diff --git a/.github/actions/change-detector/action.yml b/.github/actions/change-detector/action.yml new file mode 100644 index 000000000..d0f356e77 --- /dev/null +++ b/.github/actions/change-detector/action.yml @@ -0,0 +1,31 @@ +name: 'Change Detector' +description: 'Detects file changes for pull request and push events' +inputs: + token: + description: 'GitHub token for authentication' + required: true +outputs: + python: + description: 'Whether Python-related files were changed' + value: ${{ steps.change-detector.outputs.python }} + frontend: + description: 'Whether frontend-related files were changed' + value: ${{ steps.change-detector.outputs.frontend }} + docker: + description: 'Whether docker-related files were changed' + value: ${{ steps.change-detector.outputs.docker }} + docs: + description: 'Whether docs-related files were changed' + value: ${{ steps.change-detector.outputs.docs }} +runs: + using: 'composite' + steps: + - name: Detect file changes + id: change-detector + run: | + python --version + python scripts/change_detector.py + shell: bash + env: + GITHUB_TOKEN: ${{ inputs.token }} + GITHUB_OUTPUT: ${{ github.output }} diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index c200f9cb3..15395bbfd 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -3,13 +3,9 @@ name: "CodeQL" on: push: branches: ["master", "[0-9].[0-9]"] - paths: - - "superset/**" pull_request: # The branches below must be a subset of the branches above branches: ["master"] - paths: - - "superset/**" schedule: - cron: "0 4 * * *" @@ -37,6 +33,12 @@ jobs: - name: Checkout repository uses: actions/checkout@v4 + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + # Initializes the CodeQL tools for scanning. - name: Initialize CodeQL uses: github/codeql-action/init@v3 @@ -50,6 +52,7 @@ jobs: # queries: security-extended,security-and-quality - name: Perform CodeQL Analysis + if: steps.check.outputs.python || steps.check.outputs.frontend uses: github/codeql-action/analyze@v3 with: category: "/language:${{matrix.language}}" diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index cb9d57978..18001241a 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,28 +39,39 @@ jobs: steps: + - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Set up QEMU + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker uses: docker/setup-qemu-action@v3 - name: Set up Docker Buildx + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker uses: docker/setup-buildx-action@v3 - name: Try to login to DockerHub + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker continue-on-error: true uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@v4 - with: - persist-credentials: false - - name: Setup supersetbot + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker uses: ./.github/actions/setup-supersetbot/ - name: Build Docker Image + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker shell: bash run: | # Single platform builds in pull_request context to speed things up diff --git a/.github/workflows/no-op.yml b/.github/workflows/no-op.yml deleted file mode 100644 index 9cc9e8bec..000000000 --- a/.github/workflows/no-op.yml +++ /dev/null @@ -1,114 +0,0 @@ -# no-op.yml -# -# Purpose: -# This workflow provides a workaround for the "required status checks" feature in GitHub Actions -# when using path-specific conditions in other workflows. Required checks might remain in a "Pending" -# state if the conditions are not met, thus blocking pull requests from being merged. -# This no-op (no operation) workflow provides dummy success statuses for these required jobs when -# the real jobs do not run due to path-specific conditions. -# -# How it works: -# - It defines jobs with the same names as the required jobs in the main workflows. -# - These jobs simply execute a command (`exit 0`) to succeed immediately. -# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered. -# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses. -# - If the main workflows' jobs do run and fail, their failure statuses take precedence, -# ensuring that pull requests are not merged with failing checks. -# -# Usage: -# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows. -# - This workflow should be kept as-is, without path-specific conditions. - -name: no-op Checks -on: pull_request - -jobs: - frontend-build: - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for frontend-build to ensure a successful status." - exit 0 - - pre-commit: - strategy: - matrix: - python-version: ["3.9"] - runs-on: ubuntu-latest - steps: - - name: No-op for pre-commit - run: | - echo "This is a no-op step for pre-commit to ensure a successful status." - exit 0 - - python-lint: - strategy: - matrix: - python-version: ["3.9", "3.10"] - runs-on: ubuntu-latest - steps: - - name: No-op for python-lint - run: | - echo "This is a no-op step for python-lint to ensure a successful status." - exit 0 - test-postgres-hive: - strategy: - matrix: - python-version: ["3.9", "3.10"] - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped." - exit 0 - test-postgres-presto: - strategy: - matrix: - python-version: ["3.9", "3.10"] - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped." - exit 0 - unit-tests: - strategy: - matrix: - python-version: ["3.9", "3.10"] - runs-on: ubuntu-latest - steps: - - name: No-op for frontend-build - run: | - echo "This is a no-op step for unit-tests to ensure a successful status when skipped." - exit 0 - test-mysql: - strategy: - matrix: - python-version: ["3.9"] - runs-on: ubuntu-latest - steps: - - name: No-op for test-mysql - run: | - echo "This is a no-op step for test-mysql to ensure a successful status when skipped." - exit 0 - test-postgres: - strategy: - matrix: - python-version: ["3.9"] - runs-on: ubuntu-latest - steps: - - name: No-op for test-postgres - run: | - echo "This is a no-op step for test-postgres to ensure a successful status when skipped." - exit 0 - test-sqlite: - strategy: - matrix: - python-version: ["3.9"] - runs-on: ubuntu-latest - steps: - - name: No-op for test-sqlite - run: | - echo "This is a no-op step for test-sqlite to ensure a successful status when skipped." - exit 0 diff --git a/.github/workflows/superset-cli.yml b/.github/workflows/superset-cli.yml index f0e037197..1d9870a0b 100644 --- a/.github/workflows/superset-cli.yml +++ b/.github/workflows/superset-cli.yml @@ -44,29 +44,27 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python + if: steps.check.outputs.python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' - name: Setup Postgres - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: superset init - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | pip install -e . superset db upgrade superset load_test_users - name: superset load_examples - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | # load examples without test data superset load_examples --load-big-data diff --git a/.github/workflows/superset-e2e.yml b/.github/workflows/superset-e2e.yml index 9086c6d0e..231643d73 100644 --- a/.github/workflows/superset-e2e.yml +++ b/.github/workflows/superset-e2e.yml @@ -60,47 +60,46 @@ jobs: ref: "refs/pull/${{ github.event.number }}/merge" persist-credentials: false submodules: recursive - - name: Check if python or frontend changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python frontend + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend - name: Setup postgres - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Import test data - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: testdata - name: Setup Node.js + if: steps.check.outputs.python || steps.check.outputs.frontend uses: actions/setup-node@v4 with: node-version: "16" - name: Install npm dependencies - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: Build javascript packages - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: build-instrumented-assets - name: Install cypress - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: cypress-install - name: Run Cypress - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python || steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies env: CYPRESS_BROWSER: ${{ matrix.browser }} @@ -109,7 +108,7 @@ jobs: run: cypress-run-all - name: Upload Artifacts uses: actions/upload-artifact@v4 - if: failure() + if: steps.check.outputs.python || steps.check.outputs.frontend with: name: screenshots path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 2b078d485..20add61f7 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -5,12 +5,8 @@ on: branches: - "master" - "[0-9].[0-9]" - paths: - - "superset-frontend/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] - paths: - - "superset-frontend/**" # cancel previous workflow jobs for PRs concurrency: @@ -28,62 +24,60 @@ jobs: submodules: recursive - name: Check npm lock file version run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json - - name: Check if frontend changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh frontend + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Node.js - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend uses: actions/setup-node@v4 with: node-version: "18" - name: Install dependencies - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: eslint - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run eslint -- . --quiet - name: tsc - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run type - name: prettier - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run prettier-check - name: Build plugins packages - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: npm run plugins:build - name: Build plugins Storybook - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: npm run plugins:build-storybook - name: superset-ui/core coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run core:cover - name: unit tests - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run test -- --coverage --silent # todo: remove this step when fix generator as a project in root jest.config.js - name: generator-superset unit tests - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend/packages/generator-superset run: npx jest - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: ../.github/workflows/codecov.sh -c -F javascript diff --git a/.github/workflows/superset-python-integrationtest.yml b/.github/workflows/superset-python-integrationtest.yml index 75eea5c70..4d710d5bb 100644 --- a/.github/workflows/superset-python-integrationtest.yml +++ b/.github/workflows/superset-python-integrationtest.yml @@ -43,33 +43,31 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Setup MySQL - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: | setup-mysql - name: Run celery - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - name: Python integration tests (MySQL) - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F mysql test-postgres: @@ -102,33 +100,31 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Setup Postgres - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: | setup-postgres - name: Run celery - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - name: Python integration tests (PostgreSQL) - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F postgres @@ -156,33 +152,31 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Install dependencies - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: | # sqlite needs this working directory mkdir ${{ github.workspace }}/.temp - name: Run celery - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - name: Python integration tests (SQLite) - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | ./scripts/python_tests.sh - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F sqlite diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index 8eb34d939..2aacf8799 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -6,12 +6,8 @@ on: branches: - "master" - "[0-9].[0-9]" - paths: - - "superset/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] - paths: - - "superset/**" # cancel previous workflow jobs for PRs concurrency: @@ -30,20 +26,18 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: pylint - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python # `-j 0` run Pylint in parallel run: pylint -j 0 superset @@ -58,9 +52,16 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python + if: steps.check.outputs.python uses: ./.github/actions/setup-backend/ with: python-version: ${{ matrix.python-version }} - name: Test babel extraction + if: steps.check.outputs.python run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index dd8d6612a..66c6952c7 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -6,12 +6,8 @@ on: branches: - "master" - "[0-9].[0-9]" - paths: - - "superset/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] - paths: - - "superset/**" # cancel previous workflow jobs for PRs concurrency: @@ -59,30 +55,30 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python == 'true' - name: Setup Postgres - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: - run: setup-postgres + run: | + echo "${{ steps.check.outputs.python }}" + setup-postgres - name: Run celery - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - name: Python unit tests (PostgreSQL) - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow' - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F presto @@ -118,38 +114,36 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Create csv upload directory - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: sudo mkdir -p /tmp/.superset/uploads - name: Give write access to the csv upload directory - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: sudo chown -R $USER:$USER /tmp/.superset - name: Start hadoop and hive - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: docker compose -f scripts/databases/hive/docker-compose.yml up -d - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python - name: Setup Postgres - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python uses: ./.github/actions/cached-dependencies with: run: setup-postgres - name: Run celery - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 & - name: Python unit tests (PostgreSQL) - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow' - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F hive diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index c5bf8bf46..48e2c21c2 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -6,18 +6,8 @@ on: branches: - "master" - "[0-9].[0-9]" - paths: - - "superset/**" - - "requirements/**" - - "tests/unit_tests/**" - - "scripts/**" pull_request: types: [synchronize, opened, reopened, ready_for_review] - paths: - - "superset/**" - - "requirements/**" - - "tests/unit_tests/**" - - "scripts/**" # cancel previous workflow jobs for PRs concurrency: @@ -38,26 +28,24 @@ jobs: with: persist-credentials: false submodules: recursive - - name: Check if python changes are present + - name: Check for file changes id: check - env: - GITHUB_REPO: ${{ github.repository }} - PR_NUMBER: ${{ github.event.pull_request.number }} - continue-on-error: true - run: ./scripts/ci_check_no_file_changes.sh python + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} - name: Setup Python uses: ./.github/actions/setup-backend/ - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python with: python-version: ${{ matrix.python-version }} - name: Python unit tests - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python env: SUPERSET_TESTENV: true SUPERSET_SECRET_KEY: not-a-secret run: | pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear - name: Upload code coverage - if: steps.check.outcome == 'failure' + if: steps.check.outputs.python run: | bash .github/workflows/codecov.sh -c -F python -F unit diff --git a/.github/workflows/superset-translations.yml b/.github/workflows/superset-translations.yml index 9a8b8df1e..734e9bf75 100644 --- a/.github/workflows/superset-translations.yml +++ b/.github/workflows/superset-translations.yml @@ -14,7 +14,7 @@ concurrency: cancel-in-progress: true jobs: - frontend-check: + frontend-check-translations: runs-on: ubuntu-20.04 steps: - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" @@ -22,15 +22,25 @@ jobs: with: persist-credentials: false submodules: recursive + + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Node.js + if: steps.check.outputs.frontend uses: actions/setup-node@v4 with: node-version: '18' - name: Install dependencies + if: steps.check.outputs.frontend uses: ./.github/actions/cached-dependencies with: run: npm-install - name: lint + if: steps.check.outputs.frontend working-directory: ./superset-frontend run: | npm run check-translation @@ -46,9 +56,17 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check for file changes + id: check + uses: ./.github/actions/change-detector/ + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: Setup Python + if: steps.check.outputs.python uses: ./.github/actions/setup-backend/ with: python-version: ${{ matrix.python-version }} - name: Test babel extraction + if: steps.check.outputs.python run: ./scripts/babel_update.sh diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3549b3f47..a476dc3ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -23,6 +23,7 @@ repos: rev: v3.4.0 hooks: - id: pyupgrade + exclude: scripts/change_detector.py args: - --py39-plus - repo: https://github.com/hadialqattan/pycln diff --git a/scripts/change_detector.py b/scripts/change_detector.py new file mode 100755 index 000000000..df2c872a4 --- /dev/null +++ b/scripts/change_detector.py @@ -0,0 +1,156 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import argparse +import json +import os +import re +import subprocess +from typing import List +from urllib.error import HTTPError, URLError +from urllib.request import Request, urlopen + +# Define patterns for each group of files you're interested in +PATTERNS = { + "python": [ + r"^\.github/workflows/.*python", + r"^tests/", + r"^superset/", + r"^scripts/", + r"^setup\.py", + r"^requirements/.+\.txt", + r"^.pylintrc", + ], + "frontend": [ + r"^\.github/workflows/.*(bashlib|frontend|e2e)", + r"^superset-frontend/", + ], + "docker": [ + r"^Dockerfile$", + r"^docker/", + ], + "docs": [ + r"^docs/", + ], +} +GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN") + + +def fetch_files_github_api(url: str): # type: ignore + """Fetches data using GitHub API.""" + req = Request(url) + req.add_header("Authorization", f"token {GITHUB_TOKEN}") + req.add_header("Accept", "application/vnd.github.v3+json") + + print(f"Fetching from {url}") + with urlopen(req) as response: + body = response.read() + return json.loads(body) + + +def fetch_changed_files_pr(repo: str, pr_number: str) -> List[str]: + """Fetches files changed in a PR using the GitHub API.""" + url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files" + files = fetch_files_github_api(url) + return [file_info["filename"] for file_info in files] + + +def fetch_changed_files_push(repo: str, sha: str) -> List[str]: + """Fetches files changed in the last commit for push events using GitHub API.""" + # Fetch commit details to get the parent SHA + commit_url = f"https://api.github.com/repos/{repo}/commits/{sha}" + commit_data = fetch_files_github_api(commit_url) + if "parents" not in commit_data or len(commit_data["parents"]) < 1: + raise RuntimeError("No parent commit found for comparison.") + parent_sha = commit_data["parents"][0]["sha"] + # Compare the current commit against its parent + compare_url = f"https://api.github.com/repos/{repo}/compare/{parent_sha}...{sha}" + comparison_data = fetch_files_github_api(compare_url) + return [file["filename"] for file in comparison_data["files"]] + + +def detect_changes(files: List[str], check_patterns: List) -> bool: # type: ignore + """Detects if any of the specified files match the provided patterns.""" + for file in files: + for pattern in check_patterns: + if re.match(pattern, file): + return True + return False + + +def print_files(files: List[str]) -> None: + print("\n".join([f"- {s}" for s in files])) + + +def main(event_type: str, sha: str, repo: str) -> None: + """Main function to check for file changes based on event context.""" + print("SHA:", sha) + if event_type == "pull_request": + pr_number = os.getenv("GITHUB_REF", "").split("/")[-2] + files = fetch_changed_files_pr(repo, pr_number) + print(f"PR files:") + print_files(files) + + elif event_type == "push": + files = fetch_changed_files_push(repo, sha) + print(f"Files touched since previous commit:") + print_files(files) + else: + raise ValueError("Unsupported event type") + + changes_detected = {} + for group, regex_patterns in PATTERNS.items(): + patterns_compiled = [re.compile(p) for p in regex_patterns] + changes_detected[group] = detect_changes(files, patterns_compiled) + + # Output results + output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt" + with open(output_path, "a") as f: + for check, changed in changes_detected.items(): + if changed: + print(f"{check}={str(changed).lower()}", file=f) + print(f"Triggering group: {check}") + + +def get_git_sha() -> str: + return os.getenv("GITHUB_SHA") or subprocess.check_output( + ["git", "rev-parse", "HEAD"] + ).strip().decode("utf-8") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Detect file changes based on event context" + ) + parser.add_argument( + "--event-type", + default=os.getenv("GITHUB_EVENT_NAME") or "push", + help="The type of event that triggered the workflow", + ) + parser.add_argument( + "--sha", + default=get_git_sha(), + help="The commit SHA for push events or PR head SHA", + ) + parser.add_argument( + "--repo", + default=os.getenv("GITHUB_REPOSITORY") or "apache/superset", + help="GitHub repository in the format owner/repo", + ) + args = parser.parse_args() + + main(args.event_type, args.sha, args.repo) diff --git a/scripts/ci_check_no_file_changes.sh b/scripts/ci_check_no_file_changes.sh deleted file mode 100755 index d4851ea66..000000000 --- a/scripts/ci_check_no_file_changes.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -# Usage instructions: -# -# to check for python changes, run with CHECKS=python -# To check for frontend changes, run with CHECKS=frontend -# To check for python and frontend changes, run with CHECKS="python frontend" -if [[ -z ${PR_NUMBER} ]]; then - echo "Not a PR; Exiting with FAILURE code" - exit 1 -fi - -URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files?per_page=1000" -FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename') - -REGEXES=() -for CHECK in "$@" -do - if [[ ${CHECK} == "python" ]]; then - REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^scripts\/|^setup\.py|^requirements\/.+\.txt|^\.pylintrc)" - echo "Searching for changes in python files" - elif [[ ${CHECK} == "frontend" ]]; then - REGEX="(^\.github\/workflows\/.*(bashlib|frontend|e2e)|^superset-frontend\/)" - echo "Searching for changes in frontend files" - else - echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code" - exit 1 - fi - REGEXES=("${REGEXES[@]}" "${REGEX}") -done -echo - -cat<