fix: improve change detection for GHAs (#27904)

.github/actions/change-detector/action.yml

@@ -0,0 +1,31 @@
+name: 'Change Detector'
+description: 'Detects file changes for pull request and push events'
+inputs:
+  token:
+    description: 'GitHub token for authentication'
+    required: true
+outputs:
+  python:
+    description: 'Whether Python-related files were changed'
+    value: ${{ steps.change-detector.outputs.python }}
+  frontend:
+    description: 'Whether frontend-related files were changed'
+    value: ${{ steps.change-detector.outputs.frontend }}
+  docker:
+    description: 'Whether docker-related files were changed'
+    value: ${{ steps.change-detector.outputs.docker }}
+  docs:
+    description: 'Whether docs-related files were changed'
+    value: ${{ steps.change-detector.outputs.docs }}
+runs:
+  using: 'composite'
+  steps:
+    - name: Detect file changes
+      id: change-detector
+      run: |
+        python --version
+        python scripts/change_detector.py
+      shell: bash
+      env:
+        GITHUB_TOKEN: ${{ inputs.token }}
+        GITHUB_OUTPUT: ${{ github.output }}

.github/workflows/codeql-analysis.yml

@@ -3,13 +3,9 @@ name: "CodeQL"
 on:
   push:
     branches: ["master", "[0-9].[0-9]"]
-    paths:
-      - "superset/**"
   pull_request:
     # The branches below must be a subset of the branches above
     branches: ["master"]
-    paths:
-      - "superset/**"
   schedule:
     - cron: "0 4 * * *"
 
@@ -37,6 +33,12 @@ jobs:
       - name: Checkout repository
         uses: actions/checkout@v4
 
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
         uses: github/codeql-action/init@v3
@@ -50,6 +52,7 @@ jobs:
           # queries: security-extended,security-and-quality
 
       - name: Perform CodeQL Analysis
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: github/codeql-action/analyze@v3
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/docker.yml

@@ -39,28 +39,39 @@ jobs:
 
     steps:
 
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Set up QEMU
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         uses: docker/setup-qemu-action@v3
 
       - name: Set up Docker Buildx
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         uses: docker/setup-buildx-action@v3
 
       - name: Try to login to DockerHub
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         continue-on-error: true
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USER }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-
       - name: Setup supersetbot
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         uses: ./.github/actions/setup-supersetbot/
 
       - name: Build Docker Image
+        if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
         shell: bash
         run: |
           # Single platform builds in pull_request context to speed things up

.github/workflows/no-op.yml

@@ -1,114 +0,0 @@
-# no-op.yml
-#
-# Purpose:
-# This workflow provides a workaround for the "required status checks" feature in GitHub Actions
-# when using path-specific conditions in other workflows. Required checks might remain in a "Pending"
-# state if the conditions are not met, thus blocking pull requests from being merged.
-# This no-op (no operation) workflow provides dummy success statuses for these required jobs when
-# the real jobs do not run due to path-specific conditions.
-#
-# How it works:
-# - It defines jobs with the same names as the required jobs in the main workflows.
-# - These jobs simply execute a command (`exit 0`) to succeed immediately.
-# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered.
-# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses.
-# - If the main workflows' jobs do run and fail, their failure statuses take precedence,
-#   ensuring that pull requests are not merged with failing checks.
-#
-# Usage:
-# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows.
-# - This workflow should be kept as-is, without path-specific conditions.
-
-name: no-op Checks
-on: pull_request
-
-jobs:
-  frontend-build:
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for frontend-build to ensure a successful status."
-          exit 0
-
-  pre-commit:
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for pre-commit
-        run: |
-          echo "This is a no-op step for pre-commit to ensure a successful status."
-          exit 0
-
-  python-lint:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for python-lint
-        run: |
-          echo "This is a no-op step for python-lint to ensure a successful status."
-          exit 0
-  test-postgres-hive:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
-          exit 0
-  test-postgres-presto:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
-          exit 0
-  unit-tests:
-    strategy:
-      matrix:
-        python-version: ["3.9", "3.10"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for frontend-build
-        run: |
-          echo "This is a no-op step for unit-tests to ensure a successful status when skipped."
-          exit 0
-  test-mysql:
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for test-mysql
-        run: |
-          echo "This is a no-op step for test-mysql to ensure a successful status when skipped."
-          exit 0
-  test-postgres:
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for test-postgres
-        run: |
-          echo "This is a no-op step for test-postgres to ensure a successful status when skipped."
-          exit 0
-  test-sqlite:
-    strategy:
-      matrix:
-        python-version: ["3.9"]
-    runs-on: ubuntu-latest
-    steps:
-      - name: No-op for test-sqlite
-        run: |
-          echo "This is a no-op step for test-sqlite to ensure a successful status when skipped."
-          exit 0

.github/workflows/superset-cli.yml

@@ -44,29 +44,27 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outputs.python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
       - name: Setup Postgres
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
           run: setup-postgres
       - name: superset init
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           pip install -e .
           superset db upgrade
           superset load_test_users
       - name: superset load_examples
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           # load examples without test data
           superset load_examples --load-big-data

.github/workflows/superset-e2e.yml

@@ -60,47 +60,46 @@ jobs:
           ref: "refs/pull/${{ github.event.number }}/merge"
           persist-credentials: false
           submodules: recursive
-      - name: Check if python or frontend changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python frontend
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
       - name: Setup postgres
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: setup-postgres
       - name: Import test data
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: testdata
       - name: Setup Node.js
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: actions/setup-node@v4
         with:
           node-version: "16"
       - name: Install npm dependencies
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: npm-install
       - name: Build javascript packages
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: build-instrumented-assets
       - name: Install cypress
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: cypress-install
       - name: Run Cypress
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         env:
           CYPRESS_BROWSER: ${{ matrix.browser }}
@@ -109,7 +108,7 @@ jobs:
           run: cypress-run-all
       - name: Upload Artifacts
         uses: actions/upload-artifact@v4
-        if: failure()
+        if: steps.check.outputs.python || steps.check.outputs.frontend
         with:
           name: screenshots
           path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots

.github/workflows/superset-frontend.yml

@@ -5,12 +5,8 @@ on:
     branches:
       - "master"
       - "[0-9].[0-9]"
-    paths:
-      - "superset-frontend/**"
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
-    paths:
-      - "superset-frontend/**"
 
 # cancel previous workflow jobs for PRs
 concurrency:
@@ -28,62 +24,60 @@ jobs:
           submodules: recursive
       - name: Check npm lock file version
         run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
-      - name: Check if frontend changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh frontend
       - name: Setup Node.js
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         uses: actions/setup-node@v4
         with:
           node-version: "18"
       - name: Install dependencies
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: npm-install
       - name: eslint
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run eslint -- . --quiet
       - name: tsc
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run type
       - name: prettier
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run prettier-check
       - name: Build plugins packages
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: npm run plugins:build
       - name: Build plugins Storybook
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: npm run plugins:build-storybook
       - name: superset-ui/core coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run core:cover
       - name: unit tests
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run test -- --coverage --silent
       # todo: remove this step when fix generator as a project in root jest.config.js
       - name: generator-superset unit tests
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend/packages/generator-superset
         run: npx jest
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: ../.github/workflows/codecov.sh -c -F javascript

.github/workflows/superset-python-integrationtest.yml

@@ -43,33 +43,31 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         with:
           python-version: ${{ matrix.python-version }}
       - name: Setup MySQL
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             setup-mysql
       - name: Run celery
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
       - name: Python integration tests (MySQL)
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F mysql
   test-postgres:
@@ -102,33 +100,31 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         with:
           python-version: ${{ matrix.python-version }}
       - name: Setup Postgres
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             setup-postgres
       - name: Run celery
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
       - name: Python integration tests (PostgreSQL)
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F postgres
 
@@ -156,33 +152,31 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
             # sqlite needs this working directory
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
       - name: Python integration tests (SQLite)
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F sqlite

.github/workflows/superset-python-misc.yml

@@ -6,12 +6,8 @@ on:
     branches:
       - "master"
       - "[0-9].[0-9]"
-    paths:
-      - "superset/**"
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
-    paths:
-      - "superset/**"
 
 # cancel previous workflow jobs for PRs
 concurrency:
@@ -30,20 +26,18 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         with:
           python-version: ${{ matrix.python-version }}
       - name: pylint
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         # `-j 0` run Pylint in parallel
         run: pylint -j 0 superset
 
@@ -58,9 +52,16 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
       - name: Setup Python
+        if: steps.check.outputs.python
         uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test babel extraction
+        if: steps.check.outputs.python
         run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

.github/workflows/superset-python-presto-hive.yml

@@ -6,12 +6,8 @@ on:
     branches:
       - "master"
       - "[0-9].[0-9]"
-    paths:
-      - "superset/**"
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
-    paths:
-      - "superset/**"
 
 # cancel previous workflow jobs for PRs
 concurrency:
@@ -59,30 +55,30 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python == 'true'
       - name: Setup Postgres
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
-          run: setup-postgres
+          run: |
+            echo "${{ steps.check.outputs.python }}"
+            setup-postgres
       - name: Run celery
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
       - name: Python unit tests (PostgreSQL)
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F presto
 
@@ -118,38 +114,36 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Create csv upload directory
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: sudo mkdir -p /tmp/.superset/uploads
       - name: Give write access to the csv upload directory
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: sudo chown -R $USER:$USER /tmp/.superset
       - name: Start hadoop and hive
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
       - name: Setup Postgres
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         uses: ./.github/actions/cached-dependencies
         with:
           run: setup-postgres
       - name: Run celery
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
       - name: Python unit tests (PostgreSQL)
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           ./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F hive

.github/workflows/superset-python-unittest.yml

@@ -6,18 +6,8 @@ on:
     branches:
       - "master"
       - "[0-9].[0-9]"
-    paths:
-      - "superset/**"
-      - "requirements/**"
-      - "tests/unit_tests/**"
-      - "scripts/**"
   pull_request:
     types: [synchronize, opened, reopened, ready_for_review]
-    paths:
-      - "superset/**"
-      - "requirements/**"
-      - "tests/unit_tests/**"
-      - "scripts/**"
 
 # cancel previous workflow jobs for PRs
 concurrency:
@@ -38,26 +28,24 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
-      - name: Check if python changes are present
+      - name: Check for file changes
         id: check
-        env:
+        uses: ./.github/actions/change-detector/
-          GITHUB_REPO: ${{ github.repository }}
+        with:
-          PR_NUMBER: ${{ github.event.pull_request.number }}
+          token: ${{ secrets.GITHUB_TOKEN }}
-        continue-on-error: true
-        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: ./.github/actions/setup-backend/
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         with:
           python-version: ${{ matrix.python-version }}
       - name: Python unit tests
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         env:
           SUPERSET_TESTENV: true
           SUPERSET_SECRET_KEY: not-a-secret
         run: |
           pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
       - name: Upload code coverage
-        if: steps.check.outcome == 'failure'
+        if: steps.check.outputs.python
         run: |
           bash .github/workflows/codecov.sh -c -F python -F unit

.github/workflows/superset-translations.yml

@@ -14,7 +14,7 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  frontend-check:
+  frontend-check-translations:
     runs-on: ubuntu-20.04
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@@ -22,15 +22,25 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Node.js
+        if: steps.check.outputs.frontend
         uses: actions/setup-node@v4
         with:
           node-version:  '18'
       - name: Install dependencies
+        if: steps.check.outputs.frontend
         uses: ./.github/actions/cached-dependencies
         with:
           run: npm-install
       - name: lint
+        if: steps.check.outputs.frontend
         working-directory: ./superset-frontend
         run: |
           npm run check-translation
@@ -46,9 +56,17 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check for file changes
+        id: check
+        uses: ./.github/actions/change-detector/
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+
       - name: Setup Python
+        if: steps.check.outputs.python
         uses: ./.github/actions/setup-backend/
         with:
           python-version: ${{ matrix.python-version }}
       - name: Test babel extraction
+        if: steps.check.outputs.python
         run: ./scripts/babel_update.sh

.pre-commit-config.yaml

@@ -23,6 +23,7 @@ repos:
     rev: v3.4.0
     hooks:
       - id: pyupgrade
+        exclude: scripts/change_detector.py
         args:
           - --py39-plus
   - repo: https://github.com/hadialqattan/pycln

scripts/change_detector.py

@@ -0,0 +1,156 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import argparse
+import json
+import os
+import re
+import subprocess
+from typing import List
+from urllib.error import HTTPError, URLError
+from urllib.request import Request, urlopen
+
+# Define patterns for each group of files you're interested in
+PATTERNS = {
+    "python": [
+        r"^\.github/workflows/.*python",
+        r"^tests/",
+        r"^superset/",
+        r"^scripts/",
+        r"^setup\.py",
+        r"^requirements/.+\.txt",
+        r"^.pylintrc",
+    ],
+    "frontend": [
+        r"^\.github/workflows/.*(bashlib|frontend|e2e)",
+        r"^superset-frontend/",
+    ],
+    "docker": [
+        r"^Dockerfile$",
+        r"^docker/",
+    ],
+    "docs": [
+        r"^docs/",
+    ],
+}
+GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
+
+
+def fetch_files_github_api(url: str):  # type: ignore
+    """Fetches data using GitHub API."""
+    req = Request(url)
+    req.add_header("Authorization", f"token {GITHUB_TOKEN}")
+    req.add_header("Accept", "application/vnd.github.v3+json")
+
+    print(f"Fetching from {url}")
+    with urlopen(req) as response:
+        body = response.read()
+        return json.loads(body)
+
+
+def fetch_changed_files_pr(repo: str, pr_number: str) -> List[str]:
+    """Fetches files changed in a PR using the GitHub API."""
+    url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files"
+    files = fetch_files_github_api(url)
+    return [file_info["filename"] for file_info in files]
+
+
+def fetch_changed_files_push(repo: str, sha: str) -> List[str]:
+    """Fetches files changed in the last commit for push events using GitHub API."""
+    # Fetch commit details to get the parent SHA
+    commit_url = f"https://api.github.com/repos/{repo}/commits/{sha}"
+    commit_data = fetch_files_github_api(commit_url)
+    if "parents" not in commit_data or len(commit_data["parents"]) < 1:
+        raise RuntimeError("No parent commit found for comparison.")
+    parent_sha = commit_data["parents"][0]["sha"]
+    # Compare the current commit against its parent
+    compare_url = f"https://api.github.com/repos/{repo}/compare/{parent_sha}...{sha}"
+    comparison_data = fetch_files_github_api(compare_url)
+    return [file["filename"] for file in comparison_data["files"]]
+
+
+def detect_changes(files: List[str], check_patterns: List) -> bool:  # type: ignore
+    """Detects if any of the specified files match the provided patterns."""
+    for file in files:
+        for pattern in check_patterns:
+            if re.match(pattern, file):
+                return True
+    return False
+
+
+def print_files(files: List[str]) -> None:
+    print("\n".join([f"- {s}" for s in files]))
+
+
+def main(event_type: str, sha: str, repo: str) -> None:
+    """Main function to check for file changes based on event context."""
+    print("SHA:", sha)
+    if event_type == "pull_request":
+        pr_number = os.getenv("GITHUB_REF", "").split("/")[-2]
+        files = fetch_changed_files_pr(repo, pr_number)
+        print(f"PR files:")
+        print_files(files)
+
+    elif event_type == "push":
+        files = fetch_changed_files_push(repo, sha)
+        print(f"Files touched since previous commit:")
+        print_files(files)
+    else:
+        raise ValueError("Unsupported event type")
+
+    changes_detected = {}
+    for group, regex_patterns in PATTERNS.items():
+        patterns_compiled = [re.compile(p) for p in regex_patterns]
+        changes_detected[group] = detect_changes(files, patterns_compiled)
+
+    # Output results
+    output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt"
+    with open(output_path, "a") as f:
+        for check, changed in changes_detected.items():
+            if changed:
+                print(f"{check}={str(changed).lower()}", file=f)
+                print(f"Triggering group: {check}")
+
+
+def get_git_sha() -> str:
+    return os.getenv("GITHUB_SHA") or subprocess.check_output(
+        ["git", "rev-parse", "HEAD"]
+    ).strip().decode("utf-8")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Detect file changes based on event context"
+    )
+    parser.add_argument(
+        "--event-type",
+        default=os.getenv("GITHUB_EVENT_NAME") or "push",
+        help="The type of event that triggered the workflow",
+    )
+    parser.add_argument(
+        "--sha",
+        default=get_git_sha(),
+        help="The commit SHA for push events or PR head SHA",
+    )
+    parser.add_argument(
+        "--repo",
+        default=os.getenv("GITHUB_REPOSITORY") or "apache/superset",
+        help="GitHub repository in the format owner/repo",
+    )
+    args = parser.parse_args()
+
+    main(args.event_type, args.sha, args.repo)

scripts/ci_check_no_file_changes.sh

@@ -1,68 +0,0 @@
-#!/usr/bin/env bash
-
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements.  See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License.  You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# Usage instructions:
-#
-# to check for python changes, run with CHECKS=python
-# To check for frontend changes, run with CHECKS=frontend
-# To check for python and frontend changes, run with CHECKS="python frontend"
-if [[ -z ${PR_NUMBER} ]]; then
-  echo "Not a PR; Exiting with FAILURE code"
-  exit 1
-fi
-
-URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files?per_page=1000"
-FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')
-
-REGEXES=()
-for CHECK in "$@"
-do
-  if [[ ${CHECK} == "python" ]]; then
-    REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^scripts\/|^setup\.py|^requirements\/.+\.txt|^\.pylintrc)"
-    echo "Searching for changes in python files"
-  elif [[ ${CHECK} == "frontend" ]]; then
-    REGEX="(^\.github\/workflows\/.*(bashlib|frontend|e2e)|^superset-frontend\/)"
-    echo "Searching for changes in frontend files"
-  else
-    echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
-    exit 1
-  fi
-  REGEXES=("${REGEXES[@]}" "${REGEX}")
-done
-echo
-
-cat<<EOF
-CHANGED FILES:
-$FILES
-
-EOF
-
-for FILE in ${FILES}
-do
-  for REGEX in "${REGEXES[@]}"
-  do
-    if [[ "${FILE}" =~ ${REGEX} ]]; then
-      echo "Detected changes in following file: ${FILE}"
-      echo "Exiting with FAILURE code"
-      exit 1
-    fi
-  done
-done
-echo "No changes detected... Exiting with SUCCESS code"
-exit 0

superset/__init__.py

@@ -15,6 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 """Package's main module!"""
+
 from flask import current_app, Flask
 from werkzeug.local import LocalProxy