iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: improve change detection for GHAs (#27904)

This commit is contained in:
Maxime Beauchemin 2024-04-08 16:20:25 -07:00 committed by GitHub
parent 7c8e1bb46e
commit e80d194b8f
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 338 additions and 331 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
.github/actions/change-detector/action.yml vendored Normal file
View File

@ -0,0 +1,31 @@
name: 'Change Detector'
description: 'Detects file changes for pull request and push events'
inputs:
token:
description: 'GitHub token for authentication'
required: true
outputs:
python:
description: 'Whether Python-related files were changed'
value: ${{ steps.change-detector.outputs.python }}
frontend:
description: 'Whether frontend-related files were changed'
value: ${{ steps.change-detector.outputs.frontend }}
docker:
description: 'Whether docker-related files were changed'
value: ${{ steps.change-detector.outputs.docker }}
docs:
description: 'Whether docs-related files were changed'
value: ${{ steps.change-detector.outputs.docs }}
runs:
using: 'composite'
steps:
- name: Detect file changes
id: change-detector
run: |
python --version
python scripts/change_detector.py
shell: bash
env:
GITHUB_TOKEN: ${{ inputs.token }}
GITHUB_OUTPUT: ${{ github.output }}

11
.github/workflows/codeql-analysis.yml vendored
View File

@ -3,13 +3,9 @@ name: "CodeQL"
on:
push:
branches: ["master", "[0-9].[0-9]"]
paths:
- "superset/**"
pull_request:
# The branches below must be a subset of the branches above
branches: ["master"]
paths:
- "superset/**"
schedule:
- cron: "0 4 * * *"
@ -37,6 +33,12 @@ jobs:
- name: Checkout repository
uses: actions/checkout@v4
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
@ -50,6 +52,7 @@ jobs:
# queries: security-extended,security-and-quality
- name: Perform CodeQL Analysis
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: github/codeql-action/analyze@v3
with:
category: "/language:${{matrix.language}}"

21
.github/workflows/docker.yml vendored
View File

@ -39,28 +39,39 @@ jobs:
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Set up QEMU
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: docker/setup-buildx-action@v3
- name: Try to login to DockerHub
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
uses: actions/checkout@v4
with:
persist-credentials: false
- name: Setup supersetbot
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
uses: ./.github/actions/setup-supersetbot/
- name: Build Docker Image
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
shell: bash
run: |
# Single platform builds in pull_request context to speed things up

114
.github/workflows/no-op.yml vendored
View File

@ -1,114 +0,0 @@
# no-op.yml
#
# Purpose:
# This workflow provides a workaround for the "required status checks" feature in GitHub Actions
# when using path-specific conditions in other workflows. Required checks might remain in a "Pending"
# state if the conditions are not met, thus blocking pull requests from being merged.
# This no-op (no operation) workflow provides dummy success statuses for these required jobs when
# the real jobs do not run due to path-specific conditions.
#
# How it works:
# - It defines jobs with the same names as the required jobs in the main workflows.
# - These jobs simply execute a command (`exit 0`) to succeed immediately.
# - When a pull request is created or updated, both this no-op workflow and the main workflows are triggered.
# - If the main workflows' jobs don't run (due to path conditions), these no-op jobs provide successful statuses.
# - If the main workflows' jobs do run and fail, their failure statuses take precedence,
# ensuring that pull requests are not merged with failing checks.
#
# Usage:
# - Ensure that the job names in this workflow match exactly the names of the corresponding jobs in the main workflows.
# - This workflow should be kept as-is, without path-specific conditions.
name: no-op Checks
on: pull_request
jobs:
frontend-build:
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for frontend-build to ensure a successful status."
exit 0
pre-commit:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for pre-commit
run: |
echo "This is a no-op step for pre-commit to ensure a successful status."
exit 0
python-lint:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for python-lint
run: |
echo "This is a no-op step for python-lint to ensure a successful status."
exit 0
test-postgres-hive:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
exit 0
test-postgres-presto:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for test-postgres-postgres to ensure a successful status when skipped."
exit 0
unit-tests:
strategy:
matrix:
python-version: ["3.9", "3.10"]
runs-on: ubuntu-latest
steps:
- name: No-op for frontend-build
run: |
echo "This is a no-op step for unit-tests to ensure a successful status when skipped."
exit 0
test-mysql:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-mysql
run: |
echo "This is a no-op step for test-mysql to ensure a successful status when skipped."
exit 0
test-postgres:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-postgres
run: |
echo "This is a no-op step for test-postgres to ensure a successful status when skipped."
exit 0
test-sqlite:
strategy:
matrix:
python-version: ["3.9"]
runs-on: ubuntu-latest
steps:
- name: No-op for test-sqlite
run: |
echo "This is a no-op step for test-sqlite to ensure a successful status when skipped."
exit 0

18
.github/workflows/superset-cli.yml vendored
View File

@ -44,29 +44,27 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: superset init
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
pip install -e .
superset db upgrade
superset load_test_users
- name: superset load_examples
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
# load examples without test data
superset load_examples --load-big-data

27
.github/workflows/superset-e2e.yml vendored
View File

@ -60,47 +60,46 @@ jobs:
ref: "refs/pull/${{ github.event.number }}/merge"
persist-credentials: false
submodules: recursive
- name: Check if python or frontend changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python frontend
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
- name: Setup postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Import test data
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: testdata
- name: Setup Node.js
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "16"
- name: Install npm dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: Build javascript packages
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: build-instrumented-assets
- name: Install cypress
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: cypress-install
- name: Run Cypress
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python || steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
@ -109,7 +108,7 @@ jobs:
run: cypress-run-all
- name: Upload Artifacts
uses: actions/upload-artifact@v4
if: failure()
if: steps.check.outputs.python || steps.check.outputs.frontend
with:
name: screenshots
path: ${{ github.workspace }}/superset-frontend/cypress-base/cypress/screenshots

36
.github/workflows/superset-frontend.yml vendored
View File

@ -5,12 +5,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset-frontend/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset-frontend/**"
# cancel previous workflow jobs for PRs
concurrency:
@ -28,62 +24,60 @@ jobs:
submodules: recursive
- name: Check npm lock file version
run: ./scripts/ci_check_npm_lock_version.sh ./superset-frontend/package-lock.json
- name: Check if frontend changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh frontend
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Node.js
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: "18"
- name: Install dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: eslint
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run eslint -- . --quiet
- name: tsc
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run type
- name: prettier
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run prettier-check
- name: Build plugins packages
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build
- name: Build plugins Storybook
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: npm run plugins:build-storybook
- name: superset-ui/core coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run core:cover
- name: unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run test -- --coverage --silent
# todo: remove this step when fix generator as a project in root jest.config.js
- name: generator-superset unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend/packages/generator-superset
run: npx jest
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: ../.github/workflows/codecov.sh -c -F javascript

60
.github/workflows/superset-python-integrationtest.yml vendored
View File

@ -43,33 +43,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup MySQL
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-mysql
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (MySQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F mysql
test-postgres:
@ -102,33 +100,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F postgres
@ -156,33 +152,31 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: |
# sqlite needs this working directory
mkdir ${{ github.workspace }}/.temp
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python integration tests (SQLite)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F sqlite

25
.github/workflows/superset-python-misc.yml vendored
View File

@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@ -30,20 +26,18 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: pylint
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
# `-j 0` run Pylint in parallel
run: pylint -j 0 superset
@ -58,9 +52,16 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
if: steps.check.outputs.python
run: flask fab babel-extract --target superset/translations --output superset/translations/messages.pot --config superset/translations/babel.cfg -k _,__,t,tn,tct

54
.github/workflows/superset-python-presto-hive.yml vendored
View File

@ -6,12 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
# cancel previous workflow jobs for PRs
concurrency:
@ -59,30 +55,30 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python == 'true'
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
run: |
echo "${{ steps.check.outputs.python }}"
setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F presto
@ -118,38 +114,36 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Create csv upload directory
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: sudo mkdir -p /tmp/.superset/uploads
- name: Give write access to the csv upload directory
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: sudo chown -R $USER:$USER /tmp/.superset
- name: Start hadoop and hive
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: docker compose -f scripts/databases/hive/docker-compose.yml up -d
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
- name: Setup Postgres
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
uses: ./.github/actions/cached-dependencies
with:
run: setup-postgres
- name: Run celery
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: celery --app=superset.tasks.celery_app:app worker -Ofair -c 2 &
- name: Python unit tests (PostgreSQL)
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
./scripts/python_tests.sh -m 'chart_data_flow or sql_json_flow'
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F hive

26
.github/workflows/superset-python-unittest.yml vendored
View File

@ -6,18 +6,8 @@ on:
branches:
- "master"
- "[0-9].[0-9]"
paths:
- "superset/**"
- "requirements/**"
- "tests/unit_tests/**"
- "scripts/**"
pull_request:
types: [synchronize, opened, reopened, ready_for_review]
paths:
- "superset/**"
- "requirements/**"
- "tests/unit_tests/**"
- "scripts/**"
# cancel previous workflow jobs for PRs
concurrency:
@ -38,26 +28,24 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check if python changes are present
- name: Check for file changes
id: check
env:
GITHUB_REPO: ${{ github.repository }}
PR_NUMBER: ${{ github.event.pull_request.number }}
continue-on-error: true
run: ./scripts/ci_check_no_file_changes.sh python
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
uses: ./.github/actions/setup-backend/
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
with:
python-version: ${{ matrix.python-version }}
- name: Python unit tests
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
env:
SUPERSET_TESTENV: true
SUPERSET_SECRET_KEY: not-a-secret
run: |
pytest --durations-min=0.5 --cov-report= --cov=superset ./tests/common ./tests/unit_tests --cache-clear
- name: Upload code coverage
if: steps.check.outcome == 'failure'
if: steps.check.outputs.python
run: |
bash .github/workflows/codecov.sh -c -F python -F unit

20
.github/workflows/superset-translations.yml vendored
View File

@ -14,7 +14,7 @@ concurrency:
cancel-in-progress: true
jobs:
frontend-check:
frontend-check-translations:
runs-on: ubuntu-20.04
steps:
- name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
@ -22,15 +22,25 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Node.js
if: steps.check.outputs.frontend
uses: actions/setup-node@v4
with:
node-version: '18'
- name: Install dependencies
if: steps.check.outputs.frontend
uses: ./.github/actions/cached-dependencies
with:
run: npm-install
- name: lint
if: steps.check.outputs.frontend
working-directory: ./superset-frontend
run: |
npm run check-translation
@ -46,9 +56,17 @@ jobs:
with:
persist-credentials: false
submodules: recursive
- name: Check for file changes
id: check
uses: ./.github/actions/change-detector/
with:
token: ${{ secrets.GITHUB_TOKEN }}
- name: Setup Python
if: steps.check.outputs.python
uses: ./.github/actions/setup-backend/
with:
python-version: ${{ matrix.python-version }}
- name: Test babel extraction
if: steps.check.outputs.python
run: ./scripts/babel_update.sh

1
.pre-commit-config.yaml
View File

@ -23,6 +23,7 @@ repos:
rev: v3.4.0
hooks:
- id: pyupgrade
exclude: scripts/change_detector.py
args:
- --py39-plus
- repo: https://github.com/hadialqattan/pycln

156
scripts/change_detector.py Executable file
View File

@ -0,0 +1,156 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import argparse
import json
import os
import re
import subprocess
from typing import List
from urllib.error import HTTPError, URLError
from urllib.request import Request, urlopen
# Define patterns for each group of files you're interested in
PATTERNS = {
"python": [
r"^\.github/workflows/.*python",
r"^tests/",
r"^superset/",
r"^scripts/",
r"^setup\.py",
r"^requirements/.+\.txt",
r"^.pylintrc",
],
"frontend": [
r"^\.github/workflows/.*(bashlib|frontend|e2e)",
r"^superset-frontend/",
],
"docker": [
r"^Dockerfile$",
r"^docker/",
],
"docs": [
r"^docs/",
],
}
GITHUB_TOKEN = os.environ.get("GITHUB_TOKEN")
def fetch_files_github_api(url: str): # type: ignore
"""Fetches data using GitHub API."""
req = Request(url)
req.add_header("Authorization", f"token {GITHUB_TOKEN}")
req.add_header("Accept", "application/vnd.github.v3+json")
print(f"Fetching from {url}")
with urlopen(req) as response:
body = response.read()
return json.loads(body)
def fetch_changed_files_pr(repo: str, pr_number: str) -> List[str]:
"""Fetches files changed in a PR using the GitHub API."""
url = f"https://api.github.com/repos/{repo}/pulls/{pr_number}/files"
files = fetch_files_github_api(url)
return [file_info["filename"] for file_info in files]
def fetch_changed_files_push(repo: str, sha: str) -> List[str]:
"""Fetches files changed in the last commit for push events using GitHub API."""
# Fetch commit details to get the parent SHA
commit_url = f"https://api.github.com/repos/{repo}/commits/{sha}"
commit_data = fetch_files_github_api(commit_url)
if "parents" not in commit_data or len(commit_data["parents"]) < 1:
raise RuntimeError("No parent commit found for comparison.")
parent_sha = commit_data["parents"][0]["sha"]
# Compare the current commit against its parent
compare_url = f"https://api.github.com/repos/{repo}/compare/{parent_sha}...{sha}"
comparison_data = fetch_files_github_api(compare_url)
return [file["filename"] for file in comparison_data["files"]]
def detect_changes(files: List[str], check_patterns: List) -> bool: # type: ignore
"""Detects if any of the specified files match the provided patterns."""
for file in files:
for pattern in check_patterns:
if re.match(pattern, file):
return True
return False
def print_files(files: List[str]) -> None:
print("\n".join([f"- {s}" for s in files]))
def main(event_type: str, sha: str, repo: str) -> None:
"""Main function to check for file changes based on event context."""
print("SHA:", sha)
if event_type == "pull_request":
pr_number = os.getenv("GITHUB_REF", "").split("/")[-2]
files = fetch_changed_files_pr(repo, pr_number)
print(f"PR files:")
print_files(files)
elif event_type == "push":
files = fetch_changed_files_push(repo, sha)
print(f"Files touched since previous commit:")
print_files(files)
else:
raise ValueError("Unsupported event type")
changes_detected = {}
for group, regex_patterns in PATTERNS.items():
patterns_compiled = [re.compile(p) for p in regex_patterns]
changes_detected[group] = detect_changes(files, patterns_compiled)
# Output results
output_path = os.getenv("GITHUB_OUTPUT") or "/tmp/GITHUB_OUTPUT.txt"
with open(output_path, "a") as f:
for check, changed in changes_detected.items():
if changed:
print(f"{check}={str(changed).lower()}", file=f)
print(f"Triggering group: {check}")
def get_git_sha() -> str:
return os.getenv("GITHUB_SHA") or subprocess.check_output(
["git", "rev-parse", "HEAD"]
).strip().decode("utf-8")
if __name__ == "__main__":
parser = argparse.ArgumentParser(
description="Detect file changes based on event context"
)
parser.add_argument(
"--event-type",
default=os.getenv("GITHUB_EVENT_NAME") or "push",
help="The type of event that triggered the workflow",
)
parser.add_argument(
"--sha",
default=get_git_sha(),
help="The commit SHA for push events or PR head SHA",
)
parser.add_argument(
"--repo",
default=os.getenv("GITHUB_REPOSITORY") or "apache/superset",
help="GitHub repository in the format owner/repo",
)
args = parser.parse_args()
main(args.event_type, args.sha, args.repo)

68
scripts/ci_check_no_file_changes.sh
View File

@ -1,68 +0,0 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# Usage instructions:
#
# to check for python changes, run with CHECKS=python
# To check for frontend changes, run with CHECKS=frontend
# To check for python and frontend changes, run with CHECKS="python frontend"
if [[ -z ${PR_NUMBER} ]]; then
echo "Not a PR; Exiting with FAILURE code"
exit 1
fi
URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files?per_page=1000"
FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')
REGEXES=()
for CHECK in "$@"
do
if [[ ${CHECK} == "python" ]]; then
REGEX="(^\.github\/workflows\/.*python|^tests\/|^superset\/|^scripts\/|^setup\.py|^requirements\/.+\.txt|^\.pylintrc)"
echo "Searching for changes in python files"
elif [[ ${CHECK} == "frontend" ]]; then
REGEX="(^\.github\/workflows\/.*(bashlib|frontend|e2e)|^superset-frontend\/)"
echo "Searching for changes in frontend files"
else
echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
exit 1
fi
REGEXES=("${REGEXES[@]}" "${REGEX}")
done
echo
cat<<EOF
CHANGED FILES:
$FILES
EOF
for FILE in ${FILES}
do
for REGEX in "${REGEXES[@]}"
do
if [[ "${FILE}" =~ ${REGEX} ]]; then
echo "Detected changes in following file: ${FILE}"
echo "Exiting with FAILURE code"
exit 1
fi
done
done
echo "No changes detected... Exiting with SUCCESS code"
exit 0

1
superset/__init__.py
View File

@ -15,6 +15,7 @@
# specific language governing permissions and limitations
# under the License.
"""Package's main module!"""
from flask import current_app, Flask
from werkzeug.local import LocalProxy