From 5d55c4367cbb9d64bcabff42754c10aab6e34f12 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt <33317356+villebro@users.noreply.github.com> Date: Fri, 5 Feb 2021 17:40:08 +0200 Subject: [PATCH] ci: conditionally skip python and frontend tests (#12583) * ci: conditionally skip python tests * parametrize check script and implement for frontend * add missing script * fix incorrect check * move CHECKS to ARGS and fix typo * fix SC2086 nit --- .github/workflows/superset-frontend.yml | 11 ++++ .github/workflows/superset-python-misc.yml | 9 ++++ .../workflows/superset-python-presto-hive.yml | 27 ++++++++++ .../workflows/superset-python-unittest.yml | 36 +++++++++++++ scripts/ci_check_no_file_changes.sh | 54 +++++++++++++++++++ 5 files changed, 137 insertions(+) create mode 100755 scripts/ci_check_no_file_changes.sh diff --git a/.github/workflows/superset-frontend.yml b/.github/workflows/superset-frontend.yml index 46f3a31af..2fe7113cc 100644 --- a/.github/workflows/superset-frontend.yml +++ b/.github/workflows/superset-frontend.yml @@ -18,20 +18,31 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if frontend changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh frontend - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: npm-install - name: lint + if: steps.check.outcome == 'failure' working-directory: ./superset-frontend run: | npm run lint npm run prettier-check - name: unit tests + if: steps.check.outcome == 'failure' working-directory: ./superset-frontend run: | npm run test -- --coverage - name: Upload code coverage + if: steps.check.outcome == 'failure' working-directory: ./superset-frontend run: | bash <(curl -s https://codecov.io/bash) -cF javascript diff --git a/.github/workflows/superset-python-misc.yml b/.github/workflows/superset-python-misc.yml index 629e01d2d..4f2d742e5 100644 --- a/.github/workflows/superset-python-misc.yml +++ b/.github/workflows/superset-python-misc.yml @@ -21,11 +21,19 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -33,6 +41,7 @@ jobs: pip-upgrade pip install -r requirements/testing.txt - name: pylint + if: steps.check.outcome == 'failure' # `-j 0` run Pylint in parallel run: pylint -j 0 superset diff --git a/.github/workflows/superset-python-presto-hive.yml b/.github/workflows/superset-python-presto-hive.yml index ef6c359fb..bfc6c5c4d 100644 --- a/.github/workflows/superset-python-presto-hive.yml +++ b/.github/workflows/superset-python-presto-hive.yml @@ -51,11 +51,20 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + if: steps.check.outcome == 'failure' uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -64,11 +73,14 @@ jobs: pip install -r requirements/testing.txt setup-postgres - name: Run celery + if: steps.check.outcome == 'failure' run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (PostgreSQL) + if: steps.check.outcome == 'failure' run: | ./scripts/python_tests.sh - name: Upload code coverage + if: steps.check.outcome == 'failure' run: | bash <(curl -s https://codecov.io/bash) -cF python @@ -106,17 +118,29 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Create csv upload directory + if: steps.check.outcome == 'failure' run: sudo mkdir -p /tmp/.superset/uploads - name: Give write access to the csv upload directory + if: steps.check.outcome == 'failure' run: sudo chown -R $USER:$USER /tmp/.superset - name: Start hadoop and hive + if: steps.check.outcome == 'failure' run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d - name: Setup Python + if: steps.check.outcome == 'failure' uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -125,10 +149,13 @@ jobs: pip install -r requirements/testing.txt setup-postgres - name: Run celery + if: steps.check.outcome == 'failure' run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (PostgreSQL) + if: steps.check.outcome == 'failure' run: | ./scripts/python_tests.sh - name: Upload code coverage + if: steps.check.outcome == 'failure' run: | bash <(curl -s https://codecov.io/bash) -cF python diff --git a/.github/workflows/superset-python-unittest.yml b/.github/workflows/superset-python-unittest.yml index cb21e6bd1..86ded831f 100644 --- a/.github/workflows/superset-python-unittest.yml +++ b/.github/workflows/superset-python-unittest.yml @@ -39,11 +39,20 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + if: steps.check.outcome == 'failure' uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -52,11 +61,14 @@ jobs: pip install -r requirements/testing.txt setup-mysql - name: Run celery + if: steps.check.outcome == 'failure' run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (MySQL) + if: steps.check.outcome == 'failure' run: | ./scripts/python_tests.sh - name: Upload code coverage + if: steps.check.outcome == 'failure' run: | bash <(curl -s https://codecov.io/bash) -cF python @@ -91,11 +103,20 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + if: steps.check.outcome == 'failure' uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -104,11 +125,14 @@ jobs: pip install -r requirements/testing.txt setup-postgres - name: Run celery + if: steps.check.outcome == 'failure' run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (PostgreSQL) + if: steps.check.outcome == 'failure' run: | ./scripts/python_tests.sh - name: Upload code coverage + if: steps.check.outcome == 'failure' run: | bash <(curl -s https://codecov.io/bash) -cF python @@ -135,11 +159,20 @@ jobs: with: persist-credentials: false submodules: recursive + - name: Check if python changes are present + id: check + env: + GITHUB_REPO: ${{ github.repository }} + PR_NUMBER: ${{ github.event.pull_request.number }} + continue-on-error: true + run: ./scripts/ci_check_no_file_changes.sh python - name: Setup Python + if: steps.check.outcome == 'failure' uses: actions/setup-python@v2 with: python-version: ${{ matrix.python-version }} - name: Install dependencies + if: steps.check.outcome == 'failure' uses: ./.github/actions/cached-dependencies with: run: | @@ -148,10 +181,13 @@ jobs: pip install -r requirements/testing.txt mkdir ${{ github.workspace }}/.temp - name: Run celery + if: steps.check.outcome == 'failure' run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (SQLite) + if: steps.check.outcome == 'failure' run: | ./scripts/python_tests.sh - name: Upload code coverage + if: steps.check.outcome == 'failure' run: | bash <(curl -s https://codecov.io/bash) -cF python diff --git a/scripts/ci_check_no_file_changes.sh b/scripts/ci_check_no_file_changes.sh new file mode 100755 index 000000000..2699ae843 --- /dev/null +++ b/scripts/ci_check_no_file_changes.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Usage instructions: +# +# to check for python changes, run with CHECKS=python +# To check for frontend changes, run with CHECKS=frontend +# To check for python and frontend changes, run with CHECKS="python frontend" + +URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files" +FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename') + +cat<