ci: conditionally skip python and frontend tests (#12583)

* ci: conditionally skip python tests

* parametrize check script and implement for frontend

* add missing script

* fix incorrect check

* move CHECKS to ARGS and fix typo

* fix SC2086 nit

.github/workflows/superset-frontend.yml

@@ -18,20 +18,31 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if frontend changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh frontend
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: npm-install
       - name: lint
+        if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           npm run lint
           npm run prettier-check
       - name: unit tests
+        if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           npm run test -- --coverage
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         working-directory: ./superset-frontend
         run: |
           bash <(curl -s https://codecov.io/bash) -cF javascript

.github/workflows/superset-python-misc.yml

@@ -21,11 +21,19 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -33,6 +41,7 @@ jobs:
             pip-upgrade
             pip install -r requirements/testing.txt
       - name: pylint
+        if: steps.check.outcome == 'failure'
         # `-j 0` run Pylint in parallel
         run: pylint -j 0 superset
 

.github/workflows/superset-python-presto-hive.yml

@@ -51,11 +51,20 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -64,11 +73,14 @@ jobs:
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
+        if: steps.check.outcome == 'failure'
         run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
       - name: Python unit tests (PostgreSQL)
+        if: steps.check.outcome == 'failure'
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         run: |
           bash <(curl -s https://codecov.io/bash) -cF python
 
@@ -106,17 +118,29 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Create csv upload directory
+        if: steps.check.outcome == 'failure'
         run: sudo mkdir -p /tmp/.superset/uploads
       - name: Give write access to the csv upload directory
+        if: steps.check.outcome == 'failure'
         run: sudo chown -R $USER:$USER /tmp/.superset
       - name: Start hadoop and hive
+        if: steps.check.outcome == 'failure'
         run: docker-compose -f scripts/databases/hive/docker-compose.yml up -d
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -125,10 +149,13 @@ jobs:
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
+        if: steps.check.outcome == 'failure'
         run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
       - name: Python unit tests (PostgreSQL)
+        if: steps.check.outcome == 'failure'
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         run: |
           bash <(curl -s https://codecov.io/bash) -cF python

.github/workflows/superset-python-unittest.yml

@@ -39,11 +39,20 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -52,11 +61,14 @@ jobs:
             pip install -r requirements/testing.txt
             setup-mysql
       - name: Run celery
+        if: steps.check.outcome == 'failure'
         run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
       - name: Python unit tests (MySQL)
+        if: steps.check.outcome == 'failure'
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         run: |
           bash <(curl -s https://codecov.io/bash) -cF python
 
@@ -91,11 +103,20 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -104,11 +125,14 @@ jobs:
             pip install -r requirements/testing.txt
             setup-postgres
       - name: Run celery
+        if: steps.check.outcome == 'failure'
         run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
       - name: Python unit tests (PostgreSQL)
+        if: steps.check.outcome == 'failure'
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         run: |
           bash <(curl -s https://codecov.io/bash) -cF python
 
@@ -135,11 +159,20 @@ jobs:
         with:
           persist-credentials: false
           submodules: recursive
+      - name: Check if python changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python
       - name: Setup Python
+        if: steps.check.outcome == 'failure'
         uses: actions/setup-python@v2
         with:
           python-version: ${{ matrix.python-version }}
       - name: Install dependencies
+        if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies
         with:
           run: |
@@ -148,10 +181,13 @@ jobs:
             pip install -r requirements/testing.txt
             mkdir ${{ github.workspace }}/.temp
       - name: Run celery
+        if: steps.check.outcome == 'failure'
         run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 &
       - name: Python unit tests (SQLite)
+        if: steps.check.outcome == 'failure'
         run: |
           ./scripts/python_tests.sh
       - name: Upload code coverage
+        if: steps.check.outcome == 'failure'
         run: |
           bash <(curl -s https://codecov.io/bash) -cF python

scripts/ci_check_no_file_changes.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Usage instructions:
+#
+# to check for python changes, run with CHECKS=python
+# To check for frontend changes, run with CHECKS=frontend
+# To check for python and frontend changes, run with CHECKS="python frontend"
+
+URL="https://api.github.com/repos/${GITHUB_REPO}/pulls/${PR_NUMBER}/files"
+FILES=$(curl -s -X GET -G "${URL}" | jq -r '.[] | .filename')
+
+cat<<EOF
+CHANGED FILES:
+$FILES
+
+EOF
+
+for CHECK in "$@"
+do
+  if [[ ${CHECK} == "python" ]]; then
+    REGEX="(^tests\/|^superset\/|^setup\.py|^requirements\/.+\.txt)"
+    echo "Searching for changes in python files"
+  elif [[ ${CHECK} == "frontend" ]]; then
+    REGEX="(^superset-frontend\/)"
+    echo "Searching for changes in frontend files"
+  else
+    echo "Invalid check: \"${CHECK}\". Falling back to exiting with FAILURE code"
+    exit 1
+  fi
+
+  if [[ "${FILES}" =~ ${REGEX} ]]; then
+    echo "Detected changes... Exiting with FAILURE code"
+    exit 1
+  fi
+done
+echo "No changes detected... Exiting with SUCCESS code"
+exit 0