[Build] Collect frontend code coverage from Cypress tests (#9555)

* build: collect code coverage from Cypress

Collect frontend code coverage reports from Cypress tests and add
proper tagging for all tests.

* Fix bash script lint error from shellcheck

* Revert Cypress to 4.3.0 to see if it fixes a failing test

.github/workflows/bashlib.sh

@@ -15,10 +15,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+set -e
+
+GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
+ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
 
 # Echo only when not in parallel mode
 say() {
-  if [[ ${INPUT_PARALLEL^^} != 'TRUE' ]]; then
+  if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
     echo "$1"
   fi
 }
@@ -67,41 +71,27 @@ build-assets() {
   say "::endgroup::"
 }
 
-npm-build() {
+build-assets-cached() {
-  if [[ $1 = '--no-cache' ]]; then
+  cache-restore assets
-    build-assets
+  if [[ -f "$ASSETS_MANIFEST" ]]; then
+    echo 'Skip frontend build because static assets already exist.'
   else
-    cache-restore assets
+    build-assets
-    if [[ -f $GITHUB_WORKSPACE/superset/static/assets/manifest.json ]]; then
+    cache-save assets
-      echo 'Skip frontend build because static assets already exist.'
-    else
-      build-assets
-      cache-save assets
-    fi
   fi
 }
 
-cypress-install() {
+build-instrumented-assets() {
-  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+  cd "$GITHUB_WORKSPACE/superset-frontend"
 
-  cache-restore cypress
+  say "::group::Build static assets with JS instrumented for test coverage"
-
+  cache-restore instrumented-assets
-  say "::group::Install Cypress"
+  if [[ -f "$ASSETS_MANIFEST" ]]; then
-  npm ci
+    echo 'Skip frontend build because instrumented static assets already exist.'
-  say "::endgroup::"
+  else
-
+    npm run build-instrumented -- --no-progress
-  cache-save cypress
+    cache-save instrumented-assets
-}
+  fi
-
-testdata() {
-  cd "$GITHUB_WORKSPACE"
-  say "::group::Load test data"
-  # must specify PYTHONPATH to make `tests.superset_test_config` importable
-  export PYTHONPATH="$GITHUB_WORKSPACE"
-  superset db upgrade
-  superset load_test_users
-  superset load_examples --load-test-data
-  superset init
   say "::endgroup::"
 }
 
@@ -131,3 +121,100 @@ setup-mysql() {
 EOF
   say "::endgroup::"
 }
+
+testdata() {
+  cd "$GITHUB_WORKSPACE"
+  say "::group::Load test data"
+  # must specify PYTHONPATH to make `tests.superset_test_config` importable
+  export PYTHONPATH="$GITHUB_WORKSPACE"
+  superset db upgrade
+  superset load_test_users
+  superset load_examples --load-test-data
+  superset init
+  say "::endgroup::"
+}
+
+codecov() {
+  say "::group::Upload code coverage"
+  local codecovScript="${HOME}/codecov.sh"
+  # download bash script if needed
+  if [[ ! -f "$codecovScript" ]]; then
+    curl -s https://codecov.io/bash > "$codecovScript"
+  fi
+  bash "$codecovScript" "$@"
+  say "::endgroup::"
+}
+
+cypress-install() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+
+  cache-restore cypress
+
+  say "::group::Install Cypress"
+  npm ci
+  say "::endgroup::"
+
+  cache-save cypress
+}
+
+# Run Cypress and upload coverage reports
+cypress-run() {
+  cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
+  
+  local page=$1
+  local group=${2:-Default}
+  local cypress="./node_modules/.bin/cypress run"
+  local browser=${CYPRESS_BROWSER:-chrome}
+
+  say "::group::Run Cypress for [$page]"
+  if [[ -z $CYPRESS_RECORD_KEY ]]; then
+    $cypress --spec "cypress/integration/$page" --browser "$browser"
+  else
+    # additional flags for Cypress dashboard recording
+    $cypress --spec "cypress/integration/$page" --browser "$browser" --record \
+      --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}"
+  fi
+
+  # don't add quotes to $record because we do want word splitting
+  say "::endgroup::"
+}
+
+cypress-run-all() {
+  # Start Flask and run it in background
+  # --no-debugger means disable the interactive debugger on the 500 page
+  # so errors can print to stderr.
+  local flasklog="${HOME}/flask.log"
+  local port=8081
+
+  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "*/*"
+
+  # Upload code coverage separately so each page can have separate flags
+  # -c will clean existing coverage reports, -F means add flags
+  codecov -cF "cypress"
+
+  # After job is done, print out Flask log for debugging
+  say "::group::Flask log for default run"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # Rerun SQL Lab tests with backend persist enabled
+  export SUPERSET_CONFIG=tests.superset_test_config_sqllab_backend_persist
+
+  # Restart Flask with new configs
+  kill $flaskProcessId
+  nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "sqllab/*" "Backend persist"
+  codecov -cF "cypress"
+
+  say "::group::Flask log for backend persist"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # make sure the program exits
+  kill $flaskProcessId
+}

.github/workflows/caches.js

@@ -21,6 +21,17 @@
 const workspaceDirectory = process.env.GITHUB_WORKSPACE;
 const homeDirectory = process.env.HOME;
 
+const assetsConfig = {
+  path: [`${workspaceDirectory}/superset/static/assets`],
+  hashFiles: [
+    `${workspaceDirectory}/superset-frontend/src/**/*`,
+    `${workspaceDirectory}/superset-frontend/*.js`,
+    `${workspaceDirectory}/superset-frontend/*.json`,
+  ],
+  // dont use restore keys as it may give an invalid older build
+  restoreKeys: '',
+};
+
 // Multi-layer cache definition
 module.exports = {
   pip: {
@@ -31,18 +42,11 @@ module.exports = {
     path: [`${homeDirectory}/.npm`],
     hashFiles: ['superset-frontend/package-lock.json'],
   },
-  assets: {
+  assets: assetsConfig,
-    path: [
+  // use separate cache for instrumented JS files and regular assets
-      `${workspaceDirectory}/superset/static/assets`,
+  // one is built with `npm run build`,
-    ],
+  // another is built with `npm run build-instrumented`
-    hashFiles: [
+  'instrumented-assets': assetsConfig,
-      `${workspaceDirectory}/superset-frontend/src/**/*`,
-      `${workspaceDirectory}/superset-frontend/*.json`,
-      `${workspaceDirectory}/superset-frontend/*.js`,
-    ],
-    // dont use restore keys as it may give an invalid older build
-    restoreKeys: ''
-  },
   cypress: {
     path: [`${homeDirectory}/.cache/Cypress`],
     hashFiles: [

.github/workflows/superset-e2e.yml

@@ -7,7 +7,7 @@ jobs:
     name: Cypress
     runs-on: ubuntu-18.04
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         browser: ['chrome']
     env:
@@ -17,7 +17,6 @@ jobs:
         postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
       PYTHONPATH: ${{ github.workspace }}
       REDIS_PORT: 16379
-      CI: github-actions
       GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
     services:
       postgres:
@@ -40,42 +39,19 @@ jobs:
         python-version: '3.6'
 
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         # Run commands in parallel does help initial installation without cache
         parallel: true
         run: |
-          npm-install && npm-build
+          npm-install && build-instrumented-assets
           pip-install && setup-postgres && testdata
           cypress-install
 
-    - name: Cypress run all
+    - name: Run Cypress
+      uses: apache-superset/cached-dependencies@adc6f73
       env:
-        CYPRESS_GROUP: Default
+        CYPRESS_BROWSER: ${{ matrix.browser }}
-        CYPRESS_PATH: 'cypress/integration/*/*'
+        CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-      run: |
+      with:
-        # Start Flask and run Cypress
+        run: cypress-run-all
-
-        # --no-debugger means disable the interactive debugger on the 500 page
-        # so errors can print to stderr.
-        flask run --no-debugger --with-threads -p 8081 &
-
-        sleep 3 # wait for the Flask app to start
-
-        cd ${{ github.workspace }}/superset-frontend/cypress-base/
-        npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false
-
-    - name: Cypress run SQL Lab (with backend persist)
-      env:
-        SUPERSET_CONFIG: tests.superset_test_config_sqllab_backend_persist
-        CYPRESS_GROUP: Backend persist
-        CYPRESS_PATH: 'cypress/integration/sqllab/*'
-      run: |
-        # Start Flask with alternative config and run Cypress
-
-        killall python # exit the running Flask app
-        flask run --no-debugger --with-threads -p 8081 &
-        sleep 3 # wait for the Flask app to start
-
-        cd ${{ github.workspace }}/superset-frontend/cypress-base/
-        npm run cypress -- run --browser ${{ matrix.browser }} --spec "${{ env.CYPRESS_PATH }}" --record false

.github/workflows/superset-frontend.yml

@@ -6,13 +6,11 @@ jobs:
   frontend-build:
     name: build
     runs-on: ubuntu-18.04
-    env:
-      CI: github-actions
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: npm-install
     - name: eslint
@@ -26,4 +24,4 @@ jobs:
     - name: Upload code coverage
       working-directory: ./superset-frontend
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,javascript
+        bash <(curl -s https://codecov.io/bash) -cF javascript

.github/workflows/superset-python.yml

@@ -10,7 +10,6 @@ jobs:
         python-version: [3.6]
     env:
       PYTHON_LINT_TARGET: setup.py superset tests
-      CI: github-actions
     steps:
     - name: Checkout code
       uses: actions/checkout@v2
@@ -19,7 +18,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
     - name: black
       run: black --check $(echo $PYTHON_LINT_TARGET)
     - name: mypy
@@ -62,7 +61,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -75,7 +74,7 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,postgres
+        bash <(curl -s https://codecov.io/bash) -cF python
 
   test-mysql:
     runs-on: ubuntu-18.04
@@ -105,7 +104,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -118,7 +117,7 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,mysql
+        bash <(curl -s https://codecov.io/bash) -cF python
 
   test-sqlite:
     runs-on: ubuntu-18.04
@@ -141,7 +140,7 @@ jobs:
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
-      uses: apache-superset/cached-dependencies@ddf7d7f
+      uses: apache-superset/cached-dependencies@adc6f73
       with:
         run: |
           pip-install
@@ -154,4 +153,4 @@ jobs:
         ./scripts/python_tests.sh
     - name: Upload code coverage
       run: |
-        bash <(curl -s https://codecov.io/bash) -cF unittest,python,sqlite
+        bash <(curl -s https://codecov.io/bash) -cF python

superset-frontend/.eslintrc.js

@@ -29,6 +29,13 @@ module.exports = {
   },
   plugins: ['prettier', 'react'],
   overrides: [
+    {
+      files: ['cypress-base/**/*'],
+      rules: {
+        'import/no-unresolved': 0,
+        'global-require': 0,
+      }
+    },
     {
       files: ['*.ts', '*.tsx'],
       parser: '@typescript-eslint/parser',

superset-frontend/babel.config.js

@@ -65,5 +65,9 @@ module.exports = {
       ],
       plugins: ['babel-plugin-dynamic-import-node'],
     },
+    // build instrumented code for testing code coverage with Cypress
+    instrumented: {
+      plugins: ['istanbul'],
+    },
   },
 };

superset-frontend/cypress-base/.gitignore

@@ -1 +1,4 @@
-cypress/screenshots/
+screenshots
+.nyc_output
+coverage
+coverage.json

superset-frontend/cypress-base/cypress.json

@@ -10,5 +10,5 @@
   "videoUploadOnPasses": false,
   "viewportWidth": 1280,
   "viewportHeight": 1024,
-  "projectId": "dk2opw"
+  "projectId": "ukwxzo"
 }

superset-frontend/cypress-base/cypress/integration/dashboard/filter.js

@@ -20,9 +20,12 @@ import { WORLD_HEALTH_DASHBOARD } from './dashboard.helper';
 
 export default () =>
   describe('dashboard filter', () => {
-    let sliceIds = [];
     let filterId;
-    let dashboardId;
+    let aliases;
+
+    const getAlias = id => {
+      return `@slice_${id}`;
+    };
 
     beforeEach(() => {
       cy.server();
@@ -33,41 +36,32 @@ export default () =>
       cy.get('#app').then(data => {
         const bootstrapData = JSON.parse(data[0].dataset.bootstrap);
         const dashboard = bootstrapData.dashboard_data;
-        dashboardId = dashboard.id;
+        const sliceIds = dashboard.slices.map(slice => slice.slice_id);
-        sliceIds = dashboard.slices.map(slice => slice.slice_id);
         filterId = dashboard.slices.find(
           slice => slice.form_data.viz_type === 'filter_box',
         ).slice_id;
+        aliases = sliceIds.map(id => {
+          const alias = getAlias(id);
+          const url = `/superset/explore_json/?*{"slice_id":${id}}*`;
+          cy.route('POST', url).as(alias.slice(1));
+          return alias;
+        });
+
+        // wait the initial page load requests
+        cy.wait(aliases);
       });
     });
 
     it('should apply filter', () => {
-      const aliases = [];
+      cy.get('.Select-placeholder')
-
+        .contains('Select [region]')
-      const formData = `{"slice_id":${filterId}}`;
+        .click()
-      const filterRoute = `/superset/explore_json/?form_data=${formData}&dashboard_id=${dashboardId}`;
+        .next()
-      cy.route('POST', filterRoute).as('fetchFilter');
-      cy.wait('@fetchFilter');
-      sliceIds
-        .filter(id => parseInt(id, 10) !== filterId)
-        .forEach(id => {
-          const alias = `getJson_${id}`;
-          aliases.push(`@${alias}`);
-
-          cy.route(
-            'POST',
-            `/superset/explore_json/?form_data={"slice_id":${id}}&dashboard_id=${dashboardId}`,
-          ).as(alias);
-        });
-
-      // select filter_box and apply
-      cy.get('.Select-control')
-        .first()
         .find('input')
-        .first()
         .type('South Asia{enter}', { force: true });
 
-      cy.wait(aliases).then(requests => {
+      // wait again after applied filters
+      cy.wait(aliases.filter(x => x !== getAlias(filterId))).then(requests => {
         requests.forEach(xhr => {
           const requestFormData = xhr.request.body;
           const requestParams = JSON.parse(requestFormData.get('form_data'));

superset-frontend/cypress-base/cypress/integration/dashboard/load.js

@@ -31,16 +31,12 @@ export default () =>
 
       cy.get('#app').then(data => {
         const bootstrapData = JSON.parse(data[0].dataset.bootstrap);
-        const dashboardId = bootstrapData.dashboard_data.id;
         const slices = bootstrapData.dashboard_data.slices;
         // then define routes and create alias for each requests
         slices.forEach(slice => {
           const alias = `getJson_${slice.slice_id}`;
           const formData = `{"slice_id":${slice.slice_id}}`;
-          cy.route(
+          cy.route('POST', `/superset/explore_json/?*${formData}*`).as(alias);
-            'POST',
-            `/superset/explore_json/?form_data=${formData}&dashboard_id=${dashboardId}`,
-          ).as(alias);
           aliases.push(`@${alias}`);
         });
       });

superset-frontend/cypress-base/cypress/plugins/index.js

@@ -29,7 +29,7 @@
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 
-module.exports = (/* on, config */) => {
+module.exports = (on, config) => {
-  // `on` is used to hook into various events Cypress emits
+  require('@cypress/code-coverage/task')(on, config);
-  // `config` is the resolved Cypress config
+  return config;
 };

superset-frontend/cypress-base/cypress/support/index.js

@@ -31,6 +31,7 @@
 // https://on.cypress.io/configuration
 // ***********************************************************
 
+import '@cypress/code-coverage/support';
 import './commands';
 
 // The following is a workaround for Cypress not supporting fetch.

superset-frontend/cypress-base/package.json

@@ -12,6 +12,8 @@
     "shortid": "^2.2.15"
   },
   "devDependencies": {
-    "cypress": "^4.3.0"
+    "@cypress/code-coverage": "^3.1.0",
+    "cypress": "4.3.0",
+    "eslint-plugin-cypress": "^2.10.3"
   }
 }

superset-frontend/cypress_build.sh

@@ -29,7 +29,7 @@ flask run -p 8081 --with-threads --reload --debugger &
 
 #block on the longer running javascript process
 time npm ci
-time npm run build
+time npm run build-instrumented
 echo "[completed js build steps]"
 
 #setup cypress

superset-frontend/package-lock.json

@@ -5584,6 +5584,15 @@
         }
       }
     },
+    "@istanbuljs/nyc-config-typescript": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/@istanbuljs/nyc-config-typescript/-/nyc-config-typescript-1.0.1.tgz",
+      "integrity": "sha512-/gz6LgVpky205LuoOfwEZmnUtaSmdk0QIMcNFj9OvxhiMhPpKftMgZmGN7jNj7jR+lr8IB1Yks3QSSSNSxfoaQ==",
+      "dev": true,
+      "requires": {
+        "@istanbuljs/schema": "^0.1.2"
+      }
+    },
     "@istanbuljs/schema": {
       "version": "0.1.2",
       "resolved": "https://registry.npmjs.org/@istanbuljs/schema/-/schema-0.1.2.tgz",
@@ -32612,9 +32621,9 @@
       }
     },
     "source-map-support": {
-      "version": "0.5.13",
+      "version": "0.5.16",
-      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.13.tgz",
+      "resolved": "https://registry.npmjs.org/source-map-support/-/source-map-support-0.5.16.tgz",
-      "integrity": "sha512-SHSKFHadjVA5oR4PPqhtAVdcBWwRYVd6g6cAXnIbRiIwc2EhPrTuKUBdSLvlEKyIP3GCf89fltvcZiP9MMFA1w==",
+      "integrity": "sha512-efyLRJDr68D9hBBNIPWFjhpFzURh+KJykQwvMyW5UiZzYwoF6l4YMMDIJJEyFWxWCqfyxLzz6tSfUFR+kXXsVQ==",
       "requires": {
         "buffer-from": "^1.0.0",
         "source-map": "^0.6.0"

superset-frontend/package.json

@@ -15,6 +15,7 @@
     "dev-server": "NODE_ENV=development BABEL_ENV=development node --max_old_space_size=4096 ./node_modules/webpack-dev-server/bin/webpack-dev-server.js --mode=development --progress",
     "prod": "node --max_old_space_size=4096 ./node_modules/webpack/bin/webpack.js --mode=production --colors --progress",
     "build-dev": "cross-env NODE_OPTIONS=--max_old_space_size=8192 NODE_ENV=development webpack --mode=development --colors --progress",
+    "build-instrumented": "cross-env NODE_ENV=development BABEL_ENV=instrumented webpack --mode=development --colors --progress",
     "build": "cross-env NODE_OPTIONS=--max_old_space_size=8192 NODE_ENV=production webpack --mode=production --colors --progress",
     "lint": "prettier --check '{src,stylesheets}/**/*.{css,less,sass,scss}' && eslint --ignore-path=.eslintignore --ext .js,.jsx,.ts,.tsx .",
     "lint-fix": "eslint --fix --ignore-path=.eslintignore --ext .js,.jsx,.ts,tsx . && npm run clean-css",
@@ -180,6 +181,7 @@
     "@babel/register": "^7.8.6",
     "@hot-loader/react-dom": "^16.13.0",
     "@types/classnames": "^2.2.9",
+    "@istanbuljs/nyc-config-typescript": "^1.0.1",
     "@types/jest": "^25.1.4",
     "@types/jquery": "^3.3.32",
     "@types/react": "^16.9.23",
@@ -233,6 +235,7 @@
     "react-test-renderer": "^16.9.0",
     "redux-mock-store": "^1.2.3",
     "sinon": "^4.5.0",
+    "source-map-support": "^0.5.16",
     "speed-measure-webpack-plugin": "^1.2.3",
     "style-loader": "^1.0.0",
     "terser-webpack-plugin": "^1.1.0",