feat: Applitools Cypress workflow (#19956)

* WIP

* Attempt Github Actions integration

* Add completion notification to workflow

* Update env

* Add new line

* Add license

* Fix whitespaces

* Fix Yaml indentation

* Fix afterEach

* Add initial tests

* Update config

* Use test secret

* Clean up

* Add batchName

* Disable logs - add secret

* Create separate workflow

* Clean up

* Update workflow

* Exclude applitools tests

* Update jobs name

* Run applitools tests separetely

* Fix path pattern

* Run once

* Add more initial tests

* Enhance tests

* Add dashboard edit mode test

* Move env

* Exclude applitools from sqllab test run

* Attempt pull_request_target

* Catch Applitools failures

* Clean up tests

* Add test step

* Add test step

* Remove step

* Fix SqlLab test

* Update CURL request

* Fix Yaml

* Add empty data to batch completion

.github/workflows/bashlib.sh

@@ -183,7 +183,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "*/**/*"
+  cypress-run "*/**/!(*.applitools.test.ts)"
 
   # After job is done, print out Flask log for debugging
   say "::group::Flask log for default run"
@@ -198,7 +198,7 @@ cypress-run-all() {
   nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
   local flaskProcessId=$!
 
-  cypress-run "sqllab/*" "Backend persist"
+  cypress-run "sqllab/!(*.applitools.test.ts)" "Backend persist"
 
   # Upload code coverage separately so each page can have separate flags
   # -c will clean existing coverage reports, -F means add flags
@@ -212,3 +212,23 @@ cypress-run-all() {
   # make sure the program exits
   kill $flaskProcessId
 }
+
+cypress-run-applitools() {
+  local flasklog="${HOME}/flask.log"
+  local port=8081
+  export CYPRESS_BASE_URL="http://localhost:${port}"
+
+  nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
+  local flaskProcessId=$!
+
+  cypress-run "*/**/*.applitools.test.ts"
+
+  codecov -c -F "cypress" || true
+
+  say "::group::Flask log for default run"
+  cat "$flasklog"
+  say "::endgroup::"
+
+  # make sure the program exits
+  kill $flaskProcessId
+}

.github/workflows/superset-applitool-cypress.yml

@@ -0,0 +1,113 @@
+name: Applitools Cypress
+
+on: pull_request
+
+jobs:
+  cypress-applitools:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        browser: ["chrome"]
+    env:
+      FLASK_ENV: development
+      SUPERSET_CONFIG: tests.integration_tests.superset_test_config
+      SUPERSET__SQLALCHEMY_DATABASE_URI: postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset
+      PYTHONPATH: ${{ github.workspace }}
+      REDIS_PORT: 16379
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      APPLITOOLS_APP_NAME: Superset
+      APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+      APPLITOOLS_BATCH_ID: ${{ github.sha }}
+      APPLITOOLS_BATCH_NAME: Superset Cypress
+    services:
+      postgres:
+        image: postgres:14-alpine
+        env:
+          POSTGRES_USER: superset
+          POSTGRES_PASSWORD: superset
+        ports:
+          - 15432:5432
+      redis:
+        image: redis:5-alpine
+        ports:
+          - 16379:6379
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v2
+        with:
+          persist-credentials: false
+          submodules: recursive
+      - name: Check if python or frontend changes are present
+        id: check
+        env:
+          GITHUB_REPO: ${{ github.repository }}
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        continue-on-error: true
+        run: ./scripts/ci_check_no_file_changes.sh python frontend
+      - name: Setup Python
+        if: steps.check.outcome == 'failure'
+        uses: actions/setup-python@v2
+        with:
+          python-version: "3.8"
+      - name: OS dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: apt-get-install
+      - name: Install python dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: |
+            pip-upgrade
+            pip install -r requirements/testing.txt
+      - name: Setup postgres
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: setup-postgres
+      - name: Import test data
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: testdata
+      - name: Setup Node.js
+        uses: actions/setup-node@v2
+        with:
+          node-version: "16"
+      - name: Install npm dependencies
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: npm-install
+      - name: Build javascript packages
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: build-instrumented-assets
+      - name: Install cypress
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        with:
+          run: cypress-install
+      - name: Run Applitools Cypress
+        if: steps.check.outcome == 'failure'
+        uses: ./.github/actions/cached-dependencies
+        env:
+          CYPRESS_BROWSER: ${{ matrix.browser }}
+        with:
+          run: cypress-run-applitools
+  batch-completion-notification:
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    needs: cypress-applitools
+    steps:
+      - name: Update Applitools batch status
+        uses: wei/curl@v1.1.1
+        env:
+          APPLITOOLS_BATCH_ID: ${{ github.sha }}
+          APPLITOOLS_API_KEY: ${{ secrets.APPLITOOLS_API_KEY }}
+        with:
+          args: -d "" -X POST https://eyesapi.applitools.com/api/externals/github/servers/github.com/commit/${{ env.APPLITOOLS_BATCH_ID }}/complete?apiKey=${{ env.APPLITOOLS_API_KEY }}

.github/workflows/superset-e2e.yml

@@ -92,7 +92,7 @@ jobs:
       - name: Setup Node.js
         uses: actions/setup-node@v2
         with:
-          node-version:  '16'
+          node-version: "16"
       - name: Install npm dependencies
         if: steps.check.outcome == 'failure'
         uses: ./.github/actions/cached-dependencies

superset-frontend/cypress-base/applitools.config.js

@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+module.exports = {
+  apiKey: process.env.APPLITOOLS_API_KEY,
+  batchId: process.env.APPLITOOLS_BATCH_ID,
+  batchName: process.env.APPLITOOLS_BATCH_NAME,
+  browser: [{ width: 1000, height: 660, name: 'chrome' }],
+  failCypressOnDiff: false,
+  isDisabled: false,
+  showLogs: false,
+  testConcurrency: 10,
+};

superset-frontend/cypress-base/cypress/integration/chart_list/chartlist.applitools.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { CHART_LIST } from './chart_list.helper';
+
+describe('charts list view', () => {
+  beforeEach(() => {
+    cy.login();
+    cy.visit(CHART_LIST);
+  });
+
+  afterEach(() => {
+    cy.eyesClose();
+  });
+
+  it('should load the Charts list', () => {
+    cy.get('[aria-label="list-view"]').click();
+    cy.eyesOpen({
+      testName: 'Charts list-view',
+    });
+    cy.eyesCheckWindow('Charts loaded');
+  });
+
+  it('should load the Charts card list', () => {
+    cy.get('[aria-label="card-view"]').click();
+    cy.eyesOpen({
+      testName: 'Charts card-view',
+    });
+    cy.eyesCheckWindow('Charts loaded');
+  });
+});

superset-frontend/cypress-base/cypress/integration/dashboard/dashboard.applitools.test.ts

@@ -0,0 +1,56 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import {
+  waitForChartLoad,
+  WORLD_HEALTH_CHARTS,
+  WORLD_HEALTH_DASHBOARD,
+} from './dashboard.helper';
+
+describe('Dashboard load', () => {
+  beforeEach(() => {
+    cy.login();
+    cy.visit(WORLD_HEALTH_DASHBOARD);
+    WORLD_HEALTH_CHARTS.forEach(waitForChartLoad);
+  });
+
+  afterEach(() => {
+    cy.eyesClose();
+  });
+
+  it('should load the Dashboard', () => {
+    cy.eyesOpen({
+      testName: 'Dashboard page',
+    });
+    cy.eyesCheckWindow('Dashboard loaded');
+  });
+
+  it('should load the Dashboard in edit mode', () => {
+    cy.get('[data-test="dashboard-header"]')
+      .find('[aria-label=edit-alt]')
+      .click();
+    // wait for a chart to appear
+    cy.get('[data-test="grid-container"]').find('.box_plot', {
+      timeout: 10000,
+    });
+    cy.eyesOpen({
+      testName: 'Dashboard edit mode',
+    });
+    cy.eyesCheckWindow('Dashboard edit mode loaded');
+  });
+});

superset-frontend/cypress-base/cypress/integration/dashboard_list/dashboardlist.applitools.test.ts

@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { DASHBOARD_LIST } from './dashboard_list.helper';
+
+describe('dashboard list view', () => {
+  beforeEach(() => {
+    cy.login();
+    cy.visit(DASHBOARD_LIST);
+  });
+
+  afterEach(() => {
+    cy.eyesClose();
+  });
+
+  it('should load the Dashboards list', () => {
+    cy.get('[aria-label="list-view"]').click();
+    cy.eyesOpen({
+      testName: 'Dashboards list-view',
+    });
+    cy.eyesCheckWindow('Dashboards loaded');
+  });
+
+  it('should load the Dashboards card list', () => {
+    cy.get('[aria-label="card-view"]').click();
+    cy.eyesOpen({
+      testName: 'Dashboards card-view',
+    });
+    cy.eyesCheckWindow('Dashboards loaded');
+  });
+});

superset-frontend/cypress-base/cypress/integration/explore/explore.applitools.test.ts

@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { FORM_DATA_DEFAULTS, NUM_METRIC } from './visualizations/shared.helper';
+
+describe('explore view', () => {
+  beforeEach(() => {
+    cy.login();
+    cy.intercept('POST', '/superset/explore_json/**').as('getJson');
+  });
+
+  afterEach(() => {
+    cy.eyesClose();
+  });
+
+  it('should load Explore', () => {
+    const LINE_CHART_DEFAULTS = { ...FORM_DATA_DEFAULTS, viz_type: 'line' };
+    const formData = { ...LINE_CHART_DEFAULTS, metrics: [NUM_METRIC] };
+    cy.visitChartByParams(JSON.stringify(formData));
+    cy.verifySliceSuccess({ waitAlias: '@getJson', chartSelector: 'svg' });
+    cy.eyesOpen({
+      testName: 'Explore page',
+    });
+    cy.eyesCheckWindow('Explore loaded');
+  });
+});

superset-frontend/cypress-base/cypress/integration/sqllab/sqllab.applitools.test.ts

@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+describe('SqlLab view', () => {
+  beforeEach(() => {
+    cy.login();
+    cy.visit('/superset/sqllab');
+  });
+
+  it('should load the SqlLab', () => {
+    cy.eyesOpen({
+      testName: 'SqlLab page',
+    });
+    cy.eyesCheckWindow('SqlLab loaded');
+    cy.eyesClose();
+  });
+});

superset-frontend/cypress-base/cypress/plugins/index.js

@@ -27,3 +27,5 @@ module.exports = (on, config) => {
   on('task', { isFileExist, findFiles });
   return config;
 };
+
+require('@applitools/eyes-cypress')(module);

superset-frontend/cypress-base/cypress/support/index.ts

@@ -17,6 +17,7 @@
  * under the License.
  */
 import '@cypress/code-coverage/support';
+import '@applitools/eyes-cypress/commands';
 
 const BASE_EXPLORE_URL = '/superset/explore/?form_data=';
 const TokenName = Cypress.env('TOKEN_NAME');

superset-frontend/cypress-base/package.json

@@ -10,6 +10,7 @@
   "author": "Apache",
   "license": "Apache-2.0",
   "dependencies": {
+    "@applitools/eyes-cypress": "^3.25.3",
     "@cypress/code-coverage": "^3.9.11",
     "@superset-ui/core": "^0.18.8",
     "cy-verify-downloads": "^0.1.6",

superset-frontend/cypress-base/tsconfig.json

@@ -10,6 +10,6 @@
     "allowJs": true,
     "noEmit": true
   },
-  "files": ["cypress/support/index.d.ts"],
+  "files": ["cypress/support/index.d.ts", "./node_modules/@applitools/eyes-cypress/eyes-index.d.ts"],
   "include": ["node_modules/cypress", "cypress/**/*.ts"]
 }