iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

build: optimize GitHub Actions for building speed and stability (#10485)

This commit is contained in:
Jesse Yang 2020-07-30 22:28:56 -07:00 committed by GitHub
parent 7a329c25e9
commit 7ff1757448
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 38 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

39
.github/workflows/bashlib.sh vendored
View File

@ -20,6 +20,10 @@ set -e
GITHUB_WORKSPACE=${GITHUB_WORKSPACE:-.}
ASSETS_MANIFEST="$GITHUB_WORKSPACE/superset/static/assets/manifest.json"
# Rounded job start time, used to create a unique Cypress build id for
# parallelization so we can manually rerun a job after 20 minutes
NONCE=$(echo "$(date "+%Y%m%d%H%M") - ($(date +%M)%20)" | bc)
# Echo only when not in parallel mode
say() {
if [[ $(echo "$INPUT_PARALLEL" | tr '[:lower:]' '[:upper:]') != 'TRUE' ]]; then
@ -36,10 +40,11 @@ default-setup-command() {
pip-install() {
cd "$GITHUB_WORKSPACE"
# Don't use pip cache as it doesn't seem to help much.
# Pip cache saves at most about 20s on a good day
# cache-restore pip
say "::group::Install Python pacakges"
pip install --upgrade pip
pip install -r requirements.txt
pip install -r requirements-dev.txt
pip install -e ".[postgres,mysql]"
@ -52,15 +57,16 @@ pip-install() {
npm-install() {
cd "$GITHUB_WORKSPACE/superset-frontend"
cache-restore npm
# cache-restore npm
say "::group::Install npm packages"
echo "npm: $(npm --version)"
echo "node: $(node --version)"
npm ci
rm -rf ./node_modules
npm install
say "::endgroup::"
cache-save npm
# cache-save npm
}
build-assets() {
@ -71,16 +77,6 @@ build-assets() {
say "::endgroup::"
}
build-assets-cached() {
cache-restore assets
if [[ -f "$ASSETS_MANIFEST" ]]; then
echo 'Skip frontend build because static assets already exist.'
else
build-assets
cache-save assets
fi
}
build-instrumented-assets() {
cd "$GITHUB_WORKSPACE/superset-frontend"
@ -139,7 +135,7 @@ codecov() {
local codecovScript="${HOME}/codecov.sh"
# download bash script if needed
if [[ ! -f "$codecovScript" ]]; then
curl -s https://codecov.io/bash > "$codecovScript"
curl -s https://codecov.io/bash >"$codecovScript"
fi
bash "$codecovScript" "$@"
say "::endgroup::"
@ -160,19 +156,22 @@ cypress-install() {
# Run Cypress and upload coverage reports
cypress-run() {
cd "$GITHUB_WORKSPACE/superset-frontend/cypress-base"
local page=$1
local group=${2:-Default}
local cypress="./node_modules/.bin/cypress run"
local browser=${CYPRESS_BROWSER:-chrome}
export TERM="xterm"
say "::group::Run Cypress for [$page]"
if [[ -z $CYPRESS_RECORD_KEY ]]; then
$cypress --spec "cypress/integration/$page" --browser "$browser"
else
# additional flags for Cypress dashboard recording
$cypress --spec "cypress/integration/$page" --browser "$browser" --record \
--group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}"
$cypress --spec "cypress/integration/$page" --browser "$browser" \
--record --group "$group" --tag "${GITHUB_REPOSITORY},${GITHUB_EVENT_NAME}" \
--parallel --ci-build-id "${GITHUB_SHA:0:8}-${NONCE}"
fi
# don't add quotes to $record because we do want word splitting
@ -186,7 +185,7 @@ cypress-run-all() {
local flasklog="${HOME}/flask.log"
local port=8081
nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
local flaskProcessId=$!
cypress-run "*/**/*"
@ -205,7 +204,7 @@ cypress-run-all() {
# Restart Flask with new configs
kill $flaskProcessId
nohup flask run --no-debugger -p $port > "$flasklog" 2>&1 < /dev/null &
nohup flask run --no-debugger -p $port >"$flasklog" 2>&1 </dev/null &
local flaskProcessId=$!
cypress-run "sqllab/*" "Backend persist"

2
.github/workflows/caches.js vendored
View File

@ -40,7 +40,7 @@ module.exports = {
},
npm: {
path: [`${homeDirectory}/.npm`],
hashFiles: ['superset-frontend/package-lock.json'],
hashFiles: [`${workspaceDirectory}/superset-frontend/package-lock.json`],
},
assets: assetsConfig,
// use separate cache for instrumented JS files and regular assets

8
.github/workflows/superset-e2e.yml vendored
View File

@ -34,22 +34,20 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: '3.6'
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
# Run commands in parallel does help initial installation without cache
parallel: true
run: |
npm-install && build-instrumented-assets
pip-install && setup-postgres && testdata
cypress-install
- name: Run Cypress
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
env:
CYPRESS_BROWSER: ${{ matrix.browser }}
CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}

2
.github/workflows/superset-frontend.yml vendored
View File

@ -10,7 +10,7 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: npm-install
- name: lint

24
.github/workflows/superset-python.yml vendored
View File

@ -14,11 +14,11 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
- name: black
run: black --check $(echo $PYTHON_LINT_TARGET)
- name: mypy
@ -38,11 +38,11 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
@ -59,11 +59,11 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
@ -100,11 +100,11 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
@ -144,11 +144,11 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install
@ -181,11 +181,11 @@ jobs:
steps:
- uses: actions/checkout@v2
- name: Setup Python
uses: actions/setup-python@v1
uses: actions/setup-python@v2.1.1
with:
python-version: ${{ matrix.python-version }}
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: |
pip-install

2
.github/workflows/superset-translations.yml vendored
View File

@ -10,7 +10,7 @@ jobs:
- name: Checkout code
uses: actions/checkout@v2
- name: Install dependencies
uses: apache-superset/cached-dependencies@adc6f73
uses: apache-superset/cached-dependencies@b90713b
with:
run: npm-install
- name: lint

2
tests/schedules_test.py
View File

@ -62,7 +62,7 @@ class TestSchedules(SupersetTestCase):
)
# Pick up a random slice and dashboard
slce = db.session.query(Slice).all()[0]
slce = db.session.query(Slice).filter_by(slice_name="Participants").all()[0]
dashboard = db.session.query(Dashboard).all()[0]
dashboard_schedule = DashboardEmailSchedule(**cls.common_data)