feat: make ephemeral env use supersetbot + deprecate build_docker.py (#30870)
This commit is contained in:
Maxime Beauchemin
GitHub
parent
abf3790ea6
commit
e0deb704f9
|
|
@ -1 +1 @@
|
|||
Subproject commit a917fd15b20e8b64b94d9158ad54cd6345335584
|
||||
Subproject commit 120944e66390c2534cc1b3c62d7285ba7ff02594
|
||||
|
|
@ -1,30 +1,25 @@
|
|||
name: Ephemeral env workflow
|
||||
|
||||
# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
|
||||
|
||||
on:
|
||||
issue_comment:
|
||||
types: [created]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
comment_body:
|
||||
description: 'Comment body to simulate /testenv command'
|
||||
required: true
|
||||
default: '/testenv up'
|
||||
issue_number:
|
||||
description: 'Issue or PR number'
|
||||
required: true
|
||||
|
||||
jobs:
|
||||
config:
|
||||
runs-on: "ubuntu-22.04"
|
||||
if: github.event.issue.pull_request
|
||||
outputs:
|
||||
has-secrets: ${{ steps.check.outputs.has-secrets }}
|
||||
steps:
|
||||
- name: "Check for secrets"
|
||||
id: check
|
||||
shell: bash
|
||||
run: |
|
||||
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
|
||||
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
ephemeral-env-comment:
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
|
||||
cancel-in-progress: true
|
||||
needs: config
|
||||
if: needs.config.outputs.has-secrets
|
||||
name: Evaluate ephemeral env comment trigger (/testenv)
|
||||
runs-on: ubuntu-22.04
|
||||
permissions:
|
||||
|
|
@ -44,18 +39,18 @@ jobs:
|
|||
with:
|
||||
result-encoding: string
|
||||
script: |
|
||||
const pattern = /^\/testenv (up|down)/
|
||||
const result = pattern.exec(context.payload.comment.body)
|
||||
return result === null ? 'noop' : result[1]
|
||||
const pattern = /^\/testenv (up|down)/;
|
||||
const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}');
|
||||
return result === null ? 'noop' : result[1];
|
||||
|
||||
- name: Eval comment body for feature flags
|
||||
- name: Looking for feature flags
|
||||
uses: actions/github-script@v7
|
||||
id: eval-feature-flags
|
||||
with:
|
||||
script: |
|
||||
const pattern = /FEATURE_(\w+)=(\w+)/g;
|
||||
let results = [];
|
||||
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
|
||||
[...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => {
|
||||
const config = {
|
||||
name: `SUPERSET_FEATURE_${match[1]}`,
|
||||
value: match[2],
|
||||
|
|
@ -67,24 +62,25 @@ jobs:
|
|||
- name: Limit to committers
|
||||
if: >
|
||||
steps.eval-body.outputs.result != 'noop' &&
|
||||
github.event_name == 'issue_comment' &&
|
||||
github.event.comment.author_association != 'MEMBER' &&
|
||||
github.event.comment.author_association != 'OWNER'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
github-token: ${{ github.token }}
|
||||
script: |
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: errMsg
|
||||
})
|
||||
core.setFailed(errMsg)
|
||||
});
|
||||
core.setFailed(errMsg);
|
||||
|
||||
ephemeral-docker-build:
|
||||
concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
|
||||
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
|
||||
cancel-in-progress: true
|
||||
needs: ephemeral-env-comment
|
||||
name: ephemeral-docker-build
|
||||
|
|
@ -98,9 +94,9 @@ jobs:
|
|||
const request = {
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
pull_number: ${{ github.event.issue.number }},
|
||||
}
|
||||
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
|
||||
pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
|
||||
};
|
||||
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
|
||||
const pr = await github.rest.pulls.get(request);
|
||||
return pr.data;
|
||||
|
||||
|
|
@ -121,12 +117,17 @@ jobs:
|
|||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Setup supersetbot
|
||||
uses: ./.github/actions/setup-supersetbot/
|
||||
|
||||
- name: Build ephemeral env image
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
./scripts/build_docker.py \
|
||||
"ci" \
|
||||
"pull_request" \
|
||||
--build_context_ref ${{ github.event.issue.number }}
|
||||
supersetbot docker \
|
||||
--preset ci \
|
||||
--platform linux/amd64 \
|
||||
--context-ref "$RELEASE"
|
||||
|
||||
- name: Configure AWS credentials
|
||||
uses: aws-actions/configure-aws-credentials@v4
|
||||
|
|
@ -146,7 +147,7 @@ jobs:
|
|||
ECR_REPOSITORY: superset-ci
|
||||
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
|
||||
run: |
|
||||
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
|
||||
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
|
||||
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
|
||||
|
||||
ephemeral-env-up:
|
||||
|
|
@ -181,22 +182,22 @@ jobs:
|
|||
aws ecr describe-images \
|
||||
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
|
||||
--repository-name superset-ci \
|
||||
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
|
||||
--image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
|
||||
|
||||
- name: Fail on missing container image
|
||||
if: steps.check-image.outcome == 'failure'
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{github.token}}
|
||||
github-token: ${{ github.token }}
|
||||
script: |
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
|
||||
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: errMsg
|
||||
})
|
||||
core.setFailed(errMsg)
|
||||
});
|
||||
core.setFailed(errMsg);
|
||||
|
||||
- name: Fill in the new image ID in the Amazon ECS task definition
|
||||
id: task-def
|
||||
|
|
@ -204,7 +205,7 @@ jobs:
|
|||
with:
|
||||
task-definition: .github/workflows/ecs-task-definition.json
|
||||
container-name: superset-ci
|
||||
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
|
||||
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
|
||||
|
||||
- name: Update env vars in the Amazon ECS task definition
|
||||
run: |
|
||||
|
|
@ -213,13 +214,10 @@ jobs:
|
|||
- name: Describe ECS service
|
||||
id: describe-services
|
||||
run: |
|
||||
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
|
||||
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
|
||||
- name: Create ECS service
|
||||
if: steps.describe-services.outputs.active != 'true'
|
||||
id: create-service
|
||||
env:
|
||||
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
|
||||
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
|
||||
if: steps.describe-services.outputs.active != 'true'
|
||||
run: |
|
||||
aws ecs create-service \
|
||||
--cluster superset-ci \
|
||||
|
|
@ -230,7 +228,6 @@ jobs:
|
|||
--platform-version LATEST \
|
||||
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
|
||||
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
|
||||
|
||||
- name: Deploy Amazon ECS task definition
|
||||
id: deploy-task
|
||||
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
|
||||
|
|
@ -245,17 +242,14 @@ jobs:
|
|||
id: list-tasks
|
||||
run: |
|
||||
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Get network interface
|
||||
id: get-eni
|
||||
run: |
|
||||
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
|
||||
|
||||
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
|
||||
- name: Get public IP
|
||||
id: get-ip
|
||||
run: |
|
||||
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Comment (success)
|
||||
if: ${{ success() }}
|
||||
uses: actions/github-script@v7
|
||||
|
|
@ -263,12 +257,11 @@ jobs:
|
|||
github-token: ${{github.token}}
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
|
||||
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
|
||||
})
|
||||
|
||||
- name: Comment (failure)
|
||||
if: ${{ failure() }}
|
||||
uses: actions/github-script@v7
|
||||
|
|
@ -276,8 +269,8 @@ jobs:
|
|||
github-token: ${{github.token}}
|
||||
script: |
|
||||
github.rest.issues.createComment({
|
||||
issue_number: ${{ github.event.issue.number }},
|
||||
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
|
||||
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
|
||||
})
|
||||
|
|
|
|||
12
Dockerfile
12
Dockerfile
|
|
@ -115,7 +115,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
|
|||
libldap2-dev \
|
||||
&& touch superset/static/version_info.json \
|
||||
&& chown -R superset:superset ./* \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
|
||||
# setup.py uses the version information in package.json
|
||||
|
|
@ -128,7 +128,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
|||
&& pip install --no-cache-dir --upgrade setuptools pip \
|
||||
&& pip install --no-cache-dir -r requirements/base.txt \
|
||||
&& apt-get autoremove -yqq --purge build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
# Copy the compiled frontend assets
|
||||
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
|
||||
|
|
@ -177,7 +177,7 @@ RUN apt-get update -qq \
|
|||
libxtst6 \
|
||||
git \
|
||||
pkg-config \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
|
||||
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-cache-dir playwright
|
||||
|
|
@ -199,13 +199,13 @@ RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
|
|||
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
|
||||
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
|
||||
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
|
||||
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
|
||||
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
|
||||
fi
|
||||
|
||||
# Installing mysql client os-level dependencies in dev image only because GPL
|
||||
RUN apt-get install -yqq --no-install-recommends \
|
||||
default-libmysqlclient-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
COPY --chown=superset:superset requirements/development.txt requirements/
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
|
|
@ -213,7 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
|
|||
build-essential \
|
||||
&& pip install --no-cache-dir -r requirements/development.txt \
|
||||
&& apt-get autoremove -yqq --purge build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
USER superset
|
||||
######################################################################
|
||||
|
|
|
|||
|
|
@ -25,7 +25,6 @@ x-superset-user: &superset-user root
|
|||
x-superset-depends-on: &superset-depends-on
|
||||
- db
|
||||
- redis
|
||||
- superset-checks
|
||||
x-superset-volumes: &superset-volumes
|
||||
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
|
||||
- ./docker:/app/docker
|
||||
|
|
@ -131,23 +130,6 @@ services:
|
|||
- REDIS_PORT=6379
|
||||
- REDIS_SSL=false
|
||||
|
||||
superset-checks:
|
||||
build:
|
||||
context: .
|
||||
target: python-base
|
||||
cache_from:
|
||||
- apache/superset-cache:3.10-slim-bookworm
|
||||
container_name: superset_checks
|
||||
command: ["/app/scripts/check-env.py"]
|
||||
env_file:
|
||||
- path: docker/.env # default
|
||||
required: true
|
||||
- path: docker/.env-local # optional override
|
||||
required: false
|
||||
user: *superset-user
|
||||
healthcheck:
|
||||
disable: true
|
||||
|
||||
superset-init:
|
||||
build:
|
||||
<<: *common-build
|
||||
|
|
@ -179,6 +161,7 @@ services:
|
|||
# set this to false if you have perf issues running the npm i; npm run dev in-docker
|
||||
# if you do so, you have to run this manually on the host, which should perform better!
|
||||
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
|
||||
NPM_RUN_PRUNE: false
|
||||
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
|
||||
container_name: superset_node
|
||||
command: ["/app/docker/docker-frontend.sh"]
|
||||
|
|
|
|||
|
|
@ -27,6 +27,11 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
|
|||
echo "Building Superset frontend in dev mode inside docker container"
|
||||
cd /app/superset-frontend
|
||||
|
||||
if [ "$NPM_RUN_PRUNE" = "true" ]; then
|
||||
echo "Running `npm run prune`"
|
||||
npm run prune
|
||||
fi
|
||||
|
||||
echo "Running `npm install`"
|
||||
npm install
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ We have a set of build "presets" that each represent a combination of
|
|||
parameters for the build, mostly pointing to either different target layer
|
||||
for the build, and/or base image.
|
||||
|
||||
Here are the build presets that are exposed through the `build_docker.py` script:
|
||||
Here are the build presets that are exposed through the `supersetbot docker` utility:
|
||||
|
||||
- `lean`: The default Docker image, including both frontend and backend. Tags
|
||||
without a build_preset are lean builds (ie: `latest`, `4.0.0`, `3.0.0`, ...). `lean`
|
||||
|
|
@ -62,8 +62,8 @@ Here are the build presets that are exposed through the `build_docker.py` script
|
|||
|
||||
|
||||
For insights or modifications to the build matrix and tagging conventions,
|
||||
check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
|
||||
script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
|
||||
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
|
||||
subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
|
||||
GitHub action.
|
||||
|
||||
## Key ARGs in Dockerfile
|
||||
|
|
|
|||
|
|
@ -95,6 +95,14 @@ perform those operations. In this case, we recommend you set the env var
|
|||
Simply trigger `npm i && npm run dev`, this should be MUCH faster.
|
||||
:::
|
||||
|
||||
:::tip
|
||||
Sometimes, your npm-related state can get out-of-wack, running `npm run prune` from
|
||||
the `superset-frontend/` folder will nuke the various' packages `node_module/` folders
|
||||
and help you start fresh. In the context of `docker compose` setting
|
||||
`export NPM_RUN_PRUNE=true` prior to running `docker compose up` will trigger that
|
||||
from within docker. This will slow down the startup, but will fix various npm-related issues.
|
||||
:::
|
||||
|
||||
### Option #2 - build a set of immutable images from the local branch
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -77,10 +77,6 @@ versions officially supported by Superset. We'd recommend using a Python version
|
|||
like [pyenv](https://github.com/pyenv/pyenv)
|
||||
(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
|
||||
|
||||
:::tip
|
||||
To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).
|
||||
:::
|
||||
|
||||
Let's also make sure we have the latest version of `pip` and `setuptools`:
|
||||
|
||||
```bash
|
||||
|
|
|
|||
|
|
@ -32,7 +32,7 @@ billiard==4.2.0
|
|||
# via celery
|
||||
blinker==1.9.0
|
||||
# via flask
|
||||
bottleneck==1.3.8
|
||||
bottleneck==1.4.2
|
||||
# via pandas
|
||||
brotli==1.1.0
|
||||
# via flask-compress
|
||||
|
|
@ -148,9 +148,7 @@ geopy==2.4.1
|
|||
google-auth==2.29.0
|
||||
# via shillelagh
|
||||
greenlet==3.0.3
|
||||
# via
|
||||
# shillelagh
|
||||
# sqlalchemy
|
||||
# via shillelagh
|
||||
gunicorn==22.0.0
|
||||
# via apache-superset
|
||||
hashids==1.3.1
|
||||
|
|
|
|||
|
|
@ -1,294 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
|
||||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
from textwrap import dedent
|
||||
|
||||
import click
|
||||
|
||||
REPO = "apache/superset"
|
||||
CACHE_REPO = f"{REPO}-cache"
|
||||
BASE_PY_IMAGE = "3.10-slim-bookworm"
|
||||
|
||||
|
||||
def run_cmd(command: str, raise_on_failure: bool = True) -> str:
|
||||
process = subprocess.Popen(
|
||||
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
|
||||
)
|
||||
|
||||
output = ""
|
||||
if process.stdout is not None:
|
||||
for line in iter(process.stdout.readline, ""):
|
||||
print(line.strip()) # Print the line to stdout in real-time
|
||||
output += line
|
||||
|
||||
process.wait() # Wait for the subprocess to finish
|
||||
|
||||
if process.returncode != 0 and raise_on_failure:
|
||||
raise subprocess.CalledProcessError(process.returncode, command, output)
|
||||
return output
|
||||
|
||||
|
||||
def get_git_sha() -> str:
|
||||
return run_cmd("git rev-parse HEAD").strip()
|
||||
|
||||
|
||||
def get_build_context_ref(build_context: str) -> str:
|
||||
"""
|
||||
Given a context, return a ref:
|
||||
- if context is pull_request, return the PR's id
|
||||
- if context is push, return the branch
|
||||
- if context is release, return the release ref
|
||||
"""
|
||||
|
||||
event = os.getenv("GITHUB_EVENT_NAME")
|
||||
github_ref = os.getenv("GITHUB_REF", "")
|
||||
|
||||
if event == "pull_request":
|
||||
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
|
||||
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
|
||||
elif event == "release":
|
||||
return re.sub("refs/tags/", "", github_ref)[:40]
|
||||
elif event == "push":
|
||||
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
|
||||
return ""
|
||||
|
||||
|
||||
def is_latest_release(release: str) -> bool:
|
||||
output = (
|
||||
run_cmd(
|
||||
f"./scripts/tag_latest_release.sh {release} --dry-run",
|
||||
raise_on_failure=False,
|
||||
)
|
||||
or ""
|
||||
)
|
||||
return "SKIP_TAG::false" in output
|
||||
|
||||
|
||||
def make_docker_tag(l: list[str]) -> str: # noqa: E741
|
||||
return f"{REPO}:" + "-".join([o for o in l if o])
|
||||
|
||||
|
||||
def get_docker_tags(
|
||||
build_preset: str,
|
||||
build_platforms: list[str],
|
||||
sha: str,
|
||||
build_context: str,
|
||||
build_context_ref: str,
|
||||
force_latest: bool = False,
|
||||
) -> set[str]:
|
||||
"""
|
||||
Return a set of tags given a given build context
|
||||
"""
|
||||
tags: set[str] = set()
|
||||
tag_chunks: list[str] = []
|
||||
|
||||
is_latest = is_latest_release(build_context_ref)
|
||||
|
||||
if build_preset != "lean":
|
||||
# Always add the preset_build name if different from default (lean)
|
||||
tag_chunks += [build_preset]
|
||||
|
||||
if len(build_platforms) == 1:
|
||||
build_platform = build_platforms[0]
|
||||
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
||||
if short_build_platform != "amd":
|
||||
# Always a platform indicator if different from default (amd)
|
||||
tag_chunks += [short_build_platform]
|
||||
|
||||
# Always craft a tag for the SHA
|
||||
tags.add(make_docker_tag([sha] + tag_chunks))
|
||||
# also a short SHA, cause it's nice
|
||||
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
|
||||
|
||||
if build_context == "release":
|
||||
# add a release tag
|
||||
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
|
||||
if is_latest or force_latest:
|
||||
# add a latest tag
|
||||
tags.add(make_docker_tag(["latest"] + tag_chunks))
|
||||
elif build_context == "push" and build_context_ref == "master":
|
||||
tags.add(make_docker_tag(["master"] + tag_chunks))
|
||||
elif build_context == "pull_request":
|
||||
tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
|
||||
return tags
|
||||
|
||||
|
||||
def get_docker_command(
|
||||
build_preset: str,
|
||||
build_platforms: list[str],
|
||||
is_authenticated: bool,
|
||||
sha: str,
|
||||
build_context: str,
|
||||
build_context_ref: str,
|
||||
force_latest: bool = False,
|
||||
) -> str:
|
||||
tag = "" # noqa: F841
|
||||
build_target = ""
|
||||
py_ver = BASE_PY_IMAGE
|
||||
docker_context = "."
|
||||
|
||||
if build_preset == "dev":
|
||||
build_target = "dev"
|
||||
elif build_preset == "lean":
|
||||
build_target = "lean"
|
||||
elif build_preset == "py311":
|
||||
build_target = "lean"
|
||||
py_ver = "3.11-slim-bookworm"
|
||||
elif build_preset == "websocket":
|
||||
build_target = ""
|
||||
docker_context = "superset-websocket"
|
||||
elif build_preset == "ci":
|
||||
build_target = "ci"
|
||||
elif build_preset == "dockerize":
|
||||
build_target = ""
|
||||
docker_context = "-f dockerize.Dockerfile ."
|
||||
else:
|
||||
print(f"Invalid build preset: {build_preset}")
|
||||
exit(1)
|
||||
|
||||
# Try to get context reference if missing
|
||||
if not build_context_ref:
|
||||
build_context_ref = get_build_context_ref(build_context)
|
||||
|
||||
tags = get_docker_tags(
|
||||
build_preset,
|
||||
build_platforms,
|
||||
sha,
|
||||
build_context,
|
||||
build_context_ref,
|
||||
force_latest,
|
||||
)
|
||||
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
|
||||
|
||||
docker_args = "--load" if not is_authenticated else "--push"
|
||||
target_argument = f"--target {build_target}" if build_target else ""
|
||||
|
||||
cache_ref = f"{CACHE_REPO}:{py_ver}"
|
||||
if len(build_platforms) == 1:
|
||||
build_platform = build_platforms[0]
|
||||
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
|
||||
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
|
||||
platform_arg = "--platform " + ",".join(build_platforms)
|
||||
|
||||
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
|
||||
cache_to_arg = (
|
||||
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
|
||||
)
|
||||
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
|
||||
actor = os.getenv("GITHUB_ACTOR")
|
||||
|
||||
return dedent(
|
||||
f"""\
|
||||
docker buildx build \\
|
||||
{docker_args} \\
|
||||
{docker_tags} \\
|
||||
{cache_from_arg} \\
|
||||
{cache_to_arg} \\
|
||||
{build_arg} \\
|
||||
{platform_arg} \\
|
||||
{target_argument} \\
|
||||
--label sha={sha} \\
|
||||
--label target={build_target} \\
|
||||
--label build_trigger={build_context} \\
|
||||
--label base={py_ver} \\
|
||||
--label build_actor={actor} \\
|
||||
{docker_context}"""
|
||||
)
|
||||
|
||||
|
||||
@click.command()
|
||||
@click.argument(
|
||||
"build_preset",
|
||||
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
|
||||
)
|
||||
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
|
||||
@click.option(
|
||||
"--platform",
|
||||
type=click.Choice(["linux/arm64", "linux/amd64"]),
|
||||
default=["linux/amd64"],
|
||||
multiple=True,
|
||||
)
|
||||
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
|
||||
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
|
||||
@click.option("--verbose", is_flag=True, help="Print more info")
|
||||
@click.option(
|
||||
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
|
||||
)
|
||||
def main(
|
||||
build_preset: str,
|
||||
build_context: str,
|
||||
build_context_ref: str,
|
||||
platform: list[str],
|
||||
dry_run: bool,
|
||||
force_latest: bool,
|
||||
verbose: bool,
|
||||
) -> None:
|
||||
"""
|
||||
This script executes docker build and push commands based on given arguments.
|
||||
"""
|
||||
|
||||
is_authenticated = (
|
||||
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
|
||||
)
|
||||
|
||||
if force_latest and build_context != "release":
|
||||
print(
|
||||
"--force-latest can only be applied if the build context is set to 'release'"
|
||||
)
|
||||
exit(1)
|
||||
|
||||
if build_context == "release" and not build_context_ref.strip():
|
||||
print("Release number has to be provided")
|
||||
exit(1)
|
||||
|
||||
docker_build_command = get_docker_command(
|
||||
build_preset,
|
||||
platform,
|
||||
is_authenticated,
|
||||
get_git_sha(),
|
||||
build_context,
|
||||
build_context_ref,
|
||||
force_latest,
|
||||
)
|
||||
|
||||
if not dry_run:
|
||||
print("Executing Docker Build Command:")
|
||||
print(docker_build_command)
|
||||
script = ""
|
||||
if os.getenv("DOCKERHUB_USER"):
|
||||
script = dedent(
|
||||
f"""\
|
||||
docker logout
|
||||
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
|
||||
DOCKER_ARGS="--push"
|
||||
"""
|
||||
)
|
||||
script = script + docker_build_command
|
||||
if verbose:
|
||||
run_cmd("cat Dockerfile")
|
||||
stdout = run_cmd(script) # noqa: F841
|
||||
else:
|
||||
print("Dry Run - Docker Build Command:")
|
||||
print(docker_build_command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -1,13 +0,0 @@
|
|||
{
|
||||
"name": "@superset-ui/switchboard",
|
||||
"version": "0.18.26-0",
|
||||
"lockfileVersion": 2,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@superset-ui/switchboard",
|
||||
"version": "0.18.26-0",
|
||||
"license": "Apache-2.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -33,7 +33,6 @@ from sqlalchemy import text
|
|||
from sqlalchemy.engine.reflection import Inspector
|
||||
from sqlalchemy.engine.url import URL
|
||||
from sqlalchemy.exc import NoSuchTableError
|
||||
from trino.exceptions import HttpError
|
||||
|
||||
from superset import db
|
||||
from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY, USER_AGENT
|
||||
|
|
@ -61,6 +60,12 @@ if TYPE_CHECKING:
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
try:
|
||||
# since trino is an optional dependency, we need to handle the ImportError
|
||||
from trino.exceptions import HttpError
|
||||
except ImportError:
|
||||
HttpError = Exception
|
||||
|
||||
|
||||
class CustomTrinoAuthErrorMeta(type):
|
||||
def __instancecheck__(cls, instance: object) -> bool:
|
||||
|
|
|
|||
|
|
@ -1,290 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
|
||||
PR_ID = "666"
|
||||
OLD_REL = "2.1.0"
|
||||
NEW_REL = "2.1.1"
|
||||
REPO = "apache/superset"
|
||||
|
||||
# Add the 'scripts' directory to sys.path
|
||||
scripts_dir = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), "../../../scripts")
|
||||
)
|
||||
sys.path.append(scripts_dir)
|
||||
|
||||
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def set_env_var():
|
||||
os.environ["TEST_ENV"] = "true"
|
||||
yield
|
||||
del os.environ["TEST_ENV"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"release, expected_bool",
|
||||
[
|
||||
("2.1.0", False),
|
||||
("2.1.1", True),
|
||||
("1.0.0", False),
|
||||
("3.0.0", True),
|
||||
],
|
||||
)
|
||||
def test_is_latest_release(release, expected_bool):
|
||||
assert docker_utils.is_latest_release(release) == expected_bool
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
|
||||
[
|
||||
# PRs
|
||||
(
|
||||
"lean",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
|
||||
),
|
||||
(
|
||||
"ci",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:pr-{PR_ID}-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
|
||||
),
|
||||
# old releases
|
||||
(
|
||||
"lean",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:{OLD_REL}-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
|
||||
),
|
||||
# new releases
|
||||
(
|
||||
"lean",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-arm",
|
||||
f"{REPO}:{SHA}-arm",
|
||||
f"{REPO}:{NEW_REL}-arm",
|
||||
f"{REPO}:latest-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:{NEW_REL}-dev-arm",
|
||||
f"{REPO}:latest-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev",
|
||||
f"{REPO}:{SHA}-dev",
|
||||
f"{REPO}:{NEW_REL}-dev",
|
||||
f"{REPO}:latest-dev",
|
||||
],
|
||||
),
|
||||
# merge on master
|
||||
(
|
||||
"lean",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/arm64"],
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:master-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/amd64"],
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_docker_tags(
|
||||
build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
|
||||
):
|
||||
tags = docker_utils.get_docker_tags(
|
||||
build_preset, build_platforms, sha, build_context, build_context_ref
|
||||
)
|
||||
for tag in expected_tags:
|
||||
assert tag in tags
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
|
||||
[
|
||||
(
|
||||
"lean",
|
||||
["linux/amd64"],
|
||||
True,
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
["--push", f"-t {REPO}:master "],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
["linux/amd64"],
|
||||
False,
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
["--load", f"-t {REPO}:master-dev ", "--target dev"],
|
||||
),
|
||||
# multi-platform
|
||||
(
|
||||
"lean",
|
||||
["linux/arm64", "linux/amd64"],
|
||||
True,
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
["--platform linux/arm64,linux/amd64"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_docker_command(
|
||||
build_preset,
|
||||
build_platforms,
|
||||
is_authenticated,
|
||||
sha,
|
||||
build_context,
|
||||
build_context_ref,
|
||||
contains,
|
||||
):
|
||||
cmd = docker_utils.get_docker_command(
|
||||
build_preset,
|
||||
build_platforms,
|
||||
is_authenticated,
|
||||
sha,
|
||||
build_context,
|
||||
build_context_ref,
|
||||
)
|
||||
for s in contains:
|
||||
assert s in cmd
|
||||
|
|
@ -1,268 +0,0 @@
|
|||
# Licensed to the Apache Software Foundation (ASF) under one
|
||||
# or more contributor license agreements. See the NOTICE file
|
||||
# distributed with this work for additional information
|
||||
# regarding copyright ownership. The ASF licenses this file
|
||||
# to you under the Apache License, Version 2.0 (the
|
||||
# "License"); you may not use this file except in compliance
|
||||
# with the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing,
|
||||
# software distributed under the License is distributed on an
|
||||
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
# KIND, either express or implied. See the License for the
|
||||
# specific language governing permissions and limitations
|
||||
# under the License.
|
||||
import os
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
|
||||
PR_ID = "666"
|
||||
OLD_REL = "2.1.0"
|
||||
NEW_REL = "2.1.1"
|
||||
REPO = "apache/superset"
|
||||
|
||||
# Add the 'scripts' directory to sys.path
|
||||
scripts_dir = os.path.abspath(
|
||||
os.path.join(os.path.dirname(__file__), "../../../scripts")
|
||||
)
|
||||
sys.path.append(scripts_dir)
|
||||
|
||||
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def set_env_var():
|
||||
os.environ["TEST_ENV"] = "true"
|
||||
yield
|
||||
del os.environ["TEST_ENV"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"release, expected_bool",
|
||||
[
|
||||
("2.1.0", False),
|
||||
("2.1.1", True),
|
||||
("1.0.0", False),
|
||||
("3.0.0", True),
|
||||
],
|
||||
)
|
||||
def test_is_latest_release(release, expected_bool):
|
||||
assert docker_utils.is_latest_release(release) == expected_bool
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
|
||||
[
|
||||
# PRs
|
||||
(
|
||||
"lean",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"pull_request",
|
||||
PR_ID,
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
|
||||
),
|
||||
# old releases
|
||||
(
|
||||
"lean",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:{OLD_REL}-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"release",
|
||||
OLD_REL,
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
|
||||
),
|
||||
# new releases
|
||||
(
|
||||
"lean",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-arm",
|
||||
f"{REPO}:{SHA}-arm",
|
||||
f"{REPO}:{NEW_REL}-arm",
|
||||
f"{REPO}:latest-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:{NEW_REL}-dev-arm",
|
||||
f"{REPO}:latest-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"release",
|
||||
NEW_REL,
|
||||
[
|
||||
f"{REPO}:22e7c60-dev",
|
||||
f"{REPO}:{SHA}-dev",
|
||||
f"{REPO}:{NEW_REL}-dev",
|
||||
f"{REPO}:latest-dev",
|
||||
],
|
||||
),
|
||||
# merge on master
|
||||
(
|
||||
"lean",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
|
||||
),
|
||||
(
|
||||
"lean",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/arm64",
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[
|
||||
f"{REPO}:22e7c60-dev-arm",
|
||||
f"{REPO}:{SHA}-dev-arm",
|
||||
f"{REPO}:master-dev-arm",
|
||||
],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/amd64",
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_docker_tags(
|
||||
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
|
||||
):
|
||||
tags = docker_utils.get_docker_tags(
|
||||
build_preset, build_platform, sha, build_context, build_context_ref
|
||||
)
|
||||
for tag in expected_tags:
|
||||
assert tag in tags
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
|
||||
[
|
||||
(
|
||||
"lean",
|
||||
"linux/amd64",
|
||||
True,
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
["--push", f"-t {REPO}:master "],
|
||||
),
|
||||
(
|
||||
"dev",
|
||||
"linux/amd64",
|
||||
False,
|
||||
SHA,
|
||||
"push",
|
||||
"master",
|
||||
["--load", f"-t {REPO}:master-dev "],
|
||||
),
|
||||
],
|
||||
)
|
||||
def test_get_docker_command(
|
||||
build_preset,
|
||||
build_platform,
|
||||
is_authenticated,
|
||||
sha,
|
||||
build_context,
|
||||
build_context_ref,
|
||||
contains,
|
||||
):
|
||||
cmd = docker_utils.get_docker_command(
|
||||
build_preset,
|
||||
build_platform,
|
||||
is_authenticated,
|
||||
sha,
|
||||
build_context,
|
||||
build_context_ref,
|
||||
)
|
||||
for s in contains:
|
||||
assert s in cmd
|
||||
Loading…
Reference in New Issue