iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: make ephemeral env use supersetbot + deprecate build_docker.py (#30870)

This commit is contained in:
Maxime Beauchemin 2024-11-22 14:19:08 -08:00 committed by GitHub
parent abf3790ea6
commit e0deb704f9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 82 additions and 959 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/actions/chart-releaser-action vendored

@ -1 +1 @@
Subproject commit a917fd15b20e8b64b94d9158ad54cd6345335584
Subproject commit 120944e66390c2534cc1b3c62d7285ba7ff02594

107
.github/workflows/ephemeral-env.yml vendored
View File

@ -1,30 +1,25 @@
name: Ephemeral env workflow
# Example manual trigger: gh workflow run ephemeral-env.yml --ref fix_ephemerals --field comment_body="/testenv up" --field issue_number=666
on:
issue_comment:
types: [created]
workflow_dispatch:
inputs:
comment_body:
description: 'Comment body to simulate /testenv command'
required: true
default: '/testenv up'
issue_number:
description: 'Issue or PR number'
required: true
jobs:
config:
runs-on: "ubuntu-22.04"
if: github.event.issue.pull_request
outputs:
has-secrets: ${{ steps.check.outputs.has-secrets }}
steps:
- name: "Check for secrets"
id: check
shell: bash
run: |
if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
echo "has-secrets=1" >> "$GITHUB_OUTPUT"
fi
ephemeral-env-comment:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
cancel-in-progress: true
needs: config
if: needs.config.outputs.has-secrets
name: Evaluate ephemeral env comment trigger (/testenv)
runs-on: ubuntu-22.04
permissions:
@ -44,18 +39,18 @@ jobs:
with:
result-encoding: string
script: |
const pattern = /^\/testenv (up|down)/
const result = pattern.exec(context.payload.comment.body)
return result === null ? 'noop' : result[1]
const pattern = /^\/testenv (up|down)/;
const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}');
return result === null ? 'noop' : result[1];
- name: Eval comment body for feature flags
- name: Looking for feature flags
uses: actions/github-script@v7
id: eval-feature-flags
with:
script: |
const pattern = /FEATURE_(\w+)=(\w+)/g;
let results = [];
[...context.payload.comment.body.matchAll(pattern)].forEach(match => {
[...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => {
const config = {
name: `SUPERSET_FEATURE_${match[1]}`,
value: match[2],
@ -67,24 +62,25 @@ jobs:
- name: Limit to committers
if: >
steps.eval-body.outputs.result != 'noop' &&
github.event_name == 'issue_comment' &&
github.event.comment.author_association != 'MEMBER' &&
github.event.comment.author_association != 'OWNER'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
ephemeral-docker-build:
concurrency:
group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
cancel-in-progress: true
needs: ephemeral-env-comment
name: ephemeral-docker-build
@ -98,9 +94,9 @@ jobs:
const request = {
owner: context.repo.owner,
repo: context.repo.repo,
pull_number: ${{ github.event.issue.number }},
}
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
};
core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
const pr = await github.rest.pulls.get(request);
return pr.data;
@ -121,12 +117,17 @@ jobs:
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
- name: Build ephemeral env image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
./scripts/build_docker.py \
"ci" \
"pull_request" \
--build_context_ref ${{ github.event.issue.number }}
supersetbot docker \
--preset ci \
--platform linux/amd64 \
--context-ref "$RELEASE"
- name: Configure AWS credentials
uses: aws-actions/configure-aws-credentials@v4
@ -146,7 +147,7 @@ jobs:
ECR_REPOSITORY: superset-ci
IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
run: |
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
ephemeral-env-up:
@ -181,22 +182,22 @@ jobs:
aws ecr describe-images \
--registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
--repository-name superset-ci \
--image-ids imageTag=pr-${{ github.event.issue.number }}-ci
--image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Fail on missing container image
if: steps.check-image.outcome == 'failure'
uses: actions/github-script@v7
with:
github-token: ${{github.token}}
github-token: ${{ github.token }}
script: |
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: errMsg
})
core.setFailed(errMsg)
});
core.setFailed(errMsg);
- name: Fill in the new image ID in the Amazon ECS task definition
id: task-def
@ -204,7 +205,7 @@ jobs:
with:
task-definition: .github/workflows/ecs-task-definition.json
container-name: superset-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
- name: Update env vars in the Amazon ECS task definition
run: |
@ -213,13 +214,10 @@ jobs:
- name: Describe ECS service
id: describe-services
run: |
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
- name: Create ECS service
if: steps.describe-services.outputs.active != 'true'
id: create-service
env:
ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
if: steps.describe-services.outputs.active != 'true'
run: |
aws ecs create-service \
--cluster superset-ci \
@ -230,7 +228,6 @@ jobs:
--platform-version LATEST \
--network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
--tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
- name: Deploy Amazon ECS task definition
id: deploy-task
uses: aws-actions/amazon-ecs-deploy-task-definition@v2
@ -245,17 +242,14 @@ jobs:
id: list-tasks
run: |
echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
- name: Get network interface
id: get-eni
run: |
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
- name: Get public IP
id: get-ip
run: |
echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
- name: Comment (success)
if: ${{ success() }}
uses: actions/github-script@v7
@ -263,12 +257,11 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
})
- name: Comment (failure)
if: ${{ failure() }}
uses: actions/github-script@v7
@ -276,8 +269,8 @@ jobs:
github-token: ${{github.token}}
script: |
github.rest.issues.createComment({
issue_number: ${{ github.event.issue.number }},
issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
owner: context.repo.owner,
repo: context.repo.repo,
body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
})

12
Dockerfile
View File

@ -115,7 +115,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
libldap2-dev \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
# setup.py uses the version information in package.json
@ -128,7 +128,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
&& pip install --no-cache-dir --upgrade setuptools pip \
&& pip install --no-cache-dir -r requirements/base.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
# Copy the compiled frontend assets
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
@ -177,7 +177,7 @@ RUN apt-get update -qq \
libxtst6 \
git \
pkg-config \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
RUN --mount=type=cache,target=/root/.cache/pip \
pip install --no-cache-dir playwright
@ -199,13 +199,13 @@ RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
fi
# Installing mysql client os-level dependencies in dev image only because GPL
RUN apt-get install -yqq --no-install-recommends \
default-libmysqlclient-dev \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
COPY --chown=superset:superset requirements/development.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
@ -213,7 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
build-essential \
&& pip install --no-cache-dir -r requirements/development.txt \
&& apt-get autoremove -yqq --purge build-essential \
&& rm -rf /var/lib/apt/lists/*
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
USER superset
######################################################################

19
docker-compose.yml
View File

@ -25,7 +25,6 @@ x-superset-user: &superset-user root
x-superset-depends-on: &superset-depends-on
- db
- redis
- superset-checks
x-superset-volumes: &superset-volumes
# /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
- ./docker:/app/docker
@ -131,23 +130,6 @@ services:
- REDIS_PORT=6379
- REDIS_SSL=false
superset-checks:
build:
context: .
target: python-base
cache_from:
- apache/superset-cache:3.10-slim-bookworm
container_name: superset_checks
command: ["/app/scripts/check-env.py"]
env_file:
- path: docker/.env # default
required: true
- path: docker/.env-local # optional override
required: false
user: *superset-user
healthcheck:
disable: true
superset-init:
build:
<<: *common-build
@ -179,6 +161,7 @@ services:
# set this to false if you have perf issues running the npm i; npm run dev in-docker
# if you do so, you have to run this manually on the host, which should perform better!
BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
NPM_RUN_PRUNE: false
SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
container_name: superset_node
command: ["/app/docker/docker-frontend.sh"]

5
docker/docker-frontend.sh
View File

@ -27,6 +27,11 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
echo "Building Superset frontend in dev mode inside docker container"
cd /app/superset-frontend
if [ "$NPM_RUN_PRUNE" = "true" ]; then
echo "Running `npm run prune`"
npm run prune
fi
echo "Running `npm install`"
npm install

6
docs/docs/installation/docker-builds.mdx
View File

@ -29,7 +29,7 @@ We have a set of build "presets" that each represent a combination of
parameters for the build, mostly pointing to either different target layer
for the build, and/or base image.
Here are the build presets that are exposed through the `build_docker.py` script:
Here are the build presets that are exposed through the `supersetbot docker` utility:
- `lean`: The default Docker image, including both frontend and backend. Tags
without a build_preset are lean builds (ie: `latest`, `4.0.0`, `3.0.0`, ...). `lean`
@ -62,8 +62,8 @@ Here are the build presets that are exposed through the `build_docker.py` script
For insights or modifications to the build matrix and tagging conventions,
check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
GitHub action.
## Key ARGs in Dockerfile

8
docs/docs/installation/docker-compose.mdx
View File

@ -95,6 +95,14 @@ perform those operations. In this case, we recommend you set the env var
Simply trigger `npm i && npm run dev`, this should be MUCH faster.
:::
:::tip
Sometimes, your npm-related state can get out-of-wack, running `npm run prune` from
the `superset-frontend/` folder will nuke the various' packages `node_module/` folders
and help you start fresh. In the context of `docker compose` setting
`export NPM_RUN_PRUNE=true` prior to running `docker compose up` will trigger that
from within docker. This will slow down the startup, but will fix various npm-related issues.
:::
### Option #2 - build a set of immutable images from the local branch
```bash

4
docs/docs/installation/pypi.mdx
View File

@ -77,10 +77,6 @@ versions officially supported by Superset. We'd recommend using a Python version
like [pyenv](https://github.com/pyenv/pyenv)
(and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
:::tip
To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).
:::
Let's also make sure we have the latest version of `pip` and `setuptools`:
```bash

6
requirements/base.txt
View File

@ -32,7 +32,7 @@ billiard==4.2.0
# via celery
blinker==1.9.0
# via flask
bottleneck==1.3.8
bottleneck==1.4.2
# via pandas
brotli==1.1.0
# via flask-compress
@ -148,9 +148,7 @@ geopy==2.4.1
google-auth==2.29.0
# via shillelagh
greenlet==3.0.3
# via
# shillelagh
# sqlalchemy
# via shillelagh
gunicorn==22.0.0
# via apache-superset
hashids==1.3.1

294
scripts/build_docker.py
View File

@ -1,294 +0,0 @@
#!/usr/bin/env python3
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import re
import subprocess
from textwrap import dedent
import click
REPO = "apache/superset"
CACHE_REPO = f"{REPO}-cache"
BASE_PY_IMAGE = "3.10-slim-bookworm"
def run_cmd(command: str, raise_on_failure: bool = True) -> str:
process = subprocess.Popen(
command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
)
output = ""
if process.stdout is not None:
for line in iter(process.stdout.readline, ""):
print(line.strip()) # Print the line to stdout in real-time
output += line
process.wait() # Wait for the subprocess to finish
if process.returncode != 0 and raise_on_failure:
raise subprocess.CalledProcessError(process.returncode, command, output)
return output
def get_git_sha() -> str:
return run_cmd("git rev-parse HEAD").strip()
def get_build_context_ref(build_context: str) -> str:
"""
Given a context, return a ref:
- if context is pull_request, return the PR's id
- if context is push, return the branch
- if context is release, return the release ref
"""
event = os.getenv("GITHUB_EVENT_NAME")
github_ref = os.getenv("GITHUB_REF", "")
if event == "pull_request":
github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
elif event == "release":
return re.sub("refs/tags/", "", github_ref)[:40]
elif event == "push":
return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
return ""
def is_latest_release(release: str) -> bool:
output = (
run_cmd(
f"./scripts/tag_latest_release.sh {release} --dry-run",
raise_on_failure=False,
)
or ""
)
return "SKIP_TAG::false" in output
def make_docker_tag(l: list[str]) -> str: # noqa: E741
return f"{REPO}:" + "-".join([o for o in l if o])
def get_docker_tags(
build_preset: str,
build_platforms: list[str],
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> set[str]:
"""
Return a set of tags given a given build context
"""
tags: set[str] = set()
tag_chunks: list[str] = []
is_latest = is_latest_release(build_context_ref)
if build_preset != "lean":
# Always add the preset_build name if different from default (lean)
tag_chunks += [build_preset]
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
if short_build_platform != "amd":
# Always a platform indicator if different from default (amd)
tag_chunks += [short_build_platform]
# Always craft a tag for the SHA
tags.add(make_docker_tag([sha] + tag_chunks))
# also a short SHA, cause it's nice
tags.add(make_docker_tag([sha[:7]] + tag_chunks))
if build_context == "release":
# add a release tag
tags.add(make_docker_tag([build_context_ref] + tag_chunks))
if is_latest or force_latest:
# add a latest tag
tags.add(make_docker_tag(["latest"] + tag_chunks))
elif build_context == "push" and build_context_ref == "master":
tags.add(make_docker_tag(["master"] + tag_chunks))
elif build_context == "pull_request":
tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
return tags
def get_docker_command(
build_preset: str,
build_platforms: list[str],
is_authenticated: bool,
sha: str,
build_context: str,
build_context_ref: str,
force_latest: bool = False,
) -> str:
tag = "" # noqa: F841
build_target = ""
py_ver = BASE_PY_IMAGE
docker_context = "."
if build_preset == "dev":
build_target = "dev"
elif build_preset == "lean":
build_target = "lean"
elif build_preset == "py311":
build_target = "lean"
py_ver = "3.11-slim-bookworm"
elif build_preset == "websocket":
build_target = ""
docker_context = "superset-websocket"
elif build_preset == "ci":
build_target = "ci"
elif build_preset == "dockerize":
build_target = ""
docker_context = "-f dockerize.Dockerfile ."
else:
print(f"Invalid build preset: {build_preset}")
exit(1)
# Try to get context reference if missing
if not build_context_ref:
build_context_ref = get_build_context_ref(build_context)
tags = get_docker_tags(
build_preset,
build_platforms,
sha,
build_context,
build_context_ref,
force_latest,
)
docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
docker_args = "--load" if not is_authenticated else "--push"
target_argument = f"--target {build_target}" if build_target else ""
cache_ref = f"{CACHE_REPO}:{py_ver}"
if len(build_platforms) == 1:
build_platform = build_platforms[0]
short_build_platform = build_platform.replace("linux/", "").replace("64", "")
cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
platform_arg = "--platform " + ",".join(build_platforms)
cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
cache_to_arg = (
f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
)
build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
actor = os.getenv("GITHUB_ACTOR")
return dedent(
f"""\
docker buildx build \\
{docker_args} \\
{docker_tags} \\
{cache_from_arg} \\
{cache_to_arg} \\
{build_arg} \\
{platform_arg} \\
{target_argument} \\
--label sha={sha} \\
--label target={build_target} \\
--label build_trigger={build_context} \\
--label base={py_ver} \\
--label build_actor={actor} \\
{docker_context}"""
)
@click.command()
@click.argument(
"build_preset",
type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
)
@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
@click.option(
"--platform",
type=click.Choice(["linux/arm64", "linux/amd64"]),
default=["linux/amd64"],
multiple=True,
)
@click.option("--build_context_ref", help="a reference to the pr, release or branch")
@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
@click.option("--verbose", is_flag=True, help="Print more info")
@click.option(
"--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
)
def main(
build_preset: str,
build_context: str,
build_context_ref: str,
platform: list[str],
dry_run: bool,
force_latest: bool,
verbose: bool,
) -> None:
"""
This script executes docker build and push commands based on given arguments.
"""
is_authenticated = (
True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
)
if force_latest and build_context != "release":
print(
"--force-latest can only be applied if the build context is set to 'release'"
)
exit(1)
if build_context == "release" and not build_context_ref.strip():
print("Release number has to be provided")
exit(1)
docker_build_command = get_docker_command(
build_preset,
platform,
is_authenticated,
get_git_sha(),
build_context,
build_context_ref,
force_latest,
)
if not dry_run:
print("Executing Docker Build Command:")
print(docker_build_command)
script = ""
if os.getenv("DOCKERHUB_USER"):
script = dedent(
f"""\
docker logout
docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
DOCKER_ARGS="--push"
"""
)
script = script + docker_build_command
if verbose:
run_cmd("cat Dockerfile")
stdout = run_cmd(script) # noqa: F841
else:
print("Dry Run - Docker Build Command:")
print(docker_build_command)
if __name__ == "__main__":
main()

13
superset-frontend/packages/superset-ui-switchboard/package-lock.json generated
View File

@ -1,13 +0,0 @@
{
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"lockfileVersion": 2,
"requires": true,
"packages": {
"": {
"name": "@superset-ui/switchboard",
"version": "0.18.26-0",
"license": "Apache-2.0"
}
}
}

7
superset/db_engine_specs/trino.py
View File

@ -33,7 +33,6 @@ from sqlalchemy import text
from sqlalchemy.engine.reflection import Inspector
from sqlalchemy.engine.url import URL
from sqlalchemy.exc import NoSuchTableError
from trino.exceptions import HttpError
from superset import db
from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY, USER_AGENT
@ -61,6 +60,12 @@ if TYPE_CHECKING:
logger = logging.getLogger(__name__)
try:
# since trino is an optional dependency, we need to handle the ImportError
from trino.exceptions import HttpError
except ImportError:
HttpError = Exception
class CustomTrinoAuthErrorMeta(type):
def __instancecheck__(cls, instance: object) -> bool:

290
tests/unit_tests/scripts/docker_build.py
View File

@ -1,290 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
),
(
"ci",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
),
(
"lean",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
),
(
"dev",
["linux/arm64"],
SHA,
"pull_request",
PR_ID,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:pr-{PR_ID}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
),
# old releases
(
"lean",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
["linux/arm64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
["linux/arm64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
["linux/arm64"],
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
["linux/amd64"],
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platforms, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
["linux/amd64"],
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
["linux/amd64"],
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev ", "--target dev"],
),
# multi-platform
(
"lean",
["linux/arm64", "linux/amd64"],
True,
SHA,
"push",
"master",
["--platform linux/arm64,linux/amd64"],
),
],
)
def test_get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platforms,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd

268
tests/unit_tests/utils/docker.py
View File

@ -1,268 +0,0 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import os
import sys
import pytest
SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
PR_ID = "666"
OLD_REL = "2.1.0"
NEW_REL = "2.1.1"
REPO = "apache/superset"
# Add the 'scripts' directory to sys.path
scripts_dir = os.path.abspath(
os.path.join(os.path.dirname(__file__), "../../../scripts")
)
sys.path.append(scripts_dir)
import build_docker as docker_utils # Replace with the actual function name # noqa: E402
@pytest.fixture(autouse=True)
def set_env_var():
os.environ["TEST_ENV"] = "true"
yield
del os.environ["TEST_ENV"]
@pytest.mark.parametrize(
"release, expected_bool",
[
("2.1.0", False),
("2.1.1", True),
("1.0.0", False),
("3.0.0", True),
],
)
def test_is_latest_release(release, expected_bool):
assert docker_utils.is_latest_release(release) == expected_bool
@pytest.mark.parametrize(
"build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
[
# PRs
(
"lean",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
),
(
"dev",
"linux/arm64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
),
(
"dev",
"linux/amd64",
SHA,
"pull_request",
PR_ID,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
),
# old releases
(
"lean",
"linux/arm64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
OLD_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{OLD_REL}-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
OLD_REL,
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
),
# new releases
(
"lean",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-arm",
f"{REPO}:{SHA}-arm",
f"{REPO}:{NEW_REL}-arm",
f"{REPO}:latest-arm",
],
),
(
"lean",
"linux/amd64",
SHA,
"release",
NEW_REL,
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
),
(
"dev",
"linux/arm64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:{NEW_REL}-dev-arm",
f"{REPO}:latest-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"release",
NEW_REL,
[
f"{REPO}:22e7c60-dev",
f"{REPO}:{SHA}-dev",
f"{REPO}:{NEW_REL}-dev",
f"{REPO}:latest-dev",
],
),
# merge on master
(
"lean",
"linux/arm64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
),
(
"lean",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
),
(
"dev",
"linux/arm64",
SHA,
"push",
"master",
[
f"{REPO}:22e7c60-dev-arm",
f"{REPO}:{SHA}-dev-arm",
f"{REPO}:master-dev-arm",
],
),
(
"dev",
"linux/amd64",
SHA,
"push",
"master",
[f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
),
],
)
def test_get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
):
tags = docker_utils.get_docker_tags(
build_preset, build_platform, sha, build_context, build_context_ref
)
for tag in expected_tags:
assert tag in tags
@pytest.mark.parametrize(
"build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
[
(
"lean",
"linux/amd64",
True,
SHA,
"push",
"master",
["--push", f"-t {REPO}:master "],
),
(
"dev",
"linux/amd64",
False,
SHA,
"push",
"master",
["--load", f"-t {REPO}:master-dev "],
),
],
)
def test_get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
contains,
):
cmd = docker_utils.get_docker_command(
build_preset,
build_platform,
is_authenticated,
sha,
build_context,
build_context_ref,
)
for s in contains:
assert s in cmd