feat: make ephemeral env use supersetbot + deprecate build_docker.py (#30870)

.github/actions/chart-releaser-action

@@ -1 +1 @@
-Subproject commit a917fd15b20e8b64b94d9158ad54cd6345335584
+Subproject commit 120944e66390c2534cc1b3c62d7285ba7ff02594

.github/workflows/ephemeral-env.yml

@@ -1,30 +1,25 @@
 name: Ephemeral env workflow
 
+# Example manual trigger:  gh workflow run ephemeral-env.yml --ref fix_ephemerals  --field comment_body="/testenv up" --field issue_number=666
+
 on:
   issue_comment:
     types: [created]
+  workflow_dispatch:
+    inputs:
+      comment_body:
+        description: 'Comment body to simulate /testenv command'
+        required: true
+        default: '/testenv up'
+      issue_number:
+        description: 'Issue or PR number'
+        required: true
 
 jobs:
-  config:
-    runs-on: "ubuntu-22.04"
-    if: github.event.issue.pull_request
-    outputs:
-      has-secrets: ${{ steps.check.outputs.has-secrets }}
-    steps:
-      - name: "Check for secrets"
-        id: check
-        shell: bash
-        run: |
-          if [ -n "${{ (secrets.AWS_ACCESS_KEY_ID != '' && secrets.AWS_SECRET_ACCESS_KEY != '') || '' }}" ]; then
-            echo "has-secrets=1" >> "$GITHUB_OUTPUT"
-          fi
-
   ephemeral-env-comment:
     concurrency:
-      group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-comment
+      group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-comment
       cancel-in-progress: true
-    needs: config
-    if: needs.config.outputs.has-secrets
     name: Evaluate ephemeral env comment trigger (/testenv)
     runs-on: ubuntu-22.04
     permissions:
@@ -44,18 +39,18 @@ jobs:
       with:
         result-encoding: string
         script: |
-          const pattern = /^\/testenv (up|down)/
+          const pattern = /^\/testenv (up|down)/;
-          const result = pattern.exec(context.payload.comment.body)
+          const result = pattern.exec('${{ github.event.inputs.comment_body || github.event.comment.body }}');
-          return result === null ? 'noop' : result[1]
+          return result === null ? 'noop' : result[1];
 
-    - name: Eval comment body for feature flags
+    - name: Looking for feature flags
       uses: actions/github-script@v7
       id: eval-feature-flags
       with:
         script: |
           const pattern = /FEATURE_(\w+)=(\w+)/g;
           let results = [];
-          [...context.payload.comment.body.matchAll(pattern)].forEach(match => {
+          [...'${{ github.event.inputs.comment_body || github.event.comment.body }}'.matchAll(pattern)].forEach(match => {
             const config = {
               name: `SUPERSET_FEATURE_${match[1]}`,
               value: match[2],
@@ -67,24 +62,25 @@ jobs:
     - name: Limit to committers
       if: >
         steps.eval-body.outputs.result != 'noop' &&
+        github.event_name == 'issue_comment' &&
         github.event.comment.author_association != 'MEMBER' &&
         github.event.comment.author_association != 'OWNER'
       uses: actions/github-script@v7
       with:
-        github-token: ${{github.token}}
+        github-token: ${{ github.token }}
         script: |
-          const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.'
+          const errMsg = '@${{ github.event.comment.user.login }} Ephemeral environment creation is currently limited to committers.';
           github.rest.issues.createComment({
             issue_number: ${{ github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: errMsg
-          })
+          });
-          core.setFailed(errMsg)
+          core.setFailed(errMsg);
 
   ephemeral-docker-build:
     concurrency:
-      group: ${{ github.workflow }}-${{ github.event.issue.number || github.run_id }}-build
+      group: ${{ github.workflow }}-${{ github.event.inputs.issue_number || github.event.issue.number || github.run_id }}-build
       cancel-in-progress: true
     needs: ephemeral-env-comment
     name: ephemeral-docker-build
@@ -98,9 +94,9 @@ jobs:
             const request = {
                 owner: context.repo.owner,
                 repo: context.repo.repo,
-                pull_number: ${{ github.event.issue.number }},
+                pull_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
-            }
+            };
-            core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`)
+            core.info(`Getting PR #${request.pull_number} from ${request.owner}/${request.repo}`);
             const pr = await github.rest.pulls.get(request);
             return pr.data;
 
@@ -121,12 +117,17 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
+      - name: Setup supersetbot
+        uses: ./.github/actions/setup-supersetbot/
+
       - name: Build ephemeral env image
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
-          ./scripts/build_docker.py \
+          supersetbot docker \
-            "ci" \
+            --preset ci \
-            "pull_request" \
+            --platform  linux/amd64 \
-            --build_context_ref ${{ github.event.issue.number }}
+            --context-ref "$RELEASE"
 
       - name: Configure AWS credentials
         uses: aws-actions/configure-aws-credentials@v4
@@ -146,7 +147,7 @@ jobs:
           ECR_REPOSITORY: superset-ci
           IMAGE_TAG: apache/superset:${{ steps.get-sha.outputs.sha }}-ci
         run: |
-          docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.issue.number }}-ci
+          docker tag $IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
           docker push -a $ECR_REGISTRY/$ECR_REPOSITORY
 
   ephemeral-env-up:
@@ -181,22 +182,22 @@ jobs:
         aws ecr describe-images \
         --registry-id $(echo "${{ steps.login-ecr.outputs.registry }}" | grep -Eo "^[0-9]+") \
         --repository-name superset-ci \
-        --image-ids imageTag=pr-${{ github.event.issue.number }}-ci
+        --image-ids imageTag=pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
 
     - name: Fail on missing container image
       if: steps.check-image.outcome == 'failure'
       uses: actions/github-script@v7
       with:
-        github-token: ${{github.token}}
+        github-token: ${{ github.token }}
         script: |
-          const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.'
+          const errMsg = '@${{ github.event.comment.user.login }} Container image not yet published for this PR. Please try again when build is complete.';
           github.rest.issues.createComment({
-            issue_number: ${{ github.event.issue.number }},
+            issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
             body: errMsg
-          })
+          });
-          core.setFailed(errMsg)
+          core.setFailed(errMsg);
 
     - name: Fill in the new image ID in the Amazon ECS task definition
       id: task-def
@@ -204,7 +205,7 @@ jobs:
       with:
         task-definition: .github/workflows/ecs-task-definition.json
         container-name: superset-ci
-        image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.issue.number }}-ci
+        image: ${{ steps.login-ecr.outputs.registry }}/superset-ci:pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-ci
 
     - name: Update env vars in the Amazon ECS task definition
       run: |
@@ -213,13 +214,10 @@ jobs:
     - name: Describe ECS service
       id: describe-services
       run: |
-        echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
+        echo "active=$(aws ecs describe-services --cluster superset-ci --services pr-${{ github.event.inputs.issue_number || github.event.issue.number }}-service | jq '.services[] | select(.status == "ACTIVE") | any')" >> $GITHUB_OUTPUT
     - name: Create ECS service
-      if: steps.describe-services.outputs.active != 'true'
       id: create-service
-      env:
+      if: steps.describe-services.outputs.active != 'true'
-        ECR_SUBNETS: subnet-0e15a5034b4121710,subnet-0e8efef4a72224974
-        ECR_SECURITY_GROUP: sg-092ff3a6ae0574d91
       run: |
         aws ecs create-service \
         --cluster superset-ci \
@@ -230,7 +228,6 @@ jobs:
         --platform-version LATEST \
         --network-configuration "awsvpcConfiguration={subnets=[$ECR_SUBNETS],securityGroups=[$ECR_SECURITY_GROUP],assignPublicIp=ENABLED}" \
         --tags key=pr,value=${{ github.event.issue.number }} key=github_user,value=${{ github.actor }}
-
     - name: Deploy Amazon ECS task definition
       id: deploy-task
       uses: aws-actions/amazon-ecs-deploy-task-definition@v2
@@ -245,17 +242,14 @@ jobs:
       id: list-tasks
       run: |
         echo "task=$(aws ecs list-tasks --cluster superset-ci --service-name pr-${{ github.event.issue.number }}-service | jq '.taskArns | first')" >> $GITHUB_OUTPUT
-
     - name: Get network interface
       id: get-eni
       run: |
-        echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name=="networkInterfaceId")) | .[0] | .value')" >> $GITHUB_OUTPUT
+        echo "eni=$(aws ecs describe-tasks --cluster superset-ci --tasks ${{ steps.list-tasks.outputs.task }} | jq '.tasks | .[0] | .attachments | .[0] | .details | map(select(.name==\"networkInterfaceId\")) | .[0] | .value')" >> $GITHUB_OUTPUT
-
     - name: Get public IP
       id: get-ip
       run: |
         echo "ip=$(aws ec2 describe-network-interfaces --network-interface-ids ${{ steps.get-eni.outputs.eni }} | jq -r '.NetworkInterfaces | first | .Association.PublicIp')" >> $GITHUB_OUTPUT
-
     - name: Comment (success)
       if: ${{ success() }}
       uses: actions/github-script@v7
@@ -263,12 +257,11 @@ jobs:
         github-token: ${{github.token}}
         script: |
           github.rest.issues.createComment({
-            issue_number: ${{ github.event.issue.number }},
+            issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
-            body: '@${{ github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
+            body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment spinning up at http://${{ steps.get-ip.outputs.ip }}:8080. Credentials are `admin`/`admin`. Please allow several minutes for bootstrapping and startup.'
           })
-
     - name: Comment (failure)
       if: ${{ failure() }}
       uses: actions/github-script@v7
@@ -276,8 +269,8 @@ jobs:
         github-token: ${{github.token}}
         script: |
           github.rest.issues.createComment({
-            issue_number: ${{ github.event.issue.number }},
+            issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }},
             owner: context.repo.owner,
             repo: context.repo.repo,
-            body: '@${{ github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
+            body: '@${{ github.event.inputs.user_login || github.event.comment.user.login }} Ephemeral environment creation failed. Please check the Actions logs for details.'
           })

Dockerfile

@@ -115,7 +115,7 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
         libldap2-dev \
     && touch superset/static/version_info.json \
     && chown -R superset:superset ./* \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
 # setup.py uses the version information in package.json
@@ -128,7 +128,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
     && pip install --no-cache-dir --upgrade setuptools pip \
     && pip install --no-cache-dir -r requirements/base.txt \
     && apt-get autoremove -yqq --purge build-essential \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 # Copy the compiled frontend assets
 COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
@@ -177,7 +177,7 @@ RUN apt-get update -qq \
         libxtst6 \
         git \
         pkg-config \
-        && rm -rf /var/lib/apt/lists/*
+        && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
 
 RUN --mount=type=cache,target=/root/.cache/pip \
     pip install --no-cache-dir playwright
@@ -199,13 +199,13 @@ RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
         && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
         && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
         && ln -s /opt/firefox/firefox /usr/local/bin/firefox \
-        && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/*; \
+        && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
     fi
 
 # Installing mysql client os-level dependencies in dev image only because GPL
 RUN apt-get install -yqq --no-install-recommends \
         default-libmysqlclient-dev \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 COPY --chown=superset:superset requirements/development.txt requirements/
 RUN --mount=type=cache,target=/root/.cache/pip \
@@ -213,7 +213,7 @@ RUN --mount=type=cache,target=/root/.cache/pip \
       build-essential \
     && pip install --no-cache-dir -r requirements/development.txt \
     && apt-get autoremove -yqq --purge build-essential \
-    && rm -rf /var/lib/apt/lists/*
+    && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
 
 USER superset
 ######################################################################

docker-compose.yml

@@ -25,7 +25,6 @@ x-superset-user: &superset-user root
 x-superset-depends-on: &superset-depends-on
   - db
   - redis
-  - superset-checks
 x-superset-volumes: &superset-volumes
   # /app/pythonpath_docker will be appended to the PYTHONPATH in the final container
   - ./docker:/app/docker
@@ -131,23 +130,6 @@ services:
       - REDIS_PORT=6379
       - REDIS_SSL=false
 
-  superset-checks:
-    build:
-      context: .
-      target: python-base
-      cache_from:
-        - apache/superset-cache:3.10-slim-bookworm
-    container_name: superset_checks
-    command: ["/app/scripts/check-env.py"]
-    env_file:
-      - path: docker/.env # default
-        required: true
-      - path: docker/.env-local # optional override
-        required: false
-    user: *superset-user
-    healthcheck:
-      disable: true
-
   superset-init:
     build:
       <<: *common-build
@@ -179,6 +161,7 @@ services:
       # set this to false if you have perf issues running the npm i; npm run dev in-docker
       # if you do so, you have to run this manually on the host, which should perform better!
       BUILD_SUPERSET_FRONTEND_IN_DOCKER: true
+      NPM_RUN_PRUNE: false
       SCARF_ANALYTICS: "${SCARF_ANALYTICS:-}"
     container_name: superset_node
     command: ["/app/docker/docker-frontend.sh"]

docker/docker-frontend.sh

@@ -27,6 +27,11 @@ if [ "$BUILD_SUPERSET_FRONTEND_IN_DOCKER" = "true" ]; then
     echo "Building Superset frontend in dev mode inside docker container"
     cd /app/superset-frontend
 
+    if [ "$NPM_RUN_PRUNE" = "true" ]; then
+        echo "Running `npm run prune`"
+        npm run prune
+    fi
+
     echo "Running `npm install`"
     npm install
 

docs/docs/installation/docker-builds.mdx

@@ -29,7 +29,7 @@ We have a set of build "presets" that each represent a combination of
 parameters for the build, mostly pointing to either different target layer
 for the build, and/or base image.
 
-Here are the build presets that are exposed through the `build_docker.py` script:
+Here are the build presets that are exposed through the `supersetbot docker` utility:
 
 - `lean`: The default Docker image, including both frontend and backend. Tags
   without a build_preset are lean builds (ie: `latest`, `4.0.0`, `3.0.0`, ...). `lean`
@@ -62,8 +62,8 @@ Here are the build presets that are exposed through the `build_docker.py` script
 
 
 For insights or modifications to the build matrix and tagging conventions,
-check the [build_docker.py](https://github.com/apache/superset/blob/master/scripts/build_docker.py)
+check the [supersetbot docker](https://github.com/apache-superset/supersetbot)
-script and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
+subcommand and the [docker.yml](https://github.com/apache/superset/blob/master/.github/workflows/docker.yml)
 GitHub action.
 
 ## Key ARGs in Dockerfile

docs/docs/installation/docker-compose.mdx

@@ -95,6 +95,14 @@ perform those operations. In this case, we recommend you set the env var
 Simply trigger `npm i && npm run dev`, this should be MUCH faster.
 :::
 
+:::tip
+Sometimes, your npm-related state can get out-of-wack, running `npm run prune` from
+the `superset-frontend/` folder will nuke the various' packages `node_module/` folders
+and help you start fresh. In the context of `docker compose` setting
+`export NPM_RUN_PRUNE=true` prior to running `docker compose up` will trigger that
+from within docker. This will slow down the startup, but will fix various npm-related issues.
+:::
+
 ### Option #2 - build a set of immutable images from the local branch
 
 ```bash

docs/docs/installation/pypi.mdx

@@ -77,10 +77,6 @@ versions officially supported by Superset. We'd recommend using a Python version
 like [pyenv](https://github.com/pyenv/pyenv)
 (and also [pyenv-virtualenv](https://github.com/pyenv/pyenv-virtualenv)).
 
-:::tip
-To identify the Python version used by the official docker image, see the [Dockerfile](https://github.com/apache/superset/blob/master/Dockerfile). Additional docker images published for newer versions of Python can be found in [this file](https://github.com/apache/superset/blob/master/scripts/build_docker.py).
-:::
-
 Let's also make sure we have the latest version of `pip` and `setuptools`:
 
 ```bash

requirements/base.txt

@@ -32,7 +32,7 @@ billiard==4.2.0
     # via celery
 blinker==1.9.0
     # via flask
-bottleneck==1.3.8
+bottleneck==1.4.2
     # via pandas
 brotli==1.1.0
     # via flask-compress
@@ -148,9 +148,7 @@ geopy==2.4.1
 google-auth==2.29.0
     # via shillelagh
 greenlet==3.0.3
-    # via
+    # via shillelagh
-    #   shillelagh
-    #   sqlalchemy
 gunicorn==22.0.0
     # via apache-superset
 hashids==1.3.1

scripts/build_docker.py

@@ -1,294 +0,0 @@
-#!/usr/bin/env python3
-
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import os
-import re
-import subprocess
-from textwrap import dedent
-
-import click
-
-REPO = "apache/superset"
-CACHE_REPO = f"{REPO}-cache"
-BASE_PY_IMAGE = "3.10-slim-bookworm"
-
-
-def run_cmd(command: str, raise_on_failure: bool = True) -> str:
-    process = subprocess.Popen(
-        command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True
-    )
-
-    output = ""
-    if process.stdout is not None:
-        for line in iter(process.stdout.readline, ""):
-            print(line.strip())  # Print the line to stdout in real-time
-            output += line
-
-    process.wait()  # Wait for the subprocess to finish
-
-    if process.returncode != 0 and raise_on_failure:
-        raise subprocess.CalledProcessError(process.returncode, command, output)
-    return output
-
-
-def get_git_sha() -> str:
-    return run_cmd("git rev-parse HEAD").strip()
-
-
-def get_build_context_ref(build_context: str) -> str:
-    """
-    Given a context, return a ref:
-    - if context is pull_request, return the PR's id
-    - if context is push, return the branch
-    - if context is release, return the release ref
-    """
-
-    event = os.getenv("GITHUB_EVENT_NAME")
-    github_ref = os.getenv("GITHUB_REF", "")
-
-    if event == "pull_request":
-        github_head_ref = os.getenv("GITHUB_HEAD_REF", "")
-        return re.sub("[^a-zA-Z0-9]", "-", github_head_ref)[:40]
-    elif event == "release":
-        return re.sub("refs/tags/", "", github_ref)[:40]
-    elif event == "push":
-        return re.sub("[^a-zA-Z0-9]", "-", re.sub("refs/heads/", "", github_ref))[:40]
-    return ""
-
-
-def is_latest_release(release: str) -> bool:
-    output = (
-        run_cmd(
-            f"./scripts/tag_latest_release.sh {release} --dry-run",
-            raise_on_failure=False,
-        )
-        or ""
-    )
-    return "SKIP_TAG::false" in output
-
-
-def make_docker_tag(l: list[str]) -> str:  # noqa: E741
-    return f"{REPO}:" + "-".join([o for o in l if o])
-
-
-def get_docker_tags(
-    build_preset: str,
-    build_platforms: list[str],
-    sha: str,
-    build_context: str,
-    build_context_ref: str,
-    force_latest: bool = False,
-) -> set[str]:
-    """
-    Return a set of tags given a given build context
-    """
-    tags: set[str] = set()
-    tag_chunks: list[str] = []
-
-    is_latest = is_latest_release(build_context_ref)
-
-    if build_preset != "lean":
-        # Always add the preset_build name if different from default (lean)
-        tag_chunks += [build_preset]
-
-    if len(build_platforms) == 1:
-        build_platform = build_platforms[0]
-        short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-        if short_build_platform != "amd":
-            # Always a platform indicator if different from default (amd)
-            tag_chunks += [short_build_platform]
-
-    # Always craft a tag for the SHA
-    tags.add(make_docker_tag([sha] + tag_chunks))
-    # also a short SHA, cause it's nice
-    tags.add(make_docker_tag([sha[:7]] + tag_chunks))
-
-    if build_context == "release":
-        # add a release tag
-        tags.add(make_docker_tag([build_context_ref] + tag_chunks))
-        if is_latest or force_latest:
-            # add a latest tag
-            tags.add(make_docker_tag(["latest"] + tag_chunks))
-    elif build_context == "push" and build_context_ref == "master":
-        tags.add(make_docker_tag(["master"] + tag_chunks))
-    elif build_context == "pull_request":
-        tags.add(make_docker_tag([f"pr-{build_context_ref}"] + tag_chunks))
-    return tags
-
-
-def get_docker_command(
-    build_preset: str,
-    build_platforms: list[str],
-    is_authenticated: bool,
-    sha: str,
-    build_context: str,
-    build_context_ref: str,
-    force_latest: bool = False,
-) -> str:
-    tag = ""  # noqa: F841
-    build_target = ""
-    py_ver = BASE_PY_IMAGE
-    docker_context = "."
-
-    if build_preset == "dev":
-        build_target = "dev"
-    elif build_preset == "lean":
-        build_target = "lean"
-    elif build_preset == "py311":
-        build_target = "lean"
-        py_ver = "3.11-slim-bookworm"
-    elif build_preset == "websocket":
-        build_target = ""
-        docker_context = "superset-websocket"
-    elif build_preset == "ci":
-        build_target = "ci"
-    elif build_preset == "dockerize":
-        build_target = ""
-        docker_context = "-f dockerize.Dockerfile ."
-    else:
-        print(f"Invalid build preset: {build_preset}")
-        exit(1)
-
-    # Try to get context reference if missing
-    if not build_context_ref:
-        build_context_ref = get_build_context_ref(build_context)
-
-    tags = get_docker_tags(
-        build_preset,
-        build_platforms,
-        sha,
-        build_context,
-        build_context_ref,
-        force_latest,
-    )
-    docker_tags = ("\\\n" + 8 * " ").join([f"-t {s} " for s in tags])
-
-    docker_args = "--load" if not is_authenticated else "--push"
-    target_argument = f"--target {build_target}" if build_target else ""
-
-    cache_ref = f"{CACHE_REPO}:{py_ver}"
-    if len(build_platforms) == 1:
-        build_platform = build_platforms[0]
-        short_build_platform = build_platform.replace("linux/", "").replace("64", "")
-        cache_ref = f"{CACHE_REPO}:{py_ver}-{short_build_platform}"
-    platform_arg = "--platform " + ",".join(build_platforms)
-
-    cache_from_arg = f"--cache-from=type=registry,ref={cache_ref}"
-    cache_to_arg = (
-        f"--cache-to=type=registry,mode=max,ref={cache_ref}" if is_authenticated else ""
-    )
-    build_arg = f"--build-arg PY_VER={py_ver}" if py_ver else ""
-    actor = os.getenv("GITHUB_ACTOR")
-
-    return dedent(
-        f"""\
-        docker buildx build \\
-        {docker_args} \\
-        {docker_tags} \\
-        {cache_from_arg} \\
-        {cache_to_arg} \\
-        {build_arg} \\
-        {platform_arg} \\
-        {target_argument} \\
-        --label sha={sha} \\
-        --label target={build_target} \\
-        --label build_trigger={build_context} \\
-        --label base={py_ver} \\
-        --label build_actor={actor} \\
-        {docker_context}"""
-    )
-
-
-@click.command()
-@click.argument(
-    "build_preset",
-    type=click.Choice(["lean", "dev", "dockerize", "websocket", "py311", "ci"]),
-)
-@click.argument("build_context", type=click.Choice(["push", "pull_request", "release"]))
-@click.option(
-    "--platform",
-    type=click.Choice(["linux/arm64", "linux/amd64"]),
-    default=["linux/amd64"],
-    multiple=True,
-)
-@click.option("--build_context_ref", help="a reference to the pr, release or branch")
-@click.option("--dry-run", is_flag=True, help="Run the command in dry-run mode.")
-@click.option("--verbose", is_flag=True, help="Print more info")
-@click.option(
-    "--force-latest", is_flag=True, help="Force the 'latest' tag on the release"
-)
-def main(
-    build_preset: str,
-    build_context: str,
-    build_context_ref: str,
-    platform: list[str],
-    dry_run: bool,
-    force_latest: bool,
-    verbose: bool,
-) -> None:
-    """
-    This script executes docker build and push commands based on given arguments.
-    """
-
-    is_authenticated = (
-        True if os.getenv("DOCKERHUB_TOKEN") and os.getenv("DOCKERHUB_USER") else False
-    )
-
-    if force_latest and build_context != "release":
-        print(
-            "--force-latest can only be applied if the build context is set to 'release'"
-        )
-        exit(1)
-
-    if build_context == "release" and not build_context_ref.strip():
-        print("Release number has to be provided")
-        exit(1)
-
-    docker_build_command = get_docker_command(
-        build_preset,
-        platform,
-        is_authenticated,
-        get_git_sha(),
-        build_context,
-        build_context_ref,
-        force_latest,
-    )
-
-    if not dry_run:
-        print("Executing Docker Build Command:")
-        print(docker_build_command)
-        script = ""
-        if os.getenv("DOCKERHUB_USER"):
-            script = dedent(
-                f"""\
-                docker logout
-                docker login --username "{os.getenv("DOCKERHUB_USER")}" --password "{os.getenv("DOCKERHUB_TOKEN")}"
-                DOCKER_ARGS="--push"
-                """
-            )
-        script = script + docker_build_command
-        if verbose:
-            run_cmd("cat Dockerfile")
-        stdout = run_cmd(script)  # noqa: F841
-    else:
-        print("Dry Run - Docker Build Command:")
-        print(docker_build_command)
-
-
-if __name__ == "__main__":
-    main()

superset-frontend/packages/superset-ui-switchboard/package-lock.json

@@ -1,13 +0,0 @@
-{
-  "name": "@superset-ui/switchboard",
-  "version": "0.18.26-0",
-  "lockfileVersion": 2,
-  "requires": true,
-  "packages": {
-    "": {
-      "name": "@superset-ui/switchboard",
-      "version": "0.18.26-0",
-      "license": "Apache-2.0"
-    }
-  }
-}

superset/db_engine_specs/trino.py

@@ -33,7 +33,6 @@ from sqlalchemy import text
 from sqlalchemy.engine.reflection import Inspector
 from sqlalchemy.engine.url import URL
 from sqlalchemy.exc import NoSuchTableError
-from trino.exceptions import HttpError
 
 from superset import db
 from superset.constants import QUERY_CANCEL_KEY, QUERY_EARLY_CANCEL_KEY, USER_AGENT
@@ -61,6 +60,12 @@ if TYPE_CHECKING:
 
 logger = logging.getLogger(__name__)
 
+try:
+    # since trino is an optional dependency, we need to handle the ImportError
+    from trino.exceptions import HttpError
+except ImportError:
+    HttpError = Exception
+
 
 class CustomTrinoAuthErrorMeta(type):
     def __instancecheck__(cls, instance: object) -> bool:

tests/unit_tests/scripts/docker_build.py

@@ -1,290 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import os
-import sys
-
-import pytest
-
-SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
-PR_ID = "666"
-OLD_REL = "2.1.0"
-NEW_REL = "2.1.1"
-REPO = "apache/superset"
-
-# Add the 'scripts' directory to sys.path
-scripts_dir = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "../../../scripts")
-)
-sys.path.append(scripts_dir)
-
-import build_docker as docker_utils  # Replace with the actual function name  # noqa: E402
-
-
-@pytest.fixture(autouse=True)
-def set_env_var():
-    os.environ["TEST_ENV"] = "true"
-    yield
-    del os.environ["TEST_ENV"]
-
-
-@pytest.mark.parametrize(
-    "release, expected_bool",
-    [
-        ("2.1.0", False),
-        ("2.1.1", True),
-        ("1.0.0", False),
-        ("3.0.0", True),
-    ],
-)
-def test_is_latest_release(release, expected_bool):
-    assert docker_utils.is_latest_release(release) == expected_bool
-
-
-@pytest.mark.parametrize(
-    "build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags",
-    [
-        # PRs
-        (
-            "lean",
-            ["linux/arm64"],
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:pr-{PR_ID}-arm"],
-        ),
-        (
-            "ci",
-            ["linux/amd64"],
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-ci", f"{REPO}:{SHA}-ci", f"{REPO}:pr-{PR_ID}-ci"],
-        ),
-        (
-            "lean",
-            ["linux/amd64"],
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:pr-{PR_ID}"],
-        ),
-        (
-            "dev",
-            ["linux/arm64"],
-            SHA,
-            "pull_request",
-            PR_ID,
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:pr-{PR_ID}-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            ["linux/amd64"],
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:pr-{PR_ID}-dev"],
-        ),
-        # old releases
-        (
-            "lean",
-            ["linux/arm64"],
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
-        ),
-        (
-            "lean",
-            ["linux/amd64"],
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
-        ),
-        (
-            "dev",
-            ["linux/arm64"],
-            SHA,
-            "release",
-            OLD_REL,
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:{OLD_REL}-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            ["linux/amd64"],
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
-        ),
-        # new releases
-        (
-            "lean",
-            ["linux/arm64"],
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-arm",
-                f"{REPO}:{SHA}-arm",
-                f"{REPO}:{NEW_REL}-arm",
-                f"{REPO}:latest-arm",
-            ],
-        ),
-        (
-            "lean",
-            ["linux/amd64"],
-            SHA,
-            "release",
-            NEW_REL,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
-        ),
-        (
-            "dev",
-            ["linux/arm64"],
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:{NEW_REL}-dev-arm",
-                f"{REPO}:latest-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            ["linux/amd64"],
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-dev",
-                f"{REPO}:{SHA}-dev",
-                f"{REPO}:{NEW_REL}-dev",
-                f"{REPO}:latest-dev",
-            ],
-        ),
-        # merge on master
-        (
-            "lean",
-            ["linux/arm64"],
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
-        ),
-        (
-            "lean",
-            ["linux/amd64"],
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
-        ),
-        (
-            "dev",
-            ["linux/arm64"],
-            SHA,
-            "push",
-            "master",
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:master-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            ["linux/amd64"],
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
-        ),
-    ],
-)
-def test_get_docker_tags(
-    build_preset, build_platforms, sha, build_context, build_context_ref, expected_tags
-):
-    tags = docker_utils.get_docker_tags(
-        build_preset, build_platforms, sha, build_context, build_context_ref
-    )
-    for tag in expected_tags:
-        assert tag in tags
-
-
-@pytest.mark.parametrize(
-    "build_preset, build_platforms, is_authenticated, sha, build_context, build_context_ref, contains",
-    [
-        (
-            "lean",
-            ["linux/amd64"],
-            True,
-            SHA,
-            "push",
-            "master",
-            ["--push", f"-t {REPO}:master "],
-        ),
-        (
-            "dev",
-            ["linux/amd64"],
-            False,
-            SHA,
-            "push",
-            "master",
-            ["--load", f"-t {REPO}:master-dev ", "--target dev"],
-        ),
-        # multi-platform
-        (
-            "lean",
-            ["linux/arm64", "linux/amd64"],
-            True,
-            SHA,
-            "push",
-            "master",
-            ["--platform linux/arm64,linux/amd64"],
-        ),
-    ],
-)
-def test_get_docker_command(
-    build_preset,
-    build_platforms,
-    is_authenticated,
-    sha,
-    build_context,
-    build_context_ref,
-    contains,
-):
-    cmd = docker_utils.get_docker_command(
-        build_preset,
-        build_platforms,
-        is_authenticated,
-        sha,
-        build_context,
-        build_context_ref,
-    )
-    for s in contains:
-        assert s in cmd

tests/unit_tests/utils/docker.py

@@ -1,268 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import os
-import sys
-
-import pytest
-
-SHA = "22e7c602b9aa321ec7e0df4bb0033048664dcdf0"
-PR_ID = "666"
-OLD_REL = "2.1.0"
-NEW_REL = "2.1.1"
-REPO = "apache/superset"
-
-# Add the 'scripts' directory to sys.path
-scripts_dir = os.path.abspath(
-    os.path.join(os.path.dirname(__file__), "../../../scripts")
-)
-sys.path.append(scripts_dir)
-
-import build_docker as docker_utils  # Replace with the actual function name  # noqa: E402
-
-
-@pytest.fixture(autouse=True)
-def set_env_var():
-    os.environ["TEST_ENV"] = "true"
-    yield
-    del os.environ["TEST_ENV"]
-
-
-@pytest.mark.parametrize(
-    "release, expected_bool",
-    [
-        ("2.1.0", False),
-        ("2.1.1", True),
-        ("1.0.0", False),
-        ("3.0.0", True),
-    ],
-)
-def test_is_latest_release(release, expected_bool):
-    assert docker_utils.is_latest_release(release) == expected_bool
-
-
-@pytest.mark.parametrize(
-    "build_preset, build_platform, sha, build_context, build_context_ref, expected_tags",
-    [
-        # PRs
-        (
-            "lean",
-            "linux/arm64",
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm"],
-        ),
-        (
-            "lean",
-            "linux/amd64",
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}"],
-        ),
-        (
-            "dev",
-            "linux/arm64",
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-dev-arm", f"{REPO}:{SHA}-dev-arm"],
-        ),
-        (
-            "dev",
-            "linux/amd64",
-            SHA,
-            "pull_request",
-            PR_ID,
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev"],
-        ),
-        # old releases
-        (
-            "lean",
-            "linux/arm64",
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:{OLD_REL}-arm"],
-        ),
-        (
-            "lean",
-            "linux/amd64",
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{OLD_REL}"],
-        ),
-        (
-            "dev",
-            "linux/arm64",
-            SHA,
-            "release",
-            OLD_REL,
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:{OLD_REL}-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            "linux/amd64",
-            SHA,
-            "release",
-            OLD_REL,
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:{OLD_REL}-dev"],
-        ),
-        # new releases
-        (
-            "lean",
-            "linux/arm64",
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-arm",
-                f"{REPO}:{SHA}-arm",
-                f"{REPO}:{NEW_REL}-arm",
-                f"{REPO}:latest-arm",
-            ],
-        ),
-        (
-            "lean",
-            "linux/amd64",
-            SHA,
-            "release",
-            NEW_REL,
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:{NEW_REL}", f"{REPO}:latest"],
-        ),
-        (
-            "dev",
-            "linux/arm64",
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:{NEW_REL}-dev-arm",
-                f"{REPO}:latest-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            "linux/amd64",
-            SHA,
-            "release",
-            NEW_REL,
-            [
-                f"{REPO}:22e7c60-dev",
-                f"{REPO}:{SHA}-dev",
-                f"{REPO}:{NEW_REL}-dev",
-                f"{REPO}:latest-dev",
-            ],
-        ),
-        # merge on master
-        (
-            "lean",
-            "linux/arm64",
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60-arm", f"{REPO}:{SHA}-arm", f"{REPO}:master-arm"],
-        ),
-        (
-            "lean",
-            "linux/amd64",
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60", f"{REPO}:{SHA}", f"{REPO}:master"],
-        ),
-        (
-            "dev",
-            "linux/arm64",
-            SHA,
-            "push",
-            "master",
-            [
-                f"{REPO}:22e7c60-dev-arm",
-                f"{REPO}:{SHA}-dev-arm",
-                f"{REPO}:master-dev-arm",
-            ],
-        ),
-        (
-            "dev",
-            "linux/amd64",
-            SHA,
-            "push",
-            "master",
-            [f"{REPO}:22e7c60-dev", f"{REPO}:{SHA}-dev", f"{REPO}:master-dev"],
-        ),
-    ],
-)
-def test_get_docker_tags(
-    build_preset, build_platform, sha, build_context, build_context_ref, expected_tags
-):
-    tags = docker_utils.get_docker_tags(
-        build_preset, build_platform, sha, build_context, build_context_ref
-    )
-    for tag in expected_tags:
-        assert tag in tags
-
-
-@pytest.mark.parametrize(
-    "build_preset, build_platform, is_authenticated, sha, build_context, build_context_ref, contains",
-    [
-        (
-            "lean",
-            "linux/amd64",
-            True,
-            SHA,
-            "push",
-            "master",
-            ["--push", f"-t {REPO}:master "],
-        ),
-        (
-            "dev",
-            "linux/amd64",
-            False,
-            SHA,
-            "push",
-            "master",
-            ["--load", f"-t {REPO}:master-dev "],
-        ),
-    ],
-)
-def test_get_docker_command(
-    build_preset,
-    build_platform,
-    is_authenticated,
-    sha,
-    build_context,
-    build_context_ref,
-    contains,
-):
-    cmd = docker_utils.get_docker_command(
-        build_preset,
-        build_platform,
-        is_authenticated,
-        sha,
-        build_context,
-        build_context_ref,
-    )
-    for s in contains:
-        assert s in cmd