build: Parallelize the CI image builds (continued) (#26698)

2024-01-23 13:44:07 -08:00 · 2024-01-23 13:44:07 -08:00 · 363a8e6b07
parent e00e039ca3
commit 363a8e6b07
5 changed files with 130 additions and 139 deletions
--- a/.github/workflows/docker-release.yml
+++ b/.github/workflows/docker-release.yml
@ -22,21 +22,29 @@ jobs:
    if: needs.config.outputs.has-secrets
    name: docker-release
    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: recursive
          ref: ${{ github.ref }}
+
      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
+
      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - shell: bash
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
        env:
          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
        run: |
          GITHUB_RELEASE_TAG_NAME="${{ github.event.release.tag_name }}"
-          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME"
+          ./scripts/docker_build_push.sh "$GITHUB_RELEASE_TAG_NAME" ${{ matrix.target }} ${{ matrix.platform }}
--- a/.github/workflows/docker.yml
+++ b/.github/workflows/docker.yml
@ -24,27 +24,52 @@ jobs:
            echo "has-secrets=0" >> "$GITHUB_OUTPUT"
            echo "no secrets!"
          fi
-
  docker-build:
+    needs: config
+    if: needs.config.outputs.has-secrets
+    name: docker-build
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        target: ["dev", "lean", "lean310", "websocket", "dockerize"]
+        platform: ["linux/amd64", "linux/arm64"]
+      fail-fast: false
+    steps:
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker Image
+        shell: bash
+        env:
+          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
+          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
+        run: |
+          ./scripts/docker_build_push.sh "" ${{ matrix.target }} ${{ matrix.platform }}
+
+  ephemeral-docker-build:
    needs: config
    if: needs.config.outputs.has-secrets
    name: docker-build
    runs-on: ubuntu-latest
    steps:
      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
        with:
          persist-credentials: false
+
      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v1
+        uses: docker/setup-qemu-action@v3
+
      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v1
-      - shell: bash
-        env:
-          DOCKERHUB_USER: ${{ secrets.DOCKERHUB_USER }}
-          DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-        run: |
-          ./scripts/docker_build_push.sh
+        uses: docker/setup-buildx-action@v3

      - name: Build ephemeral env image
        if: github.event_name == 'pull_request'
@ -54,7 +79,7 @@ jobs:
          echo ${{ github.event.pull_request.number }} > ./build/PR-NUM
          docker buildx build --target ci \
            --load \
-            --cache-from=type=local,src=/tmp/superset \
+            --cache-from=type=registry,ref=apache/superset:lean \
            -t ${{ github.sha }} \
            -t "pr-${{ github.event.pull_request.number }}" \
            --platform linux/amd64 \
@ -64,7 +89,7 @@ jobs:

      - name: Upload build artifacts
        if: github.event_name == 'pull_request'
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
        with:
          name: build
          path: build/
--- a/scripts/docker_build_push.sh
+++ b/scripts/docker_build_push.sh
@ -18,9 +18,15 @@
 set -eo pipefail

 GITHUB_RELEASE_TAG_NAME="$1"
+TARGET="$2"
+BUILD_PLATFORM="$3" # should be either 'linux/amd64' or 'linux/arm64'

+# Common variables
 SHA=$(git rev-parse HEAD)
 REPO_NAME="apache/superset"
+DOCKER_ARGS="--load" # default args, change as needed
+DOCKER_CONTEXT="."
+

 if [[ "${GITHUB_EVENT_NAME}" == "pull_request" ]]; then
  REFSPEC=$(echo "${GITHUB_HEAD_REF}" | sed 's/[^a-zA-Z0-9]/-/g' | head -c 40)
@ -54,6 +60,45 @@ if [[ "${TEST_ENV}" == "true" ]]; then
  exit 0
 fi

+# for the dev image, it's ok to tag master as latest-dev
+# for production, we only want to tag the latest official release as latest
+if [ "${LATEST_TAG}" = "master" ]; then
+  DEV_TAG="${REPO_NAME}:latest-dev"
+else
+  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+fi
+
+BUILD_ARG="3.9-slim-bookworm"
+
+case "${TARGET}" in
+  "dev")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-dev -t ${REPO_NAME}:${REFSPEC}-dev -t ${DEV_TAG}"
+    BUILD_TARGET="dev"
+    ;;
+  "lean")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA} -t ${REPO_NAME}:${REFSPEC} -t ${REPO_NAME}:${LATEST_TAG}"
+    BUILD_TARGET="lean"
+    ;;
+  "lean310")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-py310 -t ${REPO_NAME}:${REFSPEC}-py310 -t ${REPO_NAME}:${LATEST_TAG}-py310"
+    BUILD_TARGET="lean"
+    BUILD_ARG="3.10-slim-bookworm"
+    ;;
+  "websocket")
+    DOCKER_TAGS="-t ${REPO_NAME}:${SHA}-websocket -t ${REPO_NAME}:${REFSPEC}-websocket -t ${REPO_NAME}:${LATEST_TAG}-websocket"
+    BUILD_TARGET=""
+	DOCKER_CONTEXT="superset-websocket"
+    ;;
+  "dockerize")
+    DOCKER_TAGS="-t ${REPO_NAME}:dockerize"
+    BUILD_TARGET=""
+	DOCKER_CONTEXT="-f dockerize.Dockerfile ."
+    ;;
+  *)
+    echo "Invalid TARGET: ${TARGET}"
+    exit 1
+    ;;
+esac

 cat<<EOF
  Rolling with tags:
@ -67,123 +112,30 @@ if [ -z "${DOCKERHUB_TOKEN}" ]; then
  echo "Skipping Docker push"
  # By default load it back
  DOCKER_ARGS="--load"
-  ARCHITECTURE_FOR_BUILD="linux/amd64 linux/arm64"
 else
  # Login and push
  docker logout
  docker login --username "${DOCKERHUB_USER}" --password "${DOCKERHUB_TOKEN}"
  DOCKER_ARGS="--push"
-  ARCHITECTURE_FOR_BUILD="linux/amd64,linux/arm64"
 fi
 set -x

-# for the dev image, it's ok to tag master as latest-dev
-# for production, we only want to tag the latest official release as latest
-if [ "${LATEST_TAG}" = "master" ]; then
-  DEV_TAG="${REPO_NAME}:latest-dev"
-else
-  DEV_TAG="${REPO_NAME}:${LATEST_TAG}-dev"
+TARGET_ARGUMENT=""
+if [[ -n "${BUILD_TARGET}" ]]; then
+  TARGET_ARGUMENT="--target ${BUILD_TARGET}"
 fi

-for BUILD_PLATFORM in $ARCHITECTURE_FOR_BUILD; do
-#
-# Build the dev image
-#
-docker buildx build --target dev \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-dev \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-dev" \
-  -t "${REPO_NAME}:${REFSPEC}-dev" \
-  -t "${DEV_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=dev" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}" \
-  -t "${REPO_NAME}:${REFSPEC}" \
-  -t "${REPO_NAME}:${LATEST_TAG}" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean310" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py310" \
-  -t "${REPO_NAME}:${REFSPEC}-py310" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py310" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.10-slim-bookworm"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean310" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "lean39" image
-#
-docker buildx build --target lean \
-  $DOCKER_ARGS \
-  --cache-from=type=local,src=/tmp/superset \
-  --cache-to=type=local,ignore-error=true,dest=/tmp/superset \
-  -t "${REPO_NAME}:${SHA}-py39" \
-  -t "${REPO_NAME}:${REFSPEC}-py39" \
-  -t "${REPO_NAME}:${LATEST_TAG}-py39" \
-  --platform ${BUILD_PLATFORM} \
-  --build-arg PY_VER="3.9-slim-bullseye"\
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "target=lean39" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  .
-
-#
-# Build the "websocket" image
-#
 docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:master-websocket \
-  -t "${REPO_NAME}:${SHA}-websocket" \
-  -t "${REPO_NAME}:${REFSPEC}-websocket" \
-  -t "${REPO_NAME}:${LATEST_TAG}-websocket" \
+  ${TARGET_ARGUMENT} \
+  ${DOCKER_ARGS} \
+  --cache-from=type=registry,ref=apache/superset:${TARGET} \
+  --cache-to=type=registry,mode=max,ref=apache/superset:${TARGET} \
+  ${DOCKER_TAGS} \
  --platform ${BUILD_PLATFORM} \
  --label "sha=${SHA}" \
  --label "built_at=$(date)" \
-  --label "target=websocket" \
+  --label "target=${TARGET}" \
+  --label "base=${PY_VER}" \
  --label "build_actor=${GITHUB_ACTOR}" \
-  superset-websocket
-
-#
-# Build the dockerize image
-#
-docker buildx build \
-  $DOCKER_ARGS \
-  --cache-from=type=registry,ref=apache/superset:dockerize \
-  -t "${REPO_NAME}:dockerize" \
-  --platform ${BUILD_PLATFORM} \
-  --label "sha=${SHA}" \
-  --label "built_at=$(date)" \
-  --label "build_actor=${GITHUB_ACTOR}" \
-  -f dockerize.Dockerfile \
-  .
-done
+  ${BUILD_ARG:+--build-arg PY_VER="${BUILD_ARG}"} \
+  ${DOCKER_CONTEXT}
--- a/tests/unit_tests/fixtures/bash_mock.py
+++ b/tests/unit_tests/fixtures/bash_mock.py
@ -32,10 +32,10 @@ class BashMock:
        return result

    @staticmethod
-    def docker_build_push(tag, branch):
-        bash_command = f"./scripts/docker_build_push.sh {tag}"
+    def docker_build_push(tag, target, platform, branch):
+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
        result = subprocess.run(
-            bash_command,
+            cmd,
            shell=True,
            capture_output=True,
            text=True,
--- a/tests/unit_tests/scripts/docker_build_push_test.py
+++ b/tests/unit_tests/scripts/docker_build_push_test.py
@ -15,30 +15,36 @@ def wrapped(*args, **kwargs):


@pytest.mark.parametrize(
-    "tag, expected_output, branch",
+    "tag, target, platform, expected_output, branch",
    [
-        ("1.0.0", "LATEST_TAG is master", "master"),
-        ("2.1.0", "LATEST_TAG is master", "master"),
-        ("2.1.1", "LATEST_TAG is latest", "master"),
-        ("3.0.0", "LATEST_TAG is latest", "master"),
-        ("2.1.0rc1", "LATEST_TAG is 2.1.0", "2.1.0"),
-        ("", "LATEST_TAG is foo", "foo"),
-        ("2.1", "LATEST_TAG is 2.1", "2.1"),
-        ("does_not_exist", "LATEST_TAG is does-not-exist", "does_not_exist"),
+        ("1.0.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.0", "lean", "linux/amd64", "LATEST_TAG is master", "master"),
+        ("2.1.1", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
+        ("3.0.0", "lean", "linux/amd64", "LATEST_TAG is latest", "master"),
+        ("2.1.0rc1", "lean", "linux/amd64", "LATEST_TAG is 2.1.0", "2.1.0"),
+        ("", "lean", "linux/amd64", "LATEST_TAG is foo", "foo"),
+        ("2.1", "lean", "linux/amd64", "LATEST_TAG is 2.1", "2.1"),
+        (
+            "does_not_exist",
+            "lean",
+            "linux/amd64",
+            "LATEST_TAG is does-not-exist",
+            "does_not_exist",
+        ),
    ],
 )
-def test_tag_latest_release(tag, expected_output, branch):
+def test_tag_latest_release(tag, target, platform, expected_output, branch):
    with mock.patch(
        "tests.unit_tests.fixtures.bash_mock.subprocess.run", wraps=wrapped
    ) as subprocess_mock:
-        result = BashMock.docker_build_push(tag, branch)
+        result = BashMock.docker_build_push(tag, target, platform, branch)

+        cmd = f'./scripts/docker_build_push.sh "{tag}" "{target}" "{platform}"'
        subprocess_mock.assert_called_once_with(
-            f"./scripts/docker_build_push.sh {tag}",
+            cmd,
            shell=True,
            capture_output=True,
            text=True,
            env={"TEST_ENV": "true", "GITHUB_REF": f"refs/heads/{branch}"},
        )
-
-        assert re.search(expected_output, result.stdout, re.MULTILINE)
+        assert re.search(expected_output, result.stdout.strip(), re.MULTILINE)