iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: docker refactor (#31385)

This commit is contained in:
Maxime Beauchemin 2024-12-13 16:17:44 -08:00 committed by GitHub
parent 15ede02c25
commit 1f17b975d6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 163 additions and 141 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore
View File

@ -45,3 +45,4 @@ superset-frontend/coverage/
superset/static/assets/
superset-websocket/dist/
venv
.venv

11
.github/workflows/ephemeral-env.yml vendored
View File

@ -148,9 +148,20 @@ jobs:
- name: Setup supersetbot
uses: ./.github/actions/setup-supersetbot/
- name: Try to login to DockerHub
if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker
continue-on-error: true
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
- name: Build ephemeral env image
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
username: ${{ secrets.DOCKERHUB_USER }}
password: ${{ secrets.DOCKERHUB_TOKEN }}
run: |
supersetbot docker \
--preset ci \

261
Dockerfile
View File

@ -22,26 +22,29 @@ ARG PY_VER=3.10-slim-bookworm
# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
######################################################################
# superset-node used for building frontend assets
######################################################################
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
ARG DEV_MODE="false" # Skip frontend build in dev mode
ENV DEV_MODE=${DEV_MODE}
COPY docker/ /app/docker/
# Arguments for build configuration
ARG NPM_BUILD_CMD="build"
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
ARG DEV_MODE="false" # Skip frontend build in dev mode
ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports
ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled
# Install system dependencies required for node-gyp
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh build-essential python3 zstd
RUN /app/docker/apt-install.sh build-essential python3 zstd
# Define environment variables for frontend build
ENV BUILD_CMD=${NPM_BUILD_CMD} \
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
# Run the frontend memory monitoring script
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/frontend-mem-nag.sh
RUN /app/docker/frontend-mem-nag.sh
WORKDIR /app/superset-frontend
@ -49,6 +52,9 @@ WORKDIR /app/superset-frontend
RUN mkdir -p /app/superset/static/assets \
/app/superset/translations
# Copy translation files
COPY superset/translations /app/superset/translations
# Mount package files and install dependencies if not in dev mode
RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
@ -61,41 +67,28 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j
# Runs the webpack build process
COPY superset-frontend /app/superset-frontend
# Copy translation files
COPY superset/translations /app/superset/translations
# Build the frontend if not in dev mode
RUN if [ "$DEV_MODE" = "false" ]; then \
BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
echo "Running 'npm run ${BUILD_CMD}'"; \
if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
npm run build-translation; \
fi; \
npm run ${BUILD_CMD}; \
else \
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
fi
# Compile .json files from .po translations (if required) and clean up .po files
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
npm run build-translation; \
else \
echo "Skipping translations as requested by build flag"; \
fi \
# removing translations files regardless
&& rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
/app/superset/translations/messages.pot
fi && \
rm -rf /app/superset/translations/*/*/*.po
# Transition to Python base image
######################################################################
# Base python layer
######################################################################
FROM python:${PY_VER} AS python-base
RUN pip install --no-cache-dir --upgrade setuptools pip uv
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS}
ARG DEV_MODE="false" # Skip frontend build in dev mode
ENV DEV_MODE=${DEV_MODE}
######################################################################
# Final lean image...
######################################################################
FROM python-base AS lean
# Build argument for including translations
ARG BUILD_TRANSLATIONS="false"
WORKDIR /app
ENV LANG=C.UTF-8 \
LC_ALL=C.UTF-8 \
SUPERSET_ENV=production \
@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \
SUPERSET_HOME="/app/superset_home" \
SUPERSET_PORT=8088
RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset
# Some bash scripts needed throughout the layers
COPY --chmod=755 docker/*.sh /app/docker/
RUN pip install --no-cache-dir --upgrade setuptools pip uv
# Using uv as it's faster/simpler than pip
RUN uv venv /app/.venv
ENV PATH="/app/.venv/bin:${PATH}"
# Install Playwright and optionally setup headless browsers
ARG INCLUDE_CHROMIUM="true"
ARG INCLUDE_FIREFOX="false"
RUN --mount=type=cache,target=/root/.cache/pip \
if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \
pip install playwright && \
playwright install-deps && \
if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \
if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \
else \
echo "Skipping browser installation"; \
fi
######################################################################
# Python translation compiler layer
######################################################################
FROM python-base AS python-translation-compiler
# Install Python dependencies using docker/pip-install.sh
COPY requirements/translations.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
/app/docker/pip-install.sh -r requirements/translations.txt
COPY superset/translations/ /app/translations_mo/
RUN pybabel compile -d /app/translations_mo | true && \
rm -f /app/translations_mo/*/*/*.po
######################################################################
# Python APP common layer
######################################################################
FROM python-base AS python-common
# Copy the entrypoints, make them executable in userspace
COPY --chmod=755 docker/entrypoints /app/docker/entrypoints
WORKDIR /app
# Set up necessary directories and user
RUN --mount=type=bind,source=./docker,target=/docker \
mkdir -p ${PYTHONPATH} \
RUN mkdir -p \
${SUPERSET_HOME} \
${PYTHONPATH} \
superset/static \
requirements \
superset-frontend \
apache_superset.egg-info \
requirements \
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
&& /docker/apt-install.sh \
&& touch superset/static/version_info.json
# Copy required files for Python build
COPY pyproject.toml setup.py MANIFEST.in README.md ./
COPY superset-frontend/package.json superset-frontend/
COPY scripts/check-env.py scripts/
# keeping for backward compatibility
COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/
# Some debian libs
RUN /app/docker/apt-install.sh \
curl \
libsasl2-dev \
libsasl2-modules-gssapi-mit \
libpq-dev \
libecpg-dev \
libldap2-dev \
&& touch superset/static/version_info.json \
&& chown -R superset:superset ./* \
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
libldap2-dev
# Copy required files for Python build
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
COPY --chown=superset:superset requirements/base.txt requirements/
COPY --chown=superset:superset scripts/check-env.py scripts/
# Copy compiled things from previous stages
COPY --from=superset-node /app/superset/static/assets superset/static/assets
# Merging translations from backend and frontend stages
COPY --from=superset-node /app/superset/translations superset/translations
COPY --from=python-translation-compiler /app/translations_mo superset/translations
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
CMD ["/app/docker/entrypoints/run-server.sh"]
EXPOSE ${SUPERSET_PORT}
######################################################################
# Final lean image...
######################################################################
FROM python-common AS lean
COPY superset superset
# Install Python dependencies using docker/pip-install.sh
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
COPY requirements/base.txt requirements/
RUN --mount=type=cache,target=/root/.cache/pip \
/app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt && \
uv pip install .
# Copy the compiled frontend assets from the node image
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
RUN python -m compileall /app/superset
# Copy the main Superset source code
COPY --chown=superset:superset superset superset
# Install Superset itself using docker/pip-install.sh
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh -e .
# Copy .json translations from the node image
COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
# Compile backend translations and clean up
COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
./scripts/translations/generate_mo_files.sh \
&& chown -R superset:superset superset/translations; \
fi \
&& rm -rf superset/translations/messages.pot \
superset/translations/*/LC_MESSAGES/*.po
# Add server run script
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
# Set user and healthcheck
USER superset
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
# Expose port and set CMD
EXPOSE ${SUPERSET_PORT}
CMD ["/usr/bin/run-server.sh"]
######################################################################
# Dev image...
######################################################################
FROM lean AS dev
FROM python-common AS dev
COPY superset superset
USER root
# Install dev dependencies
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh \
libnss3 \
libdbus-glib-1-2 \
libgtk-3-0 \
libx11-xcb1 \
libasound2 \
libxtst6 \
# Debian libs needed for dev
RUN /app/docker/apt-install.sh \
git \
pkg-config
# Install Playwright and its dependencies
RUN --mount=type=cache,target=/root/.cache/pip \
uv pip install --system playwright \
&& playwright install-deps
# Optionally install Chromium
RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
playwright install chromium; \
else \
echo "Skipping Chromium installation in dev mode"; \
fi
# Install GeckoDriver WebDriver and Firefox (if required)
ARG GECKODRIVER_VERSION=v0.34.0
ARG FIREFOX_VERSION=125.0.3
RUN --mount=type=bind,source=./docker,target=/docker \
if [ "$INCLUDE_FIREFOX" = "true" ]; then \
/docker/apt-install.sh wget bzip2 \
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
else \
echo "Skipping Firefox installation in dev mode"; \
fi
# Install MySQL client dependencies
RUN --mount=type=bind,source=./docker,target=/docker \
/docker/apt-install.sh default-libmysqlclient-dev
pkg-config \
default-libmysqlclient-dev
# Copy development requirements and install them
COPY --chown=superset:superset requirements/development.txt requirements/
RUN --mount=type=bind,source=./docker,target=/docker \
--mount=type=cache,target=/root/.cache/pip \
/docker/pip-install.sh --requires-build-essential -r requirements/development.txt
COPY requirements/*.txt requirements/
# Install Python dependencies using docker/pip-install.sh
RUN --mount=type=cache,target=/root/.cache/pip \
/app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt && \
uv pip install .
RUN python -m compileall /app/superset
USER superset
@ -232,6 +227,4 @@ USER superset
######################################################################
FROM lean AS ci
COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/
CMD ["/app/docker/docker-ci.sh"]
CMD ["/app/docker/entrypoints/docker-ci.sh"]

1
UPDATING.md
View File

@ -30,6 +30,7 @@ assists people when migrating to a new version.
- [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`.
- [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before.
- [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context.
- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ...
### Potential Downtime

1
docker/.env
View File

@ -17,6 +17,7 @@
COMPOSE_PROJECT_NAME=superset
DEV_MODE=true
# database configurations (do not modify)
DATABASE_DB=superset

5
docker/docker-bootstrap.sh
View File

@ -18,6 +18,11 @@
set -eo pipefail
# Make python interactive
if [ "$DEV_MODE" == "true" ]; then
echo "Reinstalling the app in editable mode"
uv pip install -e .
fi
REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt"
# If Cypress run – overwrite the password for admin and export env variables
if [ "$CYPRESS_CONFIG" == "true" ]; then

0
docker/docker-ci.sh → docker/entrypoints/docker-ci.sh
View File

0
docker/run-server.sh → docker/entrypoints/run-server.sh
View File

4
docker/pip-install.sh
View File

@ -47,10 +47,10 @@ fi
# Choose whether to use pip cache
if $USE_CACHE; then
echo "Using pip cache..."
uv pip install --system "${ARGS[@]}"
uv pip install "${ARGS[@]}"
else
echo "Disabling pip cache..."
uv pip install --system --no-cache-dir "${ARGS[@]}"
uv pip install --no-cache-dir "${ARGS[@]}"
fi
# Remove build-essential if it was installed

1
requirements/translations.in Normal file
View File

@ -0,0 +1 @@
babel

9
requirements/translations.txt Normal file
View File

@ -0,0 +1,9 @@
# SHA1:cad160f3d4cd7c33896f42a479eeaa1b5bedc5fb
#
# This file is autogenerated by pip-compile-multi
# To update, run:
#
# pip-compile-multi
#
babel==2.16.0
# via -r requirements/translations.in