diff --git a/.dockerignore b/.dockerignore index e79b8f6e1..33b76412b 100644 --- a/.dockerignore +++ b/.dockerignore @@ -45,3 +45,4 @@ superset-frontend/coverage/ superset/static/assets/ superset-websocket/dist/ venv +.venv diff --git a/.github/workflows/ephemeral-env.yml b/.github/workflows/ephemeral-env.yml index 9b5539ed6..acf3b0cc7 100644 --- a/.github/workflows/ephemeral-env.yml +++ b/.github/workflows/ephemeral-env.yml @@ -148,9 +148,20 @@ jobs: - name: Setup supersetbot uses: ./.github/actions/setup-supersetbot/ + - name: Try to login to DockerHub + if: steps.check.outputs.python || steps.check.outputs.frontend || steps.check.outputs.docker + continue-on-error: true + uses: docker/login-action@v3 + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} + - name: Build ephemeral env image env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + with: + username: ${{ secrets.DOCKERHUB_USER }} + password: ${{ secrets.DOCKERHUB_TOKEN }} run: | supersetbot docker \ --preset ci \ diff --git a/Dockerfile b/Dockerfile index 60ba12eab..4ee309308 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,26 +22,29 @@ ARG PY_VER=3.10-slim-bookworm # If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise). ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64} -FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node +###################################################################### +# superset-node used for building frontend assets +###################################################################### +FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node +ARG BUILD_TRANSLATIONS="false" # Include translations in the final build +ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} +ARG DEV_MODE="false" # Skip frontend build in dev mode +ENV DEV_MODE=${DEV_MODE} + +COPY docker/ /app/docker/ # Arguments for build configuration ARG NPM_BUILD_CMD="build" -ARG BUILD_TRANSLATIONS="false" # Include translations in the final build -ARG DEV_MODE="false" # Skip frontend build in dev mode -ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports -ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled # Install system dependencies required for node-gyp -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh build-essential python3 zstd +RUN /app/docker/apt-install.sh build-essential python3 zstd # Define environment variables for frontend build ENV BUILD_CMD=${NPM_BUILD_CMD} \ PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true # Run the frontend memory monitoring script -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/frontend-mem-nag.sh +RUN /app/docker/frontend-mem-nag.sh WORKDIR /app/superset-frontend @@ -49,6 +52,9 @@ WORKDIR /app/superset-frontend RUN mkdir -p /app/superset/static/assets \ /app/superset/translations +# Copy translation files +COPY superset/translations /app/superset/translations + # Mount package files and install dependencies if not in dev mode RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \ --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \ @@ -61,41 +67,28 @@ RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.j # Runs the webpack build process COPY superset-frontend /app/superset-frontend - -# Copy translation files -COPY superset/translations /app/superset/translations - # Build the frontend if not in dev mode RUN if [ "$DEV_MODE" = "false" ]; then \ - BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \ + echo "Running 'npm run ${BUILD_CMD}'"; \ + if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ + npm run build-translation; \ + fi; \ + npm run ${BUILD_CMD}; \ else \ echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \ - fi - -# Compile .json files from .po translations (if required) and clean up .po files -RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ - npm run build-translation; \ - else \ - echo "Skipping translations as requested by build flag"; \ - fi \ - # removing translations files regardless - && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \ - /app/superset/translations/messages.pot + fi && \ + rm -rf /app/superset/translations/*/*/*.po -# Transition to Python base image +###################################################################### +# Base python layer +###################################################################### FROM python:${PY_VER} AS python-base -RUN pip install --no-cache-dir --upgrade setuptools pip uv +ARG BUILD_TRANSLATIONS="false" # Include translations in the final build +ENV BUILD_TRANSLATIONS=${BUILD_TRANSLATIONS} +ARG DEV_MODE="false" # Skip frontend build in dev mode +ENV DEV_MODE=${DEV_MODE} -###################################################################### -# Final lean image... -###################################################################### -FROM python-base AS lean - -# Build argument for including translations -ARG BUILD_TRANSLATIONS="false" - -WORKDIR /app ENV LANG=C.UTF-8 \ LC_ALL=C.UTF-8 \ SUPERSET_ENV=production \ @@ -104,126 +97,128 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 + +RUN useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset + +# Some bash scripts needed throughout the layers +COPY --chmod=755 docker/*.sh /app/docker/ + +RUN pip install --no-cache-dir --upgrade setuptools pip uv + +# Using uv as it's faster/simpler than pip +RUN uv venv /app/.venv +ENV PATH="/app/.venv/bin:${PATH}" + +# Install Playwright and optionally setup headless browsers +ARG INCLUDE_CHROMIUM="true" +ARG INCLUDE_FIREFOX="false" +RUN --mount=type=cache,target=/root/.cache/pip \ + if [ "$INCLUDE_CHROMIUM" = "true" ] || [ "$INCLUDE_FIREFOX" = "true" ]; then \ + pip install playwright && \ + playwright install-deps && \ + if [ "$INCLUDE_CHROMIUM" = "true" ]; then playwright install chromium; fi && \ + if [ "$INCLUDE_FIREFOX" = "true" ]; then playwright install firefox; fi; \ + else \ + echo "Skipping browser installation"; \ + fi + +###################################################################### +# Python translation compiler layer +###################################################################### +FROM python-base AS python-translation-compiler + +# Install Python dependencies using docker/pip-install.sh +COPY requirements/translations.txt requirements/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /app/docker/pip-install.sh -r requirements/translations.txt + +COPY superset/translations/ /app/translations_mo/ +RUN pybabel compile -d /app/translations_mo | true && \ + rm -f /app/translations_mo/*/*/*.po + +###################################################################### +# Python APP common layer +###################################################################### +FROM python-base AS python-common +# Copy the entrypoints, make them executable in userspace +COPY --chmod=755 docker/entrypoints /app/docker/entrypoints + +WORKDIR /app # Set up necessary directories and user -RUN --mount=type=bind,source=./docker,target=/docker \ - mkdir -p ${PYTHONPATH} \ +RUN mkdir -p \ + ${SUPERSET_HOME} \ + ${PYTHONPATH} \ superset/static \ requirements \ superset-frontend \ apache_superset.egg-info \ requirements \ - && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && /docker/apt-install.sh \ - curl \ - libsasl2-dev \ - libsasl2-modules-gssapi-mit \ - libpq-dev \ - libecpg-dev \ - libldap2-dev \ - && touch superset/static/version_info.json \ - && chown -R superset:superset ./* \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* + && touch superset/static/version_info.json # Copy required files for Python build -COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./ -COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ -COPY --chown=superset:superset requirements/base.txt requirements/ -COPY --chown=superset:superset scripts/check-env.py scripts/ +COPY pyproject.toml setup.py MANIFEST.in README.md ./ +COPY superset-frontend/package.json superset-frontend/ +COPY scripts/check-env.py scripts/ + +# keeping for backward compatibility +COPY --chmod=755 ./docker/entrypoints/run-server.sh /usr/bin/ + +# Some debian libs +RUN /app/docker/apt-install.sh \ + curl \ + libsasl2-dev \ + libsasl2-modules-gssapi-mit \ + libpq-dev \ + libecpg-dev \ + libldap2-dev + +# Copy compiled things from previous stages +COPY --from=superset-node /app/superset/static/assets superset/static/assets + +# Merging translations from backend and frontend stages +COPY --from=superset-node /app/superset/translations superset/translations +COPY --from=python-translation-compiler /app/translations_mo superset/translations + +HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health" +CMD ["/app/docker/entrypoints/run-server.sh"] +EXPOSE ${SUPERSET_PORT} + +###################################################################### +# Final lean image... +###################################################################### +FROM python-common AS lean +COPY superset superset # Install Python dependencies using docker/pip-install.sh -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh --requires-build-essential -r requirements/base.txt +COPY requirements/base.txt requirements/ +RUN --mount=type=cache,target=/root/.cache/pip \ + /app/docker/pip-install.sh --requires-build-essential -r requirements/base.txt && \ + uv pip install . -# Copy the compiled frontend assets from the node image -COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets +RUN python -m compileall /app/superset -# Copy the main Superset source code -COPY --chown=superset:superset superset superset - -# Install Superset itself using docker/pip-install.sh -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh -e . - -# Copy .json translations from the node image -COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations - -# Compile backend translations and clean up -COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/ -RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ - ./scripts/translations/generate_mo_files.sh \ - && chown -R superset:superset superset/translations; \ - fi \ - && rm -rf superset/translations/messages.pot \ - superset/translations/*/LC_MESSAGES/*.po - -# Add server run script -COPY --chmod=755 ./docker/run-server.sh /usr/bin/ - -# Set user and healthcheck USER superset -HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health" - -# Expose port and set CMD -EXPOSE ${SUPERSET_PORT} -CMD ["/usr/bin/run-server.sh"] - ###################################################################### # Dev image... ###################################################################### -FROM lean AS dev +FROM python-common AS dev +COPY superset superset -USER root - -# Install dev dependencies -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh \ - libnss3 \ - libdbus-glib-1-2 \ - libgtk-3-0 \ - libx11-xcb1 \ - libasound2 \ - libxtst6 \ - git \ - pkg-config - -# Install Playwright and its dependencies -RUN --mount=type=cache,target=/root/.cache/pip \ - uv pip install --system playwright \ - && playwright install-deps - -# Optionally install Chromium -RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \ - playwright install chromium; \ - else \ - echo "Skipping Chromium installation in dev mode"; \ - fi - -# Install GeckoDriver WebDriver and Firefox (if required) -ARG GECKODRIVER_VERSION=v0.34.0 -ARG FIREFOX_VERSION=125.0.3 -RUN --mount=type=bind,source=./docker,target=/docker \ - if [ "$INCLUDE_FIREFOX" = "true" ]; then \ - /docker/apt-install.sh wget bzip2 \ - && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \ - && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \ - && ln -s /opt/firefox/firefox /usr/local/bin/firefox \ - && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \ - else \ - echo "Skipping Firefox installation in dev mode"; \ - fi - -# Install MySQL client dependencies -RUN --mount=type=bind,source=./docker,target=/docker \ - /docker/apt-install.sh default-libmysqlclient-dev +# Debian libs needed for dev +RUN /app/docker/apt-install.sh \ + git \ + pkg-config \ + default-libmysqlclient-dev # Copy development requirements and install them -COPY --chown=superset:superset requirements/development.txt requirements/ -RUN --mount=type=bind,source=./docker,target=/docker \ - --mount=type=cache,target=/root/.cache/pip \ - /docker/pip-install.sh --requires-build-essential -r requirements/development.txt +COPY requirements/*.txt requirements/ +# Install Python dependencies using docker/pip-install.sh +RUN --mount=type=cache,target=/root/.cache/pip \ + /app/docker/pip-install.sh --requires-build-essential -r requirements/development.txt && \ + uv pip install . + +RUN python -m compileall /app/superset USER superset @@ -232,6 +227,4 @@ USER superset ###################################################################### FROM lean AS ci -COPY --chown=superset:superset --chmod=755 ./docker/*.sh /app/docker/ - -CMD ["/app/docker/docker-ci.sh"] +CMD ["/app/docker/entrypoints/docker-ci.sh"] diff --git a/UPDATING.md b/UPDATING.md index a5ae17366..4267ae340 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -30,6 +30,7 @@ assists people when migrating to a new version. - [30099](https://github.com/apache/superset/pull/30099) Translations are no longer included in the default docker image builds. If your environment requires translations, you'll want to set the docker build arg `BUILD_TRANSACTION=true`. - [31262](https://github.com/apache/superset/pull/31262) NOTE: deprecated `pylint` in favor of `ruff` as our only python linter. Only affect development workflows positively (not the release itself). It should cover most important rules, be much faster, but some things linting rules that were enforced before may not be enforce in the exact same way as before. - [31173](https://github.com/apache/superset/pull/31173) Modified `fetch_csrf_token` to align with HTTP standards, particularly regarding how cookies are handled. If you encounter any issues related to CSRF functionality, please report them as a new issue and reference this PR for context. +- [31385](https://github.com/apache/superset/pull/31385) Significant docker refactor, reducing access levels for the `superset` user, streamlining layer building, ... ### Potential Downtime diff --git a/docker/.env b/docker/.env index 57575da76..751176656 100644 --- a/docker/.env +++ b/docker/.env @@ -17,6 +17,7 @@ COMPOSE_PROJECT_NAME=superset +DEV_MODE=true # database configurations (do not modify) DATABASE_DB=superset diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index 2f0b29ce3..1a4e04be9 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -18,6 +18,11 @@ set -eo pipefail +# Make python interactive +if [ "$DEV_MODE" == "true" ]; then + echo "Reinstalling the app in editable mode" + uv pip install -e . +fi REQUIREMENTS_LOCAL="/app/docker/requirements-local.txt" # If Cypress run – overwrite the password for admin and export env variables if [ "$CYPRESS_CONFIG" == "true" ]; then diff --git a/docker/docker-ci.sh b/docker/entrypoints/docker-ci.sh similarity index 100% rename from docker/docker-ci.sh rename to docker/entrypoints/docker-ci.sh diff --git a/docker/run-server.sh b/docker/entrypoints/run-server.sh similarity index 100% rename from docker/run-server.sh rename to docker/entrypoints/run-server.sh diff --git a/docker/pip-install.sh b/docker/pip-install.sh index 7e69a6efb..7deb4fa19 100755 --- a/docker/pip-install.sh +++ b/docker/pip-install.sh @@ -47,10 +47,10 @@ fi # Choose whether to use pip cache if $USE_CACHE; then echo "Using pip cache..." - uv pip install --system "${ARGS[@]}" + uv pip install "${ARGS[@]}" else echo "Disabling pip cache..." - uv pip install --system --no-cache-dir "${ARGS[@]}" + uv pip install --no-cache-dir "${ARGS[@]}" fi # Remove build-essential if it was installed diff --git a/requirements/translations.in b/requirements/translations.in new file mode 100644 index 000000000..98f65931c --- /dev/null +++ b/requirements/translations.in @@ -0,0 +1 @@ +babel diff --git a/requirements/translations.txt b/requirements/translations.txt new file mode 100644 index 000000000..4eab2d21f --- /dev/null +++ b/requirements/translations.txt @@ -0,0 +1,9 @@ +# SHA1:cad160f3d4cd7c33896f42a479eeaa1b5bedc5fb +# +# This file is autogenerated by pip-compile-multi +# To update, run: +# +# pip-compile-multi +# +babel==2.16.0 + # via -r requirements/translations.in