diff --git a/Dockerfile b/Dockerfile index 9b48d0396..9a89cef35 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,44 +20,38 @@ ###################################################################### ARG PY_VER=3.10-slim-bookworm -# if BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise). +# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise). ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64} FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node +# Arguments for build configuration ARG NPM_BUILD_CMD="build" +ARG BUILD_TRANSLATIONS="false" # Include translations in the final build +ARG DEV_MODE="false" # Skip frontend build in dev mode +ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports +ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled -# Include translations in the final build. The default supports en only to -# reduce complexity and weight for those only using en -ARG BUILD_TRANSLATIONS="false" - -# Used by docker-compose to skip the frontend build, -# in dev we mount the repo and build the frontend inside docker -ARG DEV_MODE="false" - -# Include headless browsers? Allows for alerts, reports & thumbnails, but bloats the images -ARG INCLUDE_CHROMIUM="true" -ARG INCLUDE_FIREFOX="false" - -# Somehow we need python3 + build-essential on this side of the house to install node-gyp -RUN apt-get update -qq \ - && apt-get install \ - -yqq --no-install-recommends \ - build-essential \ - python3 \ - zstd +# Install system dependencies required for node-gyp +RUN --mount=type=bind,source=./docker,target=/docker \ + /docker/apt-install.sh build-essential python3 zstd +# Define environment variables for frontend build ENV BUILD_CMD=${NPM_BUILD_CMD} \ PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true -# NPM ci first, as to NOT invalidate previous steps except for when package.json changes -RUN --mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \ - /frontend-mem-nag.sh +# Run the frontend memory monitoring script +RUN --mount=type=bind,source=./docker,target=/docker \ + /docker/frontend-mem-nag.sh WORKDIR /app/superset-frontend -# Creating empty folders to avoid errors when running COPY later on -RUN mkdir -p /app/superset/static/assets -RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json \ - --mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json \ + +# Create necessary folders to avoid errors in subsequent steps +RUN mkdir -p /app/superset/static/assets \ + /app/superset/translations + +# Mount package files and install dependencies if not in dev mode +RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \ + --mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \ if [ "$DEV_MODE" = "false" ]; then \ npm ci; \ else \ @@ -66,33 +60,39 @@ RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json # Runs the webpack build process COPY superset-frontend /app/superset-frontend -# This copies the .po files needed for translation -RUN mkdir -p /app/superset/translations + + +# Copy translation files COPY superset/translations /app/superset/translations + +# Build the frontend if not in dev mode RUN if [ "$DEV_MODE" = "false" ]; then \ BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \ else \ echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \ fi - -# Compiles .json files from the .po files, then deletes the .po files +# Compile .json files from .po translations (if required) and clean up .po files RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ npm run build-translation; \ else \ echo "Skipping translations as requested by build flag"; \ - fi -RUN rm /app/superset/translations/*/LC_MESSAGES/*.po -RUN rm /app/superset/translations/messages.pot + fi \ + # removing translations files regardless + && rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \ + /app/superset/translations/messages.pot + +# Transition to Python base image FROM python:${PY_VER} AS python-base +RUN pip install --no-cache-dir --upgrade setuptools pip + ###################################################################### # Final lean image... ###################################################################### FROM python-base AS lean -# Include translations in the final build. The default supports en only to -# reduce complexity and weight for those only using en +# Build argument for including translations ARG BUILD_TRANSLATIONS="false" WORKDIR /app @@ -104,9 +104,16 @@ ENV LANG=C.UTF-8 \ SUPERSET_HOME="/app/superset_home" \ SUPERSET_PORT=8088 -RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache_superset.egg-info requirements \ +# Set up necessary directories and user +RUN --mount=type=bind,source=./docker,target=/docker \ + mkdir -p ${PYTHONPATH} \ + superset/static \ + requirements \ + superset-frontend \ + apache_superset.egg-info \ + requirements \ && useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \ - && apt-get update -qq && apt-get install -yqq --no-install-recommends \ + && /docker/apt-install.sh \ curl \ libsasl2-dev \ libsasl2-modules-gssapi-mit \ @@ -117,58 +124,62 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache && chown -R superset:superset ./* \ && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* +# Copy required files for Python build COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./ -# setup.py uses the version information in package.json COPY --chown=superset:superset superset-frontend/package.json superset-frontend/ COPY --chown=superset:superset requirements/base.txt requirements/ COPY --chown=superset:superset scripts/check-env.py scripts/ -RUN --mount=type=cache,target=/root/.cache/pip \ - apt-get update -qq && apt-get install -yqq --no-install-recommends \ - build-essential \ - && pip install --no-cache-dir --upgrade setuptools pip \ - && pip install --no-cache-dir -r requirements/base.txt \ - && apt-get autoremove -yqq --purge build-essential \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* -# Copy the compiled frontend assets +# Install Python dependencies using docker/pip-install.sh +RUN --mount=type=bind,source=./docker,target=/docker \ + --mount=type=cache,target=/root/.cache/pip \ + /docker/pip-install.sh --requires-build-essential -r requirements/base.txt + +# Copy the compiled frontend assets from the node image COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets -## Lastly, let's install superset itself +# Copy the main Superset source code COPY --chown=superset:superset superset superset -RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir -e . -# Copy the .json translations from the frontend layer +# Install Superset itself using docker/pip-install.sh +RUN --mount=type=bind,source=./docker,target=/docker \ + --mount=type=cache,target=/root/.cache/pip \ + /docker/pip-install.sh -e . + +# Copy .json translations from the node image COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations -# Compile translations for the backend - this generates .mo files, then deletes the .po files +# Compile backend translations and clean up COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/ RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \ ./scripts/translations/generate_mo_files.sh \ - && chown -R superset:superset superset/translations \ - && rm superset/translations/messages.pot \ - && rm superset/translations/*/LC_MESSAGES/*.po; \ - else \ - echo "Skipping translations as requested by build flag"; \ - fi + && chown -R superset:superset superset/translations; \ + fi \ + && rm -rf superset/translations/messages.pot \ + superset/translations/*/LC_MESSAGES/*.po +# Add server run script COPY --chmod=755 ./docker/run-server.sh /usr/bin/ -USER superset +# Set user and healthcheck +USER superset HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health" +# Expose port and set CMD EXPOSE ${SUPERSET_PORT} - CMD ["/usr/bin/run-server.sh"] + ###################################################################### # Dev image... ###################################################################### FROM lean AS dev USER root -RUN apt-get update -qq \ - && apt-get install -yqq --no-install-recommends \ + +# Install dev dependencies +RUN --mount=type=bind,source=./docker,target=/docker \ + /docker/apt-install.sh \ libnss3 \ libdbus-glib-1-2 \ libgtk-3-0 \ @@ -176,46 +187,46 @@ RUN apt-get update -qq \ libasound2 \ libxtst6 \ git \ - pkg-config \ - && rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/* + pkg-config +# Install Playwright and its dependencies RUN --mount=type=cache,target=/root/.cache/pip \ - pip install --no-cache-dir playwright -RUN playwright install-deps + pip install playwright \ + && playwright install-deps +# Optionally install Chromium RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \ playwright install chromium; \ else \ - echo "Skipping translations in dev mode"; \ + echo "Skipping Chromium installation in dev mode"; \ fi -# Install GeckoDriver WebDriver -ARG GECKODRIVER_VERSION=v0.34.0 \ - FIREFOX_VERSION=125.0.3 - -RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \ - apt-get update -qq \ - && apt-get install -yqq --no-install-recommends wget bzip2 \ +# Install GeckoDriver WebDriver and Firefox (if required) +ARG GECKODRIVER_VERSION=v0.34.0 +ARG FIREFOX_VERSION=125.0.3 +RUN --mount=type=bind,source=./docker,target=/docker \ + if [ "$INCLUDE_FIREFOX" = "true" ]; then \ + /docker/apt-install.sh wget bzip2 \ && wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \ && wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \ && ln -s /opt/firefox/firefox /usr/local/bin/firefox \ && apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \ + else \ + echo "Skipping Firefox installation in dev mode"; \ fi -# Installing mysql client os-level dependencies in dev image only because GPL -RUN apt-get install -yqq --no-install-recommends \ - default-libmysqlclient-dev \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* +# Install MySQL client dependencies +RUN --mount=type=bind,source=./docker,target=/docker \ + /docker/apt-install.sh default-libmysqlclient-dev +# Copy development requirements and install them COPY --chown=superset:superset requirements/development.txt requirements/ -RUN --mount=type=cache,target=/root/.cache/pip \ - apt-get update -qq && apt-get install -yqq --no-install-recommends \ - build-essential \ - && pip install --no-cache-dir -r requirements/development.txt \ - && apt-get autoremove -yqq --purge build-essential \ - && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* +RUN --mount=type=bind,source=./docker,target=/docker \ + --mount=type=cache,target=/root/.cache/pip \ + /docker/pip-install.sh --requires-build-essential -r requirements/development.txt USER superset + ###################################################################### # CI image... ###################################################################### diff --git a/docker/apt-install.sh b/docker/apt-install.sh new file mode 100755 index 000000000..bd9152beb --- /dev/null +++ b/docker/apt-install.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +# Ensure this script is run as root +if [[ $EUID -ne 0 ]]; then + echo "This script must be run as root" >&2 + exit 1 +fi + +# Check for required arguments +if [[ $# -lt 1 ]]; then + echo "Usage: $0 [ ...]" >&2 + exit 1 +fi + +# Colors for better logging (optional) +GREEN='\033[0;32m' +RED='\033[0;31m' +RESET='\033[0m' + +# Install packages with clean-up +echo -e "${GREEN}Updating package lists...${RESET}" +apt-get update -qq + +echo -e "${GREEN}Installing packages: $@${RESET}" +apt-get install -yqq --no-install-recommends "$@" + +echo -e "${GREEN}Autoremoving unnecessary packages...${RESET}" +apt-get autoremove -y + +echo -e "${GREEN}Cleaning up package cache and metadata...${RESET}" +apt-get clean +rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/* + +echo -e "${GREEN}Installation and cleanup complete.${RESET}" diff --git a/docker/pip-install.sh b/docker/pip-install.sh new file mode 100755 index 000000000..2defc7d1e --- /dev/null +++ b/docker/pip-install.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -euo pipefail + +# Default flag +REQUIRES_BUILD_ESSENTIAL=false +USE_CACHE=true + +# Filter arguments +ARGS=() +for arg in "$@"; do + case "$arg" in + --requires-build-essential) + REQUIRES_BUILD_ESSENTIAL=true + ;; + --no-cache) + USE_CACHE=false + ;; + *) + ARGS+=("$arg") + ;; + esac +done + +# Install build-essential if required +if $REQUIRES_BUILD_ESSENTIAL; then + echo "Installing build-essential for package builds..." + apt-get update -qq \ + && apt-get install -yqq --no-install-recommends build-essential +fi + +# Choose whether to use pip cache +if $USE_CACHE; then + echo "Using pip cache..." + pip install "${ARGS[@]}" +else + echo "Disabling pip cache..." + pip install --no-cache-dir "${ARGS[@]}" +fi + +# Remove build-essential if it was installed +if $REQUIRES_BUILD_ESSENTIAL; then + echo "Removing build-essential to keep the image lean..." + apt-get autoremove -yqq --purge build-essential \ + && apt-get clean \ + && rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* +fi + +echo "Python packages installed successfully." diff --git a/superset-websocket/Dockerfile b/superset-websocket/Dockerfile index 4cc2117f0..ac6e4a299 100644 --- a/superset-websocket/Dockerfile +++ b/superset-websocket/Dockerfile @@ -12,7 +12,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -FROM node:16-alpine as build +FROM node:16-alpine AS build WORKDIR /home/superset-websocket