chore: simplify Dockerfile package install calls with bash wrappers (#31034)
This commit is contained in:
Maxime Beauchemin
GitHub
parent
fe80fb1090
commit
b3559f644c
179
Dockerfile
179
Dockerfile
|
|
@ -20,44 +20,38 @@
|
|||
######################################################################
|
||||
ARG PY_VER=3.10-slim-bookworm
|
||||
|
||||
# if BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
|
||||
# If BUILDPLATFORM is null, set it to 'amd64' (or leave as is otherwise).
|
||||
ARG BUILDPLATFORM=${BUILDPLATFORM:-amd64}
|
||||
FROM --platform=${BUILDPLATFORM} node:20-bullseye-slim AS superset-node
|
||||
|
||||
# Arguments for build configuration
|
||||
ARG NPM_BUILD_CMD="build"
|
||||
ARG BUILD_TRANSLATIONS="false" # Include translations in the final build
|
||||
ARG DEV_MODE="false" # Skip frontend build in dev mode
|
||||
ARG INCLUDE_CHROMIUM="true" # Include headless Chromium for alerts & reports
|
||||
ARG INCLUDE_FIREFOX="false" # Include headless Firefox if enabled
|
||||
|
||||
# Include translations in the final build. The default supports en only to
|
||||
# reduce complexity and weight for those only using en
|
||||
ARG BUILD_TRANSLATIONS="false"
|
||||
|
||||
# Used by docker-compose to skip the frontend build,
|
||||
# in dev we mount the repo and build the frontend inside docker
|
||||
ARG DEV_MODE="false"
|
||||
|
||||
# Include headless browsers? Allows for alerts, reports & thumbnails, but bloats the images
|
||||
ARG INCLUDE_CHROMIUM="true"
|
||||
ARG INCLUDE_FIREFOX="false"
|
||||
|
||||
# Somehow we need python3 + build-essential on this side of the house to install node-gyp
|
||||
RUN apt-get update -qq \
|
||||
&& apt-get install \
|
||||
-yqq --no-install-recommends \
|
||||
build-essential \
|
||||
python3 \
|
||||
zstd
|
||||
# Install system dependencies required for node-gyp
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
/docker/apt-install.sh build-essential python3 zstd
|
||||
|
||||
# Define environment variables for frontend build
|
||||
ENV BUILD_CMD=${NPM_BUILD_CMD} \
|
||||
PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
|
||||
# NPM ci first, as to NOT invalidate previous steps except for when package.json changes
|
||||
|
||||
RUN --mount=type=bind,target=/frontend-mem-nag.sh,src=./docker/frontend-mem-nag.sh \
|
||||
/frontend-mem-nag.sh
|
||||
# Run the frontend memory monitoring script
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
/docker/frontend-mem-nag.sh
|
||||
|
||||
WORKDIR /app/superset-frontend
|
||||
# Creating empty folders to avoid errors when running COPY later on
|
||||
RUN mkdir -p /app/superset/static/assets
|
||||
RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json \
|
||||
--mount=type=bind,target=./package-lock.json,src=./superset-frontend/package-lock.json \
|
||||
|
||||
# Create necessary folders to avoid errors in subsequent steps
|
||||
RUN mkdir -p /app/superset/static/assets \
|
||||
/app/superset/translations
|
||||
|
||||
# Mount package files and install dependencies if not in dev mode
|
||||
RUN --mount=type=bind,source=./superset-frontend/package.json,target=./package.json \
|
||||
--mount=type=bind,source=./superset-frontend/package-lock.json,target=./package-lock.json \
|
||||
if [ "$DEV_MODE" = "false" ]; then \
|
||||
npm ci; \
|
||||
else \
|
||||
|
|
@ -66,33 +60,39 @@ RUN --mount=type=bind,target=./package.json,src=./superset-frontend/package.json
|
|||
|
||||
# Runs the webpack build process
|
||||
COPY superset-frontend /app/superset-frontend
|
||||
# This copies the .po files needed for translation
|
||||
RUN mkdir -p /app/superset/translations
|
||||
|
||||
|
||||
# Copy translation files
|
||||
COPY superset/translations /app/superset/translations
|
||||
|
||||
# Build the frontend if not in dev mode
|
||||
RUN if [ "$DEV_MODE" = "false" ]; then \
|
||||
BUILD_TRANSLATIONS=$BUILD_TRANSLATIONS npm run ${BUILD_CMD}; \
|
||||
else \
|
||||
echo "Skipping 'npm run ${BUILD_CMD}' in dev mode"; \
|
||||
fi
|
||||
|
||||
|
||||
# Compiles .json files from the .po files, then deletes the .po files
|
||||
# Compile .json files from .po translations (if required) and clean up .po files
|
||||
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
|
||||
npm run build-translation; \
|
||||
else \
|
||||
echo "Skipping translations as requested by build flag"; \
|
||||
fi
|
||||
RUN rm /app/superset/translations/*/LC_MESSAGES/*.po
|
||||
RUN rm /app/superset/translations/messages.pot
|
||||
fi \
|
||||
# removing translations files regardless
|
||||
&& rm -rf /app/superset/translations/*/LC_MESSAGES/*.po \
|
||||
/app/superset/translations/messages.pot
|
||||
|
||||
|
||||
# Transition to Python base image
|
||||
FROM python:${PY_VER} AS python-base
|
||||
RUN pip install --no-cache-dir --upgrade setuptools pip
|
||||
|
||||
######################################################################
|
||||
# Final lean image...
|
||||
######################################################################
|
||||
FROM python-base AS lean
|
||||
|
||||
# Include translations in the final build. The default supports en only to
|
||||
# reduce complexity and weight for those only using en
|
||||
# Build argument for including translations
|
||||
ARG BUILD_TRANSLATIONS="false"
|
||||
|
||||
WORKDIR /app
|
||||
|
|
@ -104,9 +104,16 @@ ENV LANG=C.UTF-8 \
|
|||
SUPERSET_HOME="/app/superset_home" \
|
||||
SUPERSET_PORT=8088
|
||||
|
||||
RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache_superset.egg-info requirements \
|
||||
# Set up necessary directories and user
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
mkdir -p ${PYTHONPATH} \
|
||||
superset/static \
|
||||
requirements \
|
||||
superset-frontend \
|
||||
apache_superset.egg-info \
|
||||
requirements \
|
||||
&& useradd --user-group -d ${SUPERSET_HOME} -m --no-log-init --shell /bin/bash superset \
|
||||
&& apt-get update -qq && apt-get install -yqq --no-install-recommends \
|
||||
&& /docker/apt-install.sh \
|
||||
curl \
|
||||
libsasl2-dev \
|
||||
libsasl2-modules-gssapi-mit \
|
||||
|
|
@ -117,58 +124,62 @@ RUN mkdir -p ${PYTHONPATH} superset/static requirements superset-frontend apache
|
|||
&& chown -R superset:superset ./* \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
# Copy required files for Python build
|
||||
COPY --chown=superset:superset pyproject.toml setup.py MANIFEST.in README.md ./
|
||||
# setup.py uses the version information in package.json
|
||||
COPY --chown=superset:superset superset-frontend/package.json superset-frontend/
|
||||
COPY --chown=superset:superset requirements/base.txt requirements/
|
||||
COPY --chown=superset:superset scripts/check-env.py scripts/
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
apt-get update -qq && apt-get install -yqq --no-install-recommends \
|
||||
build-essential \
|
||||
&& pip install --no-cache-dir --upgrade setuptools pip \
|
||||
&& pip install --no-cache-dir -r requirements/base.txt \
|
||||
&& apt-get autoremove -yqq --purge build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
|
||||
# Copy the compiled frontend assets
|
||||
# Install Python dependencies using docker/pip-install.sh
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
/docker/pip-install.sh --requires-build-essential -r requirements/base.txt
|
||||
|
||||
# Copy the compiled frontend assets from the node image
|
||||
COPY --chown=superset:superset --from=superset-node /app/superset/static/assets superset/static/assets
|
||||
|
||||
## Lastly, let's install superset itself
|
||||
# Copy the main Superset source code
|
||||
COPY --chown=superset:superset superset superset
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-cache-dir -e .
|
||||
|
||||
# Copy the .json translations from the frontend layer
|
||||
# Install Superset itself using docker/pip-install.sh
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
/docker/pip-install.sh -e .
|
||||
|
||||
# Copy .json translations from the node image
|
||||
COPY --chown=superset:superset --from=superset-node /app/superset/translations superset/translations
|
||||
|
||||
# Compile translations for the backend - this generates .mo files, then deletes the .po files
|
||||
# Compile backend translations and clean up
|
||||
COPY ./scripts/translations/generate_mo_files.sh ./scripts/translations/
|
||||
RUN if [ "$BUILD_TRANSLATIONS" = "true" ]; then \
|
||||
./scripts/translations/generate_mo_files.sh \
|
||||
&& chown -R superset:superset superset/translations \
|
||||
&& rm superset/translations/messages.pot \
|
||||
&& rm superset/translations/*/LC_MESSAGES/*.po; \
|
||||
else \
|
||||
echo "Skipping translations as requested by build flag"; \
|
||||
fi
|
||||
&& chown -R superset:superset superset/translations; \
|
||||
fi \
|
||||
&& rm -rf superset/translations/messages.pot \
|
||||
superset/translations/*/LC_MESSAGES/*.po
|
||||
|
||||
# Add server run script
|
||||
COPY --chmod=755 ./docker/run-server.sh /usr/bin/
|
||||
USER superset
|
||||
|
||||
# Set user and healthcheck
|
||||
USER superset
|
||||
HEALTHCHECK CMD curl -f "http://localhost:${SUPERSET_PORT}/health"
|
||||
|
||||
# Expose port and set CMD
|
||||
EXPOSE ${SUPERSET_PORT}
|
||||
|
||||
CMD ["/usr/bin/run-server.sh"]
|
||||
|
||||
|
||||
######################################################################
|
||||
# Dev image...
|
||||
######################################################################
|
||||
FROM lean AS dev
|
||||
|
||||
USER root
|
||||
RUN apt-get update -qq \
|
||||
&& apt-get install -yqq --no-install-recommends \
|
||||
|
||||
# Install dev dependencies
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
/docker/apt-install.sh \
|
||||
libnss3 \
|
||||
libdbus-glib-1-2 \
|
||||
libgtk-3-0 \
|
||||
|
|
@ -176,46 +187,46 @@ RUN apt-get update -qq \
|
|||
libasound2 \
|
||||
libxtst6 \
|
||||
git \
|
||||
pkg-config \
|
||||
&& rm -rf /var/cache/apt/archives/* /var/lib/apt/lists/*
|
||||
pkg-config
|
||||
|
||||
# Install Playwright and its dependencies
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
pip install --no-cache-dir playwright
|
||||
RUN playwright install-deps
|
||||
pip install playwright \
|
||||
&& playwright install-deps
|
||||
|
||||
# Optionally install Chromium
|
||||
RUN if [ "$INCLUDE_CHROMIUM" = "true" ]; then \
|
||||
playwright install chromium; \
|
||||
else \
|
||||
echo "Skipping translations in dev mode"; \
|
||||
echo "Skipping Chromium installation in dev mode"; \
|
||||
fi
|
||||
|
||||
# Install GeckoDriver WebDriver
|
||||
ARG GECKODRIVER_VERSION=v0.34.0 \
|
||||
FIREFOX_VERSION=125.0.3
|
||||
|
||||
RUN if [ "$INCLUDE_FIREFOX" = "true" ]; then \
|
||||
apt-get update -qq \
|
||||
&& apt-get install -yqq --no-install-recommends wget bzip2 \
|
||||
# Install GeckoDriver WebDriver and Firefox (if required)
|
||||
ARG GECKODRIVER_VERSION=v0.34.0
|
||||
ARG FIREFOX_VERSION=125.0.3
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
if [ "$INCLUDE_FIREFOX" = "true" ]; then \
|
||||
/docker/apt-install.sh wget bzip2 \
|
||||
&& wget -q https://github.com/mozilla/geckodriver/releases/download/${GECKODRIVER_VERSION}/geckodriver-${GECKODRIVER_VERSION}-linux64.tar.gz -O - | tar xfz - -C /usr/local/bin \
|
||||
&& wget -q https://download-installer.cdn.mozilla.net/pub/firefox/releases/${FIREFOX_VERSION}/linux-x86_64/en-US/firefox-${FIREFOX_VERSION}.tar.bz2 -O - | tar xfj - -C /opt \
|
||||
&& ln -s /opt/firefox/firefox /usr/local/bin/firefox \
|
||||
&& apt-get autoremove -yqq --purge wget bzip2 && rm -rf /var/[log,tmp]/* /tmp/* /var/lib/apt/lists/* /var/cache/apt/archives/*; \
|
||||
else \
|
||||
echo "Skipping Firefox installation in dev mode"; \
|
||||
fi
|
||||
|
||||
# Installing mysql client os-level dependencies in dev image only because GPL
|
||||
RUN apt-get install -yqq --no-install-recommends \
|
||||
default-libmysqlclient-dev \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
# Install MySQL client dependencies
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
/docker/apt-install.sh default-libmysqlclient-dev
|
||||
|
||||
# Copy development requirements and install them
|
||||
COPY --chown=superset:superset requirements/development.txt requirements/
|
||||
RUN --mount=type=cache,target=/root/.cache/pip \
|
||||
apt-get update -qq && apt-get install -yqq --no-install-recommends \
|
||||
build-essential \
|
||||
&& pip install --no-cache-dir -r requirements/development.txt \
|
||||
&& apt-get autoremove -yqq --purge build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
RUN --mount=type=bind,source=./docker,target=/docker \
|
||||
--mount=type=cache,target=/root/.cache/pip \
|
||||
/docker/pip-install.sh --requires-build-essential -r requirements/development.txt
|
||||
|
||||
USER superset
|
||||
|
||||
######################################################################
|
||||
# CI image...
|
||||
######################################################################
|
||||
|
|
|
|||
|
|
@ -0,0 +1,51 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# Ensure this script is run as root
|
||||
if [[ $EUID -ne 0 ]]; then
|
||||
echo "This script must be run as root" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check for required arguments
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Usage: $0 <package1> [<package2> ...]" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Colors for better logging (optional)
|
||||
GREEN='\033[0;32m'
|
||||
RED='\033[0;31m'
|
||||
RESET='\033[0m'
|
||||
|
||||
# Install packages with clean-up
|
||||
echo -e "${GREEN}Updating package lists...${RESET}"
|
||||
apt-get update -qq
|
||||
|
||||
echo -e "${GREEN}Installing packages: $@${RESET}"
|
||||
apt-get install -yqq --no-install-recommends "$@"
|
||||
|
||||
echo -e "${GREEN}Autoremoving unnecessary packages...${RESET}"
|
||||
apt-get autoremove -y
|
||||
|
||||
echo -e "${GREEN}Cleaning up package cache and metadata...${RESET}"
|
||||
apt-get clean
|
||||
rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/* /tmp/* /var/tmp/*
|
||||
|
||||
echo -e "${GREEN}Installation and cleanup complete.${RESET}"
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
#!/usr/bin/env bash
|
||||
#
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
#
|
||||
set -euo pipefail
|
||||
|
||||
# Default flag
|
||||
REQUIRES_BUILD_ESSENTIAL=false
|
||||
USE_CACHE=true
|
||||
|
||||
# Filter arguments
|
||||
ARGS=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--requires-build-essential)
|
||||
REQUIRES_BUILD_ESSENTIAL=true
|
||||
;;
|
||||
--no-cache)
|
||||
USE_CACHE=false
|
||||
;;
|
||||
*)
|
||||
ARGS+=("$arg")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
# Install build-essential if required
|
||||
if $REQUIRES_BUILD_ESSENTIAL; then
|
||||
echo "Installing build-essential for package builds..."
|
||||
apt-get update -qq \
|
||||
&& apt-get install -yqq --no-install-recommends build-essential
|
||||
fi
|
||||
|
||||
# Choose whether to use pip cache
|
||||
if $USE_CACHE; then
|
||||
echo "Using pip cache..."
|
||||
pip install "${ARGS[@]}"
|
||||
else
|
||||
echo "Disabling pip cache..."
|
||||
pip install --no-cache-dir "${ARGS[@]}"
|
||||
fi
|
||||
|
||||
# Remove build-essential if it was installed
|
||||
if $REQUIRES_BUILD_ESSENTIAL; then
|
||||
echo "Removing build-essential to keep the image lean..."
|
||||
apt-get autoremove -yqq --purge build-essential \
|
||||
&& apt-get clean \
|
||||
&& rm -rf /var/lib/apt/lists/* /var/cache/apt/archives/*
|
||||
fi
|
||||
|
||||
echo "Python packages installed successfully."
|
||||
|
|
@ -12,7 +12,7 @@
|
|||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
FROM node:16-alpine as build
|
||||
FROM node:16-alpine AS build
|
||||
|
||||
WORKDIR /home/superset-websocket
|
||||
|
||||
|
|
|
|||
Loading…
Reference in New Issue