diff --git a/RELEASING/Dockerfile.from_local_tarball b/RELEASING/Dockerfile.from_local_tarball new file mode 100644 index 000000000..6ca3a3bb8 --- /dev/null +++ b/RELEASING/Dockerfile.from_local_tarball @@ -0,0 +1,67 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM python:3.6-jessie + +RUN useradd --user-group --create-home --no-log-init --shell /bin/bash superset + +# Configure environment +ENV LANG=C.UTF-8 \ + LC_ALL=C.UTF-8 + +RUN apt-get update -y + +# Install dependencies to fix `curl https support error` and `elaying package configuration warning` +RUN apt-get install -y apt-transport-https apt-utils + +# Install superset dependencies +# https://superset.incubator.apache.org/installation.html#os-dependencies +RUN apt-get install -y build-essential libssl-dev \ + libffi-dev python3-dev libsasl2-dev libldap2-dev libxi-dev + +# Install nodejs for custom build +# https://superset.incubator.apache.org/installation.html#making-your-own-build +# https://nodejs.org/en/download/package-manager/ +RUN curl -sL https://deb.nodesource.com/setup_10.x | bash - \ + && apt-get install -y nodejs + +RUN mkdir -p /home/superset +RUN chown superset /home/superset + +WORKDIR /home/superset +ARG VERSION +ARG SUPERSET_RELEASE_RC_TARBALL + +# Can fetch source from svn or copy tarball from local mounted directory +COPY $SUPERSET_RELEASE_RC_TARBALL ./ +RUN tar -xvf *.tar.gz +WORKDIR /home/superset/apache-superset-incubating-$VERSION/superset/assets + +RUN npm ci \ + && npm run build \ + && rm -rf node_modules + +WORKDIR /home/superset/apache-superset-incubating-$VERSION +RUN pip install --upgrade setuptools pip \ + && pip install -r requirements.txt \ + && pip install --no-cache-dir . + +RUN flask fab babel-compile --target superset/translations + +ENV PATH=/home/superset/superset/bin:$PATH \ + PYTHONPATH=/home/superset/superset/:$PYTHONPATH +COPY from_tarball_entrypoint.sh /entrypoint.sh +ENTRYPOINT ["/entrypoint.sh"] diff --git a/RELEASING/Dockerfile.from_tarball b/RELEASING/Dockerfile.from_svn_tarball similarity index 95% rename from RELEASING/Dockerfile.from_tarball rename to RELEASING/Dockerfile.from_svn_tarball index c7a9c1ca7..f3acd7702 100644 --- a/RELEASING/Dockerfile.from_tarball +++ b/RELEASING/Dockerfile.from_svn_tarball @@ -44,6 +44,7 @@ RUN chown superset /home/superset WORKDIR /home/superset ARG VERSION +# Can fetch source from svn or copy tarball from local mounted directory RUN svn co https://dist.apache.org/repos/dist/dev/incubator/superset/$VERSION ./ RUN tar -xvf *.tar.gz WORKDIR apache-superset-incubating-$VERSION @@ -56,13 +57,11 @@ RUN cd superset/assets \ WORKDIR /home/superset/apache-superset-incubating-$VERSION RUN pip install --upgrade setuptools pip \ + && pip install -r requirements.txt \ && pip install --no-cache-dir . RUN flask fab babel-compile --target superset/translations -RUN pip install -e . \ - && rm -rf /root/.cache/pip - ENV PATH=/home/superset/superset/bin:$PATH \ PYTHONPATH=/home/superset/superset/:$PYTHONPATH COPY from_tarball_entrypoint.sh /entrypoint.sh diff --git a/RELEASING/Dockerfile.make_tarball b/RELEASING/Dockerfile.make_tarball index c7d516c55..d4c434776 100644 --- a/RELEASING/Dockerfile.make_tarball +++ b/RELEASING/Dockerfile.make_tarball @@ -15,5 +15,9 @@ # limitations under the License. # FROM python:3.6-jessie -COPY make_tarball.sh /entrypoint.sh + +RUN apt-get update -y +RUN apt-get install -y jq + +COPY make_tarball_entrypoint.sh /entrypoint.sh ENTRYPOINT ["/entrypoint.sh"] diff --git a/RELEASING/README.md b/RELEASING/README.md index ed8743143..21ef756dc 100644 --- a/RELEASING/README.md +++ b/RELEASING/README.md @@ -64,13 +64,21 @@ that belong to the MAJOR.MINOR version. The MAJOR.MINOR branch is normally a "cut" from a specific point in time from the master branch. Then (if needed) apply all cherries that will make the PATCH +Next update the `CHANGELOG.md` with all the changes that are included in the release. Make sure you have +set your GITHUB_TOKEN environment variable. + +```bash +# will overwrites the local CHANGELOG.md, somehow you need to merge it in +github-changes -o apache -r incubator-superset --token $GITHUB_TOKEN --between-tags ... +``` + Finally bump the version number on `superset/static/assets/package.json`: ```json "version": "0.34.1" ``` -Commit the change with the version number, then git tag the version with the release candidate and push +Commit the change with the version number, then git tag the version with the release candidate and push to the branch ## Setting up the release environment (do every time) @@ -79,74 +87,58 @@ often stretching over several weeks calendar time if votes don't pass, chances a the same terminal session won't be used for crafting the release candidate and the final release. Therefore, it's a good idea to do the following every time you work on a new phase of the release process to make sure you aren't releasing -the wrong files/using wrong names: +the wrong files/using wrong names. There's a script to help you set correctly all the +necessary environment variables. Change you current directory to `superset/RELEASING` ```bash - # Set SUPERSET_VERSION to the release being prepared, e.g. 0.34.1. - export SUPERSET_VERSION=XX.YY.ZZ - # Set RC to the release candindate number. Replacing QQ below with 1 - # indicates rc1 i.e. first vote on version above (0.34.1rc1) - export SUPERSET_RC=QQ + # usage: set_release_env.sh "" + . ./set_release_env.sh XX.YY.ZZ QQ "YOUR PGP KEY NAME" ``` -Then you can generate other derived environment variables that are used -throughout the release process: +The script will output the exported variables, for example for 0.34.1 RC1: -```bash - # Replace SUPERSET_PGP_FULLNAME with your PGP key name for Apache - export SUPERSET_PGP_FULLNAME="YOURFULLNAMEHERE" - export SUPERSET_VERSION_RC=${SUPERSET_VERSION}rc${SUPERSET_RC} - export SUPERSET_RELEASE=apache-superset-incubating-${SUPERSET_VERSION} - export SUPERSET_RELEASE_RC=apache-superset-incubating-${SUPERSET_VERSION_RC} - export SUPERSET_RELEASE_TARBALL=${SUPERSET_RELEASE}-source.tar.gz - export SUPERSET_RELEASE_RC_TARBALL=${SUPERSET_RELEASE_RC}-source.tar.gz +``` + ------------------------------- + Set Release env variables + SUPERSET_VERSION=0.34.1 + SUPERSET_RC=1 + SUPERSET_PGP_FULLNAME=You PGP Key Name + SUPERSET_VERSION_RC=0.34.1rc1 + SUPERSET_RELEASE=apache-superset-incubating-0.34.1 + SUPERSET_RELEASE_RC=apache-superset-incubating-0.34.1rc1 + SUPERSET_RELEASE_TARBALL=apache-superset-incubating-0.34.1-source.tar.gz + SUPERSET_RELEASE_RC_TARBALL=apache-superset-incubating-0.34.1rc1-source.tar.gz + ------------------------------- ``` ## Preparing the release candidate The first step of preparing an Apache Release is packaging a release candidate -to be voted on. Start by going to the root of the repo and making sure the -prerequisites are in order: +to be voted on. Make sure you have correctly prepared and tagged the ready to ship +release on Superset's repo (MAJOR.MINOR branch), the following script will clone +the tag and create a signed source tarball from it: ```bash - # Go to the root directory of the repo, e.g. `~/src/incubator-superset` - cd ~/src/incubator-superset/ - export SUPERSET_REPO_DIR=$(pwd) - # make sure you're on the correct branch (e.g. 0.34) - git branch + # make_tarball you use the previouly set environment variables + # you can override by passing arguments: make_tarball.sh "" + ./make_tarball.sh ``` -Make sure the version number under `superset/assets/package.json` corresponds -to `SUPERSET_VERSION` above (`0.34.1` in example above), and has been committed to the -branch. +Note that `make_tarball.sh`: +- By default assumes you have already executed an SVN checkout to `$HOME/svn/superset_dev`. +This can be overriden by setting `SUPERSET_SVN_DEV_PATH` environment var to a different svn dev directory +- Will refuse to craft a new release candidate if a release already exists on your local svn dev directory +- Will check `package.json` version number and fails if it's not correctly set + +### Build and test the created source tarball + +To build and run the just created tarball ```bash - grep ${SUPERSET_VERSION} superset/assets/package.json -``` - -If nothing shows up, either the version isn't correctly set in `package.json`, -or the environment variable is misconfigured. - -### Crafting tarball and signatures - -Now let's craft a source release -```bash - # Let's create a git tag - git tag -f ${SUPERSET_VERSION_RC} - - # Create the target folder - mkdir -p ~/svn/superset_dev/${SUPERSET_VERSION_RC}/ - git archive \ - --format=tar.gz ${SUPERSET_VERSION_RC} \ - --prefix="${SUPERSET_RELEASE_RC}/" \ - -o ~/svn/superset_dev/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL} - - cd ~/svn/superset_dev/${SUPERSET_VERSION_RC}/ - ${SUPERSET_REPO_DIR}/scripts/sign.sh "${SUPERSET_RELEASE_RC_TARBALL}" "${SUPERSET_PGP_FULLNAME}" - - # To verify to signature - gpg --verify "${SUPERSET_RELEASE_RC_TARBALL}".asc "${SUPERSET_RELEASE_RC_TARBALL}" - + # Build and run a release candidate tarball + ./test_run_tarball.sh local + # you should be able to access localhost:5001 on your browser + # login using admin/admin ``` ### Shipping to SVN @@ -159,14 +151,14 @@ Now let's ship this RC into svn's dev folder svn commit -m "Release ${SUPERSET_VERSION_RC}" ``` -### Build and test from source tarball +### Build and test from SVN source tarball To make a working build given a tarball ```bash -# Build and run a release candidate tarball -./test_run_tarball.sh -# you should be able to access localhost:5001 on your browser -# login using admin/admin + # Build and run a release candidate tarball + ./test_run_tarball.sh + # you should be able to access localhost:5001 on your browser + # login using admin/admin ``` ### Voting diff --git a/RELEASING/make_tarball.sh b/RELEASING/make_tarball.sh index 5959d0fea..3d0c180f1 100755 --- a/RELEASING/make_tarball.sh +++ b/RELEASING/make_tarball.sh @@ -15,24 +15,59 @@ # See the License for the specific language governing permissions and # limitations under the License. # -set -ex +set -e -if [ -z "$VERSION" ]; then - echo "VERSION is required to run this container" +usage() { + echo "usage: make_tarball.sh " +} + +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + if [ -z "${SUPERSET_VERSION}" ] || [ -z "${SUPERSET_RC}" ] || [ -z "${SUPERSET_PGP_FULLNAME}" ]; then + echo "No parameters found an no required environment variables set" + echo "usage: make_tarball.sh " + usage; + exit 1 + fi +else + SUPERSET_VERSION="${1}" + SUPERSET_RC="${2}" + SUPERSET_PGP_FULLNAME="${3}" +fi + +SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}" + +if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then + SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev" +fi + +if [[ ! -d "${SUPERSET_SVN_DEV_PATH}" ]]; then + echo "${SUPERSET_SVN_DEV_PATH} does not exist, you need to: svn checkout" exit 1 fi +if [ -d "${SUPERSET_SVN_DEV_PATH}/${SUPERSET_VERSION_RC}" ]; then + echo "${SUPERSET_VERSION_RC} Already exists on svn, refusing to overwrite" + exit 1 +fi -echo "version: $VERSION" -cd /tmp -git clone --depth 1 --branch $VERSION https://github.com/apache/incubator-superset.git -mkdir ~/$VERSION -cd incubator-superset && \ -git archive \ - --format=tar.gz \ - --prefix=apache-superset-incubating-$VERSION/ \ - HEAD \ - -o ~/$VERSION/apache-superset-incubating.tar.gz +SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}"/"${SUPERSET_RELEASE_RC_TARBALL}" +DOCKER_SVN_PATH="/docker_svn" -gpg --armor --output apache-superset-incubating.tar.gz.asc --detach-sig apache-superset-incubating.tar.gz -gpg --print-md SHA512 apache-superset-incubating.tar.gz > apache-superset-incubating.tar.gz.sha512 +# Building docker that will produce a tarball +docker build -t apache-builder -f Dockerfile.make_tarball . + +# Running docker to produce a tarball +docker run \ + -e SUPERSET_SVN_DEV_PATH="${DOCKER_SVN_PATH}" \ + -e SUPERSET_VERSION="${SUPERSET_VERSION}" \ + -e SUPERSET_VERSION_RC="${SUPERSET_VERSION_RC}" \ + -e HOST_UID=${UID} \ + -v "${SUPERSET_SVN_DEV_PATH}":"${DOCKER_SVN_PATH}":rw \ + -ti apache-builder + +gpg --armor --local-user "${SUPERSET_PGP_FULLNAME}" --output "${SUPERSET_RELEASE_RC_TARBALL_PATH}".asc --detach-sig "${SUPERSET_RELEASE_RC_TARBALL_PATH}" +gpg --print-md --local-user "${SUPERSET_PGP_FULLNAME}" SHA512 "${SUPERSET_RELEASE_RC_TARBALL_PATH}" > "${SUPERSET_RELEASE_RC_TARBALL_PATH}".sha512 + +echo --------------------------------------- +echo Release candidate tarball is ready +echo --------------------------------------- diff --git a/RELEASING/make_tarball_entrypoint.sh b/RELEASING/make_tarball_entrypoint.sh new file mode 100755 index 000000000..dde011837 --- /dev/null +++ b/RELEASING/make_tarball_entrypoint.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +if [ -z "${SUPERSET_VERSION_RC}" ] || [ -z "${SUPERSET_SVN_DEV_PATH}" ] || [ -z "${SUPERSET_VERSION}" ]; then + echo "SUPERSET_VERSION_RC, SUPERSET_SVN_DEV_PATH and SUPERSET_VERSION are required to run this container" + exit 1 +fi + +SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}" +SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz +SUPERSET_RELEASE_RC_BASE_PATH="${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}" +SUPERSET_RELEASE_RC_TARBALL_PATH="${SUPERSET_RELEASE_RC_BASE_PATH}"/"${SUPERSET_RELEASE_RC_TARBALL}" + +# Create directory release version +mkdir -p "${SUPERSET_SVN_DEV_PATH}"/"${SUPERSET_VERSION_RC}" + +# Clone superset from tag to /tmp +cd /tmp +git clone --depth 1 --branch ${SUPERSET_VERSION_RC} https://github.com/apache/incubator-superset.git +mkdir -p "${HOME}/${SUPERSET_VERSION_RC}" +cd incubator-superset && \ + +# Check RC version +if ! jq -e --arg SUPERSET_VERSION $SUPERSET_VERSION '.version == $SUPERSET_VERSION' superset/assets/package.json +then + SOURCE_VERSION=$(jq '.version' superset/assets/package.json) + echo "Source package.json version is wrong, found: ${SOURCE_VERSION} should be: ${SUPERSET_VERSION}" + exit 1 +fi + +# Create source tarball +git archive \ + --format=tar.gz "${SUPERSET_VERSION_RC}" \ + --prefix="${SUPERSET_RELEASE_RC}/" \ + -o "${SUPERSET_RELEASE_RC_TARBALL_PATH}" + +chown -R ${HOST_UID}:${HOST_UID} "${SUPERSET_RELEASE_RC_BASE_PATH}" diff --git a/RELEASING/set_release_env.sh b/RELEASING/set_release_env.sh new file mode 100755 index 000000000..4e4423bfa --- /dev/null +++ b/RELEASING/set_release_env.sh @@ -0,0 +1,38 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +usage() { + echo "usage: . set_release_env.sh " +} + +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ]; then + usage; +else + export SUPERSET_VERSION="${1}" + export SUPERSET_RC="${2}" + export SUPERSET_PGP_FULLNAME="${3}" + export SUPERSET_VERSION_RC="${SUPERSET_VERSION}rc${SUPERSET_RC}" + export SUPERSET_RELEASE=apache-superset-incubating-"${SUPERSET_VERSION}" + export SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}" + export SUPERSET_RELEASE_TARBALL="${SUPERSET_RELEASE}"-source.tar.gz + export SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz + + echo ------------------------------- + echo Set Release env variables + env | grep SUPERSET + echo ------------------------------- +fi diff --git a/RELEASING/test_run_tarball.sh b/RELEASING/test_run_tarball.sh index 6521ac09e..74ef747ba 100755 --- a/RELEASING/test_run_tarball.sh +++ b/RELEASING/test_run_tarball.sh @@ -22,8 +22,31 @@ if [ -z "${SUPERSET_VERSION_RC}" ]; then exit 1 fi -# Building a docker from a tarball -docker build --no-cache -t apache-superset:${SUPERSET_VERSION_RC} -f Dockerfile.from_tarball . --build-arg VERSION=${SUPERSET_VERSION_RC} +if [ -z "${SUPERSET_SVN_DEV_PATH}" ]; then + SUPERSET_SVN_DEV_PATH="$HOME/svn/superset_dev" +fi + +if [ ${1} == "local" ]; then + SUPERSET_RELEASE_RC=apache-superset-incubating-"${SUPERSET_VERSION_RC}" + SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_RELEASE_RC}"-source.tar.gz + SUPERSET_TARBALL_PATH="${SUPERSET_SVN_DEV_PATH}"/${SUPERSET_VERSION_RC}/${SUPERSET_RELEASE_RC_TARBALL} + SUPERSET_TMP_TARBALL_FILENAME=_tmp_"${SUPERSET_VERSION_RC}".tar.gz + cp "${SUPERSET_TARBALL_PATH}" "${SUPERSET_TMP_TARBALL_FILENAME}" + docker build --no-cache \ + -t apache-superset:${SUPERSET_VERSION_RC} \ + -f Dockerfile.from_local_tarball . \ + --build-arg VERSION=${SUPERSET_VERSION_RC} \ + --build-arg SUPERSET_BUILD_FROM=local \ + --build-arg SUPERSET_RELEASE_RC_TARBALL="${SUPERSET_TMP_TARBALL_FILENAME}" + rm "${SUPERSET_TMP_TARBALL_FILENAME}" +else + # Building a docker from a tarball + docker build --no-cache \ + -t apache-superset:${SUPERSET_VERSION_RC} \ + -f Dockerfile.from_svn_tarball . \ + --build-arg VERSION=${SUPERSET_VERSION_RC} \ + --build-arg SUPERSET_BUILD_FROM=svn +fi echo "---------------------------------------------------" echo "After docker build and run, you should be able to access localhost:5001 on your browser"