diff --git a/RELEASING/Dockerfile.from_tarball b/RELEASING/Dockerfile.from_tarball
index f42cee392..c7a9c1ca7 100644
--- a/RELEASING/Dockerfile.from_tarball
+++ b/RELEASING/Dockerfile.from_tarball
@@ -56,9 +56,7 @@ RUN cd superset/assets \
 
 WORKDIR /home/superset/apache-superset-incubating-$VERSION
 RUN pip install --upgrade setuptools pip \
-    && pip install -r requirements.txt -r requirements-dev.txt \
-    && pip install -e . \
-    && rm -rf /root/.cache/pip
+    && pip install --no-cache-dir .
 
 RUN flask fab babel-compile --target superset/translations
 
diff --git a/RELEASING/README.md b/RELEASING/README.md
index a3248b925..ed8743143 100644
--- a/RELEASING/README.md
+++ b/RELEASING/README.md
@@ -57,16 +57,18 @@ need to be done at every release.
 ## Crafting a source release
 
 When crafting a new minor or major release we create 
-a branch named with the release MAJOR.MINOR version.
+a branch named with the release MAJOR.MINOR version (on this example 0.34).
 This new branch will hold all PATCH and release candidates 
 that belong to the MAJOR.MINOR version.
 
 The MAJOR.MINOR branch is normally a "cut" from a specific point in time from the master branch.
 Then (if needed) apply all cherries that will make the PATCH
 
-Finally bump the version number on `superset/static/assets/package.json` ::
+Finally bump the version number on `superset/static/assets/package.json`:
 
-    "version": "0.35.0"
+```json
+    "version": "0.34.1"
+```
 
 Commit the change with the version number, then git tag the version with the release candidate and push
 
@@ -157,6 +159,16 @@ Now let's ship this RC into svn's dev folder
     svn commit -m "Release ${SUPERSET_VERSION_RC}"
 ```
 
+### Build and test from source tarball
+
+To make a working build given a tarball
+```bash
+# Build and run a release candidate tarball
+./test_run_tarball.sh
+# you should be able to access localhost:5001 on your browser
+# login using admin/admin
+```
+
 ### Voting
 Now you're ready to start the [VOTE] thread. Here's an example of a
 previous release vote thread:
@@ -218,19 +230,6 @@ deprecations and upgrading-related topics,
 make sure to move the content now under the `Next Version` section under a new
 section for the new release.
 
-## Build from source tarball
-
-To make a working build given a tarball
-```bash
-# Building a docker from a tarball
-docker build -t apache-superset:${SUPERSET_VERSION_RC} -f Dockerfile.from_tarball . --build-arg VERSION=${SUPERSET_VERSION_RC}
-
-# testing the resulting docker
-docker run -p 5001:8088 apache-superset:${SUPERSET_VERSION_RC}
-# you should be able to access localhost:5001 on your browser
-# login using admin/admin
-```
-
 # Refresh documentation website
 
 Every once in a while we want to compile the documentation and publish it.
diff --git a/RELEASING/test_run_tarball.sh b/RELEASING/test_run_tarball.sh
new file mode 100755
index 000000000..6521ac09e
--- /dev/null
+++ b/RELEASING/test_run_tarball.sh
@@ -0,0 +1,36 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+set -e
+
+if [ -z "${SUPERSET_VERSION_RC}" ]; then
+  echo "SUPERSET_VERSION_RC is required to run this container"
+  exit 1
+fi
+
+# Building a docker from a tarball
+docker build --no-cache -t apache-superset:${SUPERSET_VERSION_RC} -f Dockerfile.from_tarball . --build-arg VERSION=${SUPERSET_VERSION_RC}
+
+echo "---------------------------------------------------"
+echo "After docker build and run, you should be able to access localhost:5001 on your browser"
+echo "login using admin/admin"
+echo "---------------------------------------------------"
+if ! docker run -p 5001:8088 apache-superset:${SUPERSET_VERSION_RC}; then
+  echo "---------------------------------------------------"
+  echo "[ERROR] Seems like this apache-superset:${SUPERSET_VERSION_RC} has a setup/startup problem!"
+  echo "---------------------------------------------------"
+fi