From 4427d65717563aa0b62f42bfc713970315bfcbfc Mon Sep 17 00:00:00 2001 From: Daniel Vaz Gaspar Date: Thu, 14 May 2020 16:58:30 +0100 Subject: [PATCH] tests(celery): improve celery tests infra (#9775) --- .github/workflows/superset-python.yml | 21 ++++++----- docker-compose.yml | 18 +++++++++- docker/pythonpath_dev/superset_config.py | 8 +++-- scripts/tests/README.md | 10 ++++++ scripts/tests/run.sh | 30 ++++++++++++++-- tests/celery_tests.py | 29 +++------------ tests/superset_test_config.py | 15 +++++--- tests/superset_test_config_thumbnails.py | 13 +++++-- tests/thumbnails_tests.py | 46 ++---------------------- tox.ini | 2 +- 10 files changed, 100 insertions(+), 92 deletions(-) diff --git a/.github/workflows/superset-python.yml b/.github/workflows/superset-python.yml index 5d681feae..ac4b20d2c 100644 --- a/.github/workflows/superset-python.yml +++ b/.github/workflows/superset-python.yml @@ -61,6 +61,8 @@ jobs: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.superset_test_config REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: + postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset services: postgres: image: postgres:10-alpine @@ -87,10 +89,9 @@ jobs: run: | pip-install setup-postgres + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (PostgreSQL) - env: - SUPERSET__SQLALCHEMY_DATABASE_URI: - postgresql+psycopg2://superset:superset@127.0.0.1:15432/superset run: | ./scripts/python_tests.sh - name: Upload code coverage @@ -106,6 +107,8 @@ jobs: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.superset_test_config REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true services: mysql: image: mysql:5.7 @@ -130,10 +133,9 @@ jobs: run: | pip-install setup-mysql + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (MySQL) - env: - SUPERSET__SQLALCHEMY_DATABASE_URI: | - mysql+mysqldb://superset:superset@127.0.0.1:13306/superset?charset=utf8mb4&binary_prefix=true run: | ./scripts/python_tests.sh - name: Upload code coverage @@ -149,6 +151,8 @@ jobs: PYTHONPATH: ${{ github.workspace }} SUPERSET_CONFIG: tests.superset_test_config REDIS_PORT: 16379 + SUPERSET__SQLALCHEMY_DATABASE_URI: | + sqlite:///${{ github.workspace }}/.temp/unittest.db services: redis: image: redis:5-alpine @@ -166,10 +170,9 @@ jobs: run: | pip-install mkdir ${{ github.workspace }}/.temp + - name: Run celery + run: celery worker --app=superset.tasks.celery_app:app -Ofair -c 2 & - name: Python unit tests (SQLite) - env: - SUPERSET__SQLALCHEMY_DATABASE_URI: | - sqlite:///${{ github.workspace }}/.temp/unittest.db run: | ./scripts/python_tests.sh - name: Upload code coverage diff --git a/docker-compose.yml b/docker-compose.yml index cb45e9c2b..350bb5782 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -83,12 +83,28 @@ services: superset-worker: build: *superset-build container_name: superset_worker - command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair"] + command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair", "-l", "INFO"] env_file: docker/.env restart: unless-stopped depends_on: *superset-depends-on volumes: *superset-volumes + superset-tests-worker: + build: *superset-build + container_name: superset_tests_worker + command: ["celery", "worker", "--app=superset.tasks.celery_app:app", "-Ofair", "-l", "INFO"] + env_file: docker/.env + environment: + DATABASE_HOST: localhost + DATABASE_DB: test + REDIS_CELERY_DB: 2 + REDIS_RESULTS_DB: 3 + REDIS_HOST: localhost + network_mode: host + restart: unless-stopped + depends_on: *superset-depends-on + volumes: *superset-volumes + volumes: superset_home: external: false diff --git a/docker/pythonpath_dev/superset_config.py b/docker/pythonpath_dev/superset_config.py index a2521df77..7270de3b9 100644 --- a/docker/pythonpath_dev/superset_config.py +++ b/docker/pythonpath_dev/superset_config.py @@ -63,19 +63,23 @@ SQLALCHEMY_DATABASE_URI = "%s://%s:%s@%s:%s/%s" % ( REDIS_HOST = get_env_variable("REDIS_HOST") REDIS_PORT = get_env_variable("REDIS_PORT") +REDIS_CELERY_DB = get_env_variable("REDIS_CELERY_DB", 0) +REDIS_RESULTS_DB = get_env_variable("REDIS_CELERY_DB", 1) + RESULTS_BACKEND = FileSystemCache("/app/superset_home/sqllab") class CeleryConfig(object): - BROKER_URL = "redis://%s:%s/0" % (REDIS_HOST, REDIS_PORT) + BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" CELERY_IMPORTS = ("superset.sql_lab",) - CELERY_RESULT_BACKEND = "redis://%s:%s/1" % (REDIS_HOST, REDIS_PORT) + CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" CELERY_ANNOTATIONS = {"tasks.add": {"rate_limit": "10/s"}} CELERY_TASK_PROTOCOL = 1 CELERY_CONFIG = CeleryConfig +SQLLAB_CTAS_NO_LIMIT = True # # Optionally import superset_config_docker.py (which will have been included on diff --git a/scripts/tests/README.md b/scripts/tests/README.md index 814e54235..430a155e0 100644 --- a/scripts/tests/README.md +++ b/scripts/tests/README.md @@ -49,3 +49,13 @@ scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-i ```$bash scripts/tests/run.sh tests.charts.api_tests:ChartApiTests.test_get_charts --no-reset-db ``` + +- Example for not running tests just initialize the test DB (drop/create, upgrade and load examples) +```$bash +scripts/tests/run.sh . --no-tests +``` + +- Example for just resetting the tests DB +```$bash +scripts/tests/run.sh . --reset-db +``` diff --git a/scripts/tests/run.sh b/scripts/tests/run.sh index a4d5117d0..5e9566352 100755 --- a/scripts/tests/run.sh +++ b/scripts/tests/run.sh @@ -26,8 +26,19 @@ function reset_db() { echo -------------------- echo Reseting test DB echo -------------------- - docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'DROP DATABASE IF EXISTS ${DB_NAME};'" - docker exec -i superset_db bash -c "/usr/bin/psql -h 127.0.0.1 -U ${DB_USER} -w -c 'CREATE DATABASE ${DB_NAME};'" + docker-compose stop superset-tests-worker + RESET_DB_CMD="psql \"postgresql://superset:superset@127.0.0.1:5432\" <<-EOF + DROP DATABASE IF EXISTS ${DB_NAME}; + CREATE DATABASE ${DB_NAME}; + \\c ${DB_NAME} + DROP SCHEMA IF EXISTS sqllab_test_db; + CREATE SCHEMA sqllab_test_db; + DROP SCHEMA IF EXISTS admin_database; + CREATE SCHEMA admin_database; +EOF +" + docker exec -i superset_db bash -c "${RESET_DB_CMD}" + docker-compose start superset-tests-worker } # @@ -71,6 +82,7 @@ export SUPERSET__SQLALCHEMY_DATABASE_URI=${SUPERSET__SQLALCHEMY_DATABASE_URI:-po export SUPERSET_CONFIG=${SUPERSET_CONFIG:-tests.superset_test_config} RUN_INIT=1 RUN_RESET_DB=1 +RUN_TESTS=1 TEST_MODULE="${1}" # Shift to pass the first cmd parameter for the test module @@ -88,6 +100,15 @@ while (( "$#" )); do RUN_RESET_DB=0 shift 1 ;; + --no-tests) + RUN_TESTS=0 + shift 1 + ;; + --reset-db) + RUN_TESTS=0 + RUN_INIT=0 + shift 1 + ;; --) # end argument parsing shift break @@ -122,4 +143,7 @@ then test_init fi -nosetests --exclude=load_examples_test "${TEST_MODULE}" +if [ $RUN_TESTS -eq 1 ] +then + nosetests --exclude=load_examples_test "${TEST_MODULE}" +fi diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 07950c11e..f31dcf78e 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -45,7 +45,8 @@ from superset.utils.core import get_example_database from .base_tests import SupersetTestCase CELERY_SHORT_SLEEP_TIME = 2 -CELERY_SLEEP_TIME = 5 +CELERY_SLEEP_TIME = 10 +DROP_TABLE_SLEEP_TIME = 10 class UtilityFunctionTests(SupersetTestCase): @@ -115,32 +116,9 @@ class CeleryTestCase(SupersetTestCase): @classmethod def setUpClass(cls): with app.app_context(): - - class CeleryConfig(object): - BROKER_URL = app.config["CELERY_CONFIG"].BROKER_URL - CELERY_IMPORTS = ("superset.sql_lab",) - CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}} - CONCURRENCY = 1 - - app.config["CELERY_CONFIG"] = CeleryConfig - db.session.query(Query).delete() db.session.commit() - base_dir = app.config["BASE_DIR"] - worker_command = base_dir + "/bin/superset worker -w 2" - subprocess.Popen(worker_command, shell=True, stdout=subprocess.PIPE) - - @classmethod - def tearDownClass(cls): - subprocess.call( - "ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9", shell=True - ) - subprocess.call( - "ps auxww | grep 'superset worker' | awk '{print $2}' | xargs kill -9", - shell=True, - ) - def run_sql( self, db_id, sql, client_id=None, cta=False, tmp_table="tmp", async_=False ): @@ -286,7 +264,7 @@ class CeleryTestCase(SupersetTestCase): table_name = "tmp_async_4" self.drop_table_if_exists(table_name, main_db) - time.sleep(CELERY_SLEEP_TIME) + time.sleep(DROP_TABLE_SLEEP_TIME) sql_where = "SELECT name FROM birth_names WHERE name='James' LIMIT 10" result = self.run_sql( @@ -305,6 +283,7 @@ class CeleryTestCase(SupersetTestCase): self.assertEqual(QueryStatus.SUCCESS, query.status) self.assertTrue(f"FROM {table_name}" in query.select_sql) + self.assertEqual( f"CREATE TABLE {table_name} AS \n" "SELECT name FROM birth_names " diff --git a/tests/superset_test_config.py b/tests/superset_test_config.py index 3e9e21f41..a3def4a98 100644 --- a/tests/superset_test_config.py +++ b/tests/superset_test_config.py @@ -32,8 +32,8 @@ if "SUPERSET__SQLALCHEMY_DATABASE_URI" in os.environ: if "sqlite" in SQLALCHEMY_DATABASE_URI: logger.warning( - "SQLite Database support for metadata databases will be \ - removed in a future version of Superset." + "SQLite Database support for metadata databases will be " + "removed in a future version of Superset." ) SQL_MAX_ROW = 666 @@ -57,11 +57,16 @@ ENABLE_ROW_LEVEL_SECURITY = True CACHE_CONFIG = {"CACHE_TYPE": "simple"} +REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") +REDIS_PORT = os.environ.get("REDIS_PORT", "6379") +REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2) +REDIS_RESULTS_DB = os.environ.get("REDIS_RESULTS_DB", 3) + + class CeleryConfig(object): - BROKER_URL = "redis://{}:{}".format( - os.environ.get("REDIS_HOST", "localhost"), os.environ.get("REDIS_PORT", "6379") - ) + BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" CELERY_IMPORTS = ("superset.sql_lab",) + CELERY_RESULT_BACKEND = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_RESULTS_DB}" CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}} CONCURRENCY = 1 diff --git a/tests/superset_test_config_thumbnails.py b/tests/superset_test_config_thumbnails.py index 86f1de9bd..bfcb3a31f 100644 --- a/tests/superset_test_config_thumbnails.py +++ b/tests/superset_test_config_thumbnails.py @@ -56,9 +56,14 @@ EMAIL_NOTIFICATIONS = False CACHE_CONFIG = {"CACHE_TYPE": "simple"} +REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") +REDIS_PORT = os.environ.get("REDIS_PORT", "6379") +REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2) +REDIS_RESULTS_DB = os.environ.get("REDIS_RESULTS_DB", 3) + class CeleryConfig(object): - BROKER_URL = "redis://localhost" + BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks.thumbnails") CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}} CONCURRENCY = 1 @@ -77,7 +82,11 @@ FEATURE_FLAGS = { def init_thumbnail_cache(app: Flask) -> RedisCache: return RedisCache( - host="localhost", key_prefix="superset_thumbnails_", default_timeout=10000 + host=REDIS_HOST, + port=REDIS_PORT, + db=REDIS_CELERY_DB, + key_prefix="superset_thumbnails_", + default_timeout=10000, ) diff --git a/tests/thumbnails_tests.py b/tests/thumbnails_tests.py index d2b4c5bae..a7803865a 100644 --- a/tests/thumbnails_tests.py +++ b/tests/thumbnails_tests.py @@ -38,49 +38,7 @@ from tests.test_app import app from .base_tests import SupersetTestCase -class CeleryStartMixin: - @classmethod - def setUpClass(cls): - with app.app_context(): - from cachelib.redis import RedisCache - - class CeleryConfig(object): - BROKER_URL = "redis://localhost" - CELERY_IMPORTS = ("superset.tasks.thumbnails",) - CONCURRENCY = 1 - - app.config["CELERY_CONFIG"] = CeleryConfig - - def init_thumbnail_cache(app) -> RedisCache: - return RedisCache( - host="localhost", - key_prefix="superset_thumbnails_", - default_timeout=10000, - ) - - app.config["THUMBNAIL_CACHE_CONFIG"] = init_thumbnail_cache - - base_dir = app.config["BASE_DIR"] - worker_command = base_dir + "/bin/superset worker -w 2" - subprocess.Popen( - worker_command, - shell=True, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - ) - - @classmethod - def tearDownClass(cls): - subprocess.call( - "ps auxww | grep 'celeryd' | awk '{print $2}' | xargs kill -9", shell=True - ) - subprocess.call( - "ps auxww | grep 'superset worker' | awk '{print $2}' | xargs kill -9", - shell=True, - ) - - -class ThumbnailsSeleniumLive(CeleryStartMixin, LiveServerTestCase): +class ThumbnailsSeleniumLive(LiveServerTestCase): def create_app(self): return app @@ -108,7 +66,7 @@ class ThumbnailsSeleniumLive(CeleryStartMixin, LiveServerTestCase): self.assertEqual(response.getcode(), 202) -class ThumbnailsTests(CeleryStartMixin, SupersetTestCase): +class ThumbnailsTests(SupersetTestCase): mock_image = b"bytes mock image" diff --git a/tox.ini b/tox.ini index cfcaea225..ccc39eb17 100644 --- a/tox.ini +++ b/tox.ini @@ -28,7 +28,7 @@ setenv = SUPERSET_CONFIG = tests.superset_test_config SUPERSET_HOME = {envtmpdir} py36-mysql: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - py36-postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://postgresuser:pguserpassword@localhost/superset + py36-postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test py36-sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db whitelist_externals = npm