From 0071d374daf37caf53f67c287e8774ab984c611b Mon Sep 17 00:00:00 2001 From: Bogdan Date: Mon, 10 Aug 2020 13:20:19 -0700 Subject: [PATCH] chore: move load examples to the fixture (#10545) * Move load examples to the fixture * Update tests/celery_tests.py Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com> * Address feedback Co-authored-by: bogdan kyryliuk Co-authored-by: Ville Brofeldt <33317356+villebro@users.noreply.github.com> --- scripts/python_tests.sh | 5 +- tests/celery_tests.py | 51 +++++++++++--------- tests/charts/api_tests.py | 9 ++-- tests/{load_examples_test.py => conftest.py} | 47 +++++++++--------- tests/core_tests.py | 4 +- tests/datasets/api_tests.py | 26 +++++++--- tests/schedules_test.py | 2 +- tox.ini | 3 +- 8 files changed, 81 insertions(+), 66 deletions(-) rename tests/{load_examples_test.py => conftest.py} (57%) diff --git a/scripts/python_tests.sh b/scripts/python_tests.sh index f0309f2a9..c54166edb 100755 --- a/scripts/python_tests.sh +++ b/scripts/python_tests.sh @@ -23,5 +23,6 @@ echo "Superset config module: $SUPERSET_CONFIG" superset db upgrade superset init -pytest --maxfail=1 tests/load_examples_test.py -pytest --maxfail=1 --cov=superset --ignore=load_examples_test tests/* + +echo "Running tests" +pytest --maxfail=1 --cov=superset diff --git a/tests/celery_tests.py b/tests/celery_tests.py index 53190cb7d..c3821311b 100644 --- a/tests/celery_tests.py +++ b/tests/celery_tests.py @@ -18,6 +18,7 @@ """Unit tests for Superset Celery worker""" import datetime import json +from typing import Optional from parameterized import parameterized import time @@ -26,13 +27,13 @@ import unittest.mock as mock import flask from flask import current_app -from sqlalchemy.engine import Engine from tests.test_app import app from superset import db, sql_lab from superset.result_set import SupersetResultSet from superset.db_engine_specs.base import BaseEngineSpec from superset.extensions import celery_app +from superset.models.core import Database from superset.models.helpers import QueryStatus from superset.models.sql_lab import Query from superset.sql_parse import ParsedQuery, CtasMethod @@ -219,14 +220,12 @@ class TestCelery(SupersetTestCase): query3 = self.get_query_by_id(result3["query"]["serverId"]) self.assertEqual(QueryStatus.SUCCESS, query3.status) - def drop_table_if_exists(self, table_name, table_type: CtasMethod, database=None): + def drop_table_if_exists( + self, table_name: str, table_type: CtasMethod, database: Database, + ) -> None: """Drop table if it exists, works on any DB""" - sql = f"DROP {table_type} {table_name}" - db_id = database.id - if database: - database.allow_dml = True - db.session.flush() - return self.run_sql(db_id, sql) + sql = f"DROP {table_type} IF EXISTS {table_name}" + database.get_sqla_engine().execute(sql) @parameterized.expand([CtasMethod.TABLE, CtasMethod.VIEW]) def test_run_sync_query_cta_config(self, ctas_method): @@ -234,18 +233,20 @@ class TestCelery(SupersetTestCase): "superset.views.core.get_cta_schema_name", lambda d, u, s, sql: CTAS_SCHEMA_NAME, ): - main_db = get_example_database() - db_id = main_db.id - backend = main_db.backend + examples_db = get_example_database() + db_id = examples_db.id + backend = examples_db.backend if backend == "sqlite": # sqlite doesn't support schemas return tmp_table_name = f"tmp_async_22_{ctas_method.lower()}" quote = ( - main_db.inspector.engine.dialect.identifier_preparer.quote_identifier + examples_db.inspector.engine.dialect.identifier_preparer.quote_identifier ) expected_full_table_name = f"{CTAS_SCHEMA_NAME}.{quote(tmp_table_name)}" - self.drop_table_if_exists(expected_full_table_name, ctas_method, main_db) + self.drop_table_if_exists( + expected_full_table_name, ctas_method, examples_db + ) name = "James" sql_where = f"SELECT name FROM birth_names WHERE name='{name}'" result = self.run_sql( @@ -309,24 +310,24 @@ class TestCelery(SupersetTestCase): "superset.views.core.get_cta_schema_name", lambda d, u, s, sql: CTAS_SCHEMA_NAME, ): - main_db = get_example_database() - db_id = main_db.id - if main_db.backend == "sqlite": + example_db = get_example_database() + db_id = example_db.id + if example_db.backend == "sqlite": # sqlite doesn't support schemas return tmp_table_name = f"sqllab_test_table_async_1_{ctas_method}" quote = ( - main_db.inspector.engine.dialect.identifier_preparer.quote_identifier + example_db.inspector.engine.dialect.identifier_preparer.quote_identifier ) schema_name = ( quote(CTAS_SCHEMA_NAME) - if main_db.backend == "presto" + if example_db.backend == "presto" else CTAS_SCHEMA_NAME ) expected_full_table_name = f"{schema_name}.{quote(tmp_table_name)}" - self.drop_table_if_exists(expected_full_table_name, ctas_method, main_db) + self.drop_table_if_exists(expected_full_table_name, ctas_method, example_db) sql_where = "SELECT name FROM birth_names WHERE name='James' LIMIT 10" result = self.run_sql( db_id, @@ -351,7 +352,7 @@ class TestCelery(SupersetTestCase): query.executed_sql, ) self.drop_table_if_exists( - expected_full_table_name, ctas_method, get_example_database() + f"{schema_name}.{tmp_table_name}", ctas_method, get_example_database() ) @parameterized.expand([CtasMethod.TABLE, CtasMethod.VIEW]) @@ -404,14 +405,15 @@ class TestCelery(SupersetTestCase): self.assertEqual(0, query.rows) self.assertEqual(True, query.select_as_cta) self.assertEqual(True, query.select_as_cta_used) + self.drop_table_if_exists(table_name, ctas_method, get_example_database()) @parameterized.expand([CtasMethod.TABLE, CtasMethod.VIEW]) def test_run_async_cta_query_with_lower_limit(self, ctas_method): - main_db = get_example_database() - db_backend = main_db.backend - db_id = main_db.id + example_db = get_example_database() + db_backend = example_db.backend + db_id = example_db.id tmp_table = f"tmp_async_2_{ctas_method}" - self.drop_table_if_exists(tmp_table, ctas_method, main_db) + self.drop_table_if_exists(tmp_table, ctas_method, example_db) sql_where = "SELECT name FROM birth_names LIMIT 1" result = self.run_sql( @@ -450,6 +452,7 @@ class TestCelery(SupersetTestCase): self.assertEqual(None, query.limit) self.assertEqual(True, query.select_as_cta) self.assertEqual(True, query.select_as_cta_used) + self.drop_table_if_exists(tmp_table, ctas_method, get_example_database()) def test_default_data_serialization(self): data = [("a", 4, 4.0, datetime.datetime(2019, 8, 18, 16, 39, 16, 660000))] diff --git a/tests/charts/api_tests.py b/tests/charts/api_tests.py index 5048a0a6d..48e929d66 100644 --- a/tests/charts/api_tests.py +++ b/tests/charts/api_tests.py @@ -366,14 +366,15 @@ class TestChartApi(SupersetTestCase, ApiOwnersTestCaseMixin): gamma = self.get_user("gamma") chart_id = self.insert_chart("title", [admin.id], 1).id + birth_names_table_id = SupersetTestCase.get_table_by_name("birth_names").id chart_data = { "slice_name": "title1_changed", "description": "description1", "owners": [gamma.id], "viz_type": "viz_type1", - "params": "{'a': 1}", + "params": """{"a": 1}""", "cache_timeout": 1000, - "datasource_id": 1, + "datasource_id": birth_names_table_id, "datasource_type": "table", "dashboards": [1], } @@ -388,9 +389,9 @@ class TestChartApi(SupersetTestCase, ApiOwnersTestCaseMixin): self.assertIn(admin, model.owners) self.assertIn(gamma, model.owners) self.assertEqual(model.viz_type, "viz_type1") - self.assertEqual(model.params, "{'a': 1}") + self.assertEqual(model.params, """{"a": 1}""") self.assertEqual(model.cache_timeout, 1000) - self.assertEqual(model.datasource_id, 1) + self.assertEqual(model.datasource_id, birth_names_table_id) self.assertEqual(model.datasource_type, "table") self.assertEqual(model.datasource_name, "birth_names") self.assertIn(related_dashboard, model.dashboards) diff --git a/tests/load_examples_test.py b/tests/conftest.py similarity index 57% rename from tests/load_examples_test.py rename to tests/conftest.py index 38d892b9c..760ba9795 100644 --- a/tests/load_examples_test.py +++ b/tests/conftest.py @@ -14,34 +14,35 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +from typing import Any + +import pytest + from superset.utils.core import get_example_database -from .base_tests import SupersetTestCase +from tests.test_app import app # isort:skip -class TestSupersetDataFrame(SupersetTestCase): - def setUp(self) -> None: - # Late importing here as we need an app context to be pushed... - from superset import examples - - self.examples = examples - - def test_load_css_templates(self): - self.examples.load_css_templates() - - def test_load_energy(self): - self.examples.load_energy(sample=True) - - def test_load_world_bank_health_n_pop(self): - self.examples.load_world_bank_health_n_pop(sample=True) - - def test_load_birth_names(self): - self.examples.load_birth_names(sample=True) - - def test_load_test_users_run(self): +@pytest.fixture(autouse=True, scope="session") +def setup_sample_data() -> Any: + with app.app_context(): from superset.cli import load_test_users_run load_test_users_run() - def test_load_unicode_test_data(self): - self.examples.load_unicode_test_data(sample=True) + from superset import examples + + examples.load_css_templates() + examples.load_energy(sample=True) + examples.load_world_bank_health_n_pop(sample=True) + examples.load_birth_names(sample=True) + examples.load_unicode_test_data(sample=True) + + yield + + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE energy_usage") + engine.execute("DROP TABLE wb_health_population") + engine.execute("DROP TABLE birth_names") + engine.execute("DROP TABLE unicode_test") diff --git a/tests/core_tests.py b/tests/core_tests.py index d179ac3df..4f2d1bfbe 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -361,8 +361,8 @@ class TestCore(SupersetTestCase): self.assertEqual(resp.status_code, 200) def test_get_user_slices_for_owners(self): - self.login(username="admin") - user = security_manager.find_user("admin") + self.login(username="alpha") + user = security_manager.find_user("alpha") slice_name = "Girls" # ensure user is not owner of any slices diff --git a/tests/datasets/api_tests.py b/tests/datasets/api_tests.py index 08c8d5ae2..68571c6a7 100644 --- a/tests/datasets/api_tests.py +++ b/tests/datasets/api_tests.py @@ -69,6 +69,15 @@ class TestDatasetApi(SupersetTestCase): .one() ) + @staticmethod + def get_energy_usage_dataset(): + example_db = get_example_database() + return ( + db.session.query(SqlaTable) + .filter_by(database=example_db, table_name="energy_usage") + .one() + ) + def test_get_dataset_list(self): """ Dataset API: Test get dataset list @@ -133,7 +142,7 @@ class TestDatasetApi(SupersetTestCase): """ Dataset API: Test get dataset item """ - table = self.get_birth_names_dataset() + table = self.get_energy_usage_dataset() self.login(username="admin") uri = f"api/v1/dataset/{table.id}" rv = self.get_assert_metric(uri, "get") @@ -143,21 +152,22 @@ class TestDatasetApi(SupersetTestCase): "cache_timeout": None, "database": {"database_name": "examples", "id": 1}, "default_endpoint": None, - "description": None, + "description": "Energy consumption", "fetch_values_predicate": None, - "filter_select_enabled": True, + "filter_select_enabled": False, "is_sqllab_view": False, - "main_dttm_col": "ds", + "main_dttm_col": None, "offset": 0, "owners": [], "schema": None, "sql": None, - "table_name": "birth_names", + "table_name": "energy_usage", "template_params": None, } - for key, value in expected_result.items(): - self.assertEqual(response["result"][key], expected_result[key]) - self.assertEqual(len(response["result"]["columns"]), 8) + assert { + k: v for k, v in response["result"].items() if k in expected_result + } == expected_result + self.assertEqual(len(response["result"]["columns"]), 3) self.assertEqual(len(response["result"]["metrics"]), 2) def test_get_dataset_info(self): diff --git a/tests/schedules_test.py b/tests/schedules_test.py index 3e662c72a..549a0cd98 100644 --- a/tests/schedules_test.py +++ b/tests/schedules_test.py @@ -65,7 +65,7 @@ class TestSchedules(SupersetTestCase): slce = db.session.query(Slice).filter_by(slice_name="Participants").one() dashboard = ( db.session.query(Dashboard) - .filter_by(dashboard_title="USA Births Names") + .filter_by(dashboard_title="World Bank's Data") .one() ) diff --git a/tox.ini b/tox.ini index d2171ef37..e0f4ece1b 100644 --- a/tox.ini +++ b/tox.ini @@ -21,10 +21,9 @@ commands = superset db upgrade superset init - pytest -ra -q tests/load_examples_test.py # use -s to be able to use break pointers. # no args or tests/* can be passed as an argument to run all tests - pytest --ignore=load_examples_test {posargs} + pytest {posargs} deps = -rrequirements/testing.txt setenv =