diff --git a/tests/integration_tests/access_tests.py b/tests/integration_tests/access_tests.py index 6bf6cac25..06a3ed238 100644 --- a/tests/integration_tests/access_tests.py +++ b/tests/integration_tests/access_tests.py @@ -25,12 +25,15 @@ from sqlalchemy import inspect from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app # isort:skip from superset import db, security_manager diff --git a/tests/integration_tests/base_api_tests.py b/tests/integration_tests/base_api_tests.py index a76346149..09a754e39 100644 --- a/tests/integration_tests/base_api_tests.py +++ b/tests/integration_tests/base_api_tests.py @@ -18,6 +18,7 @@ import json from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) import pytest diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index b600ab557..62edb514b 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -24,6 +24,7 @@ from superset.common.db_query_status import QueryStatus from superset.extensions import cache_manager from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/celery_tests.py b/tests/integration_tests/celery_tests.py index bfcb73271..1fd076f25 100644 --- a/tests/integration_tests/celery_tests.py +++ b/tests/integration_tests/celery_tests.py @@ -25,6 +25,7 @@ import unittest.mock as mock from typing import Optional from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index ba49c3e38..c688044eb 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -40,9 +40,11 @@ from tests.integration_tests.base_api_tests import ApiOwnersTestCaseMixin from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.importexport import ( chart_config, @@ -53,9 +55,11 @@ from tests.integration_tests.fixtures.importexport import ( ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_slice, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.insert_chart_mixin import InsertChartMixin from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/charts/commands_tests.py b/tests/integration_tests/charts/commands_tests.py index caedb6a0b..4e8ffa1af 100644 --- a/tests/integration_tests/charts/commands_tests.py +++ b/tests/integration_tests/charts/commands_tests.py @@ -34,6 +34,7 @@ from superset.models.core import Database from superset.models.slice import Slice from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/charts/data/api_tests.py b/tests/integration_tests/charts/data/api_tests.py index 2831c291c..1e6aa5601 100644 --- a/tests/integration_tests/charts/data/api_tests.py +++ b/tests/integration_tests/charts/data/api_tests.py @@ -32,6 +32,7 @@ from tests.integration_tests.base_tests import ( from tests.integration_tests.annotation_layers.fixtures import create_annotation_layers from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/charts/schema_tests.py b/tests/integration_tests/charts/schema_tests.py index 821a8d57f..c28699f33 100644 --- a/tests/integration_tests/charts/schema_tests.py +++ b/tests/integration_tests/charts/schema_tests.py @@ -26,6 +26,7 @@ from superset.charts.schemas import ChartDataQueryContextSchema from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/cli_tests.py b/tests/integration_tests/cli_tests.py index 40dfa749b..586162726 100644 --- a/tests/integration_tests/cli_tests.py +++ b/tests/integration_tests/cli_tests.py @@ -30,6 +30,7 @@ import superset.cli from superset import app from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/core_tests.py b/tests/integration_tests/core_tests.py index 87dca2d7b..67e5e5da0 100644 --- a/tests/integration_tests/core_tests.py +++ b/tests/integration_tests/core_tests.py @@ -27,6 +27,7 @@ from typing import Dict, List from urllib.parse import quote from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -44,6 +45,7 @@ from superset.utils.core import get_example_database from tests.integration_tests.conftest import with_feature_flags from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app import superset.views.utils @@ -73,6 +75,7 @@ from superset.views.database.views import DatabaseView from .base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) logger = logging.getLogger(__name__) diff --git a/tests/integration_tests/dashboard_tests.py b/tests/integration_tests/dashboard_tests.py index feac2c607..63453d85e 100644 --- a/tests/integration_tests/dashboard_tests.py +++ b/tests/integration_tests/dashboard_tests.py @@ -33,16 +33,20 @@ from superset.models.dashboard import Dashboard from superset.models.slice import Slice from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.public_role import public_role_like_gamma from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_position, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/dashboard_utils.py b/tests/integration_tests/dashboard_utils.py index 39032c923..3c314312b 100644 --- a/tests/integration_tests/dashboard_utils.py +++ b/tests/integration_tests/dashboard_utils.py @@ -29,35 +29,30 @@ from superset.models.slice import Slice from superset.utils.core import get_example_default_schema -def create_table_for_dashboard( - df: DataFrame, +def get_table( + table_name: str, database: Database, schema: Optional[str] = None, +): + schema = schema or get_example_default_schema() + table_source = ConnectorRegistry.sources["table"] + return ( + db.session.query(table_source) + .filter_by(database_id=database.id, schema=schema, table_name=table_name) + .one_or_none() + ) + + +def create_table_metadata( table_name: str, database: Database, - dtype: Dict[str, Any], table_description: str = "", fetch_values_predicate: Optional[str] = None, schema: Optional[str] = None, ) -> SqlaTable: schema = schema or get_example_default_schema() - df.to_sql( - table_name, - database.get_sqla_engine(), - if_exists="replace", - chunksize=500, - dtype=dtype, - index=False, - method="multi", - schema=schema, - ) - - table_source = ConnectorRegistry.sources["table"] - table = ( - db.session.query(table_source) - .filter_by(database_id=database.id, schema=schema, table_name=table_name) - .one_or_none() - ) + table = get_table(table_name, database, schema) if not table: + table_source = ConnectorRegistry.sources["table"] table = table_source(schema=schema, table_name=table_name) if fetch_values_predicate: table.fetch_values_predicate = fetch_values_predicate diff --git a/tests/integration_tests/dashboards/api_tests.py b/tests/integration_tests/dashboards/api_tests.py index 071cb8bee..755eb3776 100644 --- a/tests/integration_tests/dashboards/api_tests.py +++ b/tests/integration_tests/dashboards/api_tests.py @@ -53,9 +53,11 @@ from tests.integration_tests.fixtures.importexport import ( from tests.integration_tests.utils.get_dashboards import get_dashboards_ids from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) DASHBOARDS_FIXTURE_COUNT = 10 diff --git a/tests/integration_tests/dashboards/commands_tests.py b/tests/integration_tests/dashboards/commands_tests.py index 63d56eed0..7c7a2046f 100644 --- a/tests/integration_tests/dashboards/commands_tests.py +++ b/tests/integration_tests/dashboards/commands_tests.py @@ -48,6 +48,7 @@ from tests.integration_tests.fixtures.importexport import ( ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/dashboards/dao_tests.py b/tests/integration_tests/dashboards/dao_tests.py index ba3516624..4a02cd710 100644 --- a/tests/integration_tests/dashboards/dao_tests.py +++ b/tests/integration_tests/dashboards/dao_tests.py @@ -28,6 +28,7 @@ from superset.models.dashboard import Dashboard from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index 02af14c5f..bd0e2e7d9 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -30,6 +30,7 @@ from superset.models.dashboard import Dashboard from tests.integration_tests.base_tests import login from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/dashboards/security/security_dataset_tests.py b/tests/integration_tests/dashboards/security/security_dataset_tests.py index de5a14ef3..01bce014a 100644 --- a/tests/integration_tests/dashboards/security/security_dataset_tests.py +++ b/tests/integration_tests/dashboards/security/security_dataset_tests.py @@ -28,6 +28,7 @@ from tests.integration_tests.dashboards.consts import * from tests.integration_tests.dashboards.dashboard_test_utils import * from tests.integration_tests.dashboards.superset_factory_util import * from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) diff --git a/tests/integration_tests/dashboards/security/security_rbac_tests.py b/tests/integration_tests/dashboards/security/security_rbac_tests.py index 2a8ab9921..5a1f02f1e 100644 --- a/tests/integration_tests/dashboards/security/security_rbac_tests.py +++ b/tests/integration_tests/dashboards/security/security_rbac_tests.py @@ -32,6 +32,7 @@ from tests.integration_tests.dashboards.superset_factory_util import ( ) from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.public_role import public_role_like_gamma from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/databases/api_tests.py b/tests/integration_tests/databases/api_tests.py index a0e8bc3fd..a64eb3f7d 100644 --- a/tests/integration_tests/databases/api_tests.py +++ b/tests/integration_tests/databases/api_tests.py @@ -47,13 +47,16 @@ from superset.utils.core import get_example_database, get_main_database from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.certificates import ssl_certificate from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.fixtures.importexport import ( database_config, @@ -63,6 +66,7 @@ from tests.integration_tests.fixtures.importexport import ( ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_position, + load_unicode_data, ) from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/databases/commands_tests.py b/tests/integration_tests/databases/commands_tests.py index 0ec96abf9..5860b23fd 100644 --- a/tests/integration_tests/databases/commands_tests.py +++ b/tests/integration_tests/databases/commands_tests.py @@ -44,8 +44,10 @@ from superset.utils.core import backend, get_example_database from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/datasets/api_tests.py b/tests/integration_tests/datasets/api_tests.py index 09984a3db..03209fd94 100644 --- a/tests/integration_tests/datasets/api_tests.py +++ b/tests/integration_tests/datasets/api_tests.py @@ -46,8 +46,10 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.conftest import CTAS_SCHEMA_NAME from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( diff --git a/tests/integration_tests/datasets/commands_tests.py b/tests/integration_tests/datasets/commands_tests.py index d3493a4d1..67ff3a791 100644 --- a/tests/integration_tests/datasets/commands_tests.py +++ b/tests/integration_tests/datasets/commands_tests.py @@ -33,6 +33,7 @@ from superset.models.core import Database from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.energy_dashboard import ( + load_energy_table_data, load_energy_table_with_slice, ) from tests.integration_tests.fixtures.importexport import ( @@ -45,6 +46,7 @@ from tests.integration_tests.fixtures.importexport import ( ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) diff --git a/tests/integration_tests/datasource_tests.py b/tests/integration_tests/datasource_tests.py index 4c772d317..c923f2ebb 100644 --- a/tests/integration_tests/datasource_tests.py +++ b/tests/integration_tests/datasource_tests.py @@ -31,6 +31,7 @@ from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.base_tests import db_insert_temp_object, SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.datasource import get_datasource_post diff --git a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py index 294c6801e..e568b59a2 100644 --- a/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py +++ b/tests/integration_tests/db_engine_specs/base_engine_spec_tests.py @@ -35,8 +35,14 @@ from superset.utils.core import get_example_database from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec from tests.integration_tests.test_app import app -from ..fixtures.birth_names_dashboard import load_birth_names_dashboard_with_slices -from ..fixtures.energy_dashboard import load_energy_table_with_slice +from ..fixtures.birth_names_dashboard import ( + load_birth_names_dashboard_with_slices, + load_birth_names_data, +) +from ..fixtures.energy_dashboard import ( + load_energy_table_data, + load_energy_table_with_slice, +) from ..fixtures.pyodbcRow import Row diff --git a/tests/integration_tests/db_engine_specs/bigquery_tests.py b/tests/integration_tests/db_engine_specs/bigquery_tests.py index d3dd66df3..b7405092c 100644 --- a/tests/integration_tests/db_engine_specs/bigquery_tests.py +++ b/tests/integration_tests/db_engine_specs/bigquery_tests.py @@ -29,6 +29,7 @@ from superset.sql_parse import Table from tests.integration_tests.db_engine_specs.base_tests import TestDbEngineSpec from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) diff --git a/tests/integration_tests/fixtures/__init__.py b/tests/integration_tests/fixtures/__init__.py index d3531aa29..0e06be352 100644 --- a/tests/integration_tests/fixtures/__init__.py +++ b/tests/integration_tests/fixtures/__init__.py @@ -19,7 +19,7 @@ from .birth_names_dashboard import ( load_birth_names_dashboard_with_slices, load_birth_names_dashboard_with_slices_module_scope, ) -from .energy_dashboard import load_energy_table_with_slice +from .energy_dashboard import load_energy_table_data, load_energy_table_with_slice from .public_role import public_role_like_gamma, public_role_like_test_role from .unicode_dashboard import ( load_unicode_dashboard_with_position, diff --git a/tests/integration_tests/fixtures/birth_names_dashboard.py b/tests/integration_tests/fixtures/birth_names_dashboard.py index 70a120bcf..56d1bc287 100644 --- a/tests/integration_tests/fixtures/birth_names_dashboard.py +++ b/tests/integration_tests/fixtures/birth_names_dashboard.py @@ -23,7 +23,7 @@ from typing import Any, Dict, List, Optional import pandas as pd import pytest from pandas import DataFrame -from sqlalchemy import DateTime, String, TIMESTAMP +from sqlalchemy import DateTime, String from superset import ConnectorRegistry, db from superset.connectors.sqla.models import SqlaTable @@ -31,29 +31,14 @@ from superset.models.core import Database from superset.models.dashboard import Dashboard from superset.models.slice import Slice from superset.utils.core import get_example_database, get_example_default_schema -from tests.integration_tests.dashboard_utils import create_table_for_dashboard +from tests.integration_tests.dashboard_utils import create_table_metadata from tests.integration_tests.test_app import app - -@pytest.fixture() -def load_birth_names_dashboard_with_slices(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) +BIRTH_NAMES_TBL_NAME = "birth_names" -@pytest.fixture(scope="module") -def load_birth_names_dashboard_with_slices_module_scope(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) - - -def _load_data(): - table_name = "birth_names" - +@pytest.fixture(scope="session") +def load_birth_names_data(): with app.app_context(): database = get_example_database() df = _get_dataframe(database) @@ -63,35 +48,61 @@ def _load_data(): "state": String(10), "name": String(255), } - table = _create_table( - df=df, - table_name=table_name, - database=database, + + df.to_sql( + BIRTH_NAMES_TBL_NAME, + database.get_sqla_engine(), + if_exists="replace", + chunksize=500, dtype=dtype, - fetch_values_predicate="123 = 123", + index=False, + method="multi", + schema=get_example_default_schema(), ) + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS birth_names") - from superset.examples.birth_names import create_dashboard, create_slices - slices, _ = create_slices(table, admin_owner=False) - dash = create_dashboard(slices) - slices_ids_to_delete = [slice.id for slice in slices] - dash_id_to_delete = dash.id - return dash_id_to_delete, slices_ids_to_delete +@pytest.fixture() +def load_birth_names_dashboard_with_slices(load_birth_names_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = _create_dashboards() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +@pytest.fixture(scope="module") +def load_birth_names_dashboard_with_slices_module_scope(load_birth_names_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = _create_dashboards() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +def _create_dashboards(): + table = _create_table( + table_name=BIRTH_NAMES_TBL_NAME, + database=get_example_database(), + fetch_values_predicate="123 = 123", + ) + + from superset.examples.birth_names import create_dashboard, create_slices + + slices, _ = create_slices(table, admin_owner=False) + dash = create_dashboard(slices) + slices_ids_to_delete = [slice.id for slice in slices] + dash_id_to_delete = dash.id + return dash_id_to_delete, slices_ids_to_delete def _create_table( - df: DataFrame, - table_name: str, - database: "Database", - dtype: Dict[str, Any], - fetch_values_predicate: Optional[str] = None, + table_name: str, database: "Database", fetch_values_predicate: Optional[str] = None, ): - table = create_table_for_dashboard( - df=df, + table = create_table_metadata( table_name=table_name, database=database, - dtype=dtype, fetch_values_predicate=fetch_values_predicate, ) from superset.examples.birth_names import _add_table_metrics, _set_table_metadata @@ -115,8 +126,6 @@ def _cleanup(dash_id: int, slices_ids: List[int]) -> None: columns = [column for column in datasource.columns] metrics = [metric for metric in datasource.metrics] - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS birth_names") for column in columns: db.session.delete(column) for metric in metrics: diff --git a/tests/integration_tests/fixtures/energy_dashboard.py b/tests/integration_tests/fixtures/energy_dashboard.py index 144710525..f4362ea74 100644 --- a/tests/integration_tests/fixtures/energy_dashboard.py +++ b/tests/integration_tests/fixtures/energy_dashboard.py @@ -15,34 +15,50 @@ # specific language governing permissions and limitations # under the License. import random -import textwrap from typing import Dict, Set import pandas as pd import pytest -from pandas import DataFrame from sqlalchemy import column, Float, String from superset import db from superset.connectors.sqla.models import SqlaTable, SqlMetric -from superset.models.dashboard import Dashboard from superset.models.slice import Slice -from superset.utils.core import get_example_database -from tests.integration_tests.dashboard_utils import ( - create_slice, - create_table_for_dashboard, -) +from superset.utils.core import get_example_database, get_example_default_schema +from tests.integration_tests.dashboard_utils import create_slice, create_table_metadata from tests.integration_tests.test_app import app misc_dash_slices: Set[str] = set() -@pytest.fixture() -def load_energy_table_with_slice(): - table_name = "energy_usage" - df = _get_dataframe() +ENERGY_USAGE_TBL_NAME = "energy_usage" + + +@pytest.fixture(scope="session") +def load_energy_table_data(): with app.app_context(): - _create_energy_table(df, table_name) + database = get_example_database() + df = _get_dataframe() + df.to_sql( + ENERGY_USAGE_TBL_NAME, + database.get_sqla_engine(), + if_exists="replace", + chunksize=500, + index=False, + dtype={"source": String(255), "target": String(255), "value": Float()}, + method="multi", + schema=get_example_default_schema(), + ) + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS energy_usage") + + +@pytest.fixture() +def load_energy_table_with_slice(load_energy_table_data): + with app.app_context(): + _create_energy_table() yield _cleanup() @@ -52,13 +68,11 @@ def _get_dataframe(): return pd.DataFrame.from_dict(data) -def _create_energy_table(df: DataFrame, table_name: str): - database = get_example_database() - - table_description = "Energy consumption" - schema = {"source": String(255), "target": String(255), "value": Float()} - table = create_table_for_dashboard( - df, table_name, database, schema, table_description +def _create_energy_table(): + table = create_table_metadata( + table_name=ENERGY_USAGE_TBL_NAME, + database=get_example_database(), + table_description="Energy consumption", ) table.fetch_metadata() @@ -96,8 +110,6 @@ def _create_and_commit_energy_slice( def _cleanup() -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS energy_usage") for slice_data in _get_energy_slices(): slice = ( db.session.query(Slice) diff --git a/tests/integration_tests/fixtures/unicode_dashboard.py b/tests/integration_tests/fixtures/unicode_dashboard.py index c670e086b..1f1218c8f 100644 --- a/tests/integration_tests/fixtures/unicode_dashboard.py +++ b/tests/integration_tests/fixtures/unicode_dashboard.py @@ -16,42 +16,58 @@ # under the License. import pandas as pd import pytest -from pandas import DataFrame from sqlalchemy import String from superset import db from superset.connectors.sqla.models import SqlaTable from superset.models.dashboard import Dashboard from superset.models.slice import Slice -from superset.utils.core import get_example_database +from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.dashboard_utils import ( create_dashboard, create_slice, - create_table_for_dashboard, + create_table_metadata, ) from tests.integration_tests.test_app import app +UNICODE_TBL_NAME = "unicode_test" + + +@pytest.fixture(scope="session") +def load_unicode_data(): + with app.app_context(): + _get_dataframe().to_sql( + UNICODE_TBL_NAME, + get_example_database().get_sqla_engine(), + if_exists="replace", + chunksize=500, + dtype={"phrase": String(500)}, + index=False, + method="multi", + schema=get_example_default_schema(), + ) + + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS unicode_test") + @pytest.fixture() -def load_unicode_dashboard_with_slice(): - table_name = "unicode_test" +def load_unicode_dashboard_with_slice(load_unicode_data): slice_name = "Unicode Cloud" - df = _get_dataframe() with app.app_context(): - dash = _create_unicode_dashboard(df, table_name, slice_name, None) + dash = _create_unicode_dashboard(slice_name, None) yield - _cleanup(dash, slice_name) @pytest.fixture() -def load_unicode_dashboard_with_position(): - table_name = "unicode_test" +def load_unicode_dashboard_with_position(load_unicode_data): slice_name = "Unicode Cloud" - df = _get_dataframe() position = "{}" with app.app_context(): - dash = _create_unicode_dashboard(df, table_name, slice_name, position) + dash = _create_unicode_dashboard(slice_name, position) yield _cleanup(dash, slice_name) @@ -74,14 +90,8 @@ def _get_unicode_data(): ] -def _create_unicode_dashboard( - df: DataFrame, table_name: str, slice_title: str, position: str -) -> Dashboard: - database = get_example_database() - dtype = { - "phrase": String(500), - } - table = create_table_for_dashboard(df, table_name, database, dtype) +def _create_unicode_dashboard(slice_title: str, position: str) -> Dashboard: + table = create_table_metadata(UNICODE_TBL_NAME, get_example_database()) table.fetch_metadata() if slice_title: @@ -101,8 +111,6 @@ def _create_and_commit_unicode_slice(table: SqlaTable, title: str): def _cleanup(dash: Dashboard, slice_name: str) -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS unicode_test") db.session.delete(dash) if slice_name: slice = db.session.query(Slice).filter_by(slice_name=slice_name).one_or_none() diff --git a/tests/integration_tests/fixtures/world_bank_dashboard.py b/tests/integration_tests/fixtures/world_bank_dashboard.py index 96190c4b1..0782c8bdf 100644 --- a/tests/integration_tests/fixtures/world_bank_dashboard.py +++ b/tests/integration_tests/fixtures/world_bank_dashboard.py @@ -22,7 +22,7 @@ from typing import Any, Dict, List import pandas as pd import pytest from pandas import DataFrame -from sqlalchemy import DateTime, String, TIMESTAMP +from sqlalchemy import DateTime, String from superset import db from superset.connectors.sqla.models import SqlaTable @@ -32,43 +32,59 @@ from superset.models.slice import Slice from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.dashboard_utils import ( create_dashboard, - create_table_for_dashboard, + create_table_metadata, ) from tests.integration_tests.test_app import app - -@pytest.fixture() -def load_world_bank_dashboard_with_slices(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) +WB_HEALTH_POPULATION = "wb_health_population" -@pytest.fixture(scope="module") -def load_world_bank_dashboard_with_slices_module_scope(): - dash_id_to_delete, slices_ids_to_delete = _load_data() - yield - with app.app_context(): - _cleanup(dash_id_to_delete, slices_ids_to_delete) - - -def _load_data(): - table_name = "wb_health_population" - +@pytest.fixture(scope="session") +def load_world_bank_data(): with app.app_context(): database = get_example_database() - schema = get_example_default_schema() - df = _get_dataframe(database) dtype = { "year": DateTime if database.backend != "presto" else String(255), "country_code": String(3), "country_name": String(255), "region": String(255), } - table = create_table_for_dashboard( - df, table_name, database, dtype, schema=schema + _get_dataframe(database).to_sql( + WB_HEALTH_POPULATION, + get_example_database().get_sqla_engine(), + if_exists="replace", + chunksize=500, + dtype=dtype, + index=False, + method="multi", + schema=get_example_default_schema(), ) + + yield + with app.app_context(): + engine = get_example_database().get_sqla_engine() + engine.execute("DROP TABLE IF EXISTS wb_health_population") + + +@pytest.fixture() +def load_world_bank_dashboard_with_slices(load_world_bank_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +@pytest.fixture(scope="module") +def load_world_bank_dashboard_with_slices_module_scope(load_world_bank_data): + with app.app_context(): + dash_id_to_delete, slices_ids_to_delete = create_dashboard_for_loaded_data() + yield + _cleanup(dash_id_to_delete, slices_ids_to_delete) + + +def create_dashboard_for_loaded_data(): + with app.app_context(): + table = create_table_metadata(WB_HEALTH_POPULATION, get_example_database()) slices = _create_world_bank_slices(table) dash = _create_world_bank_dashboard(table, slices) slices_ids_to_delete = [slice.id for slice in slices] @@ -112,8 +128,6 @@ def _create_world_bank_dashboard(table: SqlaTable, slices: List[Slice]) -> Dashb def _cleanup(dash_id: int, slices_ids: List[int]) -> None: - engine = get_example_database().get_sqla_engine() - engine.execute("DROP TABLE IF EXISTS wb_health_population") dash = db.session.query(Dashboard).filter_by(id=dash_id).first() db.session.delete(dash) for slice_id in slices_ids: diff --git a/tests/integration_tests/import_export_tests.py b/tests/integration_tests/import_export_tests.py index 42adcb851..1745f5e01 100644 --- a/tests/integration_tests/import_export_tests.py +++ b/tests/integration_tests/import_export_tests.py @@ -20,6 +20,7 @@ import json import unittest from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -28,6 +29,7 @@ from sqlalchemy.orm.session import make_transient from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.test_app import app from superset.dashboards.commands.importers.v0 import decode_dashboards @@ -47,6 +49,7 @@ from superset.utils.core import get_example_database, get_example_default_schema from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/model_tests.py b/tests/integration_tests/model_tests.py index 7ffb173cd..bcd57726f 100644 --- a/tests/integration_tests/model_tests.py +++ b/tests/integration_tests/model_tests.py @@ -22,6 +22,7 @@ from unittest import mock from superset.exceptions import SupersetException from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import pytest @@ -38,7 +39,10 @@ from superset.models.sql_types.base import literal_dttm_type_factory from superset.utils.core import get_example_database from .base_tests import SupersetTestCase -from .fixtures.energy_dashboard import load_energy_table_with_slice +from .fixtures.energy_dashboard import ( + load_energy_table_with_slice, + load_energy_table_data, +) class TestDatabaseModel(SupersetTestCase): diff --git a/tests/integration_tests/query_context_tests.py b/tests/integration_tests/query_context_tests.py index 6df246db7..91ce67bb8 100644 --- a/tests/integration_tests/query_context_tests.py +++ b/tests/integration_tests/query_context_tests.py @@ -34,6 +34,7 @@ from superset.utils.core import AdhocMetricExpressionType, backend, TimeRangeEnd from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context diff --git a/tests/integration_tests/reports/api_tests.py b/tests/integration_tests/reports/api_tests.py index 8df8b4bc8..fa9e36d10 100644 --- a/tests/integration_tests/reports/api_tests.py +++ b/tests/integration_tests/reports/api_tests.py @@ -43,6 +43,7 @@ from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.conftest import with_feature_flags from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.reports.utils import insert_report_schedule diff --git a/tests/integration_tests/reports/commands_tests.py b/tests/integration_tests/reports/commands_tests.py index 1ed3f35a1..3a7cc873e 100644 --- a/tests/integration_tests/reports/commands_tests.py +++ b/tests/integration_tests/reports/commands_tests.py @@ -59,9 +59,11 @@ from superset.reports.commands.log_prune import AsyncPruneReportScheduleLogComma from superset.utils.core import get_example_database from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices_module_scope, + load_world_bank_data, ) from tests.integration_tests.reports.utils import insert_report_schedule from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/schedules_test.py b/tests/integration_tests/schedules_test.py index c47bf5fc8..b5cfe7160 100644 --- a/tests/integration_tests/schedules_test.py +++ b/tests/integration_tests/schedules_test.py @@ -26,6 +26,7 @@ from slack import errors, WebClient from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from tests.integration_tests.test_app import app from superset import db diff --git a/tests/integration_tests/security_tests.py b/tests/integration_tests/security_tests.py index e96ed6745..9863e1b31 100644 --- a/tests/integration_tests/security_tests.py +++ b/tests/integration_tests/security_tests.py @@ -48,9 +48,11 @@ from superset.views.access_requests import AccessRequestsModelView from .base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.energy_dashboard import ( load_energy_table_with_slice, + load_energy_table_data, ) from tests.integration_tests.fixtures.public_role import ( public_role_like_gamma, @@ -58,9 +60,11 @@ from tests.integration_tests.fixtures.public_role import ( ) from tests.integration_tests.fixtures.unicode_dashboard import ( load_unicode_dashboard_with_slice, + load_unicode_data, ) from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) NEW_SECURITY_CONVERGE_VIEWS = ( diff --git a/tests/integration_tests/sqla_models_tests.py b/tests/integration_tests/sqla_models_tests.py index 53bf031bf..00d385900 100644 --- a/tests/integration_tests/sqla_models_tests.py +++ b/tests/integration_tests/sqla_models_tests.py @@ -44,6 +44,7 @@ from superset.utils.core import ( ) from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from .base_tests import SupersetTestCase diff --git a/tests/integration_tests/sqllab_tests.py b/tests/integration_tests/sqllab_tests.py index 8a3297f05..2dce2c11f 100644 --- a/tests/integration_tests/sqllab_tests.py +++ b/tests/integration_tests/sqllab_tests.py @@ -58,6 +58,7 @@ from .base_tests import SupersetTestCase from .conftest import CTAS_SCHEMA_NAME from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) QUERY_1 = "SELECT * FROM birth_names LIMIT 1" diff --git a/tests/integration_tests/strategy_tests.py b/tests/integration_tests/strategy_tests.py index 1ba41d65b..59430f80b 100644 --- a/tests/integration_tests/strategy_tests.py +++ b/tests/integration_tests/strategy_tests.py @@ -21,6 +21,7 @@ import json from unittest.mock import MagicMock from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from sqlalchemy import String, Date, Float @@ -42,8 +43,11 @@ from superset.tasks.cache import ( ) from .base_tests import SupersetTestCase -from .dashboard_utils import create_dashboard, create_slice, create_table_for_dashboard -from .fixtures.unicode_dashboard import load_unicode_dashboard_with_slice +from .dashboard_utils import create_dashboard, create_slice, create_table_metadata +from .fixtures.unicode_dashboard import ( + load_unicode_dashboard_with_slice, + load_unicode_data, +) URL_PREFIX = "http://0.0.0.0:8081" diff --git a/tests/integration_tests/tasks/async_queries_tests.py b/tests/integration_tests/tasks/async_queries_tests.py index 6acdd88b7..388185a54 100644 --- a/tests/integration_tests/tasks/async_queries_tests.py +++ b/tests/integration_tests/tasks/async_queries_tests.py @@ -35,6 +35,7 @@ from superset.tasks.async_queries import ( from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) from tests.integration_tests.fixtures.query_context import get_query_context from tests.integration_tests.test_app import app diff --git a/tests/integration_tests/utils_tests.py b/tests/integration_tests/utils_tests.py index 3fcbe959e..fc9dba4c0 100644 --- a/tests/integration_tests/utils_tests.py +++ b/tests/integration_tests/utils_tests.py @@ -26,6 +26,7 @@ from typing import Any, Tuple, List, Optional from unittest.mock import Mock, patch from tests.integration_tests.fixtures.birth_names_dashboard import ( load_birth_names_dashboard_with_slices, + load_birth_names_data, ) import numpy as np @@ -81,6 +82,7 @@ from superset.views.utils import ( from tests.integration_tests.base_tests import SupersetTestCase from tests.integration_tests.fixtures.world_bank_dashboard import ( load_world_bank_dashboard_with_slices, + load_world_bank_data, ) from .fixtures.certificates import ssl_certificate diff --git a/tox.ini b/tox.ini index 839bce558..88a51278e 100644 --- a/tox.ini +++ b/tox.ini @@ -37,7 +37,7 @@ setenv = postgres: SUPERSET__SQLALCHEMY_DATABASE_URI = postgresql+psycopg2://superset:superset@localhost/test sqlite: SUPERSET__SQLALCHEMY_DATABASE_URI = sqlite:////{envtmpdir}/superset.db mysql-presto: SUPERSET__SQLALCHEMY_DATABASE_URI = mysql://mysqluser:mysqluserpassword@localhost/superset?charset=utf8 - # docker run -p 8080:8080 --name presto prestosql/presto + # docker run -p 8080:8080 --name presto starburstdata/presto mysql-presto: SUPERSET__SQLALCHEMY_EXAMPLES_URI = presto://localhost:8080/memory/default # based on https://github.com/big-data-europe/docker-hadoop # clone the repo & run docker-compose up -d to test locally