diff --git a/superset/cli.py b/superset/cli.py index adff31f71..bc72427db 100755 --- a/superset/cli.py +++ b/superset/cli.py @@ -235,7 +235,9 @@ def refresh_druid(datasource: str, merge: bool) -> None: ) def import_dashboards(path: str, recursive: bool, username: str) -> None: """Import dashboards from JSON""" - from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand + from superset.dashboards.commands.importers.dispatcher import ( + ImportDashboardsCommand, + ) path_object = Path(path) files: List[Path] = [] diff --git a/superset/datasets/commands/importers/v0.py b/superset/datasets/commands/importers/v0.py index d45c58c58..55f1a9c9b 100644 --- a/superset/datasets/commands/importers/v0.py +++ b/superset/datasets/commands/importers/v0.py @@ -14,6 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import json import logging from typing import Any, Callable, Dict, List, Optional @@ -301,9 +302,23 @@ class ImportDatasetsCommand(BaseCommand): def run(self) -> None: self.validate() + # TODO (betodealmeida): add rollback in case of error for file_name, config in self._configs.items(): logger.info("Importing dataset from file %s", file_name) - import_from_dict(db.session, config, sync=self.sync) + if isinstance(config, dict): + import_from_dict(db.session, config, sync=self.sync) + else: # list + for dataset in config: + # UI exports don't have the database metadata, so we assume + # the DB exists and has the same name + params = json.loads(dataset["params"]) + database = ( + db.session.query(Database) + .filter_by(database_name=params["database_name"]) + .one() + ) + dataset["database_id"] = database.id + SqlaTable.import_from_dict(db.session, dataset, sync=self.sync) def validate(self) -> None: # ensure all files are YAML @@ -314,8 +329,18 @@ class ImportDatasetsCommand(BaseCommand): logger.exception("Invalid YAML file") raise IncorrectVersionError(f"{file_name} is not a valid YAML file") - # check for keys - if DATABASES_KEY not in config and DRUID_CLUSTERS_KEY not in config: - raise IncorrectVersionError(f"{file_name} has no valid keys") + # CLI export + if isinstance(config, dict): + # TODO (betodealmeida): validate with Marshmallow + if DATABASES_KEY not in config and DRUID_CLUSTERS_KEY not in config: + raise IncorrectVersionError(f"{file_name} has no valid keys") + + # UI export + elif isinstance(config, list): + # TODO (betodealmeida): validate with Marshmallow + pass + + else: + raise IncorrectVersionError(f"{file_name} is not a valid file") self._configs[file_name] = config diff --git a/tests/dashboards/commands_tests.py b/tests/dashboards/commands_tests.py index ca03cf3ef..b081a14f3 100644 --- a/tests/dashboards/commands_tests.py +++ b/tests/dashboards/commands_tests.py @@ -25,14 +25,18 @@ import yaml from superset import db, security_manager from superset.commands.exceptions import CommandInvalidError from superset.commands.importers.exceptions import IncorrectVersionError +from superset.connectors.sqla.models import SqlaTable from superset.dashboards.commands.exceptions import DashboardNotFoundError from superset.dashboards.commands.export import ExportDashboardsCommand -from superset.dashboards.commands.importers.v1 import ImportDashboardsCommand +from superset.dashboards.commands.importers import v0, v1 +from superset.models.core import Database from superset.models.dashboard import Dashboard +from superset.models.slice import Slice from tests.base_tests import SupersetTestCase from tests.fixtures.importexport import ( chart_config, dashboard_config, + dashboard_export, dashboard_metadata_config, database_config, dataset_config, @@ -205,6 +209,45 @@ class TestExportDashboardsCommand(SupersetTestCase): class TestImportDashboardsCommand(SupersetTestCase): + def test_import_v0_dashboard_cli_export(self): + num_dashboards = db.session.query(Dashboard).count() + num_charts = db.session.query(Slice).count() + num_datasets = db.session.query(SqlaTable).count() + num_databases = db.session.query(Database).count() + + contents = { + "20201119_181105.json": json.dumps(dashboard_export), + } + command = v0.ImportDashboardsCommand(contents) + command.run() + + new_num_dashboards = db.session.query(Dashboard).count() + new_num_charts = db.session.query(Slice).count() + new_num_datasets = db.session.query(SqlaTable).count() + new_num_databases = db.session.query(Database).count() + assert new_num_dashboards == num_dashboards + 1 + assert new_num_charts == num_charts + 1 + assert new_num_datasets == num_datasets + 1 + assert new_num_databases == num_databases + + dashboard = ( + db.session.query(Dashboard).filter_by(dashboard_title="Births 2").one() + ) + assert len(dashboard.slices) == 1 + chart = dashboard.slices[0] + assert chart.slice_name == "Number of California Births" + + dataset = chart.table + assert dataset.table_name == "birth_names_2" + + database = dataset.database + assert database.database_name == "examples" + + db.session.delete(dashboard) + db.session.delete(chart) + db.session.delete(dataset) + db.session.commit() + def test_import_v1_dashboard(self): """Test that we can import a dashboard""" contents = { @@ -214,7 +257,7 @@ class TestImportDashboardsCommand(SupersetTestCase): "charts/imported_chart.yaml": yaml.safe_dump(chart_config), "dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config), } - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) command.run() dashboard = ( @@ -296,7 +339,7 @@ class TestImportDashboardsCommand(SupersetTestCase): "charts/imported_chart.yaml": yaml.safe_dump(chart_config), "dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config), } - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) command.run() command.run() @@ -325,7 +368,7 @@ class TestImportDashboardsCommand(SupersetTestCase): "charts/imported_chart.yaml": yaml.safe_dump(chart_config), "dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config), } - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) with pytest.raises(IncorrectVersionError) as excinfo: command.run() assert str(excinfo.value) == "Missing metadata.yaml" @@ -338,14 +381,14 @@ class TestImportDashboardsCommand(SupersetTestCase): "timestamp": "2020-11-04T21:27:44.423819+00:00", } ) - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) with pytest.raises(IncorrectVersionError) as excinfo: command.run() assert str(excinfo.value) == "Must be equal to 1.0.0." # type should be Database contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config) - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) with pytest.raises(CommandInvalidError) as excinfo: command.run() assert str(excinfo.value) == "Error importing dashboard" @@ -358,7 +401,7 @@ class TestImportDashboardsCommand(SupersetTestCase): del broken_config["table_name"] contents["metadata.yaml"] = yaml.safe_dump(dashboard_metadata_config) contents["datasets/imported_dataset.yaml"] = yaml.safe_dump(broken_config) - command = ImportDashboardsCommand(contents) + command = v1.ImportDashboardsCommand(contents) with pytest.raises(CommandInvalidError) as excinfo: command.run() assert str(excinfo.value) == "Error importing dashboard" diff --git a/tests/datasets/commands_tests.py b/tests/datasets/commands_tests.py index b79986e06..2e7249b9d 100644 --- a/tests/datasets/commands_tests.py +++ b/tests/datasets/commands_tests.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -# pylint: disable=no-self-use, invalid-name +# pylint: disable=no-self-use, invalid-name, line-too-long from operator import itemgetter from unittest.mock import patch @@ -29,15 +29,17 @@ from superset.connectors.sqla.models import SqlaTable from superset.databases.commands.importers.v1 import ImportDatabasesCommand from superset.datasets.commands.exceptions import DatasetNotFoundError from superset.datasets.commands.export import ExportDatasetsCommand -from superset.datasets.commands.importers.v1 import ImportDatasetsCommand +from superset.datasets.commands.importers import v0, v1 from superset.models.core import Database from superset.utils.core import get_example_database from tests.base_tests import SupersetTestCase from tests.fixtures.importexport import ( database_config, database_metadata_config, + dataset_cli_export, dataset_config, dataset_metadata_config, + dataset_ui_export, ) @@ -202,6 +204,78 @@ class TestExportDatasetsCommand(SupersetTestCase): class TestImportDatasetsCommand(SupersetTestCase): + def test_import_v0_dataset_cli_export(self): + num_datasets = db.session.query(SqlaTable).count() + + contents = { + "20201119_181105.yaml": yaml.safe_dump(dataset_cli_export), + } + command = v0.ImportDatasetsCommand(contents) + command.run() + + new_num_datasets = db.session.query(SqlaTable).count() + assert new_num_datasets == num_datasets + 1 + + dataset = ( + db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one() + ) + assert ( + dataset.params + == '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}' + ) + assert len(dataset.metrics) == 2 + assert dataset.main_dttm_col == "ds" + assert dataset.filter_select_enabled + assert [col.column_name for col in dataset.columns] == [ + "num_california", + "ds", + "state", + "gender", + "name", + "sum_boys", + "sum_girls", + "num", + ] + + db.session.delete(dataset) + db.session.commit() + + def test_import_v0_dataset_ui_export(self): + num_datasets = db.session.query(SqlaTable).count() + + contents = { + "20201119_181105.yaml": yaml.safe_dump(dataset_ui_export), + } + command = v0.ImportDatasetsCommand(contents) + command.run() + + new_num_datasets = db.session.query(SqlaTable).count() + assert new_num_datasets == num_datasets + 1 + + dataset = ( + db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one() + ) + assert ( + dataset.params + == '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}' + ) + assert len(dataset.metrics) == 2 + assert dataset.main_dttm_col == "ds" + assert dataset.filter_select_enabled + assert [col.column_name for col in dataset.columns] == [ + "num_california", + "ds", + "state", + "gender", + "name", + "sum_boys", + "sum_girls", + "num", + ] + + db.session.delete(dataset) + db.session.commit() + def test_import_v1_dataset(self): """Test that we can import a dataset""" contents = { @@ -209,7 +283,7 @@ class TestImportDatasetsCommand(SupersetTestCase): "databases/imported_database.yaml": yaml.safe_dump(database_config), "datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config), } - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) command.run() dataset = ( @@ -267,7 +341,7 @@ class TestImportDatasetsCommand(SupersetTestCase): "databases/imported_database.yaml": yaml.safe_dump(database_config), "datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config), } - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) command.run() command.run() dataset = ( @@ -285,7 +359,7 @@ class TestImportDatasetsCommand(SupersetTestCase): "databases/imported_database.yaml": yaml.safe_dump(database_config), "datasets/imported_dataset.yaml": yaml.safe_dump(new_config), } - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) command.run() dataset = ( db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one() @@ -305,7 +379,7 @@ class TestImportDatasetsCommand(SupersetTestCase): contents = { "datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config), } - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) with pytest.raises(IncorrectVersionError) as excinfo: command.run() assert str(excinfo.value) == "Missing metadata.yaml" @@ -318,14 +392,14 @@ class TestImportDatasetsCommand(SupersetTestCase): "timestamp": "2020-11-04T21:27:44.423819+00:00", } ) - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) with pytest.raises(IncorrectVersionError) as excinfo: command.run() assert str(excinfo.value) == "Must be equal to 1.0.0." # type should be SqlaTable contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config) - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) with pytest.raises(CommandInvalidError) as excinfo: command.run() assert str(excinfo.value) == "Error importing dataset" @@ -338,7 +412,7 @@ class TestImportDatasetsCommand(SupersetTestCase): del broken_config["database_name"] contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config) contents["databases/imported_database.yaml"] = yaml.safe_dump(broken_config) - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) with pytest.raises(CommandInvalidError) as excinfo: command.run() assert str(excinfo.value) == "Error importing dataset" @@ -369,10 +443,14 @@ class TestImportDatasetsCommand(SupersetTestCase): "datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config), "databases/imported_database.yaml": yaml.safe_dump(database_config), } - command = ImportDatasetsCommand(contents) + command = v1.ImportDatasetsCommand(contents) command.run() database = ( db.session.query(Database).filter_by(uuid=database_config["uuid"]).one() ) assert len(database.tables) == 1 + + db.session.delete(database.tables[0]) + db.session.delete(database) + db.session.commit() diff --git a/tests/fixtures/importexport.py b/tests/fixtures/importexport.py index 5f84088e4..638392308 100644 --- a/tests/fixtures/importexport.py +++ b/tests/fixtures/importexport.py @@ -14,10 +14,312 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +# pylint: disable=line-too-long -from typing import Any, Dict +from typing import Any, Dict, List -# example YAML files +# example V0 import/export format +dataset_ui_export: List[Dict[str, Any]] = [ + { + "columns": [ + { + "column_name": "num_california", + "expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END", + }, + {"column_name": "ds", "is_dttm": True, "type": "DATETIME"}, + {"column_name": "state", "type": "VARCHAR(10)"}, + {"column_name": "gender", "type": "VARCHAR(16)"}, + {"column_name": "name", "type": "VARCHAR(255)"}, + {"column_name": "sum_boys", "type": "BIGINT"}, + {"column_name": "sum_girls", "type": "BIGINT"}, + {"column_name": "num", "type": "BIGINT"}, + ], + "filter_select_enabled": True, + "main_dttm_col": "ds", + "metrics": [ + { + "expression": "COUNT(*)", + "metric_name": "count", + "metric_type": "count", + "verbose_name": "COUNT(*)", + }, + {"expression": "SUM(num)", "metric_name": "sum__num"}, + ], + "params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}', + "table_name": "birth_names_2", + } +] + +dataset_cli_export: Dict[str, Any] = { + "databases": [ + { + "allow_run_async": True, + "database_name": "examples", + "sqlalchemy_uri": "sqlite:////Users/beto/.superset/superset.db", + "tables": dataset_ui_export, + } + ] +} + +dashboard_export: Dict[str, Any] = { + "dashboards": [ + { + "__Dashboard__": { + "css": "", + "dashboard_title": "Births 2", + "description": None, + "json_metadata": '{"timed_refresh_immune_slices": [], "expanded_slices": {}, "refresh_frequency": 0, "default_filters": "{}", "color_scheme": null, "remote_id": 1}', + "position_json": '{"CHART--jvaBFZx78":{"children":[],"id":"CHART--jvaBFZx78","meta":{"chartId":83,"height":50,"sliceName":"Number of California Births","uuid":"c77bb4b3-09f4-4d9a-a9e2-66a627c64343","width":4},"parents":["ROOT_ID","GRID_ID","ROW-se_5H8KNiO"],"type":"CHART"},"DASHBOARD_VERSION_KEY":"v2","GRID_ID":{"children":["ROW-se_5H8KNiO"],"id":"GRID_ID","parents":["ROOT_ID"],"type":"GRID"},"HEADER_ID":{"id":"HEADER_ID","meta":{"text":"Births"},"type":"HEADER"},"ROOT_ID":{"children":["GRID_ID"],"id":"ROOT_ID","type":"ROOT"},"ROW-se_5H8KNiO":{"children":["CHART--jvaBFZx78"],"id":"ROW-se_5H8KNiO","meta":{"background":"BACKGROUND_TRANSPARENT"},"parents":["ROOT_ID","GRID_ID"],"type":"ROW"}}', + "slices": [ + { + "__Slice__": { + "cache_timeout": None, + "datasource_name": "birth_names_2", + "datasource_type": "table", + "id": 83, + "params": '{"adhoc_filters": [], "datasource": "3__table", "granularity_sqla": "ds", "header_font_size": 0.4, "metric": {"aggregate": "SUM", "column": {"column_name": "num_california", "expression": "CASE WHEN state = \'CA\' THEN num ELSE 0 END"}, "expressionType": "SIMPLE", "label": "SUM(num_california)"}, "queryFields": {"metric": "metrics"}, "slice_id": 83, "subheader_font_size": 0.15, "time_range": "100 years ago : now", "time_range_endpoints": ["unknown", "inclusive"], "url_params": {}, "viz_type": "big_number_total", "y_axis_format": "SMART_NUMBER", "remote_id": 83, "datasource_name": "birth_names_2", "schema": null, "database_name": "examples"}', + "slice_name": "Number of California Births", + "viz_type": "big_number_total", + } + } + ], + "slug": None, + } + } + ], + "datasources": [ + { + "__SqlaTable__": { + "cache_timeout": None, + "columns": [ + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "ds", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 332, + "is_active": True, + "is_dttm": True, + "python_date_format": None, + "table_id": 3, + "type": "DATETIME", + "uuid": "98e22f20-ed71-4483-b09d-31780ed1fc1b", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "gender", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 333, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "VARCHAR(16)", + "uuid": "08e08f02-fb81-4461-bba6-c8c8dfef0c02", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "name", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 334, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "VARCHAR(255)", + "uuid": "c67b14d9-fc4b-427d-a363-a53af015fb5e", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "num", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 335, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "BIGINT", + "uuid": "69835b93-7169-4a2c-baa7-c1c92f21d10a", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "state", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 336, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "VARCHAR(10)", + "uuid": "80003ad0-bdd0-48d3-ade3-8d1838e07d7a", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "sum_boys", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 337, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "BIGINT", + "uuid": "8373ed24-4d4e-4307-9eee-8deefeecbb57", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "column_name": "sum_girls", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 338, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": "BIGINT", + "uuid": "46f2de5f-c008-4024-a163-0b5c5f1d5580", + "verbose_name": None, + } + }, + { + "__TableColumn__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:32"}, + "column_name": "num_california", + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:32"}, + "description": None, + "expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END", + "filterable": True, + "groupby": True, + "id": 434, + "is_active": True, + "is_dttm": False, + "python_date_format": None, + "table_id": 3, + "type": None, + "uuid": "35e32aa6-be2b-4086-9c78-4ea3351ec079", + "verbose_name": None, + } + }, + ], + "database_id": 1000, + "default_endpoint": None, + "description": None, + "extra": None, + "fetch_values_predicate": None, + "filter_select_enabled": True, + "main_dttm_col": "ds", + "metrics": [ + { + "__SqlMetric__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "d3format": None, + "description": None, + "expression": "COUNT(*)", + "extra": None, + "id": 9, + "metric_name": "count", + "metric_type": "count", + "table_id": 3, + "uuid": "1042ef50-ebf9-4271-b44e-3aaa891f6c21", + "verbose_name": "COUNT(*)", + "warning_text": None, + } + }, + { + "__SqlMetric__": { + "changed_by_fk": None, + "changed_on": {"__datetime__": "2020-10-07T15:50:00"}, + "created_by_fk": None, + "created_on": {"__datetime__": "2020-10-07T15:50:00"}, + "d3format": None, + "description": None, + "expression": "SUM(num)", + "extra": None, + "id": 10, + "metric_name": "sum__num", + "metric_type": None, + "table_id": 3, + "uuid": "d807f208-e3c6-4b89-b790-41f521216ff6", + "verbose_name": None, + "warning_text": None, + } + }, + ], + "offset": 0, + "params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}', + "schema": None, + "sql": None, + "table_name": "birth_names_2", + "template_params": None, + } + } + ], +} + +# example V1 import/export format database_metadata_config: Dict[str, Any] = { "version": "1.0.0", "type": "Database",