iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: add unit tests for the current import functionality (#11786) 

* chore: add unit tests for the current import functionality

* Improve comment

* Fix unit test
This commit is contained in:
Beto Dealmeida 2020-12-03 14:48:43 -08:00 committed by GitHub
parent 5b1939802d
commit e0288bf76b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 474 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
superset/cli.py
View File

@ -235,7 +235,9 @@ def refresh_druid(datasource: str, merge: bool) -> None:
)
def import_dashboards(path: str, recursive: bool, username: str) -> None:
"""Import dashboards from JSON"""
from superset.dashboards.commands.importers.v0 import ImportDashboardsCommand
from superset.dashboards.commands.importers.dispatcher import (
ImportDashboardsCommand,
)
path_object = Path(path)
files: List[Path] = []

33
superset/datasets/commands/importers/v0.py
View File

@ -14,6 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
import logging
from typing import Any, Callable, Dict, List, Optional
@ -301,9 +302,23 @@ class ImportDatasetsCommand(BaseCommand):
def run(self) -> None:
self.validate()
# TODO (betodealmeida): add rollback in case of error
for file_name, config in self._configs.items():
logger.info("Importing dataset from file %s", file_name)
import_from_dict(db.session, config, sync=self.sync)
if isinstance(config, dict):
import_from_dict(db.session, config, sync=self.sync)
else: # list
for dataset in config:
# UI exports don't have the database metadata, so we assume
# the DB exists and has the same name
params = json.loads(dataset["params"])
database = (
db.session.query(Database)
.filter_by(database_name=params["database_name"])
.one()
)
dataset["database_id"] = database.id
SqlaTable.import_from_dict(db.session, dataset, sync=self.sync)
def validate(self) -> None:
# ensure all files are YAML
@ -314,8 +329,18 @@ class ImportDatasetsCommand(BaseCommand):
logger.exception("Invalid YAML file")
raise IncorrectVersionError(f"{file_name} is not a valid YAML file")
# check for keys
if DATABASES_KEY not in config and DRUID_CLUSTERS_KEY not in config:
raise IncorrectVersionError(f"{file_name} has no valid keys")
# CLI export
if isinstance(config, dict):
# TODO (betodealmeida): validate with Marshmallow
if DATABASES_KEY not in config and DRUID_CLUSTERS_KEY not in config:
raise IncorrectVersionError(f"{file_name} has no valid keys")
# UI export
elif isinstance(config, list):
# TODO (betodealmeida): validate with Marshmallow
pass
else:
raise IncorrectVersionError(f"{file_name} is not a valid file")
self._configs[file_name] = config

57
tests/dashboards/commands_tests.py
View File

@ -25,14 +25,18 @@ import yaml
from superset import db, security_manager
from superset.commands.exceptions import CommandInvalidError
from superset.commands.importers.exceptions import IncorrectVersionError
from superset.connectors.sqla.models import SqlaTable
from superset.dashboards.commands.exceptions import DashboardNotFoundError
from superset.dashboards.commands.export import ExportDashboardsCommand
from superset.dashboards.commands.importers.v1 import ImportDashboardsCommand
from superset.dashboards.commands.importers import v0, v1
from superset.models.core import Database
from superset.models.dashboard import Dashboard
from superset.models.slice import Slice
from tests.base_tests import SupersetTestCase
from tests.fixtures.importexport import (
chart_config,
dashboard_config,
dashboard_export,
dashboard_metadata_config,
database_config,
dataset_config,
@ -205,6 +209,45 @@ class TestExportDashboardsCommand(SupersetTestCase):
class TestImportDashboardsCommand(SupersetTestCase):
def test_import_v0_dashboard_cli_export(self):
num_dashboards = db.session.query(Dashboard).count()
num_charts = db.session.query(Slice).count()
num_datasets = db.session.query(SqlaTable).count()
num_databases = db.session.query(Database).count()
contents = {
"20201119_181105.json": json.dumps(dashboard_export),
}
command = v0.ImportDashboardsCommand(contents)
command.run()
new_num_dashboards = db.session.query(Dashboard).count()
new_num_charts = db.session.query(Slice).count()
new_num_datasets = db.session.query(SqlaTable).count()
new_num_databases = db.session.query(Database).count()
assert new_num_dashboards == num_dashboards + 1
assert new_num_charts == num_charts + 1
assert new_num_datasets == num_datasets + 1
assert new_num_databases == num_databases
dashboard = (
db.session.query(Dashboard).filter_by(dashboard_title="Births 2").one()
)
assert len(dashboard.slices) == 1
chart = dashboard.slices[0]
assert chart.slice_name == "Number of California Births"
dataset = chart.table
assert dataset.table_name == "birth_names_2"
database = dataset.database
assert database.database_name == "examples"
db.session.delete(dashboard)
db.session.delete(chart)
db.session.delete(dataset)
db.session.commit()
def test_import_v1_dashboard(self):
"""Test that we can import a dashboard"""
contents = {
@ -214,7 +257,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
command.run()
dashboard = (
@ -296,7 +339,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
command.run()
command.run()
@ -325,7 +368,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
"charts/imported_chart.yaml": yaml.safe_dump(chart_config),
"dashboards/imported_dashboard.yaml": yaml.safe_dump(dashboard_config),
}
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
@ -338,14 +381,14 @@ class TestImportDashboardsCommand(SupersetTestCase):
"timestamp": "2020-11-04T21:27:44.423819+00:00",
}
)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be Database
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dashboard"
@ -358,7 +401,7 @@ class TestImportDashboardsCommand(SupersetTestCase):
del broken_config["table_name"]
contents["metadata.yaml"] = yaml.safe_dump(dashboard_metadata_config)
contents["datasets/imported_dataset.yaml"] = yaml.safe_dump(broken_config)
command = ImportDashboardsCommand(contents)
command = v1.ImportDashboardsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dashboard"

98
tests/datasets/commands_tests.py
View File

@ -14,7 +14,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=no-self-use, invalid-name
# pylint: disable=no-self-use, invalid-name, line-too-long
from operator import itemgetter
from unittest.mock import patch
@ -29,15 +29,17 @@ from superset.connectors.sqla.models import SqlaTable
from superset.databases.commands.importers.v1 import ImportDatabasesCommand
from superset.datasets.commands.exceptions import DatasetNotFoundError
from superset.datasets.commands.export import ExportDatasetsCommand
from superset.datasets.commands.importers.v1 import ImportDatasetsCommand
from superset.datasets.commands.importers import v0, v1
from superset.models.core import Database
from superset.utils.core import get_example_database
from tests.base_tests import SupersetTestCase
from tests.fixtures.importexport import (
database_config,
database_metadata_config,
dataset_cli_export,
dataset_config,
dataset_metadata_config,
dataset_ui_export,
)
@ -202,6 +204,78 @@ class TestExportDatasetsCommand(SupersetTestCase):
class TestImportDatasetsCommand(SupersetTestCase):
def test_import_v0_dataset_cli_export(self):
num_datasets = db.session.query(SqlaTable).count()
contents = {
"20201119_181105.yaml": yaml.safe_dump(dataset_cli_export),
}
command = v0.ImportDatasetsCommand(contents)
command.run()
new_num_datasets = db.session.query(SqlaTable).count()
assert new_num_datasets == num_datasets + 1
dataset = (
db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one()
)
assert (
dataset.params
== '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}'
)
assert len(dataset.metrics) == 2
assert dataset.main_dttm_col == "ds"
assert dataset.filter_select_enabled
assert [col.column_name for col in dataset.columns] == [
"num_california",
"ds",
"state",
"gender",
"name",
"sum_boys",
"sum_girls",
"num",
]
db.session.delete(dataset)
db.session.commit()
def test_import_v0_dataset_ui_export(self):
num_datasets = db.session.query(SqlaTable).count()
contents = {
"20201119_181105.yaml": yaml.safe_dump(dataset_ui_export),
}
command = v0.ImportDatasetsCommand(contents)
command.run()
new_num_datasets = db.session.query(SqlaTable).count()
assert new_num_datasets == num_datasets + 1
dataset = (
db.session.query(SqlaTable).filter_by(table_name="birth_names_2").one()
)
assert (
dataset.params
== '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}'
)
assert len(dataset.metrics) == 2
assert dataset.main_dttm_col == "ds"
assert dataset.filter_select_enabled
assert [col.column_name for col in dataset.columns] == [
"num_california",
"ds",
"state",
"gender",
"name",
"sum_boys",
"sum_girls",
"num",
]
db.session.delete(dataset)
db.session.commit()
def test_import_v1_dataset(self):
"""Test that we can import a dataset"""
contents = {
@ -209,7 +283,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
dataset = (
@ -267,7 +341,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
command.run()
dataset = (
@ -285,7 +359,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
"databases/imported_database.yaml": yaml.safe_dump(database_config),
"datasets/imported_dataset.yaml": yaml.safe_dump(new_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
dataset = (
db.session.query(SqlaTable).filter_by(uuid=dataset_config["uuid"]).one()
@ -305,7 +379,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
contents = {
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Missing metadata.yaml"
@ -318,14 +392,14 @@ class TestImportDatasetsCommand(SupersetTestCase):
"timestamp": "2020-11-04T21:27:44.423819+00:00",
}
)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(IncorrectVersionError) as excinfo:
command.run()
assert str(excinfo.value) == "Must be equal to 1.0.0."
# type should be SqlaTable
contents["metadata.yaml"] = yaml.safe_dump(database_metadata_config)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dataset"
@ -338,7 +412,7 @@ class TestImportDatasetsCommand(SupersetTestCase):
del broken_config["database_name"]
contents["metadata.yaml"] = yaml.safe_dump(dataset_metadata_config)
contents["databases/imported_database.yaml"] = yaml.safe_dump(broken_config)
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
with pytest.raises(CommandInvalidError) as excinfo:
command.run()
assert str(excinfo.value) == "Error importing dataset"
@ -369,10 +443,14 @@ class TestImportDatasetsCommand(SupersetTestCase):
"datasets/imported_dataset.yaml": yaml.safe_dump(dataset_config),
"databases/imported_database.yaml": yaml.safe_dump(database_config),
}
command = ImportDatasetsCommand(contents)
command = v1.ImportDatasetsCommand(contents)
command.run()
database = (
db.session.query(Database).filter_by(uuid=database_config["uuid"]).one()
)
assert len(database.tables) == 1
db.session.delete(database.tables[0])
db.session.delete(database)
db.session.commit()

306
tests/fixtures/importexport.py vendored
View File

@ -14,10 +14,312 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
# pylint: disable=line-too-long
from typing import Any, Dict
from typing import Any, Dict, List
# example YAML files
# example V0 import/export format
dataset_ui_export: List[Dict[str, Any]] = [
{
"columns": [
{
"column_name": "num_california",
"expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END",
},
{"column_name": "ds", "is_dttm": True, "type": "DATETIME"},
{"column_name": "state", "type": "VARCHAR(10)"},
{"column_name": "gender", "type": "VARCHAR(16)"},
{"column_name": "name", "type": "VARCHAR(255)"},
{"column_name": "sum_boys", "type": "BIGINT"},
{"column_name": "sum_girls", "type": "BIGINT"},
{"column_name": "num", "type": "BIGINT"},
],
"filter_select_enabled": True,
"main_dttm_col": "ds",
"metrics": [
{
"expression": "COUNT(*)",
"metric_name": "count",
"metric_type": "count",
"verbose_name": "COUNT(*)",
},
{"expression": "SUM(num)", "metric_name": "sum__num"},
],
"params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}',
"table_name": "birth_names_2",
}
]
dataset_cli_export: Dict[str, Any] = {
"databases": [
{
"allow_run_async": True,
"database_name": "examples",
"sqlalchemy_uri": "sqlite:////Users/beto/.superset/superset.db",
"tables": dataset_ui_export,
}
]
}
dashboard_export: Dict[str, Any] = {
"dashboards": [
{
"__Dashboard__": {
"css": "",
"dashboard_title": "Births 2",
"description": None,
"json_metadata": '{"timed_refresh_immune_slices": [], "expanded_slices": {}, "refresh_frequency": 0, "default_filters": "{}", "color_scheme": null, "remote_id": 1}',
"position_json": '{"CHART--jvaBFZx78":{"children":[],"id":"CHART--jvaBFZx78","meta":{"chartId":83,"height":50,"sliceName":"Number of California Births","uuid":"c77bb4b3-09f4-4d9a-a9e2-66a627c64343","width":4},"parents":["ROOT_ID","GRID_ID","ROW-se_5H8KNiO"],"type":"CHART"},"DASHBOARD_VERSION_KEY":"v2","GRID_ID":{"children":["ROW-se_5H8KNiO"],"id":"GRID_ID","parents":["ROOT_ID"],"type":"GRID"},"HEADER_ID":{"id":"HEADER_ID","meta":{"text":"Births"},"type":"HEADER"},"ROOT_ID":{"children":["GRID_ID"],"id":"ROOT_ID","type":"ROOT"},"ROW-se_5H8KNiO":{"children":["CHART--jvaBFZx78"],"id":"ROW-se_5H8KNiO","meta":{"background":"BACKGROUND_TRANSPARENT"},"parents":["ROOT_ID","GRID_ID"],"type":"ROW"}}',
"slices": [
{
"__Slice__": {
"cache_timeout": None,
"datasource_name": "birth_names_2",
"datasource_type": "table",
"id": 83,
"params": '{"adhoc_filters": [], "datasource": "3__table", "granularity_sqla": "ds", "header_font_size": 0.4, "metric": {"aggregate": "SUM", "column": {"column_name": "num_california", "expression": "CASE WHEN state = \'CA\' THEN num ELSE 0 END"}, "expressionType": "SIMPLE", "label": "SUM(num_california)"}, "queryFields": {"metric": "metrics"}, "slice_id": 83, "subheader_font_size": 0.15, "time_range": "100 years ago : now", "time_range_endpoints": ["unknown", "inclusive"], "url_params": {}, "viz_type": "big_number_total", "y_axis_format": "SMART_NUMBER", "remote_id": 83, "datasource_name": "birth_names_2", "schema": null, "database_name": "examples"}',
"slice_name": "Number of California Births",
"viz_type": "big_number_total",
}
}
],
"slug": None,
}
}
],
"datasources": [
{
"__SqlaTable__": {
"cache_timeout": None,
"columns": [
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "ds",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 332,
"is_active": True,
"is_dttm": True,
"python_date_format": None,
"table_id": 3,
"type": "DATETIME",
"uuid": "98e22f20-ed71-4483-b09d-31780ed1fc1b",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "gender",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 333,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(16)",
"uuid": "08e08f02-fb81-4461-bba6-c8c8dfef0c02",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "name",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(255)",
"uuid": "c67b14d9-fc4b-427d-a363-a53af015fb5e",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "num",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 335,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "69835b93-7169-4a2c-baa7-c1c92f21d10a",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "state",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 336,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "VARCHAR(10)",
"uuid": "80003ad0-bdd0-48d3-ade3-8d1838e07d7a",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "sum_boys",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 337,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "8373ed24-4d4e-4307-9eee-8deefeecbb57",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"column_name": "sum_girls",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 338,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": "BIGINT",
"uuid": "46f2de5f-c008-4024-a163-0b5c5f1d5580",
"verbose_name": None,
}
},
{
"__TableColumn__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:32"},
"column_name": "num_california",
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:32"},
"description": None,
"expression": "CASE WHEN state = 'CA' THEN num ELSE 0 END",
"filterable": True,
"groupby": True,
"id": 434,
"is_active": True,
"is_dttm": False,
"python_date_format": None,
"table_id": 3,
"type": None,
"uuid": "35e32aa6-be2b-4086-9c78-4ea3351ec079",
"verbose_name": None,
}
},
],
"database_id": 1000,
"default_endpoint": None,
"description": None,
"extra": None,
"fetch_values_predicate": None,
"filter_select_enabled": True,
"main_dttm_col": "ds",
"metrics": [
{
"__SqlMetric__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"d3format": None,
"description": None,
"expression": "COUNT(*)",
"extra": None,
"id": 9,
"metric_name": "count",
"metric_type": "count",
"table_id": 3,
"uuid": "1042ef50-ebf9-4271-b44e-3aaa891f6c21",
"verbose_name": "COUNT(*)",
"warning_text": None,
}
},
{
"__SqlMetric__": {
"changed_by_fk": None,
"changed_on": {"__datetime__": "2020-10-07T15:50:00"},
"created_by_fk": None,
"created_on": {"__datetime__": "2020-10-07T15:50:00"},
"d3format": None,
"description": None,
"expression": "SUM(num)",
"extra": None,
"id": 10,
"metric_name": "sum__num",
"metric_type": None,
"table_id": 3,
"uuid": "d807f208-e3c6-4b89-b790-41f521216ff6",
"verbose_name": None,
"warning_text": None,
}
},
],
"offset": 0,
"params": '{"remote_id": 3, "database_name": "examples", "import_time": 1604342885}',
"schema": None,
"sql": None,
"table_name": "birth_names_2",
"template_params": None,
}
}
],
}
# example V1 import/export format
database_metadata_config: Dict[str, Any] = {
"version": "1.0.0",
"type": "Database",