From b512502d724fb323828110ecba14e5b9570dc09c Mon Sep 17 00:00:00 2001 From: Maxim Sukharev Date: Thu, 18 Jul 2019 07:22:10 +0200 Subject: [PATCH] Remove unnecessary fields from dashboard exported json (#7879) * Remove unnecessary fields from dashboard exported json This commit makes export respect export_fields and doesn't export unnecessary relations (like users with passwords hashes) which are ignored during the import. * Allow to import dashboard without position_json In case charts were added from chart-edit page and wasn't re-ordered on the dashboard position_json field is empty and import was failing with error. * Update export/import tests --- superset/models/core.py | 37 +++++++++++++++++++++++++----------- tests/import_export_tests.py | 15 +++++++++++++++ 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/superset/models/core.py b/superset/models/core.py index 66f2d0e76..37a7592b6 100755 --- a/superset/models/core.py +++ b/superset/models/core.py @@ -617,7 +617,10 @@ class Dashboard(Model, AuditMixinNullable, ImportMixin): existing_dashboard = dash dashboard_to_import.id = None - alter_positions(dashboard_to_import, old_to_new_slc_id_dict) + # position_json can be empty for dashboards + # with charts added from chart-edit page and without re-arranging + if dashboard_to_import.position_json: + alter_positions(dashboard_to_import, old_to_new_slc_id_dict) dashboard_to_import.alter_params(import_time=import_time) if new_expanded_slices: dashboard_to_import.alter_params(expanded_slices=new_expanded_slices) @@ -658,37 +661,49 @@ class Dashboard(Model, AuditMixinNullable, ImportMixin): for dashboard_id in dashboard_ids: # make sure that dashboard_id is an integer dashboard_id = int(dashboard_id) - copied_dashboard = ( + dashboard = ( db.session.query(Dashboard) .options(subqueryload(Dashboard.slices)) .filter_by(id=dashboard_id) .first() ) - make_transient(copied_dashboard) - for slc in copied_dashboard.slices: - make_transient(slc) + # remove ids and relations (like owners, created by, slices, ...) + copied_dashboard = dashboard.copy() + for slc in dashboard.slices: datasource_ids.add((slc.datasource_id, slc.datasource_type)) + copied_slc = slc.copy() + # save original id into json + # we need it to update dashboard's json metadata on import + copied_slc.id = slc.id # add extra params for the import - slc.alter_params( + copied_slc.alter_params( remote_id=slc.id, datasource_name=slc.datasource.name, schema=slc.datasource.schema, database_name=slc.datasource.database.name, ) + # set slices without creating ORM relations + slices = copied_dashboard.__dict__.setdefault("slices", []) + slices.append(copied_slc) copied_dashboard.alter_params(remote_id=dashboard_id) copied_dashboards.append(copied_dashboard) eager_datasources = [] - for dashboard_id, dashboard_type in datasource_ids: + for datasource_id, datasource_type in datasource_ids: eager_datasource = ConnectorRegistry.get_eager_datasource( - db.session, dashboard_type, dashboard_id + db.session, datasource_type, datasource_id ) - eager_datasource.alter_params( + copied_datasource = eager_datasource.copy() + copied_datasource.alter_params( remote_id=eager_datasource.id, database_name=eager_datasource.database.name, ) - make_transient(eager_datasource) - eager_datasources.append(eager_datasource) + datasource_class = copied_datasource.__class__ + for field_name in datasource_class.export_children: + field_val = getattr(eager_datasource, field_name).copy() + # set children without creating ORM relations + copied_datasource.__dict__[field_name] = field_val + eager_datasources.append(copied_datasource) return json.dumps( {"dashboards": copied_dashboards, "datasources": eager_datasources}, diff --git a/tests/import_export_tests.py b/tests/import_export_tests.py index d5c36091e..03068a4ed 100644 --- a/tests/import_export_tests.py +++ b/tests/import_export_tests.py @@ -204,6 +204,18 @@ class ImportExportTests(SupersetTestCase): exp_params.pop(k) self.assertEquals(exp_params, actual_params) + def assert_only_exported_slc_fields(self, expected_dash, actual_dash): + """ only exported json has this params + imported/created dashboard has relationships to other models instead + """ + expected_slices = sorted(expected_dash.slices, key=lambda s: s.slice_name or "") + actual_slices = sorted(actual_dash.slices, key=lambda s: s.slice_name or "") + for e_slc, a_slc in zip(expected_slices, actual_slices): + params = a_slc.params_dict + self.assertEqual(e_slc.datasource.name, params["datasource_name"]) + self.assertEqual(e_slc.datasource.schema, params["schema"]) + self.assertEqual(e_slc.datasource.database.name, params["database_name"]) + def test_export_1_dashboard(self): self.login("admin") birth_dash = self.get_dash_by_slug("births") @@ -216,6 +228,7 @@ class ImportExportTests(SupersetTestCase): )["dashboards"] birth_dash = self.get_dash_by_slug("births") + self.assert_only_exported_slc_fields(birth_dash, exported_dashboards[0]) self.assert_dash_equals(birth_dash, exported_dashboards[0]) self.assertEquals( birth_dash.id, @@ -250,12 +263,14 @@ class ImportExportTests(SupersetTestCase): self.assertEquals(2, len(exported_dashboards)) birth_dash = self.get_dash_by_slug("births") + self.assert_only_exported_slc_fields(birth_dash, exported_dashboards[0]) self.assert_dash_equals(birth_dash, exported_dashboards[0]) self.assertEquals( birth_dash.id, json.loads(exported_dashboards[0].json_metadata)["remote_id"] ) world_health_dash = self.get_dash_by_slug("world_health") + self.assert_only_exported_slc_fields(world_health_dash, exported_dashboards[1]) self.assert_dash_equals(world_health_dash, exported_dashboards[1]) self.assertEquals( world_health_dash.id,