iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: [migration] convert iframe chart into dashboard markdown component (#10590) 

* refactor: [migration] convert iframe chart into dashboard markdown component

* remove 3 viz_types

* fix comments
This commit is contained in:
Grace Guo 2020-08-16 14:43:30 -07:00 committed by GitHub
parent 03a62f15d8
commit ca9ca99510
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 340 additions and 139 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
UPDATING.md
View File

@ -23,6 +23,8 @@ assists people when migrating to a new version.
## Next
* [10590](https://github.com/apache/incubator-superset/pull/10590): Breaking change: this PR will convert iframe chart into dashboard markdown component, and remove all `iframe`, `separator`, and `markup` slices (and support) from Superset. If you have important data in those slices, please backup manually.
* [10562](https://github.com/apache/incubator-superset/pull/10562): EMAIL_REPORTS_WEBDRIVER is deprecated use WEBDRIVER_TYPE instead.
* [10567](https://github.com/apache/incubator-superset/pull/10567): Default WEBDRIVER_OPTION_ARGS are Chrome-specific. If you're using FF, should be `--headless` only

202
superset-frontend/src/dashboard/reducers/getInitialState.js
View File

@ -108,118 +108,116 @@ export default function (bootstrapData) {
const sliceIds = new Set();
dashboard.slices.forEach(slice => {
const key = slice.slice_id;
if (['separator', 'markup'].indexOf(slice.form_data.viz_type) === -1) {
const form_data = {
...slice.form_data,
url_params: {
...slice.form_data.url_params,
...urlParams,
},
};
chartQueries[key] = {
...chart,
id: key,
form_data,
formData: applyDefaultFormData(form_data),
};
const form_data = {
...slice.form_data,
url_params: {
...slice.form_data.url_params,
...urlParams,
},
};
chartQueries[key] = {
...chart,
id: key,
form_data,
formData: applyDefaultFormData(form_data),
};
slices[key] = {
slice_id: key,
slice_url: slice.slice_url,
slice_name: slice.slice_name,
form_data: slice.form_data,
edit_url: slice.edit_url,
viz_type: slice.form_data.viz_type,
datasource: slice.form_data.datasource,
description: slice.description,
description_markeddown: slice.description_markeddown,
owners: slice.owners,
modified: slice.modified,
changed_on: new Date(slice.changed_on).getTime(),
};
slices[key] = {
slice_id: key,
slice_url: slice.slice_url,
slice_name: slice.slice_name,
form_data: slice.form_data,
edit_url: slice.edit_url,
viz_type: slice.form_data.viz_type,
datasource: slice.form_data.datasource,
description: slice.description,
description_markeddown: slice.description_markeddown,
owners: slice.owners,
modified: slice.modified,
changed_on: new Date(slice.changed_on).getTime(),
};
sliceIds.add(key);
sliceIds.add(key);
// if there are newly added slices from explore view, fill slices into 1 or more rows
if (!chartIdToLayoutId[key] && layout[parentId]) {
if (
newSlicesContainerWidth === 0 ||
newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
) {
newSlicesContainer = newComponentFactory(
ROW_TYPE,
(parent.parents || []).slice(),
);
layout[newSlicesContainer.id] = newSlicesContainer;
parent.children.push(newSlicesContainer.id);
newSlicesContainerWidth = 0;
}
const chartHolder = newComponentFactory(
CHART_TYPE,
{
chartId: slice.slice_id,
},
(newSlicesContainer.parents || []).slice(),
// if there are newly added slices from explore view, fill slices into 1 or more rows
if (!chartIdToLayoutId[key] && layout[parentId]) {
if (
newSlicesContainerWidth === 0 ||
newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
) {
newSlicesContainer = newComponentFactory(
ROW_TYPE,
(parent.parents || []).slice(),
);
layout[chartHolder.id] = chartHolder;
newSlicesContainer.children.push(chartHolder.id);
chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
layout[newSlicesContainer.id] = newSlicesContainer;
parent.children.push(newSlicesContainer.id);
newSlicesContainerWidth = 0;
}
// build DashboardFilters for interactive filter features
if (slice.form_data.viz_type === 'filter_box') {
const configs = getFilterConfigsFromFormdata(slice.form_data);
let columns = configs.columns;
const labels = configs.labels;
if (preselectFilters[key]) {
Object.keys(columns).forEach(col => {
if (preselectFilters[key][col]) {
columns = {
...columns,
[col]: preselectFilters[key][col],
};
}
});
}
const chartHolder = newComponentFactory(
CHART_TYPE,
{
chartId: slice.slice_id,
},
(newSlicesContainer.parents || []).slice(),
);
const scopesByChartId = Object.keys(columns).reduce((map, column) => {
const scopeSettings = {
...filterScopes[key],
};
const { scope, immune } = {
...DASHBOARD_FILTER_SCOPE_GLOBAL,
...scopeSettings[column],
};
layout[chartHolder.id] = chartHolder;
newSlicesContainer.children.push(chartHolder.id);
chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
}
return {
...map,
[column]: {
scope,
immune,
},
};
}, {});
// build DashboardFilters for interactive filter features
if (slice.form_data.viz_type === 'filter_box') {
const configs = getFilterConfigsFromFormdata(slice.form_data);
let columns = configs.columns;
const labels = configs.labels;
if (preselectFilters[key]) {
Object.keys(columns).forEach(col => {
if (preselectFilters[key][col]) {
columns = {
...columns,
[col]: preselectFilters[key][col],
};
}
});
}
const componentId = chartIdToLayoutId[key];
const directPathToFilter = (layout[componentId].parents || []).slice();
directPathToFilter.push(componentId);
dashboardFilters[key] = {
...dashboardFilter,
chartId: key,
componentId,
datasourceId: slice.form_data.datasource,
filterName: slice.slice_name,
directPathToFilter,
columns,
labels,
scopes: scopesByChartId,
isInstantFilter: !!slice.form_data.instant_filtering,
isDateFilter: Object.keys(columns).includes(TIME_RANGE),
const scopesByChartId = Object.keys(columns).reduce((map, column) => {
const scopeSettings = {
...filterScopes[key],
};
}
const { scope, immune } = {
...DASHBOARD_FILTER_SCOPE_GLOBAL,
...scopeSettings[column],
};
return {
...map,
[column]: {
scope,
immune,
},
};
}, {});
const componentId = chartIdToLayoutId[key];
const directPathToFilter = (layout[componentId].parents || []).slice();
directPathToFilter.push(componentId);
dashboardFilters[key] = {
...dashboardFilter,
chartId: key,
componentId,
datasourceId: slice.form_data.datasource,
filterName: slice.slice_name,
directPathToFilter,
columns,
labels,
scopes: scopesByChartId,
isInstantFilter: !!slice.form_data.instant_filtering,
isDateFilter: Object.keys(columns).includes(TIME_RANGE),
};
}
// sync layout names with current slice names in case a slice was edited

8
superset-frontend/src/explore/components/SaveModal.jsx
View File

@ -33,8 +33,6 @@ import { CreatableSelect } from 'src/components/Select/SupersetStyledSelect';
import { t } from '@superset-ui/translation';
import ReactMarkdown from 'react-markdown';
import { EXPLORE_ONLY_VIZ_TYPE } from '../constants';
const propTypes = {
can_overwrite: PropTypes.bool,
onHide: PropTypes.func.isRequired,
@ -134,8 +132,6 @@ class SaveModal extends React.Component {
this.setState({ alert: null });
}
render() {
const canNotSaveToDash =
EXPLORE_ONLY_VIZ_TYPE.indexOf(this.state.vizType) > -1;
return (
<Modal show onHide={this.props.onHide}>
<Modal.Header closeButton>
@ -225,9 +221,7 @@ class SaveModal extends React.Component {
id="btn_modal_save_goto_dash"
bsSize="sm"
disabled={
canNotSaveToDash ||
!this.state.newSliceName ||
!this.state.newDashboardName
!this.state.newSliceName || !this.state.newDashboardName
}
onClick={this.saveOrOverwrite.bind(this, true)}
>

3
superset-frontend/src/explore/components/controls/VizTypeControl.jsx
View File

@ -74,7 +74,6 @@ const DEFAULT_ORDER = [
'line_multi',
'treemap',
'box_plot',
'separator',
'sunburst',
'sankey',
'word_cloud',
@ -85,7 +84,6 @@ const DEFAULT_ORDER = [
'bubble',
'deck_geojson',
'horizon',
'markup',
'deck_multi',
'compare',
'partition',
@ -95,7 +93,6 @@ const DEFAULT_ORDER = [
'world_map',
'paired_ttest',
'para',
'iframe',
'country_map',
];

2
superset-frontend/src/explore/constants.js
View File

@ -71,8 +71,6 @@ export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_d
export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
export const druidAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
export const EXPLORE_ONLY_VIZ_TYPE = ['separator', 'markup'];
export const TIME_FILTER_LABELS = {
time_range: t('Time Range'),
granularity_sqla: t('Time Column'),

5
superset-frontend/src/visualizations/presets/MainPreset.js
View File

@ -29,9 +29,7 @@ import ForceDirectedChartPlugin from '@superset-ui/legacy-plugin-chart-force-dir
import HeatmapChartPlugin from '@superset-ui/legacy-plugin-chart-heatmap';
import HistogramChartPlugin from '@superset-ui/legacy-plugin-chart-histogram';
import HorizonChartPlugin from '@superset-ui/legacy-plugin-chart-horizon';
import IframeChartPlugin from '@superset-ui/legacy-plugin-chart-iframe';
import MapBoxChartPlugin from '@superset-ui/legacy-plugin-chart-map-box';
import MarkupChartPlugin from '@superset-ui/legacy-plugin-chart-markup';
import PairedTTestChartPlugin from '@superset-ui/legacy-plugin-chart-paired-t-test';
import ParallelCoordinatesChartPlugin from '@superset-ui/legacy-plugin-chart-parallel-coordinates';
import PartitionChartPlugin from '@superset-ui/legacy-plugin-chart-partition';
@ -87,12 +85,9 @@ export default class MainPreset extends Preset {
new HeatmapChartPlugin().configure({ key: 'heatmap' }),
new HistogramChartPlugin().configure({ key: 'histogram' }),
new HorizonChartPlugin().configure({ key: 'horizon' }),
new IframeChartPlugin().configure({ key: 'iframe' }),
new LineChartPlugin().configure({ key: 'line' }),
new LineMultiChartPlugin().configure({ key: 'line_multi' }),
new MapBoxChartPlugin().configure({ key: 'mapbox' }),
new MarkupChartPlugin().configure({ key: 'markup' }),
new MarkupChartPlugin().configure({ key: 'separator' }),
new PairedTTestChartPlugin().configure({ key: 'paired_ttest' }),
new ParallelCoordinatesChartPlugin().configure({ key: 'para' }),
new PartitionChartPlugin().configure({ key: 'partition' }),

198
superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py Normal file
View File

@ -0,0 +1,198 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""Migrate iframe in dashboard to markdown component
Revision ID: 978245563a02
Revises: f2672aa8350a
Create Date: 2020-08-12 00:24:39.617899
"""
import collections
import json
import logging
import uuid
from collections import defaultdict
from alembic import op
from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import relationship
from superset import db
# revision identifiers, used by Alembic.
revision = "978245563a02"
down_revision = "f2672aa8350a"
Base = declarative_base()
class Slice(Base):
"""Declarative class to do query in upgrade"""
__tablename__ = "slices"
id = Column(Integer, primary_key=True)
params = Column(Text)
viz_type = Column(String(250))
dashboard_slices = Table(
"dashboard_slices",
Base.metadata,
Column("id", Integer, primary_key=True),
Column("dashboard_id", Integer, ForeignKey("dashboards.id")),
Column("slice_id", Integer, ForeignKey("slices.id")),
)
slice_user = Table(
"slice_user",
Base.metadata,
Column("id", Integer, primary_key=True),
Column("user_id", Integer, ForeignKey("ab_user.id")),
Column("slice_id", Integer, ForeignKey("slices.id")),
)
class Dashboard(Base):
__tablename__ = "dashboards"
id = Column(Integer, primary_key=True)
position_json = Column(Text)
slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards")
def create_new_markdown_component(chart_position, url):
return {
"type": "MARKDOWN",
"id": "MARKDOWN-{}".format(uuid.uuid4().hex[:8]),
"children": [],
"parents": chart_position["parents"],
"meta": {
"width": chart_position["meta"]["width"],
"height": chart_position["meta"]["height"],
"code": f'<iframe src="{url}" width="100%" height="100%"></iframe>',
},
}
def upgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
dash_to_migrate = defaultdict(list)
iframe_urls = defaultdict(list)
try:
# find iframe viz_type and its url
iframes = session.query(Slice).filter_by(viz_type="iframe").all()
iframe_ids = [slc.id for slc in iframes]
for iframe in iframes:
iframe_params = json.loads(iframe.params or "{}")
url = iframe_params.get("url")
iframe_urls[iframe.id] = url
# find iframe viz_type that used in dashboard
dash_slc = (
session.query(dashboard_slices)
.filter(dashboard_slices.c.slice_id.in_(iframe_ids))
.all()
)
for entry in dash_slc:
dash_to_migrate[entry.dashboard_id].append(entry.slice_id)
dashboard_ids = list(dash_to_migrate.keys())
# replace iframe in dashboard metadata
dashboards = (
session.query(Dashboard).filter(Dashboard.id.in_(dashboard_ids)).all()
)
for i, dashboard in enumerate(dashboards):
print(
f"scanning dashboard ({i + 1}/{len(dashboards)}) dashboard: {dashboard.id} >>>>"
)
# remove iframe slices from dashboard
dashboard.slices = [
slc for slc in dashboard.slices if slc.id not in iframe_ids
]
# find iframe chart position in metadata
# and replace it with markdown component
position_dict = json.loads(dashboard.position_json or "{}")
for key, chart_position in position_dict.items():
if (
chart_position
and isinstance(chart_position, dict)
and chart_position["type"] == "CHART"
and chart_position["meta"]
and chart_position["meta"]["chartId"] in iframe_ids
):
iframe_id = chart_position["meta"]["chartId"]
# make new markdown component
markdown = create_new_markdown_component(
chart_position, iframe_urls[iframe_id]
)
position_dict.pop(key)
position_dict[markdown["id"]] = markdown
# add markdown to layout tree
parent_id = markdown["parents"][-1]
children = position_dict[parent_id]["children"]
children.remove(key)
children.append(markdown["id"])
dashboard.position_json = json.dumps(
position_dict,
indent=None,
separators=(",", ":"),
sort_keys=True,
)
session.merge(dashboard)
# remove iframe, separator and markup charts
slices_to_remove = (
session.query(Slice)
.filter(Slice.viz_type.in_(["iframe", "separator", "markup"]))
.all()
)
slices_ids = [slc.id for slc in slices_to_remove]
# remove dependencies first
session.query(dashboard_slices).filter(
dashboard_slices.c.slice_id.in_(slices_ids)
).delete(synchronize_session=False)
session.query(slice_user).filter(slice_user.c.slice_id.in_(slices_ids)).delete(
synchronize_session=False
)
# remove slices
session.query(Slice).filter(Slice.id.in_(slices_ids)).delete(
synchronize_session=False
)
except Exception as ex:
logging.exception(f"dashboard {dashboard.id} has error: {ex}")
session.commit()
session.close()
def downgrade():
# note: this upgrade is irreversible.
# this migration removed all iframe, separator, and markup type slices,
# and Superset will not support these 3 viz_type anymore.
pass

38
superset/migrations/versions/f80a3b88324b_.py Normal file
View File

@ -0,0 +1,38 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
"""empty message
Revision ID: f80a3b88324b
Revises: ('978245563a02', 'f120347acb39')
Create Date: 2020-08-12 15:47:56.580191
"""
# revision identifiers, used by Alembic.
revision = "f80a3b88324b"
down_revision = ("978245563a02", "f120347acb39")
import sqlalchemy as sa
from alembic import op
def upgrade():
pass
def downgrade():
pass

2
superset/tasks/slack_util.py
View File

@ -26,7 +26,7 @@ from slack.web.slack_response import SlackResponse
from superset import app
# Globals
config = app.config # type: ignore
config = app.config
logger = logging.getLogger("tasks.slack_util")

19
superset/viz.py
View File

@ -2066,25 +2066,6 @@ class FilterBoxViz(BaseViz):
return d
class IFrameViz(BaseViz):
"""You can squeeze just about anything in this iFrame component"""
viz_type = "iframe"
verbose_name = _("iFrame")
credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
is_timeseries = False
def query_obj(self) -> QueryObjectDict:
return {}
def get_df(self, query_obj: Optional[QueryObjectDict] = None) -> pd.DataFrame:
return pd.DataFrame()
def get_data(self, df: pd.DataFrame) -> VizData:
return {"iframe": True}
class ParallelCoordinatesViz(BaseViz):
"""Interactive parallel coordinate implementation