refactor: [migration] convert iframe chart into dashboard markdown component (#10590)

* refactor: [migration] convert iframe chart into dashboard markdown component

* remove 3 viz_types

* fix comments

UPDATING.md

@@ -23,6 +23,8 @@ assists people when migrating to a new version.
 
 ## Next
 
+* [10590](https://github.com/apache/incubator-superset/pull/10590): Breaking change: this PR will convert iframe chart into dashboard markdown component, and remove all `iframe`, `separator`, and `markup` slices (and support) from Superset. If you have important data in those slices, please backup manually.
+
 * [10562](https://github.com/apache/incubator-superset/pull/10562): EMAIL_REPORTS_WEBDRIVER is deprecated use WEBDRIVER_TYPE instead.
 
 * [10567](https://github.com/apache/incubator-superset/pull/10567): Default WEBDRIVER_OPTION_ARGS are Chrome-specific. If you're using FF, should be `--headless` only

superset-frontend/src/dashboard/reducers/getInitialState.js

@@ -108,118 +108,116 @@ export default function (bootstrapData) {
   const sliceIds = new Set();
   dashboard.slices.forEach(slice => {
     const key = slice.slice_id;
-    if (['separator', 'markup'].indexOf(slice.form_data.viz_type) === -1) {
+    const form_data = {
-      const form_data = {
+      ...slice.form_data,
-        ...slice.form_data,
+      url_params: {
-        url_params: {
+        ...slice.form_data.url_params,
-          ...slice.form_data.url_params,
+        ...urlParams,
-          ...urlParams,
+      },
-        },
+    };
-      };
+    chartQueries[key] = {
-      chartQueries[key] = {
+      ...chart,
-        ...chart,
+      id: key,
-        id: key,
+      form_data,
-        form_data,
+      formData: applyDefaultFormData(form_data),
-        formData: applyDefaultFormData(form_data),
+    };
-      };
 
-      slices[key] = {
+    slices[key] = {
-        slice_id: key,
+      slice_id: key,
-        slice_url: slice.slice_url,
+      slice_url: slice.slice_url,
-        slice_name: slice.slice_name,
+      slice_name: slice.slice_name,
-        form_data: slice.form_data,
+      form_data: slice.form_data,
-        edit_url: slice.edit_url,
+      edit_url: slice.edit_url,
-        viz_type: slice.form_data.viz_type,
+      viz_type: slice.form_data.viz_type,
-        datasource: slice.form_data.datasource,
+      datasource: slice.form_data.datasource,
-        description: slice.description,
+      description: slice.description,
-        description_markeddown: slice.description_markeddown,
+      description_markeddown: slice.description_markeddown,
-        owners: slice.owners,
+      owners: slice.owners,
-        modified: slice.modified,
+      modified: slice.modified,
-        changed_on: new Date(slice.changed_on).getTime(),
+      changed_on: new Date(slice.changed_on).getTime(),
-      };
+    };
 
-      sliceIds.add(key);
+    sliceIds.add(key);
 
-      // if there are newly added slices from explore view, fill slices into 1 or more rows
+    // if there are newly added slices from explore view, fill slices into 1 or more rows
-      if (!chartIdToLayoutId[key] && layout[parentId]) {
+    if (!chartIdToLayoutId[key] && layout[parentId]) {
-        if (
+      if (
-          newSlicesContainerWidth === 0 ||
+        newSlicesContainerWidth === 0 ||
-          newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
+        newSlicesContainerWidth + GRID_DEFAULT_CHART_WIDTH > GRID_COLUMN_COUNT
-        ) {
+      ) {
-          newSlicesContainer = newComponentFactory(
+        newSlicesContainer = newComponentFactory(
-            ROW_TYPE,
+          ROW_TYPE,
-            (parent.parents || []).slice(),
+          (parent.parents || []).slice(),
-          );
-          layout[newSlicesContainer.id] = newSlicesContainer;
-          parent.children.push(newSlicesContainer.id);
-          newSlicesContainerWidth = 0;
-        }
-
-        const chartHolder = newComponentFactory(
-          CHART_TYPE,
-          {
-            chartId: slice.slice_id,
-          },
-          (newSlicesContainer.parents || []).slice(),
         );
-
+        layout[newSlicesContainer.id] = newSlicesContainer;
-        layout[chartHolder.id] = chartHolder;
+        parent.children.push(newSlicesContainer.id);
-        newSlicesContainer.children.push(chartHolder.id);
+        newSlicesContainerWidth = 0;
-        chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
-        newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
       }
 
-      // build DashboardFilters for interactive filter features
+      const chartHolder = newComponentFactory(
-      if (slice.form_data.viz_type === 'filter_box') {
+        CHART_TYPE,
-        const configs = getFilterConfigsFromFormdata(slice.form_data);
+        {
-        let columns = configs.columns;
+          chartId: slice.slice_id,
-        const labels = configs.labels;
+        },
-        if (preselectFilters[key]) {
+        (newSlicesContainer.parents || []).slice(),
-          Object.keys(columns).forEach(col => {
+      );
-            if (preselectFilters[key][col]) {
-              columns = {
-                ...columns,
-                [col]: preselectFilters[key][col],
-              };
-            }
-          });
-        }
 
-        const scopesByChartId = Object.keys(columns).reduce((map, column) => {
+      layout[chartHolder.id] = chartHolder;
-          const scopeSettings = {
+      newSlicesContainer.children.push(chartHolder.id);
-            ...filterScopes[key],
+      chartIdToLayoutId[chartHolder.meta.chartId] = chartHolder.id;
-          };
+      newSlicesContainerWidth += GRID_DEFAULT_CHART_WIDTH;
-          const { scope, immune } = {
+    }
-            ...DASHBOARD_FILTER_SCOPE_GLOBAL,
-            ...scopeSettings[column],
-          };
 
-          return {
+    // build DashboardFilters for interactive filter features
-            ...map,
+    if (slice.form_data.viz_type === 'filter_box') {
-            [column]: {
+      const configs = getFilterConfigsFromFormdata(slice.form_data);
-              scope,
+      let columns = configs.columns;
-              immune,
+      const labels = configs.labels;
-            },
+      if (preselectFilters[key]) {
-          };
+        Object.keys(columns).forEach(col => {
-        }, {});
+          if (preselectFilters[key][col]) {
+            columns = {
+              ...columns,
+              [col]: preselectFilters[key][col],
+            };
+          }
+        });
+      }
 
-        const componentId = chartIdToLayoutId[key];
+      const scopesByChartId = Object.keys(columns).reduce((map, column) => {
-        const directPathToFilter = (layout[componentId].parents || []).slice();
+        const scopeSettings = {
-        directPathToFilter.push(componentId);
+          ...filterScopes[key],
-        dashboardFilters[key] = {
-          ...dashboardFilter,
-          chartId: key,
-          componentId,
-          datasourceId: slice.form_data.datasource,
-          filterName: slice.slice_name,
-          directPathToFilter,
-          columns,
-          labels,
-          scopes: scopesByChartId,
-          isInstantFilter: !!slice.form_data.instant_filtering,
-          isDateFilter: Object.keys(columns).includes(TIME_RANGE),
         };
-      }
+        const { scope, immune } = {
+          ...DASHBOARD_FILTER_SCOPE_GLOBAL,
+          ...scopeSettings[column],
+        };
+
+        return {
+          ...map,
+          [column]: {
+            scope,
+            immune,
+          },
+        };
+      }, {});
+
+      const componentId = chartIdToLayoutId[key];
+      const directPathToFilter = (layout[componentId].parents || []).slice();
+      directPathToFilter.push(componentId);
+      dashboardFilters[key] = {
+        ...dashboardFilter,
+        chartId: key,
+        componentId,
+        datasourceId: slice.form_data.datasource,
+        filterName: slice.slice_name,
+        directPathToFilter,
+        columns,
+        labels,
+        scopes: scopesByChartId,
+        isInstantFilter: !!slice.form_data.instant_filtering,
+        isDateFilter: Object.keys(columns).includes(TIME_RANGE),
+      };
     }
 
     // sync layout names with current slice names in case a slice was edited

superset-frontend/src/explore/components/SaveModal.jsx

@@ -33,8 +33,6 @@ import { CreatableSelect } from 'src/components/Select/SupersetStyledSelect';
 import { t } from '@superset-ui/translation';
 import ReactMarkdown from 'react-markdown';
 
-import { EXPLORE_ONLY_VIZ_TYPE } from '../constants';
-
 const propTypes = {
   can_overwrite: PropTypes.bool,
   onHide: PropTypes.func.isRequired,
@@ -134,8 +132,6 @@ class SaveModal extends React.Component {
     this.setState({ alert: null });
   }
   render() {
-    const canNotSaveToDash =
-      EXPLORE_ONLY_VIZ_TYPE.indexOf(this.state.vizType) > -1;
     return (
       <Modal show onHide={this.props.onHide}>
         <Modal.Header closeButton>
@@ -225,9 +221,7 @@ class SaveModal extends React.Component {
               id="btn_modal_save_goto_dash"
               bsSize="sm"
               disabled={
-                canNotSaveToDash ||
+                !this.state.newSliceName || !this.state.newDashboardName
-                !this.state.newSliceName ||
-                !this.state.newDashboardName
               }
               onClick={this.saveOrOverwrite.bind(this, true)}
             >

superset-frontend/src/explore/components/controls/VizTypeControl.jsx

@@ -74,7 +74,6 @@ const DEFAULT_ORDER = [
   'line_multi',
   'treemap',
   'box_plot',
-  'separator',
   'sunburst',
   'sankey',
   'word_cloud',
@@ -85,7 +84,6 @@ const DEFAULT_ORDER = [
   'bubble',
   'deck_geojson',
   'horizon',
-  'markup',
   'deck_multi',
   'compare',
   'partition',
@@ -95,7 +93,6 @@ const DEFAULT_ORDER = [
   'world_map',
   'paired_ttest',
   'para',
-  'iframe',
   'country_map',
 ];
 

superset-frontend/src/explore/constants.js

@@ -71,8 +71,6 @@ export const sqlaAutoGeneratedMetricNameRegex = /^(sum|min|max|avg|count|count_d
 export const sqlaAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|AVG|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
 export const druidAutoGeneratedMetricRegex = /^(LONG|DOUBLE|FLOAT)?(SUM|MAX|MIN|COUNT)\([A-Z0-9_."]*\)$/i;
 
-export const EXPLORE_ONLY_VIZ_TYPE = ['separator', 'markup'];
-
 export const TIME_FILTER_LABELS = {
   time_range: t('Time Range'),
   granularity_sqla: t('Time Column'),

superset-frontend/src/visualizations/presets/MainPreset.js

@@ -29,9 +29,7 @@ import ForceDirectedChartPlugin from '@superset-ui/legacy-plugin-chart-force-dir
 import HeatmapChartPlugin from '@superset-ui/legacy-plugin-chart-heatmap';
 import HistogramChartPlugin from '@superset-ui/legacy-plugin-chart-histogram';
 import HorizonChartPlugin from '@superset-ui/legacy-plugin-chart-horizon';
-import IframeChartPlugin from '@superset-ui/legacy-plugin-chart-iframe';
 import MapBoxChartPlugin from '@superset-ui/legacy-plugin-chart-map-box';
-import MarkupChartPlugin from '@superset-ui/legacy-plugin-chart-markup';
 import PairedTTestChartPlugin from '@superset-ui/legacy-plugin-chart-paired-t-test';
 import ParallelCoordinatesChartPlugin from '@superset-ui/legacy-plugin-chart-parallel-coordinates';
 import PartitionChartPlugin from '@superset-ui/legacy-plugin-chart-partition';
@@ -87,12 +85,9 @@ export default class MainPreset extends Preset {
         new HeatmapChartPlugin().configure({ key: 'heatmap' }),
         new HistogramChartPlugin().configure({ key: 'histogram' }),
         new HorizonChartPlugin().configure({ key: 'horizon' }),
-        new IframeChartPlugin().configure({ key: 'iframe' }),
         new LineChartPlugin().configure({ key: 'line' }),
         new LineMultiChartPlugin().configure({ key: 'line_multi' }),
         new MapBoxChartPlugin().configure({ key: 'mapbox' }),
-        new MarkupChartPlugin().configure({ key: 'markup' }),
-        new MarkupChartPlugin().configure({ key: 'separator' }),
         new PairedTTestChartPlugin().configure({ key: 'paired_ttest' }),
         new ParallelCoordinatesChartPlugin().configure({ key: 'para' }),
         new PartitionChartPlugin().configure({ key: 'partition' }),

superset/migrations/versions/978245563a02_migrate_iframe_to_dash_markdown.py

@@ -0,0 +1,198 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Migrate iframe in dashboard to markdown component
+
+Revision ID: 978245563a02
+Revises: f2672aa8350a
+Create Date: 2020-08-12 00:24:39.617899
+
+"""
+import collections
+import json
+import logging
+import uuid
+from collections import defaultdict
+
+from alembic import op
+from sqlalchemy import and_, Column, ForeignKey, Integer, String, Table, Text
+from sqlalchemy.ext.declarative import declarative_base
+from sqlalchemy.orm import relationship
+
+from superset import db
+
+# revision identifiers, used by Alembic.
+revision = "978245563a02"
+down_revision = "f2672aa8350a"
+
+Base = declarative_base()
+
+
+class Slice(Base):
+    """Declarative class to do query in upgrade"""
+
+    __tablename__ = "slices"
+    id = Column(Integer, primary_key=True)
+    params = Column(Text)
+    viz_type = Column(String(250))
+
+
+dashboard_slices = Table(
+    "dashboard_slices",
+    Base.metadata,
+    Column("id", Integer, primary_key=True),
+    Column("dashboard_id", Integer, ForeignKey("dashboards.id")),
+    Column("slice_id", Integer, ForeignKey("slices.id")),
+)
+
+slice_user = Table(
+    "slice_user",
+    Base.metadata,
+    Column("id", Integer, primary_key=True),
+    Column("user_id", Integer, ForeignKey("ab_user.id")),
+    Column("slice_id", Integer, ForeignKey("slices.id")),
+)
+
+
+class Dashboard(Base):
+    __tablename__ = "dashboards"
+    id = Column(Integer, primary_key=True)
+    position_json = Column(Text)
+    slices = relationship("Slice", secondary=dashboard_slices, backref="dashboards")
+
+
+def create_new_markdown_component(chart_position, url):
+    return {
+        "type": "MARKDOWN",
+        "id": "MARKDOWN-{}".format(uuid.uuid4().hex[:8]),
+        "children": [],
+        "parents": chart_position["parents"],
+        "meta": {
+            "width": chart_position["meta"]["width"],
+            "height": chart_position["meta"]["height"],
+            "code": f'<iframe src="{url}" width="100%" height="100%"></iframe>',
+        },
+    }
+
+
+def upgrade():
+    bind = op.get_bind()
+    session = db.Session(bind=bind)
+
+    dash_to_migrate = defaultdict(list)
+    iframe_urls = defaultdict(list)
+
+    try:
+        # find iframe viz_type and its url
+        iframes = session.query(Slice).filter_by(viz_type="iframe").all()
+        iframe_ids = [slc.id for slc in iframes]
+
+        for iframe in iframes:
+            iframe_params = json.loads(iframe.params or "{}")
+            url = iframe_params.get("url")
+            iframe_urls[iframe.id] = url
+
+        # find iframe viz_type that used in dashboard
+        dash_slc = (
+            session.query(dashboard_slices)
+            .filter(dashboard_slices.c.slice_id.in_(iframe_ids))
+            .all()
+        )
+        for entry in dash_slc:
+            dash_to_migrate[entry.dashboard_id].append(entry.slice_id)
+        dashboard_ids = list(dash_to_migrate.keys())
+
+        # replace iframe in dashboard metadata
+        dashboards = (
+            session.query(Dashboard).filter(Dashboard.id.in_(dashboard_ids)).all()
+        )
+        for i, dashboard in enumerate(dashboards):
+            print(
+                f"scanning dashboard ({i + 1}/{len(dashboards)}) dashboard: {dashboard.id} >>>>"
+            )
+
+            # remove iframe slices from dashboard
+            dashboard.slices = [
+                slc for slc in dashboard.slices if slc.id not in iframe_ids
+            ]
+
+            # find iframe chart position in metadata
+            # and replace it with markdown component
+            position_dict = json.loads(dashboard.position_json or "{}")
+            for key, chart_position in position_dict.items():
+                if (
+                    chart_position
+                    and isinstance(chart_position, dict)
+                    and chart_position["type"] == "CHART"
+                    and chart_position["meta"]
+                    and chart_position["meta"]["chartId"] in iframe_ids
+                ):
+                    iframe_id = chart_position["meta"]["chartId"]
+                    # make new markdown component
+                    markdown = create_new_markdown_component(
+                        chart_position, iframe_urls[iframe_id]
+                    )
+                    position_dict.pop(key)
+                    position_dict[markdown["id"]] = markdown
+
+                    # add markdown to layout tree
+                    parent_id = markdown["parents"][-1]
+                    children = position_dict[parent_id]["children"]
+                    children.remove(key)
+                    children.append(markdown["id"])
+
+                    dashboard.position_json = json.dumps(
+                        position_dict,
+                        indent=None,
+                        separators=(",", ":"),
+                        sort_keys=True,
+                    )
+                    session.merge(dashboard)
+
+        # remove iframe, separator and markup charts
+        slices_to_remove = (
+            session.query(Slice)
+            .filter(Slice.viz_type.in_(["iframe", "separator", "markup"]))
+            .all()
+        )
+        slices_ids = [slc.id for slc in slices_to_remove]
+
+        # remove dependencies first
+        session.query(dashboard_slices).filter(
+            dashboard_slices.c.slice_id.in_(slices_ids)
+        ).delete(synchronize_session=False)
+
+        session.query(slice_user).filter(slice_user.c.slice_id.in_(slices_ids)).delete(
+            synchronize_session=False
+        )
+
+        # remove slices
+        session.query(Slice).filter(Slice.id.in_(slices_ids)).delete(
+            synchronize_session=False
+        )
+
+    except Exception as ex:
+        logging.exception(f"dashboard {dashboard.id} has error: {ex}")
+
+    session.commit()
+    session.close()
+
+
+def downgrade():
+    # note: this upgrade is irreversible.
+    # this migration removed all iframe, separator, and markup type slices,
+    # and Superset will not support these 3 viz_type anymore.
+    pass

superset/migrations/versions/f80a3b88324b_.py

@@ -0,0 +1,38 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""empty message
+
+Revision ID: f80a3b88324b
+Revises: ('978245563a02', 'f120347acb39')
+Create Date: 2020-08-12 15:47:56.580191
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = "f80a3b88324b"
+down_revision = ("978245563a02", "f120347acb39")
+
+import sqlalchemy as sa
+from alembic import op
+
+
+def upgrade():
+    pass
+
+
+def downgrade():
+    pass

superset/tasks/slack_util.py

@@ -26,7 +26,7 @@ from slack.web.slack_response import SlackResponse
 from superset import app
 
 # Globals
-config = app.config  # type: ignore
+config = app.config
 logger = logging.getLogger("tasks.slack_util")
 
 

superset/viz.py

@@ -2066,25 +2066,6 @@ class FilterBoxViz(BaseViz):
         return d
 
 
-class IFrameViz(BaseViz):
-
-    """You can squeeze just about anything in this iFrame component"""
-
-    viz_type = "iframe"
-    verbose_name = _("iFrame")
-    credits = 'a <a href="https://github.com/airbnb/superset">Superset</a> original'
-    is_timeseries = False
-
-    def query_obj(self) -> QueryObjectDict:
-        return {}
-
-    def get_df(self, query_obj: Optional[QueryObjectDict] = None) -> pd.DataFrame:
-        return pd.DataFrame()
-
-    def get_data(self, df: pd.DataFrame) -> VizData:
-        return {"iframe": True}
-
-
 class ParallelCoordinatesViz(BaseViz):
 
     """Interactive parallel coordinate implementation