From 02b3fb1d634f1e6d287060af902024c82b548c3b Mon Sep 17 00:00:00 2001 From: Bogdan Date: Fri, 10 Apr 2020 14:46:46 -0700 Subject: [PATCH] Add visualization flow to the CTA queries (#9370) * Visualize after running Ctas query (#4) * Visualize after running Ctas query. Stub for table viz Work on the viz Add JS functionality for Ctas viz Add test for the table viz flow Fix test Add js test for the ctas viz * Fix tests * Resolve comments * Leverate tmpSchema in the query object * Fix i18n string Co-authored-by: bogdan kyryliuk --- .../src/SqlLab/actions/sqlLab.js | 20 +++ .../components/ExploreCtasResultsButton.jsx | 131 ++++++++++++++++++ .../src/SqlLab/components/ResultSet.jsx | 45 ++++-- .../sqllab/ExploreCtasResultsButton_spec.jsx | 78 +++++++++++ .../components/ExploreCtasResultsButton.jsx | 131 ++++++++++++++++++ superset/connectors/sqla/models.py | 1 + superset/connectors/sqla/views.py | 35 +---- superset/models/sql_lab.py | 1 + superset/security/manager.py | 1 + superset/views/base.py | 45 +++++- superset/views/core.py | 44 ++++++ tests/sqllab_tests.py | 16 +++ 12 files changed, 505 insertions(+), 43 deletions(-) create mode 100644 superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx create mode 100644 superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx create mode 100644 superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx diff --git a/superset-frontend/src/SqlLab/actions/sqlLab.js b/superset-frontend/src/SqlLab/actions/sqlLab.js index 9dad77321..eb514b2c5 100644 --- a/superset-frontend/src/SqlLab/actions/sqlLab.js +++ b/superset-frontend/src/SqlLab/actions/sqlLab.js @@ -1249,3 +1249,23 @@ export function createDatasource(vizOptions) { }); }; } + +export function createCtasDatasource(vizOptions) { + return dispatch => { + dispatch(createDatasourceStarted()); + return SupersetClient.post({ + endpoint: '/superset/get_or_create_table/', + postPayload: { data: vizOptions }, + }) + .then(({ json }) => { + dispatch(createDatasourceSuccess(json)); + + return json; + }) + .catch(() => { + const errorMsg = t('An error occurred while creating the data source'); + dispatch(createDatasourceFailed(errorMsg)); + return Promise.reject(new Error(errorMsg)); + }); + }; +} diff --git a/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx b/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx new file mode 100644 index 000000000..b90d351b3 --- /dev/null +++ b/superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { bindActionCreators } from 'redux'; +import { connect } from 'react-redux'; +import Dialog from 'react-bootstrap-dialog'; +import { t } from '@superset-ui/translation'; + +import { exportChart } from '../../explore/exploreUtils'; +import * as actions from '../actions/sqlLab'; +import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger'; +import Button from '../../components/Button'; + +const propTypes = { + actions: PropTypes.object.isRequired, + table: PropTypes.string.isRequired, + schema: PropTypes.string, + dbId: PropTypes.number.isRequired, + errorMessage: PropTypes.string, + templateParams: PropTypes.string, +}; +const defaultProps = { + vizRequest: {}, +}; + +class ExploreCtasResultsButton extends React.PureComponent { + constructor(props) { + super(props); + this.visualize = this.visualize.bind(this); + this.onClick = this.onClick.bind(this); + } + onClick() { + this.visualize(); + } + + buildVizOptions() { + return { + datasourceName: this.props.table, + schema: this.props.schema, + dbId: this.props.dbId, + templateParams: this.props.templateParams, + }; + } + visualize() { + this.props.actions + .createCtasDatasource(this.buildVizOptions()) + .then(data => { + const formData = { + datasource: `${data.table_id}__table`, + metrics: ['count'], + groupby: [], + viz_type: 'table', + since: '100 years ago', + all_columns: [], + row_limit: 1000, + }; + this.props.actions.addInfoToast( + t('Creating a data source and creating a new tab'), + ); + + // open new window for data visualization + exportChart(formData); + }) + .catch(() => { + this.props.actions.addDangerToast( + this.props.errorMessage || t('An error occurred'), + ); + }); + } + render() { + return ( + <> + + { + this.dialog = el; + }} + /> + + ); + } +} +ExploreCtasResultsButton.propTypes = propTypes; +ExploreCtasResultsButton.defaultProps = defaultProps; + +function mapStateToProps({ sqlLab, common }) { + return { + errorMessage: sqlLab.errorMessage, + timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null, + }; +} + +function mapDispatchToProps(dispatch) { + return { + actions: bindActionCreators(actions, dispatch), + }; +} + +export { ExploreCtasResultsButton }; +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ExploreCtasResultsButton); diff --git a/superset-frontend/src/SqlLab/components/ResultSet.jsx b/superset-frontend/src/SqlLab/components/ResultSet.jsx index b7655d5fd..cbe704e52 100644 --- a/superset-frontend/src/SqlLab/components/ResultSet.jsx +++ b/superset-frontend/src/SqlLab/components/ResultSet.jsx @@ -23,6 +23,7 @@ import shortid from 'shortid'; import { t } from '@superset-ui/translation'; import Loading from '../../components/Loading'; +import ExploreCtasResultsButton from './ExploreCtasResultsButton'; import ExploreResultsButton from './ExploreResultsButton'; import HighlightedSql from './HighlightedSql'; import FilterableTable from '../../components/FilterableTable/FilterableTable'; @@ -101,13 +102,13 @@ export default class ResultSet extends React.PureComponent { clearQueryResults(query) { this.props.actions.clearQueryResults(query); } - popSelectStar() { + popSelectStar(tmpSchema, tmpTable) { const qe = { id: shortid.generate(), - title: this.props.query.tempTable, + title: tmpTable, autorun: false, dbId: this.props.query.dbId, - sql: `SELECT * FROM ${this.props.query.tempTable}`, + sql: `SELECT * FROM ${tmpSchema}.${tmpTable}`, }; this.props.actions.addQueryEditor(qe); } @@ -216,18 +217,38 @@ export default class ResultSet extends React.PureComponent { ); } else if (query.state === 'success' && query.ctas) { + // Async queries + let tmpSchema = query.tempSchema; + let tmpTable = query.tempTableName; + // Sync queries, query.results.query contains the source of truth for them. + if (query.results && query.results.query) { + tmpTable = query.results.query.tempTable; + tmpSchema = query.results.query.tempSchema; + } return (
- {t('Table')} [{query.tempTable}] {t('was created')}{' '} -   - + {t('Table')} [ + + {tmpSchema}.{tmpTable} + + ] {t('was created')}   + + + +
); diff --git a/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx b/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx new file mode 100644 index 000000000..98c8c10cf --- /dev/null +++ b/superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx @@ -0,0 +1,78 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import configureStore from 'redux-mock-store'; +import thunk from 'redux-thunk'; + +import { shallow } from 'enzyme'; +import sinon from 'sinon'; + +import sqlLabReducer from '../../../src/SqlLab/reducers/index'; +import ExploreCtasResultsButton from '../../../src/SqlLab/components/ExploreCtasResultsButton'; +import Button from '../../../src/components/Button'; + +describe('ExploreCtasResultsButton', () => { + const middlewares = [thunk]; + const mockStore = configureStore(middlewares); + const initialState = { + sqlLab: { + ...sqlLabReducer(undefined, {}), + }, + common: { + conf: { SUPERSET_WEBSERVER_TIMEOUT: 45 }, + }, + }; + const store = mockStore(initialState); + const mockedProps = { + table: 'dummy_table', + schema: 'dummy_schema', + dbId: 123, + }; + const getExploreCtasResultsButtonWrapper = (props = mockedProps) => + shallow(, { + context: { store }, + }).dive(); + + it('renders', () => { + expect(React.isValidElement()).toBe(true); + }); + + it('renders with props', () => { + expect(React.isValidElement()).toBe(true); + }); + + it('renders a Button', () => { + const wrapper = getExploreCtasResultsButtonWrapper(); + expect(wrapper.find(Button)).toHaveLength(1); + }); + + describe('datasourceName', () => { + it('should build viz options', () => { + const wrapper = getExploreCtasResultsButtonWrapper(); + const spy = sinon.spy(wrapper.instance(), 'buildVizOptions'); + wrapper.instance().buildVizOptions(); + expect(spy.returnValues[0]).toEqual({ + schema: 'dummy_schema', + dbId: 123, + templateParams: undefined, + datasourceName: 'dummy_table', + }); + }); + }); +}); diff --git a/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx b/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx new file mode 100644 index 000000000..b90d351b3 --- /dev/null +++ b/superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React from 'react'; +import PropTypes from 'prop-types'; +import { bindActionCreators } from 'redux'; +import { connect } from 'react-redux'; +import Dialog from 'react-bootstrap-dialog'; +import { t } from '@superset-ui/translation'; + +import { exportChart } from '../../explore/exploreUtils'; +import * as actions from '../actions/sqlLab'; +import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger'; +import Button from '../../components/Button'; + +const propTypes = { + actions: PropTypes.object.isRequired, + table: PropTypes.string.isRequired, + schema: PropTypes.string, + dbId: PropTypes.number.isRequired, + errorMessage: PropTypes.string, + templateParams: PropTypes.string, +}; +const defaultProps = { + vizRequest: {}, +}; + +class ExploreCtasResultsButton extends React.PureComponent { + constructor(props) { + super(props); + this.visualize = this.visualize.bind(this); + this.onClick = this.onClick.bind(this); + } + onClick() { + this.visualize(); + } + + buildVizOptions() { + return { + datasourceName: this.props.table, + schema: this.props.schema, + dbId: this.props.dbId, + templateParams: this.props.templateParams, + }; + } + visualize() { + this.props.actions + .createCtasDatasource(this.buildVizOptions()) + .then(data => { + const formData = { + datasource: `${data.table_id}__table`, + metrics: ['count'], + groupby: [], + viz_type: 'table', + since: '100 years ago', + all_columns: [], + row_limit: 1000, + }; + this.props.actions.addInfoToast( + t('Creating a data source and creating a new tab'), + ); + + // open new window for data visualization + exportChart(formData); + }) + .catch(() => { + this.props.actions.addDangerToast( + this.props.errorMessage || t('An error occurred'), + ); + }); + } + render() { + return ( + <> + + { + this.dialog = el; + }} + /> + + ); + } +} +ExploreCtasResultsButton.propTypes = propTypes; +ExploreCtasResultsButton.defaultProps = defaultProps; + +function mapStateToProps({ sqlLab, common }) { + return { + errorMessage: sqlLab.errorMessage, + timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null, + }; +} + +function mapDispatchToProps(dispatch) { + return { + actions: bindActionCreators(actions, dispatch), + }; +} + +export { ExploreCtasResultsButton }; +export default connect( + mapStateToProps, + mapDispatchToProps, +)(ExploreCtasResultsButton); diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index 4d1b9f24d..642997a10 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -1132,6 +1132,7 @@ class SqlaTable(Model, BaseDatasource): if not self.main_dttm_col: self.main_dttm_col = any_date_col self.add_missing_metrics(metrics) + db.session.merge(self) if commit: db.session.commit() diff --git a/superset/connectors/sqla/views.py b/superset/connectors/sqla/views.py index b59bf3c8e..38f78d220 100644 --- a/superset/connectors/sqla/views.py +++ b/superset/connectors/sqla/views.py @@ -29,16 +29,17 @@ from flask_babel import gettext as __, lazy_gettext as _ from wtforms.ext.sqlalchemy.fields import QuerySelectField from wtforms.validators import Regexp -from superset import app, appbuilder, db, security_manager +from superset import app, db, security_manager from superset.connectors.base.views import DatasourceModelView from superset.constants import RouteMethod from superset.utils import core as utils from superset.views.base import ( + create_table_permissions, DatasourceFilter, DeleteMixin, - get_datasource_exist_error_msg, ListWidgetWithCheckboxes, SupersetModelView, + validate_sqlatable, YamlExportMixin, ) @@ -375,37 +376,11 @@ class TableModelView(DatasourceModelView, DeleteMixin, YamlExportMixin): } def pre_add(self, table): - with db.session.no_autoflush: - table_query = db.session.query(models.SqlaTable).filter( - models.SqlaTable.table_name == table.table_name, - models.SqlaTable.schema == table.schema, - models.SqlaTable.database_id == table.database.id, - ) - if db.session.query(table_query.exists()).scalar(): - raise Exception(get_datasource_exist_error_msg(table.full_name)) - - # Fail before adding if the table can't be found - try: - table.get_sqla_table_object() - except Exception as ex: - logger.exception(f"Got an error in pre_add for {table.name}") - raise Exception( - _( - "Table [{}] could not be found, " - "please double check your " - "database connection, schema, and " - "table name, error: {}" - ).format(table.name, str(ex)) - ) + validate_sqlatable(table) def post_add(self, table, flash_message=True): table.fetch_metadata() - security_manager.add_permission_view_menu("datasource_access", table.get_perm()) - if table.schema: - security_manager.add_permission_view_menu( - "schema_access", table.schema_perm - ) - + create_table_permissions(table) if flash_message: flash( _( diff --git a/superset/models/sql_lab.py b/superset/models/sql_lab.py index 8f0254df5..654bc2fb4 100644 --- a/superset/models/sql_lab.py +++ b/superset/models/sql_lab.py @@ -120,6 +120,7 @@ class Query(Model, ExtraJSONMixin): "startDttm": self.start_time, "state": self.status.lower(), "tab": self.tab_name, + "tempSchema": self.tmp_schema_name, "tempTable": self.tmp_table_name, "userId": self.user_id, "user": user_label(self.user), diff --git a/superset/security/manager.py b/superset/security/manager.py index fe39b1924..2d720965a 100644 --- a/superset/security/manager.py +++ b/superset/security/manager.py @@ -772,6 +772,7 @@ class SupersetSecurityManager(SecurityManager): "can_csv", "can_search_queries", "can_sqllab_viz", + "can_sqllab_table_viz", "can_sqllab", } or ( diff --git a/superset/views/base.py b/superset/views/base.py index 2c16212d0..33ca79148 100644 --- a/superset/views/base.py +++ b/superset/views/base.py @@ -35,7 +35,15 @@ from sqlalchemy import or_ from werkzeug.exceptions import HTTPException from wtforms.fields.core import Field, UnboundField -from superset import appbuilder, conf, db, get_feature_flags, security_manager +from superset import ( + app as superset_app, + appbuilder, + conf, + db, + get_feature_flags, + security_manager, +) +from superset.connectors.sqla import models from superset.exceptions import SupersetException, SupersetSecurityException from superset.translations.utils import get_language_pack from superset.utils import core as utils @@ -54,6 +62,9 @@ FRONTEND_CONF_KEYS = ( ) logger = logging.getLogger(__name__) +logger = logging.getLogger(__name__) +config = superset_app.config + def get_error_msg(): if conf.get("SHOW_STACKTRACE"): @@ -150,6 +161,38 @@ def get_datasource_exist_error_msg(full_name: str) -> str: return __("Datasource %(name)s already exists", name=full_name) +def validate_sqlatable(table: models.SqlaTable) -> None: + """Checks the table existence in the database.""" + with db.session.no_autoflush: + table_query = db.session.query(models.SqlaTable).filter( + models.SqlaTable.table_name == table.table_name, + models.SqlaTable.schema == table.schema, + models.SqlaTable.database_id == table.database.id, + ) + if db.session.query(table_query.exists()).scalar(): + raise Exception(get_datasource_exist_error_msg(table.full_name)) + + # Fail before adding if the table can't be found + try: + table.get_sqla_table_object() + except Exception as ex: + logger.exception(f"Got an error in pre_add for {table.name}") + raise Exception( + _( + "Table [%{table}s] could not be found, " + "please double check your " + "database connection, schema, and " + "table name, error: {}" + ).format(table.name, str(ex)) + ) + + +def create_table_permissions(table: models.SqlaTable) -> None: + security_manager.add_permission_view_menu("datasource_access", table.get_perm()) + if table.schema: + security_manager.add_permission_view_menu("schema_access", table.schema_perm) + + def get_user_roles() -> List[Role]: if g.user.is_anonymous: public_role = conf.get("AUTH_ROLE_PUBLIC") diff --git a/superset/views/core.py b/superset/views/core.py index f574d4141..587a7b94c 100755 --- a/superset/views/core.py +++ b/superset/views/core.py @@ -98,6 +98,7 @@ from .base import ( BaseSupersetView, check_ownership, common_bootstrap_payload, + create_table_permissions, CsvResponse, data_payload_response, DeleteMixin, @@ -108,6 +109,7 @@ from .base import ( json_error_response, json_success, SupersetModelView, + validate_sqlatable, ) from .utils import ( apply_display_max_row_limit, @@ -1955,6 +1957,48 @@ class Superset(BaseSupersetView): return json_error_response(utils.error_msg_from_exception(ex)) return Response(status=201) + @has_access + @expose("/get_or_create_table/", methods=["POST"]) + @event_logger.log_this + def sqllab_table_viz(self): + """ Gets or creates a table object with attributes passed to the API. + + It expects the json with params: + * datasourceName - e.g. table name, required + * dbId - database id, required + * schema - table schema, optional + * templateParams - params for the Jinja templating syntax, optional + :return: Response + """ + SqlaTable = ConnectorRegistry.sources["table"] + data = json.loads(request.form.get("data")) + table_name = data.get("datasourceName") + database_id = data.get("dbId") + table = ( + db.session.query(SqlaTable) + .filter_by(database_id=database_id, table_name=table_name) + .one_or_none() + ) + if not table: + # Create table if doesn't exist. + with db.session.no_autoflush: + table = SqlaTable(table_name=table_name, owners=[g.user]) + table.database_id = database_id + table.database = ( + db.session.query(models.Database).filter_by(id=database_id).one() + ) + table.schema = data.get("schema") + table.template_params = data.get("templateParams") + # needed for the table validation. + validate_sqlatable(table) + + db.session.add(table) + table.fetch_metadata() + create_table_permissions(table) + db.session.commit() + + return json_success(json.dumps({"table_id": table.id})) + @has_access @expose("/sqllab_viz/", methods=["POST"]) @event_logger.log_this diff --git a/tests/sqllab_tests.py b/tests/sqllab_tests.py index ad130a8ee..bbc63d292 100644 --- a/tests/sqllab_tests.py +++ b/tests/sqllab_tests.py @@ -332,6 +332,22 @@ class SqlLabTests(SupersetTestCase): table = db.session.query(SqlaTable).filter_by(id=table_id).one() self.assertEqual([owner.username for owner in table.owners], ["admin"]) + def test_sqllab_table_viz(self): + self.login("admin") + examples_dbid = get_example_database().id + payload = {"datasourceName": "ab_role", "columns": [], "dbId": examples_dbid} + + data = {"data": json.dumps(payload)} + resp = self.get_json_resp("/superset/get_or_create_table/", data=data) + self.assertIn("table_id", resp) + + # ensure owner is set correctly + table_id = resp["table_id"] + table = db.session.query(SqlaTable).filter_by(id=table_id).one() + self.assertEqual([owner.username for owner in table.owners], ["admin"]) + db.session.delete(table) + db.session.commit() + def test_sql_limit(self): self.login("admin") test_limit = 1