iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add visualization flow to the CTA queries (#9370) 

* Visualize after running Ctas query (#4)

* Visualize after running Ctas query.

Stub for table viz

Work on the viz

Add JS functionality for Ctas viz

Add test for the table viz flow

Fix test

Add js test for the ctas viz

* Fix tests

* Resolve comments

* Leverate tmpSchema in the query object

* Fix i18n string

Co-authored-by: bogdan kyryliuk <bogdankyryliuk@dropbox.com>
This commit is contained in:
Bogdan 2020-04-10 14:46:46 -07:00 committed by GitHub
parent a8ce3bccdf
commit 02b3fb1d63
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 505 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
superset-frontend/src/SqlLab/actions/sqlLab.js
View File

@ -1249,3 +1249,23 @@ export function createDatasource(vizOptions) {
});
};
}
export function createCtasDatasource(vizOptions) {
return dispatch => {
dispatch(createDatasourceStarted());
return SupersetClient.post({
endpoint: '/superset/get_or_create_table/',
postPayload: { data: vizOptions },
})
.then(({ json }) => {
dispatch(createDatasourceSuccess(json));
return json;
})
.catch(() => {
const errorMsg = t('An error occurred while creating the data source');
dispatch(createDatasourceFailed(errorMsg));
return Promise.reject(new Error(errorMsg));
});
};
}

131
superset-frontend/src/SqlLab/components/ExploreCtasResultsButton.jsx Normal file
View File

@ -0,0 +1,131 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import PropTypes from 'prop-types';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
import Dialog from 'react-bootstrap-dialog';
import { t } from '@superset-ui/translation';
import { exportChart } from '../../explore/exploreUtils';
import * as actions from '../actions/sqlLab';
import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger';
import Button from '../../components/Button';
const propTypes = {
actions: PropTypes.object.isRequired,
table: PropTypes.string.isRequired,
schema: PropTypes.string,
dbId: PropTypes.number.isRequired,
errorMessage: PropTypes.string,
templateParams: PropTypes.string,
};
const defaultProps = {
vizRequest: {},
};
class ExploreCtasResultsButton extends React.PureComponent {
constructor(props) {
super(props);
this.visualize = this.visualize.bind(this);
this.onClick = this.onClick.bind(this);
}
onClick() {
this.visualize();
}
buildVizOptions() {
return {
datasourceName: this.props.table,
schema: this.props.schema,
dbId: this.props.dbId,
templateParams: this.props.templateParams,
};
}
visualize() {
this.props.actions
.createCtasDatasource(this.buildVizOptions())
.then(data => {
const formData = {
datasource: `${data.table_id}__table`,
metrics: ['count'],
groupby: [],
viz_type: 'table',
since: '100 years ago',
all_columns: [],
row_limit: 1000,
};
this.props.actions.addInfoToast(
t('Creating a data source and creating a new tab'),
);
// open new window for data visualization
exportChart(formData);
})
.catch(() => {
this.props.actions.addDangerToast(
this.props.errorMessage || t('An error occurred'),
);
});
}
render() {
return (
<>
<Button
bsSize="small"
onClick={this.onClick}
tooltip={t('Explore the result set in the data exploration view')}
>
<InfoTooltipWithTrigger
icon="line-chart"
placement="top"
label="explore"
/>{' '}
{t('Explore')}
</Button>
<Dialog
ref={el => {
this.dialog = el;
}}
/>
</>
);
}
}
ExploreCtasResultsButton.propTypes = propTypes;
ExploreCtasResultsButton.defaultProps = defaultProps;
function mapStateToProps({ sqlLab, common }) {
return {
errorMessage: sqlLab.errorMessage,
timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null,
};
}
function mapDispatchToProps(dispatch) {
return {
actions: bindActionCreators(actions, dispatch),
};
}
export { ExploreCtasResultsButton };
export default connect(
mapStateToProps,
mapDispatchToProps,
)(ExploreCtasResultsButton);

45
superset-frontend/src/SqlLab/components/ResultSet.jsx
View File

@ -23,6 +23,7 @@ import shortid from 'shortid';
import { t } from '@superset-ui/translation';
import Loading from '../../components/Loading';
import ExploreCtasResultsButton from './ExploreCtasResultsButton';
import ExploreResultsButton from './ExploreResultsButton';
import HighlightedSql from './HighlightedSql';
import FilterableTable from '../../components/FilterableTable/FilterableTable';
@ -101,13 +102,13 @@ export default class ResultSet extends React.PureComponent {
clearQueryResults(query) {
this.props.actions.clearQueryResults(query);
}
popSelectStar() {
popSelectStar(tmpSchema, tmpTable) {
const qe = {
id: shortid.generate(),
title: this.props.query.tempTable,
title: tmpTable,
autorun: false,
dbId: this.props.query.dbId,
sql: `SELECT * FROM ${this.props.query.tempTable}`,
sql: `SELECT * FROM ${tmpSchema}.${tmpTable}`,
};
this.props.actions.addQueryEditor(qe);
}
@ -216,18 +217,38 @@ export default class ResultSet extends React.PureComponent {
</Alert>
);
} else if (query.state === 'success' && query.ctas) {
// Async queries
let tmpSchema = query.tempSchema;
let tmpTable = query.tempTableName;
// Sync queries, query.results.query contains the source of truth for them.
if (query.results && query.results.query) {
tmpTable = query.results.query.tempTable;
tmpSchema = query.results.query.tempSchema;
}
return (
<div>
<Alert bsStyle="info">
{t('Table')} [<strong>{query.tempTable}</strong>] {t('was created')}{' '}
&nbsp;
<Button
bsSize="small"
className="m-r-5"
onClick={this.popSelectStar}
>
{t('Query in a new tab')}
</Button>
{t('Table')} [
<strong>
{tmpSchema}.{tmpTable}
</strong>
] {t('was created')} &nbsp;
<ButtonGroup>
<Button
bsSize="small"
className="m-r-5"
onClick={() => this.popSelectStar(tmpSchema, tmpTable)}
>
{t('Query in a new tab')}
</Button>
<ExploreCtasResultsButton
table={tmpTable}
schema={tmpSchema}
dbId={query.dbId}
database={this.props.database}
actions={this.props.actions}
/>
</ButtonGroup>
</Alert>
</div>
);

78
superset/assets/spec/javascripts/sqllab/ExploreCtasResultsButton_spec.jsx Normal file
View File

@ -0,0 +1,78 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import configureStore from 'redux-mock-store';
import thunk from 'redux-thunk';
import { shallow } from 'enzyme';
import sinon from 'sinon';
import sqlLabReducer from '../../../src/SqlLab/reducers/index';
import ExploreCtasResultsButton from '../../../src/SqlLab/components/ExploreCtasResultsButton';
import Button from '../../../src/components/Button';
describe('ExploreCtasResultsButton', () => {
const middlewares = [thunk];
const mockStore = configureStore(middlewares);
const initialState = {
sqlLab: {
...sqlLabReducer(undefined, {}),
},
common: {
conf: { SUPERSET_WEBSERVER_TIMEOUT: 45 },
},
};
const store = mockStore(initialState);
const mockedProps = {
table: 'dummy_table',
schema: 'dummy_schema',
dbId: 123,
};
const getExploreCtasResultsButtonWrapper = (props = mockedProps) =>
shallow(<ExploreCtasResultsButton {...props} />, {
context: { store },
}).dive();
it('renders', () => {
expect(React.isValidElement(<ExploreCtasResultsButton />)).toBe(true);
});
it('renders with props', () => {
expect(React.isValidElement(<s {...mockedProps} />)).toBe(true);
});
it('renders a Button', () => {
const wrapper = getExploreCtasResultsButtonWrapper();
expect(wrapper.find(Button)).toHaveLength(1);
});
describe('datasourceName', () => {
it('should build viz options', () => {
const wrapper = getExploreCtasResultsButtonWrapper();
const spy = sinon.spy(wrapper.instance(), 'buildVizOptions');
wrapper.instance().buildVizOptions();
expect(spy.returnValues[0]).toEqual({
schema: 'dummy_schema',
dbId: 123,
templateParams: undefined,
datasourceName: 'dummy_table',
});
});
});
});

131
superset/assets/src/SqlLab/components/ExploreCtasResultsButton.jsx Normal file
View File

@ -0,0 +1,131 @@
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
import React from 'react';
import PropTypes from 'prop-types';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
import Dialog from 'react-bootstrap-dialog';
import { t } from '@superset-ui/translation';
import { exportChart } from '../../explore/exploreUtils';
import * as actions from '../actions/sqlLab';
import InfoTooltipWithTrigger from '../../components/InfoTooltipWithTrigger';
import Button from '../../components/Button';
const propTypes = {
actions: PropTypes.object.isRequired,
table: PropTypes.string.isRequired,
schema: PropTypes.string,
dbId: PropTypes.number.isRequired,
errorMessage: PropTypes.string,
templateParams: PropTypes.string,
};
const defaultProps = {
vizRequest: {},
};
class ExploreCtasResultsButton extends React.PureComponent {
constructor(props) {
super(props);
this.visualize = this.visualize.bind(this);
this.onClick = this.onClick.bind(this);
}
onClick() {
this.visualize();
}
buildVizOptions() {
return {
datasourceName: this.props.table,
schema: this.props.schema,
dbId: this.props.dbId,
templateParams: this.props.templateParams,
};
}
visualize() {
this.props.actions
.createCtasDatasource(this.buildVizOptions())
.then(data => {
const formData = {
datasource: `${data.table_id}__table`,
metrics: ['count'],
groupby: [],
viz_type: 'table',
since: '100 years ago',
all_columns: [],
row_limit: 1000,
};
this.props.actions.addInfoToast(
t('Creating a data source and creating a new tab'),
);
// open new window for data visualization
exportChart(formData);
})
.catch(() => {
this.props.actions.addDangerToast(
this.props.errorMessage || t('An error occurred'),
);
});
}
render() {
return (
<>
<Button
bsSize="small"
onClick={this.onClick}
tooltip={t('Explore the result set in the data exploration view')}
>
<InfoTooltipWithTrigger
icon="line-chart"
placement="top"
label="explore"
/>{' '}
{t('Explore')}
</Button>
<Dialog
ref={el => {
this.dialog = el;
}}
/>
</>
);
}
}
ExploreCtasResultsButton.propTypes = propTypes;
ExploreCtasResultsButton.defaultProps = defaultProps;
function mapStateToProps({ sqlLab, common }) {
return {
errorMessage: sqlLab.errorMessage,
timeout: common.conf ? common.conf.SUPERSET_WEBSERVER_TIMEOUT : null,
};
}
function mapDispatchToProps(dispatch) {
return {
actions: bindActionCreators(actions, dispatch),
};
}
export { ExploreCtasResultsButton };
export default connect(
mapStateToProps,
mapDispatchToProps,
)(ExploreCtasResultsButton);

1
superset/connectors/sqla/models.py
View File

@ -1132,6 +1132,7 @@ class SqlaTable(Model, BaseDatasource):
if not self.main_dttm_col:
self.main_dttm_col = any_date_col
self.add_missing_metrics(metrics)
db.session.merge(self)
if commit:
db.session.commit()

35
superset/connectors/sqla/views.py
View File

@ -29,16 +29,17 @@ from flask_babel import gettext as __, lazy_gettext as _
from wtforms.ext.sqlalchemy.fields import QuerySelectField
from wtforms.validators import Regexp
from superset import app, appbuilder, db, security_manager
from superset import app, db, security_manager
from superset.connectors.base.views import DatasourceModelView
from superset.constants import RouteMethod
from superset.utils import core as utils
from superset.views.base import (
create_table_permissions,
DatasourceFilter,
DeleteMixin,
get_datasource_exist_error_msg,
ListWidgetWithCheckboxes,
SupersetModelView,
validate_sqlatable,
YamlExportMixin,
)
@ -375,37 +376,11 @@ class TableModelView(DatasourceModelView, DeleteMixin, YamlExportMixin):
}
def pre_add(self, table):
with db.session.no_autoflush:
table_query = db.session.query(models.SqlaTable).filter(
models.SqlaTable.table_name == table.table_name,
models.SqlaTable.schema == table.schema,
models.SqlaTable.database_id == table.database.id,
)
if db.session.query(table_query.exists()).scalar():
raise Exception(get_datasource_exist_error_msg(table.full_name))
# Fail before adding if the table can't be found
try:
table.get_sqla_table_object()
except Exception as ex:
logger.exception(f"Got an error in pre_add for {table.name}")
raise Exception(
_(
"Table [{}] could not be found, "
"please double check your "
"database connection, schema, and "
"table name, error: {}"
).format(table.name, str(ex))
)
validate_sqlatable(table)
def post_add(self, table, flash_message=True):
table.fetch_metadata()
security_manager.add_permission_view_menu("datasource_access", table.get_perm())
if table.schema:
security_manager.add_permission_view_menu(
"schema_access", table.schema_perm
)
create_table_permissions(table)
if flash_message:
flash(
_(

1
superset/models/sql_lab.py
View File

@ -120,6 +120,7 @@ class Query(Model, ExtraJSONMixin):
"startDttm": self.start_time,
"state": self.status.lower(),
"tab": self.tab_name,
"tempSchema": self.tmp_schema_name,
"tempTable": self.tmp_table_name,
"userId": self.user_id,
"user": user_label(self.user),

1
superset/security/manager.py
View File

@ -772,6 +772,7 @@ class SupersetSecurityManager(SecurityManager):
"can_csv",
"can_search_queries",
"can_sqllab_viz",
"can_sqllab_table_viz",
"can_sqllab",
}
or (

45
superset/views/base.py
View File

@ -35,7 +35,15 @@ from sqlalchemy import or_
from werkzeug.exceptions import HTTPException
from wtforms.fields.core import Field, UnboundField
from superset import appbuilder, conf, db, get_feature_flags, security_manager
from superset import (
app as superset_app,
appbuilder,
conf,
db,
get_feature_flags,
security_manager,
)
from superset.connectors.sqla import models
from superset.exceptions import SupersetException, SupersetSecurityException
from superset.translations.utils import get_language_pack
from superset.utils import core as utils
@ -54,6 +62,9 @@ FRONTEND_CONF_KEYS = (
)
logger = logging.getLogger(__name__)
logger = logging.getLogger(__name__)
config = superset_app.config
def get_error_msg():
if conf.get("SHOW_STACKTRACE"):
@ -150,6 +161,38 @@ def get_datasource_exist_error_msg(full_name: str) -> str:
return __("Datasource %(name)s already exists", name=full_name)
def validate_sqlatable(table: models.SqlaTable) -> None:
"""Checks the table existence in the database."""
with db.session.no_autoflush:
table_query = db.session.query(models.SqlaTable).filter(
models.SqlaTable.table_name == table.table_name,
models.SqlaTable.schema == table.schema,
models.SqlaTable.database_id == table.database.id,
)
if db.session.query(table_query.exists()).scalar():
raise Exception(get_datasource_exist_error_msg(table.full_name))
# Fail before adding if the table can't be found
try:
table.get_sqla_table_object()
except Exception as ex:
logger.exception(f"Got an error in pre_add for {table.name}")
raise Exception(
_(
"Table [%{table}s] could not be found, "
"please double check your "
"database connection, schema, and "
"table name, error: {}"
).format(table.name, str(ex))
)
def create_table_permissions(table: models.SqlaTable) -> None:
security_manager.add_permission_view_menu("datasource_access", table.get_perm())
if table.schema:
security_manager.add_permission_view_menu("schema_access", table.schema_perm)
def get_user_roles() -> List[Role]:
if g.user.is_anonymous:
public_role = conf.get("AUTH_ROLE_PUBLIC")

44
superset/views/core.py
View File

@ -98,6 +98,7 @@ from .base import (
BaseSupersetView,
check_ownership,
common_bootstrap_payload,
create_table_permissions,
CsvResponse,
data_payload_response,
DeleteMixin,
@ -108,6 +109,7 @@ from .base import (
json_error_response,
json_success,
SupersetModelView,
validate_sqlatable,
)
from .utils import (
apply_display_max_row_limit,
@ -1955,6 +1957,48 @@ class Superset(BaseSupersetView):
return json_error_response(utils.error_msg_from_exception(ex))
return Response(status=201)
@has_access
@expose("/get_or_create_table/", methods=["POST"])
@event_logger.log_this
def sqllab_table_viz(self):
""" Gets or creates a table object with attributes passed to the API.
It expects the json with params:
* datasourceName - e.g. table name, required
* dbId - database id, required
* schema - table schema, optional
* templateParams - params for the Jinja templating syntax, optional
:return: Response
"""
SqlaTable = ConnectorRegistry.sources["table"]
data = json.loads(request.form.get("data"))
table_name = data.get("datasourceName")
database_id = data.get("dbId")
table = (
db.session.query(SqlaTable)
.filter_by(database_id=database_id, table_name=table_name)
.one_or_none()
)
if not table:
# Create table if doesn't exist.
with db.session.no_autoflush:
table = SqlaTable(table_name=table_name, owners=[g.user])
table.database_id = database_id
table.database = (
db.session.query(models.Database).filter_by(id=database_id).one()
)
table.schema = data.get("schema")
table.template_params = data.get("templateParams")
# needed for the table validation.
validate_sqlatable(table)
db.session.add(table)
table.fetch_metadata()
create_table_permissions(table)
db.session.commit()
return json_success(json.dumps({"table_id": table.id}))
@has_access
@expose("/sqllab_viz/", methods=["POST"])
@event_logger.log_this

16
tests/sqllab_tests.py
View File

@ -332,6 +332,22 @@ class SqlLabTests(SupersetTestCase):
table = db.session.query(SqlaTable).filter_by(id=table_id).one()
self.assertEqual([owner.username for owner in table.owners], ["admin"])
def test_sqllab_table_viz(self):
self.login("admin")
examples_dbid = get_example_database().id
payload = {"datasourceName": "ab_role", "columns": [], "dbId": examples_dbid}
data = {"data": json.dumps(payload)}
resp = self.get_json_resp("/superset/get_or_create_table/", data=data)
self.assertIn("table_id", resp)
# ensure owner is set correctly
table_id = resp["table_id"]
table = db.session.query(SqlaTable).filter_by(id=table_id).one()
self.assertEqual([owner.username for owner in table.owners], ["admin"])
db.session.delete(table)
db.session.commit()
def test_sql_limit(self):
self.login("admin")
test_limit = 1