iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[sql lab] allow users to save their queries (#2528) 

* Allow users to save their queries

Fixing tests .

* Adding placeholder for Query Description

* initJQueryCSRF -> initJQueryAjaxCSRF
This commit is contained in:
Maxime Beauchemin 2017-04-04 20:15:19 -07:00 committed by GitHub
parent c1d9918abe
commit 122891c29b
31 changed files with 656 additions and 279 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.pylintrc
View File

@ -99,7 +99,7 @@ evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / stateme
[BASIC]
# Good variable names which should always be accepted, separated by a comma
good-names=i,j,k,ex,Run,_,d,e,v,o,l,x
good-names=i,j,k,ex,Run,_,d,e,v,o,l,x,ts
# Bad variable names which should always be refused, separated by a comma
bad-names=foo,bar,baz,toto,tutu,tata

3
superset/__init__.py
View File

@ -14,6 +14,7 @@ from flask import Flask, redirect
from flask_appbuilder import SQLA, AppBuilder, IndexView
from flask_appbuilder.baseviews import expose
from flask_migrate import Migrate
from flask_wtf.csrf import CSRFProtect
from werkzeug.contrib.fixers import ProxyFix
from superset.connectors.connector_registry import ConnectorRegistry
@ -50,6 +51,8 @@ logging.getLogger('pyhive.presto').setLevel(logging.INFO)
db = SQLA(app)
if conf.get('WTF_CSRF_ENABLED'):
csrf = CSRFProtect(app)
utils.pessimistic_connection_handling(db.engine.pool)

36
superset/assets/javascripts/SqlLab/actions.js
View File

@ -1,3 +1,4 @@
/* global notify */
import shortid from 'shortid';
import { now } from '../modules/dates';
const $ = require('jquery');
@ -33,11 +34,25 @@ export const QUERY_FAILED = 'QUERY_FAILED';
export const CLEAR_QUERY_RESULTS = 'CLEAR_QUERY_RESULTS';
export const REMOVE_DATA_PREVIEW = 'REMOVE_DATA_PREVIEW';
export const CHANGE_DATA_PREVIEW_ID = 'CHANGE_DATA_PREVIEW_ID';
export const SAVE_QUERY = 'SAVE_QUERY';
export function resetState() {
return { type: RESET_STATE };
}
export function saveQuery(query) {
const url = '/savedqueryviewapi/api/create';
$.ajax({
type: 'POST',
url,
data: query,
success: () => notify.success('Your query was saved'),
error: () => notify.error('Your query could not be saved'),
dataType: 'json',
});
return { type: SAVE_QUERY };
}
export function startQuery(query) {
Object.assign(query, {
id: query.id ? query.id : shortid.generate(),
@ -328,6 +343,27 @@ export function popStoredQuery(urlId) {
};
dispatch(addQueryEditor(queryEditorProps));
},
error: () => notify.error("The query couldn't be loaded"),
});
};
}
export function popSavedQuery(saveQueryId) {
return function (dispatch) {
$.ajax({
type: 'GET',
url: `/savedqueryviewapi/api/get/${saveQueryId}`,
success: (data) => {
const sq = data.result;
const queryEditorProps = {
title: sq.label,
dbId: sq.db_id,
schema: sq.schema,
autorun: false,
sql: sq.sql,
};
dispatch(addQueryEditor(queryEditorProps));
},
error: () => notify.error("The query couldn't be loaded"),
});
};
}

34
superset/assets/javascripts/SqlLab/components/Alerts.jsx
View File

@ -1,34 +0,0 @@
import React from 'react';
import { Alert } from 'react-bootstrap';
class Alerts extends React.PureComponent {
removeAlert(alert) {
this.props.actions.removeAlert(alert);
}
render() {
const alerts = this.props.alerts.map((alert) =>
<Alert
key={alert.id}
bsStyle={alert.bsStyle}
style={{ width: '500px', textAlign: 'midddle', margin: '10px auto' }}
>
{alert.msg}
<i
className="fa fa-close pull-right"
onClick={this.removeAlert.bind(this, alert)}
style={{ cursor: 'pointer' }}
/>
</Alert>
);
return (
<div>{alerts}</div>
);
}
}
Alerts.propTypes = {
alerts: React.PropTypes.array,
actions: React.PropTypes.object,
};
export default Alerts;

18
superset/assets/javascripts/SqlLab/components/AlertsWrapper.jsx Normal file
View File

@ -0,0 +1,18 @@
import React from 'react';
import AlertContainer from 'react-alert';
export default class AlertsWrapper extends React.PureComponent {
render() {
return (
<AlertContainer
ref={ref => {
global.notify = ref;
}}
offset={14}
position="top right"
theme="dark"
time={5000}
transition="fade"
/>);
}
}

4
superset/assets/javascripts/SqlLab/components/App.jsx
View File

@ -5,7 +5,7 @@ import React from 'react';
import TabbedSqlEditors from './TabbedSqlEditors';
import QueryAutoRefresh from './QueryAutoRefresh';
import QuerySearch from './QuerySearch';
import Alerts from './Alerts';
import AlertsWrapper from './AlertsWrapper';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
@ -64,7 +64,7 @@ class App extends React.PureComponent {
}
return (
<div className="App SqlLab">
<Alerts id="sqllab-alerts" alerts={this.props.alerts} actions={this.props.actions} />
<AlertsWrapper />
<div className="container-fluid">
{content}
</div>

2
superset/assets/javascripts/SqlLab/components/RunQueryActionButton.jsx
View File

@ -31,7 +31,7 @@ export default function RunQueryActionButton(props) {
onClick={() => props.runQuery(false)}
key="run-btn"
>
<i className="fa fa-table" /> {runBtnText}
<i className="fa fa-refresh" /> {runBtnText}
</Button>
);

131
superset/assets/javascripts/SqlLab/components/SaveQuery.jsx Normal file
View File

@ -0,0 +1,131 @@
/* global notify */
import React from 'react';
import { FormControl, FormGroup, Overlay, Popover, Row, Col } from 'react-bootstrap';
import Button from '../../components/Button';
const propTypes = {
defaultLabel: React.PropTypes.string,
sql: React.PropTypes.string,
schema: React.PropTypes.string,
dbId: React.PropTypes.number,
animation: React.PropTypes.bool,
onSave: React.PropTypes.func,
};
const defaultProps = {
defaultLabel: 'Undefined',
animation: true,
onSave: () => {},
};
class SaveQuery extends React.PureComponent {
constructor(props) {
super(props);
this.state = {
description: '',
label: props.defaultLabel,
showSave: false,
};
this.toggleSave = this.toggleSave.bind(this);
this.onSave = this.onSave.bind(this);
this.onCancel = this.onCancel.bind(this);
this.onLabelChange = this.onLabelChange.bind(this);
this.onDescriptionChange = this.onDescriptionChange.bind(this);
}
onSave() {
const query = {
label: this.state.label,
description: this.state.description,
db_id: this.props.dbId,
schema: this.props.schema,
sql: this.props.sql,
};
this.props.onSave(query);
this.setState({ showSave: false });
}
onCancel() {
this.setState({ showSave: false });
}
onLabelChange(e) {
this.setState({ label: e.target.value });
}
onDescriptionChange(e) {
this.setState({ description: e.target.value });
}
renderPopover() {
return (
<Popover id="embed-code-popover">
<FormGroup bsSize="small" style={{ width: '350px' }}>
<Row>
<Col md={12}>
<small>
<label className="control-label" htmlFor="embed-height">
Label
</label>
</small>
<FormControl
type="text"
placeholder="Label for your query"
value={this.state.label}
onChange={this.onLabelChange}
/>
</Col>
</Row>
<br />
<Row>
<Col md={12}>
<small>
<label className="control-label" htmlFor="embed-height">Description</label>
</small>
<FormControl
componentClass="textarea"
placeholder="Write a description for your query"
value={this.state.description}
onChange={this.onDescriptionChange}
/>
</Col>
</Row>
<br />
<Row>
<Col md={12}>
<Button
bsStyle="primary"
onClick={this.onSave}
className="m-r-3"
>
Save
</Button>
<Button onClick={this.onCancel} className="cancelQuery">
Cancel
</Button>
</Col>
</Row>
</FormGroup>
</Popover>
);
}
toggleSave(e) {
this.setState({ target: e.target, showSave: !this.state.showSave });
}
render() {
return (
<span className="SaveQuery">
<Overlay
trigger="click"
target={this.state.target}
show={this.state.showSave}
placement="bottom"
animation={this.props.animation}
>
{this.renderPopover()}
</Overlay>
<Button bsSize="small" className="toggleSave" onClick={this.toggleSave}>
<i className="fa fa-save" /> Save Query
</Button>
</span>
);
}
}
SaveQuery.propTypes = propTypes;
SaveQuery.defaultProps = defaultProps;
export default SaveQuery;

13
superset/assets/javascripts/SqlLab/components/SqlEditor.jsx
View File

@ -15,6 +15,7 @@ import {
import Button from '../../components/Button';
import SouthPane from './SouthPane';
import SaveQuery from './SaveQuery';
import Timer from '../../components/Timer';
import SqlEditorLeftBar from './SqlEditorLeftBar';
import AceEditorWrapper from './AceEditorWrapper';
@ -101,6 +102,7 @@ class SqlEditor extends React.PureComponent {
}
render() {
const qe = this.props.queryEditor;
let limitWarning = null;
if (this.props.latestQuery && this.props.latestQuery.limit_reached) {
const tooltip = (
@ -149,12 +151,19 @@ class SqlEditor extends React.PureComponent {
<Form inline>
<RunQueryActionButton
allowAsync={this.props.database ? this.props.database.allow_run_async : false}
dbId={this.props.queryEditor.dbId}
dbId={qe.dbId}
queryState={this.props.latestQuery && this.props.latestQuery.state}
runQuery={this.runQuery.bind(this)}
selectedText={this.props.queryEditor.selectedText}
selectedText={qe.selectedText}
stopQuery={this.stopQuery.bind(this)}
/>
<SaveQuery
defaultLabel={qe.title}
sql={qe.sql}
onSave={this.props.actions.saveQuery}
schema={qe.schema}
dbId={qe.dbId}
/>
{ctasControls}
</Form>
</div>

28
superset/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx
View File

@ -6,7 +6,7 @@ import * as Actions from '../actions';
import SqlEditor from './SqlEditor';
import CopyQueryTabUrl from './CopyQueryTabUrl';
import { areArraysShallowEqual } from '../../reduxUtils';
import { getParamFromQuery } from '../../../utils/common';
import URI from 'urijs';
const propTypes = {
actions: React.PropTypes.object.isRequired,
@ -35,19 +35,19 @@ class TabbedSqlEditors extends React.PureComponent {
};
}
componentDidMount() {
const search = window.location.search;
if (search) {
const queryString = search.substring(1);
const urlId = getParamFromQuery(queryString, 'id');
if (urlId) {
this.props.actions.popStoredQuery(urlId);
} else {
let dbId = getParamFromQuery(queryString, 'dbid');
const query = URI(window.location).search(true);
if (query.id || query.sql || query.savedQueryId) {
if (query.id) {
this.props.actions.popStoredQuery(query.id);
} else if (query.savedQueryId) {
this.props.actions.popSavedQuery(query.savedQueryId);
} else if (query.sql) {
let dbId = query.dbid;
if (dbId) {
dbId = parseInt(dbId, 10);
} else {
const databases = this.props.databases;
const dbName = getParamFromQuery(queryString, 'dbname');
const dbName = query.dbname;
if (dbName) {
Object.keys(databases).forEach((db) => {
if (databases[db].database_name === dbName) {
@ -57,11 +57,11 @@ class TabbedSqlEditors extends React.PureComponent {
}
}
const newQueryEditor = {
title: getParamFromQuery(queryString, 'title'),
title: query.title,
dbId,
schema: getParamFromQuery(queryString, 'schema'),
autorun: getParamFromQuery(queryString, 'autorun'),
sql: getParamFromQuery(queryString, 'sql'),
schema: query.schema,
autorun: query.autorun,
sql: query.sql,
};
this.props.actions.addQueryEditor(newQueryEditor);
}

2
superset/assets/javascripts/SqlLab/index.jsx
View File

@ -6,6 +6,7 @@ import React from 'react';
import { render } from 'react-dom';
import { getInitialState, sqlLabReducer } from './reducers';
import { initEnhancer } from '../reduxUtils';
import { initJQueryAjaxCSRF } from '../modules/utils';
import { createStore, compose, applyMiddleware } from 'redux';
import { Provider } from 'react-redux';
import thunkMiddleware from 'redux-thunk';
@ -14,6 +15,7 @@ import App from './components/App';
require('./main.css');
initJQueryAjaxCSRF();
const appContainer = document.getElementById('app');
const bootstrapData = JSON.parse(appContainer.getAttribute('data-bootstrap'));

14
superset/assets/javascripts/index.jsx
View File

@ -1,14 +0,0 @@
import React from 'react';
import { render } from 'react-dom';
import { Jumbotron } from 'react-bootstrap';
function App() {
return (
<Jumbotron>
<h1>Superset</h1>
<p>Extensible visualization tool for exploring data from any database.</p>
</Jumbotron>
);
}
render(<App />, document.getElementById('app'));

15
superset/assets/javascripts/modules/utils.js
View File

@ -196,3 +196,18 @@ export function getTextWidth(text, fontDetails) {
const metrics = context.measureText(text);
return metrics.width;
}
export function initJQueryAjaxCSRF() {
// Works in conjunction with a Flask-WTF token as described here:
// http://flask-wtf.readthedocs.io/en/stable/csrf.html#javascript-requests
const token = $('input#csrf_token').val();
if (token) {
$.ajaxSetup({
beforeSend(xhr, settings) {
if (!/^(GET|HEAD|OPTIONS|TRACE)$/i.test(settings.type) && !this.crossDomain) {
xhr.setRequestHeader('X-CSRFToken', token);
}
},
});
}
}

3
superset/assets/package.json
View File

@ -66,7 +66,9 @@
"nvd3": "1.8.5",
"react": "^15.3.2",
"react-ace": "^4.1.5",
"react-addons-css-transition-group": "^15.4.2",
"react-addons-shallow-compare": "^15.4.2",
"react-alert": "^1.0.14",
"react-bootstrap": "^0.30.3",
"react-bootstrap-table": "^2.3.8",
"react-dom": "^15.3.2",
@ -88,6 +90,7 @@
"style-loader": "^0.13.0",
"supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40",
"topojson": "^1.6.22",
"urijs": "^1.18.10",
"victory": "^0.17.0",
"viewport-mercator-project": "^2.1.0"
},

11
superset/assets/spec/javascripts/sqllab/AlertsWrapper_spec.jsx Normal file
View File

@ -0,0 +1,11 @@
import React from 'react';
import AlertsWrapper from '../../../javascripts/SqlLab/components/AlertsWrapper';
import { describe, it } from 'mocha';
import { expect } from 'chai';
describe('AlertsWrapper', () => {
it('is valid', () => {
expect(React.isValidElement(<AlertsWrapper />)).to.equal(true);
});
});

24
superset/assets/spec/javascripts/sqllab/Alerts_spec.jsx
View File

@ -1,24 +0,0 @@
import React from 'react';
import Alerts from '../../../javascripts/SqlLab/components/Alerts';
import { Alert } from 'react-bootstrap';
import { shallow } from 'enzyme';
import { describe, it } from 'mocha';
import { expect } from 'chai';
import { alert } from './fixtures';
describe('Alerts', () => {
const mockedProps = {
alerts: [alert],
};
it('is valid', () => {
expect(React.isValidElement(<Alerts />)).to.equal(true);
});
it('is valid with props', () => {
expect(React.isValidElement(<Alerts {...mockedProps} />)).to.equal(true);
});
it('renders an Alert', () => {
const wrapper = shallow(<Alerts {...mockedProps} />);
expect(wrapper.find(Alert)).to.have.length(1);
});
});

48
superset/assets/spec/javascripts/sqllab/SaveQuery_spec.jsx Normal file
View File

@ -0,0 +1,48 @@
import React from 'react';
import SaveQuery from '../../../javascripts/SqlLab/components/SaveQuery';
import { Overlay, Popover, FormControl } from 'react-bootstrap';
import { shallow, mount } from 'enzyme';
import { describe, it } from 'mocha';
import { expect } from 'chai';
describe('SavedQuery', () => {
const mockedProps = {
dbId: 1,
schema: 'main',
sql: 'SELECT * FROM t',
defaultLabel: 'untitled',
animation: false,
};
it('is valid', () => {
expect(
React.isValidElement(<SaveQuery />)
).to.equal(true);
});
it('is valid with props', () => {
expect(
React.isValidElement(<SaveQuery {...mockedProps} />)
).to.equal(true);
});
it('has an Overlay and a Popover', () => {
const wrapper = shallow(<SaveQuery {...mockedProps} />);
expect(wrapper.find(Overlay)).to.have.length(1);
expect(wrapper.find(Popover)).to.have.length(1);
});
it('pops and hides', () => {
const wrapper = mount(<SaveQuery {...mockedProps} />);
expect(wrapper.state().showSave).to.equal(false);
wrapper.find('.toggleSave').simulate('click');
expect(wrapper.state().showSave).to.equal(true);
wrapper.find('.toggleSave').simulate('click');
expect(wrapper.state().showSave).to.equal(false);
});
it('has a cancel button', () => {
const wrapper = shallow(<SaveQuery {...mockedProps} />);
expect(wrapper.find('.cancelQuery')).to.have.length(1);
});
it('has 2 FormControls', () => {
const wrapper = shallow(<SaveQuery {...mockedProps} />);
expect(wrapper.find(FormControl)).to.have.length(2);
});
});

2
superset/config.py
View File

@ -54,7 +54,7 @@ SQLALCHEMY_DATABASE_URI = 'sqlite:///' + os.path.join(DATA_DIR, 'superset.db')
QUERY_SEARCH_LIMIT = 1000
# Flask-WTF flag for CSRF
CSRF_ENABLED = True
WTF_CSRF_ENABLED = True
# Whether to run the web server in debug mode or not
DEBUG = False

40
superset/migrations/versions/2fcdcb35e487_saved_queries.py Normal file
View File

@ -0,0 +1,40 @@
"""saved_queries
Revision ID: 2fcdcb35e487
Revises: a6c18f869a4e
Create Date: 2017-03-29 15:04:35.734190
"""
from alembic import op
import sqlalchemy as sa
# revision identifiers, used by Alembic.
revision = '2fcdcb35e487'
down_revision = 'a6c18f869a4e'
def upgrade():
op.create_table(
'saved_query',
sa.Column('created_on', sa.DateTime(), nullable=True),
sa.Column('changed_on', sa.DateTime(), nullable=True),
sa.Column('id', sa.Integer(), nullable=False),
sa.Column('user_id', sa.Integer(), nullable=True),
sa.Column('db_id', sa.Integer(), nullable=True),
sa.Column('label', sa.String(256), nullable=True),
sa.Column('schema', sa.String(128), nullable=True),
sa.Column('sql', sa.Text(), nullable=True),
sa.Column('description', sa.Text(), nullable=True),
sa.Column('changed_by_fk', sa.Integer(), nullable=True),
sa.Column('created_by_fk', sa.Integer(), nullable=True),
sa.ForeignKeyConstraint(['changed_by_fk'], ['ab_user.id'], ),
sa.ForeignKeyConstraint(['created_by_fk'], ['ab_user.id'], ),
sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
sa.ForeignKeyConstraint(['db_id'], [u'dbs.id'], ),
sa.PrimaryKeyConstraint('id')
)
def downgrade():
op.drop_table('saved_query')

1
superset/models/__init__.py
View File

@ -1 +1,2 @@
from . import core # noqa
from . import sql_lab # noqa

104
superset/models/core.py
View File

@ -9,7 +9,6 @@ import json
import logging
import numpy
import pickle
import re
import textwrap
from future.standard_library import install_aliases
from copy import copy
@ -26,10 +25,10 @@ from flask_appbuilder.models.decorators import renders
from sqlalchemy import (
Column, Integer, String, ForeignKey, Text, Boolean,
DateTime, Date, Table, Numeric,
DateTime, Date, Table,
create_engine, MetaData, select
)
from sqlalchemy.orm import backref, relationship
from sqlalchemy.orm import relationship
from sqlalchemy.orm.session import make_transient
from sqlalchemy.sql import text
from sqlalchemy.sql.expression import TextAsFrom
@ -797,105 +796,6 @@ class FavStar(Model):
dttm = Column(DateTime, default=datetime.utcnow)
class Query(Model):
"""ORM model for SQL query"""
__tablename__ = 'query'
id = Column(Integer, primary_key=True)
client_id = Column(String(11), unique=True, nullable=False)
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
# Store the tmp table into the DB only if the user asks for it.
tmp_table_name = Column(String(256))
user_id = Column(
Integer, ForeignKey('ab_user.id'), nullable=True)
status = Column(String(16), default=QueryStatus.PENDING)
tab_name = Column(String(256))
sql_editor_id = Column(String(256))
schema = Column(String(256))
sql = Column(Text)
# Query to retrieve the results,
# used only in case of select_as_cta_used is true.
select_sql = Column(Text)
executed_sql = Column(Text)
# Could be configured in the superset config.
limit = Column(Integer)
limit_used = Column(Boolean, default=False)
select_as_cta = Column(Boolean)
select_as_cta_used = Column(Boolean, default=False)
progress = Column(Integer, default=0) # 1..100
# # of rows in the result set or rows modified.
rows = Column(Integer)
error_message = Column(Text)
# key used to store the results in the results backend
results_key = Column(String(64), index=True)
# Using Numeric in place of DateTime for sub-second precision
# stored as seconds since epoch, allowing for milliseconds
start_time = Column(Numeric(precision=3))
start_running_time = Column(Numeric(precision=3))
end_time = Column(Numeric(precision=3))
changed_on = Column(
DateTime, default=datetime.utcnow, onupdate=datetime.utcnow, nullable=True)
database = relationship(
'Database',
foreign_keys=[database_id],
backref=backref('queries', cascade='all, delete-orphan')
)
user = relationship(
'User',
backref=backref('queries', cascade='all, delete-orphan'),
foreign_keys=[user_id])
__table_args__ = (
sqla.Index('ti_user_id_changed_on', user_id, changed_on),
)
@property
def limit_reached(self):
return self.rows == self.limit if self.limit_used else False
def to_dict(self):
return {
'changedOn': self.changed_on,
'changed_on': self.changed_on.isoformat(),
'dbId': self.database_id,
'db': self.database.database_name,
'endDttm': self.end_time,
'errorMessage': self.error_message,
'executedSql': self.executed_sql,
'id': self.client_id,
'limit': self.limit,
'progress': self.progress,
'rows': self.rows,
'schema': self.schema,
'ctas': self.select_as_cta,
'serverId': self.id,
'sql': self.sql,
'sqlEditorId': self.sql_editor_id,
'startDttm': self.start_time,
'state': self.status.lower(),
'tab': self.tab_name,
'tempTable': self.tmp_table_name,
'userId': self.user_id,
'user': self.user.username,
'limit_reached': self.limit_reached,
'resultsKey': self.results_key,
}
@property
def name(self):
ts = datetime.now().isoformat()
ts = ts.replace('-', '').replace(':', '').split('.')[0]
tab = self.tab_name.replace(' ', '_').lower() if self.tab_name else 'notab'
tab = re.sub(r'\W+', '', tab)
return "sqllab_{tab}_{ts}".format(**locals())
class DatasourceAccessRequest(Model, AuditMixinNullable):
"""ORM model for the access requests for datasources and dbs."""
__tablename__ = 'access_request'

165
superset/models/sql_lab.py Normal file
View File

@ -0,0 +1,165 @@
"""A collection of ORM sqlalchemy models for SQL Lab"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function
from __future__ import unicode_literals
import re
from datetime import datetime
from future.standard_library import install_aliases
from flask import Markup
from flask_appbuilder import Model
import sqlalchemy as sqla
from sqlalchemy import (
Column, Integer, String, ForeignKey, Text, Boolean,
DateTime, Numeric,
)
from sqlalchemy.orm import backref, relationship
from superset.utils import QueryStatus
from superset.models.helpers import AuditMixinNullable
install_aliases()
class Query(Model):
"""ORM model for SQL query"""
__tablename__ = 'query'
id = Column(Integer, primary_key=True)
client_id = Column(String(11), unique=True, nullable=False)
database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
# Store the tmp table into the DB only if the user asks for it.
tmp_table_name = Column(String(256))
user_id = Column(
Integer, ForeignKey('ab_user.id'), nullable=True)
status = Column(String(16), default=QueryStatus.PENDING)
tab_name = Column(String(256))
sql_editor_id = Column(String(256))
schema = Column(String(256))
sql = Column(Text)
# Query to retrieve the results,
# used only in case of select_as_cta_used is true.
select_sql = Column(Text)
executed_sql = Column(Text)
# Could be configured in the superset config.
limit = Column(Integer)
limit_used = Column(Boolean, default=False)
select_as_cta = Column(Boolean)
select_as_cta_used = Column(Boolean, default=False)
progress = Column(Integer, default=0) # 1..100
# # of rows in the result set or rows modified.
rows = Column(Integer)
error_message = Column(Text)
# key used to store the results in the results backend
results_key = Column(String(64), index=True)
# Using Numeric in place of DateTime for sub-second precision
# stored as seconds since epoch, allowing for milliseconds
start_time = Column(Numeric(precision=3))
start_running_time = Column(Numeric(precision=3))
end_time = Column(Numeric(precision=3))
changed_on = Column(
DateTime,
default=datetime.utcnow,
onupdate=datetime.utcnow,
nullable=True)
database = relationship(
'Database',
foreign_keys=[database_id],
backref=backref('queries', cascade='all, delete-orphan')
)
user = relationship(
'User',
foreign_keys=[user_id])
__table_args__ = (
sqla.Index('ti_user_id_changed_on', user_id, changed_on),
)
@property
def limit_reached(self):
return self.rows == self.limit if self.limit_used else False
def to_dict(self):
return {
'changedOn': self.changed_on,
'changed_on': self.changed_on.isoformat(),
'dbId': self.database_id,
'db': self.database.database_name,
'endDttm': self.end_time,
'errorMessage': self.error_message,
'executedSql': self.executed_sql,
'id': self.client_id,
'limit': self.limit,
'progress': self.progress,
'rows': self.rows,
'schema': self.schema,
'ctas': self.select_as_cta,
'serverId': self.id,
'sql': self.sql,
'sqlEditorId': self.sql_editor_id,
'startDttm': self.start_time,
'state': self.status.lower(),
'tab': self.tab_name,
'tempTable': self.tmp_table_name,
'userId': self.user_id,
'user': self.user.username,
'limit_reached': self.limit_reached,
'resultsKey': self.results_key,
}
@property
def name(self):
"""Name property"""
ts = datetime.now().isoformat()
ts = ts.replace('-', '').replace(':', '').split('.')[0]
tab = (
self.tab_name.replace(' ', '_').lower()
if self.tab_name
else 'notab'
)
tab = re.sub(r'\W+', '', tab)
return "sqllab_{tab}_{ts}".format(**locals())
class SavedQuery(Model, AuditMixinNullable):
"""ORM model for SQL query"""
__tablename__ = 'saved_query'
id = Column(Integer, primary_key=True)
user_id = Column(
Integer, ForeignKey('ab_user.id'), nullable=True)
db_id = Column(
Integer, ForeignKey('dbs.id'), nullable=True)
schema = Column(String(128))
label = Column(String(256))
description = Column(Text)
sql = Column(Text)
user = relationship(
'User',
backref=backref('saved_queries', cascade='all, delete-orphan'),
foreign_keys=[user_id])
database = relationship(
'Database',
foreign_keys=[db_id],
backref=backref('saved_queries', cascade='all, delete-orphan')
)
@property
def pop_tab_link(self):
return Markup("""
<a href="/superset/sqllab?savedQueryId={self.id}">
<i class="fa fa-link"></i>
</a>
""".format(**locals()))

9
superset/sql_lab.py
View File

@ -13,7 +13,7 @@ from sqlalchemy.orm import sessionmaker
from superset import (
app, db, utils, dataframe, results_backend)
from superset.models import core as models
from superset.models.sql_lab import Query
from superset.sql_parse import SupersetQuery
from superset.db_engine_specs import LimitMethod
from superset.jinja_context import get_template_processor
@ -56,14 +56,15 @@ def get_sql_results(self, query_id, return_results=True, store_results=False):
session = db.session()
session.commit() # HACK
try:
query = session.query(models.Query).filter_by(id=query_id).one()
query = session.query(Query).filter_by(id=query_id).one()
except Exception as e:
logging.error("Query with id `{}` could not be retrieved".format(query_id))
logging.error(
"Query with id `{}` could not be retrieved".format(query_id))
logging.error("Sleeping for a sec and retrying...")
# Nasty hack to get around a race condition where the worker
# cannot find the query it's supposed to run
sleep(1)
query = session.query(models.Query).filter_by(id=query_id).one()
query = session.query(Query).filter_by(id=query_id).one()
database = query.database
db_engine_spec = database.db_engine_spec

2
superset/templates/superset/basic.html
View File

@ -1,3 +1,4 @@
{% import 'appbuilder/general/lib.html' as lib %}
<html>
<head>
<title>
@ -37,6 +38,7 @@
<div id="app" data-bootstrap="{{ bootstrap_data }}" >
<img src="/static/assets/images/loading.gif" style="width: 50px; margin: 10px;">
</div>
{{ csrf_token() if csrf_token else None }}
{% endblock %}
<!-- Modal for misc messages / alerts -->

1
superset/views/__init__.py
View File

@ -1,2 +1,3 @@
from . import base # noqa
from . import core # noqa
from . import sql_lab # noqa

42
superset/views/base.py
View File

@ -1,7 +1,9 @@
import logging
import functools
import json
import logging
import traceback
from flask import g, redirect
from flask import g, redirect, Response
from flask_babel import gettext as __
from flask_appbuilder import BaseView
@ -15,6 +17,42 @@ from superset import appbuilder, conf, db, utils, sm, sql_parse
from superset.connectors.connector_registry import ConnectorRegistry
def get_error_msg():
if conf.get("SHOW_STACKTRACE"):
error_msg = traceback.format_exc()
else:
error_msg = "FATAL ERROR \n"
error_msg += (
"Stacktrace is hidden. Change the SHOW_STACKTRACE "
"configuration setting to enable it")
return error_msg
def json_error_response(msg, status=None, stacktrace=None):
data = {'error': msg}
if stacktrace:
data['stacktrace'] = stacktrace
status = status if status else 500
return Response(
json.dumps(data),
status=status, mimetype="application/json")
def api(f):
"""
A decorator to label an endpoint as an API. Catches uncaught exceptions and
return the response in the JSON format
"""
def wraps(self, *args, **kwargs):
try:
return f(self, *args, **kwargs)
except Exception as e:
logging.exception(e)
return json_error_response(get_error_msg())
return functools.update_wrapper(wraps, f)
def get_datasource_exist_error_mgs(full_name):
return __("Datasource %(name)s already exists", name=full_name)

88
superset/views/core.py
View File

@ -13,7 +13,6 @@ import time
import traceback
import zlib
import functools
import sqlalchemy as sqla
from flask import (
@ -38,11 +37,12 @@ from superset.legacy import cast_form_data
from superset.utils import has_access
from superset.connectors.connector_registry import ConnectorRegistry
import superset.models.core as models
from superset.models.sql_lab import Query
from superset.sql_parse import SupersetQuery
from .base import (
SupersetModelView, BaseSupersetView, DeleteMixin,
SupersetFilter, get_user_roles
api, SupersetModelView, BaseSupersetView, DeleteMixin,
SupersetFilter, get_user_roles, json_error_response, get_error_msg
)
config = app.config
@ -71,46 +71,10 @@ def get_datasource_access_error_msg(datasource_name):
"`all_datasource_access` permission", name=datasource_name)
def get_error_msg():
if config.get("SHOW_STACKTRACE"):
error_msg = traceback.format_exc()
else:
error_msg = "FATAL ERROR \n"
error_msg += (
"Stacktrace is hidden. Change the SHOW_STACKTRACE "
"configuration setting to enable it")
return error_msg
def json_error_response(msg, status=None, stacktrace=None):
data = {'error': msg}
if stacktrace:
data['stacktrace'] = stacktrace
status = status if status else 500
return Response(
json.dumps(data),
status=status, mimetype="application/json")
def json_success(json_msg, status=200):
return Response(json_msg, status=status, mimetype="application/json")
def api(f):
"""
A decorator to label an endpoint as an API. Catches uncaught exceptions and
return the response in the JSON format
"""
def wraps(self, *args, **kwargs):
try:
return f(self, *args, **kwargs)
except Exception as e:
logging.exception(e)
return json_error_response(get_error_msg())
return functools.update_wrapper(wraps, f)
def is_owner(obj, user):
""" Check if user is owner of the slice """
return obj and obj.owners and user in obj.owners
@ -566,19 +530,6 @@ appbuilder.add_view(
icon="fa-list-ol")
class QueryView(SupersetModelView):
datamodel = SQLAInterface(models.Query)
list_columns = ['user', 'database', 'status', 'start_time', 'end_time']
appbuilder.add_view(
QueryView,
"Queries",
label=__("Queries"),
category="Manage",
category_label=__("Manage"),
icon="fa-search")
@app.route('/health')
def health():
return "OK"
@ -1928,7 +1879,7 @@ class Superset(BaseSupersetView):
status=410
)
query = db.session.query(models.Query).filter_by(results_key=key).one()
query = db.session.query(Query).filter_by(results_key=key).one()
rejected_tables = self.rejected_datasources(
query.sql, query.database, query.schema)
if rejected_tables:
@ -1950,7 +1901,7 @@ class Superset(BaseSupersetView):
client_id = request.form.get('client_id')
try:
query = (
db.session.query(models.Query)
db.session.query(Query)
.filter_by(client_id=client_id).one()
)
query.status = utils.QueryStatus.STOPPED
@ -1990,7 +1941,7 @@ class Superset(BaseSupersetView):
tmp_table_name
)
query = models.Query(
query = Query(
database_id=int(database_id),
limit=int(app.config.get('SQL_MAX_ROW', None)),
sql=sql,
@ -2061,7 +2012,7 @@ class Superset(BaseSupersetView):
def csv(self, client_id):
"""Download the query results as csv."""
query = (
db.session.query(models.Query)
db.session.query(Query)
.filter_by(client_id=client_id)
.one()
)
@ -2126,10 +2077,10 @@ class Superset(BaseSupersetView):
last_updated_dt = utils.EPOCH + timedelta(seconds=last_updated_ms_int / 1000)
sql_queries = (
db.session.query(models.Query)
db.session.query(Query)
.filter(
models.Query.user_id == g.user.get_id(),
models.Query.changed_on >= last_updated_dt,
Query.user_id == g.user.get_id(),
Query.changed_on >= last_updated_dt,
)
.all()
)
@ -2142,7 +2093,7 @@ class Superset(BaseSupersetView):
@log_this
def search_queries(self):
"""Search for queries."""
query = db.session.query(models.Query)
query = db.session.query(Query)
search_user_id = request.args.get('user_id')
database_id = request.args.get('database_id')
search_text = request.args.get('search_text')
@ -2153,30 +2104,30 @@ class Superset(BaseSupersetView):
if search_user_id:
# Filter on db Id
query = query.filter(models.Query.user_id == search_user_id)
query = query.filter(Query.user_id == search_user_id)
if database_id:
# Filter on db Id
query = query.filter(models.Query.database_id == database_id)
query = query.filter(Query.database_id == database_id)
if status:
# Filter on status
query = query.filter(models.Query.status == status)
query = query.filter(Query.status == status)
if search_text:
# Filter on search text
query = query \
.filter(models.Query.sql.like('%{}%'.format(search_text)))
.filter(Query.sql.like('%{}%'.format(search_text)))
if from_time:
query = query.filter(models.Query.start_time > int(from_time))
query = query.filter(Query.start_time > int(from_time))
if to_time:
query = query.filter(models.Query.start_time < int(to_time))
query = query.filter(Query.start_time < int(to_time))
query_limit = config.get('QUERY_SEARCH_LIMIT', 1000)
sql_queries = (
query.order_by(models.Query.start_time.asc())
query.order_by(Query.start_time.asc())
.limit(query_limit)
.all()
)
@ -2255,8 +2206,11 @@ class Superset(BaseSupersetView):
d = {
'defaultDbId': config.get('SQLLAB_DEFAULT_DBID'),
}
from flask_wtf import FlaskForm
ff = FlaskForm()
return self.render_template(
'superset/sqllab.html',
csrf_token=ff.csrf_token,
bootstrap_data=json.dumps(d, default=utils.json_iso_dttm_ser)
)
appbuilder.add_view_no_menu(Superset)

69
superset/views/sql_lab.py Normal file
View File

@ -0,0 +1,69 @@
from flask import redirect, g
from flask_appbuilder import expose
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_babel import gettext as __
from superset import appbuilder
from superset.models.sql_lab import Query, SavedQuery
from .base import SupersetModelView, BaseSupersetView, DeleteMixin
class QueryView(SupersetModelView):
datamodel = SQLAInterface(Query)
list_columns = ['user', 'database', 'status', 'start_time', 'end_time']
appbuilder.add_view(
QueryView,
"Queries",
label=__("Queries"),
category="Manage",
category_label=__("Manage"),
icon="fa-search")
class SavedQueryView(SupersetModelView, DeleteMixin):
datamodel = SQLAInterface(SavedQuery)
list_columns = [
'label', 'user', 'database', 'schema', 'description',
'modified', 'pop_tab_link']
show_columns = [
'id', 'label', 'user', 'database',
'description', 'sql', 'pop_tab_link']
add_columns = ['label', 'database', 'description', 'sql']
edit_columns = add_columns
base_order = ('changed_on', 'desc')
def pre_add(self, obj):
obj.user = g.user
def pre_update(self, obj):
self.pre_add(obj)
class SavedQueryViewApi(SavedQueryView):
show_columns = ['label', 'db_id', 'schema', 'description', 'sql']
add_columns = show_columns
edit_columns = add_columns
appbuilder.add_view_no_menu(SavedQueryViewApi)
appbuilder.add_view_no_menu(SavedQueryView)
appbuilder.add_link(
__('Saved Queries'),
href='/sqllab/my_queries/',
icon="fa-save",
category='SQL Lab')
class SqlLab(BaseSupersetView):
"""The base views for Superset!"""
@expose("/my_queries/")
def my_queries(self):
"""Assigns a list of found users to the given role."""
return redirect(
'/savedqueryview/list/?_flt_0_user={}'.format(g.user.id))
appbuilder.add_view_no_menu(SqlLab)

5
tests/celery_tests.py
View File

@ -16,6 +16,7 @@ import pandas as pd
from superset import app, appbuilder, cli, db, dataframe
from superset.models import core as models
from superset.models.helpers import QueryStatus
from superset.models.sql_lab import Query
from superset.security import sync_role_definitions
from superset.sql_parse import SupersetQuery
@ -73,13 +74,13 @@ class CeleryTestCase(SupersetTestCase):
def get_query_by_name(self, sql):
session = db.session
query = session.query(models.Query).filter_by(sql=sql).first()
query = session.query(Query).filter_by(sql=sql).first()
session.close()
return query
def get_query_by_id(self, id):
session = db.session
query = session.query(models.Query).filter_by(id=id).first()
query = session.query(Query).filter_by(id=id).first()
session.close()
return query

5
tests/core_tests.py
View File

@ -16,6 +16,7 @@ from flask import escape
from superset import db, utils, appbuilder, sm, jinja_context, sql_lab
from superset.models import core as models
from superset.models.sql_lab import Query
from superset.views.core import DatabaseView
from superset.connectors.sqla.models import SqlaTable
@ -38,11 +39,11 @@ class CoreTests(SupersetTestCase):
)}
def setUp(self):
db.session.query(models.Query).delete()
db.session.query(Query).delete()
db.session.query(models.DatasourceAccessRequest).delete()
def tearDown(self):
db.session.query(models.Query).delete()
db.session.query(Query).delete()
def test_login(self):
resp = self.get_resp(

14
tests/sqllab_tests.py
View File

@ -10,7 +10,7 @@ import unittest
from flask_appbuilder.security.sqla import models as ab_models
from superset import db, utils, appbuilder, sm
from superset.models import core as models
from superset.models.sql_lab import Query
from .base_tests import SupersetTestCase
@ -22,7 +22,7 @@ class SqlLabTests(SupersetTestCase):
super(SqlLabTests, self).__init__(*args, **kwargs)
def run_some_queries(self):
db.session.query(models.Query).delete()
db.session.query(Query).delete()
db.session.commit()
self.run_sql(
"SELECT * FROM ab_user",
@ -39,7 +39,7 @@ class SqlLabTests(SupersetTestCase):
self.logout()
def tearDown(self):
db.session.query(models.Query).delete()
db.session.query(Query).delete()
db.session.commit()
self.logout()
@ -77,7 +77,7 @@ class SqlLabTests(SupersetTestCase):
astronaut,
password='general')
data = self.run_sql('SELECT * FROM ab_user', "3", user_name='gagarin')
db.session.query(models.Query).delete()
db.session.query(Query).delete()
db.session.commit()
self.assertLess(0, len(data['data']))
@ -102,7 +102,7 @@ class SqlLabTests(SupersetTestCase):
self.assertEquals(4, len(data))
now = datetime.now() + timedelta(days=1)
query = db.session.query(models.Query).filter_by(
query = db.session.query(Query).filter_by(
sql='SELECT * FROM ab_user LIMIT 1').first()
query.changed_on = now
db.session.commit()
@ -176,11 +176,11 @@ class SqlLabTests(SupersetTestCase):
self.run_some_queries()
self.login('admin')
first_query_time = (
db.session.query(models.Query)
db.session.query(Query)
.filter_by(sql='SELECT * FROM ab_user').one()
).start_time
second_query_time = (
db.session.query(models.Query)
db.session.query(Query)
.filter_by(sql='SELECT * FROM ab_permission').one()
).start_time
# Test search queries on time filter