SQL Lab - A multi-tab SQL editor (#514)

* Carapal react mockup

This is really just a mock up written in React to try different
components. It could become scaffolding to build a prototype, or not.

* Merging in Alanna's theme tweaks for SQL lab

* Tweak the display of the alert message in navbar

* Sketching the middleware refresh for Queries

* Adjustments

* Implement timer sync.

* CTAS

* Refactor the queries to be stored as a dict. (#994)

* Download csv endpoint. (#992)

* CSV download engdpoint.

* Use lower case booleans.

* Replcate loop with the object lookup by key.

* First changes for the sync

* Address comments

* Fix query deletions. Update only the queries from the store.

* Sync queries using tmp_id.

* simplify

* Fix the tests in the carapal. (#1023)

* Sync queries using tmp_id.

* Fix the unit tests

* Bux fixes. Pass 2.

* Tweakin' & linting

* Adding alpha label to the SQL LAb navbar entry

* Fixing the python unit tests

.gitignore

@@ -18,6 +18,8 @@ dist
 caravel.egg-info/
 app.db
 *.bak
+.idea
+*.sqllite
 
 # Node.js, webpack artifacts
 *.entry.js

MANIFEST.in

@@ -3,6 +3,7 @@ recursive-include caravel/static *
 recursive-exclude caravel/static/assets/node_modules *
 recursive-include caravel/static/assets/node_modules/font-awesome *
 recursive-exclude caravel/static/docs *
+recursive-exclude caravel/static/spec *
 recursive-exclude tests *
 recursive-include caravel/data *
 recursive-include caravel/migrations *

caravel/assets/.eslintignore

@@ -1,5 +1,5 @@
 node_modules/*
 vendor/*
-javascripts/dist/*
+dist/*
 stylesheets/*
 spec/*

caravel/assets/javascripts/SqlLab/TODO.md

@@ -0,0 +1,19 @@
+
+# TODO
+* Figure out how to organize the left panel, integrate Search
+* collapse sql beyond 10 lines
+* Security per-database (dropdown)
+* Get a to work
+
+## Cosmetic
+* Result set font is too big
+* lmiit/timer/buttons wrap
+* table label is transparent
+* SqlEditor buttons
+* use react-bootstrap-prompt for query title input
+* Make tabs look great
+
+# PROJECT
+* Write Runbook
+* Confirm backups
+* merge chef branch

caravel/assets/javascripts/SqlLab/actions.js

@@ -0,0 +1,116 @@
+export const RESET_STATE = 'RESET_STATE';
+export const ADD_QUERY_EDITOR = 'ADD_QUERY_EDITOR';
+export const REMOVE_QUERY_EDITOR = 'REMOVE_QUERY_EDITOR';
+export const ADD_TABLE = 'ADD_TABLE';
+export const REMOVE_TABLE = 'REMOVE_TABLE';
+export const START_QUERY = 'START_QUERY';
+export const STOP_QUERY = 'STOP_QUERY';
+export const END_QUERY = 'END_QUERY';
+export const REMOVE_QUERY = 'REMOVE_QUERY';
+export const EXPAND_TABLE = 'EXPAND_TABLE';
+export const COLLAPSE_TABLE = 'COLLAPSE_TABLE';
+export const QUERY_SUCCESS = 'QUERY_SUCCESS';
+export const QUERY_FAILED = 'QUERY_FAILED';
+export const QUERY_EDITOR_SETDB = 'QUERY_EDITOR_SETDB';
+export const QUERY_EDITOR_SET_SCHEMA = 'QUERY_EDITOR_SET_SCHEMA';
+export const QUERY_EDITOR_SET_TITLE = 'QUERY_EDITOR_SET_TITLE';
+export const QUERY_EDITOR_SET_AUTORUN = 'QUERY_EDITOR_SET_AUTORUN';
+export const QUERY_EDITOR_SET_SQL = 'QUERY_EDITOR_SET_SQL';
+export const SET_WORKSPACE_DB = 'SET_WORKSPACE_DB';
+export const ADD_WORKSPACE_QUERY = 'ADD_WORKSPACE_QUERY';
+export const REMOVE_WORKSPACE_QUERY = 'REMOVE_WORKSPACE_QUERY';
+export const SET_ACTIVE_QUERY_EDITOR = 'SET_ACTIVE_QUERY_EDITOR';
+export const ADD_ALERT = 'ADD_ALERT';
+export const REMOVE_ALERT = 'REMOVE_ALERT';
+export const REFRESH_QUERIES = 'REFRESH_QUERIES';
+
+export function resetState() {
+  return { type: RESET_STATE };
+}
+
+export function addQueryEditor(queryEditor) {
+  return { type: ADD_QUERY_EDITOR, queryEditor };
+}
+
+export function addAlert(alert) {
+  return { type: ADD_ALERT, alert };
+}
+
+export function removeAlert(alert) {
+  return { type: REMOVE_ALERT, alert };
+}
+
+export function setActiveQueryEditor(queryEditor) {
+  return { type: SET_ACTIVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQueryEditor(queryEditor) {
+  return { type: REMOVE_QUERY_EDITOR, queryEditor };
+}
+
+export function removeQuery(query) {
+  return { type: REMOVE_QUERY, query };
+}
+
+export function queryEditorSetDb(queryEditor, dbId) {
+  return { type: QUERY_EDITOR_SETDB, queryEditor, dbId };
+}
+
+export function queryEditorSetSchema(queryEditor, schema) {
+  return { type: QUERY_EDITOR_SET_SCHEMA, queryEditor, schema };
+}
+
+export function queryEditorSetAutorun(queryEditor, autorun) {
+  return { type: QUERY_EDITOR_SET_AUTORUN, queryEditor, autorun };
+}
+
+export function queryEditorSetTitle(queryEditor, title) {
+  return { type: QUERY_EDITOR_SET_TITLE, queryEditor, title };
+}
+
+export function queryEditorSetSql(queryEditor, sql) {
+  return { type: QUERY_EDITOR_SET_SQL, queryEditor, sql };
+}
+
+export function addTable(table) {
+  return { type: ADD_TABLE, table };
+}
+
+export function expandTable(table) {
+  return { type: EXPAND_TABLE, table };
+}
+
+export function collapseTable(table) {
+  return { type: COLLAPSE_TABLE, table };
+}
+
+export function removeTable(table) {
+  return { type: REMOVE_TABLE, table };
+}
+
+export function startQuery(query) {
+  return { type: START_QUERY, query };
+}
+
+export function stopQuery(query) {
+  return { type: STOP_QUERY, query };
+}
+
+export function querySuccess(query, results) {
+  return { type: QUERY_SUCCESS, query, results };
+}
+
+export function queryFailed(query, msg) {
+  return { type: QUERY_FAILED, query, msg };
+}
+
+export function addWorkspaceQuery(query) {
+  return { type: ADD_WORKSPACE_QUERY, query };
+}
+
+export function removeWorkspaceQuery(query) {
+  return { type: REMOVE_WORKSPACE_QUERY, query };
+}
+export function refreshQueries(alteredQueries) {
+  return { type: REFRESH_QUERIES, alteredQueries };
+}

caravel/assets/javascripts/SqlLab/common.js

@@ -0,0 +1,6 @@
+export const STATE_BSSTYLE_MAP = {
+  failed: 'danger',
+  pending: 'info',
+  running: 'warning',
+  success: 'success',
+};

caravel/assets/javascripts/SqlLab/components/Alerts.jsx

@@ -0,0 +1,41 @@
+import React from 'react';
+import { Alert } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+class Alerts extends React.Component {
+  removeAlert(alert) {
+    this.props.actions.removeAlert(alert);
+  }
+  render() {
+    const alerts = this.props.alerts.map((alert) =>
+      <Alert
+        bsStyle={alert.bsStyle}
+        style={{ width: '500px', textAlign: 'midddle', margin: '10px auto' }}
+      >
+        {alert.msg}
+        <i
+          className="fa fa-close pull-right"
+          onClick={this.removeAlert.bind(this, alert)}
+          style={{ cursor: 'pointer' }}
+        />
+      </Alert>
+    );
+    return (
+      <div>{alerts}</div>
+    );
+  }
+}
+
+Alerts.propTypes = {
+  alerts: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(Alerts);

caravel/assets/javascripts/SqlLab/components/ButtonWithTooltip.jsx

@@ -0,0 +1,48 @@
+import React from 'react';
+import { Button, OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+const ButtonWithTooltip = (props) => {
+  let tooltip = (
+    <Tooltip id="tooltip">
+      {props.tooltip}
+    </Tooltip>
+  );
+  return (
+    <OverlayTrigger
+      overlay={tooltip}
+      delayShow={300}
+      placement={props.placement}
+      delayHide={150}
+    >
+      <Button
+        onClick={props.onClick}
+        bsStyle={props.bsStyle}
+        bsSize={props.bsSize}
+        disabled={props.disabled}
+        className={props.className}
+      >
+          {props.children}
+      </Button>
+    </OverlayTrigger>
+  );
+};
+
+ButtonWithTooltip.defaultProps = {
+  onClick: () => {},
+  disabled: false,
+  placement: 'top',
+  bsStyle: 'default',
+};
+
+ButtonWithTooltip.propTypes = {
+  bsSize: React.PropTypes.string,
+  bsStyle: React.PropTypes.string,
+  children: React.PropTypes.element,
+  className: React.PropTypes.string,
+  disabled: React.PropTypes.bool,
+  onClick: React.PropTypes.func,
+  placement: React.PropTypes.string,
+  tooltip: React.PropTypes.string,
+};
+
+export default ButtonWithTooltip;

caravel/assets/javascripts/SqlLab/components/LeftPane.jsx

@@ -0,0 +1,62 @@
+import React from 'react';
+import { Alert, Button } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryLink from './QueryLink';
+
+import 'react-select/dist/react-select.css';
+
+const LeftPane = (props) => {
+  let queryElements;
+  if (props.workspaceQueries.length > 0) {
+    queryElements = props.workspaceQueries.map((q) => <QueryLink query={q} />);
+  } else {
+    queryElements = (
+      <Alert bsStyle="info">
+        Use the save button on the SQL editor to save a query
+        into this section for future reference.
+      </Alert>
+    );
+  }
+  return (
+    <div>
+      <div className="panel panel-default LeftPane">
+        <div className="panel-heading">
+          <div className="panel-title">
+            Saved Queries
+          </div>
+        </div>
+        <div className="panel-body">
+          {queryElements}
+        </div>
+      </div>
+      <br /><br />
+      <Button onClick={props.actions.resetState.bind(this)} bsStyle="danger">
+        Reset State
+      </Button>
+    </div>
+  );
+};
+
+LeftPane.propTypes = {
+  workspaceQueries: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+LeftPane.defaultProps = {
+  workspaceQueries: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    workspaceQueries: state.workspaceQueries,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(LeftPane);

caravel/assets/javascripts/SqlLab/components/Link.jsx

@@ -0,0 +1,54 @@
+import React from 'react';
+import { OverlayTrigger, Tooltip } from 'react-bootstrap';
+
+
+class Link extends React.Component {
+  render() {
+    let tooltip = (
+      <Tooltip id="tooltip">
+        {this.props.tooltip}
+      </Tooltip>
+    );
+    const link = (
+      <a
+        href={this.props.href}
+        onClick={this.props.onClick}
+        style={this.props.style}
+        className={'Link ' + this.props.className}
+      >
+          {this.props.children}
+      </a>
+    );
+    if (this.props.tooltip) {
+      return (
+        <OverlayTrigger
+          overlay={tooltip}
+          placement={this.props.placement}
+          delayShow={300}
+          delayHide={150}
+        >
+          {link}
+        </OverlayTrigger>
+      );
+    }
+    return link;
+  }
+}
+Link.propTypes = {
+  children: React.PropTypes.object,
+  className: React.PropTypes.string,
+  href: React.PropTypes.string,
+  onClick: React.PropTypes.func,
+  placement: React.PropTypes.string,
+  style: React.PropTypes.object,
+  tooltip: React.PropTypes.string,
+};
+Link.defaultProps = {
+  disabled: false,
+  href: '#',
+  tooltip: null,
+  placement: 'top',
+  onClick: () => {},
+};
+
+export default Link;

caravel/assets/javascripts/SqlLab/components/QueryAutoRefresh.jsx

@@ -0,0 +1,55 @@
+import React from 'react';
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+
+const $ = require('jquery');
+
+
+class QueryAutoRefresh extends React.Component {
+  componentWillMount() {
+    this.startTimer();
+  }
+  componentWillUnmount() {
+    this.stopTimer();
+  }
+  startTimer() {
+    if (!(this.timer)) {
+      this.timer = setInterval(this.stopwatch.bind(this), 5000);
+    }
+  }
+  stopTimer() {
+    clearInterval(this.timer);
+    this.timer = null;
+  }
+  stopwatch() {
+    const url = '/caravel/queries/0';
+    // No updates in case of failure.
+    $.getJSON(url, (data, status) => {
+      if (status === 'success') {
+        this.props.actions.refreshQueries(data);
+      }
+    });
+  }
+  render() {
+    return null;
+  }
+}
+QueryAutoRefresh.propTypes = {
+  actions: React.PropTypes.object,
+};
+QueryAutoRefresh.defaultProps = {
+  // queries: null,
+};
+
+function mapStateToProps() {
+  return {};
+}
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryAutoRefresh);

caravel/assets/javascripts/SqlLab/components/QueryHistory.jsx

@@ -0,0 +1,58 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import QueryTable from './QueryTable';
+import { Alert } from 'react-bootstrap';
+
+const QueryHistory = (props) => {
+  const activeQeId = props.tabHistory[props.tabHistory.length - 1];
+  const queriesArray = [];
+  for (const id in props.queries) {
+    if (props.queries[id].sqlEditorId === activeQeId) {
+      queriesArray.push(props.queries[id]);
+    }
+  }
+  if (queriesArray.length > 0) {
+    return (
+      <QueryTable
+        columns={[
+          'state', 'started', 'duration', 'progress',
+          'rows', 'sql', 'output', 'actions',
+        ]}
+        queries={queriesArray}
+      />
+    );
+  }
+  return (
+    <Alert bsStyle="info">
+      No query history yet...
+    </Alert>
+  );
+};
+
+QueryHistory.defaultProps = {
+  queries: {},
+};
+
+QueryHistory.propTypes = {
+  queries: React.PropTypes.object,
+  tabHistory: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+function mapStateToProps(state) {
+  return {
+    queries: state.queries,
+    tabHistory: state.tabHistory,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryHistory);

caravel/assets/javascripts/SqlLab/components/QueryLink.jsx

@@ -0,0 +1,65 @@
+import React from 'react';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class QueryLink extends React.Component {
+  popTab() {
+    const qe = {
+      id: shortid.generate(),
+      title: this.props.query.title,
+      dbId: this.props.query.dbId,
+      autorun: false,
+      sql: this.props.query.sql,
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  render() {
+    return (
+      <div>
+        <div className="clearfix">
+          <div className="pull-left">
+            <a
+              href="#"
+              tooltip="Pop this query in a new tab"
+              onClick={this.popTab.bind(this)}
+            >
+              {this.props.query.title}
+            </a>
+          </div>
+          <div className="pull-right">
+            <Link
+              onClick={this.props.actions.removeWorkspaceQuery.bind(this, this.props.query)}
+              tooltip="Remove query from workspace"
+              href="#"
+            >
+              &times;
+            </Link>
+          </div>
+        </div>
+        <hr />
+      </div>
+    );
+  }
+}
+
+QueryLink.propTypes = {
+  query: React.PropTypes.object,
+  actions: React.PropTypes.object,
+};
+
+QueryLink.defaultProps = {
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(QueryLink);
+

caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx

@@ -0,0 +1,73 @@
+import React from 'react';
+import Select from 'react-select';
+import { Button } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import QueryTable from './QueryTable';
+
+class QuerySearch extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      queryText: '',
+    };
+  }
+  changeQueryText(value) {
+    this.setState({ queryText: value });
+  }
+  render() {
+    const queries = this.props.queries;
+    return (
+      <div>
+        <div className="pane-cell pane-west m-t-5">
+          <div className="panel panel-default Workspace">
+            <div className="panel-heading">
+              <h6>
+                <i className="fa fa-search" /> Search Queries
+              </h6>
+            </div>
+            <div className="panel-body">
+              <input type="text" className="form-control" placeholder="Query Text" />
+              <Select
+                name="select-user"
+                placeholder="[User]"
+                options={['maxime_beauchemin', 'someone else']}
+                value={'maxime_beauchemin'}
+                className="m-t-10"
+                autosize={false}
+              />
+            </div>
+          </div>
+        </div>
+        <div className="pane-cell">
+          <QueryTable
+            columns={['state', 'started', 'duration', 'rows', 'sql', 'actions']}
+            queries={queries}
+          />
+        </div>
+        <Button>Search!</Button>
+      </div>
+    );
+  }
+}
+QuerySearch.propTypes = {
+  queries: React.PropTypes.array,
+};
+QuerySearch.defaultProps = {
+  queries: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    queries: state.queries,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QuerySearch);

caravel/assets/javascripts/SqlLab/components/QueryTable.jsx

@@ -0,0 +1,142 @@
+import React from 'react';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import moment from 'moment';
+import { Table } from 'reactable';
+import { ProgressBar } from 'react-bootstrap';
+
+import SyntaxHighlighter from 'react-syntax-highlighter';
+import { github } from 'react-syntax-highlighter/dist/styles';
+
+import Link from './Link';
+import VisualizeModal from './VisualizeModal';
+import { STATE_BSSTYLE_MAP } from '../common';
+import { fDuration } from '../../modules/dates';
+
+
+class QueryTable extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      showVisualizeModal: false,
+      activeQuery: null,
+    };
+  }
+  hideVisualizeModal() {
+    this.setState({ showVisualizeModal: false });
+  }
+  showVisualizeModal(query) {
+    this.setState({ showVisualizeModal: true });
+    this.setState({ activeQuery: query });
+  }
+  restoreSql(query) {
+    this.props.actions.queryEditorSetSql({ id: query.sqlEditorId }, query.sql);
+  }
+  notImplemented() {
+    alert('Not implemented yet!');
+  }
+  render() {
+    const data = this.props.queries.map((query) => {
+      const q = Object.assign({}, query);
+      if (q.endDttm) {
+        q.duration = fDuration(q.startDttm, q.endDttm);
+      }
+      q.started = moment.utc(q.startDttm).format('HH:mm:ss');
+      const source = q.ctas ? q.executedSql : q.sql;
+      q.sql = (
+        <SyntaxHighlighter language="sql" style={github}>
+          {source || ''}
+        </SyntaxHighlighter>
+      );
+      q.output = q.tempTable;
+      q.progress = (
+        <ProgressBar
+          style={{ width: '75px' }}
+          striped
+          now={q.progress}
+          label={`${q.progress}%`}
+        />
+      );
+      let errorTooltip;
+      if (q.errorMessage) {
+        errorTooltip = (
+          <Link tooltip={q.errorMessage}>
+            <i className="fa fa-exclamation-circle text-danger" />
+          </Link>
+        );
+      }
+      q.state = (
+        <div>
+          <span className={'m-r-3 label label-' + STATE_BSSTYLE_MAP[q.state]}>
+            {q.state}
+          </span>
+          {errorTooltip}
+        </div>
+      );
+      q.actions = (
+        <div style={{ width: '75px' }}>
+          <Link
+            className="fa fa-line-chart m-r-3"
+            tooltip="Visualize the data out of this query"
+            onClick={this.showVisualizeModal.bind(this, query)}
+          />
+          <Link
+            className="fa fa-pencil m-r-3"
+            onClick={this.restoreSql.bind(this, query)}
+            tooltip="Overwrite text in editor with a query on this table"
+            placement="top"
+          />
+          <Link
+            className="fa fa-plus-circle m-r-3"
+            onClick={self.notImplemented}
+            tooltip="Run query in a new tab"
+            placement="top"
+          />
+          <Link
+            className="fa fa-trash m-r-3"
+            tooltip="Remove query from log"
+            onClick={this.props.actions.removeQuery.bind(this, query)}
+          />
+        </div>
+      );
+
+      return q;
+    }).reverse();
+    return (
+      <div>
+        <VisualizeModal
+          show={this.state.showVisualizeModal}
+          query={this.state.activeQuery}
+          onHide={this.hideVisualizeModal.bind(this)}
+        />
+        <Table
+          columns={this.props.columns}
+          className="table table-condensed"
+          data={data}
+        />
+      </div>
+    );
+  }
+}
+QueryTable.propTypes = {
+  columns: React.PropTypes.array,
+  actions: React.PropTypes.object,
+  queries: React.PropTypes.array,
+};
+QueryTable.defaultProps = {
+  columns: ['state', 'started', 'duration', 'progress', 'rows', 'sql', 'actions'],
+  queries: [],
+};
+
+function mapStateToProps() {
+  return {};
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(QueryTable);

caravel/assets/javascripts/SqlLab/components/ResultSet.jsx

@@ -0,0 +1,101 @@
+import React from 'react';
+import { Alert, Button, ButtonGroup } from 'react-bootstrap';
+import { Table } from 'reactable';
+
+import VisualizeModal from './VisualizeModal';
+
+
+class ResultSet extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      searchText: '',
+      showModal: false,
+    };
+  }
+  shouldComponentUpdate(nextProps, nextState) {
+    return (
+      this.state.searchText !== nextState.searchText ||
+      this.state.showModal !== nextState.showModal
+    );
+  }
+  changeSearch(event) {
+    this.setState({ searchText: event.target.value });
+  }
+  showModal() {
+    this.setState({ showModal: true });
+  }
+  hideModal() {
+    this.setState({ showModal: false });
+  }
+  render() {
+    const results = this.props.query.results;
+    let controls = <div className="noControls" />;
+    if (this.props.showControls) {
+      controls = (
+        <div className="ResultSetControls">
+          <div className="clearfix">
+            <div className="pull-left">
+              <ButtonGroup>
+                <Button
+                  bsSize="small"
+                  onClick={this.showModal.bind(this)}
+                >
+                  <i className="fa fa-line-chart m-l-1" /> Visualize
+                </Button>
+                <Button bsSize="small">
+                  <i className="fa fa-file-text-o" /> .CSV
+                </Button>
+              </ButtonGroup>
+            </div>
+            <div className="pull-right">
+              <input
+                type="text"
+                onChange={this.changeSearch.bind(this)}
+                className="form-control input-sm"
+                placeholder="Search Results"
+              />
+            </div>
+          </div>
+        </div>
+      );
+    }
+    if (results && results.data.length > 0) {
+      return (
+        <div>
+          <VisualizeModal
+            show={this.state.showModal}
+            query={this.props.query}
+            onHide={this.hideModal.bind(this)}
+          />
+          {controls}
+          <div className="ResultSet">
+            <Table
+              data={results.data}
+              columns={results.columns}
+              sortable
+              className="table table-condensed table-bordered"
+              filterBy={this.state.searchText}
+              filterable={results.columns}
+              hideFilterInput
+            />
+          </div>
+        </div>
+      );
+    }
+    return (<Alert bsStyle="warning">The query returned no data</Alert>);
+  }
+}
+ResultSet.propTypes = {
+  query: React.PropTypes.object,
+  showControls: React.PropTypes.boolean,
+  search: React.PropTypes.boolean,
+  searchText: React.PropTypes.string,
+};
+ResultSet.defaultProps = {
+  showControls: true,
+  search: true,
+  searchText: '',
+};
+
+export default ResultSet;

caravel/assets/javascripts/SqlLab/components/SouthPane.jsx

@@ -0,0 +1,46 @@
+import { Alert, Panel, Tab, Tabs } from 'react-bootstrap';
+import QueryHistory from './QueryHistory';
+import ResultSet from './ResultSet';
+import React from 'react';
+
+const SouthPane = function (props) {
+  let results = <div />;
+  if (props.latestQuery) {
+    if (props.latestQuery.state === 'running') {
+      results = (
+        <img className="loading" alt="Loading.." src="/static/assets/images/loading.gif" />
+      );
+    } else if (props.latestQuery.state === 'failed') {
+      results = <Alert bsStyle="danger">{props.latestQuery.msg}</Alert>;
+    } else if (props.latestQuery.state === 'success') {
+      results = <ResultSet showControls query={props.latestQuery} />;
+    }
+  } else {
+    results = <Alert bsStyle="info">Run a query to display results here</Alert>;
+  }
+  return (
+    <Tabs bsStyle="tabs">
+      <Tab title="Results" eventKey={1}>
+        <Panel>
+          <div style={{ overflow: 'auto' }}>
+            {results}
+          </div>
+        </Panel>
+      </Tab>
+      <Tab title="Query History" eventKey={2}>
+        <Panel>
+          <QueryHistory />
+        </Panel>
+      </Tab>
+    </Tabs>
+  );
+};
+
+SouthPane.propTypes = {
+  latestQuery: React.PropTypes.object,
+};
+
+SouthPane.defaultProps = {
+};
+
+export default SouthPane;

caravel/assets/javascripts/SqlLab/components/SqlEditor.jsx

@@ -0,0 +1,292 @@
+const $ = require('jquery');
+import { now } from '../../modules/dates';
+import React from 'react';
+import {
+  Button,
+  ButtonGroup,
+  FormGroup,
+  InputGroup,
+  Form,
+  FormControl,
+  DropdownButton,
+  Label,
+  MenuItem,
+  OverlayTrigger,
+  Tooltip,
+} from 'react-bootstrap';
+
+import AceEditor from 'react-ace';
+import 'brace/mode/sql';
+import 'brace/theme/github';
+import 'brace/ext/language_tools';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+
+import shortid from 'shortid';
+import ButtonWithTooltip from './ButtonWithTooltip';
+import SouthPane from './SouthPane';
+import Timer from './Timer';
+
+import SqlEditorTopToolbar from './SqlEditorTopToolbar';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditor extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      autorun: props.queryEditor.autorun,
+      sql: props.queryEditor.sql,
+      ctas: '',
+    };
+  }
+  componentDidMount() {
+    this.onMount();
+  }
+  onMount() {
+    if (this.state.autorun) {
+      this.setState({ autorun: false });
+      this.props.actions.queryEditorSetAutorun(this.props.queryEditor, false);
+      this.startQuery();
+    }
+  }
+  runQuery() {
+    this.startQuery();
+  }
+  startQuery(runAsync = false, ctas = false) {
+    const that = this;
+    const query = {
+      dbId: this.props.queryEditor.dbId,
+      id: shortid.generate(),
+      progress: 0,
+      sql: this.props.queryEditor.sql,
+      sqlEditorId: this.props.queryEditor.id,
+      startDttm: now(),
+      state: 'running',
+      tab: this.props.queryEditor.title,
+    };
+    if (runAsync) {
+      query.state = 'pending';
+    }
+
+    // Execute the Query
+    that.props.actions.startQuery(query);
+
+    const sqlJsonUrl = '/caravel/sql_json/';
+    const sqlJsonRequest = {
+      async: runAsync,
+      client_id: query.id,
+      database_id: this.props.queryEditor.dbId,
+      json: true,
+      schema: this.props.queryEditor.schema,
+      select_as_cta: ctas,
+      sql: this.props.queryEditor.sql,
+      sql_editor_id: this.props.queryEditor.id,
+      tab: this.props.queryEditor.title,
+      tmp_table_name: this.state.ctas,
+    };
+    $.ajax({
+      type: 'POST',
+      dataType: 'json',
+      url: sqlJsonUrl,
+      data: sqlJsonRequest,
+      success(results) {
+        if (!runAsync) {
+          that.props.actions.querySuccess(query, results);
+        }
+      },
+      error(err) {
+        let msg;
+        try {
+          msg = err.responseJSON.error;
+        } catch (e) {
+          msg = (err.responseText) ? err.responseText : e;
+        }
+        if (typeof(msg) !== 'string') {
+          msg = JSON.stringify(msg);
+        }
+        that.props.actions.queryFailed(query, msg);
+      },
+    });
+  }
+  stopQuery() {
+    this.props.actions.stopQuery(this.props.latestQuery);
+  }
+  createTableAs() {
+    this.startQuery(true, true);
+  }
+  textChange(text) {
+    this.setState({ sql: text });
+    this.props.actions.queryEditorSetSql(this.props.queryEditor, text);
+  }
+  addWorkspaceQuery() {
+    this.props.actions.addWorkspaceQuery({
+      id: shortid.generate(),
+      sql: this.state.sql,
+      dbId: this.props.queryEditor.dbId,
+      schema: this.props.queryEditor.schema,
+      title: this.props.queryEditor.title,
+    });
+  }
+  ctasChange() {}
+  visualize() {}
+  ctasChanged(event) {
+    this.setState({ ctas: event.target.value });
+  }
+  render() {
+    let runButtons = (
+      <ButtonGroup bsSize="small" className="inline m-r-5 pull-left">
+        <Button
+          bsSize="small"
+          bsStyle="primary"
+          style={{ width: '100px' }}
+          onClick={this.runQuery.bind(this)}
+          disabled={!(this.props.queryEditor.dbId)}
+        >
+          <i className="fa fa-table" /> Run Query
+        </Button>
+      </ButtonGroup>
+    );
+    if (this.props.latestQuery && this.props.latestQuery.state === 'running') {
+      runButtons = (
+        <ButtonGroup bsSize="small" className="inline m-r-5 pull-left">
+          <Button
+            bsStyle="primary"
+            bsSize="small"
+            style={{ width: '100px' }}
+            onClick={this.stopQuery.bind(this)}
+          >
+            <a className="fa fa-stop" /> Stop
+          </Button>
+        </ButtonGroup>
+      );
+    }
+    const rightButtons = (
+      <ButtonGroup className="inlineblock">
+        <ButtonWithTooltip
+          tooltip="Save this query in your workspace"
+          placement="left"
+          bsSize="small"
+          onClick={this.addWorkspaceQuery.bind(this)}
+        >
+          <i className="fa fa-save" />&nbsp;
+        </ButtonWithTooltip>
+        <DropdownButton
+          id="ddbtn-export"
+          pullRight
+          bsSize="small"
+          title={<i className="fa fa-file-o" />}
+        >
+          <MenuItem
+            onClick={this.notImplemented}
+          >
+            <i className="fa fa-file-text-o" /> export to .csv
+          </MenuItem>
+          <MenuItem
+            onClick={this.notImplemented}
+          >
+            <i className="fa fa-file-code-o" /> export to .json
+          </MenuItem>
+        </DropdownButton>
+      </ButtonGroup>
+    );
+    let limitWarning = null;
+    const rowLimit = 1000;
+    if (this.props.latestQuery && this.props.latestQuery.rows === rowLimit) {
+      const tooltip = (
+        <Tooltip id="tooltip">
+          It appears that the number of rows in the query results displayed
+          was limited on the server side to the {rowLimit} limit.
+        </Tooltip>
+      );
+      limitWarning = (
+        <OverlayTrigger placement="left" overlay={tooltip}>
+          <Label bsStyle="warning" className="m-r-5">LIMIT</Label>
+        </OverlayTrigger>
+      );
+    }
+    const editorBottomBar = (
+      <div className="sql-toolbar clearfix">
+        <div className="pull-left">
+          <Form inline>
+            {runButtons}
+            <FormGroup>
+              <InputGroup>
+                <FormControl
+                  type="text"
+                  bsSize="small"
+                  className="input-sm"
+                  placeholder="new table name"
+                  onChange={this.ctasChanged.bind(this)}
+                />
+                <InputGroup.Button>
+                  <Button
+                    bsSize="small"
+                    disabled={this.state.ctas.length === 0}
+                    onClick={this.createTableAs.bind(this)}
+                  >
+                    <i className="fa fa-table" /> CTAS
+                  </Button>
+                </InputGroup.Button>
+              </InputGroup>
+            </FormGroup>
+          </Form>
+        </div>
+        <div className="pull-right">
+          {limitWarning}
+          <Timer query={this.props.latestQuery} />
+          {rightButtons}
+        </div>
+      </div>
+    );
+    return (
+      <div className="SqlEditor">
+        <div>
+          <div>
+            <SqlEditorTopToolbar queryEditor={this.props.queryEditor} />
+            <AceEditor
+              mode="sql"
+              name={this.props.queryEditor.id}
+              theme="github"
+              minLines={5}
+              maxLines={30}
+              onChange={this.textChange.bind(this)}
+              height="200px"
+              width="100%"
+              editorProps={{ $blockScrolling: true }}
+              enableBasicAutocompletion
+              value={this.props.queryEditor.sql}
+            />
+            {editorBottomBar}
+            <br />
+            <SouthPane latestQuery={this.props.latestQuery} sqlEditor={this} />
+          </div>
+        </div>
+      </div>
+    );
+  }
+}
+
+SqlEditor.propTypes = {
+  queryEditor: React.PropTypes.object,
+  actions: React.PropTypes.object,
+  latestQuery: React.PropTypes.object,
+};
+
+SqlEditor.defaultProps = {
+};
+
+function mapStateToProps() {
+  return {};
+}
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditor);

caravel/assets/javascripts/SqlLab/components/SqlEditorTopToolbar.jsx

@@ -0,0 +1,271 @@
+const $ = window.$ = require('jquery');
+import React from 'react';
+import { Label, OverlayTrigger, Popover } from 'react-bootstrap';
+
+import { bindActionCreators } from 'redux';
+import { connect } from 'react-redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+import Select from 'react-select';
+import Link from './Link';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class SqlEditorTopToolbar extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      databaseLoading: false,
+      databaseOptions: [],
+      schemaLoading: false,
+      schemaOptions: [],
+      tableLoading: false,
+      tableOptions: [],
+    };
+  }
+  componentWillMount() {
+    this.fetchDatabaseOptions();
+    this.fetchSchemas();
+    this.fetchTables();
+  }
+  getSql(table) {
+    let cols = '';
+    table.columns.forEach(function (col, i) {
+      cols += col.name;
+      if (i < table.columns.length - 1) {
+        cols += ', ';
+      }
+    });
+    return `SELECT ${cols}\nFROM ${table.name}`;
+  }
+  selectStar(table) {
+    this.props.actions.queryEditorSetSql(this.props.queryEditor, this.getSql(table));
+  }
+  popTab(table) {
+    const qe = {
+      id: shortid.generate(),
+      title: table.name,
+      dbId: table.dbId,
+      schema: table.schema,
+      autorun: true,
+      sql: this.getSql(table),
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  fetchTables(dbId, schema) {
+    const actualDbId = dbId || this.props.queryEditor.dbId;
+    if (actualDbId) {
+      const actualSchema = schema || this.props.queryEditor.schema;
+      this.setState({ tableLoading: true });
+      this.setState({ tableOptions: [] });
+      const url = `/caravel/tables/${actualDbId}/${actualSchema}`;
+      $.get(url, (data) => {
+        let tableOptions = data.tables.map((s) => ({ value: s, label: s }));
+        const views = data.views.map((s) => ({ value: s, label: '[view] ' + s }));
+        tableOptions = [...tableOptions, ...views];
+        this.setState({ tableOptions });
+        this.setState({ tableLoading: false });
+      });
+    }
+  }
+  changeSchema(schemaOpt) {
+    const schema = (schemaOpt) ? schemaOpt.value : null;
+    this.props.actions.queryEditorSetSchema(this.props.queryEditor, schema);
+    this.fetchTables(this.props.queryEditor.dbId, schema);
+  }
+  fetchSchemas(dbId) {
+    const actualDbId = dbId || this.props.queryEditor.dbId;
+    if (actualDbId) {
+      this.setState({ schemaLoading: true });
+      const url = `/databasetablesasync/api/read?_flt_0_id=${actualDbId}`;
+      $.get(url, (data) => {
+        const schemas = data.result[0].all_schema_names;
+        const schemaOptions = schemas.map((s) => ({ value: s, label: s }));
+        this.setState({ schemaOptions });
+        this.setState({ schemaLoading: false });
+      });
+    }
+  }
+  changeDb(db) {
+    const val = (db) ? db.value : null;
+    this.setState({ schemaOptions: [] });
+    this.props.actions.queryEditorSetDb(this.props.queryEditor, val);
+    if (!(db)) {
+      this.setState({ tableOptions: [] });
+      return;
+    }
+    this.fetchTables(val, this.props.queryEditor.schema);
+    this.fetchSchemas(val);
+  }
+  fetchDatabaseOptions() {
+    this.setState({ databaseLoading: true });
+    const url = '/databaseasync/api/read';
+    $.get(url, (data) => {
+      const options = data.result.map((db) => ({ value: db.id, label: db.database_name }));
+      this.setState({ databaseOptions: options });
+      this.setState({ databaseLoading: false });
+    });
+  }
+  closePopover(ref) {
+    this.refs[ref].hide();
+  }
+  changeTable(tableOpt) {
+    const tableName = tableOpt.value;
+    const qe = this.props.queryEditor;
+    const url = `/caravel/table/${qe.dbId}/${tableName}/${qe.schema}/`;
+    $.get(url, (data) => {
+      this.props.actions.addTable({
+        id: shortid.generate(),
+        dbId: this.props.queryEditor.dbId,
+        queryEditorId: this.props.queryEditor.id,
+        name: data.name,
+        schema: qe.schema,
+        columns: data.columns,
+        expanded: true,
+        showPopup: true,
+      });
+    })
+    .fail(() => {
+      this.props.actions.addAlert({
+        msg: 'Error occurred while fetching metadata',
+        bsStyle: 'danger',
+      });
+    });
+  }
+  render() {
+    const tables = this.props.tables.filter((t) => (t.queryEditorId === this.props.queryEditor.id));
+    const tablesEls = tables.map((table) => {
+      let cols = [];
+      if (table.columns) {
+        cols = table.columns.map((col) => (
+          <div className="clearfix">
+            <div className="pull-left m-r-10">{col.name}</div>
+            <div className="pull-right text-muted"> {col.type}</div>
+          </div>
+        ));
+      }
+      const popoverId = 'tblPopover_' + table.name;
+      const popoverTop = (
+        <div className="clearfix">
+          <div className="pull-left">
+            <Link
+              className="fa fa-pencil"
+              onClick={this.selectStar.bind(this, table)}
+              tooltip="Overwrite text in editor with a query on this table"
+              placement="left"
+              href="#"
+            />
+            <Link
+              className="fa fa-plus-circle"
+              onClick={this.popTab.bind(this, table)}
+              tooltip="Run query in a new tab"
+              placement="left"
+              href="#"
+            />
+          </div>
+          <div className="pull-right">
+            <Link
+              className="fa fa-close"
+              onClick={this.closePopover.bind(this, popoverId)}
+              href="#"
+            />
+          </div>
+        </div>
+      );
+      const popover = (
+        <Popover
+          id={popoverId}
+          className="tablePopover"
+          title={popoverTop}
+        >
+          {cols}
+        </Popover>
+      );
+      return (
+        <Label className="m-r-5 table-label" style={{ fontSize: '100%' }}>
+          <OverlayTrigger
+            trigger="click"
+            placement="bottom"
+            overlay={popover}
+            ref={popoverId}
+          >
+            <span className="m-r-5" style={{ cursor: 'pointer' }}>
+              {table.name}
+            </span>
+          </OverlayTrigger>
+          <i
+            className="fa fa-close"
+            style={{ cursor: 'pointer' }}
+            onClick={this.props.actions.removeTable.bind(this, table)}
+          />
+        </Label>
+      );
+    });
+    return (
+      <div className="clearfix sql-toolbar">
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-db"
+            placeholder="[Database]"
+            options={this.state.databaseOptions}
+            value={this.props.queryEditor.dbId}
+            isLoading={this.state.databaseLoading}
+            autosize={false}
+            onChange={this.changeDb.bind(this)}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-schema"
+            placeholder="[Schema]"
+            options={this.state.schemaOptions}
+            value={this.props.queryEditor.schema}
+            isLoading={this.state.schemaLoading}
+            autosize={false}
+            onChange={this.changeSchema.bind(this)}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          <Select
+            name="select-table"
+            ref="selectTable"
+            isLoading={this.state.tableLoading}
+            placeholder="Add a table"
+            autosize={false}
+            value={this.state.tableName}
+            onChange={this.changeTable.bind(this)}
+            options={this.state.tableOptions}
+          />
+        </div>
+        <div className="pull-left m-r-5">
+          {tablesEls}
+        </div>
+      </div>
+    );
+  }
+}
+
+SqlEditorTopToolbar.propTypes = {
+  queryEditor: React.PropTypes.object,
+  tables: React.PropTypes.array,
+  actions: React.PropTypes.object,
+};
+
+SqlEditorTopToolbar.defaultProps = {
+  tables: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    tables: state.tables,
+  };
+}
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(SqlEditorTopToolbar);

caravel/assets/javascripts/SqlLab/components/TabbedSqlEditors.jsx

@@ -0,0 +1,120 @@
+import React from 'react';
+import { DropdownButton, MenuItem, Tab, Tabs } from 'react-bootstrap';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import SqlEditor from './SqlEditor';
+import shortid from 'shortid';
+
+let queryCount = 1;
+
+class QueryEditors extends React.Component {
+  renameTab(qe) {
+    const newTitle = prompt('Enter a new title for the tab');
+    if (newTitle) {
+      this.props.actions.queryEditorSetTitle(qe, newTitle);
+    }
+  }
+  activeQueryEditor() {
+    const qeid = this.props.tabHistory[this.props.tabHistory.length - 1];
+    for (let i = 0; i < this.props.queryEditors.length; i++) {
+      const qe = this.props.queryEditors[i];
+      if (qe.id === qeid) {
+        return qe;
+      }
+    }
+    return null;
+  }
+  newQueryEditor() {
+    queryCount++;
+    const activeQueryEditor = this.activeQueryEditor();
+    const qe = {
+      id: shortid.generate(),
+      title: `Untitled Query ${queryCount}`,
+      dbId: (activeQueryEditor) ? activeQueryEditor.dbId : null,
+      schema: (activeQueryEditor) ? activeQueryEditor.schema : null,
+      autorun: false,
+      sql: 'SELECT ...',
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  handleSelect(key) {
+    if (key === 'add_tab') {
+      this.newQueryEditor();
+    } else {
+      this.props.actions.setActiveQueryEditor({ id: key });
+    }
+  }
+  render() {
+    const editors = this.props.queryEditors.map((qe, i) => {
+      let latestQuery = this.props.queries[qe.latestQueryId];
+      const state = (latestQuery) ? latestQuery.state : '';
+      const tabTitle = (
+        <div>
+          <div className={'circle ' + state} /> {qe.title} {' '}
+          <DropdownButton
+            bsSize="small"
+            id={'ddbtn-tab-' + i}
+          >
+            <MenuItem eventKey="1" onClick={this.props.actions.removeQueryEditor.bind(this, qe)}>
+              <i className="fa fa-close" /> close tab
+            </MenuItem>
+            <MenuItem eventKey="2" onClick={this.renameTab.bind(this, qe)}>
+              <i className="fa fa-i-cursor" /> rename tab
+            </MenuItem>
+          </DropdownButton>
+        </div>
+      );
+      return (
+        <Tab
+          key={qe.id}
+          title={tabTitle}
+          eventKey={qe.id}
+        >
+          <div className="panel panel-default">
+            <div className="panel-body">
+              <SqlEditor
+                queryEditor={qe}
+                latestQuery={latestQuery}
+              />
+            </div>
+          </div>
+        </Tab>);
+    });
+    return (
+      <Tabs
+        bsStyle="tabs"
+        activeKey={this.props.tabHistory[this.props.tabHistory.length - 1]}
+        onSelect={this.handleSelect.bind(this)}
+      >
+        {editors}
+        <Tab title={<div><i className="fa fa-plus-circle" />&nbsp;</div>} eventKey="add_tab" />
+      </Tabs>
+    );
+  }
+}
+QueryEditors.propTypes = {
+  actions: React.PropTypes.object,
+  queries: React.PropTypes.object,
+  queryEditors: React.PropTypes.array,
+  tabHistory: React.PropTypes.array,
+};
+QueryEditors.defaultProps = {
+  tabHistory: [],
+  queryEditors: [],
+};
+
+function mapStateToProps(state) {
+  return {
+    queryEditors: state.queryEditors,
+    queries: state.queries,
+    tabHistory: state.tabHistory,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+export default connect(mapStateToProps, mapDispatchToProps)(QueryEditors);

caravel/assets/javascripts/SqlLab/components/TableMetadata.jsx

@@ -0,0 +1,23 @@
+import React from 'react';
+import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table';
+
+const TableMetadata = function (props) {
+  return (
+    <BootstrapTable
+      condensed
+      data={props.table.columns}
+    >
+      <TableHeaderColumn dataField="id" isKey hidden>
+        id
+      </TableHeaderColumn>
+      <TableHeaderColumn dataField="name">Name</TableHeaderColumn>
+      <TableHeaderColumn dataField="type">Type</TableHeaderColumn>
+    </BootstrapTable>
+  );
+};
+
+TableMetadata.propTypes = {
+  table: React.PropTypes.object,
+};
+
+export default TableMetadata;

caravel/assets/javascripts/SqlLab/components/TableWorkspaceElement.jsx

@@ -0,0 +1,102 @@
+import React from 'react';
+import { ButtonGroup } from 'react-bootstrap';
+import Link from './Link';
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+import shortid from 'shortid';
+
+// CSS
+import 'react-select/dist/react-select.css';
+
+class TableWorkspaceElement extends React.Component {
+  selectStar() {
+    let cols = '';
+    this.props.table.columns.forEach((col, i) => {
+      cols += col.name;
+      if (i < this.props.table.columns.length - 1) {
+        cols += ', ';
+      }
+    });
+    const sql = `SELECT ${cols}\nFROM ${this.props.table.name}`;
+    const qe = {
+      id: shortid.generate(),
+      title: this.props.table.name,
+      dbId: this.props.table.dbId,
+      autorun: true,
+      sql,
+    };
+    this.props.actions.addQueryEditor(qe);
+  }
+  render() {
+    let metadata = null;
+    let buttonToggle;
+    if (!this.props.table.expanded) {
+      buttonToggle = (
+        <Link
+          href="#"
+          onClick={this.props.actions.expandTable.bind(this, this.props.table)}
+          placement="right"
+          tooltip="Collapse the table's structure information"
+        >
+          <i className="fa fa-minus" /> {this.props.table.name}
+        </Link>
+      );
+      metadata = this.props.table.columns.map((col) =>
+        <div className="clearfix">
+          <span className="pull-left">{col.name}</span>
+          <span className="pull-right">{col.type}</span>
+        </div>
+      );
+      metadata = (
+        <div style={{ 'margin-bottom': '5px' }}>{metadata}</div>
+      );
+    } else {
+      buttonToggle = (
+        <Link
+          href="#"
+          onClick={this.props.actions.collapseTable.bind(this, this.props.table)}
+          placement="right"
+          tooltip="Expand the table's structure information"
+        >
+          <i className="fa fa-plus" /> {this.props.table.name}
+        </Link>
+      );
+    }
+    return (
+      <div className="ws-el">
+        {buttonToggle}
+        <ButtonGroup className="ws-el-controls pull-right">
+          <Link
+            className="fa fa-play"
+            onClick={this.selectStar.bind(this)}
+            tooltip="Run query in a new tab"
+            href="#"
+          />
+          <Link
+            className="fa fa-trash"
+            onClick={this.props.actions.removeTable.bind(this, this.props.table)}
+            tooltip="Remove from workspace"
+            href="#"
+          />
+        </ButtonGroup>
+        {metadata}
+      </div>
+    );
+  }
+}
+TableWorkspaceElement.propTypes = {
+  table: React.PropTypes.object,
+  actions: React.PropTypes.object,
+};
+TableWorkspaceElement.defaultProps = {
+  table: null,
+};
+
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(null, mapDispatchToProps)(TableWorkspaceElement);
+

caravel/assets/javascripts/SqlLab/components/Timer.jsx

@@ -0,0 +1,61 @@
+import React from 'react';
+import { now, fDuration } from '../../modules/dates';
+
+import { STATE_BSSTYLE_MAP } from '../common.js';
+
+class Timer extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      clockStr: '',
+    };
+  }
+  componentWillMount() {
+    this.startTimer();
+  }
+  componentWillUnmount() {
+    this.stopTimer();
+  }
+  startTimer() {
+    if (!(this.timer)) {
+      this.timer = setInterval(this.stopwatch.bind(this), 30);
+    }
+  }
+  stopTimer() {
+    clearInterval(this.timer);
+    this.timer = null;
+  }
+  stopwatch() {
+    if (this.props && this.props.query) {
+      const since = (this.props.query.endDttm) ? this.props.query.endDttm : now();
+      const clockStr = fDuration(this.props.query.startDttm, since);
+      this.setState({ clockStr });
+      if (this.props.query.state !== 'running') {
+        this.stopTimer();
+      }
+    }
+  }
+  render() {
+    if (this.props.query && this.props.query.state === 'running') {
+      this.startTimer();
+    }
+    let timerSpan = null;
+    if (this.props && this.props.query) {
+      const bsStyle = STATE_BSSTYLE_MAP[this.props.query.state];
+      timerSpan = (
+        <span className={'inlineBlock m-r-5 label label-' + bsStyle}>
+          {this.state.clockStr}
+        </span>
+      );
+    }
+    return timerSpan;
+  }
+}
+Timer.propTypes = {
+  query: React.PropTypes.object,
+};
+Timer.defaultProps = {
+  query: null,
+};
+
+export default Timer;

caravel/assets/javascripts/SqlLab/components/VisualizeModal.jsx

@@ -0,0 +1,170 @@
+import React from 'react';
+import { Button, Col, Modal } from 'react-bootstrap';
+
+import { connect } from 'react-redux';
+import { bindActionCreators } from 'redux';
+import * as Actions from '../actions';
+
+import Select from 'react-select';
+import { Table } from 'reactable';
+import shortid from 'shortid';
+
+class VisualizeModal extends React.Component {
+  constructor(props) {
+    super(props);
+    this.state = {
+      chartType: 'line',
+      datasourceName: shortid.generate(),
+      columns: {},
+    };
+  }
+  changeChartType(option) {
+    this.setState({ chartType: (option) ? option.value : null });
+  }
+  mergedColumns() {
+    const columns = Object.assign({}, this.state.columns);
+    if (this.props.query && this.props.query.results.columns) {
+      this.props.query.results.columns.forEach((col) => {
+        if (columns[col] === undefined) {
+          columns[col] = {};
+        }
+      });
+    }
+    return columns;
+  }
+  visualize() {
+    const vizOptions = {
+      chartType: this.state.chartType,
+      datasourceName: this.state.datasourceName,
+      columns: this.state.columns,
+      sql: this.props.query.sql,
+      dbId: this.props.query.dbId,
+    };
+    window.open('/caravel/sqllab_viz/?data=' + JSON.stringify(vizOptions));
+  }
+  changeDatasourceName(event) {
+    this.setState({ datasourceName: event.target.value });
+  }
+  changeCheckbox(attr, col, event) {
+    let columns = this.mergedColumns();
+    const column = Object.assign({}, columns[col], { [attr]: event.target.checked });
+    columns = Object.assign({}, columns, { [col]: column });
+    this.setState({ columns });
+  }
+  changeAggFunction(col, option) {
+    let columns = this.mergedColumns();
+    const val = (option) ? option.value : null;
+    const column = Object.assign({}, columns[col], { agg: val });
+    columns = Object.assign({}, columns, { [col]: column });
+    this.setState({ columns });
+  }
+  render() {
+    if (!(this.props.query)) {
+      return <div />;
+    }
+    const tableData = this.props.query.results.columns.map((col) => ({
+      column: col,
+      is_dimension: (
+        <input
+          type="checkbox"
+          onChange={this.changeCheckbox.bind(this, 'is_dim', col)}
+          checked={(this.state.columns[col]) ? this.state.columns[col].is_dim : false}
+          className="form-control"
+        />
+      ),
+      is_date: (
+        <input
+          type="checkbox"
+          className="form-control"
+          onChange={this.changeCheckbox.bind(this, 'is_date', col)}
+          checked={(this.state.columns[col]) ? this.state.columns[col].is_date : false}
+        />
+      ),
+      agg_func: (
+        <Select
+          options={[
+            { value: 'sum', label: 'SUM(x)' },
+            { value: 'min', label: 'MIN(x)' },
+            { value: 'max', label: 'MAX(x)' },
+            { value: 'avg', label: 'AVG(x)' },
+            { value: 'count_distinct', label: 'COUNT(DISTINCT x)' },
+          ]}
+          onChange={this.changeAggFunction.bind(this, col)}
+          value={(this.state.columns[col]) ? this.state.columns[col].agg : null}
+        />
+      ),
+    }));
+    const modal = (
+      <div className="VisualizeModal">
+        <Modal show={this.props.show} onHide={this.props.onHide}>
+          <Modal.Header closeButton>
+            <Modal.Title>
+              Visualize <span className="alert alert-danger">under construction</span>
+            </Modal.Title>
+          </Modal.Header>
+          <Modal.Body>
+            <div className="row">
+              <Col md={6}>
+                Chart Type
+                <Select
+                  name="select-chart-type"
+                  placeholder="[Chart Type]"
+                  options={[
+                    { value: 'line', label: 'Time Series - Line Chart' },
+                    { value: 'bar', label: 'Time Series - Bar Chart' },
+                    { value: 'bar_dist', label: 'Distribution - Bar Chart' },
+                    { value: 'pie', label: 'Pie Chart' },
+                  ]}
+                  value={this.state.chartType}
+                  autosize={false}
+                  onChange={this.changeChartType.bind(this)}
+                />
+              </Col>
+              <Col md={6}>
+                Datasource Name
+                <input
+                  type="text"
+                  className="form-control"
+                  placeholder="datasource name"
+                  onChange={this.changeDatasourceName.bind(this)}
+                  value={this.state.datasourceName}
+                />
+              </Col>
+            </div>
+            <hr />
+            <Table
+              className="table table-condensed"
+              columns={['column', 'is_dimension', 'is_date', 'agg_func']}
+              data={tableData}
+            />
+            <Button
+              onClick={this.visualize.bind(this)}
+              bsStyle="primary"
+            >
+              Visualize
+            </Button>
+          </Modal.Body>
+        </Modal>
+      </div>
+    );
+    return modal;
+  }
+}
+VisualizeModal.propTypes = {
+  query: React.PropTypes.object,
+  show: React.PropTypes.boolean,
+  onHide: React.PropTypes.function,
+};
+VisualizeModal.defaultProps = {
+  show: false,
+};
+
+function mapStateToProps() {
+  return {};
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+export default connect(mapStateToProps, mapDispatchToProps)(VisualizeModal);

caravel/assets/javascripts/SqlLab/index.jsx

@@ -0,0 +1,74 @@
+const $ = window.$ = require('jquery');
+const jQuery = window.jQuery = $; // eslint-disable-line
+require('bootstrap');
+
+import React from 'react';
+import { render } from 'react-dom';
+import * as Actions from './actions';
+
+import LeftPane from './components/LeftPane';
+import TabbedSqlEditors from './components/TabbedSqlEditors';
+import QueryAutoRefresh from './components/QueryAutoRefresh';
+import Alerts from './components/Alerts';
+
+import { bindActionCreators, compose, createStore } from 'redux';
+import { connect, Provider } from 'react-redux';
+
+import { initialState, sqlLabReducer } from './reducers';
+import persistState from 'redux-localstorage';
+
+require('./main.css');
+
+let enhancer = compose(persistState());
+if (process.env.NODE_ENV === 'dev') {
+  enhancer = compose(
+    persistState(), window.devToolsExtension && window.devToolsExtension()
+  );
+}
+let store = createStore(sqlLabReducer, initialState, enhancer);
+
+// jquery hack to highlight the navbar menu
+$('a[href="/caravel/sqllab"]').parent().addClass('active');
+
+const App = function (props) {
+  return (
+    <div className="App SqlLab">
+      <div className="container-fluid">
+        <QueryAutoRefresh />
+        <Alerts alerts={props.alerts} />
+        <div className="row">
+          <div className="col-md-9">
+            <TabbedSqlEditors />
+          </div>
+          <div className="col-md-3">
+            <LeftPane />
+          </div>
+        </div>
+      </div>
+    </div>
+  );
+};
+
+App.propTypes = {
+  alerts: React.PropTypes.array,
+};
+
+function mapStateToProps(state) {
+  return {
+    alerts: state.alerts,
+  };
+}
+function mapDispatchToProps(dispatch) {
+  return {
+    actions: bindActionCreators(Actions, dispatch),
+  };
+}
+
+const ReduxedApp = connect(mapStateToProps, mapDispatchToProps)(App);
+
+render(
+  <Provider store={store}>
+    <ReduxedApp />
+  </Provider>,
+  document.getElementById('app')
+);

caravel/assets/javascripts/SqlLab/main.css

@@ -0,0 +1,229 @@
+#app {
+    position: absolute;
+    top: 65;
+    left: 0;
+    right: 0;
+    bottom: 0;
+}
+.inlineBlock {
+    display: inline-block;
+}
+.valignTop {
+    vertical-align: top;
+}
+.inline {
+    display: inline;
+}
+.nopadding {
+    padding: 0px;
+}
+.panel.nopadding .panel-body {
+    padding: 0px;
+}
+
+.loading {
+    width: 50px;
+    margin-top: 15px;
+}
+.pane-cell {
+    padding: 10px;
+    overflow: auto;
+    width: 100%;
+    height: 100%;
+}
+.SqlEditor .header {
+    padding-top: 5px;
+    padding-bottom: 5px;
+}
+.Workspace .btn-sm {
+    box-shadow: 1px 1px 2px rgba(0, 0, 0, 0.1);
+    margin-top: 2px;
+    padding: 4px;
+}
+.Workspace hr {
+    margin-top: 10px;
+    margin-bottom: 10px;
+}
+div.Workspace {
+    height: 100%;
+    margin: 0px;
+}
+.SqlEditor .clock {
+    background-color: orange;
+    padding: 5px;
+}
+
+.padded {
+    padding: 10px;
+}
+
+.p-t-10 {
+    padding-top: 10px;
+}
+.p-t-5 {
+    padding-top: 5px;
+}
+.m-r-5 {
+    margin-right: 5px;
+}
+.m-r-3 {
+    margin-right: 3px;
+}
+.m-l-1 {
+    margin-left: 1px;
+}
+.m-r-10 {
+    margin-right: 10px;
+}
+.m-b-10 {
+    margin-bottom: 10px;
+}
+.m-t-5 {
+    margin-top: 5px;
+}
+.m-t-10 {
+    margin-top: 10px;
+}
+.p-t-10 {
+    padding-top: 10px;
+}
+.sqllab-toolbar {
+    padding-top: 5px;
+    border-bottom: 1px solid #DDD;
+}
+.no-shadow {
+    box-shadow: none;
+    background-color: transparent;
+}
+.pane-west {
+    height: 100%;
+    overflow: auto;
+}
+.circle {
+    border-radius: 50%;
+    width: 10px;
+    height: 10px;
+    display: inline-block;
+    background-color: #ccc;
+}
+.Pane2 {
+    width: 0;
+}
+.running {
+    background-color: lime;
+    color: black;
+}
+.success {
+    background-color: green;
+}
+.failed {
+    background-color: red;
+}
+
+.handle {
+    cursor: move;
+}
+.window {
+    z-index: 1000;
+    position: absolute;
+    width: 300px;
+    opacity: 0.85;
+    border: 1px solid #AAA;
+    max-height: 600px;
+    box-shadow: rgba(0, 0, 0, 0.8) 5px 5px 25px
+}
+
+.SqlLab pre {
+    padding: 0px !important;
+    margin: 0px;
+    border: none;
+    font-size: 11px;
+    line-height: 125%;
+    background-color: transparent !important;
+}
+
+.Resizer {
+    background: #000;
+    opacity: .2;
+    z-index: 1;
+    -moz-box-sizing: border-box;
+    -webkit-box-sizing: border-box;
+    box-sizing: border-box;
+    -moz-background-clip: padding;
+    -webkit-background-clip: padding;
+    background-clip: padding-box;
+}
+
+.Resizer:hover {
+    -webkit-transition: all 2s ease;
+    transition: all 2s ease;
+}
+
+.Resizer.horizontal {
+    height: 10px;
+    margin: -5px 0;
+    border-top: 5px solid rgba(255, 255, 255, 0);
+    border-bottom: 5px solid rgba(255, 255, 255, 0);
+    cursor: row-resize;
+    width: 100%;
+    padding: 1px;
+}
+
+.Resizer.horizontal:hover {
+    border-top: 5px solid rgba(0, 0, 0, 0.5);
+    border-bottom: 5px solid rgba(0, 0, 0, 0.5);
+}
+
+.Resizer.vertical {
+    width: 9px;
+    margin: 0 -5px;
+    border-left: 5px solid rgba(255, 255, 255, 0);
+    border-right: 5px solid rgba(255, 255, 255, 0);
+    cursor: col-resize;
+}
+
+.Resizer.vertical:hover {
+    border-left: 5px solid rgba(0, 0, 0, 0.5);
+    border-right: 5px solid rgba(0, 0, 0, 0.5);
+}
+.Resizer.disabled {
+    cursor: not-allowed;
+}
+.Resizer.disabled:hover {
+    border-color: transparent;
+}
+
+.popover{
+    max-width:400px;
+}
+.Select-menu-outer {
+    z-index: 1000;
+}
+.table-label {
+    margin-top: 5px;
+    margin-right: 10px;
+    float: left;
+}
+div.tablePopover {
+    opacity: 0.7 !important;
+}
+div.tablePopover:hover {
+    opacity: 1 !important;
+}
+.ResultSetControls {
+    padding-bottom: 3px;
+    padding-top: 3px;
+}
+
+.ace_editor {
+  border: 1px solid #ccc;
+  margin: 10px 0;
+}
+
+.Select-menu-outer {
+    min-width: 100%;
+    width: inherit;
+}
+.ace_content {
+    background-color: #f4f4f4;
+}

caravel/assets/javascripts/SqlLab/reducers.js

@@ -0,0 +1,186 @@
+import shortid from 'shortid';
+import * as actions from './actions';
+import { now } from '../modules/dates';
+
+const defaultQueryEditor = {
+  id: shortid.generate(),
+  title: 'Untitled Query',
+  sql: 'SELECT *\nFROM\nWHERE',
+  latestQueryId: null,
+  autorun: false,
+  dbId: null,
+};
+
+// TODO(bkyryliuk): document the object schemas
+export const initialState = {
+  alerts: [],
+  queries: {},
+  queryEditors: [defaultQueryEditor],
+  tabHistory: [defaultQueryEditor.id],
+  tables: [],
+  workspaceQueries: [],
+  queriesLastUpdate: 0,
+};
+
+function addToObject(state, arrKey, obj) {
+  const newObject = Object.assign({}, state[arrKey]);
+  const copiedObject = Object.assign({}, obj);
+
+  if (!copiedObject.id) {
+    copiedObject.id = shortid.generate();
+  }
+  newObject[copiedObject.id] = copiedObject;
+  return Object.assign({}, state, { [arrKey]: newObject });
+}
+
+function alterInObject(state, arrKey, obj, alterations) {
+  const newObject = Object.assign({}, state[arrKey]);
+  newObject[obj.id] = (Object.assign({}, newObject[obj.id], alterations));
+  return Object.assign({}, state, { [arrKey]: newObject });
+}
+
+function alterInArr(state, arrKey, obj, alterations) {
+  // Finds an item in an array in the state and replaces it with a
+  // new object with an altered property
+  const idKey = 'id';
+  const newArr = [];
+  state[arrKey].forEach((arrItem) => {
+    if (obj[idKey] === arrItem[idKey]) {
+      newArr.push(Object.assign({}, arrItem, alterations));
+    } else {
+      newArr.push(arrItem);
+    }
+  });
+  return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function removeFromArr(state, arrKey, obj, idKey = 'id') {
+  const newArr = [];
+  state[arrKey].forEach((arrItem) => {
+    if (!(obj[idKey] === arrItem[idKey])) {
+      newArr.push(arrItem);
+    }
+  });
+  return Object.assign({}, state, { [arrKey]: newArr });
+}
+
+function addToArr(state, arrKey, obj) {
+  const newObj = Object.assign({}, obj);
+  if (!newObj.id) {
+    newObj.id = shortid.generate();
+  }
+  const newState = {};
+  newState[arrKey] = [...state[arrKey], newObj];
+  return Object.assign({}, state, newState);
+}
+
+export const sqlLabReducer = function (state, action) {
+  const actionHandlers = {
+    [actions.ADD_QUERY_EDITOR]() {
+      const tabHistory = state.tabHistory.slice();
+      tabHistory.push(action.queryEditor.id);
+      const newState = Object.assign({}, state, { tabHistory });
+      return addToArr(newState, 'queryEditors', action.queryEditor);
+    },
+    [actions.REMOVE_QUERY_EDITOR]() {
+      let newState = removeFromArr(state, 'queryEditors', action.queryEditor);
+      // List of remaining queryEditor ids
+      const qeIds = newState.queryEditors.map((qe) => qe.id);
+      let th = state.tabHistory.slice();
+      th = th.filter((id) => qeIds.includes(id));
+      newState = Object.assign({}, newState, { tabHistory: th });
+      return newState;
+    },
+    [actions.REMOVE_QUERY]() {
+      const newQueries = Object.assign({}, state.queries);
+      delete newQueries[action.query.id];
+      return Object.assign({}, state, { queries: newQueries });
+    },
+    [actions.RESET_STATE]() {
+      return Object.assign({}, initialState);
+    },
+    [actions.ADD_TABLE]() {
+      return addToArr(state, 'tables', action.table);
+    },
+    [actions.EXPAND_TABLE]() {
+      return alterInArr(state, 'tables', action.table, { expanded: true });
+    },
+    [actions.COLLAPSE_TABLE]() {
+      return alterInArr(state, 'tables', action.table, { expanded: false });
+    },
+    [actions.REMOVE_TABLE]() {
+      return removeFromArr(state, 'tables', action.table);
+    },
+    [actions.START_QUERY]() {
+      const newState = addToObject(state, 'queries', action.query);
+      const sqlEditor = { id: action.query.sqlEditorId };
+      return alterInArr(newState, 'queryEditors', sqlEditor, { latestQueryId: action.query.id });
+    },
+    [actions.STOP_QUERY]() {
+      return alterInObject(state, 'queries', action.query, { state: 'stopped' });
+    },
+    [actions.QUERY_SUCCESS]() {
+      const alts = {
+        state: 'success',
+        results: action.results,
+        rows: action.results.data.length,
+        progress: 100,
+        endDttm: now(),
+      };
+      return alterInObject(state, 'queries', action.query, alts);
+    },
+    [actions.QUERY_FAILED]() {
+      const alts = { state: 'failed', msg: action.msg, endDttm: now() };
+      return alterInObject(state, 'queries', action.query, alts);
+    },
+    [actions.SET_ACTIVE_QUERY_EDITOR]() {
+      const qeIds = state.queryEditors.map((qe) => qe.id);
+      if (qeIds.includes(action.queryEditor.id)) {
+        const tabHistory = state.tabHistory.slice();
+        tabHistory.push(action.queryEditor.id);
+        return Object.assign({}, state, { tabHistory });
+      }
+      return state;
+    },
+    [actions.QUERY_EDITOR_SETDB]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { dbId: action.dbId });
+    },
+    [actions.QUERY_EDITOR_SET_SCHEMA]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { schema: action.schema });
+    },
+    [actions.QUERY_EDITOR_SET_TITLE]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { title: action.title });
+    },
+    [actions.QUERY_EDITOR_SET_SQL]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { sql: action.sql });
+    },
+    [actions.QUERY_EDITOR_SET_AUTORUN]() {
+      return alterInArr(state, 'queryEditors', action.queryEditor, { autorun: action.autorun });
+    },
+    [actions.ADD_WORKSPACE_QUERY]() {
+      return addToArr(state, 'workspaceQueries', action.query);
+    },
+    [actions.REMOVE_WORKSPACE_QUERY]() {
+      return removeFromArr(state, 'workspaceQueries', action.query);
+    },
+    [actions.ADD_ALERT]() {
+      return addToArr(state, 'alerts', action.alert);
+    },
+    [actions.REMOVE_ALERT]() {
+      return removeFromArr(state, 'alerts', action.alert);
+    },
+    [actions.REFRESH_QUERIES]() {
+      const newQueries = Object.assign({}, state.queries);
+      // Fetch the updates to the queries present in the store.
+      for (const queryId in state.queries) {
+        newQueries[queryId] = Object.assign(newQueries[queryId], action.alteredQueries[queryId]);
+      }
+      const queriesLastUpdate = now();
+      return Object.assign({}, state, { queries: newQueries, queriesLastUpdate });
+    },
+  };
+  if (action.type in actionHandlers) {
+    return actionHandlers[action.type]();
+  }
+  return state;
+};

caravel/assets/javascripts/modules/dates.js

@@ -1,3 +1,5 @@
+import moment from 'moment';
+
 const d3 = require('d3');
 
 function UTC(dttm) {
@@ -72,3 +74,15 @@ export const timeFormatFactory = function (d3timeFormat) {
     return f(d);
   };
 };
+export const fDuration = function (t1, t2, f = 'HH:mm:ss.SS') {
+  const diffSec = t2 - t1;
+  const duration = moment(new Date(diffSec));
+  duration.millisecond((diffSec - Math.round(diffSec)) * 1000);
+  return duration.utc().format(f);
+};
+
+export const now = function () {
+  // seconds from EPOCH as a float
+  return moment().utc().valueOf() / 1000.0;
+};
+

caravel/assets/javascripts/sql.js

@@ -1,95 +0,0 @@
-const $ = window.$ = require('jquery');
-const jQuery = window.jQuery = require('jquery'); // eslint-disable-line
-const showModal = require('./modules/utils.js').showModal;
-require('./caravel-select2.js');
-require('datatables.net-bs');
-require('../node_modules/datatables-bootstrap3-plugin/media/css/datatables-bootstrap3.css');
-require('bootstrap');
-const ace = require('brace');
-require('brace/mode/sql');
-require('brace/theme/crimson_editor');
-require('../stylesheets/sql.css');
-$(document).ready(function () {
-  function getParam(name) {
-    /* eslint no-param-reassign: 0 */
-    name = name.replace(/[\[]/, '\\[').replace(/[\]]/, '\\]');
-    const regex = new RegExp('[\\?&]' + name + '=([^&#]*)');
-    const results = regex.exec(location.search);
-    return results === null ? '' : decodeURIComponent(results[1].replace(/\+/g, ' '));
-  }
-  function initSqlEditorView() {
-    const databaseId = $('#database_id').val();
-    const editor = ace.edit('sql');
-    editor.$blockScrolling = Infinity;
-    editor.getSession().setUseWrapMode(true);
-    $('#sql').hide();
-    editor.setTheme('ace/theme/crimson_editor');
-    editor.setOptions({
-      minLines: 16,
-      maxLines: Infinity,
-    });
-    editor.getSession().setMode('ace/mode/sql');
-    editor.focus();
-
-    $('select').select2({ dropdownAutoWidth: true });
-
-    function showTableMetadata() {
-      $('.metadata').load('/caravel/table/' + databaseId + '/' + $('#dbtable').val() + '/');
-    }
-    $('#dbtable').on('change', showTableMetadata);
-    showTableMetadata();
-    $('#create_view').click(function () {
-      showModal({
-        title: 'Error',
-        body: 'Sorry, this feature is not yet implemented',
-      });
-    });
-    $('.sqlcontent').show();
-    function selectStarOnClick() {
-      const url = '/caravel/select_star/' + databaseId + '/' + $('#dbtable').val() + '/';
-      $.ajax(url).done(function (msg) {
-        editor.setValue(msg);
-      });
-    }
-    $('#select_star').click(selectStarOnClick);
-    editor.setValue(getParam('sql'));
-    $(window).bind('popstate', function () {
-      // Could do something more lightweight here, but we're not optimizing
-      // for the use of the back button anyways
-      editor.setValue(getParam('sql'));
-      $('#run').click();
-    });
-    $('#run').click(function () {
-      $('#results').hide(0);
-      $('#loading').show(0);
-      history.pushState({}, document.title, '?sql=' + encodeURIComponent(editor.getValue()));
-      $.ajax({
-        type: 'POST',
-        url: '/caravel/runsql/',
-        data: {
-          data: JSON.stringify({
-            database_id: $('#database_id').val(),
-            sql: editor.getSession().getValue(),
-          }),
-        },
-        success(data) {
-          $('#loading').hide(0);
-          $('#results').show(0);
-          $('#results').html(data);
-          $('table.sql_results').DataTable({
-            retrieve: true,
-            paging: false,
-            searching: true,
-            aaSorting: [],
-          });
-        },
-        error(err) {
-          $('#loading').hide(0);
-          $('#results').show(0);
-          $('#results').html(err.responseText);
-        },
-      });
-    });
-  }
-  initSqlEditorView();
-});

caravel/assets/package.json

@@ -8,7 +8,7 @@
   },
   "scripts": {
     "test": "npm run lint && mocha --compilers js:babel-core/register --required spec/helpers/browser.js spec/**/*_spec.*",
-    "dev": "NODE_ENV=dev webpack -d --watch --colors",
+    "dev": "NODE_ENV=dev webpack -d --watch --colors --progress",
     "prod": "NODE_ENV=production webpack -p --colors --progress",
     "lint": "npm run --silent lint:js",
     "lint:js": "eslint --ignore-path=.eslintignore --ext .js,.jsx ."
@@ -55,19 +55,29 @@
     "jquery": "^2.2.1",
     "jquery-ui": "1.10.5",
     "mapbox-gl": "^0.20.0",
+    "moment": "^2.14.1",
+    "moments": "0.0.2",
     "mustache": "^2.2.1",
     "nvd3": "1.8.4",
     "react": "^15.2.1",
-    "react-bootstrap": "^0.28.3",
-    "react-bootstrap-datetimepicker": "0.0.22",
-    "react-bootstrap-table": "^2.3.7",
+    "react-ace": "^3.4.1",
+    "react-bootstrap": "^0.30.3",
+    "react-bootstrap-table": "^2.3.8",
     "react-dom": "^0.14.8",
-    "react-grid-layout": "^0.12.3",
+    "react-draggable": "^2.1.2",
+    "react-grid-layout": "^0.13.1",
     "react-map-gl": "^1.0.0-beta-10",
+    "react-redux": "^4.4.5",
     "react-resizable": "^1.3.3",
     "react-select": "^1.0.0-beta14",
+    "react-syntax-highlighter": "^2.1.1",
+    "reactable": "^0.13.2",
+    "redux": "^3.5.2",
+    "redux-localstorage": "^0.4.1",
     "select2": "3.5",
     "select2-bootstrap-css": "^1.4.6",
+    "shortid": "^2.2.6",
+    "style-loader": "^0.13.0",
     "supercluster": "https://github.com/georgeke/supercluster/tarball/ac3492737e7ce98e07af679623aad452373bbc40",
     "topojson": "^1.6.22",
     "viewport-mercator-project": "^2.1.0"

caravel/assets/stylesheets/caravel.css

@@ -28,12 +28,6 @@ body {
 input.form-control {
     background-color: white;
 }
-input.form-control[type=text], input.form-control[type=search] {
-  border: 1px solid #ccc!important;
-  box-shadow: none!important;
-  padding: 2px;
-  border-radius: 2px;
-}
 
 .chart-header a.danger {
     color: red;
@@ -193,5 +187,10 @@ div.widget .slice_container {
 
 .navbar .alert {
     padding: 5px 10px;
-    margin-top: 15px;
+    margin-top: 8px;
+    margin-bottom: 0px
+}
+
+.table-condensed {
+    font-size: 12px;
 }

caravel/assets/stylesheets/less/cosmo/bootswatch.less

@@ -27,7 +27,8 @@
 }
 
 .navbar-inverse .navbar-nav > li > a:hover,
-.navbar-inverse .navbar-nav > li > a:focus {
+.navbar-inverse .navbar-nav > li > a:focus,
+.navbar-inverse .navbar-nav > li.active > a {
   background: transparent;
   border-bottom: 3px solid @brand-primary;
   margin-bottom: -3px;
@@ -46,6 +47,58 @@
   background-color: #efefef;
 }
 
+.nav-tabs {
+  .dropdown-toggle.btn,
+  .dropdown-toggle.btn:hover,
+  .dropdown-toggle.btn:active,
+  .dropdown-toggle.btn:focus,
+  .btn-group.open .dropdown-toggle.btn,
+  .btn-group.open .dropdown-toggle.btn:hover,
+  .btn-group.open .dropdown-toggle.btn:active,
+  .btn-group.open .dropdown-toggle.btn:focus {
+    border-color: transparent;
+    background-color: transparent;
+    box-shadow: none;
+  }
+}
+
+.caret {
+  border: none;
+  color: @gray;
+}
+
+.caret:before {
+  font-family: "FontAwesome";
+  font-size: 10px;
+  content: "\f078";
+}
+
+.caret:hover {
+  color: @gray-darker;
+}
+
+.dropdown-toggle.btn .caret {
+  margin-left: 6px;
+  margin-top: -21px;
+  margin-right: 6px;
+}
+
+.nav-tabs .dropdown-toggle.btn .caret,  {
+  margin-left: -12px;
+  margin-top: -10px;
+}
+
+.navbar-nav .caret,
+.panel-title .caret {
+  margin-left: 6px;
+  margin-top: -24px;
+  margin-right: 6px;
+}
+
+
+
+
+
 // Typography =================================================================
 
 body {
@@ -176,7 +229,7 @@ table,
 .nav-pills {
 
   & > li > a {
-    border-radius: 0;
+    border-radius: 2px;
   }
 }
 
@@ -222,7 +275,7 @@ table,
 }
 
 .label {
-  border-radius: 0;
+  border-radius: 2px;
 }
 label {
   font-weight: normal;
@@ -231,11 +284,12 @@ label {
 // Progress bars ==============================================================
 
 .progress {
-  height: 8px;
+  height: 14px;
   .box-shadow(none);
   .progress-bar {
-    font-size: 8px;
-    line-height: 8px;
+    font-size: 12px;
+    line-height: 12px;
+    padding-top: 1px;
   }
 }
 
@@ -321,10 +375,7 @@ a.list-group-item {
 .nav-tabs > li.active > a, .nav-tabs > li.active > a:hover, .nav-tabs > li.active > a:focus {
   background-color: #fff;
   font-weight: bold;
-}
-
-.tab-content {
-  padding-top: 20px;
+  border-top: 3px solid @brand-primary;
 }
 
 // Tables ==============================================================
@@ -343,3 +394,8 @@ a.list-group-item {
     color: @state-success-text;
   }
 }
+
+// Utils ==============================================================
+hr {
+  margin: 10px 0;
+}

caravel/assets/stylesheets/less/cosmo/variables.less

@@ -426,15 +426,15 @@
 @nav-disabled-link-hover-color:             @gray-light;
 
 //== Tabs
-@nav-tabs-border-color:                     #ddd;
+@nav-tabs-border-color:                     #bbb;
 
 @nav-tabs-link-hover-border-color:          @gray-lighter;
 
 @nav-tabs-active-link-hover-bg:             @body-bg;
 @nav-tabs-active-link-hover-color:          @gray;
-@nav-tabs-active-link-hover-border-color:   #ddd;
+@nav-tabs-active-link-hover-border-color:   #bbb;
 
-@nav-tabs-justified-link-border-color:            #ddd;
+@nav-tabs-justified-link-border-color:            #bbb;
 @nav-tabs-justified-active-link-border-color:     @body-bg;
 
 //== Pills

caravel/assets/webpack.config.js

@@ -5,7 +5,7 @@ const path = require('path');
 const APP_DIR = path.resolve(__dirname, './');
 
 // output dir
-const BUILD_DIR = path.resolve(__dirname, './javascripts/dist');
+const BUILD_DIR = path.resolve(__dirname, './dist');
 
 const config = {
   entry: {
@@ -13,9 +13,9 @@ const config = {
     dashboard: APP_DIR + '/javascripts/dashboard/Dashboard.jsx',
     explore: APP_DIR + '/javascripts/explore/explore.jsx',
     welcome: APP_DIR + '/javascripts/welcome.js',
-    sql: APP_DIR + '/javascripts/sql.js',
     standalone: APP_DIR + '/javascripts/standalone.js',
     common: APP_DIR + '/javascripts/common.js',
+    sqllab: APP_DIR + '/javascripts/SqlLab/index.jsx',
   },
   output: {
     path: BUILD_DIR,

caravel/bin/caravel

@@ -5,6 +5,8 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 import logging
+import celery
+from celery.bin import worker as celery_worker
 from datetime import datetime
 from subprocess import Popen
 
@@ -42,6 +44,7 @@ def runserver(debug, address, port, timeout, workers):
         app.run(
             host='0.0.0.0',
             port=int(port),
+            threaded=True,
             debug=True)
     else:
         cmd = (
@@ -126,5 +129,24 @@ def refresh_druid():
     session.commit()
 
 
+@manager.command
+def worker():
+    """Starts a Caravel worker for async SQL query execution."""
+    # celery -A tasks worker --loglevel=info
+    print("Starting SQL Celery worker.")
+    if config.get('CELERY_CONFIG'):
+        print("Celery broker url: ")
+        print(config.get('CELERY_CONFIG').BROKER_URL)
+
+    application = celery.current_app._get_current_object()
+    c_worker = celery_worker.worker(app=application)
+    options = {
+        'broker': config.get('CELERY_CONFIG').BROKER_URL,
+        'loglevel': 'INFO',
+        'traceback': True,
+    }
+    c_worker.run(**options)
+
+
 if __name__ == "__main__":
     manager.run()

caravel/config.py

@@ -179,11 +179,32 @@ BACKUP_COUNT = 30
 # Set this API key to enable Mapbox visualizations
 MAPBOX_API_KEY = ""
 
+# Maximum number of rows returned in the SQL editor
+SQL_MAX_ROW = 1000
+
 # If defined, shows this text in an alert-warning box in the navbar
 # one example use case may be "STAGING" to make it clear that this is
 # not the production version of the site.
 WARNING_MSG = None
 
+# Default celery config is to use SQLA as a broker, in a production setting
+# you'll want to use a proper broker as specified here:
+# http://docs.celeryproject.org/en/latest/getting-started/brokers/index.html
+"""
+# Example:
+class CeleryConfig(object):
+  BROKER_URL = 'sqla+sqlite:///celerydb.sqlite'
+  CELERY_IMPORTS = ('caravel.tasks', )
+  CELERY_RESULT_BACKEND = 'db+sqlite:///celery_results.sqlite'
+  CELERY_ANNOTATIONS = {'tasks.add': {'rate_limit': '10/s'}}
+CELERY_CONFIG = CeleryConfig
+"""
+CELERY_CONFIG = None
+SQL_CELERY_DB_FILE_PATH = os.path.join(DATA_DIR, 'celerydb.sqlite')
+SQL_CELERY_RESULTS_DB_FILE_PATH = os.path.join(DATA_DIR, 'celery_results.sqlite')
+
+# The db id here results in selecting this one as a default in SQL Lab
+DEFAULT_DB_ID = None
 
 try:
     from caravel_config import *  # noqa

caravel/extract_table_names.py

@@ -0,0 +1,60 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+#
+# Copyright (C) 2016 Andi Albrecht, albrecht.andi@gmail.com
+#
+# This example is part of python-sqlparse and is released under
+# the BSD License: http://www.opensource.org/licenses/bsd-license.php
+#
+# This example illustrates how to extract table names from nested
+# SELECT statements.
+#
+# See:
+# http://groups.google.com/group/sqlparse/browse_thread/thread/b0bd9a022e9d4895
+
+import sqlparse
+from sqlparse.sql import IdentifierList, Identifier
+from sqlparse.tokens import Keyword, DML
+
+
+def is_subselect(parsed):
+    if not parsed.is_group():
+        return False
+    for item in parsed.tokens:
+        if item.ttype is DML and item.value.upper() == 'SELECT':
+            return True
+    return False
+
+
+def extract_from_part(parsed):
+    from_seen = False
+    for item in parsed.tokens:
+        if from_seen:
+            if is_subselect(item):
+                for x in extract_from_part(item):
+                    yield x
+            elif item.ttype is Keyword:
+                raise StopIteration
+            else:
+                yield item
+        elif item.ttype is Keyword and item.value.upper() == 'FROM':
+            from_seen = True
+
+
+def extract_table_identifiers(token_stream):
+    for item in token_stream:
+        if isinstance(item, IdentifierList):
+            for identifier in item.get_identifiers():
+                yield identifier.get_name()
+        elif isinstance(item, Identifier):
+            yield item.get_name()
+        # It's a bug to check for Keyword here, but in the example
+        # above some tables names are identified as keywords...
+        elif item.ttype is Keyword:
+            yield item.value
+
+
+# TODO(bkyryliuk): add logic to support joins and unions.
+def extract_tables(sql):
+    stream = extract_from_part(sqlparse.parse(sql)[0])
+    return list(extract_table_identifiers(stream))

caravel/migrations/versions/3c3ffe173e4f_add_sql_string_to_table.py

@@ -0,0 +1,22 @@
+"""add_sql_string_to_table
+
+Revision ID: 3c3ffe173e4f
+Revises: ad82a75afd82
+Create Date: 2016-08-18 14:06:28.784699
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = '3c3ffe173e4f'
+down_revision = 'ad82a75afd82'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.add_column('tables', sa.Column('sql', sa.Text(), nullable=True))
+
+
+def downgrade():
+    op.drop_column('tables', 'sql')

caravel/migrations/versions/ad82a75afd82_add_query_model.py

@@ -0,0 +1,56 @@
+"""Update models to support storing the queries.
+
+Revision ID: ad82a75afd82
+Revises: f162a1dea4c4
+Create Date: 2016-07-25 17:48:12.771103
+
+"""
+
+# revision identifiers, used by Alembic.
+revision = 'ad82a75afd82'
+down_revision = 'f162a1dea4c4'
+
+from alembic import op
+import sqlalchemy as sa
+
+
+def upgrade():
+    op.create_table('query',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('client_id',  sa.String(length=11), nullable=False),
+        sa.Column('database_id', sa.Integer(), nullable=False),
+        sa.Column('tmp_table_name', sa.String(length=256), nullable=True),
+        sa.Column('tab_name', sa.String(length=256),nullable=True),
+        sa.Column('sql_editor_id', sa.String(length=256), nullable=True),
+        sa.Column('user_id', sa.Integer(), nullable=True),
+        sa.Column('status', sa.String(length=16), nullable=True),
+        sa.Column('name', sa.String(length=256), nullable=True),
+        sa.Column('schema', sa.String(length=256), nullable=True),
+        sa.Column('sql', sa.Text(), nullable=True),
+        sa.Column('select_sql', sa.Text(), nullable=True),
+        sa.Column('executed_sql', sa.Text(), nullable=True),
+        sa.Column('limit', sa.Integer(), nullable=True),
+        sa.Column('limit_used', sa.Boolean(), nullable=True),
+        sa.Column('select_as_cta', sa.Boolean(), nullable=True),
+        sa.Column('select_as_cta_used', sa.Boolean(), nullable=True),
+        sa.Column('progress', sa.Integer(), nullable=True),
+        sa.Column('rows', sa.Integer(), nullable=True),
+        sa.Column('error_message', sa.Text(), nullable=True),
+        sa.Column('start_time', sa.Numeric(precision=20, scale=6), nullable=True),
+        sa.Column('changed_on', sa.DateTime(), nullable=True),
+        sa.Column('end_time', sa.Numeric(precision=20, scale=6), nullable=True),
+        sa.ForeignKeyConstraint(['database_id'], [u'dbs.id'], ),
+        sa.ForeignKeyConstraint(['user_id'], [u'ab_user.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+    op.add_column('dbs', sa.Column('select_as_create_table_as', sa.Boolean(),
+                                   nullable=True))
+    op.create_index(
+        op.f('ti_user_id_changed_on'),
+        'query', ['user_id', 'changed_on'], unique=False)
+
+
+def downgrade():
+    op.drop_table('query')
+    op.drop_column('dbs', 'select_as_create_table_as')
+

caravel/models.py

@@ -16,6 +16,7 @@ import humanize
 import pandas as pd
 import requests
 import sqlalchemy as sqla
+from sqlalchemy.engine.url import make_url
 import sqlparse
 from dateutil.parser import parse
 
@@ -30,13 +31,16 @@ from pydruid.utils.filters import Dimension, Filter
 from pydruid.utils.postaggregator import Postaggregator
 from pydruid.utils.having import Aggregation
 from six import string_types
+
 from sqlalchemy import (
-    Column, Integer, String, ForeignKey, Text, Boolean, DateTime, Date,
-    Table, create_engine, MetaData, desc, asc, select, and_, func)
-from sqlalchemy.engine import reflection
+    Column, Integer, String, ForeignKey, Text, Boolean,
+    DateTime, Date, Table, Numeric,
+    create_engine, MetaData, desc, asc, select, and_, func
+)
 from sqlalchemy.ext.declarative import declared_attr
 from sqlalchemy.orm import relationship
 from sqlalchemy.sql import table, literal_column, text, column
+from sqlalchemy.sql.expression import TextAsFrom
 from sqlalchemy_utils import EncryptedType
 
 import caravel
@@ -310,6 +314,11 @@ class Dashboard(Model, AuditMixinNullable):
         else:
             return {}
 
+    @property
+    def sqla_metadata(self):
+        metadata = MetaData(bind=self.get_sqla_engine())
+        return metadata.reflect()
+
     def dashboard_link(self):
         return '<a href="{obj.url}">{obj.dashboard_title}</a>'.format(obj=self)
 
@@ -372,6 +381,7 @@ class Database(Model, AuditMixinNullable):
     sqlalchemy_uri = Column(String(1024))
     password = Column(EncryptedType(String(1024), config.get('SECRET_KEY')))
     cache_timeout = Column(Integer)
+    select_as_create_table_as = Column(Boolean, default=False)
     extra = Column(Text, default=textwrap.dedent("""\
     {
         "metadata_params": {},
@@ -382,14 +392,69 @@ class Database(Model, AuditMixinNullable):
     def __repr__(self):
         return self.database_name
 
-    def get_sqla_engine(self):
+    def get_sqla_engine(self, schema=None):
         extra = self.get_extra()
         params = extra.get('engine_params', {})
-        return create_engine(self.sqlalchemy_uri_decrypted, **params)
+        url = make_url(self.sqlalchemy_uri_decrypted)
+        backend = url.get_backend_name()
+        if backend == 'presto' and schema:
+            if '/' in url.database:
+                url.database = url.database.split('/')[0] + '/' + schema
+            else:
+                url.database += '/' + schema
+        elif schema:
+            url.database = schema
+        return create_engine(url, **params)
+
+    def get_df(self, sql, schema):
+        eng = self.get_sqla_engine(schema=schema)
+        cur = eng.execute(sql, schema=schema)
+        cols = [col[0] for col in cur.cursor.description]
+        df = pd.DataFrame(cur.fetchall(), columns=cols)
+        return df
+
+    def compile_sqla_query(self, qry, schema=None):
+        eng = self.get_sqla_engine(schema=schema)
+        compiled = qry.compile(eng, compile_kwargs={"literal_binds": True})
+        return '{}'.format(compiled)
+
+    def select_star(self, table_name, schema=None, limit=1000):
+        """Generates a ``select *`` statement in the proper dialect"""
+        qry = select('*').select_from(text(table_name))
+        if limit:
+            qry = qry.limit(limit)
+        return self.compile_sqla_query(qry)
+
+    def wrap_sql_limit(self, sql, limit=1000):
+        qry = (
+            select('*')
+            .select_from(TextAsFrom(text(sql), ['*'])
+            .alias('inner_qry')).limit(limit)
+        )
+        return self.compile_sqla_query(qry)
 
     def safe_sqlalchemy_uri(self):
         return self.sqlalchemy_uri
 
+    @property
+    def inspector(self):
+        engine = self.get_sqla_engine()
+        return sqla.inspect(engine)
+
+    def all_table_names(self, schema=None):
+        return sorted(self.inspector.get_table_names(schema))
+
+    def all_view_names(self, schema=None):
+        views = []
+        try:
+            views = self.inspector.get_view_names(schema)
+        except Exception as e:
+            pass
+        return views
+
+    def all_schema_names(self):
+        return sorted(self.inspector.get_schema_names())
+
     def grains(self):
         """Defines time granularity database-specific expressions.
 
@@ -508,10 +573,8 @@ class Database(Model, AuditMixinNullable):
             autoload=True,
             autoload_with=self.get_sqla_engine())
 
-    def get_columns(self, table_name):
-        engine = self.get_sqla_engine()
-        insp = reflection.Inspector.from_engine(engine)
-        return insp.get_columns(table_name)
+    def get_columns(self, table_name, schema=None):
+        return self.inspector.get_columns(table_name, schema)
 
     @property
     def sqlalchemy_uri_decrypted(self):
@@ -523,10 +586,6 @@ class Database(Model, AuditMixinNullable):
     def sql_url(self):
         return '/caravel/sql/{}/'.format(self.id)
 
-    @property
-    def sql_link(self):
-        return '<a href="{}">SQL</a>'.format(self.sql_url)
-
     @property
     def perm(self):
         return (
@@ -554,6 +613,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
     offset = Column(Integer, default=0)
     cache_timeout = Column(Integer)
     schema = Column(String(255))
+    sql = Column(Text)
     table_columns = relationship("TableColumn", back_populates="table")
 
     baselink = "tablemodelview"
@@ -632,10 +692,6 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
     def sql_url(self):
         return self.database.sql_url + "?table_name=" + str(self.table_name)
 
-    @property
-    def sql_link(self):
-        return '<a href="{}">SQL</a>'.format(self.sql_url)
-
     def get_col(self, col_name):
         columns = self.table_columns
         for col in columns:
@@ -737,6 +793,10 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
         if self.schema:
             tbl.schema = self.schema
 
+        # Supporting arbitrary SQL statements in place of tables
+        if self.sql:
+            tbl = text('(' + self.sql + ') as expr_qry ')
+
         if not columns:
             qry = qry.group_by(*groupby_exprs)
 
@@ -1690,3 +1750,86 @@ class FavStar(Model):
     class_name = Column(String(50))
     obj_id = Column(Integer)
     dttm = Column(DateTime, default=func.now())
+
+
+class QueryStatus:
+    CANCELLED = 'cancelled'
+    FAILED = 'failed'
+    PENDING = 'pending'
+    RUNNING = 'running'
+    SCHEDULED = 'scheduled'
+    SUCCESS = 'success'
+    TIMED_OUT = 'timed_out'
+
+
+class Query(Model):
+
+    """ORM model for SQL query"""
+
+    __tablename__ = 'query'
+    id = Column(Integer, primary_key=True)
+    client_id = Column(String(11))
+
+    database_id = Column(Integer, ForeignKey('dbs.id'), nullable=False)
+
+    # Store the tmp table into the DB only if the user asks for it.
+    tmp_table_name = Column(String(256))
+    user_id = Column(
+        Integer, ForeignKey('ab_user.id'), nullable=True)
+    status = Column(String(16), default=QueryStatus.PENDING)
+    name = Column(String(256))
+    tab_name = Column(String(256))
+    sql_editor_id = Column(String(256))
+    schema = Column(String(256))
+    sql = Column(Text)
+    # Query to retrieve the results,
+    # used only in case of select_as_cta_used is true.
+    select_sql = Column(Text)
+    executed_sql = Column(Text)
+    # Could be configured in the caravel config.
+    limit = Column(Integer)
+    limit_used = Column(Boolean, default=False)
+    select_as_cta = Column(Boolean)
+    select_as_cta_used = Column(Boolean, default=False)
+
+    progress = Column(Integer, default=0)  # 1..100
+    # # of rows in the result set or rows modified.
+    rows = Column(Integer)
+    error_message = Column(Text)
+
+    # Using Numeric in place of DateTime for sub-second precision
+    # stored as seconds since epoch, allowing for milliseconds
+    start_time = Column(Numeric(precision=3))
+    end_time = Column(Numeric(precision=3))
+    changed_on = Column(
+        DateTime, default=datetime.now, onupdate=datetime.now, nullable=True)
+
+    database = relationship(
+        'Database', foreign_keys=[database_id], backref='queries')
+
+    __table_args__ = (
+        sqla.Index('ti_user_id_changed_on', user_id, changed_on),
+    )
+
+    def to_dict(self):
+        return {
+            'changedOn': self.changed_on,
+            'dbId': self.database_id,
+            'endDttm': self.end_time,
+            'errorMessage': self.error_message,
+            'executedSql': self.executed_sql,
+            'id': self.client_id,
+            'limit': self.limit,
+            'progress': self.progress,
+            'rows': self.rows,
+            'schema': self.schema,
+            'ctas': self.select_as_cta,
+            'serverId': self.id,
+            'sql': self.sql,
+            'sqlEditorId': self.sql_editor_id,
+            'startDttm': self.start_time,
+            'state': self.status.lower(),
+            'tab': self.tab_name,
+            'tempTable': self.tmp_table_name,
+            'userId': self.user_id,
+        }

caravel/sql_lab.py

@@ -0,0 +1,136 @@
+import celery
+from datetime import datetime
+import pandas as pd
+import logging
+from caravel import  app, db, models, utils
+import time
+
+QueryStatus = models.QueryStatus
+
+
+celery_app = celery.Celery(config_source=app.config.get('CELERY_CONFIG'))
+
+
+def is_query_select(sql):
+    return sql.upper().startswith('SELECT')
+
+def create_table_as(sql, table_name, override=False):
+    """Reformats the query into the create table as query.
+
+    Works only for the single select SQL statements, in all other cases
+    the sql query is not modified.
+    :param sql: string, sql query that will be executed
+    :param table_name: string, will contain the results of the query execution
+    :param override, boolean, table table_name will be dropped if true
+    :return: string, create table as query
+    """
+    # TODO(bkyryliuk): enforce that all the columns have names. Presto requires it
+    #                  for the CTA operation.
+    # TODO(bkyryliuk): drop table if allowed, check the namespace and
+    #                  the permissions.
+    # TODO raise if multi-statement
+    exec_sql = ''
+    if is_query_select(sql):
+        if override:
+            exec_sql = 'DROP TABLE IF EXISTS {};\n'.format(table_name)
+        exec_sql += "CREATE TABLE {table_name} AS \n{sql}"
+    else:
+        raise Exception("Could not generate CREATE TABLE statement")
+    return exec_sql.format(**locals())
+
+
+@celery_app.task
+def get_sql_results(query_id, return_results=True):
+    """Executes the sql query returns the results."""
+    db.session.commit()  # HACK
+    q = db.session.query(models.Query).all()
+    query = db.session.query(models.Query).filter_by(id=query_id).one()
+    database = query.database
+    executed_sql = query.sql.strip().strip(';')
+
+    # Limit enforced only for retrieving the data, not for the CTA queries.
+    if is_query_select(executed_sql):
+        if query.select_as_cta:
+            if not query.tmp_table_name:
+                start_dttm = datetime.fromtimestamp(query.start_time)
+                query.tmp_table_name = 'tmp_{}_table_{}'.format(
+                    query.user_id,
+                    start_dttm.strftime('%Y_%m_%d_%H_%M_%S'))
+            executed_sql = create_table_as(executed_sql, query.tmp_table_name)
+            query.select_as_cta_used = True
+        elif query.limit:
+            executed_sql = database.wrap_sql_limit(executed_sql, query.limit)
+            query.limit_used = True
+        engine = database.get_sqla_engine(schema=query.schema)
+        try:
+            query.executed_sql = executed_sql
+            logging.info("Running query: \n{}".format(executed_sql))
+            result_proxy = engine.execute(query.executed_sql, schema=query.schema)
+        except Exception as e:
+            logging.exception(e)
+            query.error_message = utils.error_msg_from_exception(e)
+            query.status = QueryStatus.FAILED
+            query.tmp_table_name = None
+            db.session.commit()
+            raise Exception(query.error_message)
+
+        cursor = result_proxy.cursor
+        query.status = QueryStatus.RUNNING
+        db.session.flush()
+        if hasattr(cursor, "poll"):
+            polled = cursor.poll()
+            # poll returns dict -- JSON status information or ``None``
+            # if the query is done
+            # https://github.com/dropbox/PyHive/blob/
+            # b34bdbf51378b3979eaf5eca9e956f06ddc36ca0/pyhive/presto.py#L178
+            while polled:
+                # Update the object and wait for the kill signal.
+                stats = polled.get('stats', {})
+                if stats:
+                    completed_splits = float(stats.get('completedSplits'))
+                    total_splits = float(stats.get('totalSplits'))
+                    if total_splits and completed_splits:
+                        progress = 100 * (completed_splits / total_splits)
+                        if progress > query.progress:
+                            query.progress = progress
+                        db.session.commit()
+                time.sleep(200)
+
+                polled = cursor.poll()
+                # TODO(b.kyryliuk): check for the kill signal.
+
+        columns = None
+        data = None
+        if result_proxy.cursor:
+            cols = [col[0] for col in result_proxy.cursor.description]
+            data = result_proxy.fetchall()
+            df = pd.DataFrame(data, columns=cols)
+            columns = [c for c in df.columns]
+            data = df.to_dict(orient='records')
+
+        query.rows = result_proxy.rowcount
+        query.progress = 100
+        query.status = QueryStatus.SUCCESS
+        if query.rows == -1 and data:
+            # Presto doesn't provide result_proxy.row_count
+            query.rows = len(data)
+
+        # CTAs queries result in 1 cell having the # of the added rows.
+        if query.select_as_cta:
+            query.select_sql = '{}'.format(database.select_star(
+                query.tmp_table_name, limit=query.limit))
+
+        query.end_time = utils.now_as_float()
+        db.session.commit()
+
+    payload = {
+        'query_id': query.id,
+        'status': query.status,
+    }
+    if query.status == models.QueryStatus.SUCCESS:
+        payload['data'] = data
+        payload['columns'] = columns
+    else:
+        payload['error'] = query.error_message
+    if return_results:
+        return payload

caravel/templates/caravel/base.html

@@ -8,10 +8,10 @@
 
   {% block head_js %}
     {{super()}}
-    <script src="/static/assets/javascripts/dist/css-theme.entry.js"></script>
+    <script src="/static/assets/dist/css-theme.entry.js"></script>
   {% endblock %}
 
   {% block tail_js %}
       {{super()}}
-      <script src="/static/assets/javascripts/dist/common.entry.js"></script>
+      <script src="/static/assets/dist/common.entry.js"></script>
   {% endblock %}

caravel/templates/caravel/basic.html

@@ -13,7 +13,7 @@
       <link rel="icon" type="image/png" href="/static/assets/images/favicon.png">
     {% endblock %}
     {% block head_js %}
-      <script src="/static/assets/javascripts/dist/css-theme.entry.js"></script>
+      <script src="/static/assets/dist/css-theme.entry.js"></script>
     {% endblock %}
   </head>
 
@@ -29,7 +29,7 @@
     {% block body %}
       {% include 'caravel/flash_wrapper.html' %}
       <div id="app">
-        Oops! React.js is not working properly.
+        <img src="/static/assets/images/loading.gif" style="width: 50px; margin: 10px;">
       </div>
     {% endblock %}
 

caravel/templates/caravel/dashboard.html

@@ -2,7 +2,7 @@
 
 {% block head_js %}
   {{ super() }}
-  <script src="/static/assets/javascripts/dist/dashboard.entry.js"></script>
+  <script src="/static/assets/dist/dashboard.entry.js"></script>
 {% endblock %}
 {% block title %}[dashboard] {{ dashboard.dashboard_title }}{% endblock %}
 {% block body %}

caravel/templates/caravel/explore.html

@@ -339,5 +339,5 @@
 
 {% block tail_js %}
   {{ super() }}
-  <script src="/static/assets/javascripts/dist/explore.entry.js"></script>
+  <script src="/static/assets/dist/explore.entry.js"></script>
 {% endblock %}

caravel/templates/caravel/index.html

@@ -2,5 +2,5 @@
 
 {% block tail_js %}
   {{ super() }}
-  <script src="/static/assets/javascripts/dist/index.entry.js"></script>
+  <script src="/static/assets/dist/index.entry.js"></script>
 {% endblock %}

caravel/templates/caravel/paper-theme.html

@@ -0,0 +1,577 @@
+{% extends "caravel/basic.html" %}
+
+{% block body %}
+  <body role="document">
+
+    <!-- Fixed navbar -->
+    <nav class="navbar navbar-inverse navbar-fixed-top">
+      <div class="container">
+        <div class="navbar-header">
+          <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
+            <span class="sr-only">Toggle navigation</span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+            <span class="icon-bar"></span>
+          </button>
+          <a class="navbar-brand" href="#">Bootstrap theme</a>
+        </div>
+        <div id="navbar" class="navbar-collapse collapse">
+          <ul class="nav navbar-nav">
+            <li class="active"><a href="#">Home</a></li>
+            <li><a href="#">About</a></li>
+            <li><a href="#">Contact</a></li>
+            <li class="dropdown">
+              <a href="#"></span></a>
+              <ul class="dropdown-menu">
+                <li><a href="#">Action</a></li>
+                <li><a href="#">Another action</a></li>
+                <li><a href="#">Something else here</a></li>
+                <li role="separator" class="divider"></li>
+                <li class="dropdown-header">Nav header</li>
+                <li><a href="#">Separated link</a></li>
+                <li><a href="#">One more separated link</a></li>
+              </ul>
+            </li>
+          </ul>
+        </div><!--/.nav-collapse -->
+      </div>
+    </nav>
+
+    <div class="container theme-showcase" role="main">
+
+      <!-- Main jumbotron for a primary marketing message or call to action -->
+      <div class="jumbotron">
+        <h1>Theme example</h1>
+        <p>This is a template showcasing the optional theme stylesheet included in Bootstrap. Use it as a starting point to create something more unique by building on or modifying it.</p>
+      </div>
+
+
+      <div class="page-header">
+        <h1>Buttons</h1>
+      </div>
+      <p>
+        <button type="button" class="btn btn-lg btn-default">Default</button>
+        <button type="button" class="btn btn-lg btn-primary">Primary</button>
+        <button type="button" class="btn btn-lg btn-success">Success</button>
+        <button type="button" class="btn btn-lg btn-info">Info</button>
+        <button type="button" class="btn btn-lg btn-warning">Warning</button>
+        <button type="button" class="btn btn-lg btn-danger">Danger</button>
+        <button type="button" class="btn btn-lg btn-link">Link</button>
+      </p>
+      <p>
+        <button type="button" class="btn btn-default">Default</button>
+        <button type="button" class="btn btn-primary">Primary</button>
+        <button type="button" class="btn btn-success">Success</button>
+        <button type="button" class="btn btn-info">Info</button>
+        <button type="button" class="btn btn-warning">Warning</button>
+        <button type="button" class="btn btn-danger">Danger</button>
+        <button type="button" class="btn btn-link">Link</button>
+      </p>
+      <p>
+        <button type="button" class="btn btn-sm btn-default">Default</button>
+        <button type="button" class="btn btn-sm btn-primary">Primary</button>
+        <button type="button" class="btn btn-sm btn-success">Success</button>
+        <button type="button" class="btn btn-sm btn-info">Info</button>
+        <button type="button" class="btn btn-sm btn-warning">Warning</button>
+        <button type="button" class="btn btn-sm btn-danger">Danger</button>
+        <button type="button" class="btn btn-sm btn-link">Link</button>
+      </p>
+      <p>
+        <button type="button" class="btn btn-xs btn-default">Default</button>
+        <button type="button" class="btn btn-xs btn-primary">Primary</button>
+        <button type="button" class="btn btn-xs btn-success">Success</button>
+        <button type="button" class="btn btn-xs btn-info">Info</button>
+        <button type="button" class="btn btn-xs btn-warning">Warning</button>
+        <button type="button" class="btn btn-xs btn-danger">Danger</button>
+        <button type="button" class="btn btn-xs btn-link">Link</button>
+      </p>
+
+
+      <div class="page-header">
+        <h1>Tables</h1>
+      </div>
+      <div class="row">
+        <div class="col-md-6">
+          <table class="table">
+            <thead>
+              <tr>
+                <th>#</th>
+                <th>First Name</th>
+                <th>Last Name</th>
+                <th>Username</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td>1</td>
+                <td>Mark</td>
+                <td>Otto</td>
+                <td>@mdo</td>
+              </tr>
+              <tr>
+                <td>2</td>
+                <td>Jacob</td>
+                <td>Thornton</td>
+                <td>@fat</td>
+              </tr>
+              <tr>
+                <td>3</td>
+                <td>Larry</td>
+                <td>the Bird</td>
+                <td>@twitter</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+        <div class="col-md-6">
+          <table class="table table-striped">
+            <thead>
+              <tr>
+                <th>#</th>
+                <th>First Name</th>
+                <th>Last Name</th>
+                <th>Username</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td>1</td>
+                <td>Mark</td>
+                <td>Otto</td>
+                <td>@mdo</td>
+              </tr>
+              <tr>
+                <td>2</td>
+                <td>Jacob</td>
+                <td>Thornton</td>
+                <td>@fat</td>
+              </tr>
+              <tr>
+                <td>3</td>
+                <td>Larry</td>
+                <td>the Bird</td>
+                <td>@twitter</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+
+      <div class="row">
+        <div class="col-md-6">
+          <table class="table table-bordered">
+            <thead>
+              <tr>
+                <th>#</th>
+                <th>First Name</th>
+                <th>Last Name</th>
+                <th>Username</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td rowspan="2">1</td>
+                <td>Mark</td>
+                <td>Otto</td>
+                <td>@mdo</td>
+              </tr>
+              <tr>
+                <td>Mark</td>
+                <td>Otto</td>
+                <td>@TwBootstrap</td>
+              </tr>
+              <tr>
+                <td>2</td>
+                <td>Jacob</td>
+                <td>Thornton</td>
+                <td>@fat</td>
+              </tr>
+              <tr>
+                <td>3</td>
+                <td colspan="2">Larry the Bird</td>
+                <td>@twitter</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+        <div class="col-md-6">
+          <table class="table table-condensed">
+            <thead>
+              <tr>
+                <th>#</th>
+                <th>First Name</th>
+                <th>Last Name</th>
+                <th>Username</th>
+              </tr>
+            </thead>
+            <tbody>
+              <tr>
+                <td>1</td>
+                <td>Mark</td>
+                <td>Otto</td>
+                <td>@mdo</td>
+              </tr>
+              <tr>
+                <td>2</td>
+                <td>Jacob</td>
+                <td>Thornton</td>
+                <td>@fat</td>
+              </tr>
+              <tr>
+                <td>3</td>
+                <td colspan="2">Larry the Bird</td>
+                <td>@twitter</td>
+              </tr>
+            </tbody>
+          </table>
+        </div>
+      </div>
+
+
+      <div class="page-header">
+        <h1>Thumbnails</h1>
+      </div>
+      <img data-src="holder.js/200x200" class="img-thumbnail" alt="200x200" style="width: 200px; height: 200px;" src="data:image/svg+xml;base64,PD94bWwgdmVyc2lvbj0iMS4wIiBlbmNvZGluZz0iVVRGLTgiIHN0YW5kYWxvbmU9InllcyI/PjxzdmcgeG1sbnM9Imh0dHA6Ly93d3cudzMub3JnLzIwMDAvc3ZnIiB3aWR0aD0iMjAwIiBoZWlnaHQ9IjIwMCIgdmlld0JveD0iMCAwIDIwMCAyMDAiIHByZXNlcnZlQXNwZWN0UmF0aW89Im5vbmUiPjwhLS0KU291cmNlIFVSTDogaG9sZGVyLmpzLzIwMHgyMDAKQ3JlYXRlZCB3aXRoIEhvbGRlci5qcyAyLjYuMC4KTGVhcm4gbW9yZSBhdCBodHRwOi8vaG9sZGVyanMuY29tCihjKSAyMDEyLTIwMTUgSXZhbiBNYWxvcGluc2t5IC0gaHR0cDovL2ltc2t5LmNvCi0tPjxkZWZzPjxzdHlsZSB0eXBlPSJ0ZXh0L2NzcyI+PCFbQ0RBVEFbI2hvbGRlcl8xNTYwZTAxMWI3ZCB0ZXh0IHsgZmlsbDojQUFBQUFBO2ZvbnQtd2VpZ2h0OmJvbGQ7Zm9udC1mYW1pbHk6QXJpYWwsIEhlbHZldGljYSwgT3BlbiBTYW5zLCBzYW5zLXNlcmlmLCBtb25vc3BhY2U7Zm9udC1zaXplOjEwcHQgfSBdXT48L3N0eWxlPjwvZGVmcz48ZyBpZD0iaG9sZGVyXzE1NjBlMDExYjdkIj48cmVjdCB3aWR0aD0iMjAwIiBoZWlnaHQ9IjIwMCIgZmlsbD0iI0VFRUVFRSIvPjxnPjx0ZXh0IHg9Ijc0LjA1NDY4NzUiIHk9IjEwNC41Ij4yMDB4MjAwPC90ZXh0PjwvZz48L2c+PC9zdmc+" data-holder-rendered="true">
+
+
+      <div class="page-header">
+        <h1>Labels</h1>
+      </div>
+      <h1>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h1>
+      <h2>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h2>
+      <h3>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h3>
+      <h4>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h4>
+      <h5>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h5>
+      <h6>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </h6>
+      <p>
+        <span class="label label-default">Default</span>
+        <span class="label label-primary">Primary</span>
+        <span class="label label-success">Success</span>
+        <span class="label label-info">Info</span>
+        <span class="label label-warning">Warning</span>
+        <span class="label label-danger">Danger</span>
+      </p>
+
+
+      <div class="page-header">
+        <h1>Badges</h1>
+      </div>
+      <p>
+        <a href="#">42</span></a>
+      </p>
+      <ul class="nav nav-pills" role="tablist">
+        <li role="presentation" class="active"><a href="#">42</span></a></li>
+        <li role="presentation"><a href="#">Profile</a></li>
+        <li role="presentation"><a href="#">3</span></a></li>
+      </ul>
+
+
+      <div class="page-header">
+        <h1>Dropdown menus</h1>
+      </div>
+      <div class="dropdown theme-dropdown clearfix">
+        <a id="dropdownMenu1" href="#"></span></a>
+        <ul class="dropdown-menu" aria-labelledby="dropdownMenu1">
+          <li class="active"><a href="#">Action</a></li>
+          <li><a href="#">Another action</a></li>
+          <li><a href="#">Something else here</a></li>
+          <li role="separator" class="divider"></li>
+          <li><a href="#">Separated link</a></li>
+        </ul>
+      </div>
+
+
+      <div class="page-header">
+        <h1>Navs</h1>
+      </div>
+      <ul class="nav nav-tabs" role="tablist">
+        <li role="presentation" class="active"><a href="#">Home</a></li>
+        <li role="presentation"><a href="#">Profile</a></li>
+        <li role="presentation"><a href="#">Messages</a></li>
+      </ul>
+      <ul class="nav nav-pills" role="tablist">
+        <li role="presentation" class="active"><a href="#">Home</a></li>
+        <li role="presentation"><a href="#">Profile</a></li>
+        <li role="presentation"><a href="#">Messages</a></li>
+      </ul>
+
+
+      <div class="page-header">
+        <h1>Navbars</h1>
+      </div>
+
+      <nav class="navbar navbar-default">
+        <div class="container">
+          <div class="navbar-header">
+            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target=".navbar-collapse">
+              <span class="sr-only">Toggle navigation</span>
+              <span class="icon-bar"></span>
+              <span class="icon-bar"></span>
+              <span class="icon-bar"></span>
+            </button>
+            <a class="navbar-brand" href="#">Project name</a>
+          </div>
+          <div class="navbar-collapse collapse">
+            <ul class="nav navbar-nav">
+              <li class="active"><a href="#">Home</a></li>
+              <li><a href="#">About</a></li>
+              <li><a href="#">Contact</a></li>
+              <li class="dropdown">
+                <a href="#"></span></a>
+                <ul class="dropdown-menu">
+                  <li><a href="#">Action</a></li>
+                  <li><a href="#">Another action</a></li>
+                  <li><a href="#">Something else here</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li class="dropdown-header">Nav header</li>
+                  <li><a href="#">Separated link</a></li>
+                  <li><a href="#">One more separated link</a></li>
+                </ul>
+              </li>
+            </ul>
+          </div><!--/.nav-collapse -->
+        </div>
+      </nav>
+
+      <nav class="navbar navbar-inverse">
+        <div class="container">
+          <div class="navbar-header">
+            <button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target=".navbar-collapse">
+              <span class="sr-only">Toggle navigation</span>
+              <span class="icon-bar"></span>
+              <span class="icon-bar"></span>
+              <span class="icon-bar"></span>
+            </button>
+            <a class="navbar-brand" href="#">Project name</a>
+          </div>
+          <div class="navbar-collapse collapse">
+            <ul class="nav navbar-nav">
+              <li class="active"><a href="#">Home</a></li>
+              <li><a href="#">About</a></li>
+              <li><a href="#">Contact</a></li>
+              <li class="dropdown">
+                <a href="#"></span></a>
+                <ul class="dropdown-menu">
+                  <li><a href="#">Action</a></li>
+                  <li><a href="#">Another action</a></li>
+                  <li><a href="#">Something else here</a></li>
+                  <li role="separator" class="divider"></li>
+                  <li class="dropdown-header">Nav header</li>
+                  <li><a href="#">Separated link</a></li>
+                  <li><a href="#">One more separated link</a></li>
+                </ul>
+              </li>
+            </ul>
+          </div><!--/.nav-collapse -->
+        </div>
+      </nav>
+
+
+      <div class="page-header">
+        <h1>Alerts</h1>
+      </div>
+      <div class="alert alert-success" role="alert">
+        <strong>Well done!</strong> You successfully read this important alert message. With a <a href="#">link</a>. 
+      </div>
+      <div class="alert alert-info" role="alert">
+        <strong>Heads up!</strong> This alert needs your attention, but it's not super important.
+      </div>
+      <div class="alert alert-warning" role="alert">
+        <strong>Warning!</strong> Best check yo self, you're not looking too good.
+      </div>
+      <div class="alert alert-danger" role="alert">
+        <strong>Oh snap!</strong> Change a few things up and try submitting again.
+      </div>
+
+
+      <div class="page-header">
+        <h1>Progress bars</h1>
+      </div>
+      <div class="progress">
+        <div class="progress-bar" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100" style="width: 60%;"><span class="sr-only">60% Complete</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-success" role="progressbar" aria-valuenow="40" aria-valuemin="0" aria-valuemax="100" style="width: 40%"><span class="sr-only">40% Complete (success)</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-info" role="progressbar" aria-valuenow="20" aria-valuemin="0" aria-valuemax="100" style="width: 20%"><span class="sr-only">20% Complete</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-warning" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100" style="width: 60%"><span class="sr-only">60% Complete (warning)</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-danger" role="progressbar" aria-valuenow="80" aria-valuemin="0" aria-valuemax="100" style="width: 80%"><span class="sr-only">80% Complete (danger)</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-striped" role="progressbar" aria-valuenow="60" aria-valuemin="0" aria-valuemax="100" style="width: 60%"><span class="sr-only">60% Complete</span></div>
+      </div>
+      <div class="progress">
+        <div class="progress-bar progress-bar-success" style="width: 35%"><span class="sr-only">35% Complete (success)</span></div>
+        <div class="progress-bar progress-bar-warning" style="width: 20%"><span class="sr-only">20% Complete (warning)</span></div>
+        <div class="progress-bar progress-bar-danger" style="width: 10%"><span class="sr-only">10% Complete (danger)</span></div>
+      </div>
+
+
+      <div class="page-header">
+        <h1>List groups</h1>
+      </div>
+      <div class="row">
+        <div class="col-sm-4">
+          <ul class="list-group">
+            <li class="list-group-item">Cras justo odio</li>
+            <li class="list-group-item">Dapibus ac facilisis in</li>
+            <li class="list-group-item">Morbi leo risus</li>
+            <li class="list-group-item">Porta ac consectetur ac</li>
+            <li class="list-group-item">Vestibulum at eros</li>
+          </ul>
+        </div><!-- /.col-sm-4 -->
+        <div class="col-sm-4">
+          <div class="list-group">
+            <a href="#">
+              Cras justo odio
+            </a>
+            <a href="#">Dapibus ac facilisis in</a>
+            <a href="#">Morbi leo risus</a>
+            <a href="#">Porta ac consectetur ac</a>
+            <a href="#">Vestibulum at eros</a>
+          </div>
+        </div><!-- /.col-sm-4 -->
+        <div class="col-sm-4">
+          <div class="list-group">
+            <a href="#">
+              <h4 class="list-group-item-heading">List group item heading</h4>
+              <p class="list-group-item-text">Donec id elit non mi porta gravida at eget metus. Maecenas sed diam eget risus varius blandit.</p>
+            </a>
+            <a href="#">
+              <h4 class="list-group-item-heading">List group item heading</h4>
+              <p class="list-group-item-text">Donec id elit non mi porta gravida at eget metus. Maecenas sed diam eget risus varius blandit.</p>
+            </a>
+            <a href="#">
+              <h4 class="list-group-item-heading">List group item heading</h4>
+              <p class="list-group-item-text">Donec id elit non mi porta gravida at eget metus. Maecenas sed diam eget risus varius blandit.</p>
+            </a>
+          </div>
+        </div><!-- /.col-sm-4 -->
+      </div>
+
+
+      <div class="page-header">
+        <h1>Panels</h1>
+      </div>
+      <div class="row">
+        <div class="col-sm-4">
+          <div class="panel panel-default">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+          <div class="panel panel-primary">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+        </div><!-- /.col-sm-4 -->
+        <div class="col-sm-4">
+          <div class="panel panel-success">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+          <div class="panel panel-info">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+        </div><!-- /.col-sm-4 -->
+        <div class="col-sm-4">
+          <div class="panel panel-warning">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+          <div class="panel panel-danger">
+            <div class="panel-heading">
+              <h3 class="panel-title">Panel title</h3>
+            </div>
+            <div class="panel-body">
+              Panel content
+            </div>
+          </div>
+        </div><!-- /.col-sm-4 -->
+      </div>
+
+
+      <div class="page-header">
+        <h1>Wells</h1>
+      </div>
+      <div class="well">
+        <p>Lorem ipsum dolor sit amet, consectetur adipiscing elit. Maecenas sed diam eget risus varius blandit sit amet non magna. Lorem ipsum dolor sit amet, consectetur adipiscing elit. Praesent commodo cursus magna, vel scelerisque nisl consectetur et. Cras mattis consectetur purus sit amet fermentum. Duis mollis, est non commodo luctus, nisi erat porttitor ligula, eget lacinia odio sem nec elit. Aenean lacinia bibendum nulla sed consectetur.</p>
+      </div>
+      <ul class="nav nav-tabs">
+        <li role="presentation" class="active"><a href="#">Home</a></li>
+        <li role="presentation"><a href="#">Profile</a></li>
+        <li role="presentation"><a href="#">Messages</a></li>
+      </ul>
+      This is tabs above here
+
+    </div> <!-- /container -->
+
+
+{% endblock %}

caravel/templates/caravel/sql.html

@@ -1,51 +0,0 @@
-{% extends "caravel/basic.html" %}
-
-
-{% block body %}
-<div class="container-fluid">
-  {% include 'caravel/flash_wrapper.html' %}
-  <div class="sqlcontent" style="display: none;">
-    <h3>db: [{{ db }}]</h3>
-    <div class="row interactions">
-      <div class="col-xs-7">
-        <input type="hidden" id="database_id" value="{{ db.id }}">
-        <button class="btn btn-default" id="run">{{_("Run!")}}</button>
-        <button class="btn btn-default" id="create_view">{{_("Create View")}}</button>
-      </div>
-      <div class="col-xs-5">
-        <select id="dbtable">
-          {% for t in tables %}
-            <option value="{{ t }}"
-                {{ "selected" if t == table_name else '' }}>
-              {{ t }}
-            </option>
-          {% endfor %}
-        </select>
-        <button class="btn btn-default" id="select_star">SELECT *</button>
-      </div>
-    </div>
-    <div class="topsql row">
-      <div class="col-xs-7 fillheight">
-        <textarea id="sql" class="fillup"></textarea>
-      </div>
-      <div class="col-xs-5 fillheight">
-        <div class="metadata fillup bordered"></div>
-      </div>
-    </div>
-    <div id="interactive">
-    </div>
-    <div id="results_section">
-      <hr/>
-      <img id="loading" width="25" style="display: none;" src="/static/assets/images/loading.gif">
-    </div>
-    <div>
-      <div id="results" class="bordered" style="display: none;"></div>
-    </div>
-  </div>
-</div>
-{% endblock %}
-
-{% block tail_js %}
-{{ super() }}
-<script src="/static/assets/javascripts/dist/sql.entry.js"></script>
-{% endblock %}

caravel/templates/caravel/sqllab.html

@@ -0,0 +1,6 @@
+{% extends "caravel/basic.html" %}
+
+{% block tail_js %}
+  {{ super() }}
+  <script src="/static/assets/dist/sqllab.entry.js"></script>
+{% endblock %}

caravel/templates/caravel/standalone.html

@@ -23,7 +23,7 @@
       <div id="{{ viz.token }}_con" class="slice_container" style="height: 100%; width: 100%"></div>
     </div>
   </body>
-  <script src="/static/assets/javascripts/dist/css-theme.entry.js"></script>
-  <script src="/static/assets/javascripts/dist/standalone.entry.js"></script>
+  <script src="/static/assets/dist/css-theme.entry.js"></script>
+  <script src="/static/assets/dist/standalone.entry.js"></script>
 </html>
 

caravel/templates/caravel/theme.html

@@ -337,7 +337,6 @@
                   <a href="#" class="btn btn-default">Button</a>
               </div>
             </div>
-
           </div>
         </div>
       </div>
@@ -1213,15 +1212,6 @@
           </div>
           <div class="col-lg-4">
             <div class="bs-component">
-              <div class="panel panel-danger">
-                <div class="panel-heading">
-                  <h3 class="panel-title">Panel danger</h3>
-                </div>
-                <div class="panel-body">
-                  Panel content
-                </div>
-              </div>
-
               <div class="panel panel-info">
                 <div class="panel-heading">
                   <h3 class="panel-title">Panel info</h3>
@@ -1240,10 +1230,17 @@
                 </div>
               </div>
 
+              <div class="panel panel-danger">
+                <div class="panel-heading">
+                  <h3 class="panel-title">Panel danger</h3>
+                </div>
+                <div class="panel-body">
+                  Panel content
+                </div>
+              </div>
             </div>
           </div>
         </div>
-      </div>
 
       <!-- Dialogs
       ================================================== -->

caravel/templates/caravel/welcome.html

@@ -2,7 +2,7 @@
 
 {% block head_js %}
 {{ super() }}
-<script src="/static/assets/javascripts/dist/welcome.entry.js"></script>
+<script src="/static/assets/dist/welcome.entry.js"></script>
 {% endblock %}
 
 {% block title %}{{ _("Welcome!") }}{% endblock %}

caravel/utils.py

@@ -4,12 +4,13 @@ from __future__ import division
 from __future__ import print_function
 from __future__ import unicode_literals
 
-from datetime import datetime, date
+from datetime import date, datetime
 import decimal
 import functools
 import json
 import logging
 import numpy
+import time
 import uuid
 
 import parsedatetime
@@ -200,11 +201,16 @@ def init(caravel):
 
     perms = db.session.query(ab_models.PermissionView).all()
     for perm in perms:
-        if perm.permission.name in ('datasource_access', 'database_access'):
+        if (
+                perm.permission and
+                perm.permission.name in ('datasource_access', 'database_access')):
             continue
         if perm.view_menu and perm.view_menu.name not in (
-                'UserDBModelView', 'RoleModelView', 'ResetPasswordView',
-                'Security'):
+                'ResetPasswordView',
+                'RoleModelView',
+                'Security',
+                'UserDBModelView',
+                'SQL Lab'):
             sm.add_permission_role(alpha, perm)
         sm.add_permission_role(admin, perm)
     gamma = sm.add_role("Gamma")
@@ -217,7 +223,9 @@ def init(caravel):
                     'ResetPasswordView',
                     'RoleModelView',
                     'UserDBModelView',
+                    'SQL Lab',
                     'Security') and
+                perm.permission and
                 perm.permission.name not in (
                     'all_datasource_access',
                     'can_add',
@@ -304,6 +312,8 @@ def json_iso_dttm_ser(obj):
         return val
     if isinstance(obj, datetime):
         obj = obj.isoformat()
+    elif isinstance(obj, date):
+        obj = obj.isoformat()
     else:
         raise TypeError(
             "Unserializable object {} of type {}".format(obj, type(obj))
@@ -311,13 +321,21 @@ def json_iso_dttm_ser(obj):
     return obj
 
 
+def datetime_to_epoch(dttm):
+    return (dttm - EPOCH).total_seconds() * 1000
+
+
+def now_as_float():
+    return datetime_to_epoch(datetime.now())
+
+
 def json_int_dttm_ser(obj):
     """json serializer that deals with dates"""
     val = base_json_conv(obj)
     if val is not None:
         return val
     if isinstance(obj, datetime):
-        obj = (obj - EPOCH).total_seconds() * 1000
+        obj = datetime_to_epoch(obj)
     elif isinstance(obj, date):
         obj = (obj - EPOCH.date()).total_seconds() * 1000
     else:
@@ -329,13 +347,21 @@ def json_int_dttm_ser(obj):
 
 def error_msg_from_exception(e):
     """Translate exception into error message
+
     Database have different ways to handle exception. This function attempts
     to make sense of the exception object and construct a human readable
     sentence.
+
+    TODO(bkyryliuk): parse the Presto error message from the connection
+                     created via create_engine.
+    engine = create_engine('presto://localhost:3506/silver') -
+      gives an e.message as the str(dict)
+    presto.connect("localhost", port=3506, catalog='silver') - as a dict.
+    The latter version is parsed correctly by this function.
     """
     msg = ''
     if hasattr(e, 'message'):
-        if (type(e.message) is dict):
+        if type(e.message) is dict:
             msg = e.message.get('message')
         elif e.message:
             msg = "{}".format(e.message)

caravel/views.py

@@ -10,10 +10,9 @@ import re
 import sys
 import time
 import traceback
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import functools
-import pandas as pd
 import sqlalchemy as sqla
 
 from flask import (
@@ -32,7 +31,9 @@ from werkzeug.routing import BaseConverter
 from wtforms.validators import ValidationError
 
 import caravel
-from caravel import appbuilder, db, models, viz, utils, app, sm, ascii_art
+from caravel import (
+    appbuilder, db, models, viz, utils, app, sm, ascii_art, sql_lab
+)
 
 config = app.config
 log_this = models.Log.log_this
@@ -387,7 +388,7 @@ appbuilder.add_view_no_menu(DruidMetricInlineView)
 
 class DatabaseView(CaravelModelView, DeleteMixin):  # noqa
     datamodel = SQLAInterface(models.Database)
-    list_columns = ['database_name', 'sql_link', 'creator', 'changed_on_']
+    list_columns = ['database_name', 'creator', 'changed_on_']
     add_columns = [
         'database_name', 'sqlalchemy_uri', 'cache_timeout', 'extra']
     search_exclude_columns = ('password',)
@@ -423,7 +424,6 @@ class DatabaseView(CaravelModelView, DeleteMixin):  # noqa
     }
     label_columns = {
         'database_name': _("Database"),
-        'sql_link': _("SQL link"),
         'creator': _("Creator"),
         'changed_on_': _("Last Changed"),
         'sqlalchemy_uri': _("SQLAlchemy URI"),
@@ -452,18 +452,30 @@ appbuilder.add_view(
     category_icon='fa-database',)
 
 
+class DatabaseAsync(DatabaseView):
+    list_columns = ['id', 'database_name']
+
+appbuilder.add_view_no_menu(DatabaseAsync)
+
+
+class DatabaseTablesAsync(DatabaseView):
+    list_columns = ['id', 'all_table_names', 'all_schema_names']
+
+appbuilder.add_view_no_menu(DatabaseTablesAsync)
+
+
 class TableModelView(CaravelModelView, DeleteMixin):  # noqa
     datamodel = SQLAInterface(models.SqlaTable)
     list_columns = [
-        'table_link', 'database', 'sql_link', 'is_featured',
+        'table_link', 'database', 'is_featured',
         'changed_by_', 'changed_on_']
     order_columns = [
-        'table_link', 'database', 'sql_link', 'is_featured', 'changed_on_']
+        'table_link', 'database', 'is_featured', 'changed_on_']
     add_columns = [
         'table_name', 'database', 'schema',
         'default_endpoint', 'offset', 'cache_timeout']
     edit_columns = [
-        'table_name', 'is_featured', 'database', 'schema',
+        'table_name', 'sql', 'is_featured', 'database', 'schema',
         'description', 'owner',
         'main_dttm_col', 'default_endpoint', 'offset', 'cache_timeout']
     related_views = [TableColumnInlineView, SqlMetricInlineView]
@@ -476,6 +488,10 @@ class TableModelView(CaravelModelView, DeleteMixin):  # noqa
         'description': Markup(
             "Supports <a href='https://daringfireball.net/projects/markdown/'>"
             "markdown</a>"),
+        'sql': (
+            "This fields acts a Caravel view, meaning that Caravel will "
+            "run a query against this string as a subquery."
+        ),
     }
     base_filters = [['id', TableSlice, lambda: []]]
     label_columns = {
@@ -483,7 +499,6 @@ class TableModelView(CaravelModelView, DeleteMixin):  # noqa
         'changed_by_': _("Changed By"),
         'database': _("Database"),
         'changed_on_': _("Last Changed"),
-        'sql_link': _("SQL Editor"),
         'is_featured': _("Is Featured"),
         'schema': _("Schema"),
         'default_endpoint': _("Default Endpoint"),
@@ -610,7 +625,8 @@ class SliceModelView(CaravelModelView, DeleteMixin):  # noqa
             url = "/druiddatasourcemodelview/list/"
             msg = _(
                 "Click on a datasource link to create a Slice, "
-                "or click on a table link <a href='/tablemodelview/list/'>here</a> "
+                "or click on a table link "
+                "<a href='/tablemodelview/list/'>here</a> "
                 "to create a Slice for a table"
             )
         else:
@@ -892,7 +908,8 @@ class Caravel(BaseCaravelView):
         datasource_access = self.can_access(
             'datasource_access', datasource.perm)
         if not (all_datasource_access or datasource_access):
-            flash(__("You don't seem to have access to this datasource"), "danger")
+            flash(__("You don't seem to have access to this datasource"),
+                  "danger")
             return redirect(error_redirect)
 
         action = request.args.get('action')
@@ -911,7 +928,7 @@ class Caravel(BaseCaravelView):
                 form_data=request.args,
                 slice_=slc)
         except Exception as e:
-            flash(str(e), "danger")
+            flash(utils.error_msg_from_exception(e), "danger")
             return redirect(error_redirect)
         if request.args.get("json") == "true":
             status = 200
@@ -923,7 +940,7 @@ class Caravel(BaseCaravelView):
                     payload = obj.get_json()
                 except Exception as e:
                     logging.exception(e)
-                    payload = str(e)
+                    payload = utils.error_msg_from_exception(e)
                     status = 500
             resp = Response(
                 payload,
@@ -953,7 +970,7 @@ class Caravel(BaseCaravelView):
                 if config.get("DEBUG"):
                     raise(e)
                 return Response(
-                    str(e),
+                    utils.error_msg_from_exception(e),
                     status=500,
                     mimetype="application/json")
             return resp
@@ -969,7 +986,8 @@ class Caravel(BaseCaravelView):
         del d['action']
         del d['previous_viz_type']
 
-        as_list = ('metrics', 'groupby', 'columns', 'all_columns', 'mapbox_label', 'order_by_cols')
+        as_list = ('metrics', 'groupby', 'columns', 'all_columns',
+                   'mapbox_label', 'order_by_cols')
         for k in d:
             v = d.get(k)
             if k in as_list and not isinstance(v, list):
@@ -1080,9 +1098,29 @@ class Caravel(BaseCaravelView):
             .group_by(Log.dt)
             .all()
         )
-        payload = {str(time.mktime(dt.timetuple())): ccount for dt, ccount in qry if dt}
+        payload = {str(time.mktime(dt.timetuple())):
+                   ccount for dt, ccount in qry if dt}
         return Response(json.dumps(payload), mimetype="application/json")
 
+    @api
+    @has_access_api
+    @expose("/tables/<db_id>/<schema>")
+    def tables(self, db_id, schema):
+        """endpoint to power the calendar heatmap on the welcome page"""
+        schema = None if schema in ('null', 'undefined') else schema
+        database = (
+            db.session
+            .query(models.Database)
+            .filter_by(id=db_id)
+            .one()
+        )
+        payload = {
+            'tables': database.all_table_names(schema),
+            'views': database.all_view_names(schema),
+        }
+        return Response(
+            json.dumps(payload), mimetype="application/json")
+
     @api
     @has_access_api
     @expose("/save_dash/<dashboard_id>/", methods=['GET', 'POST'])
@@ -1117,9 +1155,11 @@ class Caravel(BaseCaravelView):
         data = json.loads(request.form.get('data'))
         session = db.session()
         Slice = models.Slice # noqa
-        dash = session.query(models.Dashboard).filter_by(id=dashboard_id).first()
+        dash = (
+            session.query(models.Dashboard).filter_by(id=dashboard_id).first())
         check_ownership(dash, raise_if_false=True)
-        new_slices = session.query(Slice).filter(Slice.id.in_(data['slice_ids']))
+        new_slices = session.query(Slice).filter(
+            Slice.id.in_(data['slice_ids']))
         dash.slices += new_slices
         session.merge(dash)
         session.commit()
@@ -1153,13 +1193,18 @@ class Caravel(BaseCaravelView):
         FavStar = models.FavStar  # noqa
         count = 0
         favs = session.query(FavStar).filter_by(
-            class_name=class_name, obj_id=obj_id, user_id=g.user.get_id()).all()
+            class_name=class_name, obj_id=obj_id,
+            user_id=g.user.get_id()).all()
         if action == 'select':
             if not favs:
                 session.add(
                     FavStar(
-                        class_name=class_name, obj_id=obj_id, user_id=g.user.get_id(),
-                        dttm=datetime.now()))
+                        class_name=class_name,
+                        obj_id=obj_id,
+                        user_id=g.user.get_id(),
+                        dttm=datetime.now()
+                    )
+                )
             count = 1
         elif action == 'unselect':
             for fav in favs:
@@ -1214,10 +1259,55 @@ class Caravel(BaseCaravelView):
             dash_save_perm=dash_save_perm,
             dash_edit_perm=dash_edit_perm)
 
+    @has_access
+    @expose("/sqllab_viz/")
+    @log_this
+    def sqllab_viz(self):
+        data = json.loads(request.args.get('data'))
+        table_name = data.get('datasourceName')
+        table = db.session.query(models.SqlaTable).filter_by(table_name=table_name).first()
+        if not table:
+            table = models.SqlaTable(
+                table_name=table_name,
+            )
+        table.database_id = data.get('databaseId')
+        table.sql = data.get('sql')
+        db.session.add(table)
+        cols = []
+        metrics = []
+        for column_name, config in data.get('columns').items():
+            is_dim = config.get('is_dim', False)
+            cols.append(models.TableColumn(
+                column_name=column_name,
+                filterable=is_dim,
+                groupby=is_dim,
+            ))
+            agg = config.get('agg')
+            if agg:
+                metrics.append(models.SqlMetric(
+                    metric_name="{agg}__{column_name}".format(**locals()),
+                    expression="{agg}({column_name})".format(**locals()),
+                ))
+        metrics.append(models.SqlMetric(
+            metric_name="count".format(**locals()),
+            expression="count(*)".format(**locals()),
+        ))
+        table.columns = cols
+        table.metrics = metrics
+        db.session.commit()
+        return redirect('/caravel/explore/table/{table.id}/'.format(**locals()))
+
     @has_access
     @expose("/sql/<database_id>/")
     @log_this
     def sql(self, database_id):
+        if (
+                not self.can_access(
+                    'all_datasource_access', 'all_datasource_access')):
+            flash(
+                "SQL Lab requires the `all_datasource_access` "
+                "permission", "danger")
+            return redirect("/tablemodelview/list/")
         mydb = db.session.query(
             models.Database).filter_by(id=database_id).first()
 
@@ -1240,23 +1330,35 @@ class Caravel(BaseCaravelView):
             db=mydb)
 
     @has_access
-    @expose("/table/<database_id>/<table_name>/")
+    @expose("/table/<database_id>/<table_name>/<schema>/")
     @log_this
-    def table(self, database_id, table_name):
-        mydb = db.session.query(
-            models.Database).filter_by(id=database_id).first()
-        cols = mydb.get_columns(table_name)
-        df = pd.DataFrame([(c['name'], c['type']) for c in cols])
-        df.columns = ['col', 'type']
-        tbl_cls = (
-            "dataframe table table-striped table-bordered "
-            "table-condensed sql_results").split(' ')
-        return self.render_template(
-            "caravel/ajah.html",
-            content=df.to_html(
-                index=False,
-                na_rep='',
-                classes=tbl_cls))
+    def table(self, database_id, table_name, schema):
+        schema = None if schema in ('null', 'undefined') else schema
+        mydb = db.session.query(models.Database).filter_by(id=database_id).one()
+        cols = []
+        t = mydb.get_columns(table_name, schema)
+        try:
+            t = mydb.get_columns(table_name, schema)
+        except Exception as e:
+            return Response(
+                json.dumps({'error': utils.error_msg_from_exception(e)}),
+                mimetype="application/json")
+        for col in t:
+            dtype = ""
+            try:
+                dtype = '{}'.format(col['type'])
+            except:
+                pass
+            cols.append({
+                'name': col['name'],
+                'type': dtype.split('(')[0] if '(' in dtype else dtype,
+                'longType': dtype,
+            })
+        tbl = {
+            'name': table_name,
+            'columns': cols,
+        }
+        return Response(json.dumps(tbl), mimetype="application/json")
 
     @has_access
     @expose("/select_star/<database_id>/<table_name>/")
@@ -1285,55 +1387,6 @@ class Caravel(BaseCaravelView):
             content=s
         )
 
-    @has_access
-    @expose("/runsql/", methods=['POST', 'GET'])
-    @log_this
-    def runsql(self):
-        """Runs arbitrary sql and returns and html table"""
-        # TODO deprecate in favor on `sql_json`
-        session = db.session()
-        limit = 1000
-        data = json.loads(request.form.get('data'))
-        sql = data.get('sql')
-        database_id = data.get('database_id')
-        mydb = session.query(models.Database).filter_by(id=database_id).first()
-
-        if not (self.can_access(
-                'all_datasource_access', 'all_datasource_access') or
-                self.can_access('database_access', mydb.perm)):
-            raise utils.CaravelSecurityException(_(
-                "SQL Lab requires the `all_datasource_access` or "
-                "specific db permission"))
-
-        content = ""
-        if mydb:
-            eng = mydb.get_sqla_engine()
-            if limit:
-                sql = sql.strip().strip(';')
-                qry = (
-                    select('*')
-                    .select_from(TextAsFrom(text(sql), ['*'])
-                                 .alias('inner_qry'))
-                    .limit(limit)
-                )
-                sql = '{}'.format(qry.compile(
-                    eng, compile_kwargs={"literal_binds": True}))
-            try:
-                df = pd.read_sql_query(sql=sql, con=eng)
-                content = df.to_html(
-                    index=False,
-                    na_rep='',
-                    classes=(
-                        "dataframe table table-striped table-bordered "
-                        "table-condensed sql_results").split(' '))
-            except Exception as e:
-                content = (
-                    '<div class="alert alert-danger">'
-                    "{}</div>"
-                ).format(e.message)
-        session.commit()
-        return content
-
     @expose("/theme/")
     def theme(self):
         return self.render_template('caravel/theme.html')
@@ -1343,56 +1396,133 @@ class Caravel(BaseCaravelView):
     @log_this
     def sql_json(self):
         """Runs arbitrary sql and returns and json"""
-        session = db.session()
-        limit = 1000
+        async = request.form.get('async') == 'true'
         sql = request.form.get('sql')
         database_id = request.form.get('database_id')
+
+        def json_error_response(msg, status=None):
+            return Response(json.dumps({
+                'error': msg,
+                'status': status,
+            }),
+                status=500,
+                mimetype="application/json"
+            )
+
+        session = db.session()
         mydb = session.query(models.Database).filter_by(id=database_id).first()
 
-        if not (self.can_access(
-                'all_datasource_access', 'all_datasource_access') or
-                self.can_access('database_access', mydb.perm)):
-            raise utils.CaravelSecurityException(_(
-                "SQL Lab requires the `all_datasource_access` or "
-                "specific DB permission"))
-
-        error_msg = ""
         if not mydb:
-            error_msg = "The database selected doesn't seem to exist"
-        else:
-            eng = mydb.get_sqla_engine()
-            if limit:
-                sql = sql.strip().strip(';')
-                qry = (
-                    select('*')
-                    .select_from(TextAsFrom(text(sql), ['*'])
-                                 .alias('inner_qry'))
-                    .limit(limit)
-                )
-                sql = '{}'.format(qry.compile(
-                    eng, compile_kwargs={"literal_binds": True}))
-            try:
-                df = pd.read_sql_query(sql=sql, con=eng)
-                df = df.fillna(0)  # TODO make sure NULL
-            except Exception as e:
-                logging.exception(e)
-                error_msg = utils.error_msg_from_exception(e)
+            json_error_response(
+                'Database with id {} is missing.'.format(database_id),
+                models.QueryStatus.FAILED)
 
+        if not (self.can_access('all_datasource_access', 'all_datasource_access') or
+                self.can_access('database_access', mydb.perm)):
+            json_error_response(__(
+                "SQL Lab requires the `all_datasource_access` or specific DB permission"))
         session.commit()
-        if error_msg:
+
+        query = models.Query(
+            database_id=int(database_id),
+            limit=int(app.config.get('SQL_MAX_ROW', None)),
+            sql=sql,
+            schema=request.form.get('schema'),
+            select_as_cta=request.form.get('select_as_cta') == 'true',
+            start_time=utils.now_as_float(),
+            tab_name=request.form.get('tab'),
+            sql_editor_id=request.form.get('sql_editor_id'),
+            tmp_table_name=request.form.get('tmp_table_name'),
+            user_id=int(g.user.get_id()),
+            client_id=request.form.get('client_id'),
+        )
+        session.add(query)
+        session.commit()
+        query_id = query.id
+
+        # Async request.
+        if async:
+            # Ignore the celery future object and the request may time out.
+            sql_lab.get_sql_results.delay(query_id)
             return Response(
-                json.dumps({
-                    'error': error_msg,
-                }),
-                status=500,
+                json.dumps({'query': query.to_dict()},
+                           default=utils.json_int_dttm_ser,
+                           allow_nan=False),
+                status=202,  # Accepted
                 mimetype="application/json")
-        else:
-            data = {
-                'columns': [c for c in df.columns],
-                'data': df.to_dict(orient='records'),
-            }
-            return json.dumps(
-                data, default=utils.json_int_dttm_ser, allow_nan=False)
+
+        # Sync request.
+        try:
+            data = sql_lab.get_sql_results(query_id)
+        except Exception as e:
+            logging.exception(e)
+            return Response(
+                json.dumps({'error': "{}".format(e)}),
+                status=500,
+                mimetype = "application/json")
+        data['query'] = query.to_dict()
+        return Response(
+            json.dumps(data, default=utils.json_iso_dttm_ser, allow_nan=False),
+            status=200,
+            mimetype = "application/json")
+
+    @has_access
+    @expose("/csv/<query_id>")
+    @log_this
+    def csv(self, query_id):
+        """Download the query results as csv."""
+        s = db.session()
+        query = s.query(models.Query).filter_by(id=int(query_id)).first()
+
+        if not (self.can_access('all_datasource_access', 'all_datasource_access') or
+                self.can_access('database_access', query.database.perm)):
+            flash(_(
+                "SQL Lab requires the `all_datasource_access` or specific DB permission"))
+            redirect('/')
+
+        sql = query.select_sql or query.sql
+        df = query.database.get_df(sql, query.schema)
+        # TODO(bkyryliuk): add compression=gzip for big files.
+        csv = df.to_csv(index=False)
+        response = Response(csv, mimetype='text/csv')
+        response.headers['Content-Disposition'] = (
+            'attachment; filename={}.csv'.format(query.name))
+        return response
+
+    @has_access
+    @expose("/queries/<last_updated_ms>")
+    @log_this
+    def queries(self, last_updated_ms):
+        """Get the updated queries."""
+        if not g.user.get_id():
+            return Response(
+                json.dumps({'error': "Please login to access the queries."}),
+                status=403,
+                mimetype="application/json")
+
+        # Unix time, milliseconds.
+        last_updated_ms_int = int(last_updated_ms) if last_updated_ms else 0
+
+        # Local date time, DO NOT USE IT.
+        # last_updated_dt = datetime.fromtimestamp(int(last_updated_ms) / 1000)
+
+        # UTC date time, same that is stored in the DB.
+        last_updated_dt = utils.EPOCH + timedelta(
+            seconds=last_updated_ms_int / 1000)
+
+        sql_queries = (
+            db.session.query(models.Query)
+            .filter(
+                models.Query.user_id == g.user.get_id() or
+                models.Query.changed_on >= last_updated_dt
+            )
+            .all()
+        )
+        dict_queries = {q.client_id: q.to_dict() for q in sql_queries}
+        return Response(
+            json.dumps(dict_queries, default=utils.json_int_dttm_ser),
+            status=200,
+            mimetype="application/json")
 
     @has_access
     @expose("/refresh_datasources/")
@@ -1433,6 +1563,11 @@ class Caravel(BaseCaravelView):
         """Personalized welcome page"""
         return self.render_template('caravel/welcome.html', utils=utils)
 
+    @has_access
+    @expose("/sqllab")
+    def sqlanvil(self):
+        """SQL Editor"""
+        return self.render_template('caravel/sqllab.html')
 
 appbuilder.add_view_no_menu(Caravel)
 
@@ -1462,6 +1597,11 @@ appbuilder.add_view(
     category_label=__("Sources"),
     category_icon='')
 
+appbuilder.add_link(
+    'SQL Lab <span class="label label-danger">alpha</span>',
+    href='/caravel/sqllab',
+    icon="fa-flask")
+
 
 # ---------------------------------------------------------------------
 # Redirecting URL from previous names

caravel/viz.py

@@ -285,7 +285,8 @@ class BaseViz(object):
                     cached_data = cached_data.decode('utf-8')
                 payload = json.loads(cached_data)
             except Exception as e:
-                logging.error("Error reading cache")
+                logging.error("Error reading cache: " +
+                              utils.error_msg_from_exception(e))
                 payload = None
             logging.info("Serving from cache")
 

run_tests.sh

@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 echo $DB
-rm ~/.caravel/caravel_unittests.db
+rm ~/.caravel/unittests.db
+rm ~/.caravel/celerydb.sqlite
+rm ~/.caravel/celery_results.sqlite
 rm -f .coverage
 export CARAVEL_CONFIG=tests.caravel_test_config
 set -e

setup.py

@@ -16,6 +16,7 @@ setup(
     zip_safe=False,
     scripts=['caravel/bin/caravel'],
     install_requires=[
+        'celery==3.1.23',
         'cryptography==1.4',
         'flask-appbuilder==1.8.1',
         'flask-cache==0.13.1',
@@ -29,6 +30,7 @@ setup(
         'pandas==0.18.1',
         'parsedatetime==2.0.0',
         'pydruid==0.3.0',
+        'PyHive>=0.2.1',
         'python-dateutil==2.5.3',
         'requests==2.10.0',
         'simplejson==3.8.2',
@@ -36,6 +38,8 @@ setup(
         'sqlalchemy==1.0.13',
         'sqlalchemy-utils==0.32.7',
         'sqlparse==0.1.19',
+        'thrift>=0.9.3',
+        'thrift-sasl>=0.2.1',
         'werkzeug==0.11.10',
     ],
     extras_require={

tests/base_tests.py

@@ -0,0 +1,93 @@
+"""Unit tests for Caravel"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import os
+import unittest
+
+from flask_appbuilder.security.sqla import models as ab_models
+
+import caravel
+from caravel import app, db, models, utils, appbuilder
+
+os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
+
+'''
+app.config['TESTING'] = True
+app.config['CSRF_ENABLED'] = False
+app.config['SECRET_KEY'] = 'thisismyscretkey'
+app.config['WTF_CSRF_ENABLED'] = False
+app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
+'''
+BASE_DIR = app.config.get("BASE_DIR")
+
+
+class CaravelTestCase(unittest.TestCase):
+
+    def __init__(self, *args, **kwargs):
+        super(CaravelTestCase, self).__init__(*args, **kwargs)
+        self.client = app.test_client()
+
+        utils.init(caravel)
+
+        admin = appbuilder.sm.find_user('admin')
+        if not admin:
+            appbuilder.sm.add_user(
+                'admin', 'admin', ' user', 'admin@fab.org',
+                appbuilder.sm.find_role('Admin'),
+                password='general')
+
+        gamma = appbuilder.sm.find_user('gamma')
+        if not gamma:
+            appbuilder.sm.add_user(
+                'gamma', 'gamma', 'user', 'gamma@fab.org',
+                appbuilder.sm.find_role('Gamma'),
+                password='general')
+
+        alpha = appbuilder.sm.find_user('alpha')
+        if not alpha:
+            appbuilder.sm.add_user(
+                'alpha', 'alpha', 'user', 'alpha@fab.org',
+                appbuilder.sm.find_role('Alpha'),
+                password='general')
+
+        utils.init(caravel)
+
+    def login(self, username='admin', password='general'):
+        resp = self.client.post(
+            '/login/',
+            data=dict(username=username, password=password),
+            follow_redirects=True)
+        assert 'Welcome' in resp.data.decode('utf-8')
+
+    def get_query_by_sql(self, sql):
+        session = db.create_scoped_session()
+        query = session.query(models.Query).filter_by(sql=sql).first()
+        session.close()
+        return query
+
+    def logout(self):
+        self.client.get('/logout/', follow_redirects=True)
+
+    def test_welcome(self):
+        self.login()
+        resp = self.client.get('/caravel/welcome')
+        assert 'Welcome' in resp.data.decode('utf-8')
+
+    def setup_public_access_for_dashboard(self, table_name):
+        public_role = appbuilder.sm.find_role('Public')
+        perms = db.session.query(ab_models.PermissionView).all()
+        for perm in perms:
+            if (perm.permission.name == 'datasource_access' and
+                    perm.view_menu and table_name in perm.view_menu.name):
+                appbuilder.sm.add_permission_role(public_role, perm)
+
+    def revoke_public_access(self, table_name):
+        public_role = appbuilder.sm.find_role('Public')
+        perms = db.session.query(ab_models.PermissionView).all()
+        for perm in perms:
+            if (perm.permission.name == 'datasource_access' and
+                    perm.view_menu and table_name in perm.view_menu.name):
+                appbuilder.sm.del_permission_role(public_role, perm)

tests/caravel_test_config.py

@@ -9,3 +9,23 @@ CARAVEL_WEBSERVER_PORT = 8081
 # continuous integration
 if 'CARAVEL__SQLALCHEMY_DATABASE_URI' in os.environ:
     SQLALCHEMY_DATABASE_URI = os.environ.get('CARAVEL__SQLALCHEMY_DATABASE_URI')
+
+SQL_CELERY_DB_FILE_PATH =  os.path.join(DATA_DIR, 'celerydb.sqlite')
+SQL_CELERY_RESULTS_DB_FILE_PATH =  os.path.join(DATA_DIR, 'celery_results.sqlite')
+SQL_SELECT_AS_CTA = True
+SQL_MAX_ROW = 666
+
+TESTING = True
+CSRF_ENABLED = False
+SECRET_KEY = 'thisismyscretkey'
+WTF_CSRF_ENABLED = False
+PUBLIC_ROLE_LIKE_GAMMA = True
+
+
+class CeleryConfig(object):
+    BROKER_URL = 'sqla+sqlite:///' + SQL_CELERY_DB_FILE_PATH
+    CELERY_IMPORTS = ('caravel.sql_lab', )
+    CELERY_RESULT_BACKEND = 'db+sqlite:///' + SQL_CELERY_RESULTS_DB_FILE_PATH
+    CELERY_ANNOTATIONS = {'sql_lab.add': {'rate_limit': '10/s'}}
+    CONCURRENCY = 1
+CELERY_CONFIG = CeleryConfig

tests/core_tests.py

@@ -5,112 +5,47 @@ from __future__ import print_function
 from __future__ import unicode_literals
 
 from datetime import datetime
+import csv
 import doctest
-import json
 import imp
-import os
+import json
+import io
 import unittest
-from mock import Mock, patch
+
 
 from flask import escape
 from flask_appbuilder.security.sqla import models as ab_models
 
 import caravel
 from caravel import app, db, models, utils, appbuilder, sm
-from caravel.models import DruidCluster, DruidDatasource
+from caravel.models import DruidDatasource
 
-os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config'
+from .base_tests import CaravelTestCase
 
-app.config['TESTING'] = True
-app.config['CSRF_ENABLED'] = False
-app.config['SECRET_KEY'] = 'thisismyscretkey'
-app.config['WTF_CSRF_ENABLED'] = False
-app.config['PUBLIC_ROLE_LIKE_GAMMA'] = True
 BASE_DIR = app.config.get("BASE_DIR")
 cli = imp.load_source('cli', BASE_DIR + "/bin/caravel")
 
-class CaravelTestCase(unittest.TestCase):
-
-    def __init__(self, *args, **kwargs):
-        super(CaravelTestCase, self).__init__(*args, **kwargs)
-        self.client = app.test_client()
-
-        utils.init(caravel)
-
-        admin = appbuilder.sm.find_user('admin')
-        if not admin:
-            appbuilder.sm.add_user(
-                'admin', 'admin',' user', 'admin@fab.org',
-                appbuilder.sm.find_role('Admin'),
-                password='general')
-
-        gamma = appbuilder.sm.find_user('gamma')
-        if not gamma:
-            appbuilder.sm.add_user(
-                'gamma', 'gamma', 'user', 'gamma@fab.org',
-                appbuilder.sm.find_role('Gamma'),
-                password='general')
-
-        alpha = appbuilder.sm.find_user('alpha')
-        if not alpha:
-            appbuilder.sm.add_user(
-                'alpha', 'alpha', 'user', 'alpha@fab.org',
-                appbuilder.sm.find_role('Alpha'),
-                password='general')
-
-        utils.init(caravel)
-
-    def login(self, username='admin', password='general'):
-        resp = self.client.post(
-            '/login/',
-            data=dict(username=username, password=password),
-            follow_redirects=True)
-        assert 'Welcome' in resp.data.decode('utf-8')
-
-    def logout(self):
-        self.client.get('/logout/', follow_redirects=True)
-
-    def test_welcome(self):
-        self.login()
-        resp = self.client.get('/caravel/welcome')
-        assert 'Welcome' in resp.data.decode('utf-8')
-
-    def setup_public_access_for_dashboard(self, table_name):
-        public_role = appbuilder.sm.find_role('Public')
-        perms = db.session.query(ab_models.PermissionView).all()
-        for perm in perms:
-            if (    perm.permission.name == 'datasource_access' and
-                    perm.view_menu and table_name in perm.view_menu.name):
-                appbuilder.sm.add_permission_role(public_role, perm)
-
-    def revoke_public_access(self, table_name):
-        public_role = appbuilder.sm.find_role('Public')
-        perms = db.session.query(ab_models.PermissionView).all()
-        for perm in perms:
-            if (    perm.permission.name == 'datasource_access' and
-                    perm.view_menu and table_name in perm.view_menu.name):
-                appbuilder.sm.del_permission_role(public_role, perm)
-
 
 class CoreTests(CaravelTestCase):
 
     def __init__(self, *args, **kwargs):
-        # Load examples first, so that we setup proper permission-view relations
-        # for all example data sources.
+        # Load examples first, so that we setup proper permission-view
+        # relations for all example data sources.
         super(CoreTests, self).__init__(*args, **kwargs)
 
     @classmethod
     def setUpClass(cls):
         cli.load_examples(load_test_data=True)
         utils.init(caravel)
-        cls.table_ids = {tbl.table_name: tbl.id  for tbl in (
+        cls.table_ids = {tbl.table_name: tbl.id for tbl in (
             db.session
             .query(models.SqlaTable)
             .all()
         )}
 
     def setUp(self):
-        pass
+        db.session.query(models.Query).delete()
+
 
     def tearDown(self):
         pass
@@ -126,7 +61,12 @@ class CoreTests(CaravelTestCase):
 
         copy_name = "Test Sankey Save"
         tbl_id = self.table_ids.get('energy_usage')
-        url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+        url = (
+            "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&"
+            "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+            "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&"
+            "collapsed_fieldsets=&action={}&datasource_name=energy_usage&"
+            "datasource_id=1&datasource_type=table&previous_viz_type=sankey")
 
         db.session.commit()
         resp = self.client.get(
@@ -146,6 +86,8 @@ class CoreTests(CaravelTestCase):
         for slc in db.session.query(Slc).all():
             urls += [
                 (slc.slice_name, 'slice_url', slc.slice_url),
+                (slc.slice_name, 'slice_id_endpoint', '/caravel/slices/{}'.
+                 format(slc.id)),
                 (slc.slice_name, 'json_endpoint', slc.viz.json_endpoint),
                 (slc.slice_name, 'csv_endpoint', slc.viz.csv_endpoint),
             ]
@@ -210,13 +152,20 @@ class CoreTests(CaravelTestCase):
 
     def test_shortner(self):
         self.login(username='admin')
-        data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey"
+        data = (
+            "//caravel/explore/table/1/?viz_type=sankey&groupby=source&"
+            "groupby=target&metric=sum__value&row_limit=5000&where=&having=&"
+            "flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name="
+            "Energy+Sankey&collapsed_fieldsets=&action=&datasource_name="
+            "energy_usage&datasource_id=1&datasource_type=table&"
+            "previous_viz_type=sankey")
         resp = self.client.post('/r/shortner/', data=data)
         assert '/r/' in resp.data.decode('utf-8')
 
     def test_save_dash(self, username='admin'):
         self.login(username=username)
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
         positions = []
         for i, slc in enumerate(dash.slices):
             d = {
@@ -237,18 +186,24 @@ class CoreTests(CaravelTestCase):
 
     def test_add_slices(self, username='admin'):
         self.login(username=username)
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
-        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
-        existing_slice = db.session.query(models.Slice).filter_by(slice_name="Name Cloud").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
+        new_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Mapbox Long/Lat").first()
+        existing_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Name Cloud").first()
         data = {
-            "slice_ids": [new_slice.data["slice_id"], existing_slice.data["slice_id"]]
+            "slice_ids": [new_slice.data["slice_id"],
+                          existing_slice.data["slice_id"]]
         }
         url = '/caravel/add_slices/{}/'.format(dash.id)
         resp = self.client.post(url, data=dict(data=json.dumps(data)))
         assert "SLICES ADDED" in resp.data.decode('utf-8')
 
-        dash = db.session.query(models.Dashboard).filter_by(slug="births").first()
-        new_slice = db.session.query(models.Slice).filter_by(slice_name="Mapbox Long/Lat").first()
+        dash = db.session.query(models.Dashboard).filter_by(
+            slug="births").first()
+        new_slice = db.session.query(models.Slice).filter_by(
+            slice_name="Mapbox Long/Lat").first()
         assert new_slice in dash.slices
         assert len(set(dash.slices)) == len(dash.slices)
 
@@ -256,7 +211,10 @@ class CoreTests(CaravelTestCase):
         self.login(username=username)
         url = '/slicemodelview/add'
         resp = self.client.get(url, follow_redirects=True)
-        assert "Click on a table link to create a Slice" in resp.data.decode('utf-8')
+        assert (
+            "Click on a table link to create a Slice" in
+            resp.data.decode('utf-8')
+        )
 
     def test_add_slice_redirect_to_druid(self, username='admin'):
         datasource = DruidDatasource(
@@ -268,7 +226,10 @@ class CoreTests(CaravelTestCase):
         self.login(username=username)
         url = '/slicemodelview/add'
         resp = self.client.get(url, follow_redirects=True)
-        assert "Click on a datasource link to create a Slice" in resp.data.decode('utf-8')
+        assert (
+            "Click on a datasource link to create a Slice"
+            in resp.data.decode('utf-8')
+        )
 
         db.session.delete(datasource)
         db.session.commit()
@@ -281,41 +242,42 @@ class CoreTests(CaravelTestCase):
         resp = self.client.get('/dashboardmodelview/list/')
         assert "List Dashboard" in resp.data.decode('utf-8')
 
-    def run_sql(self, sql, user_name):
+    def run_sql(self, sql, user_name, client_id='not_used'):
         self.login(username=user_name)
         dbid = (
             db.session.query(models.Database)
-            .filter_by(database_name="main")
+            .filter_by(database_name='main')
             .first().id
         )
         resp = self.client.post(
             '/caravel/sql_json/',
-            data=dict(database_id=dbid, sql=sql),
+            data=dict(database_id=dbid, sql=sql, select_as_create_as=False, client_id=client_id),
         )
         self.logout()
         return json.loads(resp.data.decode('utf-8'))
 
-    def test_sql_json_no_access(self):
-        self.assertRaises(
-            utils.CaravelSecurityException,
-            self.run_sql, "SELECT * FROM ab_user", 'gamma')
+    def test_sql_json(self):
+        data = self.run_sql('SELECT * FROM ab_user', 'admin')
+        assert len(data['data']) > 0
+
+        data = self.run_sql('SELECT * FROM unexistant_table', 'admin')
+        assert len(data['error']) > 0
 
     def test_sql_json_has_access(self):
         main_db = (
-            db.session.query(models.Database).filter_by(database_name="main")
-                .first()
+            db.session.query(models.Database).filter_by(database_name="main").first()
         )
         utils.merge_perm(sm, 'database_access', main_db.perm)
         db.session.commit()
         main_db_permission_view = (
             db.session.query(ab_models.PermissionView)
-                .join(ab_models.ViewMenu)
-                .filter(ab_models.ViewMenu.name == '[main].(id:1)')
-                .first()
+            .join(ab_models.ViewMenu)
+            .filter(ab_models.ViewMenu.name == '[main].(id:1)')
+            .first()
         )
         astronaut = sm.add_role("Astronaut")
         sm.add_permission_role(astronaut, main_db_permission_view)
-        # Astronaut role is Gamme + main db permissions
+        # Astronaut role is Gamma + main db permissions
         for gamma_perm in sm.find_role('Gamma').permissions:
             sm.add_permission_role(astronaut, gamma_perm)
 
@@ -326,14 +288,54 @@ class CoreTests(CaravelTestCase):
                 appbuilder.sm.find_role('Astronaut'),
                 password='general')
         data = self.run_sql('SELECT * FROM ab_user', 'gagarin')
+        db.session.query(models.Query).delete()
+        db.session.commit()
         assert len(data['data']) > 0
 
-    def test_sql_json(self):
-        data = self.run_sql("SELECT * FROM ab_user", 'admin')
-        assert len(data['data']) > 0
+    def test_csv_endpoint(self):
+        sql = "SELECT first_name, last_name FROM ab_user " \
+              "where first_name='admin'"
+        self.run_sql(sql, 'admin')
 
-        data = self.run_sql("SELECT * FROM unexistant_table", 'admin')
-        assert len(data['error']) > 0
+        query1_id = self.get_query_by_sql(sql).id
+        self.login('admin')
+        resp = self.client.get('/caravel/csv/{}'.format(query1_id))
+        data = csv.reader(io.StringIO(resp.data.decode('utf-8')))
+        expected_data = csv.reader(io.StringIO(
+            "first_name,last_name\nadmin, user\n"))
+
+        self.assertEqual(list(expected_data), list(data))
+        self.logout()
+
+    def test_queries_endpoint(self):
+        resp = self.client.get('/caravel/queries/{}'.format(0))
+        self.assertEquals(403, resp.status_code)
+
+        self.login('admin')
+        resp = self.client.get('/caravel/queries/{}'.format(0))
+        data = json.loads(resp.data.decode('utf-8'))
+        self.assertEquals(0, len(data))
+        self.logout()
+
+        self.run_sql("SELECT * FROM ab_user", 'admin', client_id='client_id_1')
+        self.run_sql("SELECT * FROM ab_user1", 'admin', client_id='client_id_2')
+        self.login('admin')
+        resp = self.client.get('/caravel/queries/{}'.format(0))
+        data = json.loads(resp.data.decode('utf-8'))
+        self.assertEquals(2, len(data))
+
+        query = db.session.query(models.Query).filter_by(
+            sql='SELECT * FROM ab_user').first()
+        query.changed_on = utils.EPOCH
+        db.session.commit()
+
+        resp = self.client.get('/caravel/queries/{}'.format(123456000))
+        data = json.loads(resp.data.decode('utf-8'))
+        self.assertEquals(1, len(data))
+
+        self.logout()
+        resp = self.client.get('/caravel/queries/{}'.format(0))
+        self.assertEquals(403, resp.status_code)
 
     def test_public_user_dashboard_access(self):
         # Try access before adding appropriate permissions.
@@ -365,6 +367,10 @@ class CoreTests(CaravelTestCase):
         assert 'Births' in data
 
         # Confirm that public doesn't have access to other datasets.
+        resp = self.client.get('/slicemodelview/list/')
+        data = resp.data.decode('utf-8')
+        assert 'wb_health_population</a>' not in data
+
         resp = self.client.get('/dashboardmodelview/list/')
         data = resp.data.decode('utf-8')
         assert "/caravel/dashboard/world_health/" not in data
@@ -398,108 +404,6 @@ class CoreTests(CaravelTestCase):
         db.session.commit()
         self.test_save_dash('alpha')
 
-SEGMENT_METADATA = [{
-  "id": "some_id",
-  "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ],
-  "columns": {
-    "__time": {
-        "type": "LONG", "hasMultipleValues": False,
-        "size": 407240380, "cardinality": None, "errorMessage": None },
-    "dim1": {
-        "type": "STRING", "hasMultipleValues": False,
-        "size": 100000, "cardinality": 1944, "errorMessage": None },
-    "dim2": {
-        "type": "STRING", "hasMultipleValues": True,
-        "size": 100000, "cardinality": 1504, "errorMessage": None },
-    "metric1": {
-        "type": "FLOAT", "hasMultipleValues": False,
-        "size": 100000, "cardinality": None, "errorMessage": None }
-  },
-  "aggregators": {
-    "metric1": {
-        "type": "longSum",
-        "name": "metric1",
-        "fieldName": "metric1" }
-  },
-  "size": 300000,
-  "numRows": 5000000
-}]
-
-GB_RESULT_SET = [
-  {
-    "version": "v1",
-    "timestamp": "2012-01-01T00:00:00.000Z",
-    "event": {
-      "name": 'Canada',
-      "sum__num": 12345678,
-    }
-  },
-  {
-    "version": "v1",
-    "timestamp": "2012-01-01T00:00:00.000Z",
-    "event": {
-      "name": 'USA',
-      "sum__num": 12345678 / 2,
-    }
-  },
-]
-
-
-class DruidTests(CaravelTestCase):
-
-    """Testing interactions with Druid"""
-
-    def __init__(self, *args, **kwargs):
-        super(DruidTests, self).__init__(*args, **kwargs)
-
-    @patch('caravel.models.PyDruid')
-    def test_client(self, PyDruid):
-        self.login(username='admin')
-        instance = PyDruid.return_value
-        instance.time_boundary.return_value = [
-            {'result': {'maxTime': '2016-01-01'}}]
-        instance.segment_metadata.return_value = SEGMENT_METADATA
-
-        cluster = (
-            db.session
-            .query(DruidCluster)
-            .filter_by(cluster_name='test_cluster')
-            .first()
-        )
-        if cluster:
-            db.session.delete(cluster)
-        db.session.commit()
-
-        cluster = DruidCluster(
-            cluster_name='test_cluster',
-            coordinator_host='localhost',
-            coordinator_port=7979,
-            broker_host='localhost',
-            broker_port=7980,
-            metadata_last_refreshed=datetime.now())
-
-        db.session.add(cluster)
-        cluster.get_datasources = Mock(return_value=['test_datasource'])
-        cluster.get_druid_version = Mock(return_value='0.9.1')
-        cluster.refresh_datasources()
-        datasource_id = cluster.datasources[0].id
-        db.session.commit()
-
-        resp = self.client.get('/caravel/explore/druid/{}/'.format(datasource_id))
-        assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')
-
-        nres = [
-            list(v['event'].items()) + [('timestamp', v['timestamp'])]
-            for v in GB_RESULT_SET]
-        nres = [dict(v) for v in nres]
-        import pandas as pd
-        df = pd.DataFrame(nres)
-        instance.export_pandas.return_value = df
-        instance.query_dict = {}
-        instance.query_builder.last_query.query_dict = {}
-        resp = self.client.get('/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&include_search=false&metrics=count&groupby=name&flt_col_0=dim1&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=test_datasource&datasource_id={}&datasource_type=druid&previous_viz_type=table&json=true&force=true'.format(datasource_id, datasource_id))
-        assert "Canada" in resp.data.decode('utf-8')
-
 
 if __name__ == '__main__':
     unittest.main()

tests/druid_tests.py

@@ -0,0 +1,129 @@
+"""Unit tests for Caravel"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from __future__ import unicode_literals
+
+from datetime import datetime
+import unittest
+
+from mock import Mock, patch
+
+from caravel import db
+from caravel.models import DruidCluster
+
+from .base_tests import CaravelTestCase
+
+
+SEGMENT_METADATA = [{
+  "id": "some_id",
+  "intervals": ["2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z"],
+  "columns": {
+    "__time": {
+        "type": "LONG", "hasMultipleValues": False,
+        "size": 407240380, "cardinality": None, "errorMessage": None},
+    "dim1": {
+        "type": "STRING", "hasMultipleValues": False,
+        "size": 100000, "cardinality": 1944, "errorMessage": None},
+    "dim2": {
+        "type": "STRING", "hasMultipleValues": True,
+        "size": 100000, "cardinality": 1504, "errorMessage": None},
+    "metric1": {
+        "type": "FLOAT", "hasMultipleValues": False,
+        "size": 100000, "cardinality": None, "errorMessage": None}
+  },
+  "aggregators": {
+    "metric1": {
+        "type": "longSum",
+        "name": "metric1",
+        "fieldName": "metric1"}
+  },
+  "size": 300000,
+  "numRows": 5000000
+}]
+
+GB_RESULT_SET = [
+  {
+    "version": "v1",
+    "timestamp": "2012-01-01T00:00:00.000Z",
+    "event": {
+      "name": 'Canada',
+      "sum__num": 12345678,
+    }
+  },
+  {
+    "version": "v1",
+    "timestamp": "2012-01-01T00:00:00.000Z",
+    "event": {
+      "name": 'USA',
+      "sum__num": 12345678 / 2,
+    }
+  },
+]
+
+
+class DruidTests(CaravelTestCase):
+
+    """Testing interactions with Druid"""
+
+    def __init__(self, *args, **kwargs):
+        super(DruidTests, self).__init__(*args, **kwargs)
+
+    @patch('caravel.models.PyDruid')
+    def test_client(self, PyDruid):
+        self.login(username='admin')
+        instance = PyDruid.return_value
+        instance.time_boundary.return_value = [
+            {'result': {'maxTime': '2016-01-01'}}]
+        instance.segment_metadata.return_value = SEGMENT_METADATA
+
+        cluster = (
+            db.session
+            .query(DruidCluster)
+            .filter_by(cluster_name='test_cluster')
+            .first()
+        )
+        if cluster:
+            db.session.delete(cluster)
+        db.session.commit()
+
+        cluster = DruidCluster(
+            cluster_name='test_cluster',
+            coordinator_host='localhost',
+            coordinator_port=7979,
+            broker_host='localhost',
+            broker_port=7980,
+            metadata_last_refreshed=datetime.now())
+
+        db.session.add(cluster)
+        cluster.get_datasources = Mock(return_value=['test_datasource'])
+        cluster.get_druid_version = Mock(return_value='0.9.1')
+        cluster.refresh_datasources()
+        datasource_id = cluster.datasources[0].id
+        db.session.commit()
+
+        resp = self.client.get('/caravel/explore/druid/{}/'.format(
+            datasource_id))
+        assert "[test_cluster].[test_datasource]" in resp.data.decode('utf-8')
+
+        nres = [
+            list(v['event'].items()) + [('timestamp', v['timestamp'])]
+            for v in GB_RESULT_SET]
+        nres = [dict(v) for v in nres]
+        import pandas as pd
+        df = pd.DataFrame(nres)
+        instance.export_pandas.return_value = df
+        instance.query_dict = {}
+        instance.query_builder.last_query.query_dict = {}
+        resp = self.client.get(
+            '/caravel/explore/druid/{}/?viz_type=table&granularity=one+day&'
+            'druid_time_origin=&since=7+days+ago&until=now&row_limit=5000&'
+            'include_search=false&metrics=count&groupby=name&flt_col_0=dim1&'
+            'flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&'
+            'action=&datasource_name=test_datasource&datasource_id={}&'
+            'datasource_type=druid&previous_viz_type=table&json=true&'
+            'force=true'.format(datasource_id, datasource_id))
+        assert "Canada" in resp.data.decode('utf-8')
+
+if __name__ == '__main__':
+    unittest.main()

tests/utils_tests.py

@@ -5,15 +5,14 @@ import unittest
 
 class UtilsTestCase(unittest.TestCase):
     def test_json_int_dttm_ser(self):
-        today = date.today()
-        now = datetime.now()
-        ms = utils.json_int_dttm_ser(today)
-        deser = (utils.EPOCH + timedelta(milliseconds=ms)).date()
-        assert today == deser, "Serialization error: %s is not %s" % (str(today), str(deser))
-        ms = utils.json_int_dttm_ser(now)
-        deser = (utils.EPOCH + timedelta(milliseconds=ms))
-        assert now == deser, "Serialization error: %s is not %s" % (str(now), str(deser))
+        dttm = datetime(2020, 1, 1)
+        ts = 1577836800000.0
+        json_int_dttm_ser = utils.json_int_dttm_ser
+        assert json_int_dttm_ser(dttm) == ts
+        assert json_int_dttm_ser(date(2020, 1, 1)) == ts
+        assert json_int_dttm_ser(datetime(1970, 1, 1)) == 0
+        assert json_int_dttm_ser(date(1970, 1, 1)) == 0
+        assert json_int_dttm_ser(dttm + timedelta(milliseconds=1)) == (ts + 1)
 
         with self.assertRaises(TypeError):
             utils.json_int_dttm_ser("this is not a date")
-