iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added time filter to query search page (#1329) 

* Added time filter to query search page

* Added start date

* Updated python endpoint test

* changed spec

* Added specs and tests

* Modified python/js tests and some function/file names
based on code review comments

* Resolved conflicts in DashboardSelect_spec and QuerySearch_spec

* Break python tests for separate functions, Move sql queries to setUp()

* Get around eslint error for spec

* Small changes based on comments
This commit is contained in:
vera-liu 2016-10-28 14:12:53 -07:00 committed by GitHub
parent 07a7736c71
commit 45efcb381c
11 changed files with 300 additions and 55 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
caravel/assets/javascripts/SqlLab/common.js
View File

@ -1,10 +0,0 @@
export const STATE_BSSTYLE_MAP = {
failed: 'danger',
pending: 'info',
fetching: 'info',
running: 'warning',
stopped: 'danger',
success: 'success',
};
export const STATUS_OPTIONS = ['success', 'failed', 'running'];

79
caravel/assets/javascripts/SqlLab/components/QuerySearch.jsx
View File

@ -1,11 +1,12 @@
const $ = window.$ = require('jquery');
import React from 'react';
import { Button } from 'react-bootstrap';
import Select from 'react-select';
import QueryTable from './QueryTable';
import DatabaseSelect from './DatabaseSelect';
import { STATUS_OPTIONS } from '../common';
import { now, epochTimeXHoursAgo,
epochTimeXDaysAgo, epochTimeXYearsAgo } from '../../modules/dates';
import { STATUS_OPTIONS, TIME_OPTIONS } from '../constants';
const propTypes = {
actions: React.PropTypes.object.isRequired,
@ -20,6 +21,8 @@ class QuerySearch extends React.PureComponent {
databaseId: null,
userId: null,
searchText: null,
from: null,
to: null,
status: 'success',
queriesArray: [],
};
@ -38,13 +41,44 @@ class QuerySearch extends React.PureComponent {
const val = (db) ? db.value : null;
this.setState({ databaseId: val });
}
insertParams(baseUrl, params) {
return baseUrl + '?' + params.join('&');
getTimeFromSelection(selection) {
switch (selection) {
case 'now':
return now();
case '1 hour ago':
return epochTimeXHoursAgo(1);
case '1 day ago':
return epochTimeXDaysAgo(1);
case '7 days ago':
return epochTimeXDaysAgo(7);
case '28 days ago':
return epochTimeXDaysAgo(28);
case '90 days ago':
return epochTimeXDaysAgo(90);
case '1 year ago':
return epochTimeXYearsAgo(1);
default:
return null;
}
}
changeFrom(user) {
const val = (user) ? user.value : null;
this.setState({ from: val });
}
changeTo(status) {
const val = (status) ? status.value : null;
this.setState({ to: val });
}
changeUser(user) {
const val = (user) ? user.value : null;
this.setState({ userId: val });
}
insertParams(baseUrl, params) {
const validParams = params.filter(
function (p) { return p !== ''; }
);
return baseUrl + '?' + validParams.join('&');
}
changeStatus(status) {
const val = (status) ? status.value : null;
this.setState({ status: val });
@ -67,10 +101,12 @@ class QuerySearch extends React.PureComponent {
}
refreshQueries() {
const params = [
`userId=${this.state.userId}`,
`databaseId=${this.state.databaseId}`,
`searchText=${this.state.searchText}`,
`status=${this.state.status}`,
this.state.userId ? `user_id=${this.state.userId}` : '',
this.state.databaseId ? `database_id=${this.state.databaseId}` : '',
this.state.searchText ? `search_text=${this.state.searchText}` : '',
this.state.status ? `status=${this.state.status}` : '',
this.state.from ? `from=${this.getTimeFromSelection(this.state.from)}` : '',
this.state.to ? `to=${this.getTimeFromSelection(this.state.to)}` : '',
];
const url = this.insertParams('/caravel/search_queries', params);
@ -113,9 +149,30 @@ class QuerySearch extends React.PureComponent {
placeholder="Search Results"
/>
</div>
<div className="col-sm-2">
<div className="col-sm-1">
<Select
name="select-state"
name="select-from"
placeholder="[From]-"
options={TIME_OPTIONS.
slice(1, TIME_OPTIONS.length).map((t) => ({ value: t, label: t }))}
value={this.state.from}
autosize={false}
onChange={this.changeFrom.bind(this)}
/>
</div>
<div className="col-sm-1">
<Select
name="select-to"
placeholder="[To]-"
options={TIME_OPTIONS.map((t) => ({ value: t, label: t }))}
value={this.state.to}
autosize={false}
onChange={this.changeTo.bind(this)}
/>
</div>
<div className="col-sm-1">
<Select
name="select-status"
placeholder="[Query Status]"
options={STATUS_OPTIONS.map((s) => ({ value: s, label: s }))}
value={this.state.status}
@ -130,7 +187,7 @@ class QuerySearch extends React.PureComponent {
</div>
<QueryTable
columns={[
'state', 'db', 'user',
'state', 'db', 'user', 'date',
'progress', 'rows', 'sql', 'querylink',
]}
onUserClicked={this.onUserClicked.bind(this)}

3
caravel/assets/javascripts/SqlLab/components/QueryTable.jsx
View File

@ -8,7 +8,7 @@ import VisualizeModal from './VisualizeModal';
import ResultSet from './ResultSet';
import ModalTrigger from '../../components/ModalTrigger';
import HighlightedSql from './HighlightedSql';
import { STATE_BSSTYLE_MAP } from '../common';
import { STATE_BSSTYLE_MAP } from '../constants';
import { fDuration } from '../../modules/dates';
import { getLink } from '../../../utils/common';
@ -73,6 +73,7 @@ class QueryTable extends React.PureComponent {
if (q.endDttm) {
q.duration = fDuration(q.startDttm, q.endDttm);
}
q.date = moment(q.startDttm).format('MMM Do YYYY');
q.user = (
<button
className="btn btn-link btn-xs"

2
caravel/assets/javascripts/SqlLab/components/Timer.jsx
View File

@ -1,7 +1,7 @@
import React from 'react';
import { now, fDuration } from '../../modules/dates';
import { STATE_BSSTYLE_MAP } from '../common.js';
import { STATE_BSSTYLE_MAP } from '../constants.js';
class Timer extends React.PureComponent {
constructor(props) {

24
caravel/assets/javascripts/SqlLab/constants.js Normal file
View File

@ -0,0 +1,24 @@
export const STATE_BSSTYLE_MAP = {
failed: 'danger',
pending: 'info',
fetching: 'info',
running: 'warning',
stopped: 'danger',
success: 'success',
};
export const STATUS_OPTIONS = [
'success',
'failed',
'running',
];
export const TIME_OPTIONS = [
'now',
'1 hour ago',
'1 day ago',
'7 days ago',
'28 days ago',
'90 days ago',
'1 year ago',
];

21
caravel/assets/javascripts/modules/dates.js
View File

@ -85,3 +85,24 @@ export const now = function () {
return moment().utc().valueOf();
};
export const epochTimeXHoursAgo = function (h) {
return moment()
.subtract(h, 'hours')
.utc()
.valueOf();
};
export const epochTimeXDaysAgo = function (d) {
return moment()
.subtract(d, 'days')
.utc()
.valueOf();
};
export const epochTimeXYearsAgo = function (y) {
return moment()
.subtract(y, 'years')
.utc()
.valueOf();
};

1
caravel/assets/package.json
View File

@ -116,6 +116,7 @@
"less-loader": "^2.2.2",
"mocha": "^2.4.5",
"react-addons-test-utils": "^15.3.2",
"sinon": "^1.17.6",
"style-loader": "^0.13.0",
"transform-loader": "^0.2.3",
"url-loader": "^0.5.7",

34
caravel/assets/spec/javascripts/sqllab/DatabaseSelect_spec.jsx Normal file
View File

@ -0,0 +1,34 @@
import React from 'react';
import Select from 'react-select';
import DatabaseSelect from '../../../javascripts/SqlLab/components/DatabaseSelect';
import { shallow } from 'enzyme';
import { describe, it } from 'mocha';
import { expect } from 'chai';
import sinon from 'sinon';
describe('DatabaseSelect', () => {
const mockedProps = {
actions: {},
};
it('is valid element', () => {
expect(
React.isValidElement(<DatabaseSelect {...mockedProps} />)
).to.equal(true);
});
it('has one select', () => {
const wrapper = shallow(
<DatabaseSelect {...mockedProps} />
);
expect(wrapper.find(Select)).to.have.length(1);
});
it('calls onChange on select change', () => {
const onChange = sinon.spy();
const wrapper = shallow(
<DatabaseSelect onChange={onChange} />
);
wrapper.find(Select).simulate('change', { value: 1 });
expect(onChange).to.have.property('callCount', 1);
});
});

45
caravel/assets/spec/javascripts/sqllab/QuerySearch_spec.jsx
View File

@ -5,6 +5,7 @@ import QuerySearch from '../../../javascripts/SqlLab/components/QuerySearch';
import { shallow } from 'enzyme';
import { describe, it } from 'mocha';
import { expect } from 'chai';
import sinon from 'sinon';
describe('QuerySearch', () => {
const mockedProps = {
@ -15,19 +16,53 @@ describe('QuerySearch', () => {
React.isValidElement(<QuerySearch {...mockedProps} />)
).to.equal(true);
});
const wrapper = shallow(<QuerySearch {...mockedProps} />);
it('should have two Select', () => {
const wrapper = shallow(<QuerySearch {...mockedProps} />);
expect(wrapper.find(Select)).to.have.length(2);
it('should have four Select', () => {
expect(wrapper.find(Select)).to.have.length(4);
});
it('updates userId on user selects change', () => {
wrapper.find('[name="select-user"]')
.simulate('change', { value: 1 });
expect(wrapper.state().userId).to.equal(1);
});
it('updates fromTime on user selects from time', () => {
wrapper.find('[name="select-from"]')
.simulate('change', { value: 0 });
expect(wrapper.state().from).to.equal(0);
});
it('updates toTime on user selects to time', () => {
wrapper.find('[name="select-to"]')
.simulate('change', { value: 0 });
expect(wrapper.state().to).to.equal(0);
});
it('updates status on user selects status', () => {
wrapper.find('[name="select-status"]')
.simulate('change', { value: 'success' });
expect(wrapper.state().status).to.equal('success');
});
it('should have one input for searchText', () => {
const wrapper = shallow(<QuerySearch {...mockedProps} />);
expect(wrapper.find('input')).to.have.length(1);
});
it('updates search text on user inputs search text', () => {
wrapper.find('input').simulate('change', { target: { value: 'text' } });
expect(wrapper.state().searchText).to.equal('text');
});
it('should have one Button', () => {
const wrapper = shallow(<QuerySearch {...mockedProps} />);
expect(wrapper.find(Button)).to.have.length(1);
});
it('refreshes queries when clicked', () => {
const search = sinon.spy(QuerySearch.prototype, 'refreshQueries');
wrapper.find(Button).simulate('click');
/* eslint-disable no-unused-expressions */
expect(search).to.have.been.called;
});
});

33
caravel/views.py
View File

@ -2226,30 +2226,41 @@ class Caravel(BaseCaravelView):
def search_queries(self):
"""Search for queries."""
query = db.session.query(models.Query)
userId = request.args.get('userId')
databaseId = request.args.get('databaseId')
searchText = request.args.get('searchText')
search_user_id = request.args.get('user_id')
database_id = request.args.get('database_id')
search_text = request.args.get('search_text')
status = request.args.get('status')
# From and To time stamp should be Epoch timestamp in seconds
from_time = request.args.get('from')
to_time = request.args.get('to')
if userId != 'null':
if search_user_id:
# Filter on db Id
query = query.filter(models.Query.user_id == userId)
query = query.filter(models.Query.user_id == search_user_id)
if databaseId != 'null':
if database_id:
# Filter on db Id
query = query.filter(models.Query.database_id == databaseId)
query = query.filter(models.Query.database_id == database_id)
if status != 'null':
if status:
# Filter on status
query = query.filter(models.Query.status == status)
if searchText != 'null':
if search_text:
# Filter on search text
query = query.filter(models.Query.sql.like('%{}%'.format(searchText)))
query = query \
.filter(models.Query.sql.like('%{}%'.format(search_text)))
sql_queries = query.limit(config.get("QUERY_SEARCH_LIMIT")).all()
if from_time:
query = query.filter(models.Query.start_time > int(from_time))
if to_time:
query = query.filter(models.Query.start_time < int(to_time))
query_limit = config.get('QUERY_SEARCH_LIMIT', 5000)
sql_queries = query.limit(query_limit).all()
dict_queries = {q.client_id: q.to_dict() for q in sql_queries}
return Response(
json.dumps(dict_queries, default=utils.json_int_dttm_ser),
status=200,

103
tests/core_tests.py
View File

@ -10,7 +10,7 @@ import json
import io
import random
import unittest
from datetime import datetime
from flask import escape
from flask_appbuilder.security.sqla import models as ab_models
@ -41,6 +41,14 @@ class CoreTests(CaravelTestCase):
def setUp(self):
db.session.query(models.Query).delete()
db.session.query(models.DatasourceAccessRequest).delete()
self.login('admin')
self.run_sql("SELECT * FROM ab_user", 'admin', client_id='client_id_1')
self.run_sql("SELECT * FROM NO_TABLE", 'admin', client_id='client_id_3')
self.logout()
self.login('gamma')
self.run_sql("SELECT * FROM ab_permission", 'gamma', client_id='client_id_2')
self.logout()
def tearDown(self):
pass
@ -332,35 +340,98 @@ class CoreTests(CaravelTestCase):
self.login('admin')
data = self.get_json_resp('/caravel/queries/{}'.format(0))
self.assertEquals(0, len(data))
self.assertEquals(2, len(data))
self.logout()
self.run_sql("SELECT * FROM ab_user", 'admin', client_id='client_id_1')
self.run_sql("SELECT * FROM ab_user1", 'admin', client_id='client_id_2')
self.run_sql("SELECT * FROM ab_user1", 'admin', client_id='client_id_4')
self.run_sql("SELECT * FROM ab_user2", 'admin', client_id='client_id_5')
self.login('admin')
data = self.get_json_resp('/caravel/queries/{}'.format(0))
self.assertEquals(2, len(data))
self.assertEquals(4, len(data))
query = db.session.query(models.Query).filter_by(
sql='SELECT * FROM ab_user').first()
sql='SELECT * FROM ab_user1').first()
query.changed_on = utils.EPOCH
db.session.commit()
data = self.get_json_resp('/caravel/queries/{}'.format(123456000))
self.assertEquals(1, len(data))
self.assertEquals(3, len(data))
self.logout()
resp = self.client.get('/caravel/queries/{}'.format(0))
self.assertEquals(403, resp.status_code)
def test_search_query_endpoint(self):
userId = 'userId=null'
databaseId = 'databaseId=null'
searchText = 'searchText=null'
status = 'status=success'
params = [userId, databaseId, searchText, status]
resp = self.client.get('/caravel/search_queries?'+'&'.join(params))
self.assertEquals(200, resp.status_code)
def test_search_query_on_db_id(self):
self.login('admin')
# Test search queries on database Id
resp = self.get_resp('/caravel/search_queries?database_id=1')
data = json.loads(resp)
self.assertEquals(3, len(data))
db_ids = [data[k]['dbId'] for k in data]
self.assertEquals([1, 1, 1], db_ids)
resp = self.get_resp('/caravel/search_queries?database_id=-1')
data = json.loads(resp)
self.assertEquals(0, len(data))
self.logout()
def test_search_query_on_user(self):
self.login('admin')
# Test search queries on user Id
user = appbuilder.sm.find_user('admin')
resp = self.get_resp('/caravel/search_queries?user_id={}'.format(user.id))
data = json.loads(resp)
self.assertEquals(2, len(data))
user_ids = [data[k]['userId'] for k in data]
self.assertEquals([user.id, user.id], user_ids)
user = appbuilder.sm.find_user('gamma')
resp = self.get_resp('/caravel/search_queries?user_id={}'.format(user.id))
data = json.loads(resp)
self.assertEquals(1, len(data))
self.assertEquals(list(data.values())[0]['userId'] , user.id)
self.logout()
def test_search_query_on_status(self):
self.login('admin')
# Test search queries on status
resp = self.get_resp('/caravel/search_queries?status=success')
data = json.loads(resp)
self.assertEquals(2, len(data))
states = [data[k]['state'] for k in data]
self.assertEquals(['success', 'success'], states)
resp = self.get_resp('/caravel/search_queries?status=failed')
data = json.loads(resp)
self.assertEquals(1, len(data))
self.assertEquals(list(data.values())[0]['state'], 'failed')
self.logout()
def test_search_query_on_text(self):
self.login('admin')
resp = self.get_resp('/caravel/search_queries?search_text=permission')
data = json.loads(resp)
self.assertEquals(1, len(data))
self.assertIn('permission', list(data.values())[0]['sql'])
self.logout()
def test_search_query_on_time(self):
self.login('admin')
first_query_time = db.session.query(models.Query).filter_by(
sql='SELECT * FROM ab_user').first().start_time
second_query_time = db.session.query(models.Query).filter_by(
sql='SELECT * FROM ab_permission').first().start_time
# Test search queries on time filter
from_time = 'from={}'.format(int(first_query_time))
to_time = 'to={}'.format(int(second_query_time))
params = [from_time, to_time]
resp = self.get_resp('/caravel/search_queries?'+'&'.join(params))
data = json.loads(resp)
self.assertEquals(2, len(data))
for _, v in data.items():
self.assertLess(int(first_query_time), v['startDttm'])
self.assertLess(v['startDttm'], int(second_query_time))
self.logout()
def test_public_user_dashboard_access(self):
# Try access before adding appropriate permissions.
@ -448,6 +519,6 @@ class CoreTests(CaravelTestCase):
data = self.run_sql(sql, "admin", "fdaklj3ws")
self.assertEqual(data['data'][0]['test'], "2017-01-01T00:00:00")
if __name__ == '__main__':
unittest.main()