diff --git a/TODO.md b/TODO.md index a9fd58164..a12340c54 100644 --- a/TODO.md +++ b/TODO.md @@ -34,7 +34,6 @@ List of TODO items for Caravel ## Easy-ish fix * Build matrix to include mysql using tox -* Kill switch for Druid in docs * CREATE VIEW button from SQL editor * Test button for when editing SQL expression * Slider form element diff --git a/caravel/models.py b/caravel/models.py index 8fdf14501..5cab4fd03 100644 --- a/caravel/models.py +++ b/caravel/models.py @@ -21,7 +21,7 @@ from dateutil.parser import parse from flask import flash, request, g from flask.ext.appbuilder import Model from flask.ext.appbuilder.models.mixins import AuditMixin -from pydruid import client +from pydruid.client import PyDruid from pydruid.utils.filters import Dimension, Filter from six import string_types from sqlalchemy import ( @@ -830,19 +830,21 @@ class DruidCluster(Model, AuditMixinNullable): return self.cluster_name def get_pydruid_client(self): - cli = client.PyDruid( + cli = PyDruid( "http://{0}:{1}/".format(self.broker_host, self.broker_port), self.broker_endpoint) return cli - def refresh_datasources(self): + def get_datasources(self): endpoint = ( "http://{obj.coordinator_host}:{obj.coordinator_port}/" "{obj.coordinator_endpoint}/datasources" ).format(obj=self) - datasources = json.loads(requests.get(endpoint).text) - for datasource in datasources: + return json.loads(requests.get(endpoint).text) + + def refresh_datasources(self): + for datasource in self.get_datasources(): DruidDatasource.sync_to_db(datasource, self) @@ -950,9 +952,9 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): if not datasource: datasource = cls(datasource_name=name) session.add(datasource) - flash("Adding new datasource [{}]".format(name), "success") + logging.info("Adding new datasource [{}]".format(name)) else: - flash("Refreshing datasource [{}]".format(name), "info") + logging.info("Refreshing datasource [{}]".format(name)) datasource.cluster = cluster cols = datasource.latest_metadata() @@ -977,7 +979,7 @@ class DruidDatasource(Model, AuditMixinNullable, Queryable): col_obj.datasource = datasource col_obj.generate_metrics() - def query( + def query( # druid self, groupby, metrics, granularity, from_dttm, to_dttm, diff --git a/caravel/views.py b/caravel/views.py index c2e72791d..6908e086a 100644 --- a/caravel/views.py +++ b/caravel/views.py @@ -365,13 +365,13 @@ class DruidDatasourceModelView(CaravelModelView, DeleteMixin): # noqa 'created_by_', 'created_on', 'changed_by_', 'changed_on', 'offset'] - order_columns = utils.list_minus( - list_columns, ['created_by_', 'changed_by_']) - related_views = [DruidColumnInlineView, DruidMetricInlineView] + related_views = [ + DruidColumnInlineView, DruidMetricInlineView] edit_columns = [ 'datasource_name', 'cluster', 'description', 'owner', 'is_featured', 'is_hidden', 'default_endpoint', 'offset', 'cache_timeout'] + add_columns = edit_columns page_size = 500 base_order = ('datasource_name', 'asc') description_columns = { @@ -415,7 +415,6 @@ class R(BaseView): def index(self, url_id): url = db.session.query(models.Url).filter_by(id=url_id).first() if url: - print(url.url) return redirect('/' + url.url) else: flash("URL to nowhere...", "danger") diff --git a/caravel/viz.py b/caravel/viz.py index 2765244f9..30ff920b1 100644 --- a/caravel/viz.py +++ b/caravel/viz.py @@ -68,6 +68,7 @@ class BaseViz(object): if not form.validate(): for k, v in form.errors.items(): if not data.get('json') and not data.get('async'): + logging.error("{}: {}".format(k, " ".join(v))) flash("{}: {}".format(k, " ".join(v)), 'danger') if previous_viz_type != self.viz_type: data = { diff --git a/dev-reqs.txt b/dev-reqs.txt index 73cfa730c..8bbd6bdac 100644 --- a/dev-reqs.txt +++ b/dev-reqs.txt @@ -1,4 +1,5 @@ coveralls +mock nose sphinx sphinx_bootstrap_theme diff --git a/tests/core_tests.py b/tests/core_tests.py index fb95358c4..d8c3876bc 100644 --- a/tests/core_tests.py +++ b/tests/core_tests.py @@ -1,12 +1,21 @@ +"""Unit tests for Caravel""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from __future__ import unicode_literals + +from datetime import datetime import doctest import imp import os import unittest +from mock import Mock, patch from flask import escape import caravel from caravel import app, db, models, utils, appbuilder +from caravel.models import DruidCluster os.environ['CARAVEL_CONFIG'] = 'tests.caravel_test_config' @@ -18,10 +27,10 @@ BASE_DIR = app.config.get("BASE_DIR") cli = imp.load_source('cli', BASE_DIR + "/bin/caravel") -class CaravelTests(unittest.TestCase): +class CaravelTestCase(unittest.TestCase): def __init__(self, *args, **kwargs): - super(CaravelTests, self).__init__(*args, **kwargs) + super(CaravelTestCase, self).__init__(*args, **kwargs) self.client = app.test_client() role_admin = appbuilder.sm.find_role('Admin') user = appbuilder.sm.find_user('admin') @@ -29,6 +38,23 @@ class CaravelTests(unittest.TestCase): appbuilder.sm.add_user( 'admin', 'admin',' user', 'admin@fab.org', role_admin, 'general') + + def login(self): + self.client.post( + '/login/', + data=dict(username='admin', password='general'), + follow_redirects=True) + + +class CoreTests(CaravelTestCase): + + def __init__(self, *args, **kwargs): + super(CoreTests, self).__init__(*args, **kwargs) + self.table_ids = {tbl.table_name: tbl.id for tbl in ( + db.session + .query(models.SqlaTable) + .all() + )} utils.init(caravel) self.load_examples() @@ -38,15 +64,27 @@ class CaravelTests(unittest.TestCase): def tearDown(self): pass - def login(self): - self.client.post( - '/login/', - data=dict(username='admin', password='general'), - follow_redirects=True) - def load_examples(self): cli.load_examples(sample=True) + def test_save_slice(self): + self.login() + + slice_id = db.session.query(models.Slice.id).filter_by(slice_name="Energy Sankey").scalar() + copy_name = "Test Sankey Save" + tbl_id = self.table_ids.get('energy_usage') + url = "/caravel/explore/table/{}/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id={}&slice_name={}&collapsed_fieldsets=&action={}&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey" + + db.session.commit() + resp = self.client.get( + url.format(tbl_id, slice_id, copy_name, 'save'), + follow_redirects=True) + assert copy_name in resp.data.decode('utf-8') + resp = self.client.get( + url.format(tbl_id, slice_id, copy_name, 'overwrite'), + follow_redirects=True) + assert 'Energy' in resp.data.decode('utf-8') + def test_slices(self): # Testing by running all the examples self.login() @@ -60,21 +98,12 @@ class CaravelTests(unittest.TestCase): for url in urls: self.client.get(url) - def test_csv(self): - self.client.get('/caravel/explore/table/1/?viz_type=table&granularity=ds&since=100+years&until=now&metrics=count&groupby=name&limit=50&show_brush=y&show_brush=false&show_legend=y&show_brush=false&rich_tooltip=y&show_brush=false&show_brush=false&show_brush=false&show_brush=false&y_axis_format=&x_axis_showminmax=y&show_brush=false&line_interpolation=linear&rolling_type=None&rolling_periods=&time_compare=&num_period_compare=&where=&having=&flt_col_0=gender&flt_op_0=in&flt_eq_0=&flt_col_0=gender&flt_op_0=in&flt_eq_0=&slice_id=14&slice_name=Boys&collapsed_fieldsets=&action=&datasource_name=birth_names&datasource_id=1&datasource_type=table&previous_viz_type=line&csv=true') - - def test_bubble_chart_no_time(self): - self.login() - response = self.client.get('/caravel/explore/table/1/?viz_type=bubble&series=source&entity=source&x=count&y=count&size=count&limit=50&x_log_scale=false&y_log_scale=false&show_legend=y&show_legend=false&max_bubble_size=25&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=&slice_name=&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=bubble&json=true&force=false') - self.assertEqual(response.status_code, 200) - def test_dashboard(self): self.login() urls = {} for dash in db.session.query(models.Dashboard).all(): urls[dash.dashboard_title] = dash.url for title, url in urls.items(): - print(url) assert escape(title) in self.client.get(url).data.decode('utf-8') def test_doctests(self): @@ -88,6 +117,108 @@ class CaravelTests(unittest.TestCase): assert self.client.get('/health').data.decode('utf-8') == "OK" assert self.client.get('/ping').data.decode('utf-8') == "OK" + def test_shortner(self): + self.login() + data = "//caravel/explore/table/1/?viz_type=sankey&groupby=source&groupby=target&metric=sum__value&row_limit=5000&where=&having=&flt_col_0=source&flt_op_0=in&flt_eq_0=&slice_id=78&slice_name=Energy+Sankey&collapsed_fieldsets=&action=&datasource_name=energy_usage&datasource_id=1&datasource_type=table&previous_viz_type=sankey" + resp = self.client.post('/r/shortner/', data=data) + assert '/r/' in resp.data.decode('utf-8') + + def test_save_dash(self): + self.login() + dash = db.session.query(models.Dashboard).filter_by(slug="births").first() + data = """{"positions":[{"slice_id":"131","col":8,"row":8,"size_x":2,"size_y":4},{"slice_id":"132","col":10,"row":8,"size_x":2,"size_y":4},{"slice_id":"133","col":1,"row":1,"size_x":2,"size_y":2},{"slice_id":"134","col":3,"row":1,"size_x":2,"size_y":2},{"slice_id":"135","col":5,"row":4,"size_x":3,"size_y":3},{"slice_id":"136","col":1,"row":7,"size_x":7,"size_y":4},{"slice_id":"137","col":9,"row":1,"size_x":3,"size_y":3},{"slice_id":"138","col":5,"row":1,"size_x":4,"size_y":3},{"slice_id":"139","col":1,"row":3,"size_x":4,"size_y":4},{"slice_id":"140","col":8,"row":4,"size_x":4,"size_y":4}],"css":"None","expanded_slices":{}}""" + url = '/caravel/save_dash/{}/'.format(dash.id) + resp = self.client.post(url, data=dict(data=data)) + assert "SUCCESS" in resp.data.decode('utf-8') + + +SEGMENT_METADATA = [{ + "id": "some_id", + "intervals": [ "2013-05-13T00:00:00.000Z/2013-05-14T00:00:00.000Z" ], + "columns": { + "__time": { + "type": "LONG", "hasMultipleValues": False, + "size": 407240380, "cardinality": None, "errorMessage": None }, + "dim1": { + "type": "STRING", "hasMultipleValues": False, + "size": 100000, "cardinality": 1944, "errorMessage": None }, + "dim2": { + "type": "STRING", "hasMultipleValues": True, + "size": 100000, "cardinality": 1504, "errorMessage": None }, + "metric1": { + "type": "FLOAT", "hasMultipleValues": False, + "size": 100000, "cardinality": None, "errorMessage": None } + }, + "aggregators": { + "metric1": { + "type": "longSum", + "name": "metric1", + "fieldName": "metric1" } + }, + "size": 300000, + "numRows": 5000000 +}] + +GB_RESULT_SET = [ + { + "version": "v1", + "timestamp": "2012-01-01T00:00:00.000Z", + "event": { + "name": 'Canada', + "sum__num": 12345678, + } + }, + { + "version": "v1", + "timestamp": "2012-01-01T00:00:00.000Z", + "event": { + "name": 'USA', + "sum__num": 12345678 / 2, + } + }, +] + + +class DruidTests(CaravelTestCase): + + """Testing interactions with Druid""" + + def __init__(self, *args, **kwargs): + super(DruidTests, self).__init__(*args, **kwargs) + + @patch('caravel.models.PyDruid') + def test_client(self, PyDruid): + instance = PyDruid.return_value + instance.time_boundary.return_value = [ + {'result': {'maxTime': '2016-01-01'}}] + instance.segment_metadata.return_value = SEGMENT_METADATA + instance.groupby = GB_RESULT_SET + + cluster = ( + db.session + .query(DruidCluster) + .filter_by(cluster_name='test_cluster') + .first() + ) + if cluster: + db.session.delete(cluster) + db.session.commit() + + cluster = DruidCluster( + cluster_name='test_cluster', + coordinator_host='localhost', + coordinator_port=7979, + broker_host='localhost', + broker_port=7980, + metadata_last_refreshed=datetime.now()) + + db.session.add(cluster) + cluster.get_datasources = Mock(return_value=['test_datasource']) + cluster.refresh_datasources() + db.session.commit() + + self.client.get('/caravel/explore/druid/1/') + if __name__ == '__main__': unittest.main()