diff --git a/README.md b/README.md index 93546298d..94767609d 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,31 @@ Panoramix Panoramix is a data exploration platform designed to be visual, intuitive and interactive. + +![img](http://i.imgur.com/aOaH0ty.png) + +Panoramix +--------- +Panoramix's main goal is to make it easy to slice, dice and visualize data. +It empowers its user to perform **analytics at the speed of thought**. + +Panoramix provides: +* A quick way to intuitively visualize datasets +* Create and share simple dashboards +* A rich set of visualizations to analyze your data, as well as a flexible + way to extend the capabilities +* An extensible, high granularity security model allowing intricate rules + on who can access which features, and integration with major + authentication providers (database, OpenID, LDAP, OAuth & REMOTE_USER + through Flask AppBuiler) +* A simple semantic layer, allowing to control how data sources are + displayed in the UI, + by defining which fields should show up in which dropdown and which + aggregation and function (metrics) are made available to the user +* Deep integration with Druid allows for Panoramix to stay blazing fast while + slicing and dicing large, realtime datasets + + Buzz Phrases ------------ @@ -12,17 +37,16 @@ Buzz Phrases * Realtime analytics when querying [Druid.io](http://druid.io) * Extentsible to infinity -![img](http://i.imgur.com/aOaH0ty.png) - Database Support ---------------- -Panoramix was originally designed on to of Druid.io, but quickly broadened +Panoramix was originally designed on to of Druid.io, but quickly broadened its scope to support other databases through the use of SqlAlchemy, a Python -ORM that is compatible with -[many external databases](http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html). +ORM that is compatible with +[most common databases](http://docs.sqlalchemy.org/en/rel_1_0/core/engines.html). -What's Druid? + +What is Druid? ------------- From their website at http://druid.io @@ -33,27 +57,6 @@ and fast data aggregation. Existing Druid deployments have scaled to trillions of events and petabytes of data. Druid is best used to power analytic dashboards and applications.* -Panoramix ---------- -Panoramix's main goal is to make it easy to slice, dice and visualize data -out of Druid. It empowers its user to perform **analytics -at the speed of thought**. - -Panoramix started as a hackathon project at Airbnb in while running a POC -(proof of concept) on using Druid. - -Panoramix provides: -* A way to query intuitively a Druid dataset, allowing for grouping, filtering - limiting and defining a time granularity -* Many charts and visualization to analyze your data, as well as a flexible - way to extend the visualization capabilities -* An extensible, high granularity security model allowing intricate rules - on who can access which features, and integration with major - authentication providers (through Flask AppBuiler) -* A simple semantic layer, allowing to control how Druid datasources are - displayed in the UI, - by defining which fields should show up in which dropdown and which - aggregation and function (metrics) are made available to the user Installation ------------ diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index 12a8ee44e..17bb22f76 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -60,8 +60,8 @@ def load_examples(): with gzip.open(config.basedir + '/data/birth_names.csv.gz') as f: bb_csv = csv.reader(f) - for i, (state, year, name, num, gender) in enumerate(bb_csv): - if i == 0 or not name or name=="\xc2\xa0": + for i, (state, year, name, gender, num) in enumerate(bb_csv): + if i == 0: continue if num == "NA": num = 0 @@ -71,8 +71,8 @@ def load_examples(): state=state, year=year, ds=ds, name=name, num=num, gender=gender)) - if i % 1000 == 0: - print("{} loaded out of 502619 rows".format(i)) + if i % 5000 == 0: + print("{} loaded out of 82527 rows".format(i)) session.commit() session.commit() print("Done loading table!") diff --git a/panoramix/data/birth_names.csv.gz b/panoramix/data/birth_names.csv.gz index 57acdf74e..9990ab9cc 100644 Binary files a/panoramix/data/birth_names.csv.gz and b/panoramix/data/birth_names.csv.gz differ diff --git a/panoramix/models.py b/panoramix/models.py index 037b54b69..2b23d40b3 100644 --- a/panoramix/models.py +++ b/panoramix/models.py @@ -80,7 +80,10 @@ class Slice(Model, AuditMixinNullable): @property def slice_url(self): - d = json.loads(self.params) + try: + d = json.loads(self.params) + except Exception as e: + d = {} from werkzeug.urls import Href href = Href( "/panoramix/datasource/{self.datasource_type}/" diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index 21f206534..b939b0e7d 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -105,6 +105,7 @@
+ @@ -193,14 +194,8 @@ $(this).parent().parent().slideUp("slow", function(){$(this).remove()}); }); } - $("#plus").click(add_filter); - $("#save").click(function () { - var slice_name = prompt("Name your slice!"); - $("#slice_name").val(slice_name); - $.get( "/panoramix/save/", $("#query").serialize() ); - }) - add_filter(); - $("#druidify").click(function () { + + function druidify(){ var i = 1; // removing empty filters @@ -224,7 +219,19 @@ i++; }); $("#query").submit(); - }); + } + + $("#plus").click(add_filter); + $("#save").click(function () { + var slice_name = prompt("Name your slice!"); + if (slice_name != "" && slice_name != null) { + $("#slice_name").val(slice_name); + $("#action").val("save"); + druidify(); + } + }) + add_filter(); + $("#druidify").click(druidify); function create_choices (term, data) { if ($(data).filter(function() { diff --git a/panoramix/templates/panoramix/viz_bignumber.html b/panoramix/templates/panoramix/viz_bignumber.html index 258317973..dc08c7960 100644 --- a/panoramix/templates/panoramix/viz_bignumber.html +++ b/panoramix/templates/panoramix/viz_bignumber.html @@ -36,14 +36,12 @@ var render = function(){ data = example_data(); data = json.data; var compare_suffix = ' ' + json.compare_suffix; - console.log(data); var v_compare = null; var v = data[data.length -1][1]; if (json.compare_lag >0){ pos = data.length - (json.compare_lag+1); if(pos >=0) v_compare = 1-(v / data[pos][1]); - console.log(v_compare) } var date_ext = d3.extent(data, function(d){return d[0]}); var value_ext = d3.extent(data, function(d){return d[1]}); diff --git a/panoramix/templates/panoramix/viz_table.html b/panoramix/templates/panoramix/viz_table.html index 42f7c5124..0ef758a71 100644 --- a/panoramix/templates/panoramix/viz_table.html +++ b/panoramix/templates/panoramix/viz_table.html @@ -34,6 +34,7 @@ {% macro viz_js(viz) %} {% if viz.args.get("async") != "true" %} {% endif %} {% endmacro %} diff --git a/panoramix/views.py b/panoramix/views.py index 86547fc3f..62f50b62f 100644 --- a/panoramix/views.py +++ b/panoramix/views.py @@ -144,6 +144,7 @@ appbuilder.add_view( class SliceModelView(ModelView, DeleteMixin): datamodel = SQLAInterface(models.Slice) + can_add = False list_columns = ['slice_link', 'viz_type', 'datasource', 'created_by'] edit_columns = [ 'slice_name', 'viz_type', 'druid_datasource', 'table', @@ -211,6 +212,41 @@ class Panoramix(BaseView): @has_access @expose("/datasource///") def datasource(self, datasource_type, datasource_id): + action = request.args.get('action') + if action == 'save': + session = db.session() + d = request.args.to_dict(flat=False) + del d['action'] + as_list = ('metrics', 'groupby') + for k in d: + v = d.get(k) + if k in as_list and not isinstance(v, list): + d[k] = [v] if v else [] + if k not in as_list and isinstance(v, list): + d[k] = v[0] + + table_id = druid_datasource_id = None + datasource_type = request.args.get('datasource_type') + if datasource_type == 'druid': + druid_datasource_id = request.args.get('datasource_id') + else: + table_id = request.args.get('datasource_id') + slice_name = request.args.get('slice_name') + + obj = models.Slice( + params=json.dumps(d, indent=4, sort_keys=True), + viz_type=request.args.get('viz_type'), + datasource_name=request.args.get('datasource_name'), + druid_datasource_id=druid_datasource_id, + table_id=table_id, + datasource_type=datasource_type, + slice_name=slice_name, + ) + session.add(obj) + session.commit() + flash("Slice <{}> has been added to the pie".format(slice_name), "info") + redirect(obj.slice_url) + if datasource_type == "table": datasource = ( db.session @@ -249,15 +285,13 @@ class Panoramix(BaseView): status=status, mimetype="application/json") else: - #try: - resp = self.render_template("panoramix/viz.html", viz=obj) - ''' + try: + resp = self.render_template("panoramix/viz.html", viz=obj) except Exception as e: return Response( str(e), status=500, mimetype="application/json") - ''' return resp @has_access @@ -279,36 +313,6 @@ class Panoramix(BaseView): @has_access @expose("/save/") - def save(self): - session = db.session() - d = request.args.to_dict(flat=False) - as_list = ('metrics', 'groupby') - for m in as_list: - v = d.get(m) - if v and not isinstance(d[m], list): - d[m] = [d[m]] - - table_id = druid_datasource_id = None - datasource_type = request.args.get('datasource_type') - if datasource_type == 'druid': - druid_datasource_id = request.args.get('datasource_id') - else: - table_id = request.args.get('datasource_id') - - obj = models.Slice( - params=json.dumps(d, indent=4), - viz_type=request.args.get('viz_type'), - datasource_name=request.args.get('datasource_name'), - druid_datasource_id=druid_datasource_id, - table_id=table_id, - datasource_type=datasource_type, - slice_name=request.args.get('slice_name', 'junk'), - ) - session.add(obj) - session.commit() - session.close() - - return "super!" @has_access @expose("/dashboard//") diff --git a/panoramix/viz.py b/panoramix/viz.py index 59c647572..240e65c71 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -49,6 +49,8 @@ class BaseViz(object): def get_url(self, **kwargs): d = self.args.copy() + if 'action' in d: + del d['action'] d.update(kwargs) href = Href( '/panoramix/datasource/{self.datasource.type}/' @@ -335,7 +337,9 @@ class DistributionPieViz(HighchartsViz): verbose_name = "Distribution - Pie Chart" chart_type = "pie" js_files = ['highstock.js'] - form_fields = BaseViz.form_fields + ['limit'] + form_fields = [ + 'viz_type', 'metrics', 'groupby', + ('since', 'until'), 'limit'] def query_obj(self): d = super(DistributionPieViz, self).query_obj()