diff --git a/panoramix/bin/panoramix b/panoramix/bin/panoramix index 943e10b91..1a9dfb1ff 100755 --- a/panoramix/bin/panoramix +++ b/panoramix/bin/panoramix @@ -61,6 +61,8 @@ def load_examples(sample): Column("num", Integer), Column("ds", String(20)), Column("gender", String(10)), + Column("sum_boys", Integer), + Column("sum_girls", Integer), ) try: BirthNames.drop(db.engine) @@ -82,7 +84,10 @@ def load_examples(sample): state=state, year=year, ds=ds, - name=name, num=num, gender=gender) + name=name, num=num, gender=gender, + sum_boys=num if gender == 'boy' else 0, + sum_girls=num if gender == 'girl' else 0, + ) if i % 5000 == 0: print("{} loaded out of 82527 rows".format(i)) session.commit() @@ -109,8 +114,11 @@ def load_examples(sample): obj.main_dttm_col = 'ds' obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table" obj.database = dbobj - obj.columns = [models.TableColumn( - column_name="num", sum=True, type="INTEGER")] + obj.columns = [ + models.TableColumn(column_name="num", sum=True, type="INTEGER"), + models.TableColumn(column_name="sum_boys", sum=True, type="INTEGER"), + models.TableColumn(column_name="sum_girls", sum=True, type="INTEGER"), + ] models.Table session.add(obj) session.commit() @@ -200,7 +208,7 @@ def load_examples(sample): session.add(slc) slices.append(slc) - slice_name = "States" + slice_name = "Gender by State" slc = session.query(Slice).filter_by(slice_name=slice_name).first() if not slc: slc = Slice( @@ -210,6 +218,7 @@ def load_examples(sample): table=tbl, params=get_slice_json( slice_name, flt_eq_1="other", viz_type="dist_bar", + metrics=['sum__sum_girls', 'sum__sum_boys'], groupby=['state'], flt_op_1='not in', flt_col_1='state')) session.add(slc) slices.append(slc) diff --git a/panoramix/forms.py b/panoramix/forms.py index 49c615324..e347b80fe 100644 --- a/panoramix/forms.py +++ b/panoramix/forms.py @@ -1,7 +1,9 @@ from wtforms import ( Field, Form, SelectMultipleField, SelectField, TextField, TextAreaField, - BooleanField, IntegerField) + BooleanField, IntegerField, HiddenField) from copy import copy +from panoramix import app +config = app.config class OmgWtForm(Form): @@ -27,157 +29,183 @@ class OmgWtForm(Form): return "" -def form_factory(viz): - datasource = viz.datasource - from panoramix.viz import viz_types - row_limits = [10, 50, 100, 500, 1000, 5000, 10000] +class FormFactory(object): + row_limits = [10, 50, 100, 500, 1000, 5000, 10000, 50000] series_limits = [0, 5, 10, 25, 50, 100, 500] - group_by_choices = [(s, s) for s in datasource.groupby_column_names] - # Pool of all the fields that can be used in Panoramix - px_form_fields = { - 'viz_type': SelectField( - 'Viz', - choices=[(k, v.verbose_name) for k, v in viz_types.items()], - description="The type of visualization to display"), - 'metrics': SelectMultipleField( - 'Metrics', choices=datasource.metrics_combo, - description="One or many metrics to display"), - 'metric': SelectField( - 'Metric', choices=datasource.metrics_combo, - description="One or many metrics to display"), - 'groupby': SelectMultipleField( - 'Group by', - choices=[(s, s) for s in datasource.groupby_column_names], - description="One or many fields to group by"), - 'granularity': TextField( - 'Time Granularity', default="one day", - description=( - "The time granularity for the visualization. Note that you " - "can type and use simple natural language as in '10 seconds', " - "'1 day' or '56 weeks'")), - 'since': TextField( - 'Since', default="one day ago", description=( - "Timestamp from filter. This supports free form typing and " - "natural language as in '1 day ago', '28 days' or '3 years'")), - 'until': TextField('Until', default="now"), - 'row_limit': - SelectField( - 'Row limit', choices=[(s, s) for s in row_limits]), - 'limit': - SelectField( - 'Series limit', choices=[(s, s) for s in series_limits], + + def __init__(self, viz): + self.viz = viz + from panoramix.viz import viz_types + viz = self.viz + datasource = viz.datasource + group_by_choices = [(s, s) for s in datasource.groupby_column_names] + # Pool of all the fields that can be used in Panoramix + self.field_dict = { + 'viz_type': SelectField( + 'Viz', + default='table', + choices=[(k, v.verbose_name) for k, v in viz_types.items()], + description="The type of visualization to display"), + 'metrics': SelectMultipleField( + 'Metrics', choices=datasource.metrics_combo, + default=[datasource.metrics_combo[0][0]], + description="One or many metrics to display"), + 'metric': SelectField( + 'Metric', choices=datasource.metrics_combo, + description="One or many metrics to display"), + 'groupby': SelectMultipleField( + 'Group by', + choices=[(s, s) for s in datasource.groupby_column_names], + description="One or many fields to group by"), + 'granularity': TextField( + 'Time Granularity', default="one day", description=( - "Limits the number of time series that get displayed")), - 'rolling_type': SelectField( - 'Rolling', - choices=[(s, s) for s in ['mean', 'sum', 'std']], - description=( - "Defines a rolling window function to apply")), - 'rolling_periods': TextField('Periods', description=( - "Defines the size of the rolling window function, " - "relative to the 'granularity' field")), - 'series': SelectField( - 'Series', choices=group_by_choices, - description=( - "Defines the grouping of entities. " - "Each serie is shown as a specific color on the chart and " - "has a legend toggle")), - 'entity': SelectField('Entity', choices=group_by_choices, - description="This define the element to be plotted on the chart"), - 'x': SelectField( - 'X Axis', choices=datasource.metrics_combo, - description="Metric assigned to the [X] axis"), - 'y': SelectField('Y Axis', choices=datasource.metrics_combo, - description="Metric assigned to the [Y] axis"), - 'size': SelectField('Bubble Size', choices=datasource.metrics_combo), - 'where': TextField('Custom WHERE clause'), - 'compare_lag': TextField('Comparison Period Lag', - description="Based on granularity, number of time periods to compare against"), - 'compare_suffix': TextField('Comparison suffix', - description="Suffix to apply after the percentage display"), - 'markup_type': SelectField( - "Markup Type", - choices=[(s, s) for s in ['markdown', 'html']], - default="markdown", - description="Pick your favorite markup language"), - 'rotation': SelectField( - "Rotation", - choices=[(s, s) for s in ['random', 'flat', 'square']], - default="random", - description="Rotation to apply to words in the cloud"), - 'code': TextAreaField("Code", description="Put your code here"), - 'size_from': TextField( - "Font Size From", - default="20", - description="Font size for the smallest value in the list"), - 'size_to': TextField( - "Font Size To", - default="150", - description="Font size for the biggest value in the list"), - 'show_brush': BooleanField( - "Range Selector", default=True, - description="Whether to display the time range interactive selector"), - 'show_legend': BooleanField( - "Legend", default=True, false_values=["f"], - description="Whether to display the legend (toggles)"), - 'rich_tooltip': BooleanField( - "Rich Tooltip", default=True, - description="The rich tooltip shows a list of all series for that point in time"), - 'y_axis_zero': BooleanField( - "Y Axis Zero", default=False, - description="Force the Y axis to start at 0 instead of the minimum value"), - 'y_log_scale': BooleanField( - "Y Log", default=False, - description="Use a log scale for the Y axis"), - 'x_log_scale': BooleanField( - "X Log", default=False, - description="Use a log scale for the X axis"), - 'donut': BooleanField( - "Donut", default=False, - description="Do you want a donut or a pie?"), - 'contribution': BooleanField( - "Contribution", default=False, - description="Compute the contribution to the total"), - 'num_period_compare': IntegerField( - "Period Ratio", default=None, - description=( - "Number of period to compare against, " - "this is relative to the granularity selected")), - } - field_css_classes = {k: ['form-control'] for k in px_form_fields.keys()} - select2 = [ - 'viz_type', 'metrics', 'groupby', - 'row_limit', 'rolling_type', 'series', - 'entity', 'x', 'y', 'size', 'rotation', 'metric', 'limit', - 'markup_type',] - field_css_classes['since'] += ['select2_free_since'] - field_css_classes['until'] += ['select2_free_until'] - field_css_classes['granularity'] += ['select2_free_granularity'] - for field in ('show_brush', 'show_legend', 'rich_tooltip'): - field_css_classes[field] += ['input-sm'] - for field in select2: - field_css_classes[field] += ['select2'] + "The time granularity for the visualization. Note that you " + "can type and use simple natural language as in '10 seconds', " + "'1 day' or '56 weeks'")), + 'since': TextField( + 'Since', default="one day ago", description=( + "Timestamp from filter. This supports free form typing and " + "natural language as in '1 day ago', '28 days' or '3 years'")), + 'until': TextField('Until', default="now"), + 'row_limit': + SelectField( + 'Row limit', + default=config.get("ROW_LIMIT"), + choices=[(s, s) for s in self.row_limits]), + 'limit': + SelectField( + 'Series limit', choices=[(s, s) for s in self.series_limits], + default=50, + description=( + "Limits the number of time series that get displayed")), + 'rolling_type': SelectField( + 'Rolling', + choices=[(s, s) for s in ['mean', 'sum', 'std']], + description=( + "Defines a rolling window function to apply")), + 'rolling_periods': TextField('Periods', description=( + "Defines the size of the rolling window function, " + "relative to the 'granularity' field")), + 'series': SelectField( + 'Series', choices=group_by_choices, + description=( + "Defines the grouping of entities. " + "Each serie is shown as a specific color on the chart and " + "has a legend toggle")), + 'entity': SelectField('Entity', choices=group_by_choices, + description="This define the element to be plotted on the chart"), + 'x': SelectField( + 'X Axis', choices=datasource.metrics_combo, + description="Metric assigned to the [X] axis"), + 'y': SelectField('Y Axis', choices=datasource.metrics_combo, + description="Metric assigned to the [Y] axis"), + 'size': SelectField('Bubble Size', choices=datasource.metrics_combo), + 'where': TextField('Custom WHERE clause', default=''), + 'compare_lag': TextField('Comparison Period Lag', + description="Based on granularity, number of time periods to compare against"), + 'compare_suffix': TextField('Comparison suffix', + description="Suffix to apply after the percentage display"), + 'markup_type': SelectField( + "Markup Type", + choices=[(s, s) for s in ['markdown', 'html']], + default="markdown", + description="Pick your favorite markup language"), + 'rotation': SelectField( + "Rotation", + choices=[(s, s) for s in ['random', 'flat', 'square']], + default="random", + description="Rotation to apply to words in the cloud"), + 'code': TextAreaField("Code", description="Put your code here"), + 'size_from': TextField( + "Font Size From", + default="20", + description="Font size for the smallest value in the list"), + 'size_to': TextField( + "Font Size To", + default="150", + description="Font size for the biggest value in the list"), + 'show_brush': BooleanField( + "Range Selector", default=True, + description="Whether to display the time range interactive selector"), + 'show_legend': BooleanField( + "Legend", default=True, + description="Whether to display the legend (toggles)"), + 'rich_tooltip': BooleanField( + "Rich Tooltip", default=True, + description="The rich tooltip shows a list of all series for that point in time"), + 'y_axis_zero': BooleanField( + "Y Axis Zero", default=False, + description="Force the Y axis to start at 0 instead of the minimum value"), + 'y_log_scale': BooleanField( + "Y Log", default=False, + description="Use a log scale for the Y axis"), + 'x_log_scale': BooleanField( + "X Log", default=False, + description="Use a log scale for the X axis"), + 'donut': BooleanField( + "Donut", default=False, + description="Do you want a donut or a pie?"), + 'contribution': BooleanField( + "Contribution", default=False, + description="Compute the contribution to the total"), + 'num_period_compare': IntegerField( + "Period Ratio", default=None, + description=( + "Number of period to compare against, " + "this is relative to the granularity selected")), + } - class QueryForm(OmgWtForm): - field_order = copy(viz.form_fields) - css_classes = field_css_classes + def get_form(self, previous=False): + px_form_fields = self.field_dict + viz = self.viz + datasource = viz.datasource + field_css_classes = {k: ['form-control'] for k in px_form_fields.keys()} + select2 = [ + 'viz_type', 'metrics', 'groupby', + 'row_limit', 'rolling_type', 'series', + 'entity', 'x', 'y', 'size', 'rotation', 'metric', 'limit', + 'markup_type',] + field_css_classes['since'] += ['select2_free_since'] + field_css_classes['until'] += ['select2_free_until'] + field_css_classes['granularity'] += ['select2_free_granularity'] + for field in ('show_brush', 'show_legend', 'rich_tooltip'): + field_css_classes[field] += ['input-sm'] + for field in select2: + field_css_classes[field] += ['select2'] - for i in range(10): - setattr(QueryForm, 'flt_col_' + str(i), SelectField( - 'Filter 1', - choices=[(s, s) for s in datasource.filterable_column_names])) - setattr(QueryForm, 'flt_op_' + str(i), SelectField( - 'Filter 1', choices=[(m, m) for m in ['in', 'not in']])) - setattr(QueryForm, 'flt_eq_' + str(i), TextField("Super")) - for ff in viz.form_fields: - if isinstance(ff, basestring): - ff = [ff] - for s in ff: - if s: - setattr(QueryForm, s, px_form_fields[s]) - # datasource type specific form elements - if datasource.__class__.__name__ == 'Table': - QueryForm.field_order += ['where'] - setattr(QueryForm, 'where', px_form_fields['where']) - return QueryForm + class QueryForm(OmgWtForm): + field_order = copy(viz.form_fields) + css_classes = field_css_classes + standalone = HiddenField() + async = HiddenField() + json = HiddenField() + previous_viz_type = HiddenField() + standalone = HiddenField() + + + for i in range(10): + setattr(QueryForm, 'flt_col_' + str(i), SelectField( + 'Filter 1', + default='', + choices=[(s, s) for s in datasource.filterable_column_names])) + setattr(QueryForm, 'flt_op_' + str(i), SelectField( + 'Filter 1', + default='', + choices=[(m, m) for m in ['in', 'not in']])) + setattr( + QueryForm, 'flt_eq_' + str(i), + TextField("Super", default='')) + for ff in viz.form_fields: + if isinstance(ff, basestring): + ff = [ff] + for s in ff: + if s: + setattr(QueryForm, s, px_form_fields[s]) + + # datasource type specific form elements + if datasource.__class__.__name__ == 'Table': + QueryForm.field_order += ['where'] + setattr(QueryForm, 'where', px_form_fields['where']) + return QueryForm diff --git a/panoramix/templates/panoramix/datasource.html b/panoramix/templates/panoramix/datasource.html index c88eb9d0b..308b71db1 100644 --- a/panoramix/templates/panoramix/datasource.html +++ b/panoramix/templates/panoramix/datasource.html @@ -123,6 +123,7 @@ + {{ form.previous_viz_type() }}
@@ -132,7 +133,7 @@ {{ "{0:0.4f}".format(results.duration.total_seconds()) }} s - query {% endif %} diff --git a/panoramix/templates/panoramix/viz.html b/panoramix/templates/panoramix/viz.html index 1388301d6..e6872894d 100644 --- a/panoramix/templates/panoramix/viz.html +++ b/panoramix/templates/panoramix/viz.html @@ -3,7 +3,7 @@ {% if viz.form_data.get("json") == "true" %} {{ viz.get_json() }} {% else %} - {% if viz.form_data.get("standalone") == "true" %} + {% if viz.request.args.get("standalone") == "true" %} {% extends 'panoramix/viz_standalone.html' %} {% else %} {% extends 'panoramix/datasource.html' %} @@ -16,7 +16,7 @@ {% block head_css %} {{super()}} - {% if viz.form_data.get("skip_libs") != "true" %} + {% if viz.request.args.get("skip_libs") != "true" %} {% for css in viz.css_files %} {% endfor %} @@ -27,11 +27,11 @@ {% block tail %} {{super()}} - {% if viz.form_data.get("skip_libs") != "true" %} + {% if viz.request.args.get("skip_libs") != "true" %} {% for js in viz.js_files %} {% endfor %} + {{ viz_macros.viz_js(viz) }} {% endif %} - {{ viz_macros.viz_js(viz) }} {% endblock %} {% endif %} diff --git a/panoramix/templates/panoramix/viz_nvd3.html b/panoramix/templates/panoramix/viz_nvd3.html index 17404d239..b1b072158 100644 --- a/panoramix/templates/panoramix/viz_nvd3.html +++ b/panoramix/templates/panoramix/viz_nvd3.html @@ -37,7 +37,7 @@ nv.addGraph(function() { // chart_type is {{ viz.chart_type }} {% if viz.chart_type == 'line' %} - {% if viz.form_data.show_brush == 'y' %} + {% if viz.form_data.show_brush %} var chart = nv.models.lineWithFocusChart() var xext = chart.xAxis.scale().domain(); chart @@ -52,10 +52,10 @@ chart.xAxis .showMaxMin(false) .tickFormat(function (d) {return tickMultiFormat(new Date(d)); }); - chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }}); + chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }}); chart.yAxis.tickFormat(d3.format('.3s')); - {% if viz.form_data.contribution=='y' or viz.form_data.get("num_period_compare") %} + {% if viz.form_data.contribution or viz.form_data.get("num_period_compare") %} chart.yAxis.tickFormat(d3.format('.3p')); chart.y2Axis.tickFormat(d3.format('.3p')); {% endif %} @@ -71,8 +71,8 @@ {% elif viz.chart_type == 'pie' %} var chart = nv.models.pieChart() - chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }}); - {% if viz.form_data.donut=='y' %} + chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }}); + {% if viz.form_data.donut %} chart.donut(true); chart.donutLabelsOutside(true); {% endif %} @@ -91,14 +91,14 @@ chart.xAxis .showMaxMin(false) .tickFormat(function (d) {return tickMultiFormat(new Date(d)); }); - chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }}); + chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }}); chart.yAxis.tickFormat(d3.format('.3p')); {% elif viz.chart_type == 'bubble' %} var chart = nv.models.scatterChart(); chart.xAxis.tickFormat(d3.format('.3s')); chart.yAxis.tickFormat(d3.format('.3s')); - chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }}); + chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }}); chart.pointRange([5, 5000]); {% elif viz.chart_type == 'stacked' %} @@ -107,21 +107,21 @@ chart.xAxis .showMaxMin(false) .tickFormat(function (d) {return tickMultiFormat(new Date(d)); }); - chart.showLegend({{ "{}".format(viz.form_data.show_legend=='y')|lower }}); + chart.showLegend({{ "{}".format(viz.form_data.show_legend)|lower }}); chart.yAxis.tickFormat(d3.format('.3s')); {% endif %} - {% if viz.chart_type in ("line", "stacked") and viz.form_data.rich_tooltip == 'y' %} + {% if viz.chart_type in ("line", "stacked") and viz.form_data.rich_tooltip %} chart.useInteractiveGuideline(true); {% endif %} - {% if viz.form_data.y_axis_zero == 'y' %} + {% if viz.form_data.y_axis_zero %} chart.forceY([0, 1]); - {% elif viz.form_data.y_log_scale == 'y' %} + {% elif viz.form_data.y_log_scale %} chart.yScale(d3.scale.log()); {% endif %} - {% if viz.form_data.x_log_scale == 'y' %} + {% if viz.form_data.x_log_scale %} chart.xScale(d3.scale.log()); {% endif %} diff --git a/panoramix/templates/panoramix/viz_standalone.html b/panoramix/templates/panoramix/viz_standalone.html index a79776e42..51edffff9 100644 --- a/panoramix/templates/panoramix/viz_standalone.html +++ b/panoramix/templates/panoramix/viz_standalone.html @@ -1,6 +1,6 @@ - {% if viz.form_data.get("skip_libs") != "true" %} + {% if viz.request.args.get("skip_libs") != "true" %} {% block head %} {% endblock %} diff --git a/panoramix/templates/panoramix/viz_table.html b/panoramix/templates/panoramix/viz_table.html index 90c45870d..2674019cc 100644 --- a/panoramix/templates/panoramix/viz_table.html +++ b/panoramix/templates/panoramix/viz_table.html @@ -1,5 +1,5 @@ {% macro viz_html(viz) %} - {% if viz.form_data.get("async") == "true" %} + {% if viz.request.args.get("async") == "true" %} {% set df = viz.get_df() %} diff --git a/panoramix/viz.py b/panoramix/viz.py index 9da21c809..99b664e5b 100644 --- a/panoramix/viz.py +++ b/panoramix/viz.py @@ -3,23 +3,19 @@ from datetime import datetime import json import uuid -from flask import flash +from flask import flash, request from markdown import markdown from pandas.io.json import dumps -from werkzeug.datastructures import MultiDict +from werkzeug.datastructures import ImmutableMultiDict from werkzeug.urls import Href import numpy as np import pandas as pd from panoramix import app, utils -from panoramix.forms import form_factory +from panoramix.forms import FormFactory config = app.config -CHART_ARGS = { - 'title': None, -} - class BaseViz(object): verbose_name = "Base Viz" @@ -32,12 +28,26 @@ class BaseViz(object): def __init__(self, datasource, form_data): self.datasource = datasource - form = self.form_class(form_data) - form.validate() - raise - self.form_data = form_data - if isinstance(form_data, MultiDict): - self.form_data = form_data.to_dict(flat=False) + self.request = request + + ff = FormFactory(self) + form_class = ff.get_form() + defaults = form_class().data.copy() + if isinstance(form_data, ImmutableMultiDict): + form = form_class(form_data) + else: + form = form_class(**form_data) + data = form.data.copy() + previous_viz_type = form_data.get('previous_viz_type') + if previous_viz_type in viz_types: + data = { + k: form.data[k] + for k in form_data.keys() + if k in viz_types[previous_viz_type].flat_form_fields() and k in form.data} + defaults.update(data) + self.form_data = defaults + + self.form_data['previous_viz_type'] = form_data.get("viz_type") self.token = self.form_data.get( 'token', 'token_' + uuid.uuid4().hex[:8]) @@ -48,9 +58,19 @@ class BaseViz(object): elif k not in as_list and isinstance(v, list) and v: self.form_data[k] = v[0] - self.metrics = self.form_data.get('metrics') or ['count'] + self.metrics = self.form_data.get('metrics') or [] self.groupby = self.form_data.get('groupby') or [] + @classmethod + def flat_form_fields(cls): + l = [] + for obj in cls.form_fields: + if isinstance(obj, (tuple, list)): + l += [a for a in obj] + else: + l.append(obj) + return l + def get_url(self, **kwargs): d = self.form_data.copy() if 'action' in d: @@ -80,7 +100,7 @@ class BaseViz(object): @property def form_class(self): - return form_factory(self) + return FormFactory(self).get_form() def query_filters(self): form_data = self.form_data @@ -334,11 +354,13 @@ class NVD3TimeSeriesViz(NVD3Viz): form_data = self.form_data df = super(NVD3TimeSeriesViz, self).get_df() df = df.fillna(0) - metrics = self.metrics + if form_data.get("granularity") == "all": + raise Exception("Pick a time granularity for your time series") + df = df.pivot_table( index="timestamp", - columns=self.groupby, - values=metrics,) + columns=self.form_data.get('groupby'), + values=self.form_data.get('metrics')) if self.sort_series: dfs = df.sum() @@ -379,7 +401,7 @@ class NVD3TimeSeriesViz(NVD3Viz): series_title = name else: name = ["{}".format(s) for s in name] - if len(self.metrics) > 1: + if len(self.form_data.get('metrics')) > 1: series_title = ", ".join(name) else: series_title = ", ".join(name[1:])