iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Allowing to define multiple time columns for time series on Sqla

This commit is contained in:
Maxime Beauchemin 2015-10-08 10:09:35 -07:00 committed by Maxime
parent b84ede1aef
commit cb92cac776
6 changed files with 83 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
panoramix/bin/panoramix
View File

@ -116,12 +116,13 @@ def load_examples(sample):
if not obj:
obj = TBL(table_name = 'birth_names')
obj.main_dttm_col = 'ds'
obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=one+day&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
obj.default_endpoint = "/panoramix/datasource/table/1/?viz_type=table&granularity=ds&since=100+years&until=now&row_limit=10&where=&flt_col_0=ds&flt_op_0=in&flt_eq_0=&flt_col_1=ds&flt_op_1=in&flt_eq_1=&slice_name=TEST&datasource_name=birth_names&datasource_id=1&datasource_type=table"
obj.database = dbobj
obj.columns = [
models.TableColumn(column_name="num", sum=True, type="INTEGER"),
models.TableColumn(column_name="sum_boys", sum=True, type="INTEGER"),
models.TableColumn(column_name="sum_girls", sum=True, type="INTEGER"),
models.TableColumn(column_name="ds", is_dttm=True, type="DATETIME"),
]
models.Table
session.add(obj)
@ -141,7 +142,7 @@ def load_examples(sample):
"flt_col_1": "gender",
"flt_eq_1": "",
"flt_op_1": "in",
"granularity": "all",
"granularity": "ds",
"groupby": [],
"metric": 'sum__num',
"metrics": ["sum__num"],
@ -194,7 +195,7 @@ def load_examples(sample):
datasource_type='table',
table=tbl,
params=get_slice_json(
slice_name, viz_type="big_number", granularity="1 day",
slice_name, viz_type="big_number", granularity="ds",
compare_lag="5", compare_suffix="over 5Y"))
session.add(slc)
slices.append(slc)
@ -237,7 +238,7 @@ def load_examples(sample):
table=tbl,
params=get_slice_json(
slice_name, viz_type="line", groupby=['name'],
granularity='1 day', rich_tooltip='y', show_legend='y'))
granularity='ds', rich_tooltip='y', show_legend='y'))
session.add(slc)
slices.append(slc)

16
panoramix/forms.py
View File

@ -67,6 +67,13 @@ class FormFactory(object):
"The time granularity for the visualization. Note that you "
"can type and use simple natural language as in '10 seconds', "
"'1 day' or '56 weeks'")),
'granularity_sqla': SelectField(
'Time Column', default=datasource.main_dttm_col,
choices=self.choicify(datasource.dttm_cols),
description=(
"The time granularity for the visualization. Note that you "
"can define arbitrary expression that return a DATETIME "
"column in the table editor")),
'since': TextField(
'Since', default="7 days ago", description=(
"Timestamp from filter. This supports free form typing and "
@ -217,7 +224,7 @@ class FormFactory(object):
standalone = HiddenField()
async = HiddenField()
json = HiddenField()
previous_viz_type = HiddenField()
previous_viz_type = HiddenField(default=viz.viz_type)
filter_cols = datasource.filterable_column_names or ['']
for i in range(10):
@ -243,4 +250,11 @@ class FormFactory(object):
if datasource.__class__.__name__ == 'SqlaTable':
QueryForm.field_order += ['where']
setattr(QueryForm, 'where', px_form_fields['where'])
if 'granularity' in viz.form_fields:
setattr(
QueryForm,
'granularity', px_form_fields['granularity_sqla'])
field_css_classes['granularity'] = ['form-control', 'select2']
return QueryForm

33
panoramix/models.py
View File

@ -164,6 +164,10 @@ class Queryable(object):
def column_names(self):
return sorted([c.column_name for c in self.columns])
@property
def main_dttm_col(self):
return "timestamp"
@property
def groupby_column_names(self):
return sorted([c.column_name for c in self.columns if c.groupby])
@ -172,6 +176,10 @@ class Queryable(object):
def filterable_column_names(self):
return sorted([c.column_name for c in self.columns if c.filterable])
@property
def dttm_cols(self):
return []
class Database(Model, AuditMixinNullable):
__tablename__ = 'dbs'
@ -216,6 +224,13 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
"[{self.database}].[{self.table_name}]"
"(id:{self.id})").format(self=self)
@property
def dttm_cols(self):
l = [c.column_name for c in self.columns if c.is_dttm]
if self.main_dttm_col not in l:
l.append(self.main_dttm_col)
return l
@property
def name(self):
return self.table_name
@ -339,13 +354,20 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
inner_from_dttm=None, inner_to_dttm=None,
extras=None):
# For backward compatibility
if granularity not in self.dttm_cols:
granularity = self.main_dttm_col
cols = {col.column_name: col for col in self.columns}
qry_start_dttm = datetime.now()
if not self.main_dttm_col:
raise Exception(
"Datetime column not provided as part table configuration")
timestamp = literal_column(
self.main_dttm_col).label('timestamp')
dttm_expr = cols[granularity].expression
if dttm_expr:
timestamp = ColumnClause(dttm_expr, is_literal=True).label('timestamp')
else:
timestamp = literal_column(granularity).label('timestamp')
metrics_exprs = [
literal_column(m.expression).label(m.metric_name)
for m in self.metrics if m.metric_name in metrics]
@ -381,7 +403,7 @@ class SqlaTable(Model, Queryable, AuditMixinNullable):
inner_groupby_exprs.append(inner)
inner_select_exprs.append(inner)
if granularity != "all":
if is_timeseries:
select_exprs += [timestamp]
groupby_exprs += [timestamp]
@ -562,7 +584,7 @@ class TableColumn(Model, AuditMixinNullable):
table = relationship(
'SqlaTable', backref='columns', foreign_keys=[table_id])
column_name = Column(String(256))
is_dttm = Column(Boolean, default=True)
is_dttm = Column(Boolean, default=False)
is_active = Column(Boolean, default=True)
type = Column(String(32), default='')
groupby = Column(Boolean, default=False)
@ -740,6 +762,9 @@ class Datasource(Model, AuditMixin, Queryable):
m.metric_name: m.json_obj
for m in self.metrics if m.metric_name in metrics
}
if granularity != "all":
granularity = utils.parse_human_timedelta(
granularity).total_seconds() * 1000
if not isinstance(granularity, basestring):
granularity = {"type": "duration", "duration": granularity}

2
panoramix/templates/panoramix/datasource.html
View File

@ -88,7 +88,7 @@
<input type="hidden" name="datasource_name" value="{{ datasource.name }}">
<input type="hidden" name="datasource_id" value="{{ datasource.id }}">
<input type="hidden" name="datasource_type" value="{{ datasource.type }}">
{{ form.previous_viz_type() }}
<input type="hidden" name="previous_viz_type" value="{{ viz.viz_type or "table" }}">
</form><br>
</div>

10
panoramix/views.py
View File

@ -42,12 +42,18 @@ class TableColumnInlineView(CompactCRUDMixin, PanoramixModelView):
can_delete = False
edit_columns = [
'column_name', 'description', 'groupby', 'filterable', 'table',
'count_distinct', 'sum', 'min', 'max', 'expression']
'count_distinct', 'sum', 'min', 'max', 'expression', 'is_dttm']
add_columns = edit_columns
list_columns = [
'column_name', 'type', 'groupby', 'filterable', 'count_distinct',
'sum', 'min', 'max']
'sum', 'min', 'max', 'is_dttm']
page_size = 100
description_columns = {
'is_dttm': (
"Whether to make this column available as a "
"[Time Granularity] option, column has to be DATETIME or "
"DATETIME-like"),
}
appbuilder.add_view_no_menu(TableColumnInlineView)

43
panoramix/viz.py
View File

@ -22,9 +22,13 @@ class BaseViz(object):
viz_type = None
verbose_name = "Base Viz"
template = None
is_timeseries = False
form_fields = [
'viz_type', 'metrics', 'groupby', 'granularity',
('since', 'until')]
'viz_type',
'granularity',
('since', 'until'),
'metrics', 'groupby',
]
js_files = []
css_files = []
@ -37,22 +41,21 @@ class BaseViz(object):
ff = FormFactory(self)
form_class = ff.get_form()
defaults = form_class().data.copy()
previous_viz_type = form_data.get('previous_viz_type')
if isinstance(form_data, ImmutableMultiDict):
form = form_class(form_data)
else:
form = form_class(**form_data)
data = form.data.copy()
if not form.validate():
for k, v in form.errors.items():
if not data.get('json') and not data.get('async'):
flash("{}: {}".format(k, " ".join(v)), 'danger')
previous_viz_type = form_data.get('previous_viz_type')
if previous_viz_type in viz_types and previous_viz_type != self.viz_type:
data = {
k: form.data[k]
for k in form_data.keys()
if k in viz_types[previous_viz_type].flat_form_fields() and k in form.data}
data = {
k: form.data[k]
for k in form_data.keys()
if k in form.data}
defaults.update(data)
self.form_data = defaults
@ -134,10 +137,7 @@ class BaseViz(object):
form_data = self.form_data
groupby = form_data.get("groupby") or []
metrics = form_data.get("metrics") or ['count']
granularity = form_data.get("granularity", "1 day")
if granularity != "all":
granularity = utils.parse_human_timedelta(
granularity).total_seconds() * 1000
granularity = form_data.get("granularity")
limit = int(form_data.get("limit", 0))
row_limit = int(
form_data.get("row_limit", config.get("ROW_LIMIT")))
@ -160,7 +160,7 @@ class BaseViz(object):
'granularity': granularity,
'from_dttm': from_dttm,
'to_dttm': to_dttm,
'is_timeseries': True,
'is_timeseries': self.is_timeseries,
'groupby': groupby,
'metrics': metrics,
'row_limit': row_limit,
@ -194,6 +194,7 @@ class TableViz(BaseViz):
template = 'panoramix/viz_table.html'
form_fields = BaseViz.form_fields + ['row_limit']
css_files = ['lib/dataTables/dataTables.bootstrap.css']
is_timeseries = False
js_files = [
'lib/dataTables/jquery.dataTables.min.js',
'lib/dataTables/dataTables.bootstrap.js']
@ -220,6 +221,7 @@ class MarkupViz(BaseViz):
verbose_name = "Markup Widget"
template = 'panoramix/viz_markup.html'
form_fields = ['viz_type', 'markup_type', 'code']
is_timeseries = False
def rendered(self):
markup_type = self.form_data.get("markup_type")
@ -238,6 +240,7 @@ class WordCloudViz(BaseViz):
viz_type = "word_cloud"
verbose_name = "Word Cloud"
template = 'panoramix/viz_word_cloud.html'
is_timeseries = False
form_fields = [
'viz_type',
('since', 'until'),
@ -253,7 +256,6 @@ class WordCloudViz(BaseViz):
def query_obj(self):
d = super(WordCloudViz, self).query_obj()
d['granularity'] = 'all'
metric = self.form_data.get('metric')
if not metric:
raise Exception("Pick a metric!")
@ -271,6 +273,7 @@ class NVD3Viz(BaseViz):
viz_type = None
verbose_name = "Base NVD3 Viz"
template = 'panoramix/viz_nvd3.html'
is_timeseries = False
js_files = [
'lib/d3.min.js',
'lib/nvd3/nv.d3.min.js',
@ -285,6 +288,7 @@ class NVD3Viz(BaseViz):
class BubbleViz(NVD3Viz):
viz_type = "bubble"
verbose_name = "Bubble Chart"
is_timeseries = False
form_fields = [
'viz_type',
('since', 'until'),
@ -298,7 +302,6 @@ class BubbleViz(NVD3Viz):
def query_obj(self):
form_data = self.form_data
d = super(BubbleViz, self).query_obj()
d['granularity'] = 'all'
d['groupby'] = list({
form_data.get('series'),
form_data.get('entity')
@ -349,6 +352,7 @@ class BigNumberViz(BaseViz):
viz_type = "big_number"
verbose_name = "Big Number"
template = 'panoramix/viz_bignumber.html'
is_timeseries = True
js_files = [
'lib/d3.min.js',
'widgets/viz_bignumber.js',
@ -400,6 +404,7 @@ class NVD3TimeSeriesViz(NVD3Viz):
viz_type = "line"
verbose_name = "Time Series - Line Chart"
sort_series = False
is_timeseries = True
form_fields = [
'viz_type',
'granularity', ('since', 'until'),
@ -553,16 +558,17 @@ class NVD3TimeSeriesStackedViz(NVD3TimeSeriesViz):
class DistributionPieViz(NVD3Viz):
viz_type = "pie"
verbose_name = "Distribution - NVD3 - Pie Chart"
is_timeseries = False
form_fields = [
'viz_type', 'metrics', 'groupby',
'viz_type',
('since', 'until'),
'metrics', 'groupby',
'limit',
('donut', 'show_legend'),
]
def query_obj(self):
d = super(DistributionPieViz, self).query_obj()
d['granularity'] = "all"
d['is_timeseries'] = False
return d
@ -589,6 +595,7 @@ class DistributionPieViz(NVD3Viz):
class DistributionBarViz(DistributionPieViz):
viz_type = "dist_bar"
verbose_name = "Distribution - Bar Chart"
is_timeseries = False
form_fields = [
'viz_type', 'metrics', 'groupby',
('since', 'until'),