iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Time shift difference (#5177) 

* Allow different comparisons when using time shift

* Remove test code

* Remove lead/trail NaN and improve code

* Add headers/subheader

* Update yarn

* Migration script

* Fix migration

* Small fixes

* Trigger tests

* Fix lint

* Fix javascript
This commit is contained in:
Beto Dealmeida 2018-07-16 16:30:11 -07:00 committed by Maxime Beauchemin
parent f8a6e09220
commit 7b4e6c7455
12 changed files with 285 additions and 22628 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22567
superset/assets/package-lock.json generated
View File

File diff suppressed because it is too large Load Diff

1
superset/assets/package.json
View File

@ -68,6 +68,7 @@
"geojson-extent": "^0.3.2",
"geolib": "^2.0.24",
"immutable": "^3.8.2",
"is-react": "^1.1.1",
"jed": "^1.1.1",
"jquery": "3.1.1",
"lodash.throttle": "^4.1.1",

4
superset/assets/src/SqlLab/actions.js
View File

@ -435,7 +435,9 @@ export function popDatasourceQuery(datasourceKey, sql) {
};
dispatch(addQueryEditor(queryEditorProps));
},
error: () => notify.error(t("The datasource couldn't be loaded")),
error: () => {
dispatch(addDangerToast(t("The datasource couldn't be loaded")));
},
});
};
}

22
superset/assets/src/explore/components/ControlPanelsContainer.jsx
View File

@ -1,5 +1,6 @@
/* eslint camelcase: 0 */
import React from 'react';
import isReact from 'is-react';
import PropTypes from 'prop-types';
import { bindActionCreators } from 'redux';
import { connect } from 'react-redux';
@ -29,6 +30,10 @@ class ControlPanelsContainer extends React.Component {
this.renderControlPanelSection = this.renderControlPanelSection.bind(this);
}
getControlData(controlName) {
if (isReact.element(controlName)) {
return controlName;
}
const control = this.props.controls[controlName];
// Identifying mapStateToProps function to apply (logic can't be in store)
let mapF = controls[controlName].mapStateToProps;
@ -69,10 +74,13 @@ class ControlPanelsContainer extends React.Component {
<ControlRow
key={`controlsetrow-${i}`}
className="control-row"
controls={controlSets.map(controlName => (
controlName &&
ctrls[controlName] &&
<Control
controls={controlSets.map((controlName) => {
if (!controlName) {
return null;
} else if (isReact.element(controlName)) {
return controlName;
} else if (ctrls[controlName]) {
return (<Control
name={controlName}
key={`control-${controlName}`}
value={this.props.form_data[controlName]}
@ -80,8 +88,10 @@ class ControlPanelsContainer extends React.Component {
actions={this.props.actions}
formData={ctrls[controlName].provideFormDataToProps ? this.props.form_data : null}
{...this.getControlData(controlName)}
/>
))}
/>);
}
return null;
})}
/>
))}
</ControlPanelSection>

47
superset/assets/src/explore/controls.jsx
View File

@ -864,7 +864,7 @@ export const controls = {
resample_rule: {
type: 'SelectControl',
freeForm: true,
label: t('Resample Rule'),
label: t('Rule'),
default: null,
choices: formatSelectOptions(['', '1T', '1H', '1D', '7D', '1M', '1AS']),
description: t('Pandas resample rule'),
@ -873,7 +873,7 @@ export const controls = {
resample_how: {
type: 'SelectControl',
freeForm: true,
label: t('Resample How'),
label: t('How'),
default: null,
choices: formatSelectOptions(['', 'mean', 'sum', 'median']),
description: t('Pandas resample how'),
@ -882,7 +882,7 @@ export const controls = {
resample_fillmethod: {
type: 'SelectControl',
freeForm: true,
label: t('Resample Fill Method'),
label: t('Fill Method'),
default: null,
choices: formatSelectOptions(['', 'ffill', 'bfill']),
description: t('Pandas resample fill method'),
@ -1203,11 +1203,12 @@ export const controls = {
mapStateToProps: (state) => {
const showWarning = (
state.controls &&
state.controls.num_period_compare &&
state.controls.num_period_compare.value !== '');
state.controls.comparison_type &&
state.controls.comparison_type.value === 'percentage');
return {
warning: showWarning ?
t('When `Period Ratio` is set, the Y Axis Format is forced to `.1%`') : null,
t('When `Calculation type` is set to "Percentage change", the Y ' +
'Axis Format is forced to `.1%`') : null,
disabled: showWarning,
};
},
@ -1548,30 +1549,11 @@ export const controls = {
description: t('Compute the contribution to the total'),
},
num_period_compare: {
type: 'TextControl',
label: t('Period Ratio'),
default: '',
isInt: true,
description: t('[integer] Number of period to compare against, ' +
'this is relative to the granularity selected'),
},
period_ratio_type: {
type: 'SelectControl',
label: t('Period Ratio Type'),
default: 'growth',
choices: formatSelectOptions(['factor', 'growth', 'value']),
description: t('`factor` means (new/previous), `growth` is ' +
'((new/previous) - 1), `value` is (new-previous)'),
},
time_compare: {
type: 'SelectControl',
multi: true,
freeForm: true,
label: t('Time Shift'),
default: [],
choices: formatSelectOptions([
'1 day',
'1 week',
@ -1585,6 +1567,21 @@ export const controls = {
'56 weeks, 365 days)'),
},
comparison_type: {
type: 'SelectControl',
label: t('Calculation type'),
default: 'values',
choices: [
['values', 'Actual Values'],
['absolute', 'Absolute difference'],
['percentage', 'Percentage change'],
['ratio', 'Ratio'],
],
description: t('How to display time shifts: as individual lines; as the ' +
'absolute difference between the main time series and each time shift; ' +
'as the percentage change; or as the ratio between series and time shifts.'),
},
subheader: {
type: 'TextControl',
label: t('Subheader'),

21
superset/assets/src/explore/main.css
View File

@ -225,3 +225,24 @@
text-align: center;
margin-top: 60px;
}
h1.section-header {
font-size: 14px;
font-weight: bold;
margin-bottom: 0;
margin-top: 0;
padding-bottom: 5px;
margin-left: -16px;
}
h2.section-header {
font-size: 13px;
font-weight: bold;
margin-bottom: 0;
margin-top: 0;
padding-bottom: 5px;
}
.Select {
margin-bottom: 15px;
}

5
superset/assets/src/explore/store.js
View File

@ -1,4 +1,5 @@
/* eslint camelcase: 0 */
import isReact from 'is-react';
import controls from './controls';
import visTypes, { sectionsToRender } from './visTypes';
@ -50,6 +51,10 @@ export function getControlsState(state, form_data) {
const controlOverrides = viz.controlOverrides || {};
const controlsState = {};
controlNames.forEach((k) => {
if (isReact.element(k)) {
// no state
return;
}
const control = Object.assign({}, controls[k], controlOverrides[k]);
if (control.mapStateToProps) {
Object.assign(control, control.mapStateToProps(state, control));

8
superset/assets/src/explore/visTypes.js → superset/assets/src/explore/visTypes.jsx
View File

@ -2,6 +2,7 @@
* This file defines how controls (defined in controls.js) are structured into sections
* and associated with each and every visualization type.
*/
import React from 'react';
import { D3_TIME_FORMAT_OPTIONS } from './controls';
import * as v from './validators';
import { t } from '../locales';
@ -65,9 +66,12 @@ export const sections = {
'that allow for advanced analytical post processing ' +
'of query results'),
controlSetRows: [
[<h1 className="section-header">Moving Average</h1>],
['rolling_type', 'rolling_periods', 'min_periods'],
['time_compare'],
['num_period_compare', 'period_ratio_type'],
[<h1 className="section-header">Time Comparison</h1>],
['time_compare', 'comparison_type'],
[<h1 className="section-header">Python Functions</h1>],
[<h2 className="section-header">pandas.resample</h2>],
['resample_how', 'resample_rule', 'resample_fillmethod'],
],
},

4
superset/assets/src/visualizations/nvd3_vis.js
View File

@ -402,8 +402,8 @@ export default function nvd3Vis(slice, payload) {
const yAxisFormatter = d3FormatPreset(fd.y_axis_format);
if (chart.yAxis && chart.yAxis.tickFormat) {
if (fd.num_period_compare || fd.contribution) {
// When computing a "Period Ratio" or "Contribution" selected, we force a percentage format
if (fd.contribution || fd.comparison_type === 'percentage') {
// When computing a "Percentage" or "Contribution" selected, we force a percentage format
const percentageFormat = d3.format('.1%');
chart.yAxis.tickFormat(percentageFormat);
} else {

17
superset/assets/yarn.lock
View File

@ -3598,7 +3598,7 @@ fault@^1.0.2:
dependencies:
format "^0.2.2"
fbjs@^0.8.1, fbjs@^0.8.4, fbjs@^0.8.9:
fbjs@^0.8.1, fbjs@^0.8.16, fbjs@^0.8.4, fbjs@^0.8.9:
version "0.8.17"
resolved "https://registry.yarnpkg.com/fbjs/-/fbjs-0.8.17.tgz#c4d598ead6949112653d6588b01a5cdcd9f90fdd"
dependencies:
@ -4945,6 +4945,12 @@ is-property@^1.0.0:
version "1.0.2"
resolved "https://registry.yarnpkg.com/is-property/-/is-property-1.0.2.tgz#57fe1c4e48474edd65b09911f26b1cd4095dda84"
is-react@^1.1.1:
version "1.1.1"
resolved "https://registry.yarnpkg.com/is-react/-/is-react-1.1.1.tgz#304d5541e191190f2389bd588a33da0756bfa0c7"
dependencies:
react "^16.0.0"
is-redirect@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/is-redirect/-/is-redirect-1.0.0.tgz#1d03dded53bd8db0f30c26e4f95d36fc7c87dc24"
@ -8042,6 +8048,15 @@ react@^15.6.2:
object-assign "^4.1.0"
prop-types "^15.5.10"
react@^16.0.0:
version "16.4.1"
resolved "https://registry.yarnpkg.com/react/-/react-16.4.1.tgz#de51ba5764b5dbcd1f9079037b862bd26b82fe32"
dependencies:
fbjs "^0.8.16"
loose-envify "^1.1.0"
object-assign "^4.1.1"
prop-types "^15.6.0"
reactable@1.0.2:
version "1.0.2"
resolved "https://registry.yarnpkg.com/reactable/-/reactable-1.0.2.tgz#67a579fee3af68b991b5f04df921a4a40ece0b72"

157
superset/migrations/versions/3dda56f1c4c6_migrate_num_period_compare_and_period_.py Normal file
View File

@ -0,0 +1,157 @@
"""Migrate num_period_compare and period_ratio_type
Revision ID: 3dda56f1c4c6
Revises: bddc498dd179
Create Date: 2018-07-05 15:19:14.609299
"""
from __future__ import division
# revision identifiers, used by Alembic.
revision = '3dda56f1c4c6'
down_revision = 'bddc498dd179'
import json
from alembic import op
import isodate
import sqlalchemy as sa
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy import Column, Integer, Text
from superset import db
Base = declarative_base()
class Slice(Base):
__tablename__ = 'slices'
id = Column(Integer, primary_key=True)
params = Column(Text)
comparison_type_map = {
'factor': 'ratio',
'growth': 'percentage',
'value': 'absolute',
}
db_engine_specs_map = {
'second': 'PT1S',
'minute': 'PT1M',
'5 minute': 'PT5M',
'10 minute': 'PT10M',
'half hour': 'PT0.5H',
'hour': 'PT1H',
'day': 'P1D',
'week': 'P1W',
'month': 'P1M',
'quarter': 'P0.25Y',
'year': 'P1Y',
}
def isodate_duration_to_string(obj):
if obj.tdelta:
if not obj.months and not obj.years:
return format_seconds(obj.tdelta.total_seconds())
raise Exception('Unable to convert: {0}'.format(obj))
if obj.months % 12 != 0:
months = obj.months + 12 * obj.years
return '{0} months'.format(months)
return '{0} years'.format(obj.years + obj.months // 12)
def timedelta_to_string(obj):
if obj.microseconds:
raise Exception('Unable to convert: {0}'.format(obj))
elif obj.seconds:
return format_seconds(obj.total_seconds())
elif obj.days % 7 == 0:
return '{0} weeks'.format(obj.days // 7)
else:
return '{0} days'.format(obj.days)
def format_seconds(value):
periods = [
('minute', 60),
('hour', 3600),
('day', 86400),
('week', 604800),
]
for period, multiple in periods:
if value % multiple == 0:
value //= multiple
break
else:
period = 'second'
return '{0} {1}{2}'.format(value, period, 's' if value > 1 else '')
def compute_time_compare(granularity, periods):
# convert old db_engine_spec granularity to ISO duration
if granularity in db_engine_specs_map:
granularity = db_engine_specs_map[granularity]
try:
obj = isodate.parse_duration(granularity) * periods
except isodate.isoerror.ISO8601Error:
# if parse_human_timedelta can parse it, return it directly
delta = '{0} {1}{2}'.format(periods, granularity, 's' if periods > 1 else '')
obj = parse_human_timedelta(delta)
if obj:
return delta
raise Exception('Unable to parse: {0}'.format(granularity))
if isinstance(obj, isodate.duration.Duration):
return isodate_duration_to_string(obj)
elif isinstance(obj, datetime.timedelta):
return timedelta_to_string(obj)
def upgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
for chart in session.query(Slice):
params = json.loads(chart.params)
if not params.get('num_period_compare'):
continue
num_period_compare = int(params.get('num_period_compare'))
granularity = params.get('granularity') or params.get('time_grain_sqla')
period_ratio_type = params.get('period_ratio_type', 'growth')
time_compare = compute_time_compare(granularity, num_period_compare)
comparison_type = comparison_type_map[period_ratio_type.lower()]
params['time_compare'] = [time_compare]
params['comparison_type'] = comparison_type
chart.params = json.dumps(params, sort_keys=True)
session.commit()
session.close()
def downgrade():
bind = op.get_bind()
session = db.Session(bind=bind)
for chart in session.query(Slice):
params = json.loads(chart.params)
if 'time_compare' in params or 'comparison_type' in params:
params.pop('time_compare', None)
params.pop('comparison_type', None)
chart.params = json.dumps(params, sort_keys=True)
session.commit()
session.close()

60
superset/viz.py
View File

@ -1181,18 +1181,6 @@ class NVD3TimeSeriesViz(NVD3Viz):
if min_periods:
df = df[min_periods:]
num_period_compare = fd.get('num_period_compare')
if num_period_compare:
num_period_compare = int(num_period_compare)
prt = fd.get('period_ratio_type')
if prt and prt == 'growth':
df = (df / df.shift(num_period_compare)) - 1
elif prt and prt == 'value':
df = df - df.shift(num_period_compare)
else:
df = df / df.shift(num_period_compare)
df = df[num_period_compare:]
return df
def run_extra_queries(self):
@ -1203,8 +1191,6 @@ class NVD3TimeSeriesViz(NVD3Viz):
if not isinstance(time_compare, list):
time_compare = [time_compare]
classes = ['time-shift-{}'.format(i) for i in range(10)]
i = 0
for option in time_compare:
query_object = self.query_obj()
delta = utils.parse_human_timedelta(option)
@ -1219,24 +1205,50 @@ class NVD3TimeSeriesViz(NVD3Viz):
query_object['to_dttm'] -= delta
df2 = self.get_df_payload(query_object).get('df')
if df2 is not None:
classed = classes[i % len(classes)]
i += 1
if df2 is not None and DTTM_ALIAS in df2:
label = '{} offset'. format(option)
df2[DTTM_ALIAS] += delta
df2 = self.process_data(df2)
self._extra_chart_data.extend(self.to_series(
df2, classed=classed, title_suffix=label))
self._extra_chart_data.append((label, df2))
def get_data(self, df):
fd = self.form_data
comparison_type = fd.get('comparison_type') or 'values'
df = self.process_data(df)
chart_data = self.to_series(df)
if self._extra_chart_data:
chart_data += self._extra_chart_data
chart_data = sorted(chart_data, key=lambda x: tuple(x['key']))
if comparison_type == 'values':
chart_data = self.to_series(df)
for i, (label, df2) in enumerate(self._extra_chart_data):
chart_data.extend(
self.to_series(
df2, classed='time-shift-{}'.format(i), title_suffix=label))
else:
chart_data = []
for i, (label, df2) in enumerate(self._extra_chart_data):
# reindex df2 into the df2 index
combined_index = df.index.union(df2.index)
df2 = df2.reindex(combined_index) \
.interpolate(method='time') \
.reindex(df.index)
return chart_data
if comparison_type == 'absolute':
diff = df - df2
elif comparison_type == 'percentage':
diff = (df - df2) / df2
elif comparison_type == 'ratio':
diff = df / df2
else:
raise Exception(
'Invalid `comparison_type`: {0}'.format(comparison_type))
# remove leading/trailing NaNs from the time shift difference
diff = diff[diff.first_valid_index():diff.last_valid_index()]
chart_data.extend(
self.to_series(
diff, classed='time-shift-{}'.format(i), title_suffix=label))
return sorted(chart_data, key=lambda x: tuple(x['key']))
class MultiLineViz(NVD3Viz):