iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Summary: Introduce configs for default relative end time and the epoch-tz (#6721) 

Introduce a config DEFAULT_RELATIVE_END_TIME which is used when
computing the "Last X days". The default behavior (as currently) is to
let that be 'today', which actually means the 0th second of today. We
can also let it be 'now' which means the data as of now (ie at query
time).

Secondly, also introduce another config IS_EPOCH_S_TRULY_UTC, which
means that the logged time in epoch_s format is actually in UTC.
Currently (as the default) is that it it is in the local (superset
webserver) timezone.

** There is no backward incompatibility with thes config features since
the default behavior hasn't changed. **
This commit is contained in:
agrawaldevesh 2019-02-01 16:34:13 -08:00 committed by Maxime Beauchemin
parent 11bace3808
commit cecbba3fe6
5 changed files with 44 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
superset/config.py
View File

@ -566,6 +566,16 @@ WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/'
# Send user to a link where they can report bugs
BUG_REPORT_URL = None
# What is the Last N days relative in the time selector to:
# 'today' means it is midnight (00:00:00) of today in the local timezone
# 'now' means it is relative to the query issue time
DEFAULT_RELATIVE_END_TIME = 'today'
# Is epoch_s/epoch_ms datetime format supposed to be considered since UTC ?
# If not, it is sassumed then the epoch_s/epoch_ms is seconds since 1/1/1970
# localtime (in the tz where the superset webserver is running)
IS_EPOCH_S_TRULY_UTC = False
try:
if CONFIG_PATH_ENV_VAR in os.environ:

16
superset/connectors/sqla/models.py
View File

@ -127,12 +127,13 @@ class TableColumn(Model, BaseColumn):
return self.table
def get_time_filter(self, start_dttm, end_dttm):
is_epoch_in_utc = config.get('IS_EPOCH_S_TRULY_UTC', False)
col = self.get_sqla_col(label='__time')
l = [] # noqa: E741
if start_dttm:
l.append(col >= text(self.dttm_sql_literal(start_dttm)))
l.append(col >= text(self.dttm_sql_literal(start_dttm, is_epoch_in_utc)))
if end_dttm:
l.append(col <= text(self.dttm_sql_literal(end_dttm)))
l.append(col <= text(self.dttm_sql_literal(end_dttm, is_epoch_in_utc)))
return and_(*l)
def get_timestamp_expression(self, time_grain):
@ -166,7 +167,7 @@ class TableColumn(Model, BaseColumn):
TableColumn.column_name == lookup_column.column_name).first()
return import_datasource.import_simple_obj(db.session, i_column, lookup_obj)
def dttm_sql_literal(self, dttm):
def dttm_sql_literal(self, dttm, is_epoch_in_utc):
"""Convert datetime object to a SQL expression string
If database_expression is empty, the internal dttm
@ -179,10 +180,15 @@ class TableColumn(Model, BaseColumn):
if self.database_expression:
return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S'))
elif tf:
if is_epoch_in_utc:
seconds_since_epoch = dttm.timestamp()
else:
seconds_since_epoch = (dttm - datetime(1970, 1, 1)).total_seconds()
seconds_since_epoch = int(seconds_since_epoch)
if tf == 'epoch_s':
return str((dttm - datetime(1970, 1, 1)).total_seconds())
return str(seconds_since_epoch)
elif tf == 'epoch_ms':
return str((dttm - datetime(1970, 1, 1)).total_seconds() * 1000.0)
return str(seconds_since_epoch * 1000)
return "'{}'".format(dttm.strftime(tf))
else:
s = self.table.database.db_engine_spec.convert_dttm(

2
superset/db_engine_specs.py
View File

@ -138,7 +138,7 @@ class BaseEngineSpec(object):
@classmethod
def epoch_ms_to_dttm(cls):
return cls.epoch_to_dttm().replace('{col}', '({col}/1000.000)')
return cls.epoch_to_dttm().replace('{col}', '({col}/1000)')
@classmethod
def get_datatype(cls, type_code):

25
superset/utils/core.py
View File

@ -920,7 +920,8 @@ def ensure_path_exists(path):
def get_since_until(time_range: Optional[str] = None,
since: Optional[str] = None,
until: Optional[str] = None,
time_shift: Optional[str] = None) -> (datetime, datetime):
time_shift: Optional[str] = None,
relative_end: Optional[str] = None) -> (datetime, datetime):
"""Return `since` and `until` date time tuple from string representations of
time_range, since, until and time_shift.
@ -946,13 +947,13 @@ def get_since_until(time_range: Optional[str] = None,
"""
separator = ' : '
today = parse_human_datetime('today')
relative_end = parse_human_datetime(relative_end if relative_end else 'today')
common_time_frames = {
'Last day': (today - relativedelta(days=1), today),
'Last week': (today - relativedelta(weeks=1), today),
'Last month': (today - relativedelta(months=1), today),
'Last quarter': (today - relativedelta(months=3), today),
'Last year': (today - relativedelta(years=1), today),
'Last day': (relative_end - relativedelta(days=1), relative_end),
'Last week': (relative_end - relativedelta(weeks=1), relative_end),
'Last month': (relative_end - relativedelta(months=1), relative_end),
'Last quarter': (relative_end - relativedelta(months=3), relative_end),
'Last year': (relative_end - relativedelta(years=1), relative_end),
}
if time_range:
@ -969,17 +970,17 @@ def get_since_until(time_range: Optional[str] = None,
else:
rel, num, grain = time_range.split()
if rel == 'Last':
since = today - relativedelta(**{grain: int(num)})
until = today
since = relative_end - relativedelta(**{grain: int(num)})
until = relative_end
else: # rel == 'Next'
since = today
until = today + relativedelta(**{grain: int(num)})
since = relative_end
until = relative_end + relativedelta(**{grain: int(num)})
else:
since = since or ''
if since:
since = add_ago_to_since(since)
since = parse_human_datetime(since)
until = parse_human_datetime(until or 'now')
until = parse_human_datetime(until) if until else relative_end
if time_shift:
time_shift = parse_human_timedelta(time_shift)

15
superset/viz.py
View File

@ -59,6 +59,7 @@ from superset.utils.core import (
config = app.config
stats_logger = config.get('STATS_LOGGER')
relative_end = config.get('DEFAULT_RELATIVE_END_TIME', 'today')
METRIC_KEYS = [
'metric', 'metrics', 'percent_metrics', 'metric_2', 'secondary_metric',
@ -280,9 +281,10 @@ class BaseViz(object):
# default order direction
order_desc = form_data.get('order_desc', True)
since, until = utils.get_since_until(form_data.get('time_range'),
form_data.get('since'),
form_data.get('until'))
since, until = utils.get_since_until(relative_end=relative_end,
time_range=form_data.get('time_range'),
since=form_data.get('since'),
until=form_data.get('until'))
time_shift = form_data.get('time_shift', '')
self.time_shift = utils.parse_human_timedelta(time_shift)
from_dttm = None if since is None else (since - self.time_shift)
@ -795,9 +797,10 @@ class CalHeatmapViz(BaseViz):
values[str(v / 10**9)] = obj.get(metric)
data[metric] = values
start, end = utils.get_since_until(form_data.get('time_range'),
form_data.get('since'),
form_data.get('until'))
start, end = utils.get_since_until(relative_end=relative_end,
time_range=form_data.get('time_range'),
since=form_data.get('since'),
until=form_data.get('until'))
if not start or not end:
raise Exception('Please provide both time bounds (Since and Until)')
domain = form_data.get('domain_granularity')