From cecbba3fe63cfb6ea294cec90f57e5282b6dfb6c Mon Sep 17 00:00:00 2001 From: agrawaldevesh Date: Fri, 1 Feb 2019 16:34:13 -0800 Subject: [PATCH] Summary: Introduce configs for default relative end time and the epoch-tz (#6721) Introduce a config DEFAULT_RELATIVE_END_TIME which is used when computing the "Last X days". The default behavior (as currently) is to let that be 'today', which actually means the 0th second of today. We can also let it be 'now' which means the data as of now (ie at query time). Secondly, also introduce another config IS_EPOCH_S_TRULY_UTC, which means that the logged time in epoch_s format is actually in UTC. Currently (as the default) is that it it is in the local (superset webserver) timezone. ** There is no backward incompatibility with thes config features since the default behavior hasn't changed. ** --- superset/config.py | 10 ++++++++++ superset/connectors/sqla/models.py | 16 +++++++++++----- superset/db_engine_specs.py | 2 +- superset/utils/core.py | 25 +++++++++++++------------ superset/viz.py | 15 +++++++++------ 5 files changed, 44 insertions(+), 24 deletions(-) diff --git a/superset/config.py b/superset/config.py index 93720c339..8480ef8b5 100644 --- a/superset/config.py +++ b/superset/config.py @@ -566,6 +566,16 @@ WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/' # Send user to a link where they can report bugs BUG_REPORT_URL = None +# What is the Last N days relative in the time selector to: +# 'today' means it is midnight (00:00:00) of today in the local timezone +# 'now' means it is relative to the query issue time +DEFAULT_RELATIVE_END_TIME = 'today' + +# Is epoch_s/epoch_ms datetime format supposed to be considered since UTC ? +# If not, it is sassumed then the epoch_s/epoch_ms is seconds since 1/1/1970 +# localtime (in the tz where the superset webserver is running) +IS_EPOCH_S_TRULY_UTC = False + try: if CONFIG_PATH_ENV_VAR in os.environ: diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index cfbea8512..41b5dc75a 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -127,12 +127,13 @@ class TableColumn(Model, BaseColumn): return self.table def get_time_filter(self, start_dttm, end_dttm): + is_epoch_in_utc = config.get('IS_EPOCH_S_TRULY_UTC', False) col = self.get_sqla_col(label='__time') l = [] # noqa: E741 if start_dttm: - l.append(col >= text(self.dttm_sql_literal(start_dttm))) + l.append(col >= text(self.dttm_sql_literal(start_dttm, is_epoch_in_utc))) if end_dttm: - l.append(col <= text(self.dttm_sql_literal(end_dttm))) + l.append(col <= text(self.dttm_sql_literal(end_dttm, is_epoch_in_utc))) return and_(*l) def get_timestamp_expression(self, time_grain): @@ -166,7 +167,7 @@ class TableColumn(Model, BaseColumn): TableColumn.column_name == lookup_column.column_name).first() return import_datasource.import_simple_obj(db.session, i_column, lookup_obj) - def dttm_sql_literal(self, dttm): + def dttm_sql_literal(self, dttm, is_epoch_in_utc): """Convert datetime object to a SQL expression string If database_expression is empty, the internal dttm @@ -179,10 +180,15 @@ class TableColumn(Model, BaseColumn): if self.database_expression: return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S')) elif tf: + if is_epoch_in_utc: + seconds_since_epoch = dttm.timestamp() + else: + seconds_since_epoch = (dttm - datetime(1970, 1, 1)).total_seconds() + seconds_since_epoch = int(seconds_since_epoch) if tf == 'epoch_s': - return str((dttm - datetime(1970, 1, 1)).total_seconds()) + return str(seconds_since_epoch) elif tf == 'epoch_ms': - return str((dttm - datetime(1970, 1, 1)).total_seconds() * 1000.0) + return str(seconds_since_epoch * 1000) return "'{}'".format(dttm.strftime(tf)) else: s = self.table.database.db_engine_spec.convert_dttm( diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 937537bdb..3ae433c98 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -138,7 +138,7 @@ class BaseEngineSpec(object): @classmethod def epoch_ms_to_dttm(cls): - return cls.epoch_to_dttm().replace('{col}', '({col}/1000.000)') + return cls.epoch_to_dttm().replace('{col}', '({col}/1000)') @classmethod def get_datatype(cls, type_code): diff --git a/superset/utils/core.py b/superset/utils/core.py index 3e38ea5ea..efeff7e74 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -920,7 +920,8 @@ def ensure_path_exists(path): def get_since_until(time_range: Optional[str] = None, since: Optional[str] = None, until: Optional[str] = None, - time_shift: Optional[str] = None) -> (datetime, datetime): + time_shift: Optional[str] = None, + relative_end: Optional[str] = None) -> (datetime, datetime): """Return `since` and `until` date time tuple from string representations of time_range, since, until and time_shift. @@ -946,13 +947,13 @@ def get_since_until(time_range: Optional[str] = None, """ separator = ' : ' - today = parse_human_datetime('today') + relative_end = parse_human_datetime(relative_end if relative_end else 'today') common_time_frames = { - 'Last day': (today - relativedelta(days=1), today), - 'Last week': (today - relativedelta(weeks=1), today), - 'Last month': (today - relativedelta(months=1), today), - 'Last quarter': (today - relativedelta(months=3), today), - 'Last year': (today - relativedelta(years=1), today), + 'Last day': (relative_end - relativedelta(days=1), relative_end), + 'Last week': (relative_end - relativedelta(weeks=1), relative_end), + 'Last month': (relative_end - relativedelta(months=1), relative_end), + 'Last quarter': (relative_end - relativedelta(months=3), relative_end), + 'Last year': (relative_end - relativedelta(years=1), relative_end), } if time_range: @@ -969,17 +970,17 @@ def get_since_until(time_range: Optional[str] = None, else: rel, num, grain = time_range.split() if rel == 'Last': - since = today - relativedelta(**{grain: int(num)}) - until = today + since = relative_end - relativedelta(**{grain: int(num)}) + until = relative_end else: # rel == 'Next' - since = today - until = today + relativedelta(**{grain: int(num)}) + since = relative_end + until = relative_end + relativedelta(**{grain: int(num)}) else: since = since or '' if since: since = add_ago_to_since(since) since = parse_human_datetime(since) - until = parse_human_datetime(until or 'now') + until = parse_human_datetime(until) if until else relative_end if time_shift: time_shift = parse_human_timedelta(time_shift) diff --git a/superset/viz.py b/superset/viz.py index dc84630e4..51fab12b5 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -59,6 +59,7 @@ from superset.utils.core import ( config = app.config stats_logger = config.get('STATS_LOGGER') +relative_end = config.get('DEFAULT_RELATIVE_END_TIME', 'today') METRIC_KEYS = [ 'metric', 'metrics', 'percent_metrics', 'metric_2', 'secondary_metric', @@ -280,9 +281,10 @@ class BaseViz(object): # default order direction order_desc = form_data.get('order_desc', True) - since, until = utils.get_since_until(form_data.get('time_range'), - form_data.get('since'), - form_data.get('until')) + since, until = utils.get_since_until(relative_end=relative_end, + time_range=form_data.get('time_range'), + since=form_data.get('since'), + until=form_data.get('until')) time_shift = form_data.get('time_shift', '') self.time_shift = utils.parse_human_timedelta(time_shift) from_dttm = None if since is None else (since - self.time_shift) @@ -795,9 +797,10 @@ class CalHeatmapViz(BaseViz): values[str(v / 10**9)] = obj.get(metric) data[metric] = values - start, end = utils.get_since_until(form_data.get('time_range'), - form_data.get('since'), - form_data.get('until')) + start, end = utils.get_since_until(relative_end=relative_end, + time_range=form_data.get('time_range'), + since=form_data.get('since'), + until=form_data.get('until')) if not start or not end: raise Exception('Please provide both time bounds (Since and Until)') domain = form_data.get('domain_granularity')