diff --git a/superset/config.py b/superset/config.py index 93720c339..8480ef8b5 100644 --- a/superset/config.py +++ b/superset/config.py @@ -566,6 +566,16 @@ WEBDRIVER_BASEURL = 'http://0.0.0.0:8080/' # Send user to a link where they can report bugs BUG_REPORT_URL = None +# What is the Last N days relative in the time selector to: +# 'today' means it is midnight (00:00:00) of today in the local timezone +# 'now' means it is relative to the query issue time +DEFAULT_RELATIVE_END_TIME = 'today' + +# Is epoch_s/epoch_ms datetime format supposed to be considered since UTC ? +# If not, it is sassumed then the epoch_s/epoch_ms is seconds since 1/1/1970 +# localtime (in the tz where the superset webserver is running) +IS_EPOCH_S_TRULY_UTC = False + try: if CONFIG_PATH_ENV_VAR in os.environ: diff --git a/superset/connectors/sqla/models.py b/superset/connectors/sqla/models.py index cfbea8512..41b5dc75a 100644 --- a/superset/connectors/sqla/models.py +++ b/superset/connectors/sqla/models.py @@ -127,12 +127,13 @@ class TableColumn(Model, BaseColumn): return self.table def get_time_filter(self, start_dttm, end_dttm): + is_epoch_in_utc = config.get('IS_EPOCH_S_TRULY_UTC', False) col = self.get_sqla_col(label='__time') l = [] # noqa: E741 if start_dttm: - l.append(col >= text(self.dttm_sql_literal(start_dttm))) + l.append(col >= text(self.dttm_sql_literal(start_dttm, is_epoch_in_utc))) if end_dttm: - l.append(col <= text(self.dttm_sql_literal(end_dttm))) + l.append(col <= text(self.dttm_sql_literal(end_dttm, is_epoch_in_utc))) return and_(*l) def get_timestamp_expression(self, time_grain): @@ -166,7 +167,7 @@ class TableColumn(Model, BaseColumn): TableColumn.column_name == lookup_column.column_name).first() return import_datasource.import_simple_obj(db.session, i_column, lookup_obj) - def dttm_sql_literal(self, dttm): + def dttm_sql_literal(self, dttm, is_epoch_in_utc): """Convert datetime object to a SQL expression string If database_expression is empty, the internal dttm @@ -179,10 +180,15 @@ class TableColumn(Model, BaseColumn): if self.database_expression: return self.database_expression.format(dttm.strftime('%Y-%m-%d %H:%M:%S')) elif tf: + if is_epoch_in_utc: + seconds_since_epoch = dttm.timestamp() + else: + seconds_since_epoch = (dttm - datetime(1970, 1, 1)).total_seconds() + seconds_since_epoch = int(seconds_since_epoch) if tf == 'epoch_s': - return str((dttm - datetime(1970, 1, 1)).total_seconds()) + return str(seconds_since_epoch) elif tf == 'epoch_ms': - return str((dttm - datetime(1970, 1, 1)).total_seconds() * 1000.0) + return str(seconds_since_epoch * 1000) return "'{}'".format(dttm.strftime(tf)) else: s = self.table.database.db_engine_spec.convert_dttm( diff --git a/superset/db_engine_specs.py b/superset/db_engine_specs.py index 937537bdb..3ae433c98 100644 --- a/superset/db_engine_specs.py +++ b/superset/db_engine_specs.py @@ -138,7 +138,7 @@ class BaseEngineSpec(object): @classmethod def epoch_ms_to_dttm(cls): - return cls.epoch_to_dttm().replace('{col}', '({col}/1000.000)') + return cls.epoch_to_dttm().replace('{col}', '({col}/1000)') @classmethod def get_datatype(cls, type_code): diff --git a/superset/utils/core.py b/superset/utils/core.py index 3e38ea5ea..efeff7e74 100644 --- a/superset/utils/core.py +++ b/superset/utils/core.py @@ -920,7 +920,8 @@ def ensure_path_exists(path): def get_since_until(time_range: Optional[str] = None, since: Optional[str] = None, until: Optional[str] = None, - time_shift: Optional[str] = None) -> (datetime, datetime): + time_shift: Optional[str] = None, + relative_end: Optional[str] = None) -> (datetime, datetime): """Return `since` and `until` date time tuple from string representations of time_range, since, until and time_shift. @@ -946,13 +947,13 @@ def get_since_until(time_range: Optional[str] = None, """ separator = ' : ' - today = parse_human_datetime('today') + relative_end = parse_human_datetime(relative_end if relative_end else 'today') common_time_frames = { - 'Last day': (today - relativedelta(days=1), today), - 'Last week': (today - relativedelta(weeks=1), today), - 'Last month': (today - relativedelta(months=1), today), - 'Last quarter': (today - relativedelta(months=3), today), - 'Last year': (today - relativedelta(years=1), today), + 'Last day': (relative_end - relativedelta(days=1), relative_end), + 'Last week': (relative_end - relativedelta(weeks=1), relative_end), + 'Last month': (relative_end - relativedelta(months=1), relative_end), + 'Last quarter': (relative_end - relativedelta(months=3), relative_end), + 'Last year': (relative_end - relativedelta(years=1), relative_end), } if time_range: @@ -969,17 +970,17 @@ def get_since_until(time_range: Optional[str] = None, else: rel, num, grain = time_range.split() if rel == 'Last': - since = today - relativedelta(**{grain: int(num)}) - until = today + since = relative_end - relativedelta(**{grain: int(num)}) + until = relative_end else: # rel == 'Next' - since = today - until = today + relativedelta(**{grain: int(num)}) + since = relative_end + until = relative_end + relativedelta(**{grain: int(num)}) else: since = since or '' if since: since = add_ago_to_since(since) since = parse_human_datetime(since) - until = parse_human_datetime(until or 'now') + until = parse_human_datetime(until) if until else relative_end if time_shift: time_shift = parse_human_timedelta(time_shift) diff --git a/superset/viz.py b/superset/viz.py index dc84630e4..51fab12b5 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -59,6 +59,7 @@ from superset.utils.core import ( config = app.config stats_logger = config.get('STATS_LOGGER') +relative_end = config.get('DEFAULT_RELATIVE_END_TIME', 'today') METRIC_KEYS = [ 'metric', 'metrics', 'percent_metrics', 'metric_2', 'secondary_metric', @@ -280,9 +281,10 @@ class BaseViz(object): # default order direction order_desc = form_data.get('order_desc', True) - since, until = utils.get_since_until(form_data.get('time_range'), - form_data.get('since'), - form_data.get('until')) + since, until = utils.get_since_until(relative_end=relative_end, + time_range=form_data.get('time_range'), + since=form_data.get('since'), + until=form_data.get('until')) time_shift = form_data.get('time_shift', '') self.time_shift = utils.parse_human_timedelta(time_shift) from_dttm = None if since is None else (since - self.time_shift) @@ -795,9 +797,10 @@ class CalHeatmapViz(BaseViz): values[str(v / 10**9)] = obj.get(metric) data[metric] = values - start, end = utils.get_since_until(form_data.get('time_range'), - form_data.get('since'), - form_data.get('until')) + start, end = utils.get_since_until(relative_end=relative_end, + time_range=form_data.get('time_range'), + since=form_data.get('since'), + until=form_data.get('until')) if not start or not end: raise Exception('Please provide both time bounds (Since and Until)') domain = form_data.get('domain_granularity')