feat(cross-filters): add support for temporal filters (#16139)
* feat(cross-filters): add support for temporal filters * fix test * make filter optional * remove mocks * fix more tests * remove unnecessary optionality * fix even more tests * bump superset-ui * add isExtra to schema * address comments * fix presto test
This commit is contained in:
Ville Brofeldt
GitHub
parent
5488a8a948
commit
63ace7b288
File diff suppressed because it is too large
Load Diff
|
|
@ -67,35 +67,35 @@
|
|||
"@emotion/babel-preset-css-prop": "^11.2.0",
|
||||
"@emotion/cache": "^11.1.3",
|
||||
"@emotion/react": "^11.1.5",
|
||||
"@superset-ui/chart-controls": "^0.17.79",
|
||||
"@superset-ui/core": "^0.17.75",
|
||||
"@superset-ui/legacy-plugin-chart-calendar": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-chord": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-country-map": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-event-flow": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-force-directed": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-heatmap": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-histogram": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-horizon": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-map-box": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-paired-t-test": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-parallel-coordinates": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-partition": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-pivot-table": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-rose": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-sankey": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-sankey-loop": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-sunburst": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-treemap": "^0.17.79",
|
||||
"@superset-ui/legacy-plugin-chart-world-map": "^0.17.79",
|
||||
"@superset-ui/legacy-preset-chart-big-number": "^0.17.79",
|
||||
"@superset-ui/chart-controls": "^0.17.80",
|
||||
"@superset-ui/core": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-calendar": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-chord": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-country-map": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-event-flow": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-force-directed": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-heatmap": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-histogram": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-horizon": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-map-box": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-paired-t-test": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-parallel-coordinates": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-partition": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-pivot-table": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-rose": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-sankey": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-sankey-loop": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-sunburst": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-treemap": "^0.17.80",
|
||||
"@superset-ui/legacy-plugin-chart-world-map": "^0.17.80",
|
||||
"@superset-ui/legacy-preset-chart-big-number": "^0.17.80",
|
||||
"@superset-ui/legacy-preset-chart-deckgl": "^0.4.9",
|
||||
"@superset-ui/legacy-preset-chart-nvd3": "^0.17.79",
|
||||
"@superset-ui/plugin-chart-echarts": "^0.17.79",
|
||||
"@superset-ui/plugin-chart-pivot-table": "^0.17.79",
|
||||
"@superset-ui/plugin-chart-table": "^0.17.79",
|
||||
"@superset-ui/plugin-chart-word-cloud": "^0.17.79",
|
||||
"@superset-ui/preset-chart-xy": "^0.17.79",
|
||||
"@superset-ui/legacy-preset-chart-nvd3": "^0.17.80",
|
||||
"@superset-ui/plugin-chart-echarts": "^0.17.80",
|
||||
"@superset-ui/plugin-chart-pivot-table": "^0.17.80",
|
||||
"@superset-ui/plugin-chart-table": "^0.17.80",
|
||||
"@superset-ui/plugin-chart-word-cloud": "^0.17.80",
|
||||
"@superset-ui/preset-chart-xy": "^0.17.80",
|
||||
"@vx/responsive": "^0.0.195",
|
||||
"abortcontroller-polyfill": "^1.1.9",
|
||||
"antd": "^4.9.4",
|
||||
|
|
|
|||
|
|
@ -293,6 +293,13 @@ class ChartDataAdhocMetricSchema(Schema):
|
|||
"will be generated.",
|
||||
example="metric_aec60732-fac0-4b17-b736-93f1a5c93e30",
|
||||
)
|
||||
timeGrain = fields.String(
|
||||
description="Optional time grain for temporal filters", example="PT1M",
|
||||
)
|
||||
isExtra = fields.Boolean(
|
||||
description="Indicates if the filter has been added by a filter component as "
|
||||
"opposed to being a part of the original query."
|
||||
)
|
||||
|
||||
|
||||
class ChartDataAggregateConfigField(fields.Dict):
|
||||
|
|
@ -771,6 +778,13 @@ class ChartDataFilterSchema(Schema):
|
|||
"integer, decimal or list, depending on the operator.",
|
||||
example=["China", "France", "Japan"],
|
||||
)
|
||||
grain = fields.String(
|
||||
description="Optional time grain for temporal filters", example="PT1M",
|
||||
)
|
||||
isExtra = fields.Boolean(
|
||||
description="Indicates if the filter has been added by a filter component as "
|
||||
"opposed to being a part of the original query."
|
||||
)
|
||||
|
||||
|
||||
class ChartDataExtrasSchema(Schema):
|
||||
|
|
|
|||
|
|
@ -36,6 +36,7 @@ from superset.utils.core import (
|
|||
get_metric_names,
|
||||
is_adhoc_metric,
|
||||
json_int_dttm_ser,
|
||||
QueryObjectFilterClause,
|
||||
)
|
||||
from superset.utils.date_parser import get_since_until, parse_human_timedelta
|
||||
from superset.utils.hashing import md5_sha_from_dict
|
||||
|
|
@ -85,7 +86,7 @@ class QueryObject:
|
|||
metrics: Optional[List[Metric]]
|
||||
row_limit: int
|
||||
row_offset: int
|
||||
filter: List[Dict[str, Any]]
|
||||
filter: List[QueryObjectFilterClause]
|
||||
timeseries_limit: int
|
||||
timeseries_limit_metric: Optional[Metric]
|
||||
order_desc: bool
|
||||
|
|
@ -108,7 +109,7 @@ class QueryObject:
|
|||
granularity: Optional[str] = None,
|
||||
metrics: Optional[List[Metric]] = None,
|
||||
groupby: Optional[List[str]] = None,
|
||||
filters: Optional[List[Dict[str, Any]]] = None,
|
||||
filters: Optional[List[QueryObjectFilterClause]] = None,
|
||||
time_range: Optional[str] = None,
|
||||
time_shift: Optional[str] = None,
|
||||
is_timeseries: Optional[bool] = None,
|
||||
|
|
|
|||
|
|
@ -86,7 +86,11 @@ from superset.models.helpers import AuditMixinNullable, QueryResult
|
|||
from superset.sql_parse import ParsedQuery
|
||||
from superset.typing import AdhocMetric, Metric, OrderBy, QueryObjectDict
|
||||
from superset.utils import core as utils
|
||||
from superset.utils.core import GenericDataType, remove_duplicates
|
||||
from superset.utils.core import (
|
||||
GenericDataType,
|
||||
QueryObjectFilterClause,
|
||||
remove_duplicates,
|
||||
)
|
||||
|
||||
config = app.config
|
||||
metadata = Model.metadata # pylint: disable=no-member
|
||||
|
|
@ -303,13 +307,15 @@ class TableColumn(Model, BaseColumn):
|
|||
|
||||
pdf = self.python_date_format
|
||||
is_epoch = pdf in ("epoch_s", "epoch_ms")
|
||||
column_spec = self.db_engine_spec.get_column_spec(self.type)
|
||||
type_ = column_spec.sqla_type if column_spec else DateTime
|
||||
if not self.expression and not time_grain and not is_epoch:
|
||||
sqla_col = column(self.column_name, type_=DateTime)
|
||||
sqla_col = column(self.column_name, type_=type_)
|
||||
return self.table.make_sqla_column_compatible(sqla_col, label)
|
||||
if self.expression:
|
||||
col = literal_column(self.expression)
|
||||
col = literal_column(self.expression, type_=type_)
|
||||
else:
|
||||
col = column(self.column_name)
|
||||
col = column(self.column_name, type_=type_)
|
||||
time_expr = self.db_engine_spec.get_timestamp_expr(
|
||||
col, pdf, time_grain, self.type
|
||||
)
|
||||
|
|
@ -935,7 +941,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
columns: Optional[List[str]] = None,
|
||||
groupby: Optional[List[str]] = None,
|
||||
filter: Optional[ # pylint: disable=redefined-builtin
|
||||
List[Dict[str, Any]]
|
||||
List[QueryObjectFilterClause]
|
||||
] = None,
|
||||
is_timeseries: bool = True,
|
||||
timeseries_limit: int = 15,
|
||||
|
|
@ -1056,6 +1062,8 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
# filter out the pseudo column __timestamp from columns
|
||||
columns = columns or []
|
||||
columns = [col for col in columns if col != utils.DTTM_ALIAS]
|
||||
time_grain = extras.get("time_grain_sqla")
|
||||
dttm_col = columns_by_name.get(granularity) if granularity else None
|
||||
|
||||
if need_groupby:
|
||||
# dedup columns while preserving order
|
||||
|
|
@ -1063,7 +1071,6 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
for selected in columns:
|
||||
# if groupby field/expr equals granularity field/expr
|
||||
if selected == granularity:
|
||||
time_grain = extras.get("time_grain_sqla")
|
||||
sqla_col = columns_by_name[selected]
|
||||
outer = sqla_col.get_timestamp_expression(time_grain, selected)
|
||||
# if groupby field equals a selected column
|
||||
|
|
@ -1087,15 +1094,13 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
groupby_exprs_with_timestamp = OrderedDict(groupby_exprs_sans_timestamp.items())
|
||||
|
||||
if granularity:
|
||||
if granularity not in columns_by_name:
|
||||
if granularity not in columns_by_name or not dttm_col:
|
||||
raise QueryObjectValidationError(
|
||||
_(
|
||||
'Time column "%(col)s" does not exist in dataset',
|
||||
col=granularity,
|
||||
)
|
||||
)
|
||||
dttm_col = columns_by_name[granularity]
|
||||
time_grain = extras.get("time_grain_sqla")
|
||||
time_filters = []
|
||||
|
||||
if is_timeseries:
|
||||
|
|
@ -1150,7 +1155,12 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
col = flt["col"]
|
||||
val = flt.get("val")
|
||||
op = flt["op"].upper()
|
||||
col_obj = columns_by_name.get(col)
|
||||
col_obj = (
|
||||
dttm_col
|
||||
if col == utils.DTTM_ALIAS and is_timeseries and dttm_col
|
||||
else columns_by_name.get(col)
|
||||
)
|
||||
filter_grain = flt.get("grain")
|
||||
|
||||
if is_feature_enabled("ENABLE_TEMPLATE_REMOVE_FILTERS"):
|
||||
if col in removed_filters:
|
||||
|
|
@ -1158,6 +1168,10 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
continue
|
||||
|
||||
if col_obj:
|
||||
if filter_grain:
|
||||
sqla_col = col_obj.get_timestamp_expression(filter_grain)
|
||||
else:
|
||||
sqla_col = col_obj.get_sqla_col()
|
||||
col_spec = db_engine_spec.get_column_spec(col_obj.type)
|
||||
is_list_target = op in (
|
||||
utils.FilterOperator.IN.value,
|
||||
|
|
@ -1180,24 +1194,24 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
)
|
||||
if None in eq:
|
||||
eq = [x for x in eq if x is not None]
|
||||
is_null_cond = col_obj.get_sqla_col().is_(None)
|
||||
is_null_cond = sqla_col.is_(None)
|
||||
if eq:
|
||||
cond = or_(is_null_cond, col_obj.get_sqla_col().in_(eq))
|
||||
cond = or_(is_null_cond, sqla_col.in_(eq))
|
||||
else:
|
||||
cond = is_null_cond
|
||||
else:
|
||||
cond = col_obj.get_sqla_col().in_(eq)
|
||||
cond = sqla_col.in_(eq)
|
||||
if op == utils.FilterOperator.NOT_IN.value:
|
||||
cond = ~cond
|
||||
where_clause_and.append(cond)
|
||||
elif op == utils.FilterOperator.IS_NULL.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().is_(None))
|
||||
where_clause_and.append(sqla_col.is_(None))
|
||||
elif op == utils.FilterOperator.IS_NOT_NULL.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().isnot(None))
|
||||
where_clause_and.append(sqla_col.isnot(None))
|
||||
elif op == utils.FilterOperator.IS_TRUE.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().is_(True))
|
||||
where_clause_and.append(sqla_col.is_(True))
|
||||
elif op == utils.FilterOperator.IS_FALSE.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().is_(False))
|
||||
where_clause_and.append(sqla_col.is_(False))
|
||||
else:
|
||||
if eq is None:
|
||||
raise QueryObjectValidationError(
|
||||
|
|
@ -1207,21 +1221,21 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
)
|
||||
)
|
||||
if op == utils.FilterOperator.EQUALS.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() == eq)
|
||||
where_clause_and.append(sqla_col == eq)
|
||||
elif op == utils.FilterOperator.NOT_EQUALS.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() != eq)
|
||||
where_clause_and.append(sqla_col != eq)
|
||||
elif op == utils.FilterOperator.GREATER_THAN.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() > eq)
|
||||
where_clause_and.append(sqla_col > eq)
|
||||
elif op == utils.FilterOperator.LESS_THAN.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() < eq)
|
||||
where_clause_and.append(sqla_col < eq)
|
||||
elif op == utils.FilterOperator.GREATER_THAN_OR_EQUALS.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() >= eq)
|
||||
where_clause_and.append(sqla_col >= eq)
|
||||
elif op == utils.FilterOperator.LESS_THAN_OR_EQUALS.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col() <= eq)
|
||||
where_clause_and.append(sqla_col <= eq)
|
||||
elif op == utils.FilterOperator.LIKE.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().like(eq))
|
||||
where_clause_and.append(sqla_col.like(eq))
|
||||
elif op == utils.FilterOperator.ILIKE.value:
|
||||
where_clause_and.append(col_obj.get_sqla_col().ilike(eq))
|
||||
where_clause_and.append(sqla_col.ilike(eq))
|
||||
else:
|
||||
raise QueryObjectValidationError(
|
||||
_("Invalid filter operation type: %(op)s", op=op)
|
||||
|
|
@ -1281,6 +1295,7 @@ class SqlaTable( # pylint: disable=too-many-public-methods,too-many-instance-at
|
|||
and timeseries_limit
|
||||
and not time_groupby_inline
|
||||
and groupby_exprs_sans_timestamp
|
||||
and dttm_col
|
||||
):
|
||||
if db_engine_spec.allows_joins:
|
||||
# some sql dialects require for order by expressions
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ from flask import current_app, g
|
|||
from flask_babel import gettext as __, lazy_gettext as _
|
||||
from marshmallow import fields, Schema
|
||||
from marshmallow.validate import Range
|
||||
from sqlalchemy import column, DateTime, select, types
|
||||
from sqlalchemy import column, select, types
|
||||
from sqlalchemy.engine.base import Engine
|
||||
from sqlalchemy.engine.interfaces import Compiled, Dialect
|
||||
from sqlalchemy.engine.reflection import Inspector
|
||||
|
|
@ -381,7 +381,7 @@ class BaseEngineSpec: # pylint: disable=too-many-public-methods
|
|||
elif pdf == "epoch_ms":
|
||||
time_expr = time_expr.replace("{col}", cls.epoch_ms_to_dttm())
|
||||
|
||||
return TimestampExpression(time_expr, col, type_=DateTime)
|
||||
return TimestampExpression(time_expr, col, type_=col.type)
|
||||
|
||||
@classmethod
|
||||
def get_time_grains(cls) -> Tuple[TimeGrain, ...]:
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ DbapiDescriptionRow = Tuple[
|
|||
]
|
||||
DbapiDescription = Union[List[DbapiDescriptionRow], Tuple[DbapiDescriptionRow, ...]]
|
||||
DbapiResult = Sequence[Union[List[Any], Tuple[Any, ...]]]
|
||||
FilterValue = Union[datetime, float, int, str]
|
||||
FilterValue = Union[bool, datetime, float, int, str]
|
||||
FilterValues = Union[FilterValue, List[FilterValue], Tuple[FilterValue]]
|
||||
FormData = Dict[str, Any]
|
||||
Granularity = Union[str, Dict[str, Union[str, float]]]
|
||||
|
|
|
|||
|
|
@ -96,7 +96,7 @@ from superset.exceptions import (
|
|||
SupersetException,
|
||||
SupersetTimeoutException,
|
||||
)
|
||||
from superset.typing import AdhocMetric, FlaskResponse, FormData, Metric
|
||||
from superset.typing import AdhocMetric, FilterValues, FlaskResponse, FormData, Metric
|
||||
from superset.utils.dates import datetime_to_epoch, EPOCH
|
||||
from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str
|
||||
|
||||
|
|
@ -189,6 +189,25 @@ class DatasourceDict(TypedDict):
|
|||
id: int
|
||||
|
||||
|
||||
class AdhocFilterClause(TypedDict, total=False):
|
||||
clause: str
|
||||
expressionType: str
|
||||
filterOptionName: Optional[str]
|
||||
comparator: Optional[FilterValues]
|
||||
operator: str
|
||||
subject: str
|
||||
isExtra: Optional[bool]
|
||||
sqlExpression: Optional[str]
|
||||
|
||||
|
||||
class QueryObjectFilterClause(TypedDict, total=False):
|
||||
col: str
|
||||
op: str # pylint: disable=invalid-name
|
||||
val: Optional[FilterValues]
|
||||
grain: Optional[str]
|
||||
isExtra: Optional[bool]
|
||||
|
||||
|
||||
class ExtraFiltersTimeColumnType(str, Enum):
|
||||
GRANULARITY = "__granularity"
|
||||
TIME_COL = "__time_col"
|
||||
|
|
@ -1017,28 +1036,32 @@ def zlib_decompress(blob: bytes, decode: Optional[bool] = True) -> Union[bytes,
|
|||
return decompressed.decode("utf-8") if decode else decompressed
|
||||
|
||||
|
||||
def to_adhoc(
|
||||
filt: Dict[str, Any], expression_type: str = "SIMPLE", clause: str = "where"
|
||||
) -> Dict[str, Any]:
|
||||
result = {
|
||||
def simple_filter_to_adhoc(
|
||||
filter_clause: QueryObjectFilterClause, clause: str = "where",
|
||||
) -> AdhocFilterClause:
|
||||
result: AdhocFilterClause = {
|
||||
"clause": clause.upper(),
|
||||
"expressionType": expression_type,
|
||||
"isExtra": bool(filt.get("isExtra")),
|
||||
"expressionType": "SIMPLE",
|
||||
"comparator": filter_clause.get("val"),
|
||||
"operator": filter_clause["op"],
|
||||
"subject": filter_clause["col"],
|
||||
}
|
||||
if filter_clause.get("isExtra"):
|
||||
result["isExtra"] = True
|
||||
result["filterOptionName"] = md5_sha_from_dict(cast(Dict[Any, Any], result))
|
||||
|
||||
if expression_type == "SIMPLE":
|
||||
result.update(
|
||||
{
|
||||
"comparator": filt.get("val"),
|
||||
"operator": filt.get("op"),
|
||||
"subject": filt.get("col"),
|
||||
}
|
||||
)
|
||||
elif expression_type == "SQL":
|
||||
result.update({"sqlExpression": filt.get(clause)})
|
||||
return result
|
||||
|
||||
deterministic_name = md5_sha_from_dict(result)
|
||||
result["filterOptionName"] = deterministic_name
|
||||
|
||||
def form_data_to_adhoc(form_data: Dict[str, Any], clause: str) -> AdhocFilterClause:
|
||||
if clause not in ("where", "having"):
|
||||
raise ValueError(__("Unsupported clause type: %(clause)s", clause=clause))
|
||||
result: AdhocFilterClause = {
|
||||
"clause": clause.upper(),
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": form_data.get(clause),
|
||||
}
|
||||
result["filterOptionName"] = md5_sha_from_dict(cast(Dict[Any, Any], result))
|
||||
|
||||
return result
|
||||
|
||||
|
|
@ -1050,7 +1073,7 @@ def merge_extra_form_data(form_data: Dict[str, Any]) -> None:
|
|||
"""
|
||||
filter_keys = ["filters", "adhoc_filters"]
|
||||
extra_form_data = form_data.pop("extra_form_data", {})
|
||||
append_filters = extra_form_data.get("filters", None)
|
||||
append_filters: List[QueryObjectFilterClause] = extra_form_data.get("filters", None)
|
||||
|
||||
# merge append extras
|
||||
for key in [key for key in EXTRA_FORM_DATA_APPEND_KEYS if key not in filter_keys]:
|
||||
|
|
@ -1075,13 +1098,21 @@ def merge_extra_form_data(form_data: Dict[str, Any]) -> None:
|
|||
if extras:
|
||||
form_data["extras"] = extras
|
||||
|
||||
adhoc_filters = form_data.get("adhoc_filters", [])
|
||||
adhoc_filters: List[AdhocFilterClause] = form_data.get("adhoc_filters", [])
|
||||
form_data["adhoc_filters"] = adhoc_filters
|
||||
append_adhoc_filters = extra_form_data.get("adhoc_filters", [])
|
||||
adhoc_filters.extend({"isExtra": True, **fltr} for fltr in append_adhoc_filters)
|
||||
append_adhoc_filters: List[AdhocFilterClause] = extra_form_data.get(
|
||||
"adhoc_filters", []
|
||||
)
|
||||
adhoc_filters.extend(
|
||||
{"isExtra": True, **fltr} for fltr in append_adhoc_filters # type: ignore
|
||||
)
|
||||
if append_filters:
|
||||
adhoc_filters.extend(
|
||||
to_adhoc({"isExtra": True, **fltr}) for fltr in append_filters if fltr
|
||||
simple_filter_to_adhoc(
|
||||
{"isExtra": True, **fltr} # type: ignore
|
||||
)
|
||||
for fltr in append_filters
|
||||
if fltr
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -1148,16 +1179,16 @@ def merge_extra_filters( # pylint: disable=too-many-branches
|
|||
# Add filters for unequal lists
|
||||
# order doesn't matter
|
||||
if set(existing_filters[filter_key]) != set(filtr["val"]):
|
||||
adhoc_filters.append(to_adhoc(filtr))
|
||||
adhoc_filters.append(simple_filter_to_adhoc(filtr))
|
||||
else:
|
||||
adhoc_filters.append(to_adhoc(filtr))
|
||||
adhoc_filters.append(simple_filter_to_adhoc(filtr))
|
||||
else:
|
||||
# Do not add filter if same value already exists
|
||||
if filtr["val"] != existing_filters[filter_key]:
|
||||
adhoc_filters.append(to_adhoc(filtr))
|
||||
adhoc_filters.append(simple_filter_to_adhoc(filtr))
|
||||
else:
|
||||
# Filter not found, add it
|
||||
adhoc_filters.append(to_adhoc(filtr))
|
||||
adhoc_filters.append(simple_filter_to_adhoc(filtr))
|
||||
# Remove extra filters from the form data since no longer needed
|
||||
del form_data["extra_filters"]
|
||||
|
||||
|
|
@ -1268,15 +1299,16 @@ def convert_legacy_filters_into_adhoc( # pylint: disable=invalid-name
|
|||
mapping = {"having": "having_filters", "where": "filters"}
|
||||
|
||||
if not form_data.get("adhoc_filters"):
|
||||
form_data["adhoc_filters"] = []
|
||||
adhoc_filters: List[AdhocFilterClause] = []
|
||||
form_data["adhoc_filters"] = adhoc_filters
|
||||
|
||||
for clause, filters in mapping.items():
|
||||
if clause in form_data and form_data[clause] != "":
|
||||
form_data["adhoc_filters"].append(to_adhoc(form_data, "SQL", clause))
|
||||
adhoc_filters.append(form_data_to_adhoc(form_data, clause))
|
||||
|
||||
if filters in form_data:
|
||||
for filt in filter(lambda x: x is not None, form_data[filters]):
|
||||
form_data["adhoc_filters"].append(to_adhoc(filt, "SIMPLE", clause))
|
||||
adhoc_filters.append(simple_filter_to_adhoc(filt, clause))
|
||||
|
||||
for key in ("filters", "having", "having_filters", "where"):
|
||||
if key in form_data:
|
||||
|
|
|
|||
|
|
@ -74,7 +74,7 @@ from superset.utils.core import (
|
|||
JS_MAX_INTEGER,
|
||||
merge_extra_filters,
|
||||
QueryMode,
|
||||
to_adhoc,
|
||||
simple_filter_to_adhoc,
|
||||
)
|
||||
from superset.utils.date_parser import get_since_until, parse_past_timedelta
|
||||
from superset.utils.dates import datetime_to_epoch
|
||||
|
|
@ -2475,7 +2475,9 @@ class BaseDeckGLViz(BaseViz):
|
|||
spatial_columns.add(line_column)
|
||||
|
||||
for column in sorted(spatial_columns):
|
||||
filter_ = to_adhoc({"col": column, "op": "IS NOT NULL", "val": ""})
|
||||
filter_ = simple_filter_to_adhoc(
|
||||
{"col": column, "op": "IS NOT NULL", "val": ""}
|
||||
)
|
||||
fd["adhoc_filters"].append(filter_)
|
||||
|
||||
def query_obj(self) -> QueryObjectDict:
|
||||
|
|
|
|||
|
|
@ -22,7 +22,6 @@ from tests.integration_tests.fixtures.birth_names_dashboard import (
|
|||
load_birth_names_dashboard_with_slices,
|
||||
)
|
||||
|
||||
import pandas
|
||||
import pytest
|
||||
from sqlalchemy.engine.url import make_url
|
||||
|
||||
|
|
@ -339,12 +338,19 @@ class TestDatabaseModel(SupersetTestCase):
|
|||
class TestSqlaTableModel(SupersetTestCase):
|
||||
@pytest.mark.usefixtures("load_birth_names_dashboard_with_slices")
|
||||
def test_get_timestamp_expression(self):
|
||||
col_type = (
|
||||
"VARCHAR"
|
||||
if get_example_database().backend == "presto"
|
||||
else "TemporalWrapperType"
|
||||
)
|
||||
tbl = self.get_table(name="birth_names")
|
||||
ds_col = tbl.get_column("ds")
|
||||
sqla_literal = ds_col.get_timestamp_expression(None)
|
||||
self.assertEqual(str(sqla_literal.compile()), "ds")
|
||||
assert type(sqla_literal.type).__name__ == col_type
|
||||
|
||||
sqla_literal = ds_col.get_timestamp_expression("P1D")
|
||||
assert type(sqla_literal.type).__name__ == col_type
|
||||
compiled = "{}".format(sqla_literal.compile())
|
||||
if tbl.database.backend == "mysql":
|
||||
self.assertEqual(compiled, "DATE(ds)")
|
||||
|
|
@ -352,6 +358,7 @@ class TestSqlaTableModel(SupersetTestCase):
|
|||
prev_ds_expr = ds_col.expression
|
||||
ds_col.expression = "DATE_ADD(ds, 1)"
|
||||
sqla_literal = ds_col.get_timestamp_expression("P1D")
|
||||
assert type(sqla_literal.type).__name__ == col_type
|
||||
compiled = "{}".format(sqla_literal.compile())
|
||||
if tbl.database.backend == "mysql":
|
||||
self.assertEqual(compiled, "DATE(DATE_ADD(ds, 1))")
|
||||
|
|
|
|||
|
|
@ -17,10 +17,10 @@
|
|||
# pylint: disable=no-self-use
|
||||
import pytest
|
||||
|
||||
from superset.utils.core import to_adhoc
|
||||
from superset.utils.core import form_data_to_adhoc, simple_filter_to_adhoc
|
||||
|
||||
|
||||
def test_to_adhoc_generates_deterministic_values():
|
||||
def test_simple_filter_to_adhoc_generates_deterministic_values():
|
||||
input_1 = {
|
||||
"op": "IS NOT NULL",
|
||||
"col": "LATITUDE",
|
||||
|
|
@ -30,25 +30,56 @@ def test_to_adhoc_generates_deterministic_values():
|
|||
input_2 = {**input_1, "col": "LONGITUDE"}
|
||||
|
||||
# The result is the same when given the same input
|
||||
assert to_adhoc(input_1) == to_adhoc(input_1)
|
||||
assert to_adhoc(input_1) == {
|
||||
assert simple_filter_to_adhoc(input_1) == simple_filter_to_adhoc(input_1)
|
||||
assert simple_filter_to_adhoc(input_1) == {
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"isExtra": False,
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "LATITUDE",
|
||||
"filterOptionName": "d0908f77d950131db7a69fdc820cb739",
|
||||
"filterOptionName": "6ac89d498115da22396f80a765cffc70",
|
||||
}
|
||||
|
||||
# The result is different when given different input
|
||||
assert to_adhoc(input_1) != to_adhoc(input_2)
|
||||
assert to_adhoc(input_2) == {
|
||||
assert simple_filter_to_adhoc(input_1) != simple_filter_to_adhoc(input_2)
|
||||
assert simple_filter_to_adhoc(input_2) == {
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"isExtra": False,
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "LONGITUDE",
|
||||
"filterOptionName": "c5f283f727d4dfc6258b351d4a8663bc",
|
||||
"filterOptionName": "9c984bd3714883ca859948354ce26ab9",
|
||||
}
|
||||
|
||||
|
||||
def test_form_data_to_adhoc_generates_deterministic_values():
|
||||
form_data = {"where": "1 = 1", "having": "count(*) > 1"}
|
||||
|
||||
# The result is the same when given the same input
|
||||
assert form_data_to_adhoc(form_data, "where") == form_data_to_adhoc(
|
||||
form_data, "where"
|
||||
)
|
||||
assert form_data_to_adhoc(form_data, "where") == {
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "1 = 1",
|
||||
"filterOptionName": "99fe79985afbddea4492626dc6a87b74",
|
||||
}
|
||||
|
||||
# The result is different when given different input
|
||||
assert form_data_to_adhoc(form_data, "having") == form_data_to_adhoc(
|
||||
form_data, "having"
|
||||
)
|
||||
assert form_data_to_adhoc(form_data, "having") == {
|
||||
"clause": "HAVING",
|
||||
"expressionType": "SQL",
|
||||
"sqlExpression": "count(*) > 1",
|
||||
"filterOptionName": "1da11f6b709c3190daeabb84f77fc8c2",
|
||||
}
|
||||
|
||||
|
||||
def test_form_data_to_adhoc_incorrect_clause_type():
|
||||
form_data = {"where": "1 = 1", "having": "count(*) > 1"}
|
||||
|
||||
with pytest.raises(ValueError):
|
||||
form_data_to_adhoc(form_data, "foobar")
|
||||
|
|
|
|||
|
|
@ -85,19 +85,6 @@ from tests.integration_tests.fixtures.world_bank_dashboard import (
|
|||
from .fixtures.certificates import ssl_certificate
|
||||
|
||||
|
||||
def mock_to_adhoc(filt, expressionType="SIMPLE", clause="where"):
|
||||
result = {"clause": clause.upper(), "expressionType": expressionType}
|
||||
|
||||
if expressionType == "SIMPLE":
|
||||
result.update(
|
||||
{"comparator": filt["val"], "operator": filt["op"], "subject": filt["col"]}
|
||||
)
|
||||
elif expressionType == "SQL":
|
||||
result.update({"sqlExpression": filt[clause]})
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class TestUtils(SupersetTestCase):
|
||||
def test_json_int_dttm_ser(self):
|
||||
dttm = datetime(2020, 1, 1)
|
||||
|
|
@ -137,7 +124,6 @@ class TestUtils(SupersetTestCase):
|
|||
got_str = zlib_decompress(blob)
|
||||
self.assertEqual(json_str, got_str)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters(self):
|
||||
# does nothing if no extra filters
|
||||
form_data = {"A": 1, "B": 2, "c": "test"}
|
||||
|
|
@ -168,6 +154,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
|
|
@ -175,6 +163,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "6c178d069965f1c02640661280415d96",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
|
|
@ -212,6 +202,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
|
|
@ -219,6 +211,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "6c178d069965f1c02640661280415d96",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
|
|
@ -244,6 +238,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": "hello",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "e3cbdd92a2ae23ca92c6d7fca42e36a6",
|
||||
"isExtra": True,
|
||||
"operator": "like",
|
||||
"subject": "A",
|
||||
}
|
||||
|
|
@ -264,7 +260,6 @@ class TestUtils(SupersetTestCase):
|
|||
merge_extra_filters(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters_ignores_empty_filters(self):
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
|
|
@ -276,7 +271,6 @@ class TestUtils(SupersetTestCase):
|
|||
merge_extra_filters(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters_ignores_nones(self):
|
||||
form_data = {
|
||||
"adhoc_filters": [
|
||||
|
|
@ -305,7 +299,6 @@ class TestUtils(SupersetTestCase):
|
|||
merge_extra_filters(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters_ignores_equal_filters(self):
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
|
|
@ -366,7 +359,6 @@ class TestUtils(SupersetTestCase):
|
|||
merge_extra_filters(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters_merges_different_val_types(self):
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
|
|
@ -410,6 +402,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "c11969c994b40a83a4ae7d48ff1ea28e",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
|
|
@ -460,6 +454,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "90cfb3c34852eb3bc741b0cc20053b46",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
|
|
@ -469,7 +465,6 @@ class TestUtils(SupersetTestCase):
|
|||
merge_extra_filters(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_merge_extra_filters_adds_unequal_lists(self):
|
||||
form_data = {
|
||||
"extra_filters": [
|
||||
|
|
@ -513,6 +508,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": ["g1", "g2", "g3"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "21cbb68af7b17e62b3b2f75e2190bfd7",
|
||||
"isExtra": True,
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
},
|
||||
|
|
@ -520,6 +517,8 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": ["c1", "c2", "c3"],
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "0a8dcb928f1f4bba97643c6e68d672f1",
|
||||
"isExtra": True,
|
||||
"operator": "==",
|
||||
"subject": "B",
|
||||
},
|
||||
|
|
@ -580,18 +579,21 @@ class TestUtils(SupersetTestCase):
|
|||
with self.assertRaises(SupersetException):
|
||||
validate_json(invalid)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_where(self):
|
||||
form_data = {"where": "a = 1"}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{"clause": "WHERE", "expressionType": "SQL", "sqlExpression": "a = 1"}
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SQL",
|
||||
"filterOptionName": "46fb6d7891e23596e42ae38da94a57e0",
|
||||
"sqlExpression": "a = 1",
|
||||
}
|
||||
]
|
||||
}
|
||||
convert_legacy_filters_into_adhoc(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_filters(self):
|
||||
form_data = {"filters": [{"col": "a", "op": "in", "val": "someval"}]}
|
||||
expected = {
|
||||
|
|
@ -600,6 +602,7 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "WHERE",
|
||||
"comparator": "someval",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "135c7ee246666b840a3d7a9c3a30cf38",
|
||||
"operator": "in",
|
||||
"subject": "a",
|
||||
}
|
||||
|
|
@ -608,7 +611,6 @@ class TestUtils(SupersetTestCase):
|
|||
convert_legacy_filters_into_adhoc(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_having(self):
|
||||
form_data = {"having": "COUNT(1) = 1"}
|
||||
expected = {
|
||||
|
|
@ -616,6 +618,7 @@ class TestUtils(SupersetTestCase):
|
|||
{
|
||||
"clause": "HAVING",
|
||||
"expressionType": "SQL",
|
||||
"filterOptionName": "683f1c26466ab912f75a00842e0f2f7b",
|
||||
"sqlExpression": "COUNT(1) = 1",
|
||||
}
|
||||
]
|
||||
|
|
@ -623,7 +626,6 @@ class TestUtils(SupersetTestCase):
|
|||
convert_legacy_filters_into_adhoc(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_having_filters(self):
|
||||
form_data = {"having_filters": [{"col": "COUNT(1)", "op": "==", "val": 1}]}
|
||||
expected = {
|
||||
|
|
@ -632,6 +634,7 @@ class TestUtils(SupersetTestCase):
|
|||
"clause": "HAVING",
|
||||
"comparator": 1,
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "967d0fb409f6d9c7a6c03a46cf933c9c",
|
||||
"operator": "==",
|
||||
"subject": "COUNT(1)",
|
||||
}
|
||||
|
|
@ -640,18 +643,21 @@ class TestUtils(SupersetTestCase):
|
|||
convert_legacy_filters_into_adhoc(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_present_and_empty(self):
|
||||
form_data = {"adhoc_filters": [], "where": "a = 1"}
|
||||
expected = {
|
||||
"adhoc_filters": [
|
||||
{"clause": "WHERE", "expressionType": "SQL", "sqlExpression": "a = 1"}
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SQL",
|
||||
"filterOptionName": "46fb6d7891e23596e42ae38da94a57e0",
|
||||
"sqlExpression": "a = 1",
|
||||
}
|
||||
]
|
||||
}
|
||||
convert_legacy_filters_into_adhoc(form_data)
|
||||
self.assertEqual(form_data, expected)
|
||||
|
||||
@patch("superset.utils.core.to_adhoc", mock_to_adhoc)
|
||||
def test_convert_legacy_filters_into_adhoc_present_and_nonempty(self):
|
||||
form_data = {
|
||||
"adhoc_filters": [
|
||||
|
|
|
|||
|
|
@ -1196,42 +1196,38 @@ class TestBaseDeckGLViz(SupersetTestCase):
|
|||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "bfa3a42a6f3de3c781b7d4f8e8d6613d",
|
||||
"filterOptionName": "c7f171cf3204bcbf456acfeac5cd9afd",
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "lat",
|
||||
"isExtra": False,
|
||||
},
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "2d35d87b57c6f1a5ae139f1a6b0cbd0a",
|
||||
"filterOptionName": "52634073fbb8ae0a3aa59ad48abac55e",
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "lon",
|
||||
"isExtra": False,
|
||||
},
|
||||
],
|
||||
"delimited_key": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "89cc0fafe39a4eabc5df2cd52e4d6514",
|
||||
"filterOptionName": "cae5c925c140593743da08499e6fb207",
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "lonlat",
|
||||
"isExtra": False,
|
||||
}
|
||||
],
|
||||
"geohash_key": [
|
||||
{
|
||||
"clause": "WHERE",
|
||||
"expressionType": "SIMPLE",
|
||||
"filterOptionName": "fa734d9a7bab254a53b41540d46cdb6c",
|
||||
"filterOptionName": "d84f55222d8e414e888fa5f990b341d2",
|
||||
"comparator": "",
|
||||
"operator": "IS NOT NULL",
|
||||
"subject": "geo",
|
||||
"isExtra": False,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in New Issue