iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat(time_comparison): Support all date formats when computing custom and inherit offsets (#30002)

This commit is contained in:
Antonio Rivero 2024-08-23 15:32:25 +02:00 committed by GitHub
parent ce72a0ac27
commit bc6d2dba37
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
10 changed files with 216 additions and 167 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
superset-frontend/packages/superset-ui-chart-controls/src/constants.ts
View File

@ -78,3 +78,5 @@ export const DEFAULT_XAXIS_SORT_SERIES_DATA: SortSeriesData = {
sort_series_type: SortSeriesType.Name,
sort_series_ascending: true,
};
export const DEFAULT_DATE_PATTERN = /\d{4}-\d{2}-\d{2}/g;

18
superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts
View File

@ -21,18 +21,19 @@ import { ensureIsArray } from '../utils';
import { customTimeRangeDecode } from './customTimeRangeDecode';
const DAY_IN_MS = 24 * 60 * 60 * 1000;
export const parseDttmToDate = (
dttm: string,
isEndDate = false,
computingShifts = false,
) => {
const now = new Date();
if (
dttm === 'now' ||
dttm === 'today' ||
dttm === 'No filter' ||
dttm === ''
) {
if (dttm === 'now' || dttm === 'No filter' || dttm === '') {
return now;
}
if (dttm === 'today') {
now.setHours(0, 0, 0, 0);
return now;
}
@ -280,9 +281,12 @@ export const getTimeOffset = ({
const customShift =
customStartDateTime &&
filterStartDateTime &&
Math.round((filterStartDateTime - customStartDateTime) / DAY_IN_MS);
const inInheritShift =
isInherit &&
filterEndDateTime &&
filterStartDateTime &&
Math.round((filterEndDateTime - filterStartDateTime) / DAY_IN_MS);
const newShifts = ensureIsArray(shifts)
@ -292,7 +296,7 @@ export const getTimeOffset = ({
if (includeFutureOffsets && customShift < 0) {
return `${customShift * -1} days after`;
}
if (customShift >= 0) {
if (customShift >= 0 && filterStartDateTime) {
return `${customShift} days ago`;
}
}

20
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx
View File

@ -26,7 +26,7 @@ import {
t,
useTheme,
} from '@superset-ui/core';
import { Tooltip } from '@superset-ui/chart-controls';
import { DEFAULT_DATE_PATTERN, Tooltip } from '@superset-ui/chart-controls';
import { isEmpty } from 'lodash';
import {
ColorSchemeEnum,
@ -90,21 +90,26 @@ export default function PopKPI(props: PopKPIProps) {
if (!currentTimeRangeFilter || (!shift && !startDateOffset)) {
setComparisonRange('');
} else if (!isEmpty(shift) || startDateOffset) {
const promise: any = fetchTimeRange(
dashboardTimeRange ?? (currentTimeRangeFilter as any).comparator,
currentTimeRangeFilter.subject,
);
Promise.resolve(promise).then((res: any) => {
const dates = res?.value?.match(DEFAULT_DATE_PATTERN);
const [parsedStartDate, parsedEndDate] = dates ?? [];
const newShift = getTimeOffset({
timeRangeFilter: {
...currentTimeRangeFilter,
comparator:
dashboardTimeRange ?? (currentTimeRangeFilter as any).comparator,
comparator: `${parsedStartDate} : ${parsedEndDate}`,
},
shifts: ensureIsArray(shift),
startDate: startDateOffset || '',
});
const promise: any = fetchTimeRange(
fetchTimeRange(
dashboardTimeRange ?? (currentTimeRangeFilter as any).comparator,
currentTimeRangeFilter.subject,
newShift || [],
);
Promise.resolve(promise).then((res: any) => {
ensureIsArray(newShift),
).then(res => {
const response: string[] = ensureIsArray(res.value);
const firstRange: string = response.flat()[0];
const rangeText = firstRange.split('vs\n');
@ -112,6 +117,7 @@ export default function PopKPI(props: PopKPIProps) {
rangeText.length > 1 ? rangeText[1].trim() : rangeText[0],
);
});
});
}
}, [currentTimeRangeFilter, shift, startDateOffset, dashboardTimeRange]);

56
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts
View File

@ -21,9 +21,6 @@ import {
QueryFormData,
PostProcessingRule,
ensureIsArray,
SimpleAdhocFilter,
getTimeOffset,
parseDttmToDate,
} from '@superset-ui/core';
import {
isTimeComparison,
@ -37,43 +34,30 @@ export default function buildQuery(formData: QueryFormData) {
const queryContextA = buildQueryContext(formData, baseQueryObject => {
const postProcessing: PostProcessingRule[] = [];
postProcessing.push(timeCompareOperator(formData, baseQueryObject));
const TimeRangeFilters =
formData.adhoc_filters?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
// In case the viz is using all version of controls, we try to load them
const previousCustomTimeRangeFilters: any =
formData.adhoc_custom?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
const nonCustomNorInheritShifts = ensureIsArray(
formData.time_compare,
).filter((shift: string) => shift !== 'custom' && shift !== 'inherit');
const customOrInheritShifts = ensureIsArray(formData.time_compare).filter(
(shift: string) => shift === 'custom' || shift === 'inherit',
);
let previousCustomStartDate = '';
if (
!isEmpty(previousCustomTimeRangeFilters) &&
previousCustomTimeRangeFilters[0]?.comparator !== 'No Filter'
) {
previousCustomStartDate =
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
let timeOffsets: string[] = [];
// Shifts for non-custom or non inherit time comparison
if (!isEmpty(nonCustomNorInheritShifts)) {
timeOffsets = nonCustomNorInheritShifts;
}
const timeOffsets = ensureIsArray(
isTimeComparison(formData, baseQueryObject)
? getTimeOffset({
timeRangeFilter: {
...TimeRangeFilters[0],
comparator:
baseQueryObject?.time_range ??
(TimeRangeFilters[0] as any)?.comparator,
},
shifts: formData.time_compare,
startDate:
previousCustomStartDate && !formData.start_date_offset
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: formData.start_date_offset,
})
: [],
);
// Shifts for custom or inherit time comparison
if (!isEmpty(customOrInheritShifts)) {
if (customOrInheritShifts.includes('custom')) {
timeOffsets = timeOffsets.concat([formData.start_date_offset]);
}
if (customOrInheritShifts.includes('inherit')) {
timeOffsets = timeOffsets.concat(['inherit']);
}
}
return [
{
...baseQueryObject,

34
superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts
View File

@ -24,10 +24,7 @@ import {
getNumberFormatter,
SimpleAdhocFilter,
ensureIsArray,
getTimeOffset,
parseDttmToDate,
} from '@superset-ui/core';
import { isEmpty } from 'lodash';
import { getComparisonFontSize, getHeaderFontSize } from './utils';
export const parseMetricValue = (metricValue: number | string | null) => {
@ -99,37 +96,16 @@ export default function transformProps(chartProps: ChartProps) {
(adhoc_filter: SimpleAdhocFilter) =>
adhoc_filter.operator === 'TEMPORAL_RANGE',
)?.[0];
// In case the viz is using all version of controls, we try to load them
const previousCustomTimeRangeFilters: any =
chartProps.rawFormData?.adhoc_custom?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
let previousCustomStartDate = '';
if (
!isEmpty(previousCustomTimeRangeFilters) &&
previousCustomTimeRangeFilters[0]?.comparator !== 'No Filter'
) {
previousCustomStartDate =
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
}
const isCustomOrInherit =
timeComparison === 'custom' || timeComparison === 'inherit';
let dataOffset: string[] = [];
if (isCustomOrInherit) {
dataOffset = getTimeOffset({
timeRangeFilter: {
...currentTimeRangeFilter,
comparator:
formData?.extraFormData?.time_range ??
(currentTimeRangeFilter as any)?.comparator,
},
shifts: ensureIsArray(timeComparison),
startDate:
previousCustomStartDate && !startDateOffset
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: startDateOffset,
});
if (timeComparison && timeComparison === 'custom') {
dataOffset = [startDateOffset];
} else {
dataOffset = ensureIsArray(timeComparison) || [];
}
}
const { value1, value2 } = data.reduce(

57
superset-frontend/plugins/plugin-chart-table/src/buildQuery.ts
View File

@ -21,13 +21,10 @@ import {
buildQueryContext,
ensureIsArray,
getMetricLabel,
getTimeOffset,
isPhysicalColumn,
parseDttmToDate,
QueryMode,
QueryObject,
removeDuplicates,
SimpleAdhocFilter,
} from '@superset-ui/core';
import { PostProcessingRule } from '@superset-ui/core/src/query/types/PostProcessing';
import { BuildQuery } from '@superset-ui/core/src/chart/registries/ChartBuildQueryRegistrySingleton';
@ -87,43 +84,35 @@ const buildQuery: BuildQuery<TableChartFormData> = (
let { metrics, orderby = [], columns = [] } = baseQueryObject;
const { extras = {} } = baseQueryObject;
let postProcessing: PostProcessingRule[] = [];
const TimeRangeFilters =
formData.adhoc_filters?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
const nonCustomNorInheritShifts = ensureIsArray(
formData.time_compare,
).filter((shift: string) => shift !== 'custom' && shift !== 'inherit');
const customOrInheritShifts = ensureIsArray(formData.time_compare).filter(
(shift: string) => shift === 'custom' || shift === 'inherit',
);
// In case the viz is using all version of controls, we try to load them
const previousCustomTimeRangeFilters: any =
formData.adhoc_custom?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
let timeOffsets: string[] = [];
let previousCustomStartDate = '';
// Shifts for non-custom or non inherit time comparison
if (
!isEmpty(previousCustomTimeRangeFilters) &&
previousCustomTimeRangeFilters[0]?.comparator !== 'No Filter'
isTimeComparison(formData, baseQueryObject) &&
!isEmpty(nonCustomNorInheritShifts)
) {
previousCustomStartDate =
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
timeOffsets = nonCustomNorInheritShifts;
}
const timeOffsets = ensureIsArray(
isTimeComparison(formData, baseQueryObject)
? getTimeOffset({
timeRangeFilter: {
...TimeRangeFilters[0],
comparator:
baseQueryObject?.time_range ??
(TimeRangeFilters[0] as any)?.comparator,
},
shifts: formData.time_compare,
startDate:
previousCustomStartDate && !formData.start_date_offset
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: formData.start_date_offset,
})
: [],
);
// Shifts for custom or inherit time comparison
if (
isTimeComparison(formData, baseQueryObject) &&
!isEmpty(customOrInheritShifts)
) {
if (customOrInheritShifts.includes('custom')) {
timeOffsets = timeOffsets.concat([formData.start_date_offset]);
}
if (customOrInheritShifts.includes('inherit')) {
timeOffsets = timeOffsets.concat(['inherit']);
}
}
let temporalColumAdded = false;
let temporalColum = null;

51
superset-frontend/plugins/plugin-chart-table/src/transformProps.ts
View File

@ -35,9 +35,6 @@ import {
SMART_DATE_ID,
TimeFormats,
TimeFormatter,
SimpleAdhocFilter,
getTimeOffset,
parseDttmToDate,
} from '@superset-ui/core';
import {
ColorFormatters,
@ -597,37 +594,29 @@ const transformProps = (
};
const timeGrain = extractTimegrain(formData);
const TimeRangeFilters =
chartProps.rawFormData?.adhoc_filters?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
const previousCustomTimeRangeFilters: any =
chartProps.rawFormData?.adhoc_custom?.filter(
(filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE',
) || [];
let previousCustomStartDate = '';
if (
!isEmpty(previousCustomTimeRangeFilters) &&
previousCustomTimeRangeFilters[0]?.comparator !== 'No Filter'
) {
previousCustomStartDate =
previousCustomTimeRangeFilters[0]?.comparator.split(' : ')[0];
const nonCustomNorInheritShifts = ensureIsArray(formData.time_compare).filter(
(shift: string) => shift !== 'custom' && shift !== 'inherit',
);
const customOrInheritShifts = ensureIsArray(formData.time_compare).filter(
(shift: string) => shift === 'custom' || shift === 'inherit',
);
let timeOffsets: string[] = [];
if (isUsingTimeComparison && !isEmpty(nonCustomNorInheritShifts)) {
timeOffsets = nonCustomNorInheritShifts;
}
const timeOffsets = getTimeOffset({
timeRangeFilter: {
...TimeRangeFilters[0],
comparator:
formData?.extra_form_data?.time_range ??
(TimeRangeFilters[0] as any)?.comparator,
},
shifts: formData.time_compare,
startDate:
previousCustomStartDate && !formData.start_date_offset
? parseDttmToDate(previousCustomStartDate)?.toUTCString()
: formData.start_date_offset,
});
// Shifts for custom or inherit time comparison
if (isUsingTimeComparison && !isEmpty(customOrInheritShifts)) {
if (customOrInheritShifts.includes('custom')) {
timeOffsets = timeOffsets.concat([formData.start_date_offset]);
}
if (customOrInheritShifts.includes('inherit')) {
timeOffsets = timeOffsets.concat(['inherit']);
}
}
const comparisonSuffix = isUsingTimeComparison
? ensureIsArray(timeOffsets)[0]
: '';

57
superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx
View File

@ -35,6 +35,7 @@ import ControlHeader, {
ControlHeaderProps,
} from 'src/explore/components/ControlHeader';
import { RootState } from 'src/views/store';
import { DEFAULT_DATE_PATTERN } from '@superset-ui/chart-controls';
const MOMENT_FORMAT = 'YYYY-MM-DD';
@ -108,20 +109,60 @@ export const ComparisonRangeLabel = ({
);
}
const promises = currentTimeRangeFilters.map(filter => {
const newShifts = getTimeOffset({
timeRangeFilter: filter,
shifts: shiftsArray,
startDate: useStartDate,
includeFutureOffsets: false, // So we don't trigger requests for future dates
});
const nonCustomNorInheritShifts =
shiftsArray.filter(
(shift: string) => shift !== 'custom' && shift !== 'inherit',
) || [];
const customOrInheritShifts =
shiftsArray.filter(
(shift: string) => shift === 'custom' || shift === 'inherit',
) || [];
if (!isEmpty(newShifts)) {
// There's no custom or inherit to compute, so we can just fetch the time range
if (isEmpty(customOrInheritShifts)) {
return fetchTimeRange(
filter.comparator,
filter.subject,
ensureIsArray(newShifts),
ensureIsArray(nonCustomNorInheritShifts),
);
}
// Need to compute custom or inherit shifts first and then mix with the non custom or inherit shifts
if (
(ensureIsArray(customOrInheritShifts).includes('custom') &&
startDate) ||
ensureIsArray(customOrInheritShifts).includes('inherit')
) {
return fetchTimeRange(filter.comparator, filter.subject).then(res => {
const dates = res?.value?.match(DEFAULT_DATE_PATTERN);
const [parsedStartDate, parsedEndDate] = dates ?? [];
if (parsedStartDate) {
const parsedDateMoment = moment(parseDttmToDate(parsedStartDate));
const startDateMoment = moment(parseDttmToDate(startDate));
if (
startDateMoment.isSameOrBefore(parsedDateMoment) ||
!startDate
) {
const postProcessedShifts = getTimeOffset({
timeRangeFilter: {
...filter,
comparator: `${parsedStartDate} : ${parsedEndDate}`,
},
shifts: customOrInheritShifts,
startDate: useStartDate,
includeFutureOffsets: false, // So we don't trigger requests for future dates
});
return fetchTimeRange(
filter.comparator,
filter.subject,
ensureIsArray(
postProcessedShifts.concat(nonCustomNorInheritShifts),
),
);
}
}
return Promise.resolve({ value: '' });
});
}
return Promise.resolve({ value: '' });
});
Promise.all(promises).then(res => {

14
superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx
View File

@ -26,6 +26,7 @@ import {
css,
customTimeRangeDecode,
computeCustomDateTime,
fetchTimeRange,
} from '@superset-ui/core';
import { DatePicker } from 'antd';
import { RangePickerProps } from 'antd/lib/date-picker';
@ -33,6 +34,7 @@ import { useSelector } from 'react-redux';
import ControlHeader from 'src/explore/components/ControlHeader';
import { RootState } from 'src/views/store';
import { DEFAULT_DATE_PATTERN } from '@superset-ui/chart-controls';
export interface TimeOffsetControlsProps {
label?: ReactNode;
@ -130,9 +132,15 @@ export default function TimeOffsetControls({
useEffect(() => {
if (!isEmpty(currentTimeRangeFilters)) {
customTimeRange(currentTimeRangeFilters[0]?.comparator ?? '');
const date = currentTimeRangeFilters[0]?.comparator.split(' : ')[0];
setFormatedFilterDate(moment(parseDttmToDate(date)));
fetchTimeRange(
currentTimeRangeFilters[0]?.comparator,
currentTimeRangeFilters[0]?.subject,
).then(res => {
const dates = res?.value?.match(DEFAULT_DATE_PATTERN);
const [startDate, endDate] = dates ?? [];
customTimeRange(`${startDate} : ${endDate}` ?? '');
setFormatedFilterDate(moment(parseDttmToDate(startDate)));
});
} else {
setCustomStartDateInFilter(undefined);
setFormatedFilterDate(moment(parseDttmToDate('')));

54
superset/common/query_context_processor.py
View File

@ -19,6 +19,7 @@ from __future__ import annotations
import copy
import logging
import re
from datetime import datetime
from typing import Any, cast, ClassVar, TYPE_CHECKING, TypedDict
import numpy as np
@ -369,6 +370,38 @@ class QueryContextProcessor:
axis=1,
)
def is_valid_date(self, date_string: str) -> bool:
try:
# Attempt to parse the string as a date in the format YYYY-MM-DD
datetime.strptime(date_string, "%Y-%m-%d")
return True
except ValueError:
# If parsing fails, it's not a valid date in the format YYYY-MM-DD
return False
def get_offset_custom_or_inherit(
self,
offset: str,
outer_from_dttm: datetime,
outer_to_dttm: datetime,
) -> str:
"""
Get the time offset for custom or inherit.
:param offset: The offset string.
:param outer_from_dttm: The outer from datetime.
:param outer_to_dttm: The outer to datetime.
:returns: The time offset.
"""
if offset == "inherit":
# return the difference in days between the from and the to dttm formatted as a string with the " days ago" suffix
return f"{(outer_to_dttm - outer_from_dttm).days} days ago"
if self.is_valid_date(offset):
# return the offset as the difference in days between the outer from dttm and the offset date (which is a YYYY-MM-DD string) formatted as a string with the " days ago" suffix
offset_date = datetime.strptime(offset, "%Y-%m-%d")
return f"{(outer_from_dttm - offset_date).days} days ago"
return ""
def processing_time_offsets( # pylint: disable=too-many-locals,too-many-statements
self,
df: pd.DataFrame,
@ -409,6 +442,13 @@ class QueryContextProcessor:
# time_offsets: ['1 year ago'],
# filters: [{col: 'dttm_col', op: 'TEMPORAL_RANGE', val: '2020 : 2021'}],
# }
original_offset = offset
if self.is_valid_date(offset) or offset == "inherit":
offset = self.get_offset_custom_or_inherit(
offset,
outer_from_dttm,
outer_to_dttm,
)
query_object_clone.from_dttm = get_past_or_future(
offset,
outer_from_dttm,
@ -454,9 +494,19 @@ class QueryContextProcessor:
if flt.get("col") != x_axis_label
]
# Inherit or custom start dates might compute the same offset but the response cannot be given
# using cached data unless you are using the same date of inherited range, that's why we
# set the cache cache using a custom key that includes the original offset and the computed offset
# for those two scenarios, the rest of the scenarios will use the original offset as cache key
cached_time_offset_key = (
offset if offset == original_offset else f"{offset}_{original_offset}"
)
# `offset` is added to the hash function
cache_key = self.query_cache_key(
query_object_clone, time_offset=offset, time_grain=time_grain
query_object_clone,
time_offset=cached_time_offset_key,
time_grain=time_grain,
)
cache = QueryCacheManager.get(
cache_key, CacheRegion.DATA, query_context.force
@ -471,7 +521,7 @@ class QueryContextProcessor:
query_object_clone_dct = query_object_clone.to_dict()
# rename metrics: SUM(value) => SUM(value) 1 year ago
metrics_mapping = {
metric: TIME_COMPARISON.join([metric, offset])
metric: TIME_COMPARISON.join([metric, original_offset])
for metric in metric_names
}