diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/sections/index.ts b/superset-frontend/packages/superset-ui-chart-controls/src/sections/index.ts index c0113b189..caa07faa9 100644 --- a/superset-frontend/packages/superset-ui-chart-controls/src/sections/index.ts +++ b/superset-frontend/packages/superset-ui-chart-controls/src/sections/index.ts @@ -23,3 +23,4 @@ export * from './annotationsAndLayers'; export * from './forecastInterval'; export * from './chartTitle'; export * from './echartsTimeSeriesQuery'; +export * from './timeComparison'; diff --git a/superset-frontend/packages/superset-ui-chart-controls/src/sections/timeComparison.tsx b/superset-frontend/packages/superset-ui-chart-controls/src/sections/timeComparison.tsx new file mode 100644 index 000000000..86c7eb9f9 --- /dev/null +++ b/superset-frontend/packages/superset-ui-chart-controls/src/sections/timeComparison.tsx @@ -0,0 +1,136 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { t, ComparisonType } from '@superset-ui/core'; + +import { ControlPanelSectionConfig } from '../types'; + +const fullChoices = [ + ['1 day ago', t('1 day ago')], + ['1 week ago', t('1 week ago')], + ['28 days ago', t('28 days ago')], + ['30 days ago', t('30 days ago')], + ['1 month ago', t('1 month ago')], + ['52 weeks ago', t('52 weeks ago')], + ['1 year ago', t('1 year ago')], + ['104 weeks ago', t('104 weeks ago')], + ['2 years ago', t('2 years ago')], + ['156 weeks ago', t('156 weeks ago')], + ['3 years ago', t('3 years ago')], + ['custom', t('Custom date')], + ['inherit', t('Inherit range from time filter')], +]; + +const reducedKeys = new Set([ + '1 day ago', + '1 week ago', + '1 month ago', + '1 year ago', + 'custom', + 'inherit', +]); + +// Filter fullChoices to get only the entries whose keys are in reducedKeys +const reducedChoices = fullChoices.filter(choice => reducedKeys.has(choice[0])); + +type TimeComparisonControlsType = { + multi?: boolean; + showCalculationType?: boolean; + showFullChoices?: boolean; +}; +export const timeComparisonControls: ({ + multi, + showCalculationType, + showFullChoices, +}: TimeComparisonControlsType) => ControlPanelSectionConfig = ({ + multi = true, + showCalculationType = true, + showFullChoices = true, +}) => ({ + label: t('Time Comparison'), + tabOverride: 'data', + description: t('Compare results with other time periods.'), + controlSetRows: [ + [ + { + name: 'time_compare', + config: { + type: 'SelectControl', + multi, + freeForm: true, + placeholder: t('Select or type a custom value...'), + label: t('Time shift'), + choices: showFullChoices ? fullChoices : reducedChoices, + description: t( + 'Overlay results from a relative time period. ' + + 'Expects relative time deltas ' + + 'in natural language (example: 24 hours, 7 days, ' + + '52 weeks, 365 days). Free text is supported. ' + + 'Use "Inherit range from time filters" ' + + 'to shift the comparison time range ' + + 'by the same length as your time range ' + + 'and use "Custom" to set a custom comparison range.', + ), + }, + }, + ], + [ + { + name: 'start_date_offset', + config: { + type: 'TimeOffsetControl', + label: t('shift start date'), + visibility: ({ controls }) => + controls?.time_compare.value === 'custom', + }, + }, + ], + [ + { + name: 'comparison_type', + config: { + type: 'SelectControl', + label: t('Calculation type'), + default: 'values', + choices: [ + [ComparisonType.Values, t('Actual values')], + [ComparisonType.Difference, t('Difference')], + [ComparisonType.Percentage, t('Percentage change')], + [ComparisonType.Ratio, t('Ratio')], + ], + description: t( + 'How to display time shifts: as individual lines; as the ' + + 'difference between the main time series and each time shift; ' + + 'as the percentage change; or as the ratio between series and time shifts.', + ), + visibility: () => Boolean(showCalculationType), + }, + }, + ], + [ + { + name: 'comparison_range_label', + config: { + type: 'ComparisonRangeLabel', + multi, + visibility: ({ controls }) => Boolean(controls?.time_compare.value), + }, + }, + ], + ], +}); diff --git a/superset-frontend/packages/superset-ui-core/src/time-comparison/fetchTimeRange.ts b/superset-frontend/packages/superset-ui-core/src/time-comparison/fetchTimeRange.ts index 50509af52..61c2a2f8a 100644 --- a/superset-frontend/packages/superset-ui-core/src/time-comparison/fetchTimeRange.ts +++ b/superset-frontend/packages/superset-ui-core/src/time-comparison/fetchTimeRange.ts @@ -17,7 +17,12 @@ * under the License. */ import rison from 'rison'; -import { SupersetClient, getClientErrorObject } from '@superset-ui/core'; +import { isEmpty } from 'lodash'; +import { + SupersetClient, + getClientErrorObject, + ensureIsArray, +} from '@superset-ui/core'; export const SEPARATOR = ' : '; @@ -39,20 +44,64 @@ export const formatTimeRange = ( )} ≤ ${columnPlaceholder} < ${formatDateEndpoint(splitDateRange[1])}`; }; +export const formatTimeRangeComparison = ( + initialTimeRange: string, + shiftedTimeRange: string, + columnPlaceholder = 'col', +) => { + const splitInitialDateRange = initialTimeRange.split(SEPARATOR); + const splitShiftedDateRange = shiftedTimeRange.split(SEPARATOR); + return `${columnPlaceholder}: ${formatDateEndpoint( + splitInitialDateRange[0], + true, + )} to ${formatDateEndpoint(splitInitialDateRange[1])} vs + ${formatDateEndpoint(splitShiftedDateRange[0], true)} to ${formatDateEndpoint( + splitShiftedDateRange[1], + )}`; +}; + export const fetchTimeRange = async ( timeRange: string, columnPlaceholder = 'col', + shifts?: string[], ) => { - const query = rison.encode_uri(timeRange); - const endpoint = `/api/v1/time_range/?q=${query}`; + let query; + let endpoint; + if (!isEmpty(shifts)) { + const timeRanges = ensureIsArray(shifts).map(shift => ({ + timeRange, + shift, + })); + query = rison.encode_uri([{ timeRange }, ...timeRanges]); + endpoint = `/api/v1/time_range/?q=${query}`; + } else { + query = rison.encode_uri(timeRange); + endpoint = `/api/v1/time_range/?q=${query}`; + } try { const response = await SupersetClient.get({ endpoint }); - const timeRangeString = buildTimeRangeString( - response?.json?.result[0]?.since || '', - response?.json?.result[0]?.until || '', + if (isEmpty(shifts)) { + const timeRangeString = buildTimeRangeString( + response?.json?.result[0]?.since || '', + response?.json?.result[0]?.until || '', + ); + return { + value: formatTimeRange(timeRangeString, columnPlaceholder), + }; + } + const timeRanges = response?.json?.result.map((result: any) => + buildTimeRangeString(result.since, result.until), ); return { - value: formatTimeRange(timeRangeString, columnPlaceholder), + value: timeRanges + .slice(1) + .map((timeRange: string) => + formatTimeRangeComparison( + timeRanges[0], + timeRange, + columnPlaceholder, + ), + ), }; } catch (response) { const clientError = await getClientErrorObject(response); diff --git a/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts b/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts new file mode 100644 index 000000000..aea7c6bd4 --- /dev/null +++ b/superset-frontend/packages/superset-ui-core/src/time-comparison/getTimeOffset.ts @@ -0,0 +1,121 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { ensureIsArray } from '../utils'; + +const DAY_IN_MS = 24 * 60 * 60 * 1000; +export const parseDttmToDate = (dttm: string): Date => { + const now = new Date(); + now.setUTCHours(0, 0, 0, 0); + + if (dttm === 'now' || dttm === 'today' || dttm === 'No filter') { + return now; + } + if (dttm === 'Last week') { + now.setUTCDate(now.getUTCDate() - 7); + return now; + } + if (dttm === 'Last month') { + now.setUTCMonth(now.getUTCMonth() - 1); + now.setUTCDate(1); + return now; + } + if (dttm === 'Last quarter') { + now.setUTCMonth(now.getUTCMonth() - 3); + now.setUTCDate(1); + return now; + } + if (dttm === 'Last year') { + now.setUTCFullYear(now.getUTCFullYear() - 1); + now.setUTCDate(1); + return now; + } + if (dttm === 'previous calendar week') { + now.setUTCDate(now.getUTCDate() - now.getUTCDay()); + return now; + } + if (dttm === 'previous calendar month') { + now.setUTCMonth(now.getUTCMonth() - 1, 1); + return now; + } + if (dttm === 'previous calendar year') { + now.setUTCFullYear(now.getUTCFullYear() - 1, 0, 1); + return now; + } + if (dttm?.includes('ago')) { + const parts = dttm.split(' '); + const amount = parseInt(parts[0], 10); + const unit = parts[1]; + + switch (unit) { + case 'day': + case 'days': + now.setUTCDate(now.getUTCDate() - amount); + break; + case 'week': + case 'weeks': + now.setUTCDate(now.getUTCDate() - amount * 7); + break; + case 'month': + case 'months': + now.setUTCMonth(now.getUTCMonth() - amount); + break; + case 'year': + case 'years': + now.setUTCFullYear(now.getUTCFullYear() - amount); + break; + default: + break; + } + return now; + } + const parsed = new Date(dttm); + parsed.setUTCHours(0, 0, 0, 0); + return parsed; +}; + +export const getTimeOffset = ( + timeRangeFilter: any, + shifts: string[], + startDate: string, +): string[] => { + const isCustom = shifts?.includes('custom'); + const isInherit = shifts?.includes('inherit'); + const customStartDate = isCustom && parseDttmToDate(startDate).getTime(); + const filterStartDate = parseDttmToDate( + timeRangeFilter.comparator.split(' : ')[0], + ).getTime(); + const filterEndDate = parseDttmToDate( + timeRangeFilter.comparator.split(' : ')[1], + ).getTime(); + + const customShift = + customStartDate && + Math.ceil((filterStartDate - customStartDate) / DAY_IN_MS); + const inInheritShift = + isInherit && Math.ceil((filterEndDate - filterStartDate) / DAY_IN_MS); + + let newShifts = shifts; + if (isCustom) { + newShifts = [`${customShift} days ago`]; + } + if (isInherit) { + newShifts = [`${inInheritShift} days ago`]; + } + return ensureIsArray(newShifts); +}; diff --git a/superset-frontend/packages/superset-ui-core/src/time-comparison/index.ts b/superset-frontend/packages/superset-ui-core/src/time-comparison/index.ts index ad5b5f591..7856d344f 100644 --- a/superset-frontend/packages/superset-ui-core/src/time-comparison/index.ts +++ b/superset-frontend/packages/superset-ui-core/src/time-comparison/index.ts @@ -21,4 +21,5 @@ export * from './types'; export { default as getComparisonInfo } from './getComparisonInfo'; export { default as getComparisonFilters } from './getComparisonFilters'; +export { parseDttmToDate, getTimeOffset } from './getTimeOffset'; export { SEPARATOR, fetchTimeRange } from './fetchTimeRange'; diff --git a/superset-frontend/packages/superset-ui-core/test/time-comparison/fetchTimeRange.test.ts b/superset-frontend/packages/superset-ui-core/test/time-comparison/fetchTimeRange.test.ts index e07fa8617..964761f6b 100644 --- a/superset-frontend/packages/superset-ui-core/test/time-comparison/fetchTimeRange.test.ts +++ b/superset-frontend/packages/superset-ui-core/test/time-comparison/fetchTimeRange.test.ts @@ -22,11 +22,12 @@ import { fetchTimeRange } from '@superset-ui/core'; import { buildTimeRangeString, formatTimeRange, + formatTimeRangeComparison, } from '../../src/time-comparison/fetchTimeRange'; afterEach(fetchMock.restore); -it('generates proper time range string', () => { +test('generates proper time range string', () => { expect( buildTimeRangeString('2010-07-30T00:00:00', '2020-07-30T00:00:00'), ).toBe('2010-07-30T00:00:00 : 2020-07-30T00:00:00'); @@ -36,7 +37,7 @@ it('generates proper time range string', () => { expect(buildTimeRangeString('', '')).toBe(' : '); }); -it('generates a readable time range', () => { +test('generates a readable time range', () => { expect(formatTimeRange('Last 7 days')).toBe('Last 7 days'); expect(formatTimeRange('No filter')).toBe('No filter'); expect(formatTimeRange('Yesterday : Tomorrow')).toBe( @@ -53,7 +54,7 @@ it('generates a readable time range', () => { ); }); -it('returns a formatted time range from response', async () => { +test('returns a formatted time range from response', async () => { fetchMock.get("glob:*/api/v1/time_range/?q='Last+day'", { result: [ { @@ -70,7 +71,7 @@ it('returns a formatted time range from response', async () => { }); }); -it('returns a formatted time range from empty response', async () => { +test('returns a formatted time range from empty response', async () => { fetchMock.get("glob:*/api/v1/time_range/?q='Last+day'", { result: [], }); @@ -81,7 +82,7 @@ it('returns a formatted time range from empty response', async () => { }); }); -it('returns a formatted error message from response', async () => { +test('returns a formatted error message from response', async () => { fetchMock.getOnce("glob:*/api/v1/time_range/?q='Last+day'", { throws: new Response(JSON.stringify({ message: 'Network error' })), }); @@ -116,3 +117,54 @@ it('returns a formatted error message from response', async () => { error: 'Network error', }); }); + +test('fetchTimeRange with shift', async () => { + fetchMock.getOnce( + "glob:*/api/v1/time_range/?q=!((timeRange:'Last+day'),(shift%3A'last%20month'%2CtimeRange%3A'Last%20day'))", + { + result: [ + { + since: '2021-04-13T00:00:00', + until: '2021-04-14T00:00:00', + timeRange: 'Last day', + shift: null, + }, + { + since: '2021-03-13T00:00:00', + until: '2021-03-14T00:00:00', + timeRange: 'Last day', + shift: 'last month', + }, + ], + }, + ); + + const timeRange = await fetchTimeRange('Last day', 'temporal_col', [ + 'last month', + ]); + + expect(timeRange).toEqual({ + value: [ + 'temporal_col: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14', + ], + }); +}); + +test('formatTimeRangeComparison', () => { + expect( + formatTimeRangeComparison( + '2021-04-13T00:00:00 : 2021-04-14T00:00:00', + '2021-03-13T00:00:00 : 2021-03-14T00:00:00', + ), + ).toEqual('col: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14'); + + expect( + formatTimeRangeComparison( + '2021-04-13T00:00:00 : 2021-04-14T00:00:00', + '2021-03-13T00:00:00 : 2021-03-14T00:00:00', + 'col_name', + ), + ).toEqual( + 'col_name: 2021-04-13 to 2021-04-14 vs\n 2021-03-13 to 2021-03-14', + ); +}); diff --git a/superset-frontend/packages/superset-ui-core/test/time-comparison/getTimeOffset.test.ts b/superset-frontend/packages/superset-ui-core/test/time-comparison/getTimeOffset.test.ts new file mode 100644 index 000000000..9b14e0062 --- /dev/null +++ b/superset-frontend/packages/superset-ui-core/test/time-comparison/getTimeOffset.test.ts @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { getTimeOffset } from '@superset-ui/core'; + +test('handles custom shifts', () => { + const shifts = ['custom']; + const startDate = '2023-01-01'; + const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' }; + + const result = getTimeOffset(timeRangeFilter, shifts, startDate); + expect(result).toEqual(['2 days ago']); +}); + +test('handles inherit shifts', () => { + const shifts = ['inherit']; + const startDate = ''; + const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' }; + + const result = getTimeOffset(timeRangeFilter, shifts, startDate); + expect(result).toEqual(['7 days ago']); +}); + +test('handles no custom or inherit shifts', () => { + const shifts = ['1 week ago']; + const startDate = ''; + const timeRangeFilter = { comparator: '2023-01-03 : 2023-01-10' }; + + const result = getTimeOffset(timeRangeFilter, shifts, startDate); + expect(result).toEqual(['1 week ago']); +}); diff --git a/superset-frontend/packages/superset-ui-core/test/time-comparison/parseDttmToDate.test.ts b/superset-frontend/packages/superset-ui-core/test/time-comparison/parseDttmToDate.test.ts new file mode 100644 index 000000000..30c5d285b --- /dev/null +++ b/superset-frontend/packages/superset-ui-core/test/time-comparison/parseDttmToDate.test.ts @@ -0,0 +1,131 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import { parseDttmToDate } from '@superset-ui/core'; + +test('should handle "now"', () => { + const now = parseDttmToDate('now'); + const expected = new Date(); + expected.setUTCHours(0, 0, 0, 0); + expect(expected).toEqual(now); +}); + +test('should handle "today" and "No filter"', () => { + const today = parseDttmToDate('today'); + const noFilter = parseDttmToDate('No filter'); + const expected = new Date(); + expected.setUTCHours(0, 0, 0, 0); + expect(today).toEqual(expected); + expect(noFilter).toEqual(expected); +}); + +test('should handle relative time strings', () => { + const lastWeek = parseDttmToDate('Last week'); + const lastMonth = parseDttmToDate('Last month'); + const lastQuarter = parseDttmToDate('Last quarter'); + const lastYear = parseDttmToDate('Last year'); + let now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCDate(now.getUTCDate() - 7); + expect(lastWeek).toEqual(now); + + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCMonth(now.getUTCMonth() - 1); + now.setUTCDate(1); + expect(lastMonth).toEqual(now); + + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCMonth(now.getUTCMonth() - 3); + now.setUTCDate(1); + expect(lastQuarter).toEqual(now); + + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCFullYear(now.getUTCFullYear() - 1); + now.setUTCDate(1); + expect(lastYear).toEqual(now); +}); + +test('should handle previous calendar units', () => { + let now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCDate(now.getUTCDate() - now.getUTCDay()); + const previousWeek = parseDttmToDate('previous calendar week'); + expect(previousWeek).toEqual(now); + + now = new Date(); + now.setUTCMonth(now.getUTCMonth() - 1, 1); + now.setUTCHours(0, 0, 0, 0); + const previousMonth = parseDttmToDate('previous calendar month'); + expect(previousMonth).toEqual(now); + + now = new Date(); + now.setUTCFullYear(now.getUTCFullYear() - 1, 0, 1); + now.setUTCHours(0, 0, 0, 0); + const previousYear = parseDttmToDate('previous calendar year'); + expect(previousYear).toEqual(now); +}); + +test('should handle dynamic "ago" times', () => { + const fiveDaysAgo = parseDttmToDate('5 days ago'); + const fiveDayAgo = parseDttmToDate('5 day ago'); + let now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCDate(now.getUTCDate() - 5); + expect(fiveDaysAgo).toEqual(now); + expect(fiveDayAgo).toEqual(now); + + const weeksAgo = parseDttmToDate('7 weeks ago'); + const weekAgo = parseDttmToDate('7 week ago'); + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCDate(now.getUTCDate() - 7 * 7); + expect(weeksAgo).toEqual(now); + expect(weekAgo).toEqual(now); + + const fiveMonthsAgo = parseDttmToDate('5 months ago'); + const fiveMonthAgo = parseDttmToDate('5 month ago'); + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCMonth(now.getUTCMonth() - 5); + expect(fiveMonthsAgo).toEqual(now); + expect(fiveMonthAgo).toEqual(now); + + const fiveYearsAgo = parseDttmToDate('5 years ago'); + const fiveYearAgo = parseDttmToDate('5 year ago'); + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + now.setUTCFullYear(now.getUTCFullYear() - 5); + expect(fiveYearsAgo).toEqual(now); + expect(fiveYearAgo).toEqual(now); + + // default case + const fiveHoursAgo = parseDttmToDate('5 hours ago'); + now = new Date(); + now.setUTCHours(0, 0, 0, 0); + expect(fiveHoursAgo).toEqual(now); +}); + +test('should parse valid moment strings', () => { + const specificDate = new Date('2023-01-01'); + specificDate.setUTCHours(0, 0, 0, 0); + const parsedDate = parseDttmToDate('2023-01-01'); + expect(parsedDate).toEqual(specificDate); +}); diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx index b0d9d912d..5292432fd 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/PopKPI.tsx @@ -16,9 +16,18 @@ * specific language governing permissions and limitations * under the License. */ -import React, { useMemo } from 'react'; -import { css, styled, t, useTheme } from '@superset-ui/core'; +import React, { useEffect, useMemo, useState } from 'react'; +import { + css, + ensureIsArray, + fetchTimeRange, + getTimeOffset, + styled, + t, + useTheme, +} from '@superset-ui/core'; import { Tooltip } from '@superset-ui/chart-controls'; +import { isEmpty } from 'lodash'; import { ColorSchemeEnum, PopKPIComparisonSymbolStyleProps, @@ -69,9 +78,38 @@ export default function PopKPI(props: PopKPIProps) { comparisonColorEnabled, comparisonColorScheme, percentDifferenceNumber, - comparatorText, + currentTimeRangeFilter, + startDateOffset, + shift, } = props; + const [comparisonRange, setComparisonRange] = useState(''); + + useEffect(() => { + if (!currentTimeRangeFilter || (!shift && !startDateOffset)) { + setComparisonRange(''); + } else if (!isEmpty(shift) || startDateOffset) { + const newShift = getTimeOffset( + currentTimeRangeFilter, + ensureIsArray(shift), + startDateOffset || '', + ); + const promise: any = fetchTimeRange( + (currentTimeRangeFilter as any).comparator, + currentTimeRangeFilter.subject, + newShift || [], + ); + Promise.resolve(promise).then((res: any) => { + const response: string[] = ensureIsArray(res.value); + const firstRange: string = response.flat()[0]; + const rangeText = firstRange.split('vs\n'); + setComparisonRange( + rangeText.length > 1 ? rangeText[1].trim() : rangeText[0], + ); + }); + } + }, [currentTimeRangeFilter, shift, startDateOffset]); + const theme = useTheme(); const flexGap = theme.gridUnit * 5; const wrapperDivStyles = css` @@ -150,7 +188,7 @@ export default function PopKPI(props: PopKPIProps) { { symbol: '#', value: prevNumber, - tooltipText: t('Data for %s', comparatorText), + tooltipText: t('Data for %s', comparisonRange || 'previous range'), }, { symbol: '△', @@ -164,7 +202,7 @@ export default function PopKPI(props: PopKPIProps) { }, ], [ - comparatorText, + comparisonRange, prevNumber, valueDifference, percentDifferenceFormattedString, diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts index abf911040..f9b4725cb 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/buildQuery.ts @@ -18,50 +18,50 @@ */ import { buildQueryContext, - getComparisonInfo, - ComparisonTimeRangeType, QueryFormData, + PostProcessingRule, + ensureIsArray, + SimpleAdhocFilter, + getTimeOffset, } from '@superset-ui/core'; +import { + isTimeComparison, + timeCompareOperator, +} from '@superset-ui/chart-controls'; export default function buildQuery(formData: QueryFormData) { - const { - cols: groupby, - time_comparison: timeComparison, - extra_form_data: extraFormData, - } = formData; + const { cols: groupby } = formData; - const queryContextA = buildQueryContext(formData, baseQueryObject => [ - { - ...baseQueryObject, - groupby, - }, - ]); + const queryContextA = buildQueryContext(formData, baseQueryObject => { + const postProcessing: PostProcessingRule[] = []; + postProcessing.push(timeCompareOperator(formData, baseQueryObject)); + const TimeRangeFilters = + formData.adhoc_filters?.filter( + (filter: SimpleAdhocFilter) => filter.operator === 'TEMPORAL_RANGE', + ) || []; - const comparisonFormData = getComparisonInfo( - formData, - timeComparison, - extraFormData, - ); - - const queryContextB = buildQueryContext( - comparisonFormData, - baseQueryObject => [ + const timeOffsets = ensureIsArray( + isTimeComparison(formData, baseQueryObject) + ? getTimeOffset( + TimeRangeFilters[0], + formData.time_compare, + formData.start_date_offset, + ) + : [], + ); + return [ { ...baseQueryObject, groupby, - extras: { - ...baseQueryObject.extras, - instant_time_comparison_range: - timeComparison !== ComparisonTimeRangeType.Custom - ? timeComparison - : undefined, - }, + post_processing: postProcessing, + time_offsets: isTimeComparison(formData, baseQueryObject) + ? ensureIsArray(timeOffsets) + : [], }, - ], - ); + ]; + }); return { ...queryContextA, - queries: [...queryContextA.queries, ...queryContextB.queries], }; } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/controlPanel.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/controlPanel.ts index f6f81d98d..ce934c433 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/controlPanel.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/controlPanel.ts @@ -16,20 +16,12 @@ * specific language governing permissions and limitations * under the License. */ +import { t } from '@superset-ui/core'; import { - AdhocFilter, - ComparisonTimeRangeType, - SimpleAdhocFilter, - t, - validateTimeComparisonRangeValues, -} from '@superset-ui/core'; -import { - ColumnMeta, ControlPanelConfig, - ControlPanelState, - ControlState, getStandardizedControls, sharedControls, + sections, } from '@superset-ui/chart-controls'; import { headerFontSize, subheaderFontSize } from '../sharedControls'; import { ColorSchemeEnum } from './types'; @@ -42,70 +34,6 @@ const config: ControlPanelConfig = { controlSetRows: [ ['metric'], ['adhoc_filters'], - [ - { - name: 'time_comparison', - config: { - type: 'SelectControl', - label: t('Range for Comparison'), - default: 'r', - choices: [ - ['r', 'Inherit range from time filters'], - ['y', 'Year'], - ['m', 'Month'], - ['w', 'Week'], - ['c', 'Custom'], - ], - rerender: ['adhoc_custom'], - description: t( - 'Set the time range that will be used for the comparison metrics. ' + - 'For example, "Year" will compare to the same dates one year earlier. ' + - 'Use "Inherit range from time filters" to shift the comparison time range' + - 'by the same length as your time range and use "Custom" to set a custom comparison range.', - ), - }, - }, - ], - [ - { - name: `adhoc_custom`, - config: { - ...sharedControls.adhoc_filters, - label: t('Filters for Comparison'), - description: - 'This only applies when selecting the Range for Comparison Type: Custom', - visibility: ({ controls }) => - controls?.time_comparison?.value === - ComparisonTimeRangeType.Custom, - mapStateToProps: ( - state: ControlPanelState, - controlState: ControlState, - ) => { - const originalMapStateToPropsRes = - sharedControls.adhoc_filters.mapStateToProps?.( - state, - controlState, - ) || {}; - const columns = originalMapStateToPropsRes.columns.filter( - (col: ColumnMeta) => - col.is_dttm && - (state.controls.adhoc_filters.value as AdhocFilter[]).some( - (val: SimpleAdhocFilter) => - val.subject === col.column_name, - ), - ); - return { - ...originalMapStateToPropsRes, - columns, - externalValidationErrors: validateTimeComparisonRangeValues( - state.controls?.time_comparison?.value, - controlState.value, - ), - }; - }, - }, - }, - ], [ { name: 'row_limit', @@ -180,14 +108,16 @@ const config: ControlPanelConfig = { ], ], }, + sections.timeComparisonControls({ + multi: false, + showCalculationType: false, + showFullChoices: false, + }), ], controlOverrides: { y_axis_format: { label: t('Number format'), }, - adhoc_filters: { - rerender: ['adhoc_custom'], - }, }, formDataOverrides: formData => ({ ...formData, diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts index a17fb8edd..1039df0de 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/transformProps.ts @@ -22,7 +22,9 @@ import { getMetricLabel, getValueFormatter, getNumberFormatter, - formatTimeRange, + SimpleAdhocFilter, + ensureIsArray, + getTimeOffset, } from '@superset-ui/core'; import { getComparisonFontSize, getHeaderFontSize } from './utils'; @@ -87,17 +89,49 @@ export default function transformProps(chartProps: ChartProps) { percentDifferenceFormat, } = formData; const { data: dataA = [] } = queriesData[0]; - const { - data: dataB = [], - from_dttm: comparisonFromDatetime, - to_dttm: comparisonToDatetime, - } = queriesData[1]; const data = dataA; const metricName = getMetricLabel(metric); + const timeComparison = ensureIsArray(chartProps.rawFormData?.time_compare)[0]; + const startDateOffset = chartProps.rawFormData?.start_date_offset; + const currentTimeRangeFilter = chartProps.rawFormData?.adhoc_filters?.filter( + (adhoc_filter: SimpleAdhocFilter) => + adhoc_filter.operator === 'TEMPORAL_RANGE', + )?.[0]; + const isCustomOrInherit = + timeComparison === 'custom' || timeComparison === 'inherit'; + let dataOffset: string[] = []; + if (isCustomOrInherit) { + dataOffset = getTimeOffset( + currentTimeRangeFilter, + ensureIsArray(timeComparison), + startDateOffset || '', + ); + } + + const { value1, value2 } = data.reduce( + (acc: { value1: number; value2: number }, curr: { [x: string]: any }) => { + Object.keys(curr).forEach(key => { + if ( + key.includes( + `${metricName}__${ + !isCustomOrInherit ? timeComparison : dataOffset[0] + }`, + ) + ) { + acc.value2 += curr[key]; + } else if (key.includes(metricName)) { + acc.value1 += curr[key]; + } + }); + return acc; + }, + { value1: 0, value2: 0 }, + ); + let bigNumber: number | string = - data.length === 0 ? 0 : parseMetricValue(data[0][metricName]); + data.length === 0 ? 0 : parseMetricValue(value1); let prevNumber: number | string = - data.length === 0 ? 0 : parseMetricValue(dataB[0][metricName]); + data.length === 0 ? 0 : parseMetricValue(value2); const numberFormatter = getValueFormatter( metric, @@ -133,10 +167,6 @@ export default function transformProps(chartProps: ChartProps) { prevNumber = numberFormatter(prevNumber); valueDifference = numberFormatter(valueDifference); const percentDifference: string = formatPercentChange(percentDifferenceNum); - const comparatorText = formatTimeRange('%Y-%m-%d', [ - comparisonFromDatetime, - comparisonToDatetime, - ]); return { width, @@ -155,6 +185,8 @@ export default function transformProps(chartProps: ChartProps) { comparisonColorEnabled, comparisonColorScheme, percentDifferenceNumber: percentDifferenceNum, - comparatorText, + currentTimeRangeFilter, + startDateOffset, + shift: timeComparison, }; } diff --git a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/types.ts b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/types.ts index a2282febb..e18e04261 100644 --- a/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/types.ts +++ b/superset-frontend/plugins/plugin-chart-echarts/src/BigNumber/BigNumberPeriodOverPeriod/types.ts @@ -21,6 +21,7 @@ import { supersetTheme, TimeseriesDataRecord, Metric, + SimpleAdhocFilter, } from '@superset-ui/core'; export interface PopKPIStylesProps { @@ -60,8 +61,10 @@ export type PopKPIProps = PopKPIStylesProps & percentDifferenceFormattedString: string; compType: string; percentDifferenceNumber: number; - comparatorText: string; comparisonColorScheme?: string; + currentTimeRangeFilter?: SimpleAdhocFilter; + startDateOffset?: string; + shift: string; }; export enum ColorSchemeEnum { diff --git a/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx b/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx new file mode 100644 index 000000000..cb2039698 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/ComparisonRangeLabel.tsx @@ -0,0 +1,107 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import React, { useEffect, useState } from 'react'; +import { useSelector } from 'react-redux'; +import { isEmpty, isEqual } from 'lodash'; +import { + BinaryAdhocFilter, + css, + ensureIsArray, + fetchTimeRange, + getTimeOffset, + SimpleAdhocFilter, + t, +} from '@superset-ui/core'; +import ControlHeader, { + ControlHeaderProps, +} from 'src/explore/components/ControlHeader'; +import { RootState } from 'src/views/store'; + +const isTimeRangeEqual = ( + left: BinaryAdhocFilter[], + right: BinaryAdhocFilter[], +) => isEqual(left, right); + +type ComparisonRangeLabelProps = ControlHeaderProps & { + multi?: boolean; +}; + +export const ComparisonRangeLabel = ({ + multi = true, +}: ComparisonRangeLabelProps) => { + const [labels, setLabels] = useState([]); + const currentTimeRangeFilters = useSelector( + state => + state.explore.form_data.adhoc_filters.filter( + (adhoc_filter: SimpleAdhocFilter) => + adhoc_filter.operator === 'TEMPORAL_RANGE', + ), + isTimeRangeEqual, + ); + const shifts = useSelector( + state => state.explore.form_data.time_compare, + ); + const startDate = useSelector( + state => state.explore.form_data.start_date_offset, + ); + + useEffect(() => { + const shiftsArray = ensureIsArray(shifts); + if ( + isEmpty(currentTimeRangeFilters) || + (isEmpty(shiftsArray) && !startDate) + ) { + setLabels([]); + } else if (!isEmpty(shifts) || startDate) { + const promises = currentTimeRangeFilters.map(filter => { + const newShifts = getTimeOffset(filter, shiftsArray, startDate); + + return fetchTimeRange( + filter.comparator, + filter.subject, + ensureIsArray(newShifts), + ); + }); + Promise.all(promises).then(res => { + // access the value property inside the res and set the labels with it in the state + setLabels(res.map(r => r.value ?? '')); + }); + } + }, [currentTimeRangeFilters, shifts, startDate]); + + return labels.length ? ( + <> + + {labels.flat().map(label => ( + <> +
css` + font-size: ${theme.typography.sizes.m}px; + color: ${theme.colors.grayscale.dark1}; + `} + key={label} + > + {label} +
+ + ))} + + ) : null; +}; diff --git a/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx b/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx new file mode 100644 index 000000000..1ce99e550 --- /dev/null +++ b/superset-frontend/src/explore/components/controls/TimeOffsetControl.tsx @@ -0,0 +1,87 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import React, { ReactNode } from 'react'; +import { isEqual } from 'lodash'; +import moment, { Moment } from 'moment'; +import { + parseDttmToDate, + BinaryAdhocFilter, + SimpleAdhocFilter, + css, +} from '@superset-ui/core'; +import { DatePicker } from 'antd'; +import { RangePickerProps } from 'antd/lib/date-picker'; +import { useSelector } from 'react-redux'; + +import ControlHeader from 'src/explore/components/ControlHeader'; +import { RootState } from 'src/views/store'; + +export interface TimeOffsetControlsProps { + label?: ReactNode; + startDate?: string; + description?: string; + hovered?: boolean; + value?: Moment; + onChange: (datetime: string) => void; +} +const MOMENT_FORMAT = 'YYYY-MM-DD'; + +const isTimeRangeEqual = ( + left: BinaryAdhocFilter[], + right: BinaryAdhocFilter[], +) => isEqual(left, right); + +export default function TimeOffsetControls({ + onChange, + ...props +}: TimeOffsetControlsProps) { + const currentTimeRangeFilters = useSelector( + state => + state.explore.form_data.adhoc_filters.filter( + (adhoc_filter: SimpleAdhocFilter) => + adhoc_filter.operator === 'TEMPORAL_RANGE', + ), + isTimeRangeEqual, + ); + + const startDate = currentTimeRangeFilters[0]?.comparator.split(' : ')[0]; + + const formatedDate = moment(parseDttmToDate(startDate)); + const disabledDate: RangePickerProps['disabledDate'] = current => + current && current > formatedDate; + + return ( +
+ + + onChange(datetime ? datetime.format(MOMENT_FORMAT) : '') + } + defaultPickerValue={ + startDate ? moment(formatedDate).subtract(1, 'day') : undefined + } + disabledDate={disabledDate} + defaultValue={formatedDate} + /> +
+ ); +} diff --git a/superset-frontend/src/explore/components/controls/index.js b/superset-frontend/src/explore/components/controls/index.js index a5d65f776..4c2cf0042 100644 --- a/superset-frontend/src/explore/components/controls/index.js +++ b/superset-frontend/src/explore/components/controls/index.js @@ -34,6 +34,7 @@ import SpatialControl from './SpatialControl'; import TextAreaControl from './TextAreaControl'; import TextControl from './TextControl'; import TimeSeriesColumnControl from './TimeSeriesColumnControl'; +import TimeOffsetControl from './TimeOffsetControl'; import ViewportControl from './ViewportControl'; import VizTypeControl from './VizTypeControl'; import MetricsControl from './MetricControl/MetricsControl'; @@ -48,6 +49,7 @@ import DndColumnSelectControl, { import XAxisSortControl from './XAxisSortControl'; import CurrencyControl from './CurrencyControl'; import ColumnConfigControl from './ColumnConfigControl'; +import { ComparisonRangeLabel } from './ComparisonRangeLabel'; const controlMap = { AnnotationLayerControl, @@ -80,6 +82,8 @@ const controlMap = { ConditionalFormattingControl, XAxisSortControl, ContourControl, + ComparisonRangeLabel, + TimeOffsetControl, ...sharedControlComponents, }; export default controlMap; diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index c47e295e9..75b3e4e97 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -19,7 +19,7 @@ from __future__ import annotations import copy import logging import re -from typing import Any, ClassVar, TYPE_CHECKING, TypedDict +from typing import Any, cast, ClassVar, TYPE_CHECKING, TypedDict import numpy as np import pandas as pd @@ -55,6 +55,7 @@ from superset.utils.core import ( DateColumn, DTTM_ALIAS, error_msg_from_exception, + FilterOperator, get_base_axis_labels, get_column_names_from_columns, get_column_names_from_metrics, @@ -390,11 +391,6 @@ class QueryContextProcessor: time_grain = self.get_time_grain(query_object) - if not time_grain: - raise QueryObjectValidationError( - _("Time Grain must be specified when using Time Shift.") - ) - metric_names = get_metric_names(query_object.metrics) # use columns that are not metrics as join keys @@ -429,6 +425,28 @@ class QueryContextProcessor: query_object_clone.inner_to_dttm = outer_to_dttm query_object_clone.time_offsets = [] query_object_clone.post_processing = [] + # Get time offset index + index = (get_base_axis_labels(query_object.columns) or [DTTM_ALIAS])[0] + # The comparison is not using a temporal column so we need to modify + # the temporal filter so we run the query with the correct time range + if not dataframe_utils.is_datetime_series(df.get(index)): + # Lets find the first temporal filter in the filters array and change + # its val to be the result of get_since_until with the offset + for flt in query_object_clone.filter: + if flt.get( + "op" + ) == FilterOperator.TEMPORAL_RANGE.value and isinstance( + flt.get("val"), str + ): + time_range = cast(str, flt.get("val")) + ( + new_outer_from_dttm, + new_outer_to_dttm, + ) = get_since_until_from_time_range( + time_range=time_range, + time_shift=offset, + ) + flt["val"] = f"{new_outer_from_dttm} : {new_outer_to_dttm}" query_object_clone.filter = [ flt for flt in query_object_clone.filter @@ -488,16 +506,6 @@ class QueryContextProcessor: # 2. rename extra query columns offset_metrics_df = offset_metrics_df.rename(columns=metrics_mapping) - # 3. set time offset for index - index = (get_base_axis_labels(query_object.columns) or [DTTM_ALIAS])[0] - if not dataframe_utils.is_datetime_series(offset_metrics_df.get(index)): - raise QueryObjectValidationError( - _( - "A time column must be specified " - "when using a Time Comparison." - ) - ) - # cache df and query value = { "df": offset_metrics_df, @@ -526,7 +534,7 @@ class QueryContextProcessor: self, df: pd.DataFrame, offset_dfs: dict[str, pd.DataFrame], - time_grain: str, + time_grain: str | None, join_keys: list[str], ) -> pd.DataFrame: """ @@ -541,43 +549,58 @@ class QueryContextProcessor: time_grain ) + if join_column_producer and not time_grain: + raise QueryObjectValidationError( + _("Time Grain must be specified when using Time Shift.") + ) + # iterate on offset_dfs, left join each with df for offset, offset_df in offset_dfs.items(): - # defines a column name for the offset join column - column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset + actual_join_keys = join_keys - # add offset join column to df - self.add_offset_join_column( - df, column_name, time_grain, offset, join_column_producer - ) + if time_grain: + # defines a column name for the offset join column + column_name = OFFSET_JOIN_COLUMN_SUFFIX + offset - # add offset join column to offset_df - self.add_offset_join_column( - offset_df, column_name, time_grain, None, join_column_producer - ) + # add offset join column to df + self.add_offset_join_column( + df, column_name, time_grain, offset, join_column_producer + ) - # the temporal column is the first column in the join keys - # so we use the join column instead of the temporal column - actual_join_keys = [column_name, *join_keys[1:]] + # add offset join column to offset_df + self.add_offset_join_column( + offset_df, column_name, time_grain, None, join_column_producer + ) - # left join df with offset_df - df = dataframe_utils.left_join_df( - left_df=df, - right_df=offset_df, - join_keys=actual_join_keys, - rsuffix=R_SUFFIX, - ) + # the temporal column is the first column in the join keys + # so we use the join column instead of the temporal column + actual_join_keys = [column_name, *join_keys[1:]] - # move the temporal column to the first column in df - col = df.pop(join_keys[0]) - df.insert(0, col.name, col) + if join_keys: + df = dataframe_utils.left_join_df( + left_df=df, + right_df=offset_df, + join_keys=actual_join_keys, + rsuffix=R_SUFFIX, + ) + else: + df = dataframe_utils.full_outer_join_df( + left_df=df, + right_df=offset_df, + rsuffix=R_SUFFIX, + ) - # removes columns created only for join purposes - df.drop( - list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")), - axis=1, - inplace=True, - ) + if time_grain: + # move the temporal column to the first column in df + col = df.pop(join_keys[0]) + df.insert(0, col.name, col) + + # removes columns created only for join purposes + df.drop( + list(df.filter(regex=f"{OFFSET_JOIN_COLUMN_SUFFIX}|{R_SUFFIX}")), + axis=1, + inplace=True, + ) return df @staticmethod diff --git a/superset/common/utils/dataframe_utils.py b/superset/common/utils/dataframe_utils.py index 7772ec585..89563dec0 100644 --- a/superset/common/utils/dataframe_utils.py +++ b/superset/common/utils/dataframe_utils.py @@ -40,6 +40,17 @@ def left_join_df( return df +def full_outer_join_df( + left_df: pd.DataFrame, + right_df: pd.DataFrame, + lsuffix: str = "", + rsuffix: str = "", +) -> pd.DataFrame: + df = left_df.join(right_df, lsuffix=lsuffix, rsuffix=rsuffix, how="outer") + df.reset_index(inplace=True) + return df + + def df_metrics_to_num(df: pd.DataFrame, query_object: QueryObject) -> None: """Converting metrics to numeric when pandas.read_sql cannot""" for col, dtype in df.dtypes.items(): diff --git a/superset/common/utils/time_range_utils.py b/superset/common/utils/time_range_utils.py index 2ceb9f766..496998865 100644 --- a/superset/common/utils/time_range_utils.py +++ b/superset/common/utils/time_range_utils.py @@ -21,7 +21,7 @@ from typing import Any, cast from superset import app from superset.common.query_object import QueryObject -from superset.utils.core import FilterOperator, get_xaxis_label +from superset.utils.core import FilterOperator from superset.utils.date_parser import get_since_until @@ -66,10 +66,8 @@ def get_since_until_from_query_object( time_range = None for flt in query_object.filter: - if ( - flt.get("op") == FilterOperator.TEMPORAL_RANGE.value - and flt.get("col") == get_xaxis_label(query_object.columns) - and isinstance(flt.get("val"), str) + if flt.get("op") == FilterOperator.TEMPORAL_RANGE.value and isinstance( + flt.get("val"), str ): time_range = cast(str, flt.get("val")) diff --git a/superset/migrations/versions/2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison.py b/superset/migrations/versions/2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison.py new file mode 100644 index 000000000..c349344f5 --- /dev/null +++ b/superset/migrations/versions/2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison.py @@ -0,0 +1,212 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Update charts with old time comparison controls + +Revision ID: f84fde59123a +Revises: 9621c6d56ffb +Create Date: 2024-05-10 18:02:38.891060 + +""" + +import json +import logging +from copy import deepcopy +from datetime import datetime, timedelta +from hashlib import md5 +from typing import Any + +from alembic import op +from sqlalchemy import Column, Integer, or_, String, Text +from sqlalchemy.ext.declarative import declarative_base + +from superset import db +from superset.migrations.shared.utils import paginated_update +from superset.utils.date_parser import get_since_until + +# revision identifiers, used by Alembic. +revision = "f84fde59123a" +down_revision = "9621c6d56ffb" + +logger = logging.getLogger(__name__) +Base = declarative_base() + + +class Slice(Base): + __tablename__ = "slices" + + id = Column(Integer, primary_key=True) + params = Column(Text) + viz_type = Column(String(250)) + + +time_map = { + "r": "inherit", + "y": "1 year ago", + "m": "1 month ago", + "w": "1 week ago", + "c": "custom", +} + + +def upgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]: + params = deepcopy(slice_params) + + if "enable_time_comparison" in params: + # Remove enable_time_comparison + del params["enable_time_comparison"] + + # Update time_comparison to time_compare + if "time_comparison" in params: + time_comp = params.pop("time_comparison") + params["time_compare"] = time_map.get( + time_comp, "inherit" + ) # Default to 'inherit' if not found + + # Add comparison_type + params["comparison_type"] = "values" + + # Adjust adhoc_custom + if "adhoc_custom" in params and params["adhoc_custom"]: + adhoc = params["adhoc_custom"][0] # As there's always only one element + if adhoc["comparator"] != "No filter": + # Set start_date_offset in params, not in adhoc + start_date_offset, _ = get_since_until(adhoc["comparator"]) + params["start_date_offset"] = start_date_offset.strftime("%Y-%m-%d") + # delete adhoc_custom + del params["adhoc_custom"] + + return params + + +def upgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + for slc in paginated_update( + session.query(Slice).filter( + or_(Slice.viz_type == "pop_kpi", Slice.viz_type == "table") + ) + ): + try: + params = json.loads(slc.params) + updated_slice_params = upgrade_comparison_params(params) + slc.params = json.dumps(updated_slice_params) + except Exception as ex: + session.rollback() + logger.exception( + f"An error occurred: Upgrading params for slice {slc.id} failed." + f"You need to fix it before upgrading your DB." + ) + raise Exception(f"An error occurred while upgrading slice: {ex}") + + session.commit() + session.close() + + +def downgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]: + params = deepcopy(slice_params) + + reverse_time_map = { + v: k for k, v in time_map.items() + } # Reverse the map from the upgrade function + + # Add enable_time_comparison + params["enable_time_comparison"] = True + + # Revert time_compare to time_comparison + if "time_compare" in params: + time_comp = params.pop("time_compare") + params["time_comparison"] = reverse_time_map.get( + time_comp, "r" + ) # Default to 'r' if not found + + # Remove comparison_type + if "comparison_type" in params: + del params["comparison_type"] + + # Default adhoc_custom + adhoc_custom = [ + { + "clause": "WHERE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "No filter", + "expressionType": "SIMPLE", + } + ] + + # Handle start_date_offset and adjust adhoc_custom if necessary + if "start_date_offset" in params: + start_date_offset = datetime.strptime( + params.pop("start_date_offset"), "%Y-%m-%d" + ) + adhoc_filters = params.get("adhoc_filters", []) + temporal_range_filter = next( + (f for f in adhoc_filters if f["operator"] == "TEMPORAL_RANGE"), None + ) + + if temporal_range_filter: + since, until = get_since_until(temporal_range_filter["comparator"]) + delta_days = (until - since).days + new_until_date = start_date_offset + timedelta(days=delta_days - 1) + comparator_str = f"{start_date_offset.strftime('%Y-%m-%d')} : {new_until_date.strftime('%Y-%m-%d')}" + + # Generate filterOptionName + random_string = md5(comparator_str.encode("utf-8")).hexdigest() + filter_option_name = f"filter_{random_string}" + + adhoc_custom[0] = { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": comparator_str, + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": filter_option_name, + } + + params["adhoc_custom"] = adhoc_custom + + return params + + +def downgrade(): + bind = op.get_bind() + session = db.Session(bind=bind) + + for slc in paginated_update( + session.query(Slice).filter( + Slice.viz_type == "pop_kpi" or Slice.viz_type == "table" + ) + ): + try: + params = json.loads(slc.params) + updated_slice_params = downgrade_comparison_params(params) + slc.params = json.dumps(updated_slice_params) + except Exception as ex: + session.rollback() + logger.exception( + f"An error occurred: Downgrading params for slice {slc.id} failed." + f"You need to fix it before downgrading your DB." + ) + raise Exception(f"An error occurred while downgrading slice: {ex}") + + session.commit() + session.close() diff --git a/superset/utils/date_parser.py b/superset/utils/date_parser.py index 9a7c135f5..bffe50c62 100644 --- a/superset/utils/date_parser.py +++ b/superset/utils/date_parser.py @@ -263,9 +263,10 @@ def get_since_until( # pylint: disable=too-many-arguments,too-many-locals,too-m ) if time_shift: - time_delta = parse_past_timedelta(time_shift) - _since = _since if _since is None else (_since - time_delta) - _until = _until if _until is None else (_until - time_delta) + time_delta_since = parse_past_timedelta(time_shift, _since) + time_delta_until = parse_past_timedelta(time_shift, _until) + _since = _since if _since is None else (_since - time_delta_since) + _until = _until if _until is None else (_until - time_delta_until) if instant_time_comparison_range: # This is only set using the new time comparison controls diff --git a/superset/views/api.py b/superset/views/api.py index 2e3c3b9bd..d5dd0eca4 100644 --- a/superset/views/api.py +++ b/superset/views/api.py @@ -46,6 +46,7 @@ get_time_range_schema = { "type": "object", "properties": { "timeRange": {"type": "string"}, + "shift": {"type": "string"}, }, }, } @@ -110,12 +111,16 @@ class Api(BaseSupersetView): rv = [] for time_range in time_ranges: - since, until = get_since_until(time_range["timeRange"]) + since, until = get_since_until( + time_range=time_range["timeRange"], + time_shift=time_range.get("shift"), + ) rv.append( { "since": since.isoformat() if since else "", "until": until.isoformat() if until else "", "timeRange": time_range["timeRange"], + "shift": time_range.get("shift"), } ) return self.json_response({"result": rv}) diff --git a/tests/integration_tests/charts/api_tests.py b/tests/integration_tests/charts/api_tests.py index e6c68147c..65ede9221 100644 --- a/tests/integration_tests/charts/api_tests.py +++ b/tests/integration_tests/charts/api_tests.py @@ -1520,6 +1520,20 @@ class TestChartApi(ApiOwnersTestCaseMixin, InsertChartMixin, SupersetTestCase): assert "until" in data["result"][0] assert "timeRange" in data["result"][0] + humanize_time_range = [ + {"timeRange": "2021-01-01 : 2022-02-01", "shift": "1 year ago"}, + {"timeRange": "2022-01-01 : 2023-02-01", "shift": "2 year ago"}, + ] + uri = f"api/v1/time_range/?q={prison.dumps(humanize_time_range)}" + rv = self.client.get(uri) + data = json.loads(rv.data.decode("utf-8")) + assert rv.status_code == 200 + assert len(data["result"]) == 2 + assert "since" in data["result"][0] + assert "until" in data["result"][0] + assert "timeRange" in data["result"][0] + assert "shift" in data["result"][0] + def test_query_form_data(self): """ Chart API: Test query form data diff --git a/tests/integration_tests/migrations/f84fde59123a_update_charts_with_old_time_comparison__test.py b/tests/integration_tests/migrations/f84fde59123a_update_charts_with_old_time_comparison__test.py new file mode 100644 index 000000000..5f7fe505c --- /dev/null +++ b/tests/integration_tests/migrations/f84fde59123a_update_charts_with_old_time_comparison__test.py @@ -0,0 +1,315 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from copy import deepcopy +from importlib import import_module +from typing import Any + +migrate_time_comparison_to_new_format = import_module( + "superset.migrations.versions." + "2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison", +) +downgrade_comparison_params = ( + migrate_time_comparison_to_new_format.downgrade_comparison_params +) +upgrade_comparison_params = ( + migrate_time_comparison_to_new_format.upgrade_comparison_params +) + +params_v1_with_custom: dict[str, Any] = { + "datasource": "2__table", + "viz_type": "pop_kpi", + "metric": { + "expressionType": "SIMPLE", + "column": { + "advanced_data_type": None, + "certification_details": None, + "certified_by": None, + "column_name": "num_boys", + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 334, + "is_certified": False, + "is_dttm": False, + "python_date_format": None, + "type": "BIGINT", + "type_generic": 0, + "verbose_name": None, + "warning_markdown": None, + }, + "aggregate": "SUM", + "sqlExpression": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "SUM(num_boys)", + "optionName": "metric_o6rj1h6jty_3t6mrruogfv", + }, + "adhoc_filters": [ + { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "1984 : 1986", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8", + } + ], + "row_limit": 10000, + "y_axis_format": "SMART_NUMBER", + "percentDifferenceFormat": "SMART_NUMBER", + "header_font_size": 0.2, + "subheader_font_size": 0.125, + "comparison_color_scheme": "Green", + "extra_form_data": {}, + "dashboards": [], + "time_comparison": "c", + "enable_time_comparison": True, + "adhoc_custom": [ + { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "1981-01-01 : 1983-01-01", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + } + ], +} +params_v1_other_than_custom: dict[str, Any] = { + "datasource": "2__table", + "viz_type": "pop_kpi", + "metric": { + "expressionType": "SIMPLE", + "column": { + "advanced_data_type": None, + "certification_details": None, + "certified_by": None, + "column_name": "num_boys", + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 334, + "is_certified": False, + "is_dttm": False, + "python_date_format": None, + "type": "BIGINT", + "type_generic": 0, + "verbose_name": None, + "warning_markdown": None, + }, + "aggregate": "SUM", + "sqlExpression": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "SUM(num_boys)", + "optionName": "metric_96s7b8iypsr_4wrlgm0i7il", + }, + "adhoc_filters": [ + { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "1984 : 2000", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6", + } + ], + "row_limit": 10000, + "y_axis_format": "SMART_NUMBER", + "percentDifferenceFormat": "SMART_NUMBER", + "header_font_size": 0.2, + "subheader_font_size": 0.125, + "comparison_color_scheme": "Green", + "extra_form_data": {}, + "dashboards": [], + "time_comparison": "r", + "enable_time_comparison": True, + "adhoc_custom": [ + { + "clause": "WHERE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "No filter", + "expressionType": "SIMPLE", + } + ], +} +params_v2_with_custom: dict[str, Any] = { + "datasource": "2__table", + "viz_type": "pop_kpi", + "metric": { + "expressionType": "SIMPLE", + "column": { + "advanced_data_type": None, + "certification_details": None, + "certified_by": None, + "column_name": "num_boys", + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 334, + "is_certified": False, + "is_dttm": False, + "python_date_format": None, + "type": "BIGINT", + "type_generic": 0, + "verbose_name": None, + "warning_markdown": None, + }, + "aggregate": "SUM", + "sqlExpression": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "SUM(num_boys)", + "optionName": "metric_o6rj1h6jty_3t6mrruogfv", + }, + "adhoc_filters": [ + { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "1984 : 1986", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8", + } + ], + "row_limit": 10000, + "y_axis_format": "SMART_NUMBER", + "percentDifferenceFormat": "SMART_NUMBER", + "header_font_size": 0.2, + "subheader_font_size": 0.125, + "comparison_color_scheme": "Green", + "extra_form_data": {}, + "dashboards": [], + "time_compare": "custom", + "comparison_type": "values", + "start_date_offset": "1981-01-01", +} +params_v2_other_than_custom: dict[str, Any] = { + "datasource": "2__table", + "viz_type": "pop_kpi", + "metric": { + "expressionType": "SIMPLE", + "column": { + "advanced_data_type": None, + "certification_details": None, + "certified_by": None, + "column_name": "num_boys", + "description": None, + "expression": None, + "filterable": True, + "groupby": True, + "id": 334, + "is_certified": False, + "is_dttm": False, + "python_date_format": None, + "type": "BIGINT", + "type_generic": 0, + "verbose_name": None, + "warning_markdown": None, + }, + "aggregate": "SUM", + "sqlExpression": None, + "datasourceWarning": False, + "hasCustomLabel": False, + "label": "SUM(num_boys)", + "optionName": "metric_96s7b8iypsr_4wrlgm0i7il", + }, + "adhoc_filters": [ + { + "expressionType": "SIMPLE", + "subject": "ds", + "operator": "TEMPORAL_RANGE", + "comparator": "1984 : 2000", + "clause": "WHERE", + "sqlExpression": None, + "isExtra": False, + "isNew": False, + "datasourceWarning": False, + "filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6", + } + ], + "row_limit": 10000, + "y_axis_format": "SMART_NUMBER", + "percentDifferenceFormat": "SMART_NUMBER", + "header_font_size": 0.2, + "subheader_font_size": 0.125, + "comparison_color_scheme": "Green", + "extra_form_data": {}, + "dashboards": [], + "time_compare": "inherit", + "comparison_type": "values", +} + + +def test_upgrade_chart_params_with_custom(): + """ + ensure that the new time comparison params are added + """ + original_params = deepcopy(params_v1_with_custom) + upgraded_params = upgrade_comparison_params(original_params) + assert upgraded_params == params_v2_with_custom + + +def test_downgrade_chart_params_with_custom(): + """ + ensure that the params downgrade operation produces an almost identical dict + as the original value + """ + original_params = deepcopy(params_v2_with_custom) + downgraded_params = downgrade_comparison_params(original_params) + # Ignore any property called filterOptionName simce that uses a random hash + for adhoc_custom in downgraded_params["adhoc_custom"]: + adhoc_custom.pop("filterOptionName", None) + assert downgraded_params == params_v1_with_custom + + +def test_upgrade_chart_params_other_than_custom(): + """ + ensure that the new time comparison params are added + """ + original_params = deepcopy(params_v1_other_than_custom) + upgraded_params = upgrade_comparison_params(original_params) + assert upgraded_params == params_v2_other_than_custom + + +def test_downgrade_chart_params_other_than_custom(): + """ + ensure that the params downgrade operation produces an almost identical dict + as the original value + """ + original_params = deepcopy(params_v2_other_than_custom) + downgraded_params = downgrade_comparison_params(original_params) + assert downgraded_params == params_v1_other_than_custom diff --git a/tests/unit_tests/utils/date_parser_tests.py b/tests/unit_tests/utils/date_parser_tests.py index b031e54a6..e007c17e8 100644 --- a/tests/unit_tests/utils/date_parser_tests.py +++ b/tests/unit_tests/utils/date_parser_tests.py @@ -189,6 +189,27 @@ def test_get_since_until() -> None: expected = datetime(2000, 1, 1), datetime(2018, 1, 1) assert result == expected + result = get_since_until( + time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00", + time_shift="1 year ago", + ) + expected = datetime(1999, 1, 1), datetime(2017, 1, 1) + assert result == expected + + result = get_since_until( + time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00", + time_shift="1 month ago", + ) + expected = datetime(1999, 12, 1), datetime(2017, 12, 1) + assert result == expected + + result = get_since_until( + time_range="2000-01-01T00:00:00 : 2018-01-01T00:00:00", + time_shift="1 week ago", + ) + expected = datetime(1999, 12, 25), datetime(2017, 12, 25) + assert result == expected + with pytest.raises(ValueError): get_since_until(time_range="tomorrow : yesterday")