iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(migrations): Fix the time comparison migration (#30029)

This commit is contained in:
Antonio Rivero 2024-08-27 20:55:52 +02:00 committed by GitHub
parent 07a90ad4fe
commit d80f23ed94
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
2 changed files with 75 additions and 157 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

28
superset/migrations/versions/2024-05-10_18-02_f84fde59123a_update_charts_with_old_time_comparison.py
View File

@ -65,16 +65,17 @@ time_map = {
def upgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]: def upgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]:
params = deepcopy(slice_params) params = deepcopy(slice_params)
if "enable_time_comparison" in params:
# Remove enable_time_comparison
del params["enable_time_comparison"]
# Update time_comparison to time_compare # Update time_comparison to time_compare
if "time_comparison" in params: if "time_comparison" in params:
time_comp = params.pop("time_comparison") time_comp = params.pop("time_comparison")
params["time_compare"] = time_map.get( params["time_compare"] = (
time_comp, "inherit" [time_map.get(time_comp, "inherit")]
) # Default to 'inherit' if not found if "enable_time_comparison" in params and params["enable_time_comparison"]
else []
)
if "enable_time_comparison" in params:
del params["enable_time_comparison"]
# Add comparison_type # Add comparison_type
params["comparison_type"] = "values" params["comparison_type"] = "values"
@ -119,20 +120,21 @@ def upgrade():
def downgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]: def downgrade_comparison_params(slice_params: dict[str, Any]) -> dict[str, Any]:
params = deepcopy(slice_params) params = deepcopy(slice_params)
params["enable_time_comparison"] = False
reverse_time_map = { reverse_time_map = {
v: k for k, v in time_map.items() v: k for k, v in time_map.items()
} # Reverse the map from the upgrade function } # Reverse the map from the upgrade function
# Add enable_time_comparison
params["enable_time_comparison"] = True
# Revert time_compare to time_comparison # Revert time_compare to time_comparison
if "time_compare" in params: if "time_compare" in params:
time_comp = params.pop("time_compare") time_comp = params.pop("time_compare")
params["time_comparison"] = reverse_time_map.get( # Max one element in the time_compare list
time_comp, "r" time_comp = time_comp[0] if time_comp else ""
) # Default to 'r' if not found params["time_comparison"] = reverse_time_map.get(time_comp, "r")
# If the chart was using any time compare, enable time comparison
if time_comp:
params["enable_time_comparison"] = True
# Remove comparison_type # Remove comparison_type
if "comparison_type" in params: if "comparison_type" in params:

204
tests/integration_tests/migrations/f84fde59123a_update_charts_with_old_time_comparison__test.py
View File

@ -29,7 +29,8 @@ upgrade_comparison_params = (
migrate_time_comparison_to_new_format.upgrade_comparison_params migrate_time_comparison_to_new_format.upgrade_comparison_params
) )
params_v1_with_custom: dict[str, Any] = { # Base object containing common properties
base_params: dict[str, Any] = {
"datasource": "2__table", "datasource": "2__table",
"viz_type": "pop_kpi", "viz_type": "pop_kpi",
"metric": { "metric": {
@ -57,20 +58,18 @@ params_v1_with_custom: dict[str, Any] = {
"datasourceWarning": False, "datasourceWarning": False,
"hasCustomLabel": False, "hasCustomLabel": False,
"label": "SUM(num_boys)", "label": "SUM(num_boys)",
"optionName": "metric_o6rj1h6jty_3t6mrruogfv",
}, },
"adhoc_filters": [ "adhoc_filters": [
{ {
"expressionType": "SIMPLE", "expressionType": "SIMPLE",
"subject": "ds", "subject": "ds",
"operator": "TEMPORAL_RANGE", "operator": "TEMPORAL_RANGE",
"comparator": "1984 : 1986", "comparator": "1984 : 2000",
"clause": "WHERE", "clause": "WHERE",
"sqlExpression": None, "sqlExpression": None,
"isExtra": False, "isExtra": False,
"isNew": False, "isNew": False,
"datasourceWarning": False, "datasourceWarning": False,
"filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8",
} }
], ],
"row_limit": 10000, "row_limit": 10000,
@ -81,6 +80,22 @@ params_v1_with_custom: dict[str, Any] = {
"comparison_color_scheme": "Green", "comparison_color_scheme": "Green",
"extra_form_data": {}, "extra_form_data": {},
"dashboards": [], "dashboards": [],
}
# Specific parameter objects overriding only the differing properties
params_v1_with_custom: dict[str, Any] = {
**base_params,
"metric": {
**base_params["metric"],
"optionName": "metric_o6rj1h6jty_3t6mrruogfv",
},
"adhoc_filters": [
{
**base_params["adhoc_filters"][0],
"comparator": "1984 : 1986",
"filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8",
}
],
"time_comparison": "c", "time_comparison": "c",
"enable_time_comparison": True, "enable_time_comparison": True,
"adhoc_custom": [ "adhoc_custom": [
@ -97,58 +112,13 @@ params_v1_with_custom: dict[str, Any] = {
} }
], ],
} }
params_v1_other_than_custom: dict[str, Any] = { params_v1_other_than_custom: dict[str, Any] = {
"datasource": "2__table", **base_params,
"viz_type": "pop_kpi",
"metric": { "metric": {
"expressionType": "SIMPLE", **base_params["metric"],
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_96s7b8iypsr_4wrlgm0i7il", "optionName": "metric_96s7b8iypsr_4wrlgm0i7il",
}, },
"adhoc_filters": [
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 2000",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_comparison": "r", "time_comparison": "r",
"enable_time_comparison": True, "enable_time_comparison": True,
"adhoc_custom": [ "adhoc_custom": [
@ -161,118 +131,45 @@ params_v1_other_than_custom: dict[str, Any] = {
} }
], ],
} }
params_v1_other_than_custom_false: dict[str, Any] = {
**params_v1_other_than_custom,
"enable_time_comparison": False,
}
params_v2_with_custom: dict[str, Any] = { params_v2_with_custom: dict[str, Any] = {
"datasource": "2__table", **base_params,
"viz_type": "pop_kpi",
"metric": { "metric": {
"expressionType": "SIMPLE", **base_params["metric"],
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_o6rj1h6jty_3t6mrruogfv", "optionName": "metric_o6rj1h6jty_3t6mrruogfv",
}, },
"adhoc_filters": [ "adhoc_filters": [
{ {
"expressionType": "SIMPLE", **base_params["adhoc_filters"][0],
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 1986", "comparator": "1984 : 1986",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8", "filterOptionName": "filter_p50i4xw50d_8x8e4ypwjs8",
} }
], ],
"row_limit": 10000, "time_compare": ["custom"],
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_compare": "custom",
"comparison_type": "values", "comparison_type": "values",
"start_date_offset": "1981-01-01", "start_date_offset": "1981-01-01",
} }
params_v2_other_than_custom: dict[str, Any] = { params_v2_other_than_custom: dict[str, Any] = {
"datasource": "2__table", **base_params,
"viz_type": "pop_kpi",
"metric": { "metric": {
"expressionType": "SIMPLE", **base_params["metric"],
"column": {
"advanced_data_type": None,
"certification_details": None,
"certified_by": None,
"column_name": "num_boys",
"description": None,
"expression": None,
"filterable": True,
"groupby": True,
"id": 334,
"is_certified": False,
"is_dttm": False,
"python_date_format": None,
"type": "BIGINT",
"type_generic": 0,
"verbose_name": None,
"warning_markdown": None,
},
"aggregate": "SUM",
"sqlExpression": None,
"datasourceWarning": False,
"hasCustomLabel": False,
"label": "SUM(num_boys)",
"optionName": "metric_96s7b8iypsr_4wrlgm0i7il", "optionName": "metric_96s7b8iypsr_4wrlgm0i7il",
}, },
"adhoc_filters": [ "time_compare": ["inherit"],
{
"expressionType": "SIMPLE",
"subject": "ds",
"operator": "TEMPORAL_RANGE",
"comparator": "1984 : 2000",
"clause": "WHERE",
"sqlExpression": None,
"isExtra": False,
"isNew": False,
"datasourceWarning": False,
"filterOptionName": "filter_2sefqq1rwb7_lhqvw7ukc6",
}
],
"row_limit": 10000,
"y_axis_format": "SMART_NUMBER",
"percentDifferenceFormat": "SMART_NUMBER",
"header_font_size": 0.2,
"subheader_font_size": 0.125,
"comparison_color_scheme": "Green",
"extra_form_data": {},
"dashboards": [],
"time_compare": "inherit",
"comparison_type": "values", "comparison_type": "values",
} }
params_v2_other_than_custom_false: dict[str, Any] = {
**params_v2_other_than_custom,
"time_compare": [],
}
def test_upgrade_chart_params_with_custom(): def test_upgrade_chart_params_with_custom():
""" """
@ -313,3 +210,22 @@ def test_downgrade_chart_params_other_than_custom():
original_params = deepcopy(params_v2_other_than_custom) original_params = deepcopy(params_v2_other_than_custom)
downgraded_params = downgrade_comparison_params(original_params) downgraded_params = downgrade_comparison_params(original_params)
assert downgraded_params == params_v1_other_than_custom assert downgraded_params == params_v1_other_than_custom
def test_upgrade_chart_params_other_than_custom_false():
"""
ensure that the new time comparison params are added
"""
original_params = deepcopy(params_v1_other_than_custom_false)
upgraded_params = upgrade_comparison_params(original_params)
assert upgraded_params == params_v2_other_than_custom_false
def test_downgrade_chart_params_other_than_custom_false():
"""
ensure that the params downgrade operation produces an almost identical dict
as the original value
"""
original_params = deepcopy(params_v2_other_than_custom_false)
downgraded_params = downgrade_comparison_params(original_params)
assert downgraded_params == params_v1_other_than_custom_false