iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: rename `apply_post_process` (#31511)

This commit is contained in:
Beto Dealmeida 2024-12-19 10:13:28 -05:00 committed by GitHub
parent 531f1b6aa4
commit 7458c4bbd5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 46 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
superset/charts/post_processing.py → superset/charts/client_processing.py
View File

@ -15,15 +15,13 @@
# specific language governing permissions and limitations
# under the License.
"""
Functions to reproduce the post-processing of data on text charts.
Functions to reproduce the client post-processing of data on charts.
Some text-based charts (pivot tables and t-test table) perform
post-processing of the data in JavaScript. When sending the data
to users in reports we want to show the same data they would see
on Explore.
Some text-based charts (pivot tables and t-test table) perform post-processing of the
data in JavaScript. When sending the data to users in reports we want to show the same
data they would see on Explore.
In order to do that, we reproduce the post-processing in Python
for these chart types.
In order to do that, we reproduce the post-processing in Python for these chart types.
"""
from io import StringIO
@ -298,7 +296,7 @@ post_processors = {
@event_logger.log_this
def apply_post_process( # noqa: C901
def apply_client_processing( # noqa: C901
result: dict[Any, Any],
form_data: Optional[dict[str, Any]] = None,
datasource: Optional[Union["BaseDatasource", "Query"]] = None,

4
superset/charts/data/api.py
View File

@ -28,8 +28,8 @@ from marshmallow import ValidationError
from superset import is_feature_enabled, security_manager
from superset.async_events.async_query_manager import AsyncQueryTokenException
from superset.charts.api import ChartRestApi
from superset.charts.client_processing import apply_client_processing
from superset.charts.data.query_context_cache_loader import QueryContextCacheLoader
from superset.charts.post_processing import apply_post_process
from superset.charts.schemas import ChartDataQueryContextSchema
from superset.commands.chart.data.create_async_job_command import (
CreateAsyncChartDataJobCommand,
@ -356,7 +356,7 @@ class ChartDataRestApi(ChartRestApi):
# This is needed for sending reports based on text charts that do the
# post-processing of data, eg, the pivot table.
if result_type == ChartDataResultType.POST_PROCESSED:
result = apply_post_process(result, form_data, datasource)
result = apply_client_processing(result, form_data, datasource)
if result_format in ChartDataResultFormat.table_like():
# Verify user has permission to export file

32
superset/common/query_object.py
View File

@ -33,6 +33,7 @@ from superset.exceptions import (
QueryClauseValidationException,
QueryObjectValidationError,
)
from superset.extensions import event_logger
from superset.sql_parse import sanitize_clause
from superset.superset_typing import Column, Metric, OrderBy
from superset.utils import json, pandas_postprocessing
@ -428,19 +429,20 @@ class QueryObject: # pylint: disable=too-many-instance-attributes
is incorrect
"""
logger.debug("post_processing: \n %s", pformat(self.post_processing))
for post_process in self.post_processing:
operation = post_process.get("operation")
if not operation:
raise InvalidPostProcessingError(
_("`operation` property of post processing object undefined")
)
if not hasattr(pandas_postprocessing, operation):
raise InvalidPostProcessingError(
_(
"Unsupported post processing operation: %(operation)s",
type=operation,
with event_logger.log_context(f"{self.__class__.__name__}.post_processing"):
for post_process in self.post_processing:
operation = post_process.get("operation")
if not operation:
raise InvalidPostProcessingError(
_("`operation` property of post processing object undefined")
)
)
options = post_process.get("options", {})
df = getattr(pandas_postprocessing, operation)(df, **options)
return df
if not hasattr(pandas_postprocessing, operation):
raise InvalidPostProcessingError(
_(
"Unsupported post processing operation: %(operation)s",
type=operation,
)
)
options = post_process.get("options", {})
df = getattr(pandas_postprocessing, operation)(df, **options)
return df

42
tests/unit_tests/charts/test_post_processing.py → tests/unit_tests/charts/test_client_processing.py
View File

@ -21,7 +21,7 @@ import pytest
from flask_babel import lazy_gettext as _
from sqlalchemy.orm.session import Session
from superset.charts.post_processing import apply_post_process, pivot_df, table
from superset.charts.client_processing import apply_client_processing, pivot_df, table
from superset.common.chart_data import ChartDataResultFormat
from superset.utils.core import GenericDataType
@ -1841,16 +1841,16 @@ def test_table():
)
def test_apply_post_process_no_form_invalid_viz_type():
def test_apply_client_processing_no_form_invalid_viz_type():
"""
Test with invalid viz type. It should just return the result
"""
result = {"foo": "bar"}
form_data = {"viz_type": "baz"}
assert apply_post_process(result, form_data) == result
assert apply_client_processing(result, form_data) == result
def test_apply_post_process_without_result_format():
def test_apply_client_processing_without_result_format():
"""
A query without result_format should raise an exception
"""
@ -1858,12 +1858,12 @@ def test_apply_post_process_without_result_format():
form_data = {"viz_type": "pivot_table_v2"}
with pytest.raises(Exception) as ex: # noqa: PT011
apply_post_process(result, form_data)
apply_client_processing(result, form_data)
assert ex.match("Result format foo not supported") is True # noqa: E712
def test_apply_post_process_json_format():
def test_apply_client_processing_json_format():
"""
It should be able to process json results
"""
@ -1944,7 +1944,7 @@ def test_apply_post_process_json_format():
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [
{
"result_format": ChartDataResultFormat.JSON,
@ -1966,7 +1966,7 @@ def test_apply_post_process_json_format():
}
def test_apply_post_process_csv_format():
def test_apply_client_processing_csv_format():
"""
It should be able to process csv results
"""
@ -2042,7 +2042,7 @@ COUNT(is_software_dev)
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [
{
"result_format": ChartDataResultFormat.CSV,
@ -2056,7 +2056,7 @@ COUNT(is_software_dev)
}
def test_apply_post_process_csv_format_empty_string():
def test_apply_client_processing_csv_format_empty_string():
"""
It should be able to process csv results with no data
"""
@ -2122,13 +2122,13 @@ def test_apply_post_process_csv_format_empty_string():
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [{"result_format": ChartDataResultFormat.CSV, "data": ""}]
}
@pytest.mark.parametrize("data", [None, "", "\n"])
def test_apply_post_process_csv_format_no_data(data):
def test_apply_client_processing_csv_format_no_data(data):
"""
It should be able to process csv results with no data
"""
@ -2194,12 +2194,12 @@ def test_apply_post_process_csv_format_no_data(data):
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [{"result_format": ChartDataResultFormat.CSV, "data": data}]
}
def test_apply_post_process_csv_format_no_data_multiple_queries():
def test_apply_client_processing_csv_format_no_data_multiple_queries():
"""
It should be able to process csv results multiple queries if one query has no data
"""
@ -2276,7 +2276,7 @@ COUNT(is_software_dev)
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [
{"result_format": ChartDataResultFormat.CSV, "data": ""},
{
@ -2291,7 +2291,7 @@ COUNT(is_software_dev)
}
def test_apply_post_process_json_format_empty_string():
def test_apply_client_processing_json_format_empty_string():
"""
It should be able to process json results with no data
"""
@ -2357,12 +2357,12 @@ def test_apply_post_process_json_format_empty_string():
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [{"result_format": ChartDataResultFormat.JSON, "data": ""}]
}
def test_apply_post_process_json_format_data_is_none():
def test_apply_client_processing_json_format_data_is_none():
"""
It should be able to process json results with no data
"""
@ -2428,12 +2428,12 @@ def test_apply_post_process_json_format_data_is_none():
"result_type": "results",
}
assert apply_post_process(result, form_data) == {
assert apply_client_processing(result, form_data) == {
"queries": [{"result_format": ChartDataResultFormat.JSON, "data": None}]
}
def test_apply_post_process_verbose_map(session: Session):
def test_apply_client_processing_verbose_map(session: Session):
from superset import db
from superset.connectors.sqla.models import SqlaTable, SqlMetric
from superset.models.core import Database
@ -2487,7 +2487,7 @@ def test_apply_post_process_verbose_map(session: Session):
"result_type": "results",
}
assert apply_post_process(result, form_data, datasource=sqla_table) == {
assert apply_client_processing(result, form_data, datasource=sqla_table) == {
"queries": [
{
"result_format": ChartDataResultFormat.JSON,