iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: Migration of json utilities from core (#28522) 

Co-authored-by: Eyal Ezer <eyal.ezer@ge.com>
This commit is contained in:
Eyal Ezer 2024-05-20 12:27:22 -05:00 committed by GitHub
parent e954360eec
commit 56f0fc4ec2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
30 changed files with 431 additions and 316 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
superset/annotation_layers/annotations/schemas.py
View File

@ -20,7 +20,7 @@ from marshmallow import fields, Schema, ValidationError
from marshmallow.validate import Length
from superset.exceptions import SupersetException
from superset.utils import core as utils
from superset.utils import json as json_utils
openapi_spec_methods_override = {
"get": {"get": {"summary": "Get an annotation layer"}},
@ -51,7 +51,7 @@ annotation_json_metadata = "JSON metadata"
def validate_json(value: Union[bytes, bytearray, str]) -> None:
try:
utils.validate_json(value)
json_utils.validate_json(value)
except SupersetException as ex:
raise ValidationError("JSON not valid") from ex

7
superset/charts/data/api.py
View File

@ -21,7 +21,6 @@ import json
import logging
from typing import Any, TYPE_CHECKING
import simplejson
from flask import current_app, g, make_response, request, Response
from flask_appbuilder.api import expose, protect
from flask_babel import gettext as _
@ -47,11 +46,11 @@ from superset.daos.exceptions import DatasourceNotFound
from superset.exceptions import QueryObjectValidationError
from superset.extensions import event_logger
from superset.models.sql_lab import Query
from superset.utils import json as json_utils
from superset.utils.core import (
create_zip,
DatasourceType,
get_user_id,
json_int_dttm_ser,
)
from superset.utils.decorators import logs_context
from superset.views.base import CsvResponse, generate_download_headers, XlsxResponse
@ -396,9 +395,9 @@ class ChartDataRestApi(ChartRestApi):
)
if result_format == ChartDataResultFormat.JSON:
response_data = simplejson.dumps(
response_data = json_utils.dumps(
{"result": result["queries"]},
default=json_int_dttm_ser,
default=json_utils.json_int_dttm_ser,
ignore_nan=True,
)
resp = make_response(response_data, 200)

2
superset/common/query_object.py
View File

@ -43,10 +43,10 @@ from superset.utils.core import (
get_column_names,
get_metric_names,
is_adhoc_metric,
json_int_dttm_ser,
QueryObjectFilterClause,
)
from superset.utils.hashing import md5_sha_from_dict
from superset.utils.json import json_int_dttm_ser
if TYPE_CHECKING:
from superset.connectors.sqla.models import BaseDatasource

4
superset/dashboards/schemas.py
View File

@ -24,7 +24,7 @@ from marshmallow.validate import Length, ValidationError
from superset import security_manager
from superset.exceptions import SupersetException
from superset.tags.models import TagType
from superset.utils import core as utils
from superset.utils import json as json_utils
get_delete_ids_schema = {"type": "array", "items": {"type": "integer"}}
get_export_ids_schema = {"type": "array", "items": {"type": "integer"}}
@ -88,7 +88,7 @@ openapi_spec_methods_override = {
def validate_json(value: Union[bytes, bytearray, str]) -> None:
try:
utils.validate_json(value)
json_utils.validate_json(value)
except SupersetException as ex:
raise ValidationError("JSON not valid") from ex

7
superset/embedded/view.py
View File

@ -14,7 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import json
from typing import Callable
from flask import abort, request
@ -25,7 +24,7 @@ from flask_wtf.csrf import same_origin
from superset import event_logger, is_feature_enabled
from superset.daos.dashboard import EmbeddedDashboardDAO
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import json as json_utils
from superset.views.base import BaseSupersetView, common_bootstrap_payload
@ -87,7 +86,7 @@ class EmbeddedView(BaseSupersetView):
return self.render_template(
"superset/spa.html",
entry="embedded",
bootstrap_data=json.dumps(
bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser
),
)

2
superset/key_value/utils.py
View File

@ -26,7 +26,7 @@ from flask_babel import gettext as _
from superset.key_value.exceptions import KeyValueParseKeyError
from superset.key_value.types import KeyValueFilter, KeyValueResource
from superset.utils.core import json_dumps_w_dates
from superset.utils.json import json_dumps_w_dates
HASHIDS_MIN_LENGTH = 11

6
superset/models/core.py
View File

@ -76,7 +76,7 @@ from superset.models.helpers import AuditMixinNullable, ImportExportMixin
from superset.result_set import SupersetResultSet
from superset.sql_parse import Table
from superset.superset_typing import OAuth2ClientConfig, ResultSetColumnType
from superset.utils import cache as cache_util, core as utils
from superset.utils import cache as cache_util, core as utils, json as json_utils
from superset.utils.backports import StrEnum
from superset.utils.core import DatasourceName, get_username
from superset.utils.oauth2 import get_oauth2_access_token
@ -601,7 +601,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
for col, coltype in df.dtypes.to_dict().items():
if coltype == numpy.object_ and column_needs_conversion(df[col]):
df[col] = df[col].apply(utils.json_dumps_w_dates)
df[col] = df[col].apply(json_utils.json_dumps_w_dates)
return df
@property
@ -957,7 +957,7 @@ class Database(Model, AuditMixinNullable, ImportExportMixin): # pylint: disable
def _convert(value: Any) -> Any:
try:
return utils.base_json_conv(value)
return json_utils.base_json_conv(value)
except TypeError:
return None

4
superset/models/dashboard.py
View File

@ -51,7 +51,7 @@ from superset.models.user_attributes import UserAttribute
from superset.tasks.thumbnails import cache_dashboard_thumbnail
from superset.tasks.utils import get_current_user
from superset.thumbnails.digest import get_dashboard_digest
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
metadata = Model.metadata # pylint: disable=no-member
config = app.config
@ -372,7 +372,7 @@ class Dashboard(AuditMixinNullable, ImportExportMixin, Model):
return json.dumps(
{"dashboards": copied_dashboards, "datasources": eager_datasources},
cls=utils.DashboardEncoder,
cls=json_utils.DashboardEncoder,
indent=4,
)

4
superset/result_set.py
View File

@ -28,7 +28,7 @@ from numpy.typing import NDArray
from superset.db_engine_specs import BaseEngineSpec
from superset.superset_typing import DbapiDescription, DbapiResult, ResultSetColumnType
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
from superset.utils.core import GenericDataType
logger = logging.getLogger(__name__)
@ -61,7 +61,7 @@ def dedup(l: list[str], suffix: str = "__", case_sensitive: bool = True) -> list
def stringify(obj: Any) -> str:
return json.dumps(obj, default=utils.json_iso_dttm_ser)
return json_utils.dumps(obj, default=json_utils.json_iso_dttm_ser)
def stringify_values(array: NDArray[Any]) -> NDArray[Any]:

11
superset/sql_lab.py
View File

@ -24,7 +24,6 @@ from typing import Any, cast, Optional, Union
import backoff
import msgpack
import simplejson as json
from celery.exceptions import SoftTimeLimitExceeded
from flask_babel import gettext as __
@ -59,8 +58,8 @@ from superset.sql_parse import (
)
from superset.sqllab.limiting_factor import LimitingFactor
from superset.sqllab.utils import write_ipc_buffer
from superset.utils import json as json_utils
from superset.utils.core import (
json_iso_dttm_ser,
override_user,
QuerySource,
zlib_compress,
@ -349,9 +348,13 @@ def _serialize_payload(
) -> Union[bytes, str]:
logger.debug("Serializing to msgpack: %r", use_msgpack)
if use_msgpack:
return msgpack.dumps(payload, default=json_iso_dttm_ser, use_bin_type=True)
return msgpack.dumps(
payload, default=json_utils.json_iso_dttm_ser, use_bin_type=True
)
return json.dumps(payload, default=json_iso_dttm_ser, ignore_nan=True)
return json_utils.dumps(
payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True
)
def _serialize_and_expand_data(

11
superset/sqllab/api.py
View File

@ -18,7 +18,6 @@ import logging
from typing import Any, cast, Optional
from urllib import parse
import simplejson as json
from flask import request, Response
from flask_appbuilder import permission_name
from flask_appbuilder.api import expose, protect, rison, safe
@ -62,7 +61,7 @@ from superset.sqllab.sqllab_execution_context import SqlJsonExecutionContext
from superset.sqllab.utils import bootstrap_sqllab_data
from superset.sqllab.validators import CanAccessQueryValidatorImpl
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
from superset.views.base import CsvResponse, generate_download_headers, json_success
from superset.views.base_api import BaseSupersetApi, requires_json, statsd_metrics
@ -132,9 +131,9 @@ class SqlLabRestApi(BaseSupersetApi):
result = bootstrap_sqllab_data(user_id)
return json_success(
json.dumps(
json_utils.dumps(
{"result": result},
default=utils.json_iso_dttm_ser,
default=json_utils.json_iso_dttm_ser,
ignore_nan=True,
),
200,
@ -343,9 +342,9 @@ class SqlLabRestApi(BaseSupersetApi):
# Using pessimistic json serialization since some database drivers can return
# unserializeable types at times
payload = json.dumps(
payload = json_utils.dumps(
result,
default=utils.pessimistic_json_iso_dttm_ser,
default=json_utils.pessimistic_json_iso_dttm_ser,
ignore_nan=True,
)
return json_success(payload, 200)

31
superset/sqllab/execution_context_convertor.py
View File

@ -19,11 +19,9 @@ from __future__ import annotations
import logging
from typing import Any, TYPE_CHECKING
import simplejson as json
import superset.utils.core as utils
from superset.sqllab.command_status import SqlJsonExecutionStatus
from superset.sqllab.utils import apply_display_max_row_configuration_if_require
from superset.utils import json as json_utils
logger = logging.getLogger(__name__)
@ -52,23 +50,16 @@ class ExecutionContextConvertor:
def serialize_payload(self) -> str:
if self._exc_status == SqlJsonExecutionStatus.HAS_RESULTS:
sql_results = apply_display_max_row_configuration_if_require(
self.payload, self._max_row_in_display_configuration
return json_utils.dumps(
apply_display_max_row_configuration_if_require(
self.payload, self._max_row_in_display_configuration
),
default=json_utils.pessimistic_json_iso_dttm_ser,
ignore_nan=True,
)
try:
return json.dumps(
sql_results,
default=utils.pessimistic_json_iso_dttm_ser,
ignore_nan=True,
)
except UnicodeDecodeError:
return json.dumps(
sql_results,
default=utils.pessimistic_json_iso_dttm_ser,
ensure_ascii=False,
ignore_nan=True,
)
return json.dumps(
{"query": self.payload}, default=utils.json_int_dttm_ser, ignore_nan=True
return json_utils.dumps(
{"query": self.payload},
default=json_utils.json_int_dttm_ser,
ignore_nan=True,
)

2
superset/utils/cache.py
View File

@ -30,8 +30,8 @@ from werkzeug.wrappers import Response
from superset import db
from superset.extensions import cache_manager
from superset.models.cache import CacheKey
from superset.utils.core import json_int_dttm_ser
from superset.utils.hashing import md5_sha_from_dict
from superset.utils.json import json_int_dttm_ser
if TYPE_CHECKING:
from superset.stats_logger import BaseStatsLogger

146
superset/utils/core.py
View File

@ -21,9 +21,7 @@ from __future__ import annotations
import _thread
import collections
import decimal
import errno
import json
import logging
import os
import platform
@ -40,7 +38,7 @@ import zlib
from collections.abc import Iterable, Iterator, Sequence
from contextlib import closing, contextmanager
from dataclasses import dataclass
from datetime import date, datetime, time, timedelta
from datetime import timedelta
from email.mime.application import MIMEApplication
from email.mime.image import MIMEImage
from email.mime.multipart import MIMEMultipart
@ -56,7 +54,6 @@ from zipfile import ZipFile
import markdown as md
import nh3
import numpy as np
import pandas as pd
import sqlalchemy as sa
from cryptography.hazmat.backends import default_backend
@ -65,7 +62,6 @@ from flask import current_app, g, request
from flask_appbuilder import SQLA
from flask_appbuilder.security.sqla.models import User
from flask_babel import gettext as __
from flask_babel.speaklater import LazyString
from markupsafe import Markup
from pandas.api.types import infer_dtype
from pandas.core.dtypes.common import is_numeric_dtype
@ -103,7 +99,6 @@ from superset.superset_typing import (
from superset.utils.backports import StrEnum
from superset.utils.database import get_example_database
from superset.utils.date_parser import parse_human_timedelta
from superset.utils.dates import datetime_to_epoch, EPOCH
from superset.utils.hashing import md5_sha_from_dict, md5_sha_from_str
if TYPE_CHECKING:
@ -418,136 +413,6 @@ def cast_to_boolean(value: Any) -> bool | None:
return False
class DashboardEncoder(json.JSONEncoder):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.sort_keys = True
def default(self, o: Any) -> dict[Any, Any] | str:
if isinstance(o, uuid.UUID):
return str(o)
try:
vals = {k: v for k, v in o.__dict__.items() if k != "_sa_instance_state"}
return {f"__{o.__class__.__name__}__": vals}
except Exception: # pylint: disable=broad-except
if isinstance(o, datetime):
return {"__datetime__": o.replace(microsecond=0).isoformat()}
return json.JSONEncoder(sort_keys=True).default(o)
def format_timedelta(time_delta: timedelta) -> str:
"""
Ensures negative time deltas are easily interpreted by humans
>>> td = timedelta(0) - timedelta(days=1, hours=5,minutes=6)
>>> str(td)
'-2 days, 18:54:00'
>>> format_timedelta(td)
'-1 day, 5:06:00'
"""
if time_delta < timedelta(0):
return "-" + str(abs(time_delta))
# Change this to format positive time deltas the way you want
return str(time_delta)
def base_json_conv(obj: Any) -> Any:
"""
Tries to convert additional types to JSON compatible forms.
:param obj: The serializable object
:returns: The JSON compatible form
:raises TypeError: If the object cannot be serialized
:see: https://docs.python.org/3/library/json.html#encoders-and-decoders
"""
if isinstance(obj, memoryview):
obj = obj.tobytes()
if isinstance(obj, np.int64):
return int(obj)
if isinstance(obj, np.bool_):
return bool(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
if isinstance(obj, set):
return list(obj)
if isinstance(obj, decimal.Decimal):
return float(obj)
if isinstance(obj, (uuid.UUID, time, LazyString)):
return str(obj)
if isinstance(obj, timedelta):
return format_timedelta(obj)
if isinstance(obj, bytes):
try:
return obj.decode("utf-8")
except Exception: # pylint: disable=broad-except
try:
return obj.decode("utf-16")
except Exception: # pylint: disable=broad-except
return "[bytes]"
raise TypeError(f"Unserializable object {obj} of type {type(obj)}")
def json_iso_dttm_ser(obj: Any, pessimistic: bool = False) -> Any:
"""
A JSON serializer that deals with dates by serializing them to ISO 8601.
>>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_iso_dttm_ser)
'{"dttm": "1970-01-01T00:00:00"}'
:param obj: The serializable object
:param pessimistic: Whether to be pessimistic regarding serialization
:returns: The JSON compatible form
:raises TypeError: If the non-pessimistic object cannot be serialized
"""
if isinstance(obj, (datetime, date, pd.Timestamp)):
return obj.isoformat()
try:
return base_json_conv(obj)
except TypeError as ex:
if pessimistic:
logger.error("Failed to serialize %s", obj)
return f"Unserializable [{type(obj)}]"
raise ex
def pessimistic_json_iso_dttm_ser(obj: Any) -> Any:
"""Proxy to call json_iso_dttm_ser in a pessimistic way
If one of object is not serializable to json, it will still succeed"""
return json_iso_dttm_ser(obj, pessimistic=True)
def json_int_dttm_ser(obj: Any) -> Any:
"""
A JSON serializer that deals with dates by serializing them to EPOCH.
>>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_int_dttm_ser)
'{"dttm": 0.0}'
:param obj: The serializable object
:returns: The JSON compatible form
:raises TypeError: If the object cannot be serialized
"""
if isinstance(obj, (datetime, pd.Timestamp)):
return datetime_to_epoch(obj)
if isinstance(obj, date):
return (obj - EPOCH.date()).total_seconds() * 1000
return base_json_conv(obj)
def json_dumps_w_dates(payload: dict[Any, Any], sort_keys: bool = False) -> str:
"""Dumps payload to JSON with Datetime objects properly converted"""
return json.dumps(payload, default=json_int_dttm_ser, sort_keys=sort_keys)
def error_msg_from_exception(ex: Exception) -> str:
"""Translate exception into error message
@ -691,15 +556,6 @@ def get_datasource_full_name(
return ".".join([f"[{part}]" for part in parts if part])
def validate_json(obj: bytes | bytearray | str) -> None:
if obj:
try:
json.loads(obj)
except Exception as ex:
logger.error("JSON is not valid %s", str(ex), exc_info=True)
raise SupersetException("JSON is not valid") from ex
class SigalrmTimeout:
"""
To be used in a ``with`` block and timeout its content.

211
superset/utils/json.py Normal file
View File

@ -0,0 +1,211 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import decimal
import json
import logging
import uuid
from datetime import date, datetime, time, timedelta
from typing import Any, Callable, Optional, Union
import numpy as np
import pandas as pd
import simplejson
from flask_babel.speaklater import LazyString
from superset.exceptions import SupersetException
from superset.utils.dates import datetime_to_epoch, EPOCH
logging.getLogger("MARKDOWN").setLevel(logging.INFO)
logger = logging.getLogger(__name__)
class DashboardEncoder(json.JSONEncoder):
def __init__(self, *args: Any, **kwargs: Any) -> None:
super().__init__(*args, **kwargs)
self.sort_keys = True
def default(self, o: Any) -> Union[dict[Any, Any], str]:
if isinstance(o, uuid.UUID):
return str(o)
try:
vals = {k: v for k, v in o.__dict__.items() if k != "_sa_instance_state"}
return {f"__{o.__class__.__name__}__": vals}
except Exception: # pylint: disable=broad-except
if isinstance(o, datetime):
return {"__datetime__": o.replace(microsecond=0).isoformat()}
return json.JSONEncoder(sort_keys=True).default(o)
def format_timedelta(time_delta: timedelta) -> str:
"""
Ensures negative time deltas are easily interpreted by humans
>>> td = timedelta(0) - timedelta(days=1, hours=5,minutes=6)
>>> str(td)
'-2 days, 18:54:00'
>>> format_timedelta(td)
'-1 day, 5:06:00'
"""
if time_delta < timedelta(0):
return "-" + str(abs(time_delta))
# Change this to format positive time deltas the way you want
return str(time_delta)
def base_json_conv(obj: Any) -> Any:
"""
Tries to convert additional types to JSON compatible forms.
:param obj: The serializable object
:returns: The JSON compatible form
:raises TypeError: If the object cannot be serialized
:see: https://docs.python.org/3/library/json.html#encoders-and-decoders
"""
if isinstance(obj, memoryview):
obj = obj.tobytes()
if isinstance(obj, np.int64):
return int(obj)
if isinstance(obj, np.bool_):
return bool(obj)
if isinstance(obj, np.ndarray):
return obj.tolist()
if isinstance(obj, set):
return list(obj)
if isinstance(obj, decimal.Decimal):
return float(obj)
if isinstance(obj, (uuid.UUID, time, LazyString)):
return str(obj)
if isinstance(obj, timedelta):
return format_timedelta(obj)
if isinstance(obj, bytes):
try:
return obj.decode("utf-8")
except Exception: # pylint: disable=broad-except
try:
return obj.decode("utf-16")
except Exception: # pylint: disable=broad-except
return "[bytes]"
raise TypeError(f"Unserializable object {obj} of type {type(obj)}")
def json_iso_dttm_ser(obj: Any, pessimistic: bool = False) -> Any:
"""
A JSON serializer that deals with dates by serializing them to ISO 8601.
>>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_iso_dttm_ser)
'{"dttm": "1970-01-01T00:00:00"}'
:param obj: The serializable object
:param pessimistic: Whether to be pessimistic regarding serialization
:returns: The JSON compatible form
:raises TypeError: If the non-pessimistic object cannot be serialized
"""
if isinstance(obj, (datetime, date, pd.Timestamp)):
return obj.isoformat()
try:
return base_json_conv(obj)
except TypeError as ex:
if pessimistic:
logger.error("Failed to serialize %s", obj)
return f"Unserializable [{type(obj)}]"
raise ex
def pessimistic_json_iso_dttm_ser(obj: Any) -> Any:
"""Proxy to call json_iso_dttm_ser in a pessimistic way
If one of object is not serializable to json, it will still succeed"""
return json_iso_dttm_ser(obj, pessimistic=True)
def json_int_dttm_ser(obj: Any) -> Any:
"""
A JSON serializer that deals with dates by serializing them to EPOCH.
>>> json.dumps({'dttm': datetime(1970, 1, 1)}, default=json_int_dttm_ser)
'{"dttm": 0.0}'
:param obj: The serializable object
:returns: The JSON compatible form
:raises TypeError: If the object cannot be serialized
"""
if isinstance(obj, (datetime, pd.Timestamp)):
return datetime_to_epoch(obj)
if isinstance(obj, date):
return (obj - EPOCH.date()).total_seconds() * 1000
return base_json_conv(obj)
def json_dumps_w_dates(payload: dict[Any, Any], sort_keys: bool = False) -> str:
"""Dumps payload to JSON with Datetime objects properly converted"""
return dumps(payload, default=json_int_dttm_ser, sort_keys=sort_keys)
def validate_json(obj: Union[bytes, bytearray, str]) -> None:
"""
A JSON Validator that validates an object of bytes, bytes array or string
to be in valid JSON format
:raises SupersetException: if obj is not serializable to JSON
:param obj: an object that should be parseable to JSON
"""
if obj:
try:
json.loads(obj)
except Exception as ex:
logger.error("JSON is not valid %s", str(ex), exc_info=True)
raise SupersetException("JSON is not valid") from ex
def dumps(
obj: Any,
default: Optional[Callable[[Any], Any]] = json_iso_dttm_ser,
ignore_nan: bool = True,
sort_keys: bool = False,
) -> str:
"""
Dumps object to compatible JSON format
:param obj: The serializable object
:param default: function that should return a serializable version of obj
:param ignore_nan: when set to True nan values will be ignored
:param sort_keys: when set to True keys will be sorted
:returns: String object in the JSON compatible form
"""
results_string = ""
try:
results_string = simplejson.dumps(
obj, default=default, ignore_nan=ignore_nan, sort_keys=sort_keys
)
except UnicodeDecodeError:
results_string = simplejson.dumps( # type: ignore[call-overload]
obj,
default=default,
ignore_nan=ignore_nan,
sort_keys=sort_keys,
encoding=None,
)
return results_string

4
superset/utils/schema.py
View File

@ -19,7 +19,7 @@ from typing import Any, Union
from marshmallow import validate, ValidationError
from superset.exceptions import SupersetException
from superset.utils import core as utils
from superset.utils import json as json_utils
class OneOfCaseInsensitive(validate.OneOf):
@ -49,6 +49,6 @@ def validate_json(value: Union[bytes, bytearray, str]) -> None:
:param value: an object that should be parseable to JSON
"""
try:
utils.validate_json(value)
json_utils.validate_json(value)
except SupersetException as ex:
raise ValidationError("JSON not valid") from ex

6
superset/views/api.py
View File

@ -33,7 +33,7 @@ from superset.commands.chart.exceptions import (
from superset.legacy import update_time_range
from superset.models.slice import Slice
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import json as json_utils
from superset.utils.date_parser import get_since_until
from superset.views.base import api, BaseSupersetView, handle_api_exception
@ -73,8 +73,8 @@ class Api(BaseSupersetView):
query_context.raise_for_access()
result = query_context.get_payload()
payload_json = result["queries"]
return json.dumps(
payload_json, default=utils.json_int_dttm_ser, ignore_nan=True
return json_utils.dumps(
payload_json, default=json_utils.json_int_dttm_ser, ignore_nan=True
)
@event_logger.log_this

40
superset/views/base.py
View File

@ -25,7 +25,6 @@ from datetime import datetime
from importlib.resources import files
from typing import Any, Callable, cast
import simplejson as json
import yaml
from babel import Locale
from flask import (
@ -51,14 +50,13 @@ from flask_appbuilder.security.decorators import (
)
from flask_appbuilder.security.sqla.models import User
from flask_appbuilder.widgets import ListWidget
from flask_babel import get_locale, gettext as __, lazy_gettext as _
from flask_babel import get_locale, gettext as __
from flask_jwt_extended.exceptions import NoAuthorizationError
from flask_wtf.csrf import CSRFError
from flask_wtf.form import FlaskForm
from sqlalchemy import exc
from sqlalchemy.orm import Query
from werkzeug.exceptions import HTTPException
from wtforms import Form
from wtforms.fields.core import Field, UnboundField
from superset import (
@ -85,7 +83,7 @@ from superset.models.helpers import ImportExportMixin
from superset.reports.models import ReportRecipientType
from superset.superset_typing import FlaskResponse
from superset.translations.utils import get_language_pack
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
from superset.utils.filters import get_dataset_access_filters
from .utils import bootstrap_user_data
@ -155,7 +153,9 @@ def json_error_response(
payload = payload or {"error": f"{msg}"}
return Response(
json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True),
json_utils.dumps(
payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True
),
status=status,
mimetype="application/json",
)
@ -170,7 +170,9 @@ def json_errors_response(
payload["errors"] = [dataclasses.asdict(error) for error in errors]
return Response(
json.dumps(payload, default=utils.json_iso_dttm_ser, ignore_nan=True),
json_utils.dumps(
payload, default=json_utils.json_iso_dttm_ser, ignore_nan=True
),
status=status,
mimetype="application/json; charset=utf-8",
)
@ -292,7 +294,9 @@ class BaseSupersetView(BaseView):
@staticmethod
def json_response(obj: Any, status: int = 200) -> FlaskResponse:
return Response(
json.dumps(obj, default=utils.json_int_dttm_ser, ignore_nan=True),
json_utils.dumps(
obj, default=json_utils.json_int_dttm_ser, ignore_nan=True
),
status=status,
mimetype="application/json",
)
@ -308,8 +312,8 @@ class BaseSupersetView(BaseView):
return self.render_template(
"superset/spa.html",
entry="spa",
bootstrap_data=json.dumps(
payload, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
payload, default=json_utils.pessimistic_json_iso_dttm_ser
),
)
@ -543,9 +547,9 @@ def show_unexpected_exception(ex: Exception) -> FlaskResponse:
@superset_app.context_processor
def get_common_bootstrap_data() -> dict[str, Any]:
def serialize_bootstrap_data() -> str:
return json.dumps(
return json_utils.dumps(
{"common": common_bootstrap_payload()},
default=utils.pessimistic_json_iso_dttm_ser,
default=json_utils.pessimistic_json_iso_dttm_ser,
)
return {"bootstrap_data": serialize_bootstrap_data}
@ -626,8 +630,8 @@ class SupersetModelView(ModelView):
return self.render_template(
"superset/spa.html",
entry="spa",
bootstrap_data=json.dumps(
payload, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
payload, default=json_utils.pessimistic_json_iso_dttm_ser
),
)
@ -640,16 +644,6 @@ class ListWidgetWithCheckboxes(ListWidget): # pylint: disable=too-few-public-me
template = "superset/fab_overrides/list_with_checkboxes.html"
def validate_json(form: Form, field: Field) -> None: # pylint: disable=unused-argument
try:
json.loads(field.data)
except Exception as ex:
logger.exception(ex)
raise Exception( # pylint: disable=broad-exception-raised
_("json isn't valid")
) from ex
class YamlExportMixin: # pylint: disable=too-few-public-methods
"""
Override this if you want a dict response instead, with a certain key.

6
superset/views/chart/views.py
View File

@ -22,7 +22,7 @@ from superset import security_manager
from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod
from superset.models.slice import Slice
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import json as json_utils
from superset.views.base import DeleteMixin, DeprecateModelViewMixin, SupersetModelView
from superset.views.chart.mixin import SliceMixin
@ -41,10 +41,10 @@ class SliceModelView(
method_permission_name = MODEL_VIEW_RW_METHOD_PERMISSION_MAP
def pre_add(self, item: "SliceModelView") -> None:
utils.validate_json(item.params)
json_utils.validate_json(item.params)
def pre_update(self, item: "SliceModelView") -> None:
utils.validate_json(item.params)
json_utils.validate_json(item.params)
security_manager.raise_for_ownership(item)
def pre_delete(self, item: "SliceModelView") -> None:

19
superset/views/core.py
View File

@ -71,10 +71,9 @@ from superset.models.slice import Slice
from superset.models.sql_lab import Query
from superset.models.user_attributes import UserAttribute
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
from superset.utils.cache import etag_cache
from superset.utils.core import (
base_json_conv,
DatasourceType,
get_user_id,
ReservedUrlParameters,
@ -576,8 +575,8 @@ class Superset(BaseSupersetView):
return self.render_template(
"superset/basic.html",
bootstrap_data=json.dumps(
bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser
),
entry="explore",
title=title,
@ -753,7 +752,7 @@ class Superset(BaseSupersetView):
)
return json_success(
json.dumps(
json_utils.dumps(
[
{
"slice_id" if key == "chart_id" else key: value
@ -765,7 +764,7 @@ class Superset(BaseSupersetView):
}
for slc in slices
],
default=base_json_conv,
default=json_utils.base_json_conv,
),
)
@ -814,12 +813,12 @@ class Superset(BaseSupersetView):
"superset/spa.html",
entry="spa",
title=dashboard.dashboard_title, # dashboard title is always visible
bootstrap_data=json.dumps(
bootstrap_data=json_utils.dumps(
{
"user": bootstrap_user_data(g.user, include_perms=True),
"common": common_bootstrap_payload(),
},
default=utils.pessimistic_json_iso_dttm_ser,
default=json_utils.pessimistic_json_iso_dttm_ser,
),
standalone_mode=ReservedUrlParameters.is_standalone_mode(),
)
@ -919,8 +918,8 @@ class Superset(BaseSupersetView):
return self.render_template(
"superset/spa.html",
entry="spa",
bootstrap_data=json.dumps(
payload, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
payload, default=json_utils.pessimistic_json_iso_dttm_ser
),
)

7
superset/views/dashboard/views.py
View File

@ -15,7 +15,6 @@
# specific language governing permissions and limitations
# under the License.
import builtins
import json
from typing import Callable, Union
from flask import g, redirect, request, Response
@ -35,7 +34,7 @@ from superset import db, event_logger, is_feature_enabled
from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod
from superset.models.dashboard import Dashboard as DashboardModel
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import json as json_utils
from superset.views.base import (
BaseSupersetView,
common_bootstrap_payload,
@ -160,8 +159,8 @@ class Dashboard(BaseSupersetView):
return self.render_template(
"superset/spa.html",
entry="embedded",
bootstrap_data=json.dumps(
bootstrap_data, default=utils.pessimistic_json_iso_dttm_ser
bootstrap_data=json_utils.dumps(
bootstrap_data, default=json_utils.pessimistic_json_iso_dttm_ser
),
)

4
superset/views/sql_lab/views.py
View File

@ -28,7 +28,7 @@ from superset import db
from superset.constants import MODEL_VIEW_RW_METHOD_PERMISSION_MAP, RouteMethod
from superset.models.sql_lab import Query, SavedQuery, TableSchema, TabState
from superset.superset_typing import FlaskResponse
from superset.utils import core as utils
from superset.utils import json as json_utils
from superset.utils.core import get_user_id
from superset.views.base import (
BaseSupersetView,
@ -140,7 +140,7 @@ class TabStateView(BaseSupersetView):
if tab_state is None:
return Response(status=404)
return json_success(
json.dumps(tab_state.to_dict(), default=utils.json_iso_dttm_ser)
json.dumps(tab_state.to_dict(), default=json_utils.json_iso_dttm_ser)
)
@has_access_api

8
superset/views/tags.py
View File

@ -17,16 +17,16 @@
import logging
import simplejson as json
from flask_appbuilder import expose
from flask_appbuilder.hooks import before_request
from flask_appbuilder.models.sqla.interface import SQLAInterface
from flask_appbuilder.security.decorators import has_access, has_access_api
from werkzeug.exceptions import NotFound
from superset import db, is_feature_enabled, utils
from superset import db, is_feature_enabled
from superset.superset_typing import FlaskResponse
from superset.tags.models import Tag
from superset.utils import json as json_utils
from superset.views.base import SupersetModelView
from .base import BaseSupersetView, json_success
@ -74,4 +74,6 @@ class TagView(BaseSupersetView):
}
for obj in query
]
return json_success(json.dumps(results, default=utils.core.json_int_dttm_ser))
return json_success(
json_utils.dumps(results, default=json_utils.json_int_dttm_ser)
)

10
superset/viz.py
View File

@ -66,7 +66,7 @@ from superset.superset_typing import (
VizData,
VizPayload,
)
from superset.utils import core as utils, csv
from superset.utils import core as utils, csv, json as json_utils
from superset.utils.cache import set_and_log_cache
from superset.utils.core import (
apply_max_row_limit,
@ -440,8 +440,8 @@ class BaseViz: # pylint: disable=too-many-public-methods
@deprecated(deprecated_in="3.0")
def get_json(self) -> str:
return json.dumps(
self.get_payload(), default=utils.json_int_dttm_ser, ignore_nan=True
return json_utils.dumps(
self.get_payload(), default=json_utils.json_int_dttm_ser, ignore_nan=True
)
@deprecated(deprecated_in="3.0")
@ -641,9 +641,9 @@ class BaseViz: # pylint: disable=too-many-public-methods
@staticmethod
@deprecated(deprecated_in="3.0")
def json_dumps(query_obj: Any, sort_keys: bool = False) -> str:
return json.dumps(
return json_utils.dumps(
query_obj,
default=utils.json_int_dttm_ser,
default=json_utils.json_int_dttm_ser,
ignore_nan=True,
sort_keys=sort_keys,
)

2
tests/integration_tests/conftest.py
View File

@ -29,8 +29,8 @@ from sqlalchemy.engine import Engine
from superset import db, security_manager
from superset.extensions import feature_flag_manager
from superset.utils.core import json_dumps_w_dates
from superset.utils.database import get_example_database, remove_database
from superset.utils.json import json_dumps_w_dates
from tests.integration_tests.test_app import app, login
if TYPE_CHECKING:

6
tests/integration_tests/core_tests.py
View File

@ -51,7 +51,7 @@ from superset.models.slice import Slice
from superset.models.sql_lab import Query
from superset.result_set import SupersetResultSet
from superset.sql_parse import Table
from superset.utils import core as utils
from superset.utils import core as utils, json as json_utils
from superset.utils.core import backend
from superset.utils.database import get_example_database
from superset.views.database.views import DatabaseView
@ -502,7 +502,7 @@ class TestCore(SupersetTestCase):
results = SupersetResultSet(list(data), [["data"]], BaseEngineSpec)
df = results.to_pandas_df()
data = dataframe.df_to_records(df)
json_str = json.dumps(data, default=utils.pessimistic_json_iso_dttm_ser)
json_str = json.dumps(data, default=json_utils.pessimistic_json_iso_dttm_ser)
self.assertDictEqual(
data[0], {"data": pd.Timestamp("2017-11-18 21:53:00.219225+0100", tz=tz)}
)
@ -943,7 +943,7 @@ class TestCore(SupersetTestCase):
encoded = json.dumps(
{"FOO": lambda x: 1, "super": "set"},
default=utils.pessimistic_json_iso_dttm_ser,
default=json_utils.pessimistic_json_iso_dttm_ser,
)
html_string = (
html.escape(encoded, quote=False)

6
tests/integration_tests/sqllab_tests.py
View File

@ -44,10 +44,8 @@ from superset.sql_lab import (
apply_limit_if_exists,
)
from superset.sql_parse import CtasMethod
from superset.utils.core import (
backend,
datetime_to_epoch, # noqa: F401
)
from superset.utils.core import backend
from superset.utils.json import datetime_to_epoch # noqa: F401
from superset.utils.database import get_example_database, get_main_database
from tests.integration_tests.base_tests import SupersetTestCase

12
tests/integration_tests/utils_tests.py
View File

@ -45,20 +45,16 @@ from superset.models.core import Database, Log
from superset.models.dashboard import Dashboard # noqa: F401
from superset.models.slice import Slice # noqa: F401
from superset.utils.core import (
base_json_conv,
cast_to_num,
convert_legacy_filters_into_adhoc,
create_ssl_cert_file,
DTTM_ALIAS,
extract_dataframe_dtypes,
format_timedelta,
GenericDataType,
get_form_data_token,
as_list,
get_email_address_list,
get_stacktrace,
json_int_dttm_ser,
json_iso_dttm_ser,
merge_extra_filters,
merge_extra_form_data,
merge_request_params,
@ -66,11 +62,17 @@ from superset.utils.core import (
parse_ssl_cert,
parse_js_uri_path_item,
split,
validate_json,
zlib_compress,
zlib_decompress,
DateColumn,
)
from superset.utils.json import (
base_json_conv,
format_timedelta,
json_int_dttm_ser,
json_iso_dttm_ser,
validate_json,
)
from superset.utils.database import get_or_create_db
from superset.utils import schema
from superset.utils.hashing import md5_sha_from_str

114
tests/unit_tests/utils/json_tests.py Normal file
View File

@ -0,0 +1,114 @@
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import datetime
import json
from unittest.mock import MagicMock
import pytest
from superset.exceptions import SupersetException
from superset.utils.json import (
dumps,
json_iso_dttm_ser,
pessimistic_json_iso_dttm_ser,
validate_json,
)
def test_json_dumps():
data = {
"str": "some string",
"int": 123456789,
"float": 0.12345,
"bool": True,
}
json_str = dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["str"] == "some string"
assert reloaded_data["int"] == 123456789
assert reloaded_data["float"] == 0.12345
assert reloaded_data["bool"] is True
def test_json_dumps_encoding():
data = {
"utf8": b"Hello World",
"utf16": b"\xff\xfeH\x00e\x00l\x00l\x00o\x00 \x00W\x00o\x00r\x00l\x00d\x00",
"bytes": b"\xff",
}
json_str = dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["utf8"] == "Hello World"
assert reloaded_data["utf16"] == "Hello World"
assert reloaded_data["bytes"] == "[bytes]"
def test_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
}
json_str = json.dumps(data, default=json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
assert reloaded_data["date"] == "2021-01-01"
def test_pessimistic_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
"UNSERIALIZABLE": MagicMock(),
}
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
assert reloaded_data["date"] == "2021-01-01"
assert (
reloaded_data["UNSERIALIZABLE"]
== "Unserializable [<class 'unittest.mock.MagicMock'>]"
)
def test_pessimistic_json_iso_dttm_ser_nonutf8():
data = {
"INVALID_UTF8_BYTES": b"\xff",
}
assert isinstance(data["INVALID_UTF8_BYTES"], bytes)
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["INVALID_UTF8_BYTES"] == "[bytes]"
def test_pessimistic_json_iso_dttm_ser_utf16():
data = {
"VALID_UTF16_BYTES": b"\xff\xfeS0\x930k0a0o0\x16NLu",
}
assert isinstance(data["VALID_UTF16_BYTES"], bytes)
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["VALID_UTF16_BYTES"] == "こんにちは世界"
def test_validate_json():
valid = '{"a": 5, "b": [1, 5, ["g", "h"]]}'
assert validate_json(valid) is None
invalid = '{"a": 5, "b": [1, 5, ["g", "h]]}'
with pytest.raises(SupersetException) as excinfo:
validate_json(invalid)
assert str(excinfo.value) == "JSON is not valid"

51
tests/unit_tests/utils/test_core.py
View File

@ -14,8 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
import datetime
import json
import os
from dataclasses import dataclass
from typing import Any, Optional
@ -33,10 +31,8 @@ from superset.utils.core import (
generic_find_fk_constraint_name,
get_datasource_full_name,
is_test,
json_iso_dttm_ser,
normalize_dttm_col,
parse_boolean_string,
pessimistic_json_iso_dttm_ser,
QueryObjectFilterClause,
remove_extra_adhoc_filters,
)
@ -400,50 +396,3 @@ def test_get_datasource_full_name():
get_datasource_full_name("db", "table", "catalog", None)
== "[db].[catalog].[table]"
)
def test_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
}
json_str = json.dumps(data, default=json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
assert reloaded_data["date"] == "2021-01-01"
def test_pessimistic_json_iso_dttm_ser():
data = {
"datetime": datetime.datetime(2021, 1, 1, 0, 0, 0),
"date": datetime.date(2021, 1, 1),
"UNSERIALIZABLE": MagicMock(),
}
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["datetime"] == "2021-01-01T00:00:00"
assert reloaded_data["date"] == "2021-01-01"
assert (
reloaded_data["UNSERIALIZABLE"]
== "Unserializable [<class 'unittest.mock.MagicMock'>]"
)
def test_pessimistic_json_iso_dttm_ser_nonutf8():
data = {
"INVALID_UTF8_BYTES": b"\xff",
}
assert isinstance(data["INVALID_UTF8_BYTES"], bytes)
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["INVALID_UTF8_BYTES"] == "[bytes]"
def test_pessimistic_json_iso_dttm_ser_utf16():
data = {
"VALID_UTF16_BYTES": b"\xff\xfeS0\x930k0a0o0\x16NLu",
}
assert isinstance(data["VALID_UTF16_BYTES"], bytes)
json_str = json.dumps(data, default=pessimistic_json_iso_dttm_ser)
reloaded_data = json.loads(json_str)
assert reloaded_data["VALID_UTF16_BYTES"] == "こんにちは世界"