iop
/
superset
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: Update time grain expressions for Spark >= 3.x (#18690) 

* Fix the time grain expressions for Spark >= 2.3.0

Spark removed date format string 'u' in Spark 3.0. Switch to using date_trunc which has been around since 2.3

* Review: Pull out time_grain_expressoins into its own thing
This commit is contained in:
Thomas Desrosiers 2022-03-08 16:21:13 -05:00 committed by GitHub
parent f9c7405e0e
commit 03b2b06e90
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 21 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
superset/db_engine_specs/databricks.py
View File

@ -21,6 +21,24 @@ from typing import Any, Dict, Optional
from superset.db_engine_specs.base import BaseEngineSpec
from superset.db_engine_specs.hive import HiveEngineSpec
time_grain_expressions = {
None: "{col}",
"PT1S": "date_trunc('second', {col})",
"PT1M": "date_trunc('minute', {col})",
"PT1H": "date_trunc('hour', {col})",
"P1D": "date_trunc('day', {col})",
"P1W": "date_trunc('week', {col})",
"P1M": "date_trunc('month', {col})",
"P3M": "date_trunc('quarter', {col})",
"P1Y": "date_trunc('year', {col})",
"P1W/1970-01-03T00:00:00Z": (
"date_trunc('week', {col} + interval '1 day') + interval '5 days'"
),
"1969-12-28T00:00:00Z/P1W": (
"date_trunc('week', {col} + interval '1 day') - interval '1 day'"
),
}
class DatabricksHiveEngineSpec(HiveEngineSpec):
engine = "databricks"
@ -28,16 +46,15 @@ class DatabricksHiveEngineSpec(HiveEngineSpec):
driver = "pyhive"
_show_functions_column = "function"
_time_grain_expressions = time_grain_expressions
class DatabricksODBCEngineSpec(BaseEngineSpec):
engine = "databricks"
engine_name = "Databricks SQL Endpoint"
driver = "pyodbc"
# the syntax for the ODBC engine is identical to the Hive one, so
# we can reuse the expressions from `HiveEngineSpec`
# pylint: disable=protected-access
_time_grain_expressions = HiveEngineSpec._time_grain_expressions
_time_grain_expressions = time_grain_expressions
@classmethod
def convert_dttm(