From 500d90058f44dcbd4851f9110d9edf5b117fe92c Mon Sep 17 00:00:00 2001 From: Ville Brofeldt <33317356+villebro@users.noreply.github.com> Date: Fri, 31 Mar 2023 11:41:57 +0300 Subject: [PATCH] feat: add ability to disable cache (#23439) --- docs/docs/installation/cache.mdx | 3 +++ .../components/Datasource/DatasourceEditor.jsx | 2 +- .../components/Datasource/DatasourceModal.tsx | 4 ++++ .../explore/components/PropertiesModal/index.tsx | 2 +- .../data/database/DatabaseModal/ExtraOptions.tsx | 2 +- superset/common/query_context.py | 5 ++++- superset/common/query_context_processor.py | 12 +++++++----- superset/viz.py | 16 +++++++++------- tests/integration_tests/charts/data/api_tests.py | 8 ++++++++ 9 files changed, 38 insertions(+), 16 deletions(-) diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx index 58b4bcb2b..f69028ef8 100644 --- a/docs/docs/installation/cache.mdx +++ b/docs/docs/installation/cache.mdx @@ -66,6 +66,9 @@ The cache timeout for charts may be overridden by the settings for an individual database. Each of these configurations will be checked in order before falling back to the default value defined in `DATA_CACHE_CONFIG. +Note, that by setting the cache timeout to `-1`, caching for charting data can be disabled, either +per chart, dataset or database, or by default if set in `DATA_CACHE_CONFIG`. + ### SQL Lab Query Results Caching for SQL Lab query results is used when async queries are enabled and is configured using diff --git a/superset-frontend/src/components/Datasource/DatasourceEditor.jsx b/superset-frontend/src/components/Datasource/DatasourceEditor.jsx index f4d7e4857..34a702077 100644 --- a/superset-frontend/src/components/Datasource/DatasourceEditor.jsx +++ b/superset-frontend/src/components/Datasource/DatasourceEditor.jsx @@ -944,7 +944,7 @@ class DatasourceEditor extends React.PureComponent { fieldKey="cache_timeout" label={t('Cache timeout')} description={t( - 'The duration of time in seconds before the cache is invalidated', + 'The duration of time in seconds before the cache is invalidated. Set to -1 to bypass the cache.', )} control={} /> diff --git a/superset-frontend/src/components/Datasource/DatasourceModal.tsx b/superset-frontend/src/components/Datasource/DatasourceModal.tsx index 3f5e37746..90135f40f 100644 --- a/superset-frontend/src/components/Datasource/DatasourceModal.tsx +++ b/superset-frontend/src/components/Datasource/DatasourceModal.tsx @@ -101,6 +101,10 @@ const DatasourceModal: FunctionComponent = ({ postPayload: { data: { ...currentDatasource, + cache_timeout: + currentDatasource.cache_timeout === '' + ? null + : currentDatasource.cache_timeout, schema, metrics: currentDatasource?.metrics?.map( (metric: Record) => ({ diff --git a/superset-frontend/src/explore/components/PropertiesModal/index.tsx b/superset-frontend/src/explore/components/PropertiesModal/index.tsx index 9465fc87b..4cfd4b600 100644 --- a/superset-frontend/src/explore/components/PropertiesModal/index.tsx +++ b/superset-frontend/src/explore/components/PropertiesModal/index.tsx @@ -399,7 +399,7 @@ function PropertiesModal({ {t( - "Duration (in seconds) of the caching timeout for this chart. Note this defaults to the dataset's timeout if undefined.", + "Duration (in seconds) of the caching timeout for this chart. Set to -1 to bypass the cache. Note this defaults to the dataset's timeout if undefined.", )} diff --git a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx index f50d7597f..e171a2d26 100644 --- a/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx +++ b/superset-frontend/src/views/CRUD/data/database/DatabaseModal/ExtraOptions.tsx @@ -238,7 +238,7 @@ const ExtraOptions = ({
{t( 'Duration (in seconds) of the caching timeout for charts of this database.' + - ' A timeout of 0 indicates that the cache never expires.' + + ' A timeout of 0 indicates that the cache never expires, and -1 bypasses the cache.' + ' Note this defaults to the global timeout if undefined.', )}
diff --git a/superset/common/query_context.py b/superset/common/query_context.py index 3f3667709..78eb8800c 100644 --- a/superset/common/query_context.py +++ b/superset/common/query_context.py @@ -118,7 +118,10 @@ class QueryContext: query_obj: QueryObject, force_cached: Optional[bool] = False, ) -> Dict[str, Any]: - return self._processor.get_df_payload(query_obj, force_cached) + return self._processor.get_df_payload( + query_obj=query_obj, + force_cached=force_cached, + ) def get_query_result(self, query_object: QueryObject) -> QueryResult: return self._processor.get_query_result(query_object) diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index 703e1d71d..019422db3 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -106,11 +106,13 @@ class QueryContextProcessor: ) -> Dict[str, Any]: """Handles caching around the df payload retrieval""" cache_key = self.query_cache_key(query_obj) + timeout = self.get_cache_timeout() + force_query = self._query_context.force or timeout == -1 cache = QueryCacheManager.get( - cache_key, - CacheRegion.DATA, - self._query_context.force, - force_cached, + key=cache_key, + region=CacheRegion.DATA, + force_query=force_query, + force_cached=force_cached, ) if query_obj and cache_key and not cache.is_loaded: @@ -139,7 +141,7 @@ class QueryContextProcessor: key=cache_key, query_result=query_result, annotation_data=annotation_data, - force_query=self._query_context.force, + force_query=force_query, timeout=self.get_cache_timeout(), datasource_uid=self._qc_datasource.uid, region=CacheRegion.DATA, diff --git a/superset/viz.py b/superset/viz.py index 8c027a132..b021acd92 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -526,7 +526,9 @@ class BaseViz: # pylint: disable=too-many-public-methods is_loaded = False stacktrace = None df = None - if cache_key and cache_manager.data_cache and not self.force: + cache_timeout = self.cache_timeout + force = self.force or cache_timeout == -1 + if cache_key and cache_manager.data_cache and not force: cache_value = cache_manager.data_cache.get(cache_key) if cache_value: stats_logger.incr("loading_from_cache") @@ -609,16 +611,16 @@ class BaseViz: # pylint: disable=too-many-public-methods if is_loaded and cache_key and self.status != QueryStatus.FAILED: set_and_log_cache( - cache_manager.data_cache, - cache_key, - {"df": df, "query": self.query}, - self.cache_timeout, - self.datasource.uid, + cache_instance=cache_manager.data_cache, + cache_key=cache_key, + cache_value={"df": df, "query": self.query}, + cache_timeout=cache_timeout, + datasource_uid=self.datasource.uid, ) return { "cache_key": cache_key, "cached_dttm": cache_value["dttm"] if cache_value is not None else None, - "cache_timeout": self.cache_timeout, + "cache_timeout": cache_timeout, "df": df, "errors": self.errors, "form_data": self.form_data, diff --git a/tests/integration_tests/charts/data/api_tests.py b/tests/integration_tests/charts/data/api_tests.py index 2818793af..821c80ec4 100644 --- a/tests/integration_tests/charts/data/api_tests.py +++ b/tests/integration_tests/charts/data/api_tests.py @@ -1132,6 +1132,14 @@ def test_custom_cache_timeout(test_client, login_as_admin, physical_query_contex assert rv.json["result"][0]["cache_timeout"] == 5678 +def test_force_cache_timeout(test_client, login_as_admin, physical_query_context): + physical_query_context["custom_cache_timeout"] = -1 + test_client.post(CHART_DATA_URI, json=physical_query_context) + rv = test_client.post(CHART_DATA_URI, json=physical_query_context) + assert rv.json["result"][0]["cached_dttm"] is None + assert rv.json["result"][0]["is_cached"] is None + + @mock.patch( "superset.common.query_context_processor.config", {