From 1214e7c78cf6ef940507332fa0b504b763ff4af2 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 10:53:43 +0200 Subject: [PATCH 01/15] chore(cache): default to SimpleCache in debug mode --- UPDATING.md | 1 + superset/common/query_context_processor.py | 8 +++-- superset/config.py | 41 ++++++++++++++-------- superset/sql_lab.py | 3 +- superset/utils/cache.py | 15 ++++++-- superset/utils/cache_manager.py | 30 ++++------------ superset/viz.py | 4 +-- 7 files changed, 56 insertions(+), 46 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index e00532fb4eddc..1ebbd1adda6f6 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -26,6 +26,7 @@ assists people when migrating to a new version. ### Breaking Changes +- [18976](https://github.com/apache/superset/pull/18976): A new `DEFAULT_CACHE_CONFIG_FUNC` parameter has been introduced in `config.py` which makes it possible to define a default cache config that will be used as the basis for all cache configs. When running the app in debug mode, the app will default to use `SimpleCache`; in other cases the default cache type will be `NullCache`. In addition, `DEFAULT_CACHE_TIMEOUT` has been deprecated and moved into `DEFAULT_CACHE_CONFIG_FUNC` (will be removed in Superset 2.0). For installations using Redis or other caching backends, it is recommended to set the default cache options in `DEFAULT_CACHE_CONFIG_FUNC` to ensure the primary cache is always used if new caches are added. - [17881](https://github.com/apache/superset/pull/17881): Previously simple adhoc filter values on string columns were stripped of enclosing single and double quotes. To fully support literal quotes in filters, both single and double quotes will no longer be removed from filter values. - [17984](https://github.com/apache/superset/pull/17984): Default Flask SECRET_KEY has changed for security reasons. You should always override with your own secret. Set `PREVIOUS_SECRET_KEY` (ex: PREVIOUS_SECRET_KEY = "\2\1thisismyscretkey\1\2\\e\\y\\y\\h") with your previous key and use `superset re-encrypt-secrets` to rotate you current secrets - [15254](https://github.com/apache/superset/pull/15254): Previously `QUERY_COST_FORMATTERS_BY_ENGINE`, `SQL_VALIDATORS_BY_ENGINE` and `SCHEDULED_QUERIES` were expected to be defined in the feature flag dictionary in the `config.py` file. These should now be defined as a top-level config, with the feature flag dictionary being reserved for boolean only values. diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index a313d9d8258d6..5def6618671fa 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -40,7 +40,11 @@ from superset.extensions import cache_manager, security_manager from superset.models.helpers import QueryResult from superset.utils import csv -from superset.utils.cache import generate_cache_key, set_and_log_cache +from superset.utils.cache import ( + generate_cache_key, + get_default_cache_config, + set_and_log_cache, +) from superset.utils.core import ( DTTM_ALIAS, error_msg_from_exception, @@ -385,7 +389,7 @@ def get_cache_timeout(self) -> int: cache_timeout_rv = self._query_context.get_cache_timeout() if cache_timeout_rv: return cache_timeout_rv - return config["CACHE_DEFAULT_TIMEOUT"] + return get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] def cache_key(self, **extra: Any) -> str: """ diff --git a/superset/config.py b/superset/config.py index bad83615ce3ed..841ca9ecb4e71 100644 --- a/superset/config.py +++ b/superset/config.py @@ -36,7 +36,7 @@ from cachelib.base import BaseCache from celery.schedules import crontab from dateutil import tz -from flask import Blueprint +from flask import Blueprint, Flask from flask_appbuilder.security.manager import AUTH_DB from pandas._libs.parsers import STR_NA_VALUES # pylint: disable=no-name-in-module from typing_extensions import Literal @@ -543,8 +543,9 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Also used by Alerts & Reports # --------------------------------------------------- THUMBNAIL_SELENIUM_USER = "admin" +# thumbnail cache (merged with default cache config) THUMBNAIL_CACHE_CONFIG: CacheConfig = { - "CACHE_TYPE": "null", + "CACHE_TYPE": "NullCache", "CACHE_NO_NULL_WARNING": True, } @@ -576,29 +577,41 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Setup image size default is (300, 200, True) # IMG_SIZE = (300, 200, True) -# Default cache timeout, applies to all cache backends unless specifically overridden in + +# Default cache config, applies to all cache backends unless specifically overridden in # each cache config. -CACHE_DEFAULT_TIMEOUT = int(timedelta(days=1).total_seconds()) +def DEFAULT_CACHE_CONFIG_FUNC(app: Flask) -> Dict[str, Any]: + default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") + if default_timeout is None: + default_timeout = int(timedelta(days=1).total_seconds()) + else: + logger.warning( + "The config flag CACHE_DEFAULT_TIMEOUT has been deprecated " + "and will be removed in Superset 2.0. Please set default cache options in " + "DEFAULT_CACHE_CONFIG_FUNC" + ) + + return { + "CACHE_TYPE": "SimpleCache" if app.debug else "NullCache", + "CACHE_DEFAULT_TIMEOUT": default_timeout, + } + -# Default cache for Superset objects -CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "null"} +# Default cache for Superset objects (merged with default cache config) +CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} -# Cache for datasource metadata and query results -DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "null"} +# Cache for datasource metadata and query results (merged with default cache config) +DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} -# Cache for filters state +# Cache for filters state (merged with default cache config) FILTER_STATE_CACHE_CONFIG: CacheConfig = { - "CACHE_TYPE": "FileSystemCache", - "CACHE_DIR": os.path.join(DATA_DIR, "cache"), "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()), "CACHE_THRESHOLD": 0, "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } -# Cache for chart form data +# Cache for chart form data (merged with default cache config) EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = { - "CACHE_TYPE": "FileSystemCache", - "CACHE_DIR": os.path.join(DATA_DIR, "cache"), "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()), "CACHE_THRESHOLD": 0, "REFRESH_TIMEOUT_ON_RETRIEVAL": True, diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 8fac419cf0ba6..25b9a52add06d 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -43,6 +43,7 @@ from superset.result_set import SupersetResultSet from superset.sql_parse import CtasMethod, ParsedQuery from superset.sqllab.limiting_factor import LimitingFactor +from superset.utils.cache import get_default_cache_config from superset.utils.celery import session_scope from superset.utils.core import json_iso_dttm_ser, QuerySource, zlib_compress from superset.utils.dates import now_as_float @@ -538,7 +539,7 @@ def execute_sql_statements( # pylint: disable=too-many-arguments, too-many-loca ) cache_timeout = database.cache_timeout if cache_timeout is None: - cache_timeout = config["CACHE_DEFAULT_TIMEOUT"] + cache_timeout = get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] compressed = zlib_compress(serialized_payload) logger.debug( diff --git a/superset/utils/cache.py b/superset/utils/cache.py index e7bdc35acda3e..ddd07fa9fe616 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -21,7 +21,7 @@ from functools import wraps from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, Union -from flask import current_app as app, request +from flask import current_app as app, Flask, request from flask_caching import Cache from flask_caching.backends import NullCache from werkzeug.wrappers.etag import ETagResponseMixin @@ -29,6 +29,7 @@ from superset import db from superset.extensions import cache_manager from superset.models.cache import CacheKey +from superset.typing import CacheConfig from superset.utils.core import json_int_dttm_ser from superset.utils.hashing import md5_sha_from_dict @@ -40,6 +41,10 @@ logger = logging.getLogger(__name__) +def get_default_cache_config(flask_app: Flask) -> Dict[str, Any]: + return flask_app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) + + def generate_cache_key(values_dict: Dict[str, Any], key_prefix: str = "") -> str: hash_str = md5_sha_from_dict(values_dict, default=json_int_dttm_ser) return f"{key_prefix}{hash_str}" @@ -55,7 +60,11 @@ def set_and_log_cache( if isinstance(cache_instance.cache, NullCache): return - timeout = cache_timeout if cache_timeout else config["CACHE_DEFAULT_TIMEOUT"] + timeout = ( + cache_timeout + if cache_timeout + else get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + ) try: dttm = datetime.utcnow().isoformat().split(".")[0] value = {**cache_value, "dttm": dttm} @@ -146,7 +155,7 @@ def etag_cache( """ if max_age is None: - max_age = app.config["CACHE_DEFAULT_TIMEOUT"] + max_age = get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] def decorator(f: Callable[..., Any]) -> Callable[..., Any]: @wraps(f) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index e92d930d2ef8f..50c4bf05a9195 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -29,40 +29,22 @@ def __init__(self) -> None: self._explore_form_data_cache = Cache() def init_app(self, app: Flask) -> None: + default_cache_config = app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) self._cache.init_app( - app, - { - "CACHE_DEFAULT_TIMEOUT": app.config["CACHE_DEFAULT_TIMEOUT"], - **app.config["CACHE_CONFIG"], - }, + app, {**default_cache_config, **app.config["CACHE_CONFIG"]}, ) self._data_cache.init_app( - app, - { - "CACHE_DEFAULT_TIMEOUT": app.config["CACHE_DEFAULT_TIMEOUT"], - **app.config["DATA_CACHE_CONFIG"], - }, + app, {**default_cache_config, **app.config["DATA_CACHE_CONFIG"]}, ) self._thumbnail_cache.init_app( - app, - { - "CACHE_DEFAULT_TIMEOUT": app.config["CACHE_DEFAULT_TIMEOUT"], - **app.config["THUMBNAIL_CACHE_CONFIG"], - }, + app, {**default_cache_config, **app.config["THUMBNAIL_CACHE_CONFIG"]}, ) self._filter_state_cache.init_app( - app, - { - "CACHE_DEFAULT_TIMEOUT": app.config["CACHE_DEFAULT_TIMEOUT"], - **app.config["FILTER_STATE_CACHE_CONFIG"], - }, + app, {**default_cache_config, **app.config["FILTER_STATE_CACHE_CONFIG"]}, ) self._explore_form_data_cache.init_app( app, - { - "CACHE_DEFAULT_TIMEOUT": app.config["CACHE_DEFAULT_TIMEOUT"], - **app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"], - }, + {**default_cache_config, **app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"],}, ) @property diff --git a/superset/viz.py b/superset/viz.py index 3a519bb1f9e83..563f665aaf2e3 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -72,7 +72,7 @@ from superset.sql_parse import validate_filter_clause from superset.typing import Column, Metric, QueryObjectDict, VizData, VizPayload from superset.utils import core as utils, csv -from superset.utils.cache import set_and_log_cache +from superset.utils.cache import get_default_cache_config, set_and_log_cache from superset.utils.core import ( apply_max_row_limit, DTTM_ALIAS, @@ -429,7 +429,7 @@ def cache_timeout(self) -> int: return self.datasource.database.cache_timeout if config["DATA_CACHE_CONFIG"].get("CACHE_DEFAULT_TIMEOUT") is not None: return config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] - return config["CACHE_DEFAULT_TIMEOUT"] + return get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] def get_json(self) -> str: return json.dumps( From 8044865544fd00644705fda5b5d4f10821f65214 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 12:55:31 +0200 Subject: [PATCH 02/15] lint --- superset/config.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/config.py b/superset/config.py index 841ca9ecb4e71..eb611556697cf 100644 --- a/superset/config.py +++ b/superset/config.py @@ -580,7 +580,9 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Default cache config, applies to all cache backends unless specifically overridden in # each cache config. -def DEFAULT_CACHE_CONFIG_FUNC(app: Flask) -> Dict[str, Any]: +def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name + app: Flask, +) -> Dict[str, Any]: default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") if default_timeout is None: default_timeout = int(timedelta(days=1).total_seconds()) From 507d57989512db7651c6e581c48b98dee662a7bc Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 14:29:04 +0200 Subject: [PATCH 03/15] clean up type --- superset/config.py | 2 +- superset/typing.py | 16 ++-------------- superset/utils/cache.py | 2 +- 3 files changed, 4 insertions(+), 16 deletions(-) diff --git a/superset/config.py b/superset/config.py index eb611556697cf..13f28380ea4a9 100644 --- a/superset/config.py +++ b/superset/config.py @@ -582,7 +582,7 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # each cache config. def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name app: Flask, -) -> Dict[str, Any]: +) -> CacheConfig: default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") if default_timeout is None: default_timeout = int(timedelta(days=1).total_seconds()) diff --git a/superset/typing.py b/superset/typing.py index 66b6cd4c38491..253d2b63551a8 100644 --- a/superset/typing.py +++ b/superset/typing.py @@ -15,20 +15,8 @@ # specific language governing permissions and limitations # under the License. from datetime import datetime -from typing import ( - Any, - Callable, - Dict, - List, - Optional, - Sequence, - Tuple, - TYPE_CHECKING, - Union, -) +from typing import Any, Dict, List, Optional, Sequence, Tuple, TYPE_CHECKING, Union -from flask import Flask -from flask_caching import Cache from typing_extensions import Literal, TypedDict from werkzeug.wrappers import Response @@ -69,7 +57,7 @@ class AdhocColumn(TypedDict, total=False): sqlExpression: Optional[str] -CacheConfig = Union[Callable[[Flask], Cache], Dict[str, Any]] +CacheConfig = Dict[str, Any] DbapiDescriptionRow = Tuple[ str, str, Optional[str], Optional[str], Optional[int], Optional[int], bool ] diff --git a/superset/utils/cache.py b/superset/utils/cache.py index ddd07fa9fe616..194a7ce5064ee 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -41,7 +41,7 @@ logger = logging.getLogger(__name__) -def get_default_cache_config(flask_app: Flask) -> Dict[str, Any]: +def get_default_cache_config(flask_app: Flask) -> CacheConfig: return flask_app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) From f0f90659db59bebb3b23ba1f4f7b6a0f6f24f595 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 15:50:53 +0200 Subject: [PATCH 04/15] use util --- superset/utils/cache_manager.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index 50c4bf05a9195..a7395c32323fd 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -17,6 +17,8 @@ from flask import Flask from flask_caching import Cache +from superset.utils.cache import get_default_cache_config + class CacheManager: def __init__(self) -> None: @@ -29,7 +31,7 @@ def __init__(self) -> None: self._explore_form_data_cache = Cache() def init_app(self, app: Flask) -> None: - default_cache_config = app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) + default_cache_config = get_default_cache_config(app) self._cache.init_app( app, {**default_cache_config, **app.config["CACHE_CONFIG"]}, ) From fdacbaf87ad064ae34a44751e7bf1bf6c904da83 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 17:05:19 +0200 Subject: [PATCH 05/15] fix integration test cache configs --- tests/integration_tests/cache_tests.py | 3 +-- .../dashboards/filter_state/api_tests.py | 1 - tests/integration_tests/explore/form_data/api_tests.py | 1 - tests/integration_tests/superset_test_config.py | 10 ++++++++-- .../superset_test_config_thumbnails.py | 4 +--- 5 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index 62edb514b35cc..bfcb89a95a16c 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -43,7 +43,7 @@ def tearDown(self): @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_no_data_cache(self): data_cache_config = app.config["DATA_CACHE_CONFIG"] - app.config["DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "null"} + app.config["DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "NullCache"} cache_manager.init_app(app) slc = self.get_slice("Girls", db.session) @@ -68,7 +68,6 @@ def test_slice_data_cache(self): cache_default_timeout = app.config["CACHE_DEFAULT_TIMEOUT"] app.config["CACHE_DEFAULT_TIMEOUT"] = 100 app.config["DATA_CACHE_CONFIG"] = { - "CACHE_TYPE": "simple", "CACHE_DEFAULT_TIMEOUT": 10, "CACHE_KEY_PREFIX": "superset_data_cache", } diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index f89efce29f3f7..2a11980535efd 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -62,7 +62,6 @@ def admin_id() -> int: @pytest.fixture(autouse=True) def cache(dashboard_id, admin_id): - app.config["FILTER_STATE_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: Entry = {"owner": admin_id, "value": value} cache_manager.filter_state_cache.set(cache_key(dashboard_id, key), entry) diff --git a/tests/integration_tests/explore/form_data/api_tests.py b/tests/integration_tests/explore/form_data/api_tests.py index 5e97aae6b7654..112eec2bb839b 100644 --- a/tests/integration_tests/explore/form_data/api_tests.py +++ b/tests/integration_tests/explore/form_data/api_tests.py @@ -74,7 +74,6 @@ def dataset_id() -> int: @pytest.fixture(autouse=True) def cache(chart_id, admin_id, dataset_id): - app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: TemporaryExploreState = { "owner": admin_id, diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index 698440c36383c..79582310eafc6 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -85,10 +85,16 @@ def GET_FEATURE_FLAGS_FUNC(ff): REDIS_RESULTS_DB = os.environ.get("REDIS_RESULTS_DB", 3) REDIS_CACHE_DB = os.environ.get("REDIS_CACHE_DB", 4) -CACHE_DEFAULT_TIMEOUT = int(timedelta(minutes=10).total_seconds()) + +def DEFAULT_CACHE_CONFIG_FUNC(app): + return { + "CACHE_TYPE": "SimpleCache", + "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), + } + CACHE_CONFIG = { - "CACHE_TYPE": "redis", + "CACHE_TYPE": "RedisCache", "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=1).total_seconds()), "CACHE_KEY_PREFIX": "superset_cache", "CACHE_REDIS_URL": f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CACHE_DB}", diff --git a/tests/integration_tests/superset_test_config_thumbnails.py b/tests/integration_tests/superset_test_config_thumbnails.py index 964164cf79cee..2e375931c3867 100644 --- a/tests/integration_tests/superset_test_config_thumbnails.py +++ b/tests/integration_tests/superset_test_config_thumbnails.py @@ -53,8 +53,6 @@ def GET_FEATURE_FLAGS_FUNC(ff): AUTH_ROLE_PUBLIC = "Public" EMAIL_NOTIFICATIONS = False -CACHE_CONFIG = {"CACHE_TYPE": "simple"} - REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") REDIS_PORT = os.environ.get("REDIS_PORT", "6379") REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2) @@ -79,7 +77,7 @@ class CeleryConfig(object): } THUMBNAIL_CACHE_CONFIG = { - "CACHE_TYPE": "redis", + "CACHE_TYPE": "RedisCache", "CACHE_DEFAULT_TIMEOUT": 10000, "CACHE_KEY_PREFIX": "superset_thumbnails_", "CACHE_REDIS_HOST": REDIS_HOST, From b953fd4552707d776ed2a9526f0c8ec2258fa6eb Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 17:56:21 +0200 Subject: [PATCH 06/15] remove util from cache manager --- superset/utils/cache_manager.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index a7395c32323fd..50c4bf05a9195 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -17,8 +17,6 @@ from flask import Flask from flask_caching import Cache -from superset.utils.cache import get_default_cache_config - class CacheManager: def __init__(self) -> None: @@ -31,7 +29,7 @@ def __init__(self) -> None: self._explore_form_data_cache = Cache() def init_app(self, app: Flask) -> None: - default_cache_config = get_default_cache_config(app) + default_cache_config = app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) self._cache.init_app( app, {**default_cache_config, **app.config["CACHE_CONFIG"]}, ) From bcf98496f67013fc749247006e428dd6bc885c4e Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 18:11:17 +0200 Subject: [PATCH 07/15] remove trailing comma --- superset/utils/cache_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index 50c4bf05a9195..46e8b7c607e12 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -44,7 +44,7 @@ def init_app(self, app: Flask) -> None: ) self._explore_form_data_cache.init_app( app, - {**default_cache_config, **app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"],}, + {**default_cache_config, **app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"]}, ) @property From 410140760bebe2469f5c16145c0eb33a883e1d89 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 19:53:41 +0200 Subject: [PATCH 08/15] fix more tests --- tests/integration_tests/cache_tests.py | 12 ++++++++++-- tests/integration_tests/viz_tests.py | 5 ++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index bfcb89a95a16c..c0738be6c7843 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -64,8 +64,16 @@ def test_no_data_cache(self): @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_slice_data_cache(self): # Override cache config + default_cache_config_func = app.config["DEFAULT_CACHE_CONFIG_FUNC"] + + def func(flask_app): + config = default_cache_config_func(flask_app) + config.update({"CACHE_DEFAULT_TIMEOUT": 100}) + return config + + app.config["DEFAULT_CACHE_CONFIG_FUNC"] = func data_cache_config = app.config["DATA_CACHE_CONFIG"] - cache_default_timeout = app.config["CACHE_DEFAULT_TIMEOUT"] + app.config["CACHE_DEFAULT_TIMEOUT"] = 100 app.config["DATA_CACHE_CONFIG"] = { "CACHE_DEFAULT_TIMEOUT": 10, @@ -100,5 +108,5 @@ def test_slice_data_cache(self): # reset cache config app.config["DATA_CACHE_CONFIG"] = data_cache_config - app.config["CACHE_DEFAULT_TIMEOUT"] = cache_default_timeout + app.config["DEFAULT_CACHE_CONFIG_FUNC"] = default_cache_config_func cache_manager.init_app(app) diff --git a/tests/integration_tests/viz_tests.py b/tests/integration_tests/viz_tests.py index 465fdb26ef581..493d7943cae11 100644 --- a/tests/integration_tests/viz_tests.py +++ b/tests/integration_tests/viz_tests.py @@ -173,7 +173,10 @@ def test_cache_timeout(self): app.config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] = None datasource.database.cache_timeout = None test_viz = viz.BaseViz(datasource, form_data={}) - self.assertEqual(app.config["CACHE_DEFAULT_TIMEOUT"], test_viz.cache_timeout) + self.assertEqual( + app.config["DEFAULT_CACHE_CONFIG_FUNC"](app)["CACHE_DEFAULT_TIMEOUT"], + test_viz.cache_timeout, + ) # restore DATA_CACHE_CONFIG timeout app.config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] = data_cache_timeout From d1a0ad124402c9d9c447beef48bbdb604111fa8e Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 19:55:21 +0200 Subject: [PATCH 09/15] fix truthiness check --- superset/utils/cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/superset/utils/cache.py b/superset/utils/cache.py index 194a7ce5064ee..515c2c4da5ab8 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -62,7 +62,7 @@ def set_and_log_cache( timeout = ( cache_timeout - if cache_timeout + if cache_timeout is not None else get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] ) try: From e5d3a11ccc1e2981a8cdacc7e52e5cfe6b6251fa Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 20:41:10 +0200 Subject: [PATCH 10/15] fix tests and improve deprecation notice --- superset/config.py | 2 +- tests/integration_tests/dashboards/filter_state/api_tests.py | 1 + tests/integration_tests/explore/form_data/api_tests.py | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/superset/config.py b/superset/config.py index 13f28380ea4a9..c345672341a3d 100644 --- a/superset/config.py +++ b/superset/config.py @@ -588,7 +588,7 @@ def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name default_timeout = int(timedelta(days=1).total_seconds()) else: logger.warning( - "The config flag CACHE_DEFAULT_TIMEOUT has been deprecated " + "The global config flag CACHE_DEFAULT_TIMEOUT has been deprecated " "and will be removed in Superset 2.0. Please set default cache options in " "DEFAULT_CACHE_CONFIG_FUNC" ) diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index 2a11980535efd..f89efce29f3f7 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -62,6 +62,7 @@ def admin_id() -> int: @pytest.fixture(autouse=True) def cache(dashboard_id, admin_id): + app.config["FILTER_STATE_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: Entry = {"owner": admin_id, "value": value} cache_manager.filter_state_cache.set(cache_key(dashboard_id, key), entry) diff --git a/tests/integration_tests/explore/form_data/api_tests.py b/tests/integration_tests/explore/form_data/api_tests.py index 112eec2bb839b..5e97aae6b7654 100644 --- a/tests/integration_tests/explore/form_data/api_tests.py +++ b/tests/integration_tests/explore/form_data/api_tests.py @@ -74,6 +74,7 @@ def dataset_id() -> int: @pytest.fixture(autouse=True) def cache(chart_id, admin_id, dataset_id): + app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: TemporaryExploreState = { "owner": admin_id, From 9a92f9a6d1e6f5f785ec83504b4d857146c0ae1f Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Tue, 1 Mar 2022 23:05:48 +0200 Subject: [PATCH 11/15] fix default cache threshold --- superset/config.py | 4 ++-- tests/integration_tests/dashboards/filter_state/api_tests.py | 1 - tests/integration_tests/explore/form_data/api_tests.py | 1 - 3 files changed, 2 insertions(+), 4 deletions(-) diff --git a/superset/config.py b/superset/config.py index c345672341a3d..311e7ab31f457 100644 --- a/superset/config.py +++ b/superset/config.py @@ -25,6 +25,7 @@ import importlib.util import json import logging +import math import os import re import sys @@ -595,6 +596,7 @@ def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name return { "CACHE_TYPE": "SimpleCache" if app.debug else "NullCache", + "CACHE_THRESHOLD": math.inf, "CACHE_DEFAULT_TIMEOUT": default_timeout, } @@ -608,14 +610,12 @@ def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name # Cache for filters state (merged with default cache config) FILTER_STATE_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()), - "CACHE_THRESHOLD": 0, "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } # Cache for chart form data (merged with default cache config) EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()), - "CACHE_THRESHOLD": 0, "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index f89efce29f3f7..2a11980535efd 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -62,7 +62,6 @@ def admin_id() -> int: @pytest.fixture(autouse=True) def cache(dashboard_id, admin_id): - app.config["FILTER_STATE_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: Entry = {"owner": admin_id, "value": value} cache_manager.filter_state_cache.set(cache_key(dashboard_id, key), entry) diff --git a/tests/integration_tests/explore/form_data/api_tests.py b/tests/integration_tests/explore/form_data/api_tests.py index 5e97aae6b7654..112eec2bb839b 100644 --- a/tests/integration_tests/explore/form_data/api_tests.py +++ b/tests/integration_tests/explore/form_data/api_tests.py @@ -74,7 +74,6 @@ def dataset_id() -> int: @pytest.fixture(autouse=True) def cache(chart_id, admin_id, dataset_id): - app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"] = {"CACHE_TYPE": "SimpleCache"} cache_manager.init_app(app) entry: TemporaryExploreState = { "owner": admin_id, From d9c86394146df53b3ee275797097016f9b49ff24 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 2 Mar 2022 12:15:39 +0200 Subject: [PATCH 12/15] move debug check to cache_manager --- superset/common/query_context_processor.py | 8 +-- superset/config.py | 44 ++++-------- superset/sql_lab.py | 5 +- superset/utils/cache.py | 8 +-- superset/utils/cache_manager.py | 71 +++++++++++++++---- superset/viz.py | 4 +- tests/integration_tests/cache_tests.py | 15 ++-- .../integration_tests/superset_test_config.py | 11 +-- tests/integration_tests/viz_tests.py | 2 +- 9 files changed, 92 insertions(+), 76 deletions(-) diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index 5def6618671fa..f247512d47628 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -40,11 +40,7 @@ from superset.extensions import cache_manager, security_manager from superset.models.helpers import QueryResult from superset.utils import csv -from superset.utils.cache import ( - generate_cache_key, - get_default_cache_config, - set_and_log_cache, -) +from superset.utils.cache import generate_cache_key, set_and_log_cache from superset.utils.core import ( DTTM_ALIAS, error_msg_from_exception, @@ -389,7 +385,7 @@ def get_cache_timeout(self) -> int: cache_timeout_rv = self._query_context.get_cache_timeout() if cache_timeout_rv: return cache_timeout_rv - return get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + return app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] def cache_key(self, **extra: Any) -> str: """ diff --git a/superset/config.py b/superset/config.py index 311e7ab31f457..d751eda480ed7 100644 --- a/superset/config.py +++ b/superset/config.py @@ -25,7 +25,6 @@ import importlib.util import json import logging -import math import os import re import sys @@ -544,9 +543,8 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Also used by Alerts & Reports # --------------------------------------------------- THUMBNAIL_SELENIUM_USER = "admin" -# thumbnail cache (merged with default cache config) +# thumbnail cache (will be merged with DEFAULT_CACHE_CONFIG) THUMBNAIL_CACHE_CONFIG: CacheConfig = { - "CACHE_TYPE": "NullCache", "CACHE_NO_NULL_WARNING": True, } @@ -578,42 +576,26 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Setup image size default is (300, 200, True) # IMG_SIZE = (300, 200, True) +# Default cache for Superset objects (will be used as the base for all cache configs) +DEFAULT_CACHE_CONFIG: CacheConfig = { + "CACHE_TYPE": "NullCache", + "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=1).total_seconds()), +} -# Default cache config, applies to all cache backends unless specifically overridden in -# each cache config. -def DEFAULT_CACHE_CONFIG_FUNC( # pylint: disable=invalid-name - app: Flask, -) -> CacheConfig: - default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") - if default_timeout is None: - default_timeout = int(timedelta(days=1).total_seconds()) - else: - logger.warning( - "The global config flag CACHE_DEFAULT_TIMEOUT has been deprecated " - "and will be removed in Superset 2.0. Please set default cache options in " - "DEFAULT_CACHE_CONFIG_FUNC" - ) - - return { - "CACHE_TYPE": "SimpleCache" if app.debug else "NullCache", - "CACHE_THRESHOLD": math.inf, - "CACHE_DEFAULT_TIMEOUT": default_timeout, - } - - -# Default cache for Superset objects (merged with default cache config) -CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} +# Default cache for Superset objects (will be merged with DEFAULT_CACHE_CONFIG) +CACHE_CONFIG: CacheConfig = {} -# Cache for datasource metadata and query results (merged with default cache config) -DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} +# Cache for datasource metadata and query results (will be merged with +# DEFAULT_CACHE_CONFIG) +DATA_CACHE_CONFIG: CacheConfig = {} -# Cache for filters state (merged with default cache config) +# Cache for filters state (will be merged with DEFAULT_CACHE_CONFIG) FILTER_STATE_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()), "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } -# Cache for chart form data (merged with default cache config) +# Cache for chart form data (will be merged with DEFAULT_CACHE_CONFIG) EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()), "REFRESH_TIMEOUT_ON_RETRIEVAL": True, diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 25b9a52add06d..90349aa58bf9d 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -43,7 +43,6 @@ from superset.result_set import SupersetResultSet from superset.sql_parse import CtasMethod, ParsedQuery from superset.sqllab.limiting_factor import LimitingFactor -from superset.utils.cache import get_default_cache_config from superset.utils.celery import session_scope from superset.utils.core import json_iso_dttm_ser, QuerySource, zlib_compress from superset.utils.dates import now_as_float @@ -539,7 +538,9 @@ def execute_sql_statements( # pylint: disable=too-many-arguments, too-many-loca ) cache_timeout = database.cache_timeout if cache_timeout is None: - cache_timeout = get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + cache_timeout = app.config["DEFAULT_CACHE_CONFIG"][ + "CACHE_DEFAULT_TIMEOUT" + ] compressed = zlib_compress(serialized_payload) logger.debug( diff --git a/superset/utils/cache.py b/superset/utils/cache.py index 515c2c4da5ab8..1adb8af7b9f17 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -41,10 +41,6 @@ logger = logging.getLogger(__name__) -def get_default_cache_config(flask_app: Flask) -> CacheConfig: - return flask_app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) - - def generate_cache_key(values_dict: Dict[str, Any], key_prefix: str = "") -> str: hash_str = md5_sha_from_dict(values_dict, default=json_int_dttm_ser) return f"{key_prefix}{hash_str}" @@ -63,7 +59,7 @@ def set_and_log_cache( timeout = ( cache_timeout if cache_timeout is not None - else get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + else app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] ) try: dttm = datetime.utcnow().isoformat().split(".")[0] @@ -155,7 +151,7 @@ def etag_cache( """ if max_age is None: - max_age = get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + max_age = app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] def decorator(f: Callable[..., Any]) -> Callable[..., Any]: @wraps(f) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index 46e8b7c607e12..63ef5b8063d70 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -14,37 +14,80 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. +import logging +import math + from flask import Flask +from flask_babel import gettext as _ from flask_caching import Cache +from superset.typing import CacheConfig + +logger = logging.getLogger(__name__) + class CacheManager: def __init__(self) -> None: super().__init__() + self._default_cache_config: CacheConfig = {} self._cache = Cache() self._data_cache = Cache() self._thumbnail_cache = Cache() self._filter_state_cache = Cache() self._explore_form_data_cache = Cache() + def _init_cache( + self, app: Flask, cache: Cache, cache_config_key: str, required: bool = False + ) -> None: + config = {**self._default_cache_config, **app.config[cache_config_key]} + if required and config["CACHE_TYPE"] in ("null", "NullCache"): + raise Exception( + _( + "The CACHE_TYPE `%(cache_type)s` for `%(cache_config_key)s` is not " + "supported. It is recommended to use `RedisCache`, `MemcachedCache` " + "or another dedicated caching backend for production deployments", + cache_type=config["CACHE_TYPE"], + cache_config_key=cache_config_key, + ), + ) + cache.init_app(app, config) + def init_app(self, app: Flask) -> None: - default_cache_config = app.config["DEFAULT_CACHE_CONFIG_FUNC"](app) - self._cache.init_app( - app, {**default_cache_config, **app.config["CACHE_CONFIG"]}, - ) - self._data_cache.init_app( - app, {**default_cache_config, **app.config["DATA_CACHE_CONFIG"]}, - ) - self._thumbnail_cache.init_app( - app, {**default_cache_config, **app.config["THUMBNAIL_CACHE_CONFIG"]}, - ) - self._filter_state_cache.init_app( - app, {**default_cache_config, **app.config["FILTER_STATE_CACHE_CONFIG"]}, + if app.debug: + self._default_cache_config = { + "CACHE_TYPE": "SimpleCache", + "CACHE_THRESHOLD": math.inf, + } + else: + self._default_cache_config = {} + + default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") + if default_timeout is not None: + self._default_cache_config["CACHE_DEFAULT_TIMEOUT"] = default_timeout + logger.warning( + _( + "The global config flag `CACHE_DEFAULT_TIMEOUT` has been " + "deprecated and will be removed in Superset 2.0. Please set " + "default cache options in the `DEFAULT_CACHE_CONFIG` parameter" + ), + ) + self._default_cache_config = { + **self._default_cache_config, + **app.config["DEFAULT_CACHE_CONFIG"], + } + + self._init_cache(app, self._cache, "CACHE_CONFIG") + self._init_cache(app, self._data_cache, "DATA_CACHE_CONFIG") + self._init_cache(app, self._thumbnail_cache, "THUMBNAIL_CACHE_CONFIG") + self._init_cache( + app, self._filter_state_cache, "FILTER_STATE_CACHE_CONFIG", required=True ) - self._explore_form_data_cache.init_app( + self._init_cache( app, - {**default_cache_config, **app.config["EXPLORE_FORM_DATA_CACHE_CONFIG"]}, + self._explore_form_data_cache, + "EXPLORE_FORM_DATA_CACHE_CONFIG", + required=True, ) @property diff --git a/superset/viz.py b/superset/viz.py index 563f665aaf2e3..f1ce2c5543b21 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -72,7 +72,7 @@ from superset.sql_parse import validate_filter_clause from superset.typing import Column, Metric, QueryObjectDict, VizData, VizPayload from superset.utils import core as utils, csv -from superset.utils.cache import get_default_cache_config, set_and_log_cache +from superset.utils.cache import set_and_log_cache from superset.utils.core import ( apply_max_row_limit, DTTM_ALIAS, @@ -429,7 +429,7 @@ def cache_timeout(self) -> int: return self.datasource.database.cache_timeout if config["DATA_CACHE_CONFIG"].get("CACHE_DEFAULT_TIMEOUT") is not None: return config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] - return get_default_cache_config(app)["CACHE_DEFAULT_TIMEOUT"] + return app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] def get_json(self) -> str: return json.dumps( diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index c0738be6c7843..ddb9d91c6d424 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -64,17 +64,14 @@ def test_no_data_cache(self): @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_slice_data_cache(self): # Override cache config - default_cache_config_func = app.config["DEFAULT_CACHE_CONFIG_FUNC"] + default_cache_config = app.config["DEFAULT_CACHE_CONFIG"] - def func(flask_app): - config = default_cache_config_func(flask_app) - config.update({"CACHE_DEFAULT_TIMEOUT": 100}) - return config - - app.config["DEFAULT_CACHE_CONFIG_FUNC"] = func + app.config["DEFAULT_CACHE_CONFIG"] = { + **default_cache_config, + "CACHE_DEFAULT_TIMEOUT": 100, + } data_cache_config = app.config["DATA_CACHE_CONFIG"] - app.config["CACHE_DEFAULT_TIMEOUT"] = 100 app.config["DATA_CACHE_CONFIG"] = { "CACHE_DEFAULT_TIMEOUT": 10, "CACHE_KEY_PREFIX": "superset_data_cache", @@ -108,5 +105,5 @@ def func(flask_app): # reset cache config app.config["DATA_CACHE_CONFIG"] = data_cache_config - app.config["DEFAULT_CACHE_CONFIG_FUNC"] = default_cache_config_func + app.config["DEFAULT_CACHE_CONFIG"] = default_cache_config cache_manager.init_app(app) diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index 79582310eafc6..3e3b6151f8986 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -15,6 +15,7 @@ # specific language governing permissions and limitations # under the License. # type: ignore +import math from copy import copy from datetime import timedelta @@ -86,11 +87,11 @@ def GET_FEATURE_FLAGS_FUNC(ff): REDIS_CACHE_DB = os.environ.get("REDIS_CACHE_DB", 4) -def DEFAULT_CACHE_CONFIG_FUNC(app): - return { - "CACHE_TYPE": "SimpleCache", - "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), - } +DEFAULT_CACHE_CONFIG = { + "CACHE_TYPE": "SimpleCache", + "CACHE_THRESHOLD": math.inf, + "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), +} CACHE_CONFIG = { diff --git a/tests/integration_tests/viz_tests.py b/tests/integration_tests/viz_tests.py index 493d7943cae11..56af2240f480d 100644 --- a/tests/integration_tests/viz_tests.py +++ b/tests/integration_tests/viz_tests.py @@ -174,7 +174,7 @@ def test_cache_timeout(self): datasource.database.cache_timeout = None test_viz = viz.BaseViz(datasource, form_data={}) self.assertEqual( - app.config["DEFAULT_CACHE_CONFIG_FUNC"](app)["CACHE_DEFAULT_TIMEOUT"], + app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"], test_viz.cache_timeout, ) # restore DATA_CACHE_CONFIG timeout From dd77329c47040e9890c1237a4390de2c4aa7ea2d Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 2 Mar 2022 12:53:30 +0200 Subject: [PATCH 13/15] remove separate getter --- superset/utils/cache_manager.py | 40 +++++++++------------------------ 1 file changed, 10 insertions(+), 30 deletions(-) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index 63ef5b8063d70..f40e6018950e9 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -31,11 +31,11 @@ def __init__(self) -> None: super().__init__() self._default_cache_config: CacheConfig = {} - self._cache = Cache() - self._data_cache = Cache() - self._thumbnail_cache = Cache() - self._filter_state_cache = Cache() - self._explore_form_data_cache = Cache() + self.cache = Cache() + self.data_cache = Cache() + self.thumbnail_cache = Cache() + self.filter_state_cache = Cache() + self.explore_form_data_cache = Cache() def _init_cache( self, app: Flask, cache: Cache, cache_config_key: str, required: bool = False @@ -77,35 +77,15 @@ def init_app(self, app: Flask) -> None: **app.config["DEFAULT_CACHE_CONFIG"], } - self._init_cache(app, self._cache, "CACHE_CONFIG") - self._init_cache(app, self._data_cache, "DATA_CACHE_CONFIG") - self._init_cache(app, self._thumbnail_cache, "THUMBNAIL_CACHE_CONFIG") + self._init_cache(app, self.cache, "CACHE_CONFIG") + self._init_cache(app, self.data_cache, "DATA_CACHE_CONFIG") + self._init_cache(app, self.thumbnail_cache, "THUMBNAIL_CACHE_CONFIG") self._init_cache( - app, self._filter_state_cache, "FILTER_STATE_CACHE_CONFIG", required=True + app, self.filter_state_cache, "FILTER_STATE_CACHE_CONFIG", required=True ) self._init_cache( app, - self._explore_form_data_cache, + self.explore_form_data_cache, "EXPLORE_FORM_DATA_CACHE_CONFIG", required=True, ) - - @property - def data_cache(self) -> Cache: - return self._data_cache - - @property - def cache(self) -> Cache: - return self._cache - - @property - def thumbnail_cache(self) -> Cache: - return self._thumbnail_cache - - @property - def filter_state_cache(self) -> Cache: - return self._filter_state_cache - - @property - def explore_form_data_cache(self) -> Cache: - return self._explore_form_data_cache From 9c5f20920d3181450a9feefc340e77935f13ec1c Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 2 Mar 2022 12:55:34 +0200 Subject: [PATCH 14/15] update docs --- UPDATING.md | 2 +- docs/docs/installation/cache.mdx | 37 ++++++++++++++++---------------- superset/config.py | 2 ++ 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 1ebbd1adda6f6..77a9f7da9b754 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -26,7 +26,7 @@ assists people when migrating to a new version. ### Breaking Changes -- [18976](https://github.com/apache/superset/pull/18976): A new `DEFAULT_CACHE_CONFIG_FUNC` parameter has been introduced in `config.py` which makes it possible to define a default cache config that will be used as the basis for all cache configs. When running the app in debug mode, the app will default to use `SimpleCache`; in other cases the default cache type will be `NullCache`. In addition, `DEFAULT_CACHE_TIMEOUT` has been deprecated and moved into `DEFAULT_CACHE_CONFIG_FUNC` (will be removed in Superset 2.0). For installations using Redis or other caching backends, it is recommended to set the default cache options in `DEFAULT_CACHE_CONFIG_FUNC` to ensure the primary cache is always used if new caches are added. +- [18976](https://github.com/apache/superset/pull/18976): A new `DEFAULT_CACHE_CONFIG` parameter has been introduced in `config.py` which makes it possible to define a default cache config that will be used as the basis for all cache configs. When running the app in debug mode, the app will default to use `SimpleCache`; in other cases the default cache type will be `NullCache`. In addition, `DEFAULT_CACHE_TIMEOUT` has been deprecated and moved into `DEFAULT_CACHE_CONFIG` (will be removed in Superset 2.0). For installations using Redis or other caching backends, it is recommended to set the default cache options in `DEFAULT_CACHE_CONFIG` to ensure the primary cache is always used if new caches are added. - [17881](https://github.com/apache/superset/pull/17881): Previously simple adhoc filter values on string columns were stripped of enclosing single and double quotes. To fully support literal quotes in filters, both single and double quotes will no longer be removed from filter values. - [17984](https://github.com/apache/superset/pull/17984): Default Flask SECRET_KEY has changed for security reasons. You should always override with your own secret. Set `PREVIOUS_SECRET_KEY` (ex: PREVIOUS_SECRET_KEY = "\2\1thisismyscretkey\1\2\\e\\y\\y\\h") with your previous key and use `superset re-encrypt-secrets` to rotate you current secrets - [15254](https://github.com/apache/superset/pull/15254): Previously `QUERY_COST_FORMATTERS_BY_ENGINE`, `SQL_VALIDATORS_BY_ENGINE` and `SCHEDULED_QUERIES` were expected to be defined in the feature flag dictionary in the `config.py` file. These should now be defined as a top-level config, with the feature flag dictionary being reserved for boolean only values. diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx index 4a4258a60e4a5..c91cd51176004 100644 --- a/docs/docs/installation/cache.mdx +++ b/docs/docs/installation/cache.mdx @@ -7,20 +7,29 @@ version: 1 ## Caching -Superset uses [Flask-Caching](https://flask-caching.readthedocs.io/) for caching purpose. For security reasons, -there are two separate cache configs for Superset's own metadata (`CACHE_CONFIG`) and charting data queried from -connected datasources (`DATA_CACHE_CONFIG`). However, Query results from SQL Lab are stored in another backend -called `RESULTS_BACKEND`, See [Async Queries via Celery](/docs/installation/async-queries-celery) for details. - -Configuring caching is as easy as providing `CACHE_CONFIG` and `DATA_CACHE_CONFIG` in your +Superset uses [Flask-Caching](https://flask-caching.readthedocs.io/) for caching purpose. Default caching options +can be set by overriding the `DEFAULT_CACHE_CONFIG` in your `superset_config.py`. Unless overridden, the default +cache type will be set to `SimpleCache` when running in debug mode, and `NullCache` otherwise. + +Currently there are five separate cache configurations to provide additional security and more granular customization options: +- Metadata cache (optional): `CACHE_CONFIG` +- Charting data queried from datasets (optional): `DATA_CACHE_CONFIG` +- SQL Lab query results (optional): `RESULTS_BACKEND`. See [Async Queries via Celery](/docs/installation/async-queries-celery) for details +- Dashboard filter state (required): `FILTER_STATE_CACHE_CONFIG`. +- Explore chart form data (required): `EXPLORE_FORM_DATA_CACHE_CONFIG` + +Configuring caching is as easy as providing a custom cache config in your `superset_config.py` that complies with [the Flask-Caching specifications](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching). - Flask-Caching supports various caching backends, including Redis, Memcached, SimpleCache (in-memory), or the -local filesystem. +local filesystem. Custom cache backends are also supported. See [here](https://flask-caching.readthedocs.io/en/latest/#custom-cache-backends) for specifics. +Note that Dashboard and Explore caching is required, and configuring the application with either of these caches set to `NullCache` will +cause the application to fail on startup. Also keep in mind, tht when running Superset on a multi-worker setup, a dedicated cache is required. +For this we recommend running either Redis or Memcached: + +- Redis (recommended): we recommend the [redis](https://pypi.python.org/pypi/redis) Python package - Memcached: we recommend using [pylibmc](https://pypi.org/project/pylibmc/) client library as `python-memcached` does not handle storing binary data correctly. -- Redis: we recommend the [redis](https://pypi.python.org/pypi/redis) Python package Both of these libraries can be installed using pip. @@ -28,16 +37,6 @@ For chart data, Superset goes up a “timeout search path”, from a slice's con to the datasource’s, the database’s, then ultimately falls back to the global default defined in `DATA_CACHE_CONFIG`. -``` -DATA_CACHE_CONFIG = { - 'CACHE_TYPE': 'redis', - 'CACHE_DEFAULT_TIMEOUT': 60 * 60 * 24, # 1 day default (in secs) - 'CACHE_KEY_PREFIX': 'superset_results', - 'CACHE_REDIS_URL': 'redis://localhost:6379/0', -} -``` - -Custom cache backends are also supported. See [here](https://flask-caching.readthedocs.io/en/latest/#custom-cache-backends) for specifics. Superset has a Celery task that will periodically warm up the cache based on different strategies. To use it, add the following to the `CELERYBEAT_SCHEDULE` section in `config.py`: diff --git a/superset/config.py b/superset/config.py index d751eda480ed7..59eb069e5dd51 100644 --- a/superset/config.py +++ b/superset/config.py @@ -592,12 +592,14 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Cache for filters state (will be merged with DEFAULT_CACHE_CONFIG) FILTER_STATE_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()), + # should the timeout be reset when retrieving a cached value "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } # Cache for chart form data (will be merged with DEFAULT_CACHE_CONFIG) EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()), + # should the timeout be reset when retrieving a cached value "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } From d5f8d42007763a953cd83dae2439eba4d258f1d8 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Wed, 2 Mar 2022 15:39:16 +0200 Subject: [PATCH 15/15] remove default cache config --- UPDATING.md | 2 +- docs/docs/installation/cache.mdx | 23 ++--- superset/common/query_context_processor.py | 2 +- superset/config.py | 27 +++--- superset/sql_lab.py | 4 +- superset/utils/cache.py | 7 +- superset/utils/cache_manager.py | 93 ++++++++++--------- superset/viz.py | 2 +- tests/integration_tests/cache_tests.py | 13 +-- .../dashboards/filter_state/api_tests.py | 1 - .../explore/form_data/api_tests.py | 1 - .../integration_tests/superset_test_config.py | 19 ++-- .../superset_test_config_thumbnails.py | 2 + tests/integration_tests/viz_tests.py | 5 +- 14 files changed, 99 insertions(+), 102 deletions(-) diff --git a/UPDATING.md b/UPDATING.md index 77a9f7da9b754..c3d1de14d56e6 100644 --- a/UPDATING.md +++ b/UPDATING.md @@ -26,7 +26,7 @@ assists people when migrating to a new version. ### Breaking Changes -- [18976](https://github.com/apache/superset/pull/18976): A new `DEFAULT_CACHE_CONFIG` parameter has been introduced in `config.py` which makes it possible to define a default cache config that will be used as the basis for all cache configs. When running the app in debug mode, the app will default to use `SimpleCache`; in other cases the default cache type will be `NullCache`. In addition, `DEFAULT_CACHE_TIMEOUT` has been deprecated and moved into `DEFAULT_CACHE_CONFIG` (will be removed in Superset 2.0). For installations using Redis or other caching backends, it is recommended to set the default cache options in `DEFAULT_CACHE_CONFIG` to ensure the primary cache is always used if new caches are added. +- [18976](https://github.com/apache/superset/pull/18976): When running the app in debug mode, the app will default to use `SimpleCache` for `FILTER_STATE_CACHE_CONFIG` and `EXPLORE_FORM_DATA_CACHE_CONFIG`. When running in non-debug mode, a cache backend will need to be defined, otherwise the application will fail to start. For installations using Redis or other caching backends, it is recommended to use the same backend for both cache configs. - [17881](https://github.com/apache/superset/pull/17881): Previously simple adhoc filter values on string columns were stripped of enclosing single and double quotes. To fully support literal quotes in filters, both single and double quotes will no longer be removed from filter values. - [17984](https://github.com/apache/superset/pull/17984): Default Flask SECRET_KEY has changed for security reasons. You should always override with your own secret. Set `PREVIOUS_SECRET_KEY` (ex: PREVIOUS_SECRET_KEY = "\2\1thisismyscretkey\1\2\\e\\y\\y\\h") with your previous key and use `superset re-encrypt-secrets` to rotate you current secrets - [15254](https://github.com/apache/superset/pull/15254): Previously `QUERY_COST_FORMATTERS_BY_ENGINE`, `SQL_VALIDATORS_BY_ENGINE` and `SCHEDULED_QUERIES` were expected to be defined in the feature flag dictionary in the `config.py` file. These should now be defined as a top-level config, with the feature flag dictionary being reserved for boolean only values. diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx index c91cd51176004..e86382b3c16c4 100644 --- a/docs/docs/installation/cache.mdx +++ b/docs/docs/installation/cache.mdx @@ -7,25 +7,21 @@ version: 1 ## Caching -Superset uses [Flask-Caching](https://flask-caching.readthedocs.io/) for caching purpose. Default caching options -can be set by overriding the `DEFAULT_CACHE_CONFIG` in your `superset_config.py`. Unless overridden, the default -cache type will be set to `SimpleCache` when running in debug mode, and `NullCache` otherwise. - -Currently there are five separate cache configurations to provide additional security and more granular customization options: +Superset uses [Flask-Caching](https://flask-caching.readthedocs.io/) for caching purpose. Configuring caching is as easy as providing a custom cache config in your +`superset_config.py` that complies with [the Flask-Caching specifications](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching). +Flask-Caching supports various caching backends, including Redis, Memcached, SimpleCache (in-memory), or the +local filesystem. Custom cache backends are also supported. See [here](https://flask-caching.readthedocs.io/en/latest/#custom-cache-backends) for specifics. +The following cache configurations can be customized: - Metadata cache (optional): `CACHE_CONFIG` - Charting data queried from datasets (optional): `DATA_CACHE_CONFIG` - SQL Lab query results (optional): `RESULTS_BACKEND`. See [Async Queries via Celery](/docs/installation/async-queries-celery) for details - Dashboard filter state (required): `FILTER_STATE_CACHE_CONFIG`. - Explore chart form data (required): `EXPLORE_FORM_DATA_CACHE_CONFIG` -Configuring caching is as easy as providing a custom cache config in your -`superset_config.py` that complies with [the Flask-Caching specifications](https://flask-caching.readthedocs.io/en/latest/#configuring-flask-caching). -Flask-Caching supports various caching backends, including Redis, Memcached, SimpleCache (in-memory), or the -local filesystem. Custom cache backends are also supported. See [here](https://flask-caching.readthedocs.io/en/latest/#custom-cache-backends) for specifics. - -Note that Dashboard and Explore caching is required, and configuring the application with either of these caches set to `NullCache` will -cause the application to fail on startup. Also keep in mind, tht when running Superset on a multi-worker setup, a dedicated cache is required. -For this we recommend running either Redis or Memcached: +Please note, that Dashboard and Explore caching is required. When running Superset in debug mode, both Explore and Dashboard caches will default to `SimpleCache`; +However, trying to run Superset in non-debug mode without defining a cache for these will cause the application to fail on startup. When running +superset in single-worker mode, any cache backend is supported. However, when running Superset in on a multi-worker setup, a dedicated cache is required. For this +we recommend using either Redis or Memcached: - Redis (recommended): we recommend the [redis](https://pypi.python.org/pypi/redis) Python package - Memcached: we recommend using [pylibmc](https://pypi.org/project/pylibmc/) client library as @@ -37,6 +33,7 @@ For chart data, Superset goes up a “timeout search path”, from a slice's con to the datasource’s, the database’s, then ultimately falls back to the global default defined in `DATA_CACHE_CONFIG`. +## Celery beat Superset has a Celery task that will periodically warm up the cache based on different strategies. To use it, add the following to the `CELERYBEAT_SCHEDULE` section in `config.py`: diff --git a/superset/common/query_context_processor.py b/superset/common/query_context_processor.py index f247512d47628..a313d9d8258d6 100644 --- a/superset/common/query_context_processor.py +++ b/superset/common/query_context_processor.py @@ -385,7 +385,7 @@ def get_cache_timeout(self) -> int: cache_timeout_rv = self._query_context.get_cache_timeout() if cache_timeout_rv: return cache_timeout_rv - return app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] + return config["CACHE_DEFAULT_TIMEOUT"] def cache_key(self, **extra: Any) -> str: """ diff --git a/superset/config.py b/superset/config.py index 59eb069e5dd51..86d7b45c4711a 100644 --- a/superset/config.py +++ b/superset/config.py @@ -36,7 +36,7 @@ from cachelib.base import BaseCache from celery.schedules import crontab from dateutil import tz -from flask import Blueprint, Flask +from flask import Blueprint from flask_appbuilder.security.manager import AUTH_DB from pandas._libs.parsers import STR_NA_VALUES # pylint: disable=no-name-in-module from typing_extensions import Literal @@ -543,8 +543,8 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Also used by Alerts & Reports # --------------------------------------------------- THUMBNAIL_SELENIUM_USER = "admin" -# thumbnail cache (will be merged with DEFAULT_CACHE_CONFIG) THUMBNAIL_CACHE_CONFIG: CacheConfig = { + "CACHE_TYPE": "NullCache", "CACHE_NO_NULL_WARNING": True, } @@ -576,27 +576,26 @@ def _try_json_readsha(filepath: str, length: int) -> Optional[str]: # Setup image size default is (300, 200, True) # IMG_SIZE = (300, 200, True) -# Default cache for Superset objects (will be used as the base for all cache configs) -DEFAULT_CACHE_CONFIG: CacheConfig = { - "CACHE_TYPE": "NullCache", - "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=1).total_seconds()), -} +# Default cache timeout, applies to all cache backends unless specifically overridden in +# each cache config. +CACHE_DEFAULT_TIMEOUT = int(timedelta(days=1).total_seconds()) -# Default cache for Superset objects (will be merged with DEFAULT_CACHE_CONFIG) -CACHE_CONFIG: CacheConfig = {} +# Default cache for Superset objects +CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} -# Cache for datasource metadata and query results (will be merged with -# DEFAULT_CACHE_CONFIG) -DATA_CACHE_CONFIG: CacheConfig = {} +# Cache for datasource metadata and query results +DATA_CACHE_CONFIG: CacheConfig = {"CACHE_TYPE": "NullCache"} -# Cache for filters state (will be merged with DEFAULT_CACHE_CONFIG) +# Cache for dashboard filter state (`CACHE_TYPE` defaults to `SimpleCache` when +# running in debug mode unless overridden) FILTER_STATE_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=90).total_seconds()), # should the timeout be reset when retrieving a cached value "REFRESH_TIMEOUT_ON_RETRIEVAL": True, } -# Cache for chart form data (will be merged with DEFAULT_CACHE_CONFIG) +# Cache for explore form data state (`CACHE_TYPE` defaults to `SimpleCache` when +# running in debug mode unless overridden) EXPLORE_FORM_DATA_CACHE_CONFIG: CacheConfig = { "CACHE_DEFAULT_TIMEOUT": int(timedelta(days=7).total_seconds()), # should the timeout be reset when retrieving a cached value diff --git a/superset/sql_lab.py b/superset/sql_lab.py index 90349aa58bf9d..8fac419cf0ba6 100644 --- a/superset/sql_lab.py +++ b/superset/sql_lab.py @@ -538,9 +538,7 @@ def execute_sql_statements( # pylint: disable=too-many-arguments, too-many-loca ) cache_timeout = database.cache_timeout if cache_timeout is None: - cache_timeout = app.config["DEFAULT_CACHE_CONFIG"][ - "CACHE_DEFAULT_TIMEOUT" - ] + cache_timeout = config["CACHE_DEFAULT_TIMEOUT"] compressed = zlib_compress(serialized_payload) logger.debug( diff --git a/superset/utils/cache.py b/superset/utils/cache.py index 1adb8af7b9f17..c10f296e1cac4 100644 --- a/superset/utils/cache.py +++ b/superset/utils/cache.py @@ -21,7 +21,7 @@ from functools import wraps from typing import Any, Callable, Dict, Optional, TYPE_CHECKING, Union -from flask import current_app as app, Flask, request +from flask import current_app as app, request from flask_caching import Cache from flask_caching.backends import NullCache from werkzeug.wrappers.etag import ETagResponseMixin @@ -29,7 +29,6 @@ from superset import db from superset.extensions import cache_manager from superset.models.cache import CacheKey -from superset.typing import CacheConfig from superset.utils.core import json_int_dttm_ser from superset.utils.hashing import md5_sha_from_dict @@ -59,7 +58,7 @@ def set_and_log_cache( timeout = ( cache_timeout if cache_timeout is not None - else app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] + else app.config["CACHE_DEFAULT_TIMEOUT"] ) try: dttm = datetime.utcnow().isoformat().split(".")[0] @@ -151,7 +150,7 @@ def etag_cache( """ if max_age is None: - max_age = app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] + max_age = app.config["CACHE_DEFAULT_TIMEOUT"] def decorator(f: Callable[..., Any]) -> Callable[..., Any]: @wraps(f) diff --git a/superset/utils/cache_manager.py b/superset/utils/cache_manager.py index f40e6018950e9..a0c759035fc2b 100644 --- a/superset/utils/cache_manager.py +++ b/superset/utils/cache_manager.py @@ -21,8 +21,6 @@ from flask_babel import gettext as _ from flask_caching import Cache -from superset.typing import CacheConfig - logger = logging.getLogger(__name__) @@ -30,62 +28,71 @@ class CacheManager: def __init__(self) -> None: super().__init__() - self._default_cache_config: CacheConfig = {} - self.cache = Cache() - self.data_cache = Cache() - self.thumbnail_cache = Cache() - self.filter_state_cache = Cache() - self.explore_form_data_cache = Cache() + self._cache = Cache() + self._data_cache = Cache() + self._thumbnail_cache = Cache() + self._filter_state_cache = Cache() + self._explore_form_data_cache = Cache() + @staticmethod def _init_cache( - self, app: Flask, cache: Cache, cache_config_key: str, required: bool = False + app: Flask, cache: Cache, cache_config_key: str, required: bool = False ) -> None: - config = {**self._default_cache_config, **app.config[cache_config_key]} - if required and config["CACHE_TYPE"] in ("null", "NullCache"): + cache_config = app.config[cache_config_key] + cache_type = cache_config.get("CACHE_TYPE") + if app.debug and cache_type is None: + cache_threshold = cache_config.get("CACHE_THRESHOLD", math.inf) + cache_config.update( + {"CACHE_TYPE": "SimpleCache", "CACHE_THRESHOLD": cache_threshold,} + ) + + if "CACHE_DEFAULT_TIMEOUT" not in cache_config: + default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") + cache_config["CACHE_DEFAULT_TIMEOUT"] = default_timeout + + if required and cache_type in ("null", "NullCache"): raise Exception( _( "The CACHE_TYPE `%(cache_type)s` for `%(cache_config_key)s` is not " - "supported. It is recommended to use `RedisCache`, `MemcachedCache` " - "or another dedicated caching backend for production deployments", - cache_type=config["CACHE_TYPE"], + "supported. It is recommended to use `RedisCache`, " + "`MemcachedCache` or another dedicated caching backend for " + "production deployments", + cache_type=cache_config["CACHE_TYPE"], cache_config_key=cache_config_key, ), ) - cache.init_app(app, config) + cache.init_app(app, cache_config) def init_app(self, app: Flask) -> None: - if app.debug: - self._default_cache_config = { - "CACHE_TYPE": "SimpleCache", - "CACHE_THRESHOLD": math.inf, - } - else: - self._default_cache_config = {} - - default_timeout = app.config.get("CACHE_DEFAULT_TIMEOUT") - if default_timeout is not None: - self._default_cache_config["CACHE_DEFAULT_TIMEOUT"] = default_timeout - logger.warning( - _( - "The global config flag `CACHE_DEFAULT_TIMEOUT` has been " - "deprecated and will be removed in Superset 2.0. Please set " - "default cache options in the `DEFAULT_CACHE_CONFIG` parameter" - ), - ) - self._default_cache_config = { - **self._default_cache_config, - **app.config["DEFAULT_CACHE_CONFIG"], - } - - self._init_cache(app, self.cache, "CACHE_CONFIG") - self._init_cache(app, self.data_cache, "DATA_CACHE_CONFIG") - self._init_cache(app, self.thumbnail_cache, "THUMBNAIL_CACHE_CONFIG") + self._init_cache(app, self._cache, "CACHE_CONFIG") + self._init_cache(app, self._data_cache, "DATA_CACHE_CONFIG") + self._init_cache(app, self._thumbnail_cache, "THUMBNAIL_CACHE_CONFIG") self._init_cache( - app, self.filter_state_cache, "FILTER_STATE_CACHE_CONFIG", required=True + app, self._filter_state_cache, "FILTER_STATE_CACHE_CONFIG", required=True ) self._init_cache( app, - self.explore_form_data_cache, + self._explore_form_data_cache, "EXPLORE_FORM_DATA_CACHE_CONFIG", required=True, ) + + @property + def data_cache(self) -> Cache: + return self._data_cache + + @property + def cache(self) -> Cache: + return self._cache + + @property + def thumbnail_cache(self) -> Cache: + return self._thumbnail_cache + + @property + def filter_state_cache(self) -> Cache: + return self._filter_state_cache + + @property + def explore_form_data_cache(self) -> Cache: + return self._explore_form_data_cache diff --git a/superset/viz.py b/superset/viz.py index f1ce2c5543b21..3a519bb1f9e83 100644 --- a/superset/viz.py +++ b/superset/viz.py @@ -429,7 +429,7 @@ def cache_timeout(self) -> int: return self.datasource.database.cache_timeout if config["DATA_CACHE_CONFIG"].get("CACHE_DEFAULT_TIMEOUT") is not None: return config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] - return app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] + return config["CACHE_DEFAULT_TIMEOUT"] def get_json(self) -> str: return json.dumps( diff --git a/tests/integration_tests/cache_tests.py b/tests/integration_tests/cache_tests.py index ddb9d91c6d424..a7da8a50d2a59 100644 --- a/tests/integration_tests/cache_tests.py +++ b/tests/integration_tests/cache_tests.py @@ -64,17 +64,12 @@ def test_no_data_cache(self): @pytest.mark.usefixtures("load_birth_names_dashboard_with_slices") def test_slice_data_cache(self): # Override cache config - default_cache_config = app.config["DEFAULT_CACHE_CONFIG"] - - app.config["DEFAULT_CACHE_CONFIG"] = { - **default_cache_config, - "CACHE_DEFAULT_TIMEOUT": 100, - } data_cache_config = app.config["DATA_CACHE_CONFIG"] - + cache_default_timeout = app.config["CACHE_DEFAULT_TIMEOUT"] + app.config["CACHE_DEFAULT_TIMEOUT"] = 100 app.config["DATA_CACHE_CONFIG"] = { + "CACHE_TYPE": "SimpleCache", "CACHE_DEFAULT_TIMEOUT": 10, - "CACHE_KEY_PREFIX": "superset_data_cache", } cache_manager.init_app(app) @@ -105,5 +100,5 @@ def test_slice_data_cache(self): # reset cache config app.config["DATA_CACHE_CONFIG"] = data_cache_config - app.config["DEFAULT_CACHE_CONFIG"] = default_cache_config + app.config["CACHE_DEFAULT_TIMEOUT"] = cache_default_timeout cache_manager.init_app(app) diff --git a/tests/integration_tests/dashboards/filter_state/api_tests.py b/tests/integration_tests/dashboards/filter_state/api_tests.py index 2a11980535efd..3816f6ac87337 100644 --- a/tests/integration_tests/dashboards/filter_state/api_tests.py +++ b/tests/integration_tests/dashboards/filter_state/api_tests.py @@ -62,7 +62,6 @@ def admin_id() -> int: @pytest.fixture(autouse=True) def cache(dashboard_id, admin_id): - cache_manager.init_app(app) entry: Entry = {"owner": admin_id, "value": value} cache_manager.filter_state_cache.set(cache_key(dashboard_id, key), entry) diff --git a/tests/integration_tests/explore/form_data/api_tests.py b/tests/integration_tests/explore/form_data/api_tests.py index 112eec2bb839b..4b646a03586de 100644 --- a/tests/integration_tests/explore/form_data/api_tests.py +++ b/tests/integration_tests/explore/form_data/api_tests.py @@ -74,7 +74,6 @@ def dataset_id() -> int: @pytest.fixture(autouse=True) def cache(chart_id, admin_id, dataset_id): - cache_manager.init_app(app) entry: TemporaryExploreState = { "owner": admin_id, "dataset_id": dataset_id, diff --git a/tests/integration_tests/superset_test_config.py b/tests/integration_tests/superset_test_config.py index 3e3b6151f8986..7c862328294b5 100644 --- a/tests/integration_tests/superset_test_config.py +++ b/tests/integration_tests/superset_test_config.py @@ -87,13 +87,6 @@ def GET_FEATURE_FLAGS_FUNC(ff): REDIS_CACHE_DB = os.environ.get("REDIS_CACHE_DB", 4) -DEFAULT_CACHE_CONFIG = { - "CACHE_TYPE": "SimpleCache", - "CACHE_THRESHOLD": math.inf, - "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), -} - - CACHE_CONFIG = { "CACHE_TYPE": "RedisCache", "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=1).total_seconds()), @@ -107,6 +100,18 @@ def GET_FEATURE_FLAGS_FUNC(ff): "CACHE_KEY_PREFIX": "superset_data_cache", } +FILTER_STATE_CACHE_CONFIG = { + "CACHE_TYPE": "SimpleCache", + "CACHE_THRESHOLD": math.inf, + "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), +} + +EXPLORE_FORM_DATA_CACHE_CONFIG = { + "CACHE_TYPE": "SimpleCache", + "CACHE_THRESHOLD": math.inf, + "CACHE_DEFAULT_TIMEOUT": int(timedelta(minutes=10).total_seconds()), +} + GLOBAL_ASYNC_QUERIES_JWT_SECRET = "test-secret-change-me-test-secret-change-me" ALERT_REPORTS_WORKING_TIME_OUT_KILL = True diff --git a/tests/integration_tests/superset_test_config_thumbnails.py b/tests/integration_tests/superset_test_config_thumbnails.py index 2e375931c3867..9f621efabbf4d 100644 --- a/tests/integration_tests/superset_test_config_thumbnails.py +++ b/tests/integration_tests/superset_test_config_thumbnails.py @@ -53,6 +53,8 @@ def GET_FEATURE_FLAGS_FUNC(ff): AUTH_ROLE_PUBLIC = "Public" EMAIL_NOTIFICATIONS = False +CACHE_CONFIG = {"CACHE_TYPE": "SimpleCache"} + REDIS_HOST = os.environ.get("REDIS_HOST", "localhost") REDIS_PORT = os.environ.get("REDIS_PORT", "6379") REDIS_CELERY_DB = os.environ.get("REDIS_CELERY_DB", 2) diff --git a/tests/integration_tests/viz_tests.py b/tests/integration_tests/viz_tests.py index 56af2240f480d..465fdb26ef581 100644 --- a/tests/integration_tests/viz_tests.py +++ b/tests/integration_tests/viz_tests.py @@ -173,10 +173,7 @@ def test_cache_timeout(self): app.config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] = None datasource.database.cache_timeout = None test_viz = viz.BaseViz(datasource, form_data={}) - self.assertEqual( - app.config["DEFAULT_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"], - test_viz.cache_timeout, - ) + self.assertEqual(app.config["CACHE_DEFAULT_TIMEOUT"], test_viz.cache_timeout) # restore DATA_CACHE_CONFIG timeout app.config["DATA_CACHE_CONFIG"]["CACHE_DEFAULT_TIMEOUT"] = data_cache_timeout