From fdc2698ba1a40082198bdb4f041740cea08818d5 Mon Sep 17 00:00:00 2001 From: Ville Brofeldt Date: Fri, 9 Dec 2022 15:51:54 +0200 Subject: [PATCH] move tasks back and fix digest func --- docs/docs/installation/alerts-reports.mdx | 2 +- docs/docs/installation/cache.mdx | 2 +- .../installation/running-on-kubernetes.mdx | 2 +- superset/charts/api.py | 2 +- superset/cli/thumbnails.py | 2 +- superset/dashboards/api.py | 2 +- superset/models/dashboard.py | 2 +- superset/models/slice.py | 2 +- superset/tasks/thumbnails.py | 86 +++++++++++++-- superset/thumbnails/digest.py | 3 +- superset/thumbnails/tasks.py | 101 ------------------ .../superset_test_config_thumbnails.py | 2 +- tests/unit_tests/thumbnails/test_digest.py | 10 +- 13 files changed, 94 insertions(+), 124 deletions(-) delete mode 100644 superset/thumbnails/tasks.py diff --git a/docs/docs/installation/alerts-reports.mdx b/docs/docs/installation/alerts-reports.mdx index 5551641392e67..3538ca1479e42 100644 --- a/docs/docs/installation/alerts-reports.mdx +++ b/docs/docs/installation/alerts-reports.mdx @@ -90,7 +90,7 @@ REDIS_PORT = "6379" class CeleryConfig: broker_url = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) - imports = ('superset.sql_lab', "superset.tasks", "superset.thumbnails.tasks", ) + imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) result_backend = 'redis://%s:%s/0' % (REDIS_HOST, REDIS_PORT) worker_prefetch_multiplier = 10 task_acks_late = True diff --git a/docs/docs/installation/cache.mdx b/docs/docs/installation/cache.mdx index 9b00121a7bee3..4838fc47e61c1 100644 --- a/docs/docs/installation/cache.mdx +++ b/docs/docs/installation/cache.mdx @@ -73,7 +73,7 @@ from s3cache.s3cache import S3Cache class CeleryConfig(object): broker_url = "redis://localhost:6379/0" - imports = ("superset.sql_lab", "superset.tasks", "superset.thumbnails.tasks") + imports = ("superset.sql_lab", "superset.tasks", "superset.tasks.thumbnails") result_backend = "redis://localhost:6379/0" worker_prefetch_multiplier = 10 task_acks_late = True diff --git a/docs/docs/installation/running-on-kubernetes.mdx b/docs/docs/installation/running-on-kubernetes.mdx index e2c1f92613255..1b75e0f5c6101 100644 --- a/docs/docs/installation/running-on-kubernetes.mdx +++ b/docs/docs/installation/running-on-kubernetes.mdx @@ -345,7 +345,7 @@ configOverrides: class CeleryConfig(object): broker_url = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0" - imports = ('superset.sql_lab', "superset.tasks", "superset.thumbnails.tasks", ) + imports = ('superset.sql_lab', "superset.tasks", "superset.tasks.thumbnails", ) result_backend = f"redis://{env('REDIS_HOST')}:{env('REDIS_PORT')}/0" task_annotations = { 'sql_lab.get_sql_results': { diff --git a/superset/charts/api.py b/superset/charts/api.py index bc5fc2f0a0874..c5e0eb77d8013 100644 --- a/superset/charts/api.py +++ b/superset/charts/api.py @@ -74,8 +74,8 @@ from superset.constants import MODEL_API_RW_METHOD_PERMISSION_MAP, RouteMethod from superset.extensions import event_logger from superset.models.slice import Slice +from superset.tasks.thumbnails import cache_chart_thumbnail from superset.tasks.utils import get_current_user -from superset.thumbnails.tasks import cache_chart_thumbnail from superset.utils.screenshots import ChartScreenshot from superset.utils.urls import get_url_path from superset.views.base_api import ( diff --git a/superset/cli/thumbnails.py b/superset/cli/thumbnails.py index 46d1d126f08d3..5556cff92f620 100755 --- a/superset/cli/thumbnails.py +++ b/superset/cli/thumbnails.py @@ -69,7 +69,7 @@ def compute_thumbnails( # pylint: disable=import-outside-toplevel from superset.models.dashboard import Dashboard from superset.models.slice import Slice - from superset.thumbnails.tasks import ( + from superset.tasks.thumbnails import ( cache_chart_thumbnail, cache_dashboard_thumbnail, ) diff --git a/superset/dashboards/api.py b/superset/dashboards/api.py index 71c85d4688af1..79255d19211b2 100644 --- a/superset/dashboards/api.py +++ b/superset/dashboards/api.py @@ -82,8 +82,8 @@ from superset.extensions import event_logger from superset.models.dashboard import Dashboard from superset.models.embedded_dashboard import EmbeddedDashboard +from superset.tasks.thumbnails import cache_dashboard_thumbnail from superset.tasks.utils import get_current_user -from superset.thumbnails.tasks import cache_dashboard_thumbnail from superset.utils.cache import etag_cache from superset.utils.screenshots import DashboardScreenshot from superset.utils.urls import get_url_path diff --git a/superset/models/dashboard.py b/superset/models/dashboard.py index 5e6af2de61bd9..ae6bae4b733ff 100644 --- a/superset/models/dashboard.py +++ b/superset/models/dashboard.py @@ -54,9 +54,9 @@ from superset.models.helpers import AuditMixinNullable, ImportExportMixin from superset.models.slice import Slice from superset.models.user_attributes import UserAttribute +from superset.tasks.thumbnails import cache_dashboard_thumbnail from superset.tasks.utils import get_current_user from superset.thumbnails.digest import get_dashboard_digest -from superset.thumbnails.tasks import cache_dashboard_thumbnail from superset.utils import core as utils from superset.utils.core import get_user_id from superset.utils.decorators import debounce diff --git a/superset/models/slice.py b/superset/models/slice.py index 6571f9b2c9d86..657ff7d38a079 100644 --- a/superset/models/slice.py +++ b/superset/models/slice.py @@ -42,9 +42,9 @@ from superset import db, is_feature_enabled, security_manager from superset.legacy import update_time_range from superset.models.helpers import AuditMixinNullable, ImportExportMixin +from superset.tasks.thumbnails import cache_chart_thumbnail from superset.tasks.utils import get_current_user from superset.thumbnails.digest import get_chart_digest -from superset.thumbnails.tasks import cache_chart_thumbnail from superset.utils import core as utils from superset.utils.memoized import memoized from superset.viz import BaseViz, viz_types diff --git a/superset/tasks/thumbnails.py b/superset/tasks/thumbnails.py index 247d865001be6..c03d13b0bdbeb 100644 --- a/superset/tasks/thumbnails.py +++ b/superset/tasks/thumbnails.py @@ -15,15 +15,87 @@ # specific language governing permissions and limitations # under the License. +"""Utility functions used across Superset""" + import logging +from typing import cast, Optional + +from flask import current_app -# TODO(villebro): deprecate in 3.0 -# pylint: disable=unused-wildcard-import,wildcard-import -from superset.thumbnails.tasks import * +from superset import security_manager, thumbnail_cache +from superset.extensions import celery_app +from superset.tasks.utils import get_executor +from superset.utils.core import override_user +from superset.utils.screenshots import ChartScreenshot, DashboardScreenshot +from superset.utils.urls import get_url_path +from superset.utils.webdriver import WindowSize logger = logging.getLogger(__name__) -logger.warning( - "The import path superset.tasks.thumbnails is deprecated and will be removed in" - "3.0. Please use superset.thumbnails.tasks instead." -) + +@celery_app.task(name="cache_chart_thumbnail", soft_time_limit=300) +def cache_chart_thumbnail( + current_user: Optional[str], + chart_id: int, + force: bool = False, + window_size: Optional[WindowSize] = None, + thumb_size: Optional[WindowSize] = None, +) -> None: + # pylint: disable=import-outside-toplevel + from superset.models.slice import Slice + + if not thumbnail_cache: + logger.warning("No cache set, refusing to compute") + return None + chart = cast(Slice, Slice.get(chart_id)) + url = get_url_path("Superset.slice", slice_id=chart.id, standalone="true") + logger.info("Caching chart: %s", url) + _, username = get_executor( + executor_types=current_app.config["THUMBNAIL_EXECUTE_AS"], + model=chart, + current_user=current_user, + ) + user = security_manager.find_user(username) + with override_user(user): + screenshot = ChartScreenshot(url, chart.digest) + screenshot.compute_and_cache( + user=user, + cache=thumbnail_cache, + force=force, + window_size=window_size, + thumb_size=thumb_size, + ) + return None + + +@celery_app.task(name="cache_dashboard_thumbnail", soft_time_limit=300) +def cache_dashboard_thumbnail( + current_user: Optional[str], + dashboard_id: int, + force: bool = False, + thumb_size: Optional[WindowSize] = None, +) -> None: + # pylint: disable=import-outside-toplevel + from superset.models.dashboard import Dashboard + + if not thumbnail_cache: + logging.warning("No cache set, refusing to compute") + return + dashboard = Dashboard.get(dashboard_id) + url = get_url_path("Superset.dashboard", dashboard_id_or_slug=dashboard.id) + + logger.info("Caching dashboard: %s", url) + _, username = get_executor( + executor_types=current_app.config["THUMBNAIL_EXECUTE_AS"], + model=dashboard, + current_user=current_user, + ) + user = security_manager.find_user(username) + with override_user(user): + screenshot = DashboardScreenshot(url, dashboard.digest) + screenshot.compute_and_cache( + user=user, + cache=thumbnail_cache, + force=force, + thumb_size=thumb_size, + ) diff --git a/superset/thumbnails/digest.py b/superset/thumbnails/digest.py index 077ab89251c2a..543983e21bd90 100644 --- a/superset/thumbnails/digest.py +++ b/superset/thumbnails/digest.py @@ -60,8 +60,7 @@ def get_dashboard_digest(dashboard: Dashboard) -> str: return func(dashboard, executor_type, executor) unique_string = ( - f"{dashboard.position_json}.{dashboard.css}" - f".{dashboard.json_metadata}.{executor}" + f"{dashboard.position_json}.{dashboard.css}.{dashboard.json_metadata}" ) unique_string = _adjust_string_for_executor(unique_string, executor_type, executor) diff --git a/superset/thumbnails/tasks.py b/superset/thumbnails/tasks.py deleted file mode 100644 index c03d13b0bdbeb..0000000000000 --- a/superset/thumbnails/tasks.py +++ /dev/null @@ -1,101 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -"""Utility functions used across Superset""" - -import logging -from typing import cast, Optional - -from flask import current_app - -from superset import security_manager, thumbnail_cache -from superset.extensions import celery_app -from superset.tasks.utils import get_executor -from superset.utils.core import override_user -from superset.utils.screenshots import ChartScreenshot, DashboardScreenshot -from superset.utils.urls import get_url_path -from superset.utils.webdriver import WindowSize - -logger = logging.getLogger(__name__) - - -@celery_app.task(name="cache_chart_thumbnail", soft_time_limit=300) -def cache_chart_thumbnail( - current_user: Optional[str], - chart_id: int, - force: bool = False, - window_size: Optional[WindowSize] = None, - thumb_size: Optional[WindowSize] = None, -) -> None: - # pylint: disable=import-outside-toplevel - from superset.models.slice import Slice - - if not thumbnail_cache: - logger.warning("No cache set, refusing to compute") - return None - chart = cast(Slice, Slice.get(chart_id)) - url = get_url_path("Superset.slice", slice_id=chart.id, standalone="true") - logger.info("Caching chart: %s", url) - _, username = get_executor( - executor_types=current_app.config["THUMBNAIL_EXECUTE_AS"], - model=chart, - current_user=current_user, - ) - user = security_manager.find_user(username) - with override_user(user): - screenshot = ChartScreenshot(url, chart.digest) - screenshot.compute_and_cache( - user=user, - cache=thumbnail_cache, - force=force, - window_size=window_size, - thumb_size=thumb_size, - ) - return None - - -@celery_app.task(name="cache_dashboard_thumbnail", soft_time_limit=300) -def cache_dashboard_thumbnail( - current_user: Optional[str], - dashboard_id: int, - force: bool = False, - thumb_size: Optional[WindowSize] = None, -) -> None: - # pylint: disable=import-outside-toplevel - from superset.models.dashboard import Dashboard - - if not thumbnail_cache: - logging.warning("No cache set, refusing to compute") - return - dashboard = Dashboard.get(dashboard_id) - url = get_url_path("Superset.dashboard", dashboard_id_or_slug=dashboard.id) - - logger.info("Caching dashboard: %s", url) - _, username = get_executor( - executor_types=current_app.config["THUMBNAIL_EXECUTE_AS"], - model=dashboard, - current_user=current_user, - ) - user = security_manager.find_user(username) - with override_user(user): - screenshot = DashboardScreenshot(url, dashboard.digest) - screenshot.compute_and_cache( - user=user, - cache=thumbnail_cache, - force=force, - thumb_size=thumb_size, - ) diff --git a/tests/integration_tests/superset_test_config_thumbnails.py b/tests/integration_tests/superset_test_config_thumbnails.py index 58fd367c42305..9f621efabbf4d 100644 --- a/tests/integration_tests/superset_test_config_thumbnails.py +++ b/tests/integration_tests/superset_test_config_thumbnails.py @@ -63,7 +63,7 @@ def GET_FEATURE_FLAGS_FUNC(ff): class CeleryConfig(object): BROKER_URL = f"redis://{REDIS_HOST}:{REDIS_PORT}/{REDIS_CELERY_DB}" - CELERY_IMPORTS = ("superset.sql_lab", "superset.thumbnails.tasks") + CELERY_IMPORTS = ("superset.sql_lab", "superset.tasks.thumbnails") CELERY_ANNOTATIONS = {"sql_lab.add": {"rate_limit": "10/s"}} CONCURRENCY = 1 diff --git a/tests/unit_tests/thumbnails/test_digest.py b/tests/unit_tests/thumbnails/test_digest.py index 72a293f85950e..c30d93f5eb81a 100644 --- a/tests/unit_tests/thumbnails/test_digest.py +++ b/tests/unit_tests/thumbnails/test_digest.py @@ -68,14 +68,14 @@ def CUSTOM_CHART_FUNC( [ExecutorType.SELENIUM], False, False, - "9dfd9e0685911ca56f041e57b63bd950", + "0f94edd2d7dfd71724ce8236c05a6bf5", ), ( None, [ExecutorType.CURRENT_USER], True, False, - "55fa9f78f4d8c96464fd5b369a8f2367", + "6b05b0eff27796f266e538a26a155cf7", ), ( { @@ -84,7 +84,7 @@ def CUSTOM_CHART_FUNC( [ExecutorType.CURRENT_USER], True, False, - "9725aa2717974238f03c3fc29bef243b", + "c9c06fd8058f8b1d3bb84e2699aaae0f", ), ( { @@ -93,7 +93,7 @@ def CUSTOM_CHART_FUNC( [ExecutorType.CURRENT_USER], True, False, - "234e168024483a520b705ecf71cf4fca", + "be15e84a631755e537bd2706beeae70a", ), ( { @@ -102,7 +102,7 @@ def CUSTOM_CHART_FUNC( [ExecutorType.CURRENT_USER], True, False, - "430dc5a4ab07928f4465c43a32b4c846", + "9996e1f6cf8ebbf2aebe2938a7d59a0e", ), ( None,