From 72e0e6a7f0826d811f9f26191b99838f59adebfa Mon Sep 17 00:00:00 2001 From: Oscar Domingo Date: Mon, 29 Jan 2024 15:43:22 +0000 Subject: [PATCH] Refactor `ayon_server` modules to use the new `AddonLibrary` and `RezRepo` Modified all modules where the interface with `AddonLibrary` changed or where it was now necessary to use the `RezRepo`. --- ayon_server/addons/__init__.py | 4 +- ayon_server/addons/addon.py | 26 ++-- ayon_server/api/server.py | 218 +++++++++++++++--------------- ayon_server/installer/__init__.py | 6 +- ayon_server/installer/addons.py | 183 ------------------------- ayon_server/metrics/bundles.py | 11 +- ayon_server/metrics/settings.py | 6 +- 7 files changed, 143 insertions(+), 311 deletions(-) delete mode 100644 ayon_server/installer/addons.py diff --git a/ayon_server/addons/__init__.py b/ayon_server/addons/__init__.py index 2bbb945b..1d9ad624 100644 --- a/ayon_server/addons/__init__.py +++ b/ayon_server/addons/__init__.py @@ -1,6 +1,8 @@ -__all__ = ["AddonLibrary", "BaseServerAddon", "SSOOption"] +__all__ = ["AddonLibrary", "BaseServerAddon", "SSOOption", "RezRepo"] + from ayon_server.addons.addon import BaseServerAddon from ayon_server.addons.library import AddonLibrary from ayon_server.addons.models import SSOOption +from ayon_server.addons.rezrepo import RezRepo diff --git a/ayon_server/addons/addon.py b/ayon_server/addons/addon.py index fe91f8c4..4ded4445 100644 --- a/ayon_server/addons/addon.py +++ b/ayon_server/addons/addon.py @@ -15,7 +15,7 @@ from ayon_server.settings import BaseSettingsModel, apply_overrides if TYPE_CHECKING: - from ayon_server.addons.definition import ServerAddonDefinition + from rez.packages import Package class BaseServerAddon: @@ -23,7 +23,7 @@ class BaseServerAddon: version: str title: str | None = None app_host_name: str | None = None - definition: "ServerAddonDefinition" + rez_package: "Package" endpoints: list[dict[str, Any]] settings_model: Type[BaseSettingsModel] | None = None site_settings_model: Type[BaseSettingsModel] | None = None @@ -31,9 +31,9 @@ class BaseServerAddon: services: dict[str, Any] = {} system: bool = False # Hide settings for non-admins and make the addon mandatory - def __init__(self, definition: "ServerAddonDefinition", addon_dir: str): + def __init__(self, rez_package: "Package", addon_dir: str): assert self.name and self.version - self.definition = definition + self.rez_package = rez_package self.addon_dir = addon_dir self.endpoints = [] self.restart_requested = False @@ -41,12 +41,20 @@ def __init__(self, definition: "ServerAddonDefinition", addon_dir: str): self.initialize() def __repr__(self) -> str: - return f"" + return f"" + + @property + def definition(self) -> "Package": + return self.rez_package @property def friendly_name(self) -> str: """Return the friendly name of the addon.""" - return f"{self.definition.friendly_name} {self.version}" + try: + friendly_name = self.rez_package.friendly_name + except AttributeError: + friendly_name = self.name.capitalize() + return f"{friendly_name} {self.version}" async def is_production(self) -> bool: """Return True if the addon is in production bundle.""" @@ -224,7 +232,7 @@ async def get_studio_overrides(self, variant: str = "production") -> dict[str, A WHERE addon_name = $1 AND addon_version = $2 AND variant = $3 """ - res = await Postgres.fetch(query, self.definition.name, self.version, variant) + res = await Postgres.fetch(query, self.rez_package.name, self.version, variant) if res: return dict(res[0]["data"]) return {} @@ -243,7 +251,7 @@ async def get_project_overrides( try: res = await Postgres.fetch( - query, self.definition.name, self.version, variant + query, self.rez_package.name, self.version, variant ) except Postgres.UndefinedTableError: raise NotFoundException(f"Project {project_name} does not exists") from None @@ -265,7 +273,7 @@ async def get_project_site_overrides( WHERE addon_name = $1 AND addon_version = $2 AND user_name = $3 AND site_id = $4 """, - self.definition.name, + self.rez_package.name, self.version, user_name, site_id, diff --git a/ayon_server/api/server.py b/ayon_server/api/server.py index 83a521f6..2de605f9 100644 --- a/ayon_server/api/server.py +++ b/ayon_server/api/server.py @@ -13,8 +13,7 @@ from nxtools import log_to_file, log_traceback, logging, slugify from ayon_server.access.access_groups import AccessGroups -from ayon_server.addons import get_enabled_addons -from ayon_server.addons.rezrepo import RezRepo +from ayon_server.addons import AddonLibrary, RezRepo from ayon_server.api.messaging import Messaging from ayon_server.api.metadata import app_meta, tags_meta from ayon_server.api.responses import ErrorResponse @@ -303,64 +302,57 @@ def init_api(target_app: fastapi.FastAPI, plugin_dir: str = "api") -> None: route.operation_id = route.name -def init_addon_endpoints(target_app: fastapi.FastAPI) -> None: - # library = AddonLibrary.getinstance() - for addon in get_enabled_addons(): - print(addon) # get_addon(addon) - # for addon_name, addon_definition in library.items(): - # for version in addon_definition.versions: - # addon = addon_definition.versions[version] - # for endpoint in addon.endpoints: - # path = endpoint["path"].lstrip("/") - # first_element = path.split("/")[0] - # # TODO: site settings? other routes? - # if first_element in ["settings", "schema", "overrides"]: - # logging.error(f"Unable to assing path to endpoint: {path}") - # continue - - # path = f"/api/addons/{addon_name}/{version}/{path}" - # target_app.add_api_route( - # path, - # endpoint["handler"], - # methods=[endpoint["method"]], - # name=endpoint["name"], - # tags=[f"{addon_definition.friendly_name} {version}"], - # operation_id=slugify( - # f"{addon_name}_{version}_{endpoint['name']}", - # separator="_", - # ), - # ) - - -def init_addon_static(target_app: fastapi.FastAPI) -> None: +def init_addon_endpoints(target_app: fastapi.FastAPI, addons: list) -> None: + for addon_name, addon in addons.items(): + for endpoint in addon.endpoints: + path = endpoint["path"].lstrip("/") + first_element = path.split("/")[0] + # TODO: site settings? other routes? + if first_element in ["settings", "schema", "overrides"]: + logging.error(f"Unable to assing path to endpoint: {path}") + continue + + path = f"/api/addons/{addon_name}/{addon.version}/{path}" + target_app.add_api_route( + path, + endpoint["handler"], + methods=[endpoint["method"]], + name=endpoint["name"], + tags=[f"{addon.title} {addon>addon.version}"], + operation_id=slugify( + f"{addon.name}_{addon.version}_{endpoint['name']}", + separator="_", + ), + ) + + +def init_addon_static(target_app: fastapi.FastAPI, addons: list) -> None: """Serve static files for addon frontends.""" - # for addon_name, addon_definition in AddonLibrary.items(): - # for version in addon_definition.versions: - # addon = addon_definition.versions[version] - # static_dirs = [] - # if (fedir := addon.get_frontend_dir()) is not None: - # static_dirs.append("frontend") - # target_app.mount( - # f"/addons/{addon_name}/{version}/frontend/", - # StaticFiles(directory=fedir, html=True), - # ) - # if (resdir := addon.get_public_dir()) is not None: - # static_dirs.append("public") - # target_app.mount( - # f"/addons/{addon_name}/{version}/public/", - # StaticFiles(directory=resdir), - # ) - # if (resdir := addon.get_private_dir()) is not None: - # static_dirs.append("private") - # target_app.mount( - # f"/addons/{addon_name}/{version}/private/", - # AuthStaticFiles(directory=resdir), - # ) - - # if static_dirs: - # logging.debug( - # f"Initialized static dirs for {addon_name}:{version}: {', '.join(static_dirs)}" - # ) + for addon_name, addon in addons.items(): + static_dirs = [] + if (fedir := addon.get_frontend_dir()) is not None: + static_dirs.append("frontend") + target_app.mount( + f"/addons/{addon.name}/{addon.version}/frontend/", + StaticFiles(directory=fedir, html=True), + ) + if (resdir := addon.get_public_dir()) is not None: + static_dirs.append("public") + target_app.mount( + f"/addons/{addon.name}/{addon.version}/public/", + StaticFiles(directory=resdir), + ) + if (resdir := addon.get_private_dir()) is not None: + static_dirs.append("private") + target_app.mount( + f"/addons/{addon.name}/{addon.version}/private/", + AuthStaticFiles(directory=resdir), + ) + + if static_dirs: + logging.debug( + f"Initialized static dirs for {addon.name}:{addon.version}: {', '.join(static_dirs)}" + ) def init_frontend(target_app: fastapi.FastAPI, frontend_dir: str) -> None: @@ -422,69 +414,72 @@ async def startup_event() -> None: messaging.start() # Initialize addons - start_event = await dispatch_event("server.started", finished=False) - library = AddonLibrary.getinstance() - addon_records = list(AddonLibrary.items()) - if library.restart_requested: - logging.warning("Restart requested, skipping addon setup") + rezrepo = RezRepo.get_instance() + + # In case we installed the `ayon_server` + if rezrepo.restart_requested: await dispatch_event( "server.restart_requested", description="Server restart requested during addon initialization", ) - return + # addon_library = .get_instance() + initialized_addons, broken_addons = await AddonsLibrary.initialize_enabled_addons() restart_requested = False bad_addons = {} - for addon_name, addon in rezrepo.packages: - for version in addon.versions.values(): - try: - if inspect.iscoroutinefunction(version.pre_setup): - # Since setup may, but does not have to be async, we need to - # silence mypy here. - await version.pre_setup() # type: ignore - else: - version.pre_setup() - if (not restart_requested) and version.restart_requested: - logging.warning( - f"Restart requested during addon {addon_name} pre-setup." - ) - restart_requested = True - except Exception as e: - log_traceback(f"Error during {addon_name} {version.version} pre-setup") - reason = { - "error": str(e), - "traceback": traceback.format_exc(), - } - bad_addons[(addon_name, version.version)] = reason - - for addon_name, addon in addon_records: - for version in addon.versions.values(): - try: - if inspect.iscoroutinefunction(version.setup): - await version.setup() - else: - version.setup() - if (not restart_requested) and version.restart_requested: - logging.warning( - f"Restart requested during addon {addon_name} setup." - ) - restart_requested = True - except Exception as e: - log_traceback(f"Error during {addon_name} {version.version} setup") - reason = { - "error": str(e), - "traceback": traceback.format_exc(), - } - bad_addons[(addon_name, version.version)] = reason + + async def _run_addon_method(addon, method_name): + method_object = getattr(addon, method_name, False) + + try: + if inspect.iscoroutinefunction(method_object): + # Since setup may, but does not have to be async, we need to + # silence mypy here. + await method_object() # type: ignore + else: + method_object() + except Exception as e: + log_traceback(f"Error during {addon} pre-setup") + reason = { + "error": str(e), + "traceback": traceback.format_exc(), + } + return (False, reason) + + return (True, None) + + # Perform Addon's `pre_setup` method + for addon_name, addon_class in initialized_addons.items(): + ran, result = await _run_addon_method(addon_class, "pre_setup") + + if not ran: + bad_addons[(addon_class.name, addon_class.version)] = result + + if (not restart_requested) and addon_class.restart_requested: + logging.warning( + f"Restart requested during addon {addon_name}-{addon_class.version} setup." + ) + restart_requested = True + + # Perform Addon's `pre_setup` method + for addon_name, addon_class in initialized_addons.items(): + ran, result = await _run_addon_method(addon_class, "setup") + + if not ran: + bad_addons[(addon_class.name, addon_class.version)] = result + + if (not restart_requested) and addon_class.restart_requested: + logging.warning( + f"Restart requested during addon {addon_name}-{addon_class.version} setup." + ) + restart_requested = True for _addon_name, _addon_version in bad_addons: logging.error( - f"Addon {_addon_name} {_addon_version} failed to initialize. Unloading." + f"Addon {_addon_name} {_addon_version} failed to initialize." ) - reason = bad_addons[(_addon_name, _addon_version)] - library.unload_addon(_addon_name, _addon_version, reason=reason) if restart_requested: await dispatch_event( @@ -493,10 +488,10 @@ async def startup_event() -> None: ) else: # Initialize endpoints for active addons - init_addon_endpoints(app) + init_addon_endpoints(app, initialized_addons) # Addon static dirs must stay exactly here - init_addon_static(app) + init_addon_static(app, initialized_addons) # Frontend must be initialized last (since it is mounted to /) init_frontend(app, ayonconfig.frontend_dir) @@ -519,3 +514,4 @@ async def shutdown_event() -> None: await messaging.shutdown() await Postgres.shutdown() logging.info("Server stopped", handlers=None) + diff --git a/ayon_server/installer/__init__.py b/ayon_server/installer/__init__.py index 6145b3e8..d603bb86 100644 --- a/ayon_server/installer/__init__.py +++ b/ayon_server/installer/__init__.py @@ -4,7 +4,7 @@ from ayon_server.background.background_worker import BackgroundWorker from ayon_server.events import update_event -from ayon_server.installer.addons import install_addon_from_url, unpack_addon +from ayon_server.addons import AddonLibrary from ayon_server.installer.dependency_packages import download_dependency_package from ayon_server.installer.installers import download_installer from ayon_server.lib.postgres import Postgres @@ -48,7 +48,7 @@ async def process_event(self, event_id: str): logging.info(f"Background installer: processing {topic} event: {event_id}") if topic == "addon.install": - await unpack_addon( + await AddonLibrary.install_addon( event_id, summary["zip_path"], summary["addon_name"], @@ -56,7 +56,7 @@ async def process_event(self, event_id: str): ) elif topic == "addon.install_from_url": - await install_addon_from_url(event_id, summary["url"]) + await AddonLibrary.install_addon_from_url(event_id, summary["url"]) elif topic == "dependency_package.install_from_url": await download_dependency_package(event_id, summary["url"]) diff --git a/ayon_server/installer/addons.py b/ayon_server/installer/addons.py deleted file mode 100644 index ef26e339..00000000 --- a/ayon_server/installer/addons.py +++ /dev/null @@ -1,183 +0,0 @@ -import asyncio -import json -import os -import shutil -import tempfile -import time -import zipfile -from concurrent.futures import ThreadPoolExecutor - -import aiofiles -import httpx -from nxtools import logging - -from ayon_server.config import ayonconfig -from ayon_server.events import update_event - - -def get_addon_zip_info(path: str) -> tuple[str, str]: - """Returns the addon name and version from the zip file""" - with zipfile.ZipFile(path, "r") as zip_ref: - names = zip_ref.namelist() - if "manifest.json" not in names: - raise RuntimeError("Addon manifest not found in zip file") - - if "addon/__init__.py" not in names: - raise RuntimeError("Addon __init__.py not found in zip file") - - with zip_ref.open("manifest.json") as manifest_file: - manifest = json.load(manifest_file) - - addon_name = manifest.get("addon_name") - addon_version = manifest.get("addon_version") - - if not (addon_name and addon_version): - raise RuntimeError("Addon name or version not found in manifest") - return addon_name, addon_version - - -def unpack_addon_sync(zip_path: str, addon_name: str, addon_version) -> None: - addon_root_dir = ayonconfig.addons_dir - os.makedirs(addon_root_dir, exist_ok=True) - target_dir = os.path.join(addon_root_dir, addon_name, addon_version) - - with tempfile.TemporaryDirectory(dir=addon_root_dir) as tmpdirname: - with zipfile.ZipFile(zip_path, "r") as zip_ref: - for member in zip_ref.infolist(): - extracted_path = zip_ref.extract(member, tmpdirname) - - # Preserve the file permissions - original_mode = member.external_attr >> 16 - if original_mode: - os.chmod(extracted_path, original_mode) - - if os.path.isdir(target_dir): - logging.info(f"Removing existing addon {addon_name} {addon_version}") - shutil.rmtree(target_dir) - - # move the extracted files to the target directory - shutil.move(os.path.join(tmpdirname, "addon"), target_dir) - - -async def unpack_addon( - event_id: str, - zip_path: str, - addon_name: str, - addon_version: str, -): - """Unpack the addon from the zip file and install it - - Unpacking is done in a separate thread to avoid blocking the main thread - (unzipping is a synchronous operation and it is also cpu-bound) - - After the addon is unpacked, the event is finalized and the zip file is removed. - """ - - await update_event( - event_id, - description=f"Unpacking addon {addon_name} {addon_version}", - status="in_progress", - ) - - loop = asyncio.get_event_loop() - - try: - with ThreadPoolExecutor() as executor: - task = loop.run_in_executor( - executor, - unpack_addon_sync, - zip_path, - addon_name, - addon_version, - ) - await asyncio.gather(task) - except Exception as e: - logging.error(f"Error while unpacking addon: {e}") - await update_event( - event_id, - description=f"Error while unpacking addon: {e}", - status="failed", - ) - - try: - os.remove(zip_path) - except Exception as e: - logging.error(f"Error while removing zip file: {e}") - - await update_event( - event_id, - description=f"Addon {addon_name} {addon_version} installed", - status="finished", - ) - - -async def install_addon_from_url(event_id: str, url: str) -> None: - """Download the addon zip file from the URL and install it""" - - await update_event( - event_id, - description=f"Downloading addon from URL {url}", - status="in_progress", - ) - - # Download the zip file - # we do not use download_file() here because using NamedTemporaryFile - # is much more convenient than manually creating a temporary file - - file_size = 0 - last_time = 0.0 - - i = 0 - with tempfile.NamedTemporaryFile(dir=ayonconfig.addons_dir) as temporary_file: - zip_path = temporary_file.name - async with httpx.AsyncClient(timeout=ayonconfig.http_timeout) as client: - async with client.stream("GET", url) as response: - file_size = int(response.headers.get("content-length", 0)) - async with aiofiles.open(zip_path, "wb") as f: - async for chunk in response.aiter_bytes(): - await f.write(chunk) - i += len(chunk) - - if file_size and (time.time() - last_time > 1): - percent = int(i / file_size * 100) - await update_event( - event_id, - progress=int(percent / 2), - store=False, - ) - last_time = time.time() - - # Get the addon name and version from the zip file - - addon_name, addon_version = get_addon_zip_info(zip_path) - await update_event( - event_id, - description=f"Installing addon {addon_name} {addon_version}", - status="in_progress", - summary={ - "addon_name": addon_name, - "addon_version": addon_version, - "url": url, - }, - progress=50, - ) - - # Unpack the addon - - loop = asyncio.get_event_loop() - - with ThreadPoolExecutor() as executor: - task = loop.run_in_executor( - executor, - unpack_addon_sync, - zip_path, - addon_name, - addon_version, - ) - await asyncio.gather(task) - - await update_event( - event_id, - description=f"Addon {addon_name} {addon_version} installed", - status="finished", - ) diff --git a/ayon_server/metrics/bundles.py b/ayon_server/metrics/bundles.py index f5679b0e..d69e7b7e 100644 --- a/ayon_server/metrics/bundles.py +++ b/ayon_server/metrics/bundles.py @@ -1,4 +1,4 @@ -from ayon_server.addons.rezrepo import RezRepo +from ayon_server.addons import RezRepo from ayon_server.lib.postgres import Postgres from ayon_server.types import Field, OPModel @@ -51,5 +51,12 @@ async def get_installed_addons(saturated: bool) -> list[tuple[str, str]]: """ rezrepo = RezRepo.get_instance() - return rezrepo.packages + installed_packages = [] + for package, package_dict in rezrepo.packages.items(): + for version in package_dict.get("versions"): + [[version_number, rez_package]] = version.items() + installed_packages.append((str(package), str(version_number))) + + return installed_packages + diff --git a/ayon_server/metrics/settings.py b/ayon_server/metrics/settings.py index c0195ccb..07aba2d9 100644 --- a/ayon_server/metrics/settings.py +++ b/ayon_server/metrics/settings.py @@ -1,4 +1,4 @@ -from ayon_server.addons import get_addon +from ayon_server.addons import AddonLibrary from ayon_server.lib.postgres import Postgres from ayon_server.settings.overrides import list_overrides from ayon_server.types import Field, OPModel @@ -44,7 +44,9 @@ async def get_studio_settings_overrides(saturated: bool) -> list[SettingsOverrid data = row["data"] try: - addon = AddonLibrary.addon(addon_name, addon_version) + addon = AddonLibrary.get_addon(addon_name, addon_version) + if not addon: + continue except Exception: continue