[CDF-23593] 😃 Refactor dump method

CHANGELOG.cdf-tk.md

@@ -25,6 +25,7 @@ Changes are grouped as follows:
 ### Fixed
 
 - No more warning about missing `.env` file when running in `Google Cloud Build`.
+- When deploying a `Sequence` resource, Cognite Toolkit now replaces `dataSetExternalId` with `dataSetId`.
 
 ## [0.3.23] - 2024-12-13
 

cognite_toolkit/_cdf_tk/client/api/lookup.py

@@ -16,14 +16,15 @@
 from cognite.client.utils.useful_types import SequenceNotStr
 
 from cognite_toolkit._cdf_tk.client.api_client import ToolkitAPI
+from cognite_toolkit._cdf_tk.constants import DRY_RUN_ID
 from cognite_toolkit._cdf_tk.exceptions import ResourceRetrievalError
 
 if TYPE_CHECKING:
     from cognite_toolkit._cdf_tk.client._toolkit_client import ToolkitClient
 
 
 class LookUpAPI(ToolkitAPI, ABC):
-    dry_run_id: int = -1
+    dry_run_id: int = DRY_RUN_ID
 
     def __init__(self, config: ClientConfig, api_version: str | None, cognite_client: "ToolkitClient") -> None:
         super().__init__(config, api_version, cognite_client)
@@ -82,7 +83,7 @@ def external_id(
         id: int | Sequence[int],
     ) -> str | list[str]:
         ids = [id] if isinstance(id, int) else id
-        missing = [id_ for id_ in ids if id not in self._reverse_cache]
+        missing = [id_ for id_ in ids if id_ not in self._reverse_cache]
         if missing:
             try:
                 lookup = self._external_id(missing)

cognite_toolkit/_cdf_tk/client/data_classes/streamlit_.py

@@ -23,6 +23,7 @@ def __init__(
         theme: Literal["Light", "Dark"] = "Light",
         thumbnail: str | None = None,
         data_set_id: int | None = None,
+        cognite_toolkit_app_hash: str = "MISSING",
     ) -> None:
         self.external_id = external_id
         self.name = name
@@ -33,6 +34,7 @@ def __init__(
         self.theme = theme
         self.thumbnail = thumbnail
         self.data_set_id = data_set_id
+        self.cognite_toolkit_app_hash = cognite_toolkit_app_hash
 
     def _as_file_args(self) -> dict[str, Any]:
         metadata = {
@@ -42,6 +44,7 @@ def _as_file_args(self) -> dict[str, Any]:
             "published": self.published,
             "theme": self.theme,
             "entrypoint": self.entrypoint,
+            "cdf-toolkit-app-hash": self.cognite_toolkit_app_hash,
         }
         if self.thumbnail:
             metadata["thumbnail"] = self.thumbnail
@@ -70,6 +73,8 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None =
         for key in ["published", "theme"]:
             if key in resource:
                 args[key] = resource[key]
+        if "cogniteToolkitAppHash" in resource:
+            args["cognite_toolkit_app_hash"] = resource["cogniteToolkitAppHash"]
         return cls(**args)
 
     def as_write(self) -> "StreamlitWrite":
@@ -88,15 +93,25 @@ def __init__(
         entrypoint: str,
         created_time: int,
         last_updated_time: int,
-        app_hash: str = "MISSING",
         description: str | None = None,
         published: bool = False,
         theme: Literal["Light", "Dark"] = "Light",
         thumbnail: str | None = None,
         data_set_id: int | None = None,
+        cognite_toolkit_app_hash: str = "MISSING",
     ) -> None:
-        super().__init__(external_id, name, creator, entrypoint, description, published, theme, thumbnail, data_set_id)
-        self.app_hash = app_hash
+        super().__init__(
+            external_id,
+            name,
+            creator,
+            entrypoint,
+            description,
+            published,
+            theme,
+            thumbnail,
+            data_set_id,
+            cognite_toolkit_app_hash,
+        )
         self.created_time = created_time
         self.last_updated_time = last_updated_time
 
@@ -117,6 +132,8 @@ def _load(cls, resource: dict[str, Any], cognite_client: CogniteClient | None =
         for key in ["theme", "app_hash"]:
             if key in resource:
                 args[key] = resource[key]
+        if "cogniteToolkitAppHash" in resource:
+            args["cognite_toolkit_app_hash"] = resource["cogniteToolkitAppHash"]
         if "published" in resource:
             if isinstance(resource["published"], str):
                 args["published"] = resource["published"].strip().lower() == "true"
@@ -130,7 +147,7 @@ def from_file(cls, file: FileMetadata) -> "Streamlit":
         if "metadata" in dumped:
             dumped.update(dumped.pop("metadata"))
         if "cdf-toolkit-app-hash" in dumped:
-            dumped["app_hash"] = dumped.pop("cdf-toolkit-app-hash")
+            dumped["cogniteToolkitAppHash"] = dumped.pop("cdf-toolkit-app-hash")
         if "entrypoint" not in dumped:
             dumped["entrypoint"] = "MISSING"
         return cls._load(dumped)
@@ -146,6 +163,7 @@ def as_write(self) -> StreamlitWrite:
             theme=self.theme,
             thumbnail=self.thumbnail,
             data_set_id=self.data_set_id,
+            cognite_toolkit_app_hash=self.cognite_toolkit_app_hash,
         )
 
     def as_file(self) -> FileMetadata:

cognite_toolkit/_cdf_tk/commands/_base.py

@@ -2,26 +2,15 @@
 
 import sys
 from collections.abc import Callable
-from pathlib import Path
 from typing import Any
 
-from cognite.client.data_classes._base import T_CogniteResourceList, T_WritableCogniteResource, T_WriteClass
 from rich import print
 
-from cognite_toolkit._cdf_tk.exceptions import ToolkitRequiredValueError, ToolkitTypeError
-from cognite_toolkit._cdf_tk.loaders import (
-    ResourceLoader,
-)
-from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, T_WritableCogniteResourceList
 from cognite_toolkit._cdf_tk.tk_warnings import (
-    LowSeverityWarning,
     ToolkitWarning,
     WarningList,
 )
 from cognite_toolkit._cdf_tk.tracker import Tracker
-from cognite_toolkit._cdf_tk.utils import (
-    CDFToolConfig,
-)
 
 _HAS_PRINTED_COLLECT_MESSAGE = False
 
@@ -75,39 +64,3 @@ def warn(self, warning: ToolkitWarning) -> None:
     def console(self, message: str, prefix: str = "[bold green]INFO:[/] ") -> None:
         if not self.silent:
             print(f"{prefix}{message}")
-
-    def _load_files(
-        self,
-        loader: ResourceLoader[
-            T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
-        ],
-        filepaths: list[Path],
-        ToolGlobals: CDFToolConfig,
-        skip_validation: bool,
-    ) -> T_CogniteResourceList:
-        loaded_resources = loader.list_write_cls([])
-        for filepath in filepaths:
-            try:
-                resource = loader.load_resource_file(filepath, ToolGlobals, skip_validation)
-            except KeyError as e:
-                # KeyError means that we are missing a required field in the yaml file.
-                raise ToolkitRequiredValueError(
-                    f"Failed to load {filepath.name} with {loader.display_name}. Missing required field: {e}."
-                    f"\nPlease compare with the API specification at {loader.doc_url()}."
-                )
-            except TypeError as e:
-                raise ToolkitTypeError(
-                    f"Failed to load {filepath.name} with {loader.display_name}. Wrong type {e!r}"
-                ) from e
-            if resource is None:
-                # This is intentional. It is, for example, used by the AuthLoader to skip groups with resource scopes.
-                continue
-            if isinstance(resource, loader.list_write_cls) and not resource:
-                self.warn(LowSeverityWarning(f"Skipping {filepath.name}. No data to load."))
-                continue
-
-            if isinstance(resource, loader.list_write_cls):
-                loaded_resources.extend(resource)
-            else:
-                loaded_resources.append(resource)
-        return loaded_resources

cognite_toolkit/_cdf_tk/commands/clean.py

@@ -23,6 +23,7 @@
     ResourceContainerDeployResult,
     ResourceDeployResult,
 )
+from cognite_toolkit._cdf_tk.data_classes._module_directories import ReadModule
 from cognite_toolkit._cdf_tk.exceptions import (
     ToolkitCleanResourceError,
     ToolkitNotADirectoryError,
@@ -34,6 +35,7 @@
     RawDatabaseLoader,
     ResourceContainerLoader,
     ResourceLoader,
+    ResourceWorker,
 )
 from cognite_toolkit._cdf_tk.loaders._base_loaders import T_ID, Loader, T_WritableCogniteResourceList
 from cognite_toolkit._cdf_tk.tk_warnings import (
@@ -48,7 +50,7 @@
     read_yaml_file,
 )
 
-from ._utils import _print_ids_or_length, _remove_duplicates
+from ._utils import _print_ids_or_length
 
 
 class CleanCommand(ToolkitCommand):
@@ -58,6 +60,7 @@ def clean_resources(
             T_ID, T_WriteClass, T_WritableCogniteResource, T_CogniteResourceList, T_WritableCogniteResourceList
         ],
         ToolGlobals: CDFToolConfig,
+        read_modules: list[ReadModule],
         dry_run: bool = False,
         drop: bool = True,
         drop_data: bool = False,
@@ -77,25 +80,16 @@ def clean_resources(
             )
             return ResourceContainerDeployResult(name=loader.display_name, item_name=loader.item_name)
 
-        filepaths = loader.find_files()
-
+        worker = ResourceWorker(loader)
+        files = worker.load_files(read_modules=read_modules)
         # Since we do a clean, we do not want to verify that everything exists wrt data sets, spaces etc.
-        loaded_resources = self._load_files(loader, filepaths, ToolGlobals, skip_validation=True)
-
-        # Duplicates are warned in the build step, but the use might continue, so we
-        # need to check for duplicates here as well.
-        loaded_resources, duplicates = _remove_duplicates(loaded_resources, loader)
-
-        capabilities = loader.get_required_capability(loaded_resources, read_only=dry_run)
-
-        if capabilities and (missing := ToolGlobals.toolkit_client.verify.authorization(capabilities)):
-            raise ToolGlobals.toolkit_client.verify.create_error(missing, action=f"clean {loader.display_name}")
-
-        nr_of_items = len(loaded_resources)
-        if nr_of_items == 0:
-            return ResourceDeployResult(name=loader.display_name)
-
-        existing_resources = loader.retrieve(loader.get_ids(loaded_resources))
+        existing_resources, duplicated = worker.load_resources(
+            filepaths=files,
+            return_existing=True,
+            environment_variables=ToolGlobals.environment_variables(),
+            is_dry_run=True,
+            verbose=verbose,
+        )
         nr_of_existing = len(existing_resources)
 
         if drop:
@@ -106,7 +100,7 @@ def clean_resources(
             with_data = ""
         print(f"[bold]{prefix} {nr_of_existing} {loader.display_name} {with_data}from CDF...[/]")
         if not isinstance(loader, RawDatabaseLoader):
-            for duplicate in duplicates:
+            for duplicate in duplicated:
                 self.warn(LowSeverityWarning(f"Duplicate {loader.display_name} {duplicate}."))
 
         # Deleting resources.
@@ -121,15 +115,15 @@ def clean_resources(
             return ResourceContainerDeployResult(
                 name=loader.display_name,
                 deleted=nr_of_deleted,
-                total=nr_of_items,
+                total=nr_of_existing,
                 dropped_datapoints=nr_of_dropped_datapoints,
                 item_name=loader.item_name,
             )
         elif not isinstance(self, ResourceContainerLoader) and drop:
             nr_of_deleted = self._delete_resources(existing_resources, loader, dry_run, verbose)
             if verbose:
                 print("")
-            return ResourceDeployResult(name=loader.display_name, deleted=nr_of_deleted, total=nr_of_items)
+            return ResourceDeployResult(name=loader.display_name, deleted=nr_of_deleted, total=nr_of_existing)
         else:
             return ResourceDeployResult(name=loader.display_name)
 
@@ -275,6 +269,7 @@ def execute(
             result = self.clean_resources(
                 loader,
                 ToolGlobals,
+                read_modules=clean_state.read_modules,
                 drop=True,
                 dry_run=dry_run,
                 drop_data=True,