[CDF-23474] 💥 Purge round 2

CHANGELOG.cdf-tk.md

@@ -17,10 +17,23 @@ Changes are grouped as follows:
 
 ## TBD
 
+### Added
+
+- [alpha feature] `cdf purge dataset` now supports purging resources with internal IDs.
+
 ### Fixed
 
 - Running `cdf build` on an older module will no longer raise an `KeyError` if the `module.toml` does
   not have a `package` key.
+- [alpha feature] `cdf purge dataset` no longer deletes `LocationFilters`
+
+### Changed
+
+- [alpha feature] `cdf purge` now requires a confirmation before deleting resources.
+
+### Improved
+
+- Consistent display names of resources in output table of `cdf deploy` and `cdf clean`.
 
 ## [0.3.18] - 2024-12-03
 

cognite_toolkit/_cdf_tk/apps/_purge.py

@@ -17,7 +17,7 @@ def __init__(self, *args: Any, **kwargs: Any) -> None:
     def main(self, ctx: typer.Context) -> None:
         """Commands purge functionality"""
         if ctx.invoked_subcommand is None:
-            print("Use [bold yellow]cdf pull --help[/] for more information.")
+            print("Use [bold yellow]cdf purge --help[/] for more information.")
 
     def purge_dataset(
         self,
@@ -44,6 +44,14 @@ def purge_dataset(
                 help="Whether to do a dry-run, do dry-run if present.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="Automatically confirm that you are sure you want to purge the dataset.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -61,6 +69,7 @@ def purge_dataset(
                 external_id,
                 include_dataset,
                 dry_run,
+                auto_yes,
                 verbose,
             )
         )
@@ -90,6 +99,14 @@ def purge_space(
                 help="Whether to do a dry-run, do dry-run if present.",
             ),
         ] = False,
+        auto_yes: Annotated[
+            bool,
+            typer.Option(
+                "--yes",
+                "-y",
+                help="Automatically confirm that you are sure you want to purge the space.",
+            ),
+        ] = False,
         verbose: Annotated[
             bool,
             typer.Option(
@@ -109,6 +126,7 @@ def purge_space(
                 space,
                 include_space,
                 dry_run,
+                auto_yes,
                 verbose,
             )
         )

cognite_toolkit/_cdf_tk/commands/_purge.py

@@ -44,15 +44,23 @@ def space(
         space: str | None = None,
         include_space: bool = False,
         dry_run: bool = False,
+        auto_yes: bool = False,
         verbose: bool = False,
     ) -> None:
         """Purge a space and all its content"""
         selected_space = self._get_selected_space(space, ToolGlobals.toolkit_client)
-        self._print_panel("space", selected_space)
         if space is None:
             # Interactive mode
             include_space = questionary.confirm("Do you also want to delete the space itself?", default=False).ask()
             dry_run = questionary.confirm("Dry run?", default=True).ask()
+        if not dry_run:
+            self._print_panel("space", selected_space)
+            if not auto_yes:
+                confirm = questionary.confirm(
+                    f"Are you really sure you want to purge the {selected_space!r} space?", default=False
+                ).ask()
+                if not confirm:
+                    return
 
         loaders = self._get_dependencies(
             SpaceLoader,
@@ -118,17 +126,25 @@ def dataset(
         external_id: str | None = None,
         include_dataset: bool = False,
         dry_run: bool = False,
+        auto_yes: bool = False,
         verbose: bool = False,
     ) -> None:
         """Purge a dataset and all its content"""
         selected_dataset = self._get_selected_dataset(external_id, ToolGlobals.toolkit_client)
-        self._print_panel("dataset", selected_dataset)
         if external_id is None:
             # Interactive mode
             include_dataset = questionary.confirm(
                 "Do you want to archive the dataset itself after the purge?", default=False
             ).ask()
             dry_run = questionary.confirm("Dry run?", default=True).ask()
+        if not dry_run:
+            self._print_panel("dataset", selected_dataset)
+            if not auto_yes:
+                confirm = questionary.confirm(
+                    f"Are you really sure you want to purge the {selected_dataset!r} dataset?", default=False
+                ).ask()
+                if not confirm:
+                    return
 
         loaders = self._get_dependencies(
             DataSetsLoader,
@@ -139,6 +155,7 @@ def dataset(
                 StreamlitLoader,
                 HostedExtractorDestinationLoader,
                 FunctionLoader,
+                LocationFilterLoader,
             },
         )
         is_purged = self._purge(
@@ -245,9 +262,14 @@ def _purge(
                 try:
                     batch_ids.append(loader.get_id(resource))
                 except ToolkitRequiredValueError as e:
-                    self.warn(HighSeverityWarning(f"Cannot delete {resource.dump()!r}. Failed to obtain ID: {e}"))
-                    is_purged = False
-                    continue
+                    try:
+                        batch_ids.append(loader.get_internal_id(resource))
+                    except (AttributeError, NotImplementedError):
+                        self.warn(
+                            HighSeverityWarning(f"Cannot delete {type(resource).__name__}. Failed to obtain ID: {e}")
+                        )
+                        is_purged = False
+                        continue
 
                 if len(batch_ids) >= batch_size:
                     child_deletion = self._delete_children(batch_ids, child_loaders, dry_run, verbose)

cognite_toolkit/_cdf_tk/loaders/_base_loaders.py

@@ -249,6 +249,23 @@ def get_dependent_items(cls, item: dict) -> Iterable[tuple[type[ResourceLoader],
         return
         yield
 
+    @classmethod
+    def get_internal_id(cls, item: T_WritableCogniteResource | dict) -> int:
+        raise NotImplementedError(f"{cls.__name__} does not have an internal id.")
+
+    @classmethod
+    def _split_ids(cls, ids: T_ID | int | SequenceNotStr[T_ID | int] | None) -> tuple[list[int], list[str]]:
+        # Used by subclasses to split the ids into external and internal ids
+        if ids is None:
+            return [], []
+        if isinstance(ids, int):
+            return [ids], []
+        if isinstance(ids, str):
+            return [], [ids]
+        if isinstance(ids, Sequence):
+            return [id for id in ids if isinstance(id, int)], [id for id in ids if isinstance(id, str)]
+        raise ValueError(f"Invalid ids: {ids}")
+
     def load_resource(
         self, filepath: Path, ToolGlobals: CDFToolConfig, skip_validation: bool
     ) -> T_WriteClass | T_CogniteResourceList | None:

cognite_toolkit/_cdf_tk/loaders/_data_loaders.py

@@ -32,7 +32,7 @@ class DatapointsLoader(DataLoader):
 
     @property
     def display_name(self) -> str:
-        return "timeseries.datapoints"
+        return "timeseries datapoints"
 
     def upload(self, state: BuildEnvironment, ToolGlobals: CDFToolConfig, dry_run: bool) -> Iterable[tuple[str, int]]:
         if self.folder_name not in state.built_resources:
@@ -92,7 +92,7 @@ class FileLoader(DataLoader):
 
     @property
     def display_name(self) -> str:
-        return "file contents"
+        return "file content"
 
     def upload(self, state: BuildEnvironment, ToolGlobals: CDFToolConfig, dry_run: bool) -> Iterable[tuple[str, int]]:
         if self.folder_name not in state.built_resources:
@@ -157,7 +157,7 @@ class RawFileLoader(DataLoader):
 
     @property
     def display_name(self) -> str:
-        return "raw.rows"
+        return "raw rows"
 
     def upload(self, state: BuildEnvironment, ToolGlobals: CDFToolConfig, dry_run: bool) -> Iterable[tuple[str, int]]:
         if self.folder_name not in state.built_resources:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/auth_loaders.py

@@ -100,7 +100,7 @@ def __init__(
 
     @property
     def display_name(self) -> str:
-        return f"iam.groups({self.target_scopes.removesuffix('_only')})"
+        return f"groups({self.target_scopes.removesuffix('_only')})"
 
     @classmethod
     def create_loader(
@@ -478,6 +478,10 @@ class GroupAllScopedLoader(GroupLoader):
     def __init__(self, client: ToolkitClient, build_dir: Path | None):
         super().__init__(client, build_dir, "all_scoped_only")
 
+    @property
+    def display_name(self) -> str:
+        return "all-scoped groups"
+
 
 @final
 class SecurityCategoryLoader(
@@ -495,7 +499,7 @@ class SecurityCategoryLoader(
 
     @property
     def display_name(self) -> str:
-        return "security.categories"
+        return "security categories"
 
     @classmethod
     def get_id(cls, item: SecurityCategoryWrite | SecurityCategory | dict) -> str:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/classic_loaders.py

@@ -48,7 +48,7 @@ class AssetLoader(ResourceLoader[str, AssetWrite, Asset, AssetWriteList, AssetLi
 
     @property
     def display_name(self) -> str:
-        return self.kind
+        return "assets"
 
     @classmethod
     def get_id(cls, item: Asset | AssetWrite | dict) -> str:
@@ -58,6 +58,14 @@ def get_id(cls, item: Asset | AssetWrite | dict) -> str:
             raise KeyError("Asset must have external_id")
         return item.external_id
 
+    @classmethod
+    def get_internal_id(cls, item: Asset | dict) -> int:
+        if isinstance(item, dict):
+            return item["id"]
+        if not item.id:
+            raise KeyError("Asset must have id")
+        return item.id
+
     @classmethod
     def dump_id(cls, id: str) -> dict[str, Any]:
         return {"externalId": id}
@@ -94,13 +102,15 @@ def retrieve(self, ids: SequenceNotStr[str]) -> AssetList:
     def update(self, items: AssetWriteList) -> AssetList:
         return self.client.assets.update(items, mode="replace")
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
+    def delete(self, ids: SequenceNotStr[str | int]) -> int:
+        internal_ids, external_ids = self._split_ids(ids)
         try:
-            self.client.assets.delete(external_id=ids)
+            self.client.assets.delete(id=internal_ids, external_id=external_ids)
         except (CogniteAPIError, CogniteNotFoundError) as e:
             non_existing = set(e.failed or [])
             if existing := [id_ for id_ in ids if id_ not in non_existing]:
-                self.client.assets.delete(external_id=existing)
+                internal_ids, external_ids = self._split_ids(existing)
+                self.client.assets.delete(id=internal_ids, external_id=external_ids)
             return len(existing)
         else:
             return len(ids)
@@ -230,7 +240,7 @@ class SequenceLoader(ResourceLoader[str, SequenceWrite, Sequence, SequenceWriteL
 
     @property
     def display_name(self) -> str:
-        return self.kind
+        return "sequences"
 
     @classmethod
     def get_id(cls, item: Sequence | SequenceWrite | dict) -> str:
@@ -240,6 +250,14 @@ def get_id(cls, item: Sequence | SequenceWrite | dict) -> str:
             raise KeyError("Sequence must have external_id")
         return item.external_id
 
+    @classmethod
+    def get_internal_id(cls, item: Sequence | dict) -> int:
+        if isinstance(item, dict):
+            return item["id"]
+        if not item.id:
+            raise KeyError("Sequence must have id")
+        return item.id
+
     @classmethod
     def dump_id(cls, id: str) -> dict[str, Any]:
         return {"externalId": id}
@@ -273,13 +291,15 @@ def retrieve(self, ids: SequenceNotStr[str]) -> SequenceList:
     def update(self, items: SequenceWriteList) -> SequenceList:
         return self.client.sequences.update(items, mode="replace")
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
+    def delete(self, ids: SequenceNotStr[str | int]) -> int:
+        internal_ids, external_ids = self._split_ids(ids)
         try:
-            self.client.sequences.delete(external_id=ids)
+            self.client.sequences.delete(id=internal_ids, external_id=external_ids)
         except (CogniteAPIError, CogniteNotFoundError) as e:
             non_existing = set(e.failed or [])
             if existing := [id_ for id_ in ids if id_ not in non_existing]:
-                self.client.sequences.delete(external_id=existing)
+                internal_ids, external_ids = self._split_ids(existing)
+                self.client.sequences.delete(id=internal_ids, external_id=external_ids)
             return len(existing)
         else:
             return len(ids)
@@ -346,7 +366,7 @@ class EventLoader(ResourceLoader[str, EventWrite, Event, EventWriteList, EventLi
 
     @property
     def display_name(self) -> str:
-        return self.kind
+        return "events"
 
     @classmethod
     def get_id(cls, item: Event | EventWrite | dict) -> str:
@@ -356,6 +376,14 @@ def get_id(cls, item: Event | EventWrite | dict) -> str:
             raise KeyError("Event must have external_id")
         return item.external_id
 
+    @classmethod
+    def get_internal_id(cls, item: Event | dict) -> int:
+        if isinstance(item, dict):
+            return item["id"]
+        if not item.id:
+            raise KeyError("Event must have id")
+        return item.id
+
     @classmethod
     def dump_id(cls, id: str) -> dict[str, Any]:
         return {"externalId": id}
@@ -392,13 +420,15 @@ def retrieve(self, ids: SequenceNotStr[str]) -> EventList:
     def update(self, items: EventWriteList) -> EventList:
         return self.client.events.update(items, mode="replace")
 
-    def delete(self, ids: SequenceNotStr[str]) -> int:
+    def delete(self, ids: SequenceNotStr[str | int]) -> int:
+        internal_ids, external_ids = self._split_ids(ids)
         try:
-            self.client.events.delete(external_id=ids)
+            self.client.events.delete(id=internal_ids, external_id=external_ids)
         except (CogniteAPIError, CogniteNotFoundError) as e:
             non_existing = set(e.failed or [])
             if existing := [id_ for id_ in ids if id_ not in non_existing]:
-                self.client.events.delete(external_id=existing)
+                internal_ids, external_ids = self._split_ids(existing)
+                self.client.events.delete(id=internal_ids, external_id=external_ids)
             return len(existing)
         else:
             return len(ids)

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/data_organization_loaders.py

@@ -63,6 +63,10 @@ class DataSetsLoader(ResourceLoader[str, DataSetWrite, DataSet, DataSetWriteList
     dependencies = frozenset({GroupAllScopedLoader})
     _doc_url = "Data-sets/operation/createDataSets"
 
+    @property
+    def display_name(self) -> str:
+        return "data sets"
+
     @classmethod
     def get_required_capability(cls, items: DataSetWriteList | None, read_only: bool) -> Capability | list[Capability]:
         if not items and items is not None:
@@ -177,7 +181,7 @@ class LabelLoader(
 
     @property
     def display_name(self) -> str:
-        return self.kind
+        return "labels"
 
     @classmethod
     def get_id(cls, item: LabelDefinition | LabelDefinitionWrite | dict) -> str:

cognite_toolkit/_cdf_tk/loaders/_resource_loaders/datamodel_loaders.py

@@ -1060,7 +1060,7 @@ def __init__(self, client: ToolkitClient, build_dir: Path) -> None:
 
     @property
     def display_name(self) -> str:
-        return "GraphQL schemas"
+        return "graph QL schemas"
 
     @classmethod
     def get_id(cls, item: GraphQLDataModelWrite | GraphQLDataModel | dict) -> DataModelId: