Add api endpoints to delete collections and collection versions.

fixes: pulp#879

CHANGES/879.feature

@@ -0,0 +1 @@
+Add api endpoints to delete collections and collection versions.

pulp_ansible/app/galaxy/v3/views.py

@@ -18,11 +18,13 @@
 from jinja2 import Template
 from rest_framework import mixins
 from rest_framework.response import Response
+from rest_framework.request import Request
 from rest_framework.reverse import reverse, reverse_lazy
 from rest_framework import serializers
 from rest_framework import status as http_status
 from rest_framework import viewsets, views
 from rest_framework.exceptions import NotFound
+from rest_framework import status
 
 from pulpcore.plugin.exceptions import DigestValidationError
 from pulpcore.plugin.models import PulpTemporaryFile, Content
@@ -56,6 +58,7 @@
 from pulp_ansible.app.galaxy.v3.pagination import LimitOffsetPagination
 from pulp_ansible.app.viewsets import CollectionVersionFilter
 
+from pulp_ansible.app.tasks.deletion import delete_collection_version, delete_collection
 
 _PERMISSIVE_ACCESS_POLICY = {
     "statements": [
@@ -359,6 +362,73 @@ def update(self, request, *args, **kwargs):
         )
         return OperationPostponedResponse(task, request)
 
+    @extend_schema(
+        description="Trigger an asynchronous delete task",
+        responses={202: AsyncOperationResponseSerializer},
+    )
+    def destroy(self, request: Request, *args, **kwargs) -> Response:
+        """
+        Allow a Collection to be deleted.
+
+        1. Perform Dependency Check to verify that each CollectionVersion
+           inside Collection can be deleted
+        2. If the Collection can’t be deleted, return the reason why
+        3. If it can, dispatch task to delete each CollectionVersion
+           and the Collection
+        """
+        collection = self.get_object()
+
+        # dependency check
+        dependents = get_collection_dependents(collection)
+        if dependents:
+            return Response(
+                {
+                    "detail": _(
+                        "Collection {namespace}.{name} could not be deleted "
+                        "because there are other collections that require it."
+                    ).format(
+                        namespace=collection.namespace,
+                        name=collection.name,
+                    ),
+                    "dependent_collection_versions": [
+                        f"{dep.namespace}.{dep.name} {dep.version}" for dep in dependents
+                    ],
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        repositories = set()
+        for version in collection.versions.all():
+            for repo in version.repositories.all():
+                repositories.add(repo)
+
+        async_result = dispatch(
+            delete_collection,
+            exclusive_resources=list(repositories),
+            kwargs={"collection_pk": collection.pk},
+        )
+
+        return OperationPostponedResponse(async_result, request)
+
+
+def get_collection_dependents(parent):
+    """Given a parent collection, return a list of collection versions that depend on it."""
+    key = f"{parent.namespace}.{parent.name}"
+    return list(
+        CollectionVersion.objects.exclude(collection=parent).filter(dependencies__has_key=key)
+    )
+
+
+def get_dependents(parent):
+    """Given a parent collection version, return a list of collection versions that depend on it."""
+    key = f"{parent.namespace}.{parent.name}"
+    dependents = []
+    for child in CollectionVersion.objects.filter(dependencies__has_key=key):
+        spec = semantic_version.SimpleSpec(child.dependencies[key])
+        if spec.match(semantic_version.Version(parent.version)):
+            dependents.append(child)
+    return dependents
+
 
 class UnpaginatedCollectionViewSet(CollectionViewSet):
     """Unpaginated ViewSet for Collections."""
@@ -542,6 +612,50 @@ def list(self, request, *args, **kwargs):
         serializer = self.get_list_serializer(queryset, many=True, context=context)
         return Response(serializer.data)
 
+    @extend_schema(
+        description="Trigger an asynchronous delete task",
+        responses={202: AsyncOperationResponseSerializer},
+    )
+    def destroy(self, request: Request, *args, **kwargs) -> Response:
+        """
+        Allow a CollectionVersion to be deleted.
+
+        1. Perform Dependency Check to verify that the collection version can be deleted
+        2. If the collection version can’t be deleted, return the reason why
+        3. If it can, dispatch task to delete CollectionVersion and clean up repository.
+           If the version being deleted is the last collection version in the collection,
+           remove the collection object as well.
+        """
+        collection_version = self.get_object()
+
+        # dependency check
+        dependents = get_dependents(collection_version)
+        if dependents:
+            return Response(
+                {
+                    "detail": _(
+                        "Collection version {namespace}.{name} {version} could not be "
+                        "deleted because there are other collections that require it."
+                    ).format(
+                        namespace=collection_version.namespace,
+                        name=collection_version.collection.name,
+                        version=collection_version.version,
+                    ),
+                    "dependent_collection_versions": [
+                        f"{dep.namespace}.{dep.name} {dep.version}" for dep in dependents
+                    ],
+                },
+                status=status.HTTP_400_BAD_REQUEST,
+            )
+
+        async_result = dispatch(
+            delete_collection_version,
+            exclusive_resources=collection_version.repositories.all(),
+            kwargs={"collection_version_pk": collection_version.pk},
+        )
+
+        return OperationPostponedResponse(async_result, request)
+
 
 class UnpaginatedCollectionVersionViewSet(CollectionVersionViewSet):
     """Unpaginated ViewSet for CollectionVersions."""
@@ -709,7 +823,11 @@ def redirect_view_generator(actions, url, viewset, distro_view=True, responses={
     # TODO: Might be able to load serializer info directly from spectacular
     # schema that gets set on the viewset
     def get_responses(action):
-        default = list_serializer if action == "list" else serializer
+        default = serializer
+        if action == "list":
+            default = list_serializer
+        elif action == "destroy":
+            default = AsyncOperationResponseSerializer
         return {302: None, 202: responses.get(action, default)}
 
     # subclasses viewset to make .as_view work correctly on non viewset views
@@ -764,6 +882,14 @@ def get_redirect_url(self, *args, **kwargs):
         def retrieve(self, request, *args, **kwargs):
             return self._get(request, *args, **kwargs)
 
+        @extend_schema(
+            description=description,
+            responses=get_responses("destroy"),
+            deprecated=True,
+        )
+        def destroy(self, request, *args, **kwargs):
+            return self._get(request, *args, **kwargs)
+
         @extend_schema(
             description=description,
             responses=get_responses("list"),

pulp_ansible/app/migrations/0041_alter_collectionversion_collection.py

@@ -0,0 +1,19 @@
+# Generated by Django 3.2.12 on 2022-04-01 13:59
+
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('ansible', '0040_ansiblerepository_keyring'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='collectionversion',
+            name='collection',
+            field=models.ForeignKey(editable=False, on_delete=django.db.models.deletion.PROTECT, related_name='versions', to='ansible.collection'),
+        ),
+    ]

pulp_ansible/app/models.py

@@ -151,7 +151,7 @@ class CollectionVersion(Content):
 
     # Foreign Key Fields
     collection = models.ForeignKey(
-        Collection, on_delete=models.CASCADE, related_name="versions", editable=False
+        Collection, on_delete=models.PROTECT, related_name="versions", editable=False
     )
     tags = models.ManyToManyField(Tag, editable=False)
 

pulp_ansible/app/tasks/deletion.py

@@ -0,0 +1,82 @@
+"""
+This module includes tasks related to deleting Content.
+
+You can remove Content from a Repository by making a new RepositoryVersion
+without the Content. If an API endpoint uses a Distribution which points to
+the latest_version of the Repository then the Content is unavailable,
+however it is not deleted.
+
+Content can only be deleted if it exists in no RepositoryVersion.
+
+Content cannot be removed from a RepositoryVersion since it is immutable.
+
+Pulp's orphan_cleanup task deletes any Content not part of a RepositoryVersion.
+"""
+
+import logging
+
+from pulp_ansible.app.models import Collection, CollectionVersion
+from pulpcore.plugin.tasking import add_and_remove, orphan_cleanup
+
+log = logging.getLogger(__name__)
+
+
+def _cleanup_old_versions(repo):
+    """Delete all the old versions of the given repository."""
+    for version in repo.versions.complete().order_by("-number")[1:]:
+        version.delete()
+
+
+def _remove_collection_version_from_repos(collection_version):
+    """Remove CollectionVersion from latest RepositoryVersion of each repo."""
+    for repo in collection_version.repositories.all():
+        add_and_remove(repo.pk, add_content_units=[], remove_content_units=[collection_version.pk])
+        _cleanup_old_versions(repo)
+
+
+def delete_collection_version(collection_version_pk):
+    """Task to delete CollectionVersion object.
+
+    Sequentially do the following in a single task:
+    1. Call _remove_collection_version_from_repos
+    2. Run orphan_cleanup to delete the CollectionVersion
+    3. Delete Collection if it has no more CollectionVersion
+    """
+    collection_version = CollectionVersion.objects.get(pk=collection_version_pk)
+    collection = collection_version.collection
+
+    _remove_collection_version_from_repos(collection_version)
+
+    log.info("Running orphan_cleanup to delete CollectionVersion object and artifact")
+    # Running orphan_protection_time=0 should be safe since we're specifying the content
+    # to be deleted. This will prevent orphan_cleanup from deleting content that is in
+    # the process of being uploaded.
+    orphan_cleanup(content_pks=[collection_version.pk], orphan_protection_time=0)
+
+    if not collection.versions.exists():
+        log.info("Collection has no more versions, deleting collection {}".format(collection))
+        collection.delete()
+
+
+def delete_collection(collection_pk):
+    """Task to delete Collection object.
+
+    Sequentially do the following in a single task:
+    1. For each CollectionVersion call _remove_collection_version_from_repos
+    2. Run orphan_cleanup to delete the CollectionVersions
+    3. Delete Collection
+    """
+    collection = Collection.objects.get(pk=collection_pk)
+    version_pks = []
+    for version in collection.versions.all():
+        _remove_collection_version_from_repos(version)
+        version_pks.append(version.pk)
+
+    log.info("Running orphan_cleanup to delete CollectionVersion objects and artifacts")
+    # Running orphan_protection_time=0 should be safe since we're specifying the content
+    # to be deleted. This will prevent orphan_cleanup from deleting content that is in
+    # the process of being uploaded.
+    orphan_cleanup(content_pks=version_pks, orphan_protection_time=0)
+
+    log.info("Deleting collection {}".format(collection))
+    collection.delete()

pulp_ansible/app/urls.py

@@ -67,7 +67,7 @@
     path(
         "collections/<str:namespace>/<str:name>/",
         views_v3.redirect_view_generator(
-            {"get": "retrieve", "patch": "update"},
+            {"get": "retrieve", "patch": "update", "delete": "destroy"},
             url="collections-detail",
             viewset=views_v3.CollectionViewSet,
             responses={"update": AsyncOperationResponseSerializer},
@@ -86,7 +86,7 @@
     path(
         "collections/<str:namespace>/<str:name>/versions/<str:version>/",
         views_v3.redirect_view_generator(
-            {"get": "retrieve"},
+            {"get": "retrieve", "delete": "destroy"},
             url="collection-versions-detail",
             viewset=views_v3.CollectionVersionViewSet,
         ),
@@ -157,7 +157,9 @@
     ),
     path(
         "index/<str:namespace>/<str:name>/",
-        views_v3.CollectionViewSet.as_view({"get": "retrieve", "patch": "update"}),
+        views_v3.CollectionViewSet.as_view(
+            {"get": "retrieve", "patch": "update", "delete": "destroy"}
+        ),
         name="collections-detail",
     ),
     path(
@@ -167,7 +169,7 @@
     ),
     path(
         "index/<str:namespace>/<str:name>/versions/<str:version>/",
-        views_v3.CollectionVersionViewSet.as_view({"get": "retrieve"}),
+        views_v3.CollectionVersionViewSet.as_view({"get": "retrieve", "delete": "destroy"}),
         name="collection-versions-detail",
     ),
     path(