Merge pull request #60 from c3g/curation-delete-individual

New curation script to delete individuals.

backend/fms_core/management/commands/_delete_individual.py

@@ -0,0 +1,81 @@
+from django.apps import apps
+import reversion
+import json
+import logging
+from reversion.models import Version
+from fms_core.models import Individual
+
+# Parameters required for this curation
+ACTION = "action"                     # = delete_individual
+CURATION_INDEX = "curation_index"     # Number indicating the order in which this action was performed during the curation.
+COMMENT = "comment"                   # An optional comment to be stored in the logs
+INDIVIDUAL_NAMES = "individual_names" # An array of individual name that are to be deleted.
+USER_ID = "requester_user_id"         # The user id of the person requesting the curation. Optional. If left empty, uses biobankadmin id.
+
+# Curation params template
+# { CURATION_INDEX: 1,
+#   ACTION: "delete_individual",
+#   COMMENT: "Dr. No asked to delete these individuals that are not used.",
+#   INDIVIDUAL_NAMES: ["Roger", "TiPaul", "Gertrude"], # List of individual names (unique) to be deleted
+#   USER_ID: 5
+# }
+
+# This curation deletes only individual that are not referenced. Remove individual from samples references using "update_field_value" first.
+# To delete parents, a first curation need to be run to delete the child individual (or an update), then parents will be unreferenced.
+
+# function that checks the references to an individual and list them.
+def list_references_to(individual):
+    links = []
+    mother_of = list(individual.mother_of.all())
+    if mother_of:
+        links.append(f"Mother of {[child.name for child in mother_of]}.")
+    father_of = list(individual.father_of.all())
+    if father_of:
+        links.append(f"Father of {[child.name for child in father_of]}.")
+    samples = list(individual.samples.all())
+    if samples:
+        links.append(f"Has samples {[sample.id for sample in samples]}")
+    return links
+
+
+def delete_individual(params, objects_to_delete, log):
+    log.info("Action [" + str(params[CURATION_INDEX]) + "] Update Field Value started.")
+    log.info("Comment [" + str(params.get(COMMENT, "None")) + "].")
+    log.info("Individual names : " + str(params[INDIVIDUAL_NAMES]) + ".")
+    log.info("Requester id : " + str(params.get(USER_ID)))
+
+    # initialize the curation
+    curation_code = params.get(CURATION_INDEX, "Invalid index")
+    error_found = False
+    name_array = params[INDIVIDUAL_NAMES]
+    user_id = params.get(USER_ID)
+
+    try:
+        individual_model = apps.get_model("fms_core", "Individual")
+        count_deleted = 0
+        for name in name_array:
+            try:
+                individual = individual_model.objects.get(name=name)
+                links = list_references_to(individual)
+                if links:
+                    log.error(f"Individual [{name}] is still referenced. {links}")
+                    error_found = True
+                else:
+                    log.info(f"Deleted [Individual] name [{individual.name}] id [{individual.id}]].")
+                    individual.deleted = True
+                    individual.save(requester_id=user_id) # save using the id of the requester (using the default admin user if None)
+                    objects_to_delete.append(individual)  # Delay deletion until after the revision block so the object get a version
+                    count_deleted += 1
+            except individual_model.DoesNotExist:
+                log.error(f"No individual found for name [{name}].")
+                error_found = True
+            except individual_model.MultipleObjectsReturned:
+                log.error(f"Multiple individuals found for name [{name}]. Provide a unique identifier.")
+                error_found = True
+    except LookupError:
+        log.error("Model [Individual] does not exist.")
+        error_found = True
+    if not error_found:
+        curation_code = None
+        log.info(f"Deleted [{count_deleted}] individuals.")
+    return curation_code

backend/fms_core/management/commands/_update_field_value.py

@@ -8,18 +8,18 @@
 # Parameters required for this curation
 ACTION = "action"                     # = update_field_value
 CURATION_INDEX = "curation_index"     # Number indicating the order in which this action was performed during the curation.
-COMMENT = "comment"                   # An optional comment to be stored in the logs
+COMMENT = "comment"                   # A comment to be stored in the logs. Optional.
 ENTITY_MODEL = "entity_model"         # The name of the model for the target entity.
 ENTITY_DICT_ID = "entity_identifier"  # An array of dictionary that contains the fields required to uniquely identify the targeted entity.
 FIELD_NAME = "field_name"             # The name of the field that need to be updated.
-VALUE_OLD = "value_old"               # The old value of the entity's field (used for validation). Optional, validation skipped if empty.
+VALUE_OLD = "value_old"               # The old value of the entity's field (used for validation). Optional. Validation skipped if empty.
 VALUE_NEW = "value_new"               # The new value of the entity's field.
-USER_ID = "requester_user_id"         # The user id of the person requesting the curation. Optional, if left empty use biobankadmin id.
+USER_ID = "requester_user_id"         # The user id of the person requesting the curation. Optional. If left empty, uses biobankadmin id.
 
 # Curation params template
 # { CURATION_INDEX: 1,
 #   ACTION: "update_field_value",
-#   COMMENT: "Dr. No asled the samples to be changed from BLOOD to PLASMA to correct an error at submission.",
+#   COMMENT: "Dr. No asked the samples to be changed from BLOOD to PLASMA to correct an error at submission.",
 #   ENTITY_MODEL: "Sample",
 #   ENTITY_DICT_ID: [{"name": "Sample_test", "id": 42, "container_id": 5823}], # Any subset of fields that identifies uniquely the entity
 #   FIELD_NAME: "sample_kind",
@@ -31,7 +31,7 @@
 # ENTITY_DICT_ID is an array to allow an identical change to be performed on multiple entities. If the changes are different,
 # add more field_value_actions to the curation.
 
-def update_field_value(params, log):
+def update_field_value(params, objects_to_delete, log):
     log.info("Action [" + str(params[CURATION_INDEX]) + "] Update Field Value started.")
     log.info("Comment [" + str(params.get(COMMENT, "None")) + "].")
     log.info("Targeted model : " + str(params[ENTITY_MODEL]))

backend/fms_core/management/commands/curation.py

@@ -14,11 +14,14 @@
 from ._rollback_extraction import rollback_extraction
 from ._rollback_curation import rollback_curation
 from ._update_field_value import update_field_value
+from ._delete_individual import delete_individual
+
 
 # Available actions
 ACTION_ROLLBACK_CURATION = "rollback_curation"
 ACTION_ROLLBACK_EXTRACTION = "rollback_extraction"
 ACTION_UPDATE_FIELD_VALUE = "update_field_value"
+ACTION_DELETE_INDIVIDUAL = "delete_individual"
 
 # Curation params template
 # [CURATION_ACTION_TEMPLATE_1,CURATION_ACTION_TEMPLATE_2,...]
@@ -39,6 +42,7 @@ class Command(BaseCommand):
         ACTION_ROLLBACK_EXTRACTION: rollback_extraction,
         ACTION_ROLLBACK_CURATION: rollback_curation,
         ACTION_UPDATE_FIELD_VALUE: update_field_value,
+        ACTION_DELETE_INDIVIDUAL: delete_individual,
     }
 
     def init_logging(self, log_name, timestamp):
@@ -84,14 +88,15 @@ def handle(self, *args, **options):
             error_found = False
             try:
                 with transaction.atomic():
+                    objects_to_delete = []
                     # Create a curation revision to eventually rollback the current curation
                     with reversion.create_revision():
                         # Launch each individual curation
                         for curation in params:
                             self.stdout.write(self.style.SUCCESS('Launching action [' + str(curation["curation_index"]) + '] "%s"' % curation["action"]) + '.')
                             action = self.curation_switch.get(curation["action"])
                             if action:
-                                curation_failed = action(curation, log)
+                                curation_failed = action(curation, objects_to_delete, log)
                                 if curation_failed:
                                     self.stdout.write(self.style.ERROR("Action [" + str(curation_failed) + "] failed."))
                                     error_found = True
@@ -104,6 +109,11 @@ def handle(self, *args, **options):
                     if error_found:
                         raise IntegrityError
                     else:
+                        # Operate the deletions here instead of inside the revision scope so objects get a version even when deleted
+                        with reversion.create_revision(manage_manually=True):  # Shadowing the revision blocks for deletes to prevent them
+                            for object in objects_to_delete:
+                                log.info(f"Completing deletion of object {object.__class__.__name__} id [{object.id}].")
+                                object.delete()
                         self.stdout.write(self.style.SUCCESS("Completed curation."))
             except IntegrityError:
                 log.info("Curation operation transaction rolled back.")

backend/fms_core/migrations/0019_v3_1_2.py

@@ -1,5 +1,6 @@
 from django.conf import settings
-from django.db import migrations
+from django.db import migrations, models
+import django.db.models.deletion
 from django.contrib.auth.models import User
 import reversion
 import datetime
@@ -80,4 +81,9 @@ class Migration(migrations.Migration):
             fix_process_data,
             reverse_code=migrations.RunPython.noop,
         ),
+        migrations.AlterField(
+            model_name='sample',
+            name='individual',
+            field=models.ForeignKey(help_text='Individual associated with the sample.', on_delete=django.db.models.deletion.PROTECT, related_name='samples', to='fms_core.individual'),
+        ),
     ]

backend/fms_core/models/sample.py

@@ -121,7 +121,7 @@ class Sample(TrackedModel):
                             help_text="Sample name.")
     alias = models.CharField(max_length=200, blank=True, help_text="Alternative sample name given by the "
                                                                    "collaborator or customer.")
-    individual = models.ForeignKey("Individual", on_delete=models.PROTECT, help_text="Individual associated "
+    individual = models.ForeignKey("Individual", on_delete=models.PROTECT, related_name="samples", help_text="Individual associated "
                                                                                      "with the sample.")
 
     volume = models.DecimalField(max_digits=20, decimal_places=3, help_text="Current volume of the sample, in µL. ")
@@ -240,11 +240,11 @@ def source_depleted(self) -> bool:
         return self.extracted_from.depleted if self.extracted_from else None
 
     @property
-    def extracted_from(self) -> ["Sample"]:
+    def extracted_from(self) -> "Sample":
         return self.child_of.filter(parent_sample__child=self, parent_sample__process_sample__process__protocol__name="Extraction").first() if self.id else None
 
     @property
-    def transferred_from(self) -> ["Sample"]:
+    def transferred_from(self) -> "Sample":
         return self.child_of.filter(parent_sample__child=self, parent_sample__process_sample__process__protocol__name="Transfer").first() if self.id else None
 
     # Representations

backend/fms_core/models/tracked_model.py

@@ -36,7 +36,11 @@ def save(self, *args, **kwargs):
         super().save()
 
     def delete(self, *args, **kwargs):
-        user = get_current_user()
+        requester = kwargs.get("requester_id")
+        if requester:
+            user = User.objects.get(pk=requester)
+        else:
+            user = get_current_user()
         if user and not user.pk:
             user = User.objects.get(username=ADMIN_USERNAME)
 
@@ -48,7 +52,7 @@ def delete(self, *args, **kwargs):
             reversion.set_user(user)
             reversion.set_comment(f'Deletion of object id ${self.id}')
 
-        super().delete(*args, **kwargs)
+        super().delete()