From 623aa6d5584cc41aa734f176489451428f646c80 Mon Sep 17 00:00:00 2001 From: UlysseFG Date: Mon, 10 May 2021 10:19:16 -0400 Subject: [PATCH 1/3] first draft field curation script --- .../commands/_field_value_update.py | 84 +++++++++++++++++++ 1 file changed, 84 insertions(+) create mode 100644 backend/fms_core/management/commands/_field_value_update.py diff --git a/backend/fms_core/management/commands/_field_value_update.py b/backend/fms_core/management/commands/_field_value_update.py new file mode 100644 index 000000000..e47015537 --- /dev/null +++ b/backend/fms_core/management/commands/_field_value_update.py @@ -0,0 +1,84 @@ +from django.apps import apps +import reversion +import json +import logging +from reversion.models import Version +from django.contrib.auth.models import User +from fms_core.models import Sample, Container, Individual, Protocol, SampleKind, SampleLineage, Process, ProcessSample + +# Parameters required for this curation +ACTION = "action" # = field_value_update +CURATION_INDEX = "curation_index" # Number indicating the order in which this action was performed during the curation. +ENTITY_MODEL = "entity_model" # The name of the model for the target entity. +ENTITY_DICT_ID = "entity_identifier" # A dictionary that contains the fields required to uniquely identify the targeted entity. +FIELD_NAME = "field_name" # The name of the field that need to be updated. +VALUE_OLD = "value_old" # The old value of the entity's field (used for validation). Optional, validation skipped if empty. +VALUE_NEW = "value_new" # The new value of the entity's field. +USER_ID = "requester_user_id" # The user id of the person requesting the curation. Optional, if left empty use biobankadmin id. + +# Curation params template +# { CURATION_INDEX: 1, +# ACTION: "field_value_update", +# ENTITY_MODEL: "Sample", +# ENTITY_DICT_ID: [{"name": "Sample_test", "id": 42, "container_id": 5823}], # Any subset of fields that identifies uniquely the entity +# FIELD_NAME: "sample_kind", +# VALUE_OLD: 5, # Matches BLOOD +# VALUE_NEW: 9, # Matches PLASMA +# USER_ID: 5 +# } + +# ENTITY_DICT_ID is an array to allow an identical change to be performed on multiple entities. If the changes are different, +# add more field_value_actions to the curation. + +# Other constants +REVERSION_ADMIN_USER = "biobankadmin" + +def field_value_update(params, log): + log.info("Action [" + str(params[CURATION_INDEX]) + "] Field Value Update started.") + log.info("Targeted model : " + str(params[ENTITY_MODEL])) + log.info("Identifier used : " + str(params[ENTITY_DICT_ID])) + log.info("Field to update : " + str(params[FIELD_NAME])) + log.info("Old value : " + str(params.get(VALUE_OLD))) + log.info("New value : " + str(params[VALUE_NEW])) + log.info("Old value : " + str(params.get(USER_ID))) + + # initialize the curation + curation_code = params.get(CURATION_INDEX, "Invalid index") + error_found = False + model = params[ENTITY_MODEL] + id_array = params[ENTITY_DICT_ID] + field = params[FIELD_NAME] + old_value = params.get(VALUE_OLD) + new_value = params[VALUE_NEW] + user_id = params.get(USER_ID, User.objects.get(username=REVERSION_ADMIN_USER)) + + + try: + entity_model = apps.get_model("fms_core", model) + for id in id_array: + try: + entity = entity_model.objects.get(**id) + try: + db_old_value = getattr(entity, field) + if old_value and old_value == db_old_value: + setattr(entity, field, new_value) + setattr(entity, "updated_by_id", user_id) # Record manually the user_id since the request may originate from a user + entity.save() + else: + log.error(f"Content of field [{field}] do not match the old_value expected [{old_value}].") + error_found = True + except AttributeError: + log.error(f"Field [{field}] does not exist for model [{model}].") + error_found = True + except entity_model.DoesNotExist: + log.error(f"No entity found for id [{id}].") + error_found = True + except entity_model.MultipleObjectsReturned: + log.error(f"Multiple entities found for id [{id}]. Provide a unique identifier.") + error_found = True + except LookupError as e: + log.error("Model [" + str(model) + "] does not exist.") + error_found = True + if not error_found: + curation_code = None + return curation_code From c41884c39be83a85b8311a0d659db16994dae348 Mon Sep 17 00:00:00 2001 From: UlysseFG Date: Mon, 10 May 2021 16:37:45 -0400 Subject: [PATCH 2/3] completed modifications to fied_value update script --- .../commands/_field_value_update.py | 20 ++++++++++--------- .../fms_core/management/commands/curation.py | 5 ++++- backend/fms_core/models/tracked_model.py | 8 ++++++-- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/backend/fms_core/management/commands/_field_value_update.py b/backend/fms_core/management/commands/_field_value_update.py index e47015537..fcfc341a9 100644 --- a/backend/fms_core/management/commands/_field_value_update.py +++ b/backend/fms_core/management/commands/_field_value_update.py @@ -3,14 +3,13 @@ import json import logging from reversion.models import Version -from django.contrib.auth.models import User from fms_core.models import Sample, Container, Individual, Protocol, SampleKind, SampleLineage, Process, ProcessSample # Parameters required for this curation ACTION = "action" # = field_value_update CURATION_INDEX = "curation_index" # Number indicating the order in which this action was performed during the curation. ENTITY_MODEL = "entity_model" # The name of the model for the target entity. -ENTITY_DICT_ID = "entity_identifier" # A dictionary that contains the fields required to uniquely identify the targeted entity. +ENTITY_DICT_ID = "entity_identifier" # An array of dictionary that contains the fields required to uniquely identify the targeted entity. FIELD_NAME = "field_name" # The name of the field that need to be updated. VALUE_OLD = "value_old" # The old value of the entity's field (used for validation). Optional, validation skipped if empty. VALUE_NEW = "value_new" # The new value of the entity's field. @@ -30,9 +29,6 @@ # ENTITY_DICT_ID is an array to allow an identical change to be performed on multiple entities. If the changes are different, # add more field_value_actions to the curation. -# Other constants -REVERSION_ADMIN_USER = "biobankadmin" - def field_value_update(params, log): log.info("Action [" + str(params[CURATION_INDEX]) + "] Field Value Update started.") log.info("Targeted model : " + str(params[ENTITY_MODEL])) @@ -40,7 +36,7 @@ def field_value_update(params, log): log.info("Field to update : " + str(params[FIELD_NAME])) log.info("Old value : " + str(params.get(VALUE_OLD))) log.info("New value : " + str(params[VALUE_NEW])) - log.info("Old value : " + str(params.get(USER_ID))) + log.info("Requester id : " + str(params.get(USER_ID))) # initialize the curation curation_code = params.get(CURATION_INDEX, "Invalid index") @@ -50,11 +46,12 @@ def field_value_update(params, log): field = params[FIELD_NAME] old_value = params.get(VALUE_OLD) new_value = params[VALUE_NEW] - user_id = params.get(USER_ID, User.objects.get(username=REVERSION_ADMIN_USER)) + user_id = params.get(USER_ID) try: entity_model = apps.get_model("fms_core", model) + count_updates = 0 for id in id_array: try: entity = entity_model.objects.get(**id) @@ -62,8 +59,12 @@ def field_value_update(params, log): db_old_value = getattr(entity, field) if old_value and old_value == db_old_value: setattr(entity, field, new_value) - setattr(entity, "updated_by_id", user_id) # Record manually the user_id since the request may originate from a user - entity.save() + log.info(f"Updated model [{model}] id [{id} field [{field}] old value [{old_value}] new value [{new_value}].") + if user_id: + entity.save(requester_id=user_id) # Save using th id of the requester + else: + entity.save() # Save using the default admin user + count_updates += 1 else: log.error(f"Content of field [{field}] do not match the old_value expected [{old_value}].") error_found = True @@ -81,4 +82,5 @@ def field_value_update(params, log): error_found = True if not error_found: curation_code = None + log.info(f"Updated [{count_updates}] field values.") return curation_code diff --git a/backend/fms_core/management/commands/curation.py b/backend/fms_core/management/commands/curation.py index 626e250c0..7b5dd8fbb 100644 --- a/backend/fms_core/management/commands/curation.py +++ b/backend/fms_core/management/commands/curation.py @@ -13,10 +13,12 @@ # Import the functions of the various curations available. from ._rollback_extraction import rollback_extraction from ._rollback_curation import rollback_curation +from ._field_value_update import field_value_update # Available actions ACTION_ROLLBACK_CURATION = "rollback_curation" ACTION_ROLLBACK_EXTRACTION = "rollback_extraction" +ACTION_FIELD_UPDATE = "field_value_update" # Curation params template # [CURATION_ACTION_TEMPLATE_1,CURATION_ACTION_TEMPLATE_2,...] @@ -35,7 +37,8 @@ class Command(BaseCommand): curation_switch = { ACTION_ROLLBACK_EXTRACTION: rollback_extraction, - ACTION_ROLLBACK_CURATION: rollback_curation + ACTION_ROLLBACK_CURATION: rollback_curation, + ACTION_FIELD_UPDATE: field_value_update, } def init_logging(self, log_name, timestamp): diff --git a/backend/fms_core/models/tracked_model.py b/backend/fms_core/models/tracked_model.py index 81d4acb0f..5061c93d9 100644 --- a/backend/fms_core/models/tracked_model.py +++ b/backend/fms_core/models/tracked_model.py @@ -19,7 +19,11 @@ class Meta: abstract = True def save(self, *args, **kwargs): - user = get_current_user() + requester = kwargs.get("requester_id") + if requester: + user = User.objects.get(pk=requester) + else: + user = get_current_user() if not user or (user and not user.pk): user = User.objects.get(username=ADMIN_USERNAME) # if the instance has not been saved to the DB yet @@ -29,7 +33,7 @@ def save(self, *args, **kwargs): # Set modified by user each time we save self.updated_by = user - super().save(*args, **kwargs) + super().save() def delete(self, *args, **kwargs): user = get_current_user() From 238242c5de18b39be0f6060cae6d3a8558c86cf9 Mon Sep 17 00:00:00 2001 From: UlysseFG Date: Tue, 11 May 2021 11:54:57 -0400 Subject: [PATCH 3/3] Added comment and generic models. Renamed action. --- ...field_value_update.py => _update_field_value.py} | 13 ++++++++----- backend/fms_core/management/commands/curation.py | 6 +++--- 2 files changed, 11 insertions(+), 8 deletions(-) rename backend/fms_core/management/commands/{_field_value_update.py => _update_field_value.py} (88%) diff --git a/backend/fms_core/management/commands/_field_value_update.py b/backend/fms_core/management/commands/_update_field_value.py similarity index 88% rename from backend/fms_core/management/commands/_field_value_update.py rename to backend/fms_core/management/commands/_update_field_value.py index fcfc341a9..76fcbba98 100644 --- a/backend/fms_core/management/commands/_field_value_update.py +++ b/backend/fms_core/management/commands/_update_field_value.py @@ -3,11 +3,12 @@ import json import logging from reversion.models import Version -from fms_core.models import Sample, Container, Individual, Protocol, SampleKind, SampleLineage, Process, ProcessSample +from fms_core.models import * # Parameters required for this curation -ACTION = "action" # = field_value_update +ACTION = "action" # = update_field_value CURATION_INDEX = "curation_index" # Number indicating the order in which this action was performed during the curation. +COMMENT = "comment" # An optional comment to be stored in the logs ENTITY_MODEL = "entity_model" # The name of the model for the target entity. ENTITY_DICT_ID = "entity_identifier" # An array of dictionary that contains the fields required to uniquely identify the targeted entity. FIELD_NAME = "field_name" # The name of the field that need to be updated. @@ -17,7 +18,8 @@ # Curation params template # { CURATION_INDEX: 1, -# ACTION: "field_value_update", +# ACTION: "update_field_value", +# COMMENT: "Dr. No asled the samples to be changed from BLOOD to PLASMA to correct an error at submission.", # ENTITY_MODEL: "Sample", # ENTITY_DICT_ID: [{"name": "Sample_test", "id": 42, "container_id": 5823}], # Any subset of fields that identifies uniquely the entity # FIELD_NAME: "sample_kind", @@ -29,8 +31,9 @@ # ENTITY_DICT_ID is an array to allow an identical change to be performed on multiple entities. If the changes are different, # add more field_value_actions to the curation. -def field_value_update(params, log): - log.info("Action [" + str(params[CURATION_INDEX]) + "] Field Value Update started.") +def update_field_value(params, log): + log.info("Action [" + str(params[CURATION_INDEX]) + "] Update Field Value started.") + log.info("Comment [" + str(params.get(COMMENT, "None")) + "].") log.info("Targeted model : " + str(params[ENTITY_MODEL])) log.info("Identifier used : " + str(params[ENTITY_DICT_ID])) log.info("Field to update : " + str(params[FIELD_NAME])) diff --git a/backend/fms_core/management/commands/curation.py b/backend/fms_core/management/commands/curation.py index 7b5dd8fbb..f51c071fa 100644 --- a/backend/fms_core/management/commands/curation.py +++ b/backend/fms_core/management/commands/curation.py @@ -13,12 +13,12 @@ # Import the functions of the various curations available. from ._rollback_extraction import rollback_extraction from ._rollback_curation import rollback_curation -from ._field_value_update import field_value_update +from ._update_field_value import update_field_value # Available actions ACTION_ROLLBACK_CURATION = "rollback_curation" ACTION_ROLLBACK_EXTRACTION = "rollback_extraction" -ACTION_FIELD_UPDATE = "field_value_update" +ACTION_UPDATE_FIELD_VALUE = "update_field_value" # Curation params template # [CURATION_ACTION_TEMPLATE_1,CURATION_ACTION_TEMPLATE_2,...] @@ -38,7 +38,7 @@ class Command(BaseCommand): curation_switch = { ACTION_ROLLBACK_EXTRACTION: rollback_extraction, ACTION_ROLLBACK_CURATION: rollback_curation, - ACTION_FIELD_UPDATE: field_value_update, + ACTION_UPDATE_FIELD_VALUE: update_field_value, } def init_logging(self, log_name, timestamp):