From 0aa8e3ee178e63616bf5825def19e9a383029748 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Tue, 20 Jun 2023 16:18:38 -0400 Subject: [PATCH 1/7] add new endpoint to handle multiple URIs --- process_request/routines.py | 23 +++++++++++++++++++++++ process_request/views.py | 14 +++++++++++++- request_broker/urls.py | 4 +++- 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/process_request/routines.py b/process_request/routines.py index d30f985..4f2cb4b 100644 --- a/process_request/routines.py +++ b/process_request/routines.py @@ -118,6 +118,8 @@ def parse_item(self, uri, baseurl): """Parses requested items to determine which are submittable. Adds a `submit` and `submit_reason` attribute to each item. + NOTE: This method is deprecated and will be removed in a future release + Args: uri (str): An AS archival object URI. baseurl (str): base URL for links to objects in DIMES @@ -131,6 +133,27 @@ def parse_item(self, uri, baseurl): submit, reason = self.is_submittable(data[0]) return {"uri": uri, "submit": submit, "submit_reason": reason} + def parse_items(self, uris, baseurl): + """Parses requested items to determine which are submittable. Adds a + `submit` and `submit_reason` attribute to each item. + + Args: + uris (str): A list of AS archival object URIs. + baseurl (str): base URL for links to objects in DIMES + + Returns: + parsed (list): A list of dicts containing parsed item information. + """ + parsed = [] + data = self.get_data(uris, baseurl) + for item in data: + submit, reason = self.is_submittable(data[0]) + parsed.append({"uri": item["uri"], "submit": submit, "submit_reason": reason}) + missing_uris = [m for m in uris if m not in [p["uri"] for p in parsed]] + for uri in missing_uris: + parsed.append({"uri": uri, "submit": False, "submit_reason": "This item is currently unavailable for request. It will not be included in request. Reason: This item cannot be found."}) + return parsed + class Mailer(object): """Email delivery class.""" diff --git a/process_request/views.py b/process_request/views.py index 11ebed5..bfccb6f 100644 --- a/process_request/views.py +++ b/process_request/views.py @@ -27,7 +27,10 @@ def post(self, request, format=None): class ParseRequestView(BaseRequestView): - """Parses an item to determine whether or not it is submittable.""" + """Parses an item to determine whether or not it is submittable. + + NOTE: This view is currently deprecated and will be removed in a future release. + """ def get_response_data(self, request): uri = request.data.get("item") @@ -35,6 +38,15 @@ def get_response_data(self, request): return Processor().parse_item(uri, baseurl) +class ParseMultipleRequestView(BaseRequestView): + """Parses multiple items to determine whether or not they are submittable.""" + + def get_response_data(self, request): + uris = request.data.get("items") + baseurl = request.META.get("HTTP_ORIGIN", settings.DIMES_BASEURL) + return Processor().parse_items(uris, baseurl) + + class MailerView(BaseRequestView): """Delivers email messages containing data.""" diff --git a/request_broker/urls.py b/request_broker/urls.py index df92cfa..9001aca 100644 --- a/request_broker/urls.py +++ b/request_broker/urls.py @@ -19,7 +19,8 @@ from process_request.views import (DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, LinkResolverView, - MailerView, ParseRequestView, PingView) + MailerView, ParseMultipleRequestView, + ParseRequestView, PingView) urlpatterns = [ path("admin/", admin.site.urls), @@ -27,6 +28,7 @@ path("api/deliver-request/duplication", DeliverDuplicationRequestView.as_view(), name="deliver-duplication"), path("api/deliver-request/reading-room", DeliverReadingRoomRequestView.as_view(), name="deliver-readingroom"), path("api/process-request/parse", ParseRequestView.as_view(), name="parse-request"), + path("api/process-request/parse_multiple", ParseMultipleRequestView.as_view(), name="parse-multiple"), path("api/process-request/resolve", LinkResolverView.as_view(), name="resolve-request"), path("api/download-csv/", DownloadCSVView.as_view(), name="download-csv"), path("api/status/", PingView.as_view(), name="ping") From 552abede20a0393104a3c00daeea5587c5d1ef88 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Tue, 20 Jun 2023 16:19:21 -0400 Subject: [PATCH 2/7] update tests --- process_request/tests.py | 50 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/process_request/tests.py b/process_request/tests.py index 281bd88..e0722b7 100644 --- a/process_request/tests.py +++ b/process_request/tests.py @@ -23,7 +23,7 @@ from .test_helpers import json_from_fixture, random_list, random_string from .views import (DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, MailerView, - ParseRequestView) + ParseMultipleRequestView, ParseRequestView) aspace_vcr = vcr.VCR( serializer='json', @@ -284,6 +284,45 @@ def test_parse_item(self, mock_get_data): self.assertEqual(parsed["submit"], False) self.assertEqual(parsed["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: This item cannot be found.") + @patch("process_request.routines.Processor.get_data") + def test_parse_items(self, mock_get_data): + item = json_from_fixture("as_data.json") + mock_get_data.return_value = [item] + + # Ensure objects return correct messages + for restrictions, text, submit, reason in [ + ("closed", "foo", False, "This item is currently unavailable for request. It will not be included in request. Reason: foo"), + ("open", "bar", True, None), + ("conditional", "foobar", True, "This item may be currently unavailable for request. It will be included in request. Reason: foobar")]: + mock_get_data.return_value[0]["restrictions"] = restrictions + mock_get_data.return_value[0]["restrictions_text"] = text + parsed = Processor().parse_items([mock_get_data.return_value[0]["uri"]], "https://dimes.rockarch.org") + self.assertIsInstance(parsed, list) + self.assertEqual(len(parsed), 1) + item = parsed[0] + self.assertIsInstance(item, dict) + self.assertEqual(item["submit"], submit) + self.assertEqual(item["submit_reason"], reason) + + # Ensure objects with attached digital objects return correct message + for format, submit in [ + ("Digital", True), ("digital_object", False), ("Mixed materials", True), ("microfilm", True)]: + mock_get_data.return_value[0]["preferred_instance"]["format"] = format + item = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + self.assertEqual(item["submit"], submit) + + # Ensure objects without instances return correct message + mock_get_data.return_value[0]["preferred_instance"] = {"format": None, "container": None, + "subcontainer": None, "location": None, "barcode": None, "uri": None} + item = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + self.assertEqual(item["submit"], False) + self.assertEqual(item["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: Required information about the physical container of this item is not available.") + + mock_get_data.return_value = [] + parsed = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + self.assertEqual(parsed["submit"], False) + self.assertEqual(parsed["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: This item cannot be found.") + @patch("process_request.routines.Processor.get_data") def test_deliver_email(self, mock_get_data): mock_get_data.return_value = [json_from_fixture("as_data.json")] @@ -395,6 +434,15 @@ def test_parse_request_view(self, mock_parse): self.assert_handles_exceptions( mock_parse, "bar", "parse-request", ParseRequestView) + @patch("process_request.routines.Processor.parse_items") + def test_parse_multiple_view(self, mock_parse): + parsed = [{"foo": "bar"}] + mock_parse.return_value = parsed + self.assert_handles_routine( + {"item": [random_string()]}, "parse-multiple", ParseMultipleRequestView) + self.assert_handles_exceptions( + mock_parse, "bar", "parse-multiple", ParseMultipleRequestView) + @patch("process_request.routines.AeonRequester.get_request_data") def test_deliver_readingroomrequest_view(self, mock_send): delivered = {"foo": "bar"} From d790d99e7ac9356da089c29f69d7addad0f890c5 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Tue, 20 Jun 2023 17:06:42 -0400 Subject: [PATCH 3/7] remove dist --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 172c25c..5d84bf8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -dist: bionic language: python python: "3.10" From d3635901ca4bee7cacba7780b617ca278a5ead8a Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Tue, 20 Jun 2023 17:15:20 -0400 Subject: [PATCH 4/7] update dist --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 5d84bf8..c91f6fb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +dist: focal language: python python: "3.10" From 5b11bd87911551703739af6fb507d27497ce7426 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 21 Jun 2023 14:24:48 -0400 Subject: [PATCH 5/7] update names and docstrings --- process_request/routines.py | 4 +--- process_request/tests.py | 12 ++++++------ process_request/views.py | 7 ++----- request_broker/urls.py | 6 +++--- 4 files changed, 12 insertions(+), 17 deletions(-) diff --git a/process_request/routines.py b/process_request/routines.py index 4f2cb4b..34ba4af 100644 --- a/process_request/routines.py +++ b/process_request/routines.py @@ -115,11 +115,9 @@ def is_submittable(self, item): return submit, reason def parse_item(self, uri, baseurl): - """Parses requested items to determine which are submittable. Adds a + """Parses requested item to determine if it is submittable. Adds a `submit` and `submit_reason` attribute to each item. - NOTE: This method is deprecated and will be removed in a future release - Args: uri (str): An AS archival object URI. baseurl (str): base URL for links to objects in DIMES diff --git a/process_request/tests.py b/process_request/tests.py index e0722b7..f9e07cf 100644 --- a/process_request/tests.py +++ b/process_request/tests.py @@ -23,7 +23,7 @@ from .test_helpers import json_from_fixture, random_list, random_string from .views import (DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, MailerView, - ParseMultipleRequestView, ParseRequestView) + ParseBatchRequestView, ParseRequestView) aspace_vcr = vcr.VCR( serializer='json', @@ -430,18 +430,18 @@ def test_parse_request_view(self, mock_parse): parsed = {"foo": "bar"} mock_parse.return_value = parsed self.assert_handles_routine( - {"item": random_string()}, "parse-request", ParseRequestView) + {"item": random_string()}, "parse-individual", ParseRequestView) self.assert_handles_exceptions( - mock_parse, "bar", "parse-request", ParseRequestView) + mock_parse, "bar", "parse-individual", ParseRequestView) @patch("process_request.routines.Processor.parse_items") - def test_parse_multiple_view(self, mock_parse): + def test_parse_batch_view(self, mock_parse): parsed = [{"foo": "bar"}] mock_parse.return_value = parsed self.assert_handles_routine( - {"item": [random_string()]}, "parse-multiple", ParseMultipleRequestView) + {"item": [random_string()]}, "parse-batch", ParseBatchRequestView) self.assert_handles_exceptions( - mock_parse, "bar", "parse-multiple", ParseMultipleRequestView) + mock_parse, "bar", "parse-batch", ParseBatchRequestView) @patch("process_request.routines.AeonRequester.get_request_data") def test_deliver_readingroomrequest_view(self, mock_send): diff --git a/process_request/views.py b/process_request/views.py index bfccb6f..da8e1f7 100644 --- a/process_request/views.py +++ b/process_request/views.py @@ -27,10 +27,7 @@ def post(self, request, format=None): class ParseRequestView(BaseRequestView): - """Parses an item to determine whether or not it is submittable. - - NOTE: This view is currently deprecated and will be removed in a future release. - """ + """Parses an item to determine whether or not it is submittable.""" def get_response_data(self, request): uri = request.data.get("item") @@ -38,7 +35,7 @@ def get_response_data(self, request): return Processor().parse_item(uri, baseurl) -class ParseMultipleRequestView(BaseRequestView): +class ParseBatchRequestView(BaseRequestView): """Parses multiple items to determine whether or not they are submittable.""" def get_response_data(self, request): diff --git a/request_broker/urls.py b/request_broker/urls.py index 9001aca..7bd8e26 100644 --- a/request_broker/urls.py +++ b/request_broker/urls.py @@ -19,7 +19,7 @@ from process_request.views import (DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, LinkResolverView, - MailerView, ParseMultipleRequestView, + MailerView, ParseBatchRequestView, ParseRequestView, PingView) urlpatterns = [ @@ -27,8 +27,8 @@ path("api/deliver-request/email", MailerView.as_view(), name="deliver-email"), path("api/deliver-request/duplication", DeliverDuplicationRequestView.as_view(), name="deliver-duplication"), path("api/deliver-request/reading-room", DeliverReadingRoomRequestView.as_view(), name="deliver-readingroom"), - path("api/process-request/parse", ParseRequestView.as_view(), name="parse-request"), - path("api/process-request/parse_multiple", ParseMultipleRequestView.as_view(), name="parse-multiple"), + path("api/process-request/parse", ParseRequestView.as_view(), name="parse-individual"), + path("api/process-request/parse_multiple", ParseBatchRequestView.as_view(), name="parse-batch"), path("api/process-request/resolve", LinkResolverView.as_view(), name="resolve-request"), path("api/download-csv/", DownloadCSVView.as_view(), name="download-csv"), path("api/status/", PingView.as_view(), name="ping") From 2205572229de1c71dd4b221dbcd9d7480db8aa23 Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 21 Jun 2023 15:51:31 -0400 Subject: [PATCH 6/7] rename method --- process_request/routines.py | 2 +- process_request/tests.py | 12 ++++++------ process_request/views.py | 2 +- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/process_request/routines.py b/process_request/routines.py index 34ba4af..57a04b8 100644 --- a/process_request/routines.py +++ b/process_request/routines.py @@ -131,7 +131,7 @@ def parse_item(self, uri, baseurl): submit, reason = self.is_submittable(data[0]) return {"uri": uri, "submit": submit, "submit_reason": reason} - def parse_items(self, uris, baseurl): + def parse_batch(self, uris, baseurl): """Parses requested items to determine which are submittable. Adds a `submit` and `submit_reason` attribute to each item. diff --git a/process_request/tests.py b/process_request/tests.py index f9e07cf..f820d39 100644 --- a/process_request/tests.py +++ b/process_request/tests.py @@ -285,7 +285,7 @@ def test_parse_item(self, mock_get_data): self.assertEqual(parsed["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: This item cannot be found.") @patch("process_request.routines.Processor.get_data") - def test_parse_items(self, mock_get_data): + def test_parse_batch(self, mock_get_data): item = json_from_fixture("as_data.json") mock_get_data.return_value = [item] @@ -296,7 +296,7 @@ def test_parse_items(self, mock_get_data): ("conditional", "foobar", True, "This item may be currently unavailable for request. It will be included in request. Reason: foobar")]: mock_get_data.return_value[0]["restrictions"] = restrictions mock_get_data.return_value[0]["restrictions_text"] = text - parsed = Processor().parse_items([mock_get_data.return_value[0]["uri"]], "https://dimes.rockarch.org") + parsed = Processor().parse_batch([mock_get_data.return_value[0]["uri"]], "https://dimes.rockarch.org") self.assertIsInstance(parsed, list) self.assertEqual(len(parsed), 1) item = parsed[0] @@ -308,18 +308,18 @@ def test_parse_items(self, mock_get_data): for format, submit in [ ("Digital", True), ("digital_object", False), ("Mixed materials", True), ("microfilm", True)]: mock_get_data.return_value[0]["preferred_instance"]["format"] = format - item = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + item = Processor().parse_batch([item["uri"]], "https://dimes.rockarch.org")[0] self.assertEqual(item["submit"], submit) # Ensure objects without instances return correct message mock_get_data.return_value[0]["preferred_instance"] = {"format": None, "container": None, "subcontainer": None, "location": None, "barcode": None, "uri": None} - item = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + item = Processor().parse_batch([item["uri"]], "https://dimes.rockarch.org")[0] self.assertEqual(item["submit"], False) self.assertEqual(item["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: Required information about the physical container of this item is not available.") mock_get_data.return_value = [] - parsed = Processor().parse_items([item["uri"]], "https://dimes.rockarch.org")[0] + parsed = Processor().parse_batch([item["uri"]], "https://dimes.rockarch.org")[0] self.assertEqual(parsed["submit"], False) self.assertEqual(parsed["submit_reason"], "This item is currently unavailable for request. It will not be included in request. Reason: This item cannot be found.") @@ -434,7 +434,7 @@ def test_parse_request_view(self, mock_parse): self.assert_handles_exceptions( mock_parse, "bar", "parse-individual", ParseRequestView) - @patch("process_request.routines.Processor.parse_items") + @patch("process_request.routines.Processor.parse_batch") def test_parse_batch_view(self, mock_parse): parsed = [{"foo": "bar"}] mock_parse.return_value = parsed diff --git a/process_request/views.py b/process_request/views.py index da8e1f7..b1ba454 100644 --- a/process_request/views.py +++ b/process_request/views.py @@ -41,7 +41,7 @@ class ParseBatchRequestView(BaseRequestView): def get_response_data(self, request): uris = request.data.get("items") baseurl = request.META.get("HTTP_ORIGIN", settings.DIMES_BASEURL) - return Processor().parse_items(uris, baseurl) + return Processor().parse_batch(uris, baseurl) class MailerView(BaseRequestView): From 89edb252b68dcd63f814be8a06c034c7ee03a8be Mon Sep 17 00:00:00 2001 From: Hillel Arnold Date: Wed, 21 Jun 2023 15:55:02 -0400 Subject: [PATCH 7/7] update view name --- process_request/tests.py | 6 +++--- process_request/views.py | 2 +- request_broker/urls.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/process_request/tests.py b/process_request/tests.py index f820d39..126ec7e 100644 --- a/process_request/tests.py +++ b/process_request/tests.py @@ -23,7 +23,7 @@ from .test_helpers import json_from_fixture, random_list, random_string from .views import (DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, MailerView, - ParseBatchRequestView, ParseRequestView) + ParseBatchRequestView, ParseItemRequestView) aspace_vcr = vcr.VCR( serializer='json', @@ -430,9 +430,9 @@ def test_parse_request_view(self, mock_parse): parsed = {"foo": "bar"} mock_parse.return_value = parsed self.assert_handles_routine( - {"item": random_string()}, "parse-individual", ParseRequestView) + {"item": random_string()}, "parse-individual", ParseItemRequestView) self.assert_handles_exceptions( - mock_parse, "bar", "parse-individual", ParseRequestView) + mock_parse, "bar", "parse-individual", ParseItemRequestView) @patch("process_request.routines.Processor.parse_batch") def test_parse_batch_view(self, mock_parse): diff --git a/process_request/views.py b/process_request/views.py index b1ba454..ab9b6ab 100644 --- a/process_request/views.py +++ b/process_request/views.py @@ -26,7 +26,7 @@ def post(self, request, format=None): return Response({"detail": str(e)}, status=500) -class ParseRequestView(BaseRequestView): +class ParseItemRequestView(BaseRequestView): """Parses an item to determine whether or not it is submittable.""" def get_response_data(self, request): diff --git a/request_broker/urls.py b/request_broker/urls.py index 7bd8e26..7e8b459 100644 --- a/request_broker/urls.py +++ b/request_broker/urls.py @@ -20,14 +20,14 @@ DeliverReadingRoomRequestView, DownloadCSVView, LinkResolverView, MailerView, ParseBatchRequestView, - ParseRequestView, PingView) + ParseItemRequestView, PingView) urlpatterns = [ path("admin/", admin.site.urls), path("api/deliver-request/email", MailerView.as_view(), name="deliver-email"), path("api/deliver-request/duplication", DeliverDuplicationRequestView.as_view(), name="deliver-duplication"), path("api/deliver-request/reading-room", DeliverReadingRoomRequestView.as_view(), name="deliver-readingroom"), - path("api/process-request/parse", ParseRequestView.as_view(), name="parse-individual"), + path("api/process-request/parse", ParseItemRequestView.as_view(), name="parse-individual"), path("api/process-request/parse_multiple", ParseBatchRequestView.as_view(), name="parse-batch"), path("api/process-request/resolve", LinkResolverView.as_view(), name="resolve-request"), path("api/download-csv/", DownloadCSVView.as_view(), name="download-csv"),