From 4b7dd31111cdc3e5af21aa3172aa127fa18daf02 Mon Sep 17 00:00:00 2001 From: Argyris Galamatis Date: Fri, 9 Sep 2022 13:26:21 +0100 Subject: [PATCH 01/27] testing the tests and setup --- Makefile | 2 +- dataworkspace/dataworkspace/zendesk.py | 91 ++++++++++++++------------ requirements.txt | 1 + 3 files changed, 50 insertions(+), 44 deletions(-) diff --git a/Makefile b/Makefile index 0dde95418a..8a4a741eb0 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ docker-build: .PHONY: docker-test-unit docker-test-unit: docker-build - docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace + docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace --disable-warnings .PHONY: docker-test-integration diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 06bea9f231..be413eaaad 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -8,6 +8,8 @@ from dataworkspace.notify import generate_token, send_email +# from helpdesk_client.zenddesk_manager import ZendeskManager + logger = logging.getLogger("app") zendesk_service_field_id = settings.ZENDESK_SERVICE_FIELD_ID @@ -73,12 +75,6 @@ def build_private_comment_text(catalogue_item, approval_url): def create_zendesk_ticket(request, access_request, catalogue_item=None): - client = Zenpy( - subdomain=settings.ZENDESK_SUBDOMAIN, - email=settings.ZENDESK_EMAIL, - token=settings.ZENDESK_TOKEN, - ) - access_request_url = request.build_absolute_uri( reverse("admin:request_access_accessrequest_change", args=(access_request.id,)) ) @@ -99,39 +95,48 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): subject = f"Access Request for {catalogue_item if catalogue_item else username}" - ticket_audit = client.tickets.create( - Ticket( - subject=subject, - description=ticket_description, - requester=User(email=access_request.requester.email, name=username), - custom_fields=[ - CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) - ], - ) - ) + # client = Zenpy( + # subdomain=settings.ZENDESK_SUBDOMAIN, + # email=settings.ZENDESK_EMAIL, + # token=settings.ZENDESK_TOKEN, + # ) - ticket_audit.ticket.comment = Comment(body=private_comment, public=False) - client.tickets.update(ticket_audit.ticket) + # ticket_audit = client.tickets.create( + # Ticket( + # subject=subject, + # description=ticket_description, + # requester=User(email=access_request.requester.email, name=username), + # custom_fields=[ + # CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) + # ], + # ) + # ) - return ticket_audit.ticket.id + # ticket_audit.ticket.comment = Comment(body=private_comment, public=False) + # client.tickets.update(ticket_audit.ticket) + + # return ticket_audit.ticket.id + + return 0 def update_zendesk_ticket(ticket_id, comment=None, status=None): - client = Zenpy( - subdomain=settings.ZENDESK_SUBDOMAIN, - email=settings.ZENDESK_EMAIL, - token=settings.ZENDESK_TOKEN, - ) + # client = Zenpy( + # subdomain=settings.ZENDESK_SUBDOMAIN, + # email=settings.ZENDESK_EMAIL, + # token=settings.ZENDESK_TOKEN, + # ) - ticket = client.tickets(id=ticket_id) + # ticket = client.tickets(id=ticket_id) if comment: - ticket.comment = Comment(body=comment, public=False) + # ticket.comment = Comment(body=comment, public=False) + pass if status: ticket.status = status - client.tickets.update(ticket) + # client.tickets.update(ticket) return ticket @@ -200,20 +205,20 @@ def notify_visualisation_access_request(request, access_request, dataset): def create_support_request(user, email, message, tag=None, subject=None): - client = Zenpy( - subdomain=settings.ZENDESK_SUBDOMAIN, - email=settings.ZENDESK_EMAIL, - token=settings.ZENDESK_TOKEN, - ) - ticket_audit = client.tickets.create( - Ticket( - subject=subject or "Data Workspace Support Request", - description=message, - requester=User(email=email, name=user.get_full_name()), - tags=[tag] if tag else None, - custom_fields=[ - CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) - ], - ) - ) + # client = Zenpy( + # subdomain=settings.ZENDESK_SUBDOMAIN, + # email=settings.ZENDESK_EMAIL, + # token=settings.ZENDESK_TOKEN, + # ) + # ticket_audit = client.tickets.create( + # Ticket( + # subject=subject or "Data Workspace Support Request", + # description=message, + # requester=User(email=email, name=user.get_full_name()), + # tags=[tag] if tag else None, + # custom_fields=[ + # CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) + # ], + # ) + # ) return ticket_audit.ticket.id diff --git a/requirements.txt b/requirements.txt index 07f024a093..eae27ff9ae 100644 --- a/requirements.txt +++ b/requirements.txt @@ -321,6 +321,7 @@ zope-event==4.5.0 # via gevent zope-interface==5.1.0 # via gevent +git+https://github.com/uktrade/helpdesk-client.git@main # The following packages are considered to be unsafe in a requirements file: # setuptools From aa42ce1f1b2a79b3cc1860cd45ed675cc1cc1a60 Mon Sep 17 00:00:00 2001 From: Argyris Galamatis Date: Wed, 14 Sep 2022 16:20:26 +0100 Subject: [PATCH 02/27] experiment branch to check with the ticketing test, IGNORE --- .envs/sample.env | 3 + Makefile | 2 +- dataworkspace/dataworkspace/settings/base.py | 6 ++ .../tests/request_access/test_views.py | 32 +++++-- dataworkspace/dataworkspace/zendesk.py | 89 ++++++++++++++++++- requirements.txt | 5 +- 6 files changed, 121 insertions(+), 16 deletions(-) diff --git a/.envs/sample.env b/.envs/sample.env index 10bb6c1c34..9a0c678563 100644 --- a/.envs/sample.env +++ b/.envs/sample.env @@ -89,6 +89,9 @@ ZENDESK_TOKEN=abcd ZENDESK_SERVICE_FIELD_ID=numeric_field_id ZENDESK_SERVICE_FIELD_VALUE=field_value +# helpdesk abstraction +HELP_DESK_INTERFACE=helpdesk_client.interfaces.HelpDeskStubbed + NOTIFY_API_KEY=notify-token FERNET_EMAIL_TOKEN_KEY=generate-using-fernet-generate-key diff --git a/Makefile b/Makefile index 8a4a741eb0..79367779f4 100644 --- a/Makefile +++ b/Makefile @@ -32,7 +32,7 @@ docker-build: .PHONY: docker-test-unit docker-test-unit: docker-build - docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace --disable-warnings + docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace/tests/request_access/test_views.py -s --disable-warnings .PHONY: docker-test-integration diff --git a/dataworkspace/dataworkspace/settings/base.py b/dataworkspace/dataworkspace/settings/base.py index f0aa98f365..92bb321f69 100644 --- a/dataworkspace/dataworkspace/settings/base.py +++ b/dataworkspace/dataworkspace/settings/base.py @@ -290,6 +290,12 @@ def aws_fargate_private_ip(): ZENDESK_SERVICE_FIELD_ID = env["ZENDESK_SERVICE_FIELD_ID"] ZENDESK_SERVICE_FIELD_VALUE = env["ZENDESK_SERVICE_FIELD_VALUE"] +# helpdesk abstraction +import environ +hd_env = environ.Env() +HELP_DESK_INTERFACE = hd_env("HELP_DESK_INTERFACE", default="") +HELP_DESK_CREDS = hd_env.dict("HELP_DESK_CREDS", default={}) + NOTIFY_API_KEY = env["NOTIFY_API_KEY"] FERNET_EMAIL_TOKEN_KEY = env["FERNET_EMAIL_TOKEN_KEY"] diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 700bd68afd..a7d6e86d3f 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -15,6 +15,8 @@ from dataworkspace.apps.core.storage import ClamAVResponse +# from helpdesk_client.interfaces import HelpDeskStubbed +import helpdesk_client class TestDatasetAccessOnly: def test_user_sees_appropriate_message_on_dataset_page(self, client, user, metadata_db): @@ -288,12 +290,14 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - @mock.patch("dataworkspace.apps.request_access.views.zendesk.Zenpy") + # @mock.patch("dataworkspace.apps.request_access.views.zendesk.Zenpy") + @mock.patch.object(helpdesk_client.interfaces.HelpDeskStubbed, "create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_zendesk_ticket_created_after_form_submission( self, mock_upload_to_clamav, - mock_zendesk_client, + # mock_zendesk_client, + mock_helpdesk, mock_boto, client, metadata_db, @@ -302,12 +306,13 @@ def test_zendesk_ticket_created_after_form_submission( class MockTicket: @property def ticket(self): - return type("ticket", (object,), {"id": 1})() + # return type("ticket", (object,), {"id": 1})() + return type("HelpDeskTicket", (object,), {"id": 1})() - mock_zenpy_client = mock.MagicMock() - mock_zenpy_client.tickets.create.return_value = MockTicket() - - mock_zendesk_client.return_value = mock_zenpy_client + # mock_zenpy_client = mock.MagicMock() + # # mock_zenpy_client.tickets.create.return_value = MockTicket() + # mock_zenpy_client.create_ticket.return_value = MockTicket() + # mock_zendesk_client.return_value = mock_zenpy_client mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) @@ -329,8 +334,17 @@ def ticket(self): follow=True, ) - assert len(mock_zenpy_client.tickets.create.call_args_list) == 1 - call_args, _ = mock_zenpy_client.tickets.create.call_args_list[0] + print("TEST_PRINT\n"*10) + print("") + print("==") + print() + print("*"*10) + print("TEST_PRINT\n"*10) + + # assert len(mock_zenpy_client.tickets.create.call_args_list) == 1 + assert len(mock_helpdesk.create_ticket.call_args_list) == 1 + # call_args, _ = mock_zenpy_client.tickets.create.call_args_list[0] + call_args, _ = mock_helpdesk.create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index be413eaaad..67ac4b9709 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -8,7 +8,25 @@ from dataworkspace.notify import generate_token, send_email -# from helpdesk_client.zenddesk_manager import ZendeskManager +print("x"*80) +print("x"*80) +print("x"*80) +print(settings.HELP_DESK_INTERFACE) +print("x"*80) +print("x"*80) +print("x"*80) +print(settings.HELP_DESK_CREDS) +print("x"*80) +print("x"*80) +print("x"*80) + +from helpdesk_client import get_helpdesk_interface +from helpdesk_client.interfaces import ( + HelpDeskTicket, + HelpDeskUser, + HelpDeskCustomField, + HelpDeskComment, +) logger = logging.getLogger("app") @@ -74,6 +92,12 @@ def build_private_comment_text(catalogue_item, approval_url): return private_comment +# configure and instantiate the client +helpdesk_interface = get_helpdesk_interface("helpdesk_client.interfaces.HelpDeskStubbed") +# helpdesk_interface = get_helpdesk_interface(settings.HELP_DESK_INTERFACE) +helpdesk = helpdesk_interface(credentials=settings.HELP_DESK_CREDS) + + def create_zendesk_ticket(request, access_request, catalogue_item=None): access_request_url = request.build_absolute_uri( reverse("admin:request_access_accessrequest_change", args=(access_request.id,)) @@ -117,7 +141,36 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): # return ticket_audit.ticket.id - return 0 + hdt = HelpDeskTicket( + subject=subject, + description=ticket_description, + user=HelpDeskUser( + full_name=username, + email=access_request.requester.email + ), + custom_fields=[ + HelpDeskCustomField( + id=zendesk_service_field_id, + value=zendesk_service_field_value + ) + ], + comment=HelpDeskComment( + body=private_comment, + public=False + ) + ) + + print("X"*120, flush=True) + print(hdt, flush=True) + print("Y"*120, flush=True) + + ticket_audit = helpdesk.create_ticket(hdt) + + print("X"*120, flush=True) + print(ticket_audit.id, flush=True) + print("Y"*120, flush=True) + + return ticket_audit.id def update_zendesk_ticket(ticket_id, comment=None, status=None): @@ -131,10 +184,18 @@ def update_zendesk_ticket(ticket_id, comment=None, status=None): if comment: # ticket.comment = Comment(body=comment, public=False) - pass + helpdesk.helpdesk.add_comment( + ticket_id=ticket_id, + comment=comment( + body=private_comment, + public=False + ) + ) if status: + ticket = helpdesk.get_ticket(ticket_id=ticket_id) ticket.status = status + ticket = helpdesk.update_ticket(ticket=ticket) # client.tickets.update(ticket) @@ -221,4 +282,24 @@ def create_support_request(user, email, message, tag=None, subject=None): # ], # ) # ) - return ticket_audit.ticket.id + # return ticket_audit.ticket.id + + ticket_audit = helpdesk.create_ticket( + HelpDeskTicket( + subject=subject, + description=ticket_description, + user=HelpDeskUser( + full_name=username, + email=access_request.requester.email + ), + custom_fields=[ + HelpDeskCustomField( + id=zendesk_service_field_id, + value=zendesk_service_field_value + ) + ], + tags=[tag] if tag else None + ) + ) + + return ticket_audit.id diff --git a/requirements.txt b/requirements.txt index eae27ff9ae..81eb06e688 100644 --- a/requirements.txt +++ b/requirements.txt @@ -315,13 +315,14 @@ xlsxwriter==1.2.8 # via -r requirements.in yarl==1.5.1 # via aiohttp -zenpy==2.0.21 +zenpy==2.0.25 +# zenpy==2.0.21 # via -r requirements.in zope-event==4.5.0 # via gevent zope-interface==5.1.0 # via gevent -git+https://github.com/uktrade/helpdesk-client.git@main +helpdesk-client==0.1.6 # The following packages are considered to be unsafe in a requirements file: # setuptools From 3fd82cf2f96492572f8b15bf688ad05b06d38fb7 Mon Sep 17 00:00:00 2001 From: Argyris Galamatis Date: Wed, 21 Sep 2022 08:26:35 +0100 Subject: [PATCH 03/27] sorting out tests --- .../tests/request_access/test_views.py | 19 +++-------- requirements-dev.txt | 34 +++++++++++++------ requirements.in | 4 +-- requirements.txt | 21 +++++++++--- 4 files changed, 47 insertions(+), 31 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index a7d6e86d3f..6bfb48559a 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -291,29 +291,18 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") # @mock.patch("dataworkspace.apps.request_access.views.zendesk.Zenpy") - @mock.patch.object(helpdesk_client.interfaces.HelpDeskStubbed, "create_ticket") + @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_zendesk_ticket_created_after_form_submission( self, mock_upload_to_clamav, # mock_zendesk_client, - mock_helpdesk, + mock_helpdesk_create_ticket, mock_boto, client, metadata_db, access_type, ): - class MockTicket: - @property - def ticket(self): - # return type("ticket", (object,), {"id": 1})() - return type("HelpDeskTicket", (object,), {"id": 1})() - - # mock_zenpy_client = mock.MagicMock() - # # mock_zenpy_client.tickets.create.return_value = MockTicket() - # mock_zenpy_client.create_ticket.return_value = MockTicket() - # mock_zendesk_client.return_value = mock_zenpy_client - mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) dataset = DatasetsCommon()._create_master(user_access_type=access_type) @@ -342,9 +331,9 @@ def ticket(self): print("TEST_PRINT\n"*10) # assert len(mock_zenpy_client.tickets.create.call_args_list) == 1 - assert len(mock_helpdesk.create_ticket.call_args_list) == 1 + assert len(mock_helpdesk_create_ticket.call_args_list) == 1 # call_args, _ = mock_zenpy_client.tickets.create.call_args_list[0] - call_args, _ = mock_helpdesk.create_ticket.call_args_list[0] + call_args, _ = mock_helpdesk_create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" diff --git a/requirements-dev.txt b/requirements-dev.txt index 39d87dc897..335d42ce69 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -229,6 +229,8 @@ h11==0.13.0 # via wsproto hawk-server-asyncio==0.0.13 # via -r requirements.txt +helpdesk-client==0.1.6 + # via -r requirements.txt hiredis==1.1.0 # via # -r requirements.txt @@ -247,7 +249,9 @@ ijson==3.1.3 importlib-metadata==4.11.3 # via pyppeteer iniconfig==1.1.1 - # via pytest + # via + # -r requirements.txt + # pytest ipdb==0.13.7 # via -r requirements-dev.in ipython==7.31.1 @@ -355,8 +359,10 @@ platformdirs==2.4.0 # pylint plotly==5.6.0 # via -r requirements.txt -pluggy==0.13.1 - # via pytest +pluggy==1.0.0 + # via + # -r requirements.txt + # pytest prompt-toolkit==3.0.24 # via # -r requirements.txt @@ -375,8 +381,10 @@ psycopg2-binary==2.9.2 # aiopg ptyprocess==0.7.0 # via pexpect -py==1.10.0 - # via pytest +py==1.11.0 + # via + # -r requirements.txt + # pytest pycodestyle==2.7.0 # via flake8 pycparser==2.20 @@ -418,9 +426,11 @@ pyrsistent==0.17.3 # jsonschema pysocks==1.7.1 # via urllib3 -pytest==6.2.4 +pytest==7.1.3 # via # -r requirements-dev.in + # -r requirements.txt + # helpdesk-client # pytest-cov # pytest-django # pytest-mock @@ -549,9 +559,11 @@ toml==0.10.2 # ipdb # pep517 # pylint +tomli==2.0.1 + # via + # -r requirements.txt + # black # pytest -tomli==1.2.2 - # via black tqdm==4.56.0 # via pyppeteer traitlets==5.0.5 @@ -620,8 +632,10 @@ yarl==1.5.1 # via # -r requirements.txt # aiohttp -zenpy==2.0.21 - # via -r requirements.txt +zenpy==2.0.25 + # via + # -r requirements.txt + # helpdesk-client zipp==3.8.0 # via importlib-metadata zope-event==4.5.0 diff --git a/requirements.in b/requirements.in index a18397336f..4090119718 100644 --- a/requirements.in +++ b/requirements.in @@ -49,5 +49,5 @@ sqlalchemy tableschema Werkzeug XlsxWriter -zenpy -django-webpack-loader \ No newline at end of file +django-webpack-loader +helpdesk-client diff --git a/requirements.txt b/requirements.txt index 81eb06e688..c29c6114fa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ attrs==20.2.0 # via # aiohttp # jsonschema + # pytest aws-requests-auth==0.4.3 # via -r requirements.in billiard==3.6.4.0 @@ -143,6 +144,8 @@ greenlet==1.1.0 # via gevent hawk-server-asyncio==0.0.13 # via -r requirements.in +helpdesk-client==0.1.6 + # via -r requirements.in hiredis==1.1.0 # via aioredis idna==2.10 @@ -151,6 +154,8 @@ idna==2.10 # yarl ijson==3.1.3 # via tabulator +iniconfig==1.1.1 + # via pytest isodate==0.6.0 # via tableschema jdcal==1.4.1 @@ -186,13 +191,17 @@ oauthlib==3.1.0 openpyxl==3.0.6 # via tabulator packaging==20.4 - # via bleach + # via + # bleach + # pytest paste==3.4.3 # via -r requirements.in pglast==3.10 # via -r requirements.in plotly==5.6.0 # via -r requirements.in +pluggy==1.0.0 + # via pytest prompt-toolkit==3.0.24 # via click-repl psutil==5.7.2 @@ -203,6 +212,8 @@ psycogreen==1.0.2 # django-db-geventpool psycopg2-binary==2.9.2 # via -r requirements.in +py==1.11.0 + # via pytest pycparser==2.20 # via cffi pyjwt==2.4.0 @@ -213,6 +224,8 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema +pytest==7.1.3 + # via helpdesk-client python-dateutil==2.8.1 # via # botocore @@ -285,6 +298,8 @@ tenacity==6.2.0 # via # celery-redbeat # plotly +tomli==2.0.1 + # via pytest typing-extensions==4.0.0 # via aiohttp unicodecsv==0.14.1 @@ -316,13 +331,11 @@ xlsxwriter==1.2.8 yarl==1.5.1 # via aiohttp zenpy==2.0.25 -# zenpy==2.0.21 - # via -r requirements.in + # via helpdesk-client zope-event==4.5.0 # via gevent zope-interface==5.1.0 # via gevent -helpdesk-client==0.1.6 # The following packages are considered to be unsafe in a requirements file: # setuptools From 7cae664313c79cea9ffe32ce3ae00b80bcb3edfc Mon Sep 17 00:00:00 2001 From: Argyris Galamatis Date: Wed, 21 Sep 2022 09:55:51 +0100 Subject: [PATCH 04/27] ross update --- .../tests/request_access/test_views.py | 59 ++++++++++++++----- dataworkspace/dataworkspace/zendesk.py | 49 +-------------- 2 files changed, 47 insertions(+), 61 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 6bfb48559a..c14b62c943 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -285,19 +285,57 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "request_access:summary-page", kwargs={"pk": access_requests[0].pk} ) + # Hier... + # @pytest.mark.django_db + # @pytest.mark.parametrize( + # "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) + # ) + # @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") + # @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") + # @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") + # def test_helpdesk( + # self, + # mock_upload_to_clamav, + # helpdesk_create_ticket, + # mock_boto, + # client, + # metadata_db, + # access_type, + # ): + # mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) + + # dataset = DatasetsCommon()._create_master(user_access_type=access_type) + # client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + # access_requests = AccessRequest.objects.all() + + # screenshot = SimpleUploadedFile("file.txt", b"file_content") + # client.post( + # reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), + # {"training_screenshot": screenshot}, + # ) + # client.post( + # reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), + # follow=True, + # ) + # client.post( + # reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), + # follow=True, + # ) + + # assert len(helpdesk_create_ticket.call_args_list) == 1 + + @pytest.mark.django_db @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - # @mock.patch("dataworkspace.apps.request_access.views.zendesk.Zenpy") @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - def test_zendesk_ticket_created_after_form_submission( + def test_helpdesk( self, mock_upload_to_clamav, - # mock_zendesk_client, - mock_helpdesk_create_ticket, + helpdesk_create_ticket, mock_boto, client, metadata_db, @@ -323,17 +361,8 @@ def test_zendesk_ticket_created_after_form_submission( follow=True, ) - print("TEST_PRINT\n"*10) - print("") - print("==") - print() - print("*"*10) - print("TEST_PRINT\n"*10) - - # assert len(mock_zenpy_client.tickets.create.call_args_list) == 1 - assert len(mock_helpdesk_create_ticket.call_args_list) == 1 - # call_args, _ = mock_zenpy_client.tickets.create.call_args_list[0] - call_args, _ = mock_helpdesk_create_ticket.call_args_list[0] + assert len(helpdesk_create_ticket.call_args_list) == 1 + call_args, _ = helpdesk_create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 67ac4b9709..edfe906861 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -8,18 +8,6 @@ from dataworkspace.notify import generate_token, send_email -print("x"*80) -print("x"*80) -print("x"*80) -print(settings.HELP_DESK_INTERFACE) -print("x"*80) -print("x"*80) -print("x"*80) -print(settings.HELP_DESK_CREDS) -print("x"*80) -print("x"*80) -print("x"*80) - from helpdesk_client import get_helpdesk_interface from helpdesk_client.interfaces import ( HelpDeskTicket, @@ -94,7 +82,6 @@ def build_private_comment_text(catalogue_item, approval_url): # configure and instantiate the client helpdesk_interface = get_helpdesk_interface("helpdesk_client.interfaces.HelpDeskStubbed") -# helpdesk_interface = get_helpdesk_interface(settings.HELP_DESK_INTERFACE) helpdesk = helpdesk_interface(credentials=settings.HELP_DESK_CREDS) @@ -119,29 +106,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): subject = f"Access Request for {catalogue_item if catalogue_item else username}" - # client = Zenpy( - # subdomain=settings.ZENDESK_SUBDOMAIN, - # email=settings.ZENDESK_EMAIL, - # token=settings.ZENDESK_TOKEN, - # ) - - # ticket_audit = client.tickets.create( - # Ticket( - # subject=subject, - # description=ticket_description, - # requester=User(email=access_request.requester.email, name=username), - # custom_fields=[ - # CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) - # ], - # ) - # ) - - # ticket_audit.ticket.comment = Comment(body=private_comment, public=False) - # client.tickets.update(ticket_audit.ticket) - - # return ticket_audit.ticket.id - - hdt = HelpDeskTicket( + helpdesk_ticket = HelpDeskTicket( subject=subject, description=ticket_description, user=HelpDeskUser( @@ -160,15 +125,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): ) ) - print("X"*120, flush=True) - print(hdt, flush=True) - print("Y"*120, flush=True) - - ticket_audit = helpdesk.create_ticket(hdt) - - print("X"*120, flush=True) - print(ticket_audit.id, flush=True) - print("Y"*120, flush=True) + ticket_audit = helpdesk.create_ticket(helpdesk_ticket) return ticket_audit.id @@ -302,4 +259,4 @@ def create_support_request(user, email, message, tag=None, subject=None): ) ) - return ticket_audit.id + return ticket_audit.id \ No newline at end of file From de3fee2a05fb9715d136c3f853483cc54bee0704 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 21 Sep 2022 12:14:23 +0100 Subject: [PATCH 05/27] WIP --- Makefile | 8 +- dataworkspace/dataworkspace/tests/conftest.py | 2 + .../tests/request_access/test_views.py | 139 +++++++----------- 3 files changed, 59 insertions(+), 90 deletions(-) diff --git a/Makefile b/Makefile index 79367779f4..02c9d2fd4c 100644 --- a/Makefile +++ b/Makefile @@ -31,9 +31,12 @@ docker-build: .PHONY: docker-test-unit -docker-test-unit: docker-build +docker-test-unit: docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace/tests/request_access/test_views.py -s --disable-warnings +down: + docker-compose -f docker-compose-test.yml -p data-workspace-test down + .PHONY: docker-test-integration docker-test-integration: docker-build @@ -98,7 +101,8 @@ docker-test-unit-local: if [ -z "$$TEST_DIR" ]; \ then TEST_DIR="/dataworkspace/dataworkspace"; \ fi; \ - docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR -x -v --disable-warnings --reuse-db + docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR/tests/request_access/test_views.py -k 'TestToolsAccessOnly' -x -v --disable-warnings + #docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR -x -v --disable-warnings --reuse-db .PHONY: docker-test-shell-local docker-test-shell-local: diff --git a/dataworkspace/dataworkspace/tests/conftest.py b/dataworkspace/dataworkspace/tests/conftest.py index ffc55cc773..76b4ad9cf8 100644 --- a/dataworkspace/dataworkspace/tests/conftest.py +++ b/dataworkspace/dataworkspace/tests/conftest.py @@ -142,6 +142,7 @@ def metadata_db(db): ) as conn, conn.cursor() as cursor: cursor.execute( """ + BEGIN; CREATE SCHEMA IF NOT EXISTS dataflow; CREATE TABLE IF NOT EXISTS dataflow.metadata ( id SERIAL, @@ -164,6 +165,7 @@ def metadata_db(db): ('public','table2','2020-09-01 00:01:00.0','2020-09-02 00:01:00.0',NULL,1), ('public','table1','2020-01-01 00:01:00.0','2020-09-02 00:01:00.0',NULL,1), ('public','table4', NULL,'2021-12-01 00:00:00.0',NULL,1); + COMMIT; """ ) conn.commit() diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index c14b62c943..026f666178 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -92,20 +92,22 @@ def test_user_redirected_to_confirmation_page_after_form_submission( ) @pytest.mark.django_db - @mock.patch("dataworkspace.apps.request_access.views.zendesk.Zenpy") + @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - def test_zendesk_ticket_created_after_form_submission( - self, mock_upload_to_clamav, mock_zendesk_client, client, user, metadata_db + def test_helpdesk_ticket_created_after_form_submission( + self, + mock_upload_to_clamav, + helpdesk_create_ticket, + client, + user, + metadata_db, ): class MockTicket: @property - def ticket(self): - return type("ticket", (object,), {"id": 1})() + def id(self): + return 1 - mock_zenpy_client = mock.MagicMock() - mock_zenpy_client.tickets.create.return_value = MockTicket() - - mock_zendesk_client.return_value = mock_zenpy_client + helpdesk_create_ticket.return_value = MockTicket() mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) @@ -138,8 +140,8 @@ def ticket(self): follow=True, ) - assert len(mock_zenpy_client.tickets.create.call_args_list) == 1 - call_args, _ = mock_zenpy_client.tickets.create.call_args_list[0] + assert len(helpdesk_create_ticket.call_args_list) == 1 + call_args, _ = helpdesk_create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" @@ -285,47 +287,7 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "request_access:summary-page", kwargs={"pk": access_requests[0].pk} ) - # Hier... - # @pytest.mark.django_db - # @pytest.mark.parametrize( - # "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) - # ) - # @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - # @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") - # @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - # def test_helpdesk( - # self, - # mock_upload_to_clamav, - # helpdesk_create_ticket, - # mock_boto, - # client, - # metadata_db, - # access_type, - # ): - # mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) - - # dataset = DatasetsCommon()._create_master(user_access_type=access_type) - # client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) - # access_requests = AccessRequest.objects.all() - - # screenshot = SimpleUploadedFile("file.txt", b"file_content") - # client.post( - # reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), - # {"training_screenshot": screenshot}, - # ) - # client.post( - # reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), - # follow=True, - # ) - # client.post( - # reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), - # follow=True, - # ) - - # assert len(helpdesk_create_ticket.call_args_list) == 1 - - - @pytest.mark.django_db + @pytest.mark.django_db(transaction=True) @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @@ -344,45 +306,46 @@ def test_helpdesk( mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) dataset = DatasetsCommon()._create_master(user_access_type=access_type) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() - screenshot = SimpleUploadedFile("file.txt", b"file_content") - client.post( - reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), - {"training_screenshot": screenshot}, - ) - client.post( - reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), - follow=True, - ) - client.post( - reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), - follow=True, - ) - - assert len(helpdesk_create_ticket.call_args_list) == 1 - call_args, _ = helpdesk_create_ticket.call_args_list[0] - ticket = call_args[0] - - assert ticket.subject == "Access Request for A master" - assert ( - ticket.description - == f"""Access request for - -Username: Frank Exampleson -Journey: Tools access -Dataset: A master -SSO Login: frank.exampleson@test.com -People search: https://people.trade.gov.uk/search?search_filters[]=people&query=Frank%20Exampleson - - -Details for the request can be found at - -http://testserver/admin/request_access/accessrequest/{access_requests[0].pk}/change/ - -""" - ) +# screenshot = SimpleUploadedFile("file.txt", b"file_content") +# client.post( +# reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), +# {"training_screenshot": screenshot}, +# ) +# client.post( +# reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), +# follow=True, +# ) +# client.post( +# reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), +# follow=True, +# ) + +# assert len(helpdesk_create_ticket.call_args_list) == 1 +# call_args, _ = helpdesk_create_ticket.call_args_list[0] +# ticket = call_args[0] + +# assert ticket.subject == "Access Request for A master" +# assert ( +# ticket.description +# == f"""Access request for + +# Username: Frank Exampleson +# Journey: Tools access +# Dataset: A master +# SSO Login: frank.exampleson@test.com +# People search: https://people.trade.gov.uk/search?search_filters[]=people&query=Frank%20Exampleson + + +# Details for the request can be found at + +# http://testserver/admin/request_access/accessrequest/{access_requests[0].pk}/change/ + +# """ +# ) class TestDatasetAndToolsAccess: From df4cec071925002917051684634077b7c6c09a75 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 21 Sep 2022 12:27:23 +0100 Subject: [PATCH 06/27] WIP --- dataworkspace/dataworkspace/tests/conftest.py | 2 - .../tests/request_access/test_views.py | 84 ++++++++++--------- 2 files changed, 46 insertions(+), 40 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/conftest.py b/dataworkspace/dataworkspace/tests/conftest.py index 76b4ad9cf8..ffc55cc773 100644 --- a/dataworkspace/dataworkspace/tests/conftest.py +++ b/dataworkspace/dataworkspace/tests/conftest.py @@ -142,7 +142,6 @@ def metadata_db(db): ) as conn, conn.cursor() as cursor: cursor.execute( """ - BEGIN; CREATE SCHEMA IF NOT EXISTS dataflow; CREATE TABLE IF NOT EXISTS dataflow.metadata ( id SERIAL, @@ -165,7 +164,6 @@ def metadata_db(db): ('public','table2','2020-09-01 00:01:00.0','2020-09-02 00:01:00.0',NULL,1), ('public','table1','2020-01-01 00:01:00.0','2020-09-02 00:01:00.0',NULL,1), ('public','table4', NULL,'2021-12-01 00:00:00.0',NULL,1); - COMMIT; """ ) conn.commit() diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 026f666178..f4b14dcfd1 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -287,14 +287,14 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "request_access:summary-page", kwargs={"pk": access_requests[0].pk} ) - @pytest.mark.django_db(transaction=True) + @pytest.mark.django_db() @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - def test_helpdesk( + def test_helpdesk_ticket_created_after_form_submission( self, mock_upload_to_clamav, helpdesk_create_ticket, @@ -303,6 +303,14 @@ def test_helpdesk( metadata_db, access_type, ): + + class MockTicket: + @property + def id(self): + return 1 + + helpdesk_create_ticket.return_value = MockTicket() + mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) dataset = DatasetsCommon()._create_master(user_access_type=access_type) @@ -310,42 +318,42 @@ def test_helpdesk( client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() -# screenshot = SimpleUploadedFile("file.txt", b"file_content") -# client.post( -# reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), -# {"training_screenshot": screenshot}, -# ) -# client.post( -# reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), -# follow=True, -# ) -# client.post( -# reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), -# follow=True, -# ) - -# assert len(helpdesk_create_ticket.call_args_list) == 1 -# call_args, _ = helpdesk_create_ticket.call_args_list[0] -# ticket = call_args[0] - -# assert ticket.subject == "Access Request for A master" -# assert ( -# ticket.description -# == f"""Access request for - -# Username: Frank Exampleson -# Journey: Tools access -# Dataset: A master -# SSO Login: frank.exampleson@test.com -# People search: https://people.trade.gov.uk/search?search_filters[]=people&query=Frank%20Exampleson - - -# Details for the request can be found at - -# http://testserver/admin/request_access/accessrequest/{access_requests[0].pk}/change/ - -# """ -# ) + screenshot = SimpleUploadedFile("file.txt", b"file_content") + client.post( + reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}), + {"training_screenshot": screenshot}, + ) + client.post( + reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}), + follow=True, + ) + client.post( + reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), + follow=True, + ) + + assert len(helpdesk_create_ticket.call_args_list) == 1 + call_args, _ = helpdesk_create_ticket.call_args_list[0] + ticket = call_args[0] + + assert ticket.subject == "Access Request for A master" + assert ( + ticket.description + == f"""Access request for + +Username: Frank Exampleson +Journey: Tools access +Dataset: A master +SSO Login: frank.exampleson@test.com +People search: https://people.trade.gov.uk/search?search_filters[]=people&query=Frank%20Exampleson + + +Details for the request can be found at + +http://testserver/admin/request_access/accessrequest/{access_requests[0].pk}/change/ + +""" + ) class TestDatasetAndToolsAccess: From df75a79f92815ecc567a26935543f3434f9d07bb Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 21 Sep 2022 12:57:56 +0100 Subject: [PATCH 07/27] WIP --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 02c9d2fd4c..6daa047a7c 100644 --- a/Makefile +++ b/Makefile @@ -101,8 +101,7 @@ docker-test-unit-local: if [ -z "$$TEST_DIR" ]; \ then TEST_DIR="/dataworkspace/dataworkspace"; \ fi; \ - docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR/tests/request_access/test_views.py -k 'TestToolsAccessOnly' -x -v --disable-warnings - #docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR -x -v --disable-warnings --reuse-db + docker-compose -f docker-compose-test-local.yml -p data-workspace-test run data-workspace-test pytest $$TEST_DIR -x -v --disable-warnings --reuse-db .PHONY: docker-test-shell-local docker-test-shell-local: From 478f21ea78638f2ed7d6eca83dd7706f61de4711 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 21 Sep 2022 15:47:13 +0100 Subject: [PATCH 08/27] Tidy up --- Makefile | 8 ++--- .../dataworkspace/apps/applications/views.py | 4 +-- dataworkspace/dataworkspace/settings/base.py | 7 ++-- .../tests/request_access/test_views.py | 4 +-- dataworkspace/dataworkspace/zendesk.py | 33 ++----------------- 5 files changed, 10 insertions(+), 46 deletions(-) diff --git a/Makefile b/Makefile index 6daa047a7c..59c4044988 100644 --- a/Makefile +++ b/Makefile @@ -30,12 +30,8 @@ docker-build: docker-compose -f docker-compose-test.yml build -.PHONY: docker-test-unit -docker-test-unit: - docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace/tests/request_access/test_views.py -s --disable-warnings - -down: - docker-compose -f docker-compose-test.yml -p data-workspace-test down +docker-test-unit: docker-build + docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace .PHONY: docker-test-integration diff --git a/dataworkspace/dataworkspace/apps/applications/views.py b/dataworkspace/dataworkspace/apps/applications/views.py index 148763c055..69b70c364d 100644 --- a/dataworkspace/dataworkspace/apps/applications/views.py +++ b/dataworkspace/dataworkspace/apps/applications/views.py @@ -83,7 +83,7 @@ from dataworkspace.apps.eventlog.models import EventLog from dataworkspace.apps.eventlog.utils import log_event from dataworkspace.notify import decrypt_token, send_email -from dataworkspace.zendesk import update_zendesk_ticket +from dataworkspace.zendesk import update_helpdesk_ticket TOOL_LOADING_MESSAGES = [ { @@ -726,7 +726,7 @@ def error(email_address_error): return error(f"{user.get_full_name()} already has access") if email_address == token_data.get("email", "").strip().lower(): - update_zendesk_ticket( + update_helpdesk_ticket( token_data["ticket"], comment=f"Access granted by {request.user.email}", status="solved", diff --git a/dataworkspace/dataworkspace/settings/base.py b/dataworkspace/dataworkspace/settings/base.py index 92bb321f69..be115e8d69 100644 --- a/dataworkspace/dataworkspace/settings/base.py +++ b/dataworkspace/dataworkspace/settings/base.py @@ -291,11 +291,8 @@ def aws_fargate_private_ip(): ZENDESK_SERVICE_FIELD_VALUE = env["ZENDESK_SERVICE_FIELD_VALUE"] # helpdesk abstraction -import environ -hd_env = environ.Env() -HELP_DESK_INTERFACE = hd_env("HELP_DESK_INTERFACE", default="") -HELP_DESK_CREDS = hd_env.dict("HELP_DESK_CREDS", default={}) - +HELP_DESK_INTERFACE = env.get("HELP_DESK_INTERFACE", "") +HELP_DESK_CREDS = env.get("HELP_DESK_CREDS", "") NOTIFY_API_KEY = env["NOTIFY_API_KEY"] FERNET_EMAIL_TOKEN_KEY = env["FERNET_EMAIL_TOKEN_KEY"] diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index f4b14dcfd1..7c8701ad55 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -6,6 +6,8 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse +import helpdesk_client + from dataworkspace.apps.applications.models import ApplicationInstance from dataworkspace.apps.datasets.constants import DataSetType, UserAccessType from dataworkspace.apps.request_access.models import AccessRequest @@ -15,8 +17,6 @@ from dataworkspace.apps.core.storage import ClamAVResponse -# from helpdesk_client.interfaces import HelpDeskStubbed -import helpdesk_client class TestDatasetAccessOnly: def test_user_sees_appropriate_message_on_dataset_page(self, client, user, metadata_db): diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index edfe906861..2c8ab076fd 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -130,17 +130,8 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): return ticket_audit.id -def update_zendesk_ticket(ticket_id, comment=None, status=None): - # client = Zenpy( - # subdomain=settings.ZENDESK_SUBDOMAIN, - # email=settings.ZENDESK_EMAIL, - # token=settings.ZENDESK_TOKEN, - # ) - - # ticket = client.tickets(id=ticket_id) - +def update_helpdesk_ticket(ticket_id, comment=None, status=None): if comment: - # ticket.comment = Comment(body=comment, public=False) helpdesk.helpdesk.add_comment( ticket_id=ticket_id, comment=comment( @@ -154,8 +145,6 @@ def update_zendesk_ticket(ticket_id, comment=None, status=None): ticket.status = status ticket = helpdesk.update_ticket(ticket=ticket) - # client.tickets.update(ticket) - return ticket @@ -223,24 +212,6 @@ def notify_visualisation_access_request(request, access_request, dataset): def create_support_request(user, email, message, tag=None, subject=None): - # client = Zenpy( - # subdomain=settings.ZENDESK_SUBDOMAIN, - # email=settings.ZENDESK_EMAIL, - # token=settings.ZENDESK_TOKEN, - # ) - # ticket_audit = client.tickets.create( - # Ticket( - # subject=subject or "Data Workspace Support Request", - # description=message, - # requester=User(email=email, name=user.get_full_name()), - # tags=[tag] if tag else None, - # custom_fields=[ - # CustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) - # ], - # ) - # ) - # return ticket_audit.ticket.id - ticket_audit = helpdesk.create_ticket( HelpDeskTicket( subject=subject, @@ -259,4 +230,4 @@ def create_support_request(user, email, message, tag=None, subject=None): ) ) - return ticket_audit.id \ No newline at end of file + return ticket_audit.id From 834f57c45aee4f2f3c18dca3a4475670bfa9e8b9 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 21 Sep 2022 15:52:29 +0100 Subject: [PATCH 09/27] Restore line removed in error --- Makefile | 1 + 1 file changed, 1 insertion(+) diff --git a/Makefile b/Makefile index 59c4044988..0dde95418a 100644 --- a/Makefile +++ b/Makefile @@ -30,6 +30,7 @@ docker-build: docker-compose -f docker-compose-test.yml build +.PHONY: docker-test-unit docker-test-unit: docker-build docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest /dataworkspace/dataworkspace From 2b6f73cf5a14997a162e8cac74ae8152d9ba6c71 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 07:36:57 +0100 Subject: [PATCH 10/27] Fix linting, support request logic --- .../tests/request_access/test_views.py | 2 -- dataworkspace/dataworkspace/zendesk.py | 15 +++++---------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 7c8701ad55..f0ae6ba89b 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -6,8 +6,6 @@ from django.core.files.uploadedfile import SimpleUploadedFile from django.urls import reverse -import helpdesk_client - from dataworkspace.apps.applications.models import ApplicationInstance from dataworkspace.apps.datasets.constants import DataSetType, UserAccessType from dataworkspace.apps.request_access.models import AccessRequest diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 2c8ab076fd..23fd3c7595 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -3,8 +3,6 @@ from django.conf import settings from django.urls import reverse -from zenpy import Zenpy -from zenpy.lib.api_objects import Ticket, User, Comment, CustomField from dataworkspace.notify import generate_token, send_email @@ -134,8 +132,8 @@ def update_helpdesk_ticket(ticket_id, comment=None, status=None): if comment: helpdesk.helpdesk.add_comment( ticket_id=ticket_id, - comment=comment( - body=private_comment, + comment=HelpDeskComment( + body=comment, public=False ) ) @@ -214,12 +212,9 @@ def notify_visualisation_access_request(request, access_request, dataset): def create_support_request(user, email, message, tag=None, subject=None): ticket_audit = helpdesk.create_ticket( HelpDeskTicket( - subject=subject, - description=ticket_description, - user=HelpDeskUser( - full_name=username, - email=access_request.requester.email - ), + subject=subject or "Data Workspace Support Request",, + description=message, + user=User(email=email, name=user.get_full_name()),, custom_fields=[ HelpDeskCustomField( id=zendesk_service_field_id, From 9732e09bd7d66af6df0e43cdf7e1dc95cc7992c5 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 07:40:33 +0100 Subject: [PATCH 11/27] Linting --- dataworkspace/dataworkspace/zendesk.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 23fd3c7595..ffa3e57fac 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -212,9 +212,9 @@ def notify_visualisation_access_request(request, access_request, dataset): def create_support_request(user, email, message, tag=None, subject=None): ticket_audit = helpdesk.create_ticket( HelpDeskTicket( - subject=subject or "Data Workspace Support Request",, + subject=subject or "Data Workspace Support Request", description=message, - user=User(email=email, name=user.get_full_name()),, + user=User(email=email, name=user.get_full_name()), custom_fields=[ HelpDeskCustomField( id=zendesk_service_field_id, From ecf9b290b2a24bb4edc73f13fdafbaa9b999ea0a Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 07:42:53 +0100 Subject: [PATCH 12/27] Use HelpDeskUser --- dataworkspace/dataworkspace/zendesk.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index ffa3e57fac..6c6c2a5be4 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -214,7 +214,10 @@ def create_support_request(user, email, message, tag=None, subject=None): HelpDeskTicket( subject=subject or "Data Workspace Support Request", description=message, - user=User(email=email, name=user.get_full_name()), + user=HelpDeskUser( + full_name=user.get_full_name(), + email=email, + ), custom_fields=[ HelpDeskCustomField( id=zendesk_service_field_id, From 7ce53157365507e58b2402e3d3ff7cb9405aaf21 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 07:48:21 +0100 Subject: [PATCH 13/27] Linting --- .../tests/request_access/test_views.py | 106 +++++++++++++----- dataworkspace/dataworkspace/zendesk.py | 36 +++--- 2 files changed, 93 insertions(+), 49 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index f0ae6ba89b..881f3d166d 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -17,7 +17,9 @@ class TestDatasetAccessOnly: - def test_user_sees_appropriate_message_on_dataset_page(self, client, user, metadata_db): + def test_user_sees_appropriate_message_on_dataset_page( + self, client, user, metadata_db + ): dataset = DatasetsCommon()._create_master( user_access_type=UserAccessType.REQUIRES_AUTHORIZATION ) @@ -30,7 +32,9 @@ def test_user_sees_appropriate_message_on_dataset_page(self, client, user, metad resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert "You need to request access to view this data." in resp.content.decode(resp.charset) + assert "You need to request access to view this data." in resp.content.decode( + resp.charset + ) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." not in resp.content.decode(resp.charset) @@ -46,7 +50,9 @@ def test_request_access_form_is_single_page(self, client, user, metadata_db): ) user.user_permissions.add(permission) - resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + resp = client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) assert resp.status_code == 200 assert "Submit" in resp.content.decode(resp.charset) @@ -90,7 +96,9 @@ def test_user_redirected_to_confirmation_page_after_form_submission( ) @pytest.mark.django_db - @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") + @mock.patch( + "dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket" + ) @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, @@ -133,7 +141,9 @@ def id(self): # Submit summary page client.post( - reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), + reverse( + "request_access:summary-page", kwargs={"pk": access_requests[0].pk} + ), {"contact_email": "test@example.com", "reason_for_access": "I need it"}, follow=True, ) @@ -166,13 +176,16 @@ class TestToolsAccessOnly: @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) - def test_user_sees_appropriate_message_on_dataset_page(self, access_type, client, metadata_db): + def test_user_sees_appropriate_message_on_dataset_page( + self, access_type, client, metadata_db + ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert "You need to request access to tools to analyse this data." in resp.content.decode( - resp.charset + assert ( + "You need to request access to tools to analyse this data." + in resp.content.decode(resp.charset) ) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." @@ -182,15 +195,23 @@ def test_user_sees_appropriate_message_on_dataset_page(self, access_type, client @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) - def test_request_access_form_is_multipage_form(self, access_type, client, metadata_db): + def test_request_access_form_is_multipage_form( + self, access_type, client, metadata_db + ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + resp = client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() assert resp.status_code == 302 - assert resp.url == reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) + assert resp.url == reverse( + "request_access:tools-1", kwargs={"pk": access_requests[0].pk} + ) - resp = client.get(reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk})) + resp = client.get( + reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) + ) assert "Continue" in resp.content.decode(resp.charset) @pytest.mark.parametrize( @@ -204,7 +225,9 @@ def test_user_redirected_to_step_2_after_step_1_form_submission( _upload_to_clamav.return_value = ClamAVResponse({"malware": False}) dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() screenshot = SimpleUploadedFile("file.txt", b"file_content") @@ -216,7 +239,9 @@ def test_user_redirected_to_step_2_after_step_1_form_submission( assert len(access_requests) == 1 assert access_requests[0].training_screenshot.name.startswith("file.txt") assert resp.status_code == 302 - assert resp.url == reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}) + assert resp.url == reverse( + "request_access:tools-2", kwargs={"pk": access_requests[0].pk} + ) @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) @@ -225,7 +250,9 @@ def test_user_redirected_to_step_3_after_responding_yes_in_step_2( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() resp = client.post( @@ -236,7 +263,9 @@ def test_user_redirected_to_step_3_after_responding_yes_in_step_2( assert len(access_requests) == 1 assert access_requests[0].spss_and_stata is True assert resp.status_code == 302 - assert resp.url == reverse("request_access:tools-3", kwargs={"pk": access_requests[0].pk}) + assert resp.url == reverse( + "request_access:tools-3", kwargs={"pk": access_requests[0].pk} + ) @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) @@ -245,7 +274,9 @@ def test_user_redirected_to_summary_page_after_responding_no_in_step_2( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() resp = client.post( @@ -266,7 +297,9 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() resp = client.post( @@ -290,7 +323,9 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") + @mock.patch( + "dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket" + ) @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, @@ -301,7 +336,6 @@ def test_helpdesk_ticket_created_after_form_submission( metadata_db, access_type, ): - class MockTicket: @property def id(self): @@ -313,7 +347,9 @@ def id(self): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) access_requests = AccessRequest.objects.all() screenshot = SimpleUploadedFile("file.txt", b"file_content") @@ -326,7 +362,9 @@ def id(self): follow=True, ) client.post( - reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), + reverse( + "request_access:summary-page", kwargs={"pk": access_requests[0].pk} + ), follow=True, ) @@ -362,7 +400,9 @@ def test_user_sees_appropriate_message_on_dataset_page(self, client, metadata_db resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert "You need to request access to view this data." in resp.content.decode(resp.charset) + assert "You need to request access to view this data." in resp.content.decode( + resp.charset + ) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." in resp.content.decode(resp.charset) @@ -372,7 +412,9 @@ def test_request_access_form_is_multipage_form(self, client, metadata_db): dataset = DatasetsCommon()._create_master( user_access_type=UserAccessType.REQUIRES_AUTHORIZATION ) - resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) + resp = client.get( + reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + ) assert "Continue" in resp.content.decode(resp.charset) def test_user_redirected_to_tools_form_after_dataset_request_access_form_submission( @@ -393,7 +435,9 @@ def test_user_redirected_to_tools_form_after_dataset_request_access_form_submiss assert access_requests[0].reason_for_access == "I need it" assert access_requests[0].journey == AccessRequest.JOURNEY_DATASET_ACCESS assert resp.status_code == 302 - assert resp.url == reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) + assert resp.url == reverse( + "request_access:tools-1", kwargs={"pk": access_requests[0].pk} + ) def test_tools_not_required_for_data_cut(self, client, metadata_db): datacut = DataSetFactory.create( @@ -458,7 +502,9 @@ def test_edit_eligibility_criteria(self, client): {"meet_criteria": "yes"}, ) assert resp.status_code == 302 - assert resp.url == reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) + assert resp.url == reverse( + "request_access:dataset", kwargs={"dataset_uuid": dataset.id} + ) assert access_request.id == AccessRequest.objects.latest("created_date").id def test_edit_dataset_request_fields(self, client, user): @@ -489,7 +535,9 @@ def test_edit_dataset_request_fields(self, client, user): @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - def test_edit_training_screenshot(self, mock_upload_to_clamav, mock_boto, client, user): + def test_edit_training_screenshot( + self, mock_upload_to_clamav, mock_boto, client, user + ): mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) screenshot1 = SimpleUploadedFile("original-file.txt", b"file_content") @@ -500,7 +548,9 @@ def test_edit_training_screenshot(self, mock_upload_to_clamav, mock_boto, client ) # Ensure the original file name is displayed in the form - resp = client.get(reverse("request_access:tools-1", kwargs={"pk": access_request.pk})) + resp = client.get( + reverse("request_access:tools-1", kwargs={"pk": access_request.pk}) + ) assert "original-file.txt" in resp.content.decode(resp.charset) # Ensure the file can be updated diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 6c6c2a5be4..e28b68bc1d 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -30,7 +30,9 @@ def get_people_url(name): ) -def build_ticket_description_text(access_request, access_request_url, catalogue_item=None): +def build_ticket_description_text( + access_request, access_request_url, catalogue_item=None +): username = get_username(access_request.requester) people_url = get_people_url(username) ticket_description = f"""Access request for @@ -79,7 +81,9 @@ def build_private_comment_text(catalogue_item, approval_url): # configure and instantiate the client -helpdesk_interface = get_helpdesk_interface("helpdesk_client.interfaces.HelpDeskStubbed") +helpdesk_interface = get_helpdesk_interface( + "helpdesk_client.interfaces.HelpDeskStubbed" +) helpdesk = helpdesk_interface(credentials=settings.HELP_DESK_CREDS) @@ -97,7 +101,9 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): ) private_comment = ( - build_private_comment_text(catalogue_item, authorize_url) if catalogue_item else None + build_private_comment_text(catalogue_item, authorize_url) + if catalogue_item + else None ) username = get_username(access_request.requester) @@ -107,20 +113,13 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): helpdesk_ticket = HelpDeskTicket( subject=subject, description=ticket_description, - user=HelpDeskUser( - full_name=username, - email=access_request.requester.email - ), + user=HelpDeskUser(full_name=username, email=access_request.requester.email), custom_fields=[ HelpDeskCustomField( - id=zendesk_service_field_id, - value=zendesk_service_field_value + id=zendesk_service_field_id, value=zendesk_service_field_value ) ], - comment=HelpDeskComment( - body=private_comment, - public=False - ) + comment=HelpDeskComment(body=private_comment, public=False), ) ticket_audit = helpdesk.create_ticket(helpdesk_ticket) @@ -131,11 +130,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): def update_helpdesk_ticket(ticket_id, comment=None, status=None): if comment: helpdesk.helpdesk.add_comment( - ticket_id=ticket_id, - comment=HelpDeskComment( - body=comment, - public=False - ) + ticket_id=ticket_id, comment=HelpDeskComment(body=comment, public=False) ) if status: @@ -220,11 +215,10 @@ def create_support_request(user, email, message, tag=None, subject=None): ), custom_fields=[ HelpDeskCustomField( - id=zendesk_service_field_id, - value=zendesk_service_field_value + id=zendesk_service_field_id, value=zendesk_service_field_value ) ], - tags=[tag] if tag else None + tags=[tag] if tag else None, ) ) From 63f0441743468b0a6d6d194ac29cefe51b0515f4 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 07:54:29 +0100 Subject: [PATCH 14/27] Black --- .../tests/request_access/test_views.py | 105 +++++------------- dataworkspace/dataworkspace/zendesk.py | 20 +--- 2 files changed, 32 insertions(+), 93 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 881f3d166d..3c36acaeaf 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -17,9 +17,7 @@ class TestDatasetAccessOnly: - def test_user_sees_appropriate_message_on_dataset_page( - self, client, user, metadata_db - ): + def test_user_sees_appropriate_message_on_dataset_page(self, client, user, metadata_db): dataset = DatasetsCommon()._create_master( user_access_type=UserAccessType.REQUIRES_AUTHORIZATION ) @@ -32,9 +30,7 @@ def test_user_sees_appropriate_message_on_dataset_page( resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert "You need to request access to view this data." in resp.content.decode( - resp.charset - ) + assert "You need to request access to view this data." in resp.content.decode(resp.charset) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." not in resp.content.decode(resp.charset) @@ -50,9 +46,7 @@ def test_request_access_form_is_single_page(self, client, user, metadata_db): ) user.user_permissions.add(permission) - resp = client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) assert resp.status_code == 200 assert "Submit" in resp.content.decode(resp.charset) @@ -96,9 +90,7 @@ def test_user_redirected_to_confirmation_page_after_form_submission( ) @pytest.mark.django_db - @mock.patch( - "dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket" - ) + @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, @@ -141,9 +133,7 @@ def id(self): # Submit summary page client.post( - reverse( - "request_access:summary-page", kwargs={"pk": access_requests[0].pk} - ), + reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), {"contact_email": "test@example.com", "reason_for_access": "I need it"}, follow=True, ) @@ -176,16 +166,13 @@ class TestToolsAccessOnly: @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) - def test_user_sees_appropriate_message_on_dataset_page( - self, access_type, client, metadata_db - ): + def test_user_sees_appropriate_message_on_dataset_page(self, access_type, client, metadata_db): dataset = DatasetsCommon()._create_master(user_access_type=access_type) resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert ( - "You need to request access to tools to analyse this data." - in resp.content.decode(resp.charset) + assert "You need to request access to tools to analyse this data." in resp.content.decode( + resp.charset ) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." @@ -195,23 +182,15 @@ def test_user_sees_appropriate_message_on_dataset_page( @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) - def test_request_access_form_is_multipage_form( - self, access_type, client, metadata_db - ): + def test_request_access_form_is_multipage_form(self, access_type, client, metadata_db): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - resp = client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() assert resp.status_code == 302 - assert resp.url == reverse( - "request_access:tools-1", kwargs={"pk": access_requests[0].pk} - ) + assert resp.url == reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) - resp = client.get( - reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) - ) + resp = client.get(reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk})) assert "Continue" in resp.content.decode(resp.charset) @pytest.mark.parametrize( @@ -225,9 +204,7 @@ def test_user_redirected_to_step_2_after_step_1_form_submission( _upload_to_clamav.return_value = ClamAVResponse({"malware": False}) dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() screenshot = SimpleUploadedFile("file.txt", b"file_content") @@ -239,9 +216,7 @@ def test_user_redirected_to_step_2_after_step_1_form_submission( assert len(access_requests) == 1 assert access_requests[0].training_screenshot.name.startswith("file.txt") assert resp.status_code == 302 - assert resp.url == reverse( - "request_access:tools-2", kwargs={"pk": access_requests[0].pk} - ) + assert resp.url == reverse("request_access:tools-2", kwargs={"pk": access_requests[0].pk}) @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) @@ -250,9 +225,7 @@ def test_user_redirected_to_step_3_after_responding_yes_in_step_2( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() resp = client.post( @@ -263,9 +236,7 @@ def test_user_redirected_to_step_3_after_responding_yes_in_step_2( assert len(access_requests) == 1 assert access_requests[0].spss_and_stata is True assert resp.status_code == 302 - assert resp.url == reverse( - "request_access:tools-3", kwargs={"pk": access_requests[0].pk} - ) + assert resp.url == reverse("request_access:tools-3", kwargs={"pk": access_requests[0].pk}) @pytest.mark.parametrize( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) @@ -274,9 +245,7 @@ def test_user_redirected_to_summary_page_after_responding_no_in_step_2( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() resp = client.post( @@ -297,9 +266,7 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( self, access_type, client, metadata_db ): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() resp = client.post( @@ -323,9 +290,7 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - @mock.patch( - "dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket" - ) + @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, @@ -347,9 +312,7 @@ def id(self): dataset = DatasetsCommon()._create_master(user_access_type=access_type) - client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) access_requests = AccessRequest.objects.all() screenshot = SimpleUploadedFile("file.txt", b"file_content") @@ -362,9 +325,7 @@ def id(self): follow=True, ) client.post( - reverse( - "request_access:summary-page", kwargs={"pk": access_requests[0].pk} - ), + reverse("request_access:summary-page", kwargs={"pk": access_requests[0].pk}), follow=True, ) @@ -400,9 +361,7 @@ def test_user_sees_appropriate_message_on_dataset_page(self, client, metadata_db resp = client.get(dataset.get_absolute_url()) assert resp.status_code == 200 - assert "You need to request access to view this data." in resp.content.decode( - resp.charset - ) + assert "You need to request access to view this data." in resp.content.decode(resp.charset) assert ( "We will ask you some questions so we can give you access to the tools you need to analyse this data." in resp.content.decode(resp.charset) @@ -412,9 +371,7 @@ def test_request_access_form_is_multipage_form(self, client, metadata_db): dataset = DatasetsCommon()._create_master( user_access_type=UserAccessType.REQUIRES_AUTHORIZATION ) - resp = client.get( - reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) - ) + resp = client.get(reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id})) assert "Continue" in resp.content.decode(resp.charset) def test_user_redirected_to_tools_form_after_dataset_request_access_form_submission( @@ -435,9 +392,7 @@ def test_user_redirected_to_tools_form_after_dataset_request_access_form_submiss assert access_requests[0].reason_for_access == "I need it" assert access_requests[0].journey == AccessRequest.JOURNEY_DATASET_ACCESS assert resp.status_code == 302 - assert resp.url == reverse( - "request_access:tools-1", kwargs={"pk": access_requests[0].pk} - ) + assert resp.url == reverse("request_access:tools-1", kwargs={"pk": access_requests[0].pk}) def test_tools_not_required_for_data_cut(self, client, metadata_db): datacut = DataSetFactory.create( @@ -502,9 +457,7 @@ def test_edit_eligibility_criteria(self, client): {"meet_criteria": "yes"}, ) assert resp.status_code == 302 - assert resp.url == reverse( - "request_access:dataset", kwargs={"dataset_uuid": dataset.id} - ) + assert resp.url == reverse("request_access:dataset", kwargs={"dataset_uuid": dataset.id}) assert access_request.id == AccessRequest.objects.latest("created_date").id def test_edit_dataset_request_fields(self, client, user): @@ -535,9 +488,7 @@ def test_edit_dataset_request_fields(self, client, user): @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") - def test_edit_training_screenshot( - self, mock_upload_to_clamav, mock_boto, client, user - ): + def test_edit_training_screenshot(self, mock_upload_to_clamav, mock_boto, client, user): mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) screenshot1 = SimpleUploadedFile("original-file.txt", b"file_content") @@ -548,9 +499,7 @@ def test_edit_training_screenshot( ) # Ensure the original file name is displayed in the form - resp = client.get( - reverse("request_access:tools-1", kwargs={"pk": access_request.pk}) - ) + resp = client.get(reverse("request_access:tools-1", kwargs={"pk": access_request.pk})) assert "original-file.txt" in resp.content.decode(resp.charset) # Ensure the file can be updated diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index e28b68bc1d..06da9ab796 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -30,9 +30,7 @@ def get_people_url(name): ) -def build_ticket_description_text( - access_request, access_request_url, catalogue_item=None -): +def build_ticket_description_text(access_request, access_request_url, catalogue_item=None): username = get_username(access_request.requester) people_url = get_people_url(username) ticket_description = f"""Access request for @@ -81,9 +79,7 @@ def build_private_comment_text(catalogue_item, approval_url): # configure and instantiate the client -helpdesk_interface = get_helpdesk_interface( - "helpdesk_client.interfaces.HelpDeskStubbed" -) +helpdesk_interface = get_helpdesk_interface("helpdesk_client.interfaces.HelpDeskStubbed") helpdesk = helpdesk_interface(credentials=settings.HELP_DESK_CREDS) @@ -101,9 +97,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): ) private_comment = ( - build_private_comment_text(catalogue_item, authorize_url) - if catalogue_item - else None + build_private_comment_text(catalogue_item, authorize_url) if catalogue_item else None ) username = get_username(access_request.requester) @@ -115,9 +109,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): description=ticket_description, user=HelpDeskUser(full_name=username, email=access_request.requester.email), custom_fields=[ - HelpDeskCustomField( - id=zendesk_service_field_id, value=zendesk_service_field_value - ) + HelpDeskCustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) ], comment=HelpDeskComment(body=private_comment, public=False), ) @@ -214,9 +206,7 @@ def create_support_request(user, email, message, tag=None, subject=None): email=email, ), custom_fields=[ - HelpDeskCustomField( - id=zendesk_service_field_id, value=zendesk_service_field_value - ) + HelpDeskCustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) ], tags=[tag] if tag else None, ) From f8ac64849666f7095c6411d27212470a70e9997f Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Thu, 22 Sep 2022 08:00:12 +0100 Subject: [PATCH 15/27] pylint --- dataworkspace/dataworkspace/zendesk.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/zendesk.py index 06da9ab796..d06945e85c 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/zendesk.py @@ -4,16 +4,16 @@ from django.conf import settings from django.urls import reverse -from dataworkspace.notify import generate_token, send_email - from helpdesk_client import get_helpdesk_interface from helpdesk_client.interfaces import ( + HelpDeskComment, + HelpDeskCustomField, HelpDeskTicket, HelpDeskUser, - HelpDeskCustomField, - HelpDeskComment, ) +from dataworkspace.notify import generate_token, send_email + logger = logging.getLogger("app") zendesk_service_field_id = settings.ZENDESK_SERVICE_FIELD_ID From 288a80e6da846695c94df88a4e30742a939df55c Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Mon, 26 Sep 2022 17:30:29 +0100 Subject: [PATCH 16/27] Fix tests, update refs to Zendesk to help desk --- .envs/sample.env | 16 +++--- .envs/test.env | 11 ++-- .../dataworkspace/apps/applications/views.py | 2 +- .../dataworkspace/apps/core/views.py | 6 +-- .../apps/request_access/models.py | 4 +- .../apps/request_access/views.py | 14 ++--- .../dataworkspace/apps/request_data/models.py | 2 +- .../dataworkspace/apps/request_data/views.py | 15 ++++-- .../dataworkspace/context_processors.py | 2 +- .../{zendesk.py => help_desk.py} | 14 ++--- dataworkspace/dataworkspace/settings/base.py | 8 +-- .../settings/integration_tests.py | 51 +++++++++++++++++++ .../subscriptions/unsubscribe_confirm.html | 2 +- .../request_access/confirmation-page.html | 2 +- .../request_data/confirmation-page.html | 2 +- .../tests/request_access/test_views.py | 4 +- .../tests/request_data/test_views.py | 6 +-- infra/ecs_main_admin.tf | 8 ++- .../ecs_main_admin_container_definitions.json | 20 +++----- infra/main.tf | 8 ++- test/selenium/conftest.py | 20 ++++---- test/selenium/test_request_data.py | 20 +++++--- .../{zendesk => help_desk}/create-ticket.json | 2 +- 23 files changed, 143 insertions(+), 96 deletions(-) rename dataworkspace/dataworkspace/{zendesk.py => help_desk.py} (91%) rename test/stubs/{zendesk => help_desk}/create-ticket.json (99%) diff --git a/.envs/sample.env b/.envs/sample.env index 9a0c678563..3380015d29 100644 --- a/.envs/sample.env +++ b/.envs/sample.env @@ -78,19 +78,15 @@ S3_POLICY_DOCUMENT_TEMPLATE_BASE64=e30= S3_PERMISSIONS_BOUNDARY_ARN=my-arn S3_ROLE_PREFIX=my-prefix -ZENDESK_EMAIL=test@test.com -# ZENDESK_SUBDOMAIN just requires the subdomain part not the full url -# i.e. for subdomain.zendesk.com the value should be subdomain -ZENDESK_SUBDOMAIN=subdomain -ZENDESK_TOKEN=abcd - -# ZENDESK_SERVICE_FIELD_ID is a numeric value for a custom field within zendesk which is -# set to the ZENDESK_SERVIE_FIELD_VALUE when requesting access to datasets zendesk.py -ZENDESK_SERVICE_FIELD_ID=numeric_field_id -ZENDESK_SERVICE_FIELD_VALUE=field_value +# HELP_DESK_SERVICE_FIELD_ID is a numeric value for a custom field within the help desk which is +# set to the HELPDESK_SERVICE_FIELD_VALUE when requesting access to datasets help_desk.py +HELP_DESK_EMAIL=test@test.com +HELP_DESK_SERVICE_FIELD_ID=numeric_field_id +HELP_DESK_SERVICE_FIELD_VALUE=field_value # helpdesk abstraction HELP_DESK_INTERFACE=helpdesk_client.interfaces.HelpDeskStubbed +HELP_DESK_CREDS=xxx NOTIFY_API_KEY=notify-token FERNET_EMAIL_TOKEN_KEY=generate-using-fernet-generate-key diff --git a/.envs/test.env b/.envs/test.env index f21ff488d1..759cdb4be8 100644 --- a/.envs/test.env +++ b/.envs/test.env @@ -54,11 +54,9 @@ AWS_ECR_ENDPOINT_URL=http://api.ecr.my-region-1.amazonaws.com:8008/ PROMETHEUS_DOMAIN=some.domain.com METRICS_SERVICE_DISCOVERY_BASIC_AUTH_USER=user METRICS_SERVICE_DISCOVERY_BASIC_AUTH_PASSWORD=password -ZENDESK_EMAIL=test@test.com -ZENDESK_SUBDOMAIN=subdomain -ZENDESK_TOKEN=abcd -ZENDESK_SERVICE_FIELD_ID=654321 -ZENDESK_SERVICE_FIELD_VALUE=field_value +HELP_DESK_EMAIL=test@test.com +HELP_DESK_SERVICE_FIELD_ID=654321 +HELP_DESK_SERVICE_FIELD_VALUE=field_value S3_ASSUME_ROLE_POLICY_DOCUMENT_BASE64=e30= S3_POLICY_NAME=my-policy S3_POLICY_DOCUMENT_TEMPLATE_BASE64=e30= @@ -67,6 +65,9 @@ S3_ROLE_PREFIX=my-prefix ALLOWED_HOSTS='dataworkspace.test' +HELP_DESK_INTERFACE=helpdesk_client.interfaces.HelpDeskStubbed + + UPLOADS_BUCKET=an-upload-bucket MIRROR_REMOTE_ROOT=http://127.0.0.1:8006/some-remote-folder/ diff --git a/dataworkspace/dataworkspace/apps/applications/views.py b/dataworkspace/dataworkspace/apps/applications/views.py index 69b70c364d..d83b246f0e 100644 --- a/dataworkspace/dataworkspace/apps/applications/views.py +++ b/dataworkspace/dataworkspace/apps/applications/views.py @@ -83,7 +83,7 @@ from dataworkspace.apps.eventlog.models import EventLog from dataworkspace.apps.eventlog.utils import log_event from dataworkspace.notify import decrypt_token, send_email -from dataworkspace.zendesk import update_helpdesk_ticket +from dataworkspace.help_desk import update_helpdesk_ticket TOOL_LOADING_MESSAGES = [ { diff --git a/dataworkspace/dataworkspace/apps/core/views.py b/dataworkspace/dataworkspace/apps/core/views.py index 7567604cc3..47c68a5e2b 100644 --- a/dataworkspace/dataworkspace/apps/core/views.py +++ b/dataworkspace/dataworkspace/apps/core/views.py @@ -43,7 +43,7 @@ ) from dataworkspace.apps.eventlog.models import EventLog from dataworkspace.apps.eventlog.utils import log_event -from dataworkspace.zendesk import create_support_request +from dataworkspace.help_desk import create_support_request logger = logging.getLogger("app") @@ -101,7 +101,7 @@ class SupportView(FormView): form_class = SupportForm template_name = "core/support.html" - ZENDESK_TAGS = {"data-request": "data_request"} + HELP_DESK_TAGS = {"data-request": "data_request"} def get_context_data(self, **kwargs): ctx = super().get_context_data() @@ -122,7 +122,7 @@ def form_valid(self, form): if cleaned["support_type"] == form.SupportTypes.TECH_SUPPORT: return HttpResponseRedirect(f'{reverse("technical-support")}?email={cleaned["email"]}') - tag = self.ZENDESK_TAGS.get(self.request.GET.get("tag")) + tag = self.HELP_DESK_TAGS.get(self.request.GET.get("tag")) ticket_id = create_support_request( self.request.user, cleaned["email"], cleaned["message"], tag=tag ) diff --git a/dataworkspace/dataworkspace/apps/request_access/models.py b/dataworkspace/dataworkspace/apps/request_access/models.py index 318b94aa43..d0b263f8b9 100644 --- a/dataworkspace/dataworkspace/apps/request_access/models.py +++ b/dataworkspace/dataworkspace/apps/request_access/models.py @@ -31,10 +31,10 @@ class AccessRequest(TimeStampedModel): spss_and_stata = models.BooleanField(default=False, blank=True) line_manager_email_address = models.CharField(max_length=256, null=True) reason_for_spss_and_stata = models.TextField(null=True) - zendesk_reference_number = models.CharField(max_length=256, null=True) + help_desk_reference_number = models.CharField(max_length=256, null=True) def __str__(self): - return f"{self.requester} - Zendesk reference number: {self.zendesk_reference_number}" + return f"{self.requester} - Help desk reference number: {self.help_desk_reference_number}" @property def journey(self): diff --git a/dataworkspace/dataworkspace/apps/request_access/views.py b/dataworkspace/dataworkspace/apps/request_access/views.py index 2538052d8e..6712ea7b43 100644 --- a/dataworkspace/dataworkspace/apps/request_access/views.py +++ b/dataworkspace/dataworkspace/apps/request_access/views.py @@ -18,7 +18,7 @@ ) from dataworkspace.apps.request_access import models -from dataworkspace import zendesk +from dataworkspace import help_desk class DatasetAccessRequest(CreateView): @@ -173,7 +173,7 @@ class AccessRequestConfirmationPage(RequestAccessMixin, DetailView): def get(self, request, *args, **kwargs): access_request = self.get_object() - if not access_request.zendesk_reference_number: + if not access_request.help_desk_reference_number: catalogue_item = ( find_dataset(access_request.catalogue_item_id, self.request.user) if access_request.catalogue_item_id @@ -184,15 +184,15 @@ def get(self, request, *args, **kwargs): isinstance(catalogue_item, VisualisationCatalogueItem) and catalogue_item.visualisation_template is not None ): - access_request.zendesk_reference_number = ( - zendesk.notify_visualisation_access_request( + access_request.help_desk_reference_number = ( + help_desk.notify_visualisation_access_request( request, access_request, catalogue_item, ) ) else: - access_request.zendesk_reference_number = zendesk.create_zendesk_ticket( + access_request.help_desk_reference_number = help_desk.create_help_desk_ticket( request, access_request, catalogue_item, @@ -205,7 +205,7 @@ def get(self, request, *args, **kwargs): EventLog.TYPE_DATASET_ACCESS_REQUEST, catalogue_item, extra={ - "ticket_reference": access_request.zendesk_reference_number, + "ticket_reference": access_request.help_desk_reference_number, }, ) else: @@ -213,7 +213,7 @@ def get(self, request, *args, **kwargs): request.user, EventLog.TYPE_TOOLS_ACCESS_REQUEST, extra={ - "ticket_reference": access_request.zendesk_reference_number, + "ticket_reference": access_request.help_desk_reference_number, }, ) return super().get(request, *args, **kwargs) diff --git a/dataworkspace/dataworkspace/apps/request_data/models.py b/dataworkspace/dataworkspace/apps/request_data/models.py index 1d1a0a95e2..d2f12a9519 100644 --- a/dataworkspace/dataworkspace/apps/request_data/models.py +++ b/dataworkspace/dataworkspace/apps/request_data/models.py @@ -47,4 +47,4 @@ class DataRequest(TimeStampedModel): choices=DataRequestStatus.choices, default=DataRequestStatus.draft, ) - zendesk_ticket_id = models.CharField(max_length=256) + help_desk_ticket_id = models.CharField(max_length=256) diff --git a/dataworkspace/dataworkspace/apps/request_data/views.py b/dataworkspace/dataworkspace/apps/request_data/views.py index 0a8f56d6a2..a7bf496689 100644 --- a/dataworkspace/dataworkspace/apps/request_data/views.py +++ b/dataworkspace/dataworkspace/apps/request_data/views.py @@ -17,7 +17,7 @@ RoleType, DataRequestStatus, ) -from dataworkspace.zendesk import create_support_request # pylint: disable=import-error +from dataworkspace.help_desk import create_support_request # pylint: disable=import-error class RequestData(TemplateView): @@ -249,11 +249,11 @@ def post(self, request, *args, **kwargs): # If they've hacked the URL, some required fields might be blank. try: - obj.clean_fields(exclude=["zendesk_ticket_id"]) + obj.clean_fields(exclude=["help_desk_ticket_id"]) except ValidationError: return HttpResponseBadRequest() - zendesk_message = f""" + help_desk_message = f""" A request for a new dataset on Data Workspace has been submitted. Here are the details: # Request details @@ -288,16 +288,21 @@ def post(self, request, *args, **kwargs): {obj.requester.email} """ + from django.conf import settings + + print("settings:::", flush=True) + print(settings.HELP_DESK_INTERFACE, flush=True) + ticket_id = create_support_request( obj.requester, obj.requester.email, - zendesk_message, + help_desk_message, tag="request-for-data", subject="Request for new dataset on Data Workspace", ) obj.status = DataRequestStatus.submitted - obj.zendesk_ticket_id = ticket_id + obj.help_desk_ticket_id = ticket_id obj.save() return HttpResponseRedirect( diff --git a/dataworkspace/dataworkspace/context_processors.py b/dataworkspace/dataworkspace/context_processors.py index cd0da65be7..aaa143e3b3 100644 --- a/dataworkspace/dataworkspace/context_processors.py +++ b/dataworkspace/dataworkspace/context_processors.py @@ -31,7 +31,7 @@ def common(request): "NOTIFY_ON_MASTER_DATASET_CHANGE_FLAG": settings.NOTIFY_ON_MASTER_DATASET_CHANGE_FLAG, "NOTIFY_ON_DATACUT_CHANGE_FLAG": settings.NOTIFY_ON_DATACUT_CHANGE_FLAG, "NOTIFY_ON_REFERENCE_DATASET_CHANGE_FLAG": settings.NOTIFY_ON_REFERENCE_DATASET_CHANGE_FLAG, - "ZENDESK_EMAIL": settings.ZENDESK_EMAIL, + "HELP_DESK_EMAIL": settings.HELP_DESK_EMAIL, "TEAMS_DATA_WORKSPACE_COMMUNITY_URL": settings.TEAMS_DATA_WORKSPACE_COMMUNITY_URL, "DATA_WORKSPACE_ROADMAP_URL": settings.DATA_WORKSPACE_ROADMAP_URL, "SSO_USER_ID": request.META.get("HTTP_SSO_PROFILE_USER_ID", ""), diff --git a/dataworkspace/dataworkspace/zendesk.py b/dataworkspace/dataworkspace/help_desk.py similarity index 91% rename from dataworkspace/dataworkspace/zendesk.py rename to dataworkspace/dataworkspace/help_desk.py index d06945e85c..0f500fd5fc 100644 --- a/dataworkspace/dataworkspace/zendesk.py +++ b/dataworkspace/dataworkspace/help_desk.py @@ -16,8 +16,8 @@ logger = logging.getLogger("app") -zendesk_service_field_id = settings.ZENDESK_SERVICE_FIELD_ID -zendesk_service_field_value = settings.ZENDESK_SERVICE_FIELD_VALUE +help_desk_service_field_id = settings.HELP_DESK_SERVICE_FIELD_ID +help_desk_service_field_value = settings.HELP_DESK_SERVICE_FIELD_VALUE def get_username(user): @@ -79,11 +79,11 @@ def build_private_comment_text(catalogue_item, approval_url): # configure and instantiate the client -helpdesk_interface = get_helpdesk_interface("helpdesk_client.interfaces.HelpDeskStubbed") +helpdesk_interface = get_helpdesk_interface(settings.HELP_DESK_INTERFACE) helpdesk = helpdesk_interface(credentials=settings.HELP_DESK_CREDS) -def create_zendesk_ticket(request, access_request, catalogue_item=None): +def create_help_desk_ticket(request, access_request, catalogue_item=None): access_request_url = request.build_absolute_uri( reverse("admin:request_access_accessrequest_change", args=(access_request.id,)) ) @@ -109,7 +109,7 @@ def create_zendesk_ticket(request, access_request, catalogue_item=None): description=ticket_description, user=HelpDeskUser(full_name=username, email=access_request.requester.email), custom_fields=[ - HelpDeskCustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) + HelpDeskCustomField(id=help_desk_service_field_id, value=help_desk_service_field_value) ], comment=HelpDeskComment(body=private_comment, public=False), ) @@ -152,7 +152,7 @@ def notify_visualisation_access_request(request, access_request, dataset): Secondary contact: {dataset.secondary_enquiries_contact.email if dataset.secondary_enquiries_contact else 'Not set'} -If access has not been granted to the requestor within 5 working days, this will trigger an update to this Zendesk ticket to resolve the request. +If access has not been granted to the requestor within 5 working days, this will trigger an update to this help desk ticket to resolve the request. """ ticket_reference = create_support_request( @@ -206,7 +206,7 @@ def create_support_request(user, email, message, tag=None, subject=None): email=email, ), custom_fields=[ - HelpDeskCustomField(id=zendesk_service_field_id, value=zendesk_service_field_value) + HelpDeskCustomField(id=help_desk_service_field_id, value=help_desk_service_field_value) ], tags=[tag] if tag else None, ) diff --git a/dataworkspace/dataworkspace/settings/base.py b/dataworkspace/dataworkspace/settings/base.py index be115e8d69..91a7b9292a 100644 --- a/dataworkspace/dataworkspace/settings/base.py +++ b/dataworkspace/dataworkspace/settings/base.py @@ -283,12 +283,8 @@ def aws_fargate_private_ip(): ] -ZENDESK_EMAIL = env["ZENDESK_EMAIL"] -ZENDESK_SUBDOMAIN = env["ZENDESK_SUBDOMAIN"] -ZENDESK_TOKEN = env["ZENDESK_TOKEN"] - -ZENDESK_SERVICE_FIELD_ID = env["ZENDESK_SERVICE_FIELD_ID"] -ZENDESK_SERVICE_FIELD_VALUE = env["ZENDESK_SERVICE_FIELD_VALUE"] +HELP_DESK_SERVICE_FIELD_ID = env["HELP_DESK_SERVICE_FIELD_ID"] +HELP_DESK_SERVICE_FIELD_VALUE = env["HELP_DESK_SERVICE_FIELD_VALUE"] # helpdesk abstraction HELP_DESK_INTERFACE = env.get("HELP_DESK_INTERFACE", "") diff --git a/dataworkspace/dataworkspace/settings/integration_tests.py b/dataworkspace/dataworkspace/settings/integration_tests.py index 3a07468615..e810aa6e3f 100644 --- a/dataworkspace/dataworkspace/settings/integration_tests.py +++ b/dataworkspace/dataworkspace/settings/integration_tests.py @@ -2,6 +2,15 @@ from .base import * # noqa +from helpdesk_client.interfaces import ( + HelpDeskBase, + HelpDeskComment, + HelpDeskUser, + HelpDeskTicket, +) + +import requests + DEBUG = True LOGGING = { @@ -19,3 +28,45 @@ "celery": {"handlers": ["dev"], "level": "INFO", "propagate": False}, }, } + + +class HelpDeskTest(HelpDeskBase): + def __init__(self, *args, **kwargs) -> None: + self._ticket_id = 1234567890987654321 + + def get_or_create_user(self, user: HelpDeskUser) -> HelpDeskUser: + raise NotImplementedError + + def create_ticket(self, ticket: HelpDeskTicket) -> HelpDeskTicket: + # Call help desk test server with ticket details + ticket.id = self._ticket_id + + resp = requests.post( + "http://dataworkspace.test:8006/api/v2/tickets.json", + json={ + "ticket": { + "id": ticket.id, + "subject": ticket.subject, + "description": ticket.description, + "status": "new", + "requester_id": 1, + "submitter_id": 1 + }, + } + ) + + return ticket + + def get_ticket(self, ticket_id: int) -> HelpDeskTicket: + raise NotImplementedError + + def close_ticket(self, ticket_id: int) -> HelpDeskTicket: + raise NotImplementedError + + def add_comment(self, ticket_id: int, comment: HelpDeskComment) -> HelpDeskTicket: + raise NotImplementedError + + def update_ticket(self, ticket: HelpDeskTicket) -> HelpDeskTicket: + raise NotImplementedError + +HELP_DESK_INTERFACE="dataworkspace.settings.integration_tests.HelpDeskTest" diff --git a/dataworkspace/dataworkspace/templates/datasets/subscriptions/unsubscribe_confirm.html b/dataworkspace/dataworkspace/templates/datasets/subscriptions/unsubscribe_confirm.html index 835ca7ffdf..a457fbfbdf 100644 --- a/dataworkspace/dataworkspace/templates/datasets/subscriptions/unsubscribe_confirm.html +++ b/dataworkspace/dataworkspace/templates/datasets/subscriptions/unsubscribe_confirm.html @@ -59,7 +59,7 @@

What happens next

If you have any questions, contact {{ ZENDESK_EMAIL }}. + href="mailto:{{ HELP_DESK_EMAIL }}">{{ HELP_DESK_EMAIL }}.

diff --git a/dataworkspace/dataworkspace/templates/request_access/confirmation-page.html b/dataworkspace/dataworkspace/templates/request_access/confirmation-page.html index 81d50c451a..e67169c0c7 100644 --- a/dataworkspace/dataworkspace/templates/request_access/confirmation-page.html +++ b/dataworkspace/dataworkspace/templates/request_access/confirmation-page.html @@ -19,7 +19,7 @@

Request complete

- Your reference number
{{ object.zendesk_reference_number }} + Your reference number
{{ object.help_desk_reference_number }}
diff --git a/dataworkspace/dataworkspace/templates/request_data/confirmation-page.html b/dataworkspace/dataworkspace/templates/request_data/confirmation-page.html index 9baa36216f..fb799f53a5 100644 --- a/dataworkspace/dataworkspace/templates/request_data/confirmation-page.html +++ b/dataworkspace/dataworkspace/templates/request_data/confirmation-page.html @@ -23,7 +23,7 @@

Application complete

- Your reference number is
{{ object.zendesk_ticket_id }} + Your reference number is
{{ object.help_desk_ticket_id }}
diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 3c36acaeaf..9cfd2b9744 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -90,7 +90,7 @@ def test_user_redirected_to_confirmation_page_after_form_submission( ) @pytest.mark.django_db - @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") + @mock.patch("dataworkspace.apps.request_access.views.help_desk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, @@ -290,7 +290,7 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( "access_type", (UserAccessType.REQUIRES_AUTHENTICATION, UserAccessType.OPEN) ) @mock.patch("dataworkspace.apps.core.boto3_client.boto3.client") - @mock.patch("dataworkspace.apps.request_access.views.zendesk.helpdesk.create_ticket") + @mock.patch("dataworkspace.apps.request_access.views.help_desk.helpdesk.create_ticket") @mock.patch("dataworkspace.apps.core.storage._upload_to_clamav") def test_helpdesk_ticket_created_after_form_submission( self, diff --git a/dataworkspace/dataworkspace/tests/request_data/test_views.py b/dataworkspace/dataworkspace/tests/request_data/test_views.py index da229f846c..35eef6a287 100644 --- a/dataworkspace/dataworkspace/tests/request_data/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_data/test_views.py @@ -209,7 +209,7 @@ def test_submitting_answers_fails_on_invalid_data( assert resp.status_code == expected_status_code @mock.patch("dataworkspace.apps.request_data.views.create_support_request") - def test_submitting_data_request_that_has_been_submitted_does_not_create_new_zendesk_ticket( + def test_submitting_data_request_that_has_been_submitted_does_not_create_new_help_desk_ticket( self, create_support_request_mock, client ): dr = DataRequestFactory.create(status=DataRequestStatus.submitted) @@ -241,8 +241,8 @@ def test_page_available(self, client): resp = client.get(reverse("request-data:confirmation-page", kwargs={"pk": dr.pk})) assert resp.status_code == 200 - def test_shows_zendesk_ticket_id(self, client): - dr = DataRequestFactory.create(zendesk_ticket_id="123456789-IM-A-REFERENCE-987654321") + def test_shows_help_desk_ticket_id(self, client): + dr = DataRequestFactory.create(help_desk_ticket_id="123456789-IM-A-REFERENCE-987654321") resp = client.get(reverse("request-data:confirmation-page", kwargs={"pk": dr.pk})) assert resp.status_code == 200 assert "123456789-IM-A-REFERENCE-987654321" in resp.content.decode(resp.charset) diff --git a/infra/ecs_main_admin.tf b/infra/ecs_main_admin.tf index f6bccd51d2..218397aa2b 100644 --- a/infra/ecs_main_admin.tf +++ b/infra/ecs_main_admin.tf @@ -64,11 +64,9 @@ locals { fargate_spawner__user_provided_task_role__policy_document_template_base64 = "${base64encode(data.aws_iam_policy_document.user_provided_access_template.json)}" fargate_spawner__user_provided_ecr_repository__name = "${aws_ecr_repository.user_provided.name}" - zendesk_email = "${var.zendesk_email}" - zendesk_subdomain = "${var.zendesk_subdomain}" - zendesk_token = "${var.zendesk_token}" - zendesk_service_field_id = "${var.zendesk_service_field_id}" - zendesk_service_field_value = "${var.zendesk_service_field_value}" + help_desk_email = "${var.help_desk_email}" + help_desk_service_field_id = "${var.help_desk_service_field_id}" + help_desk_service_field_value = "${var.help_desk_service_field_value}" prometheus_domain = "${var.prometheus_domain}" metrics_service_discovery_basic_auth_user = "${var.metrics_service_discovery_basic_auth_user}" diff --git a/infra/ecs_main_admin_container_definitions.json b/infra/ecs_main_admin_container_definitions.json index 725c01dee8..4a0921b76c 100644 --- a/infra/ecs_main_admin_container_definitions.json +++ b/infra/ecs_main_admin_container_definitions.json @@ -829,24 +829,16 @@ "value": "${mirror_remote_root}" }, { - "name": "ZENDESK_EMAIL", - "value": "${zendesk_email}" + "name": "HELP_DESK_EMAIL", + "value": "${help_desk_email}" }, { - "name": "ZENDESK_SUBDOMAIN", - "value": "${zendesk_subdomain}" + "name": "HELP_DESK_SERVICE_FIELD_ID", + "value": "${help_desk_service_field_id}" }, { - "name": "ZENDESK_TOKEN", - "value": "${zendesk_token}" - }, - { - "name": "ZENDESK_SERVICE_FIELD_ID", - "value": "${zendesk_service_field_id}" - }, - { - "name": "ZENDESK_SERVICE_FIELD_VALUE", - "value": "${zendesk_service_field_value}" + "name": "ZHELP_DESK_SERVICE_FIELD_VALUE", + "value": "${help_desk_service_field_value}" }, { "name": "QUICKSIGHT_DASHBOARD_EMBEDDING_ROLE_ARN", diff --git a/infra/main.tf b/infra/main.tf index 16623d989e..c9135db226 100644 --- a/infra/main.tf +++ b/infra/main.tf @@ -78,11 +78,9 @@ variable healthcheck_domain {} variable prometheus_domain {} variable cloudwatch_subscription_filter {} -variable zendesk_email {} -variable zendesk_subdomain {} -variable zendesk_token {} -variable zendesk_service_field_id {} -variable zendesk_service_field_value {} +variable help_desk_email {} +variable help_desk_service_field_id {} +variable help_desk_service_field_value {} variable prometheus_whitelist { type = "list" diff --git a/test/selenium/conftest.py b/test/selenium/conftest.py index f60d6ede38..663083d837 100644 --- a/test/selenium/conftest.py +++ b/test/selenium/conftest.py @@ -269,11 +269,11 @@ def create_sso(is_logged_in, codes, tokens, auth_to_me): proc.join() -class ZendeskServer(multiprocessing.Process): +class HelpDeskServer(multiprocessing.Process): def run(self): submitted_tickets = [] - class ZendeskHandler(BaseHTTPRequestHandler): + class HelpDeskHandler(BaseHTTPRequestHandler): def do_GET(self): nonlocal submitted_tickets if self.path == "/_meta/read-submitted-tickets": @@ -296,17 +296,19 @@ def do_POST(self): self.send_header("Content-Type", "application/json") self.end_headers() + response_content = self.rfile.read(int(self.headers["content-length"])).decode("ascii") + ticket_dict = json.loads(response_content) + submitted_tickets.append( - json.loads( - self.rfile.read(int(self.headers["content-length"])).decode("ascii") - ) + ticket_dict, ) - with open("test/stubs/zendesk/create-ticket.json", "rb") as stub: + with open("test/stubs/help_desk/create-ticket.json", "rb") as stub: response = BytesIO() response.write(stub.read()) self.wfile.write(response.getvalue()) + return self.send_response(500) @@ -317,7 +319,7 @@ def sys_exit(_, __): signal.signal(signal.SIGTERM, sys_exit) - httpd = HTTPServer(("0.0.0.0", 8006), ZendeskHandler) + httpd = HTTPServer(("0.0.0.0", 8006), HelpDeskHandler) try: httpd.serve_forever() finally: @@ -325,8 +327,8 @@ def sys_exit(_, __): @contextmanager -def create_zendesk(): - proc = ZendeskServer() +def create_help_desk(): + proc = HelpDeskServer() proc.start() yield proc diff --git a/test/selenium/test_request_data.py b/test/selenium/test_request_data.py index 91abd760cc..93896eecbf 100644 --- a/test/selenium/test_request_data.py +++ b/test/selenium/test_request_data.py @@ -1,3 +1,5 @@ +from unittest import mock + import time import pytest @@ -9,7 +11,7 @@ from test.selenium.common import get_driver # pylint: disable=wrong-import-order from test.selenium.conftest import ( # pylint: disable=wrong-import-order create_sso, - create_zendesk, + create_help_desk, ) from test.selenium.workspace_pages import ( # pylint: disable=wrong-import-order HomePage, @@ -23,6 +25,11 @@ RequestDataLicencePage, ) +from django.conf import settings + + +from helpdesk_client.interfaces import HelpDeskBase, HelpDeskUser, HelpDeskTicket, HelpDeskComment + class TestRequestData: driver = None @@ -45,7 +52,7 @@ def _application(self, create_application): } with create_sso( is_logged_in, codes, tokens, auth_to_me - ) as sso, create_zendesk() as zendesk: + ) as sso, create_help_desk() as help_desk: tries = 240 # 60 seconds while tries >= 0: try: @@ -60,14 +67,14 @@ def _application(self, create_application): self.__class__.driver = get_driver() self.__class__.sso = sso - self.__class__.zendesk = zendesk + self.__class__.help_desk = help_desk cache.clear() yield - def test_happy_path(self, _application): + def test_happy_path(self, _application): home_page = HomePage(self.driver) home_page.open() @@ -114,15 +121,16 @@ def test_happy_path(self, _application): # Confirm that the answers are correct confirmation_page = check_answers_page.click_accept() + assert "1234567890987654321" in confirmation_page.get_html() - # Check that the request data has been posted to Zendesk + # Check that the request data has been posted to the help desk submitted_tickets = requests.get( "http://dataworkspace.test:8006/_meta/read-submitted-tickets" ).json() assert len(submitted_tickets) == 1 - ticket_data = submitted_tickets[0]["ticket"] + assert "Bobby Tables" in ticket_data["description"] assert "It’s data about DIT" in ticket_data["description"] assert ( diff --git a/test/stubs/zendesk/create-ticket.json b/test/stubs/help_desk/create-ticket.json similarity index 99% rename from test/stubs/zendesk/create-ticket.json rename to test/stubs/help_desk/create-ticket.json index 0c7d3bddff..7dcfc115fb 100644 --- a/test/stubs/zendesk/create-ticket.json +++ b/test/stubs/help_desk/create-ticket.json @@ -12,4 +12,4 @@ "ticket_id": 1234567890987654321, "author_id": 1 } -} +} \ No newline at end of file From eeb6d14e9b42bf560d9602be7d33ef56bc5148f5 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Mon, 26 Sep 2022 17:42:46 +0100 Subject: [PATCH 17/27] Linting and tidy up --- dataworkspace/dataworkspace/help_desk.py | 4 +++- .../dataworkspace/settings/integration_tests.py | 9 +++++---- test/selenium/conftest.py | 4 +++- test/selenium/test_request_data.py | 11 +++-------- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/dataworkspace/dataworkspace/help_desk.py b/dataworkspace/dataworkspace/help_desk.py index 0f500fd5fc..08e671c922 100644 --- a/dataworkspace/dataworkspace/help_desk.py +++ b/dataworkspace/dataworkspace/help_desk.py @@ -206,7 +206,9 @@ def create_support_request(user, email, message, tag=None, subject=None): email=email, ), custom_fields=[ - HelpDeskCustomField(id=help_desk_service_field_id, value=help_desk_service_field_value) + HelpDeskCustomField( + id=help_desk_service_field_id, value=help_desk_service_field_value + ) ], tags=[tag] if tag else None, ) diff --git a/dataworkspace/dataworkspace/settings/integration_tests.py b/dataworkspace/dataworkspace/settings/integration_tests.py index e810aa6e3f..b1cf96d43c 100644 --- a/dataworkspace/dataworkspace/settings/integration_tests.py +++ b/dataworkspace/dataworkspace/settings/integration_tests.py @@ -41,7 +41,7 @@ def create_ticket(self, ticket: HelpDeskTicket) -> HelpDeskTicket: # Call help desk test server with ticket details ticket.id = self._ticket_id - resp = requests.post( + requests.post( "http://dataworkspace.test:8006/api/v2/tickets.json", json={ "ticket": { @@ -50,9 +50,9 @@ def create_ticket(self, ticket: HelpDeskTicket) -> HelpDeskTicket: "description": ticket.description, "status": "new", "requester_id": 1, - "submitter_id": 1 + "submitter_id": 1, }, - } + }, ) return ticket @@ -69,4 +69,5 @@ def add_comment(self, ticket_id: int, comment: HelpDeskComment) -> HelpDeskTicke def update_ticket(self, ticket: HelpDeskTicket) -> HelpDeskTicket: raise NotImplementedError -HELP_DESK_INTERFACE="dataworkspace.settings.integration_tests.HelpDeskTest" + +HELP_DESK_INTERFACE = "dataworkspace.settings.integration_tests.HelpDeskTest" diff --git a/test/selenium/conftest.py b/test/selenium/conftest.py index 663083d837..90073f2368 100644 --- a/test/selenium/conftest.py +++ b/test/selenium/conftest.py @@ -296,7 +296,9 @@ def do_POST(self): self.send_header("Content-Type", "application/json") self.end_headers() - response_content = self.rfile.read(int(self.headers["content-length"])).decode("ascii") + response_content = self.rfile.read(int(self.headers["content-length"])).decode( + "ascii" + ) ticket_dict = json.loads(response_content) submitted_tickets.append( diff --git a/test/selenium/test_request_data.py b/test/selenium/test_request_data.py index 93896eecbf..623899d06e 100644 --- a/test/selenium/test_request_data.py +++ b/test/selenium/test_request_data.py @@ -1,5 +1,3 @@ -from unittest import mock - import time import pytest @@ -8,6 +6,7 @@ from django.core.cache import cache from dataworkspace.apps.request_data.models import RoleType, SecurityClassificationType + from test.selenium.common import get_driver # pylint: disable=wrong-import-order from test.selenium.conftest import ( # pylint: disable=wrong-import-order create_sso, @@ -25,11 +24,6 @@ RequestDataLicencePage, ) -from django.conf import settings - - -from helpdesk_client.interfaces import HelpDeskBase, HelpDeskUser, HelpDeskTicket, HelpDeskComment - class TestRequestData: driver = None @@ -73,7 +67,6 @@ def _application(self, create_application): yield - def test_happy_path(self, _application): home_page = HomePage(self.driver) home_page.open() @@ -131,6 +124,8 @@ def test_happy_path(self, _application): assert len(submitted_tickets) == 1 + ticket_data = submitted_tickets[0]["ticket"] + assert "Bobby Tables" in ticket_data["description"] assert "It’s data about DIT" in ticket_data["description"] assert ( From d4899074a2719132718cf853d7fecf4f95bf3e4d Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Mon, 26 Sep 2022 18:41:15 +0100 Subject: [PATCH 18/27] Add missing migrations --- ...name_zendesk_reference_number_accessrequest | 18 ++++++++++++++++++ ...name_zendesk_ticket_id_datarequest_help_des | 18 ++++++++++++++++++ dataworkspace/dataworkspace/settings/base.py | 1 - 3 files changed, 36 insertions(+), 1 deletion(-) create mode 100644 dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest create mode 100644 dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des diff --git a/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest b/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest new file mode 100644 index 0000000000..630808e7fd --- /dev/null +++ b/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-09-26 17:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('request_access', '0004_alter_accessrequest_training_screenshot'), + ] + + operations = [ + migrations.RenameField( + model_name='accessrequest', + old_name='zendesk_reference_number', + new_name='help_desk_reference_number', + ), + ] diff --git a/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des b/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des new file mode 100644 index 0000000000..b6fa7b4740 --- /dev/null +++ b/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des @@ -0,0 +1,18 @@ +# Generated by Django 3.2.15 on 2022-09-26 17:38 + +from django.db import migrations + + +class Migration(migrations.Migration): + + dependencies = [ + ('request_data', '0004_datarequest_data_licence'), + ] + + operations = [ + migrations.RenameField( + model_name='datarequest', + old_name='zendesk_ticket_id', + new_name='help_desk_ticket_id', + ), + ] diff --git a/dataworkspace/dataworkspace/settings/base.py b/dataworkspace/dataworkspace/settings/base.py index 91a7b9292a..d13504d3ea 100644 --- a/dataworkspace/dataworkspace/settings/base.py +++ b/dataworkspace/dataworkspace/settings/base.py @@ -282,7 +282,6 @@ def aws_fargate_private_ip(): f"ws://{APPLICATION_ROOT_DOMAIN.split(':')[0]}:3000", ] - HELP_DESK_SERVICE_FIELD_ID = env["HELP_DESK_SERVICE_FIELD_ID"] HELP_DESK_SERVICE_FIELD_VALUE = env["HELP_DESK_SERVICE_FIELD_VALUE"] From ecb3992d2cc6d79bbbaa19e793081b74f5f593b8 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 27 Sep 2022 09:16:27 +0100 Subject: [PATCH 19/27] Add missing migrations --- ...=> 0005_rename_zendesk_reference_number_accessrequest.py} | 0 ...> 0005_rename_zendesk_ticket_id_datarequest_help_desk.py} | 0 dataworkspace/dataworkspace/settings/base.py | 5 +++-- 3 files changed, 3 insertions(+), 2 deletions(-) rename dataworkspace/dataworkspace/apps/request_access/migrations/{0005_rename_zendesk_reference_number_accessrequest => 0005_rename_zendesk_reference_number_accessrequest.py} (100%) rename dataworkspace/dataworkspace/apps/request_data/migrations/{0005_rename_zendesk_ticket_id_datarequest_help_des => 0005_rename_zendesk_ticket_id_datarequest_help_desk.py} (100%) diff --git a/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest b/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py similarity index 100% rename from dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest rename to dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py diff --git a/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des b/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py similarity index 100% rename from dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_des rename to dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py diff --git a/dataworkspace/dataworkspace/settings/base.py b/dataworkspace/dataworkspace/settings/base.py index d13504d3ea..886c41fe89 100644 --- a/dataworkspace/dataworkspace/settings/base.py +++ b/dataworkspace/dataworkspace/settings/base.py @@ -282,8 +282,9 @@ def aws_fargate_private_ip(): f"ws://{APPLICATION_ROOT_DOMAIN.split(':')[0]}:3000", ] -HELP_DESK_SERVICE_FIELD_ID = env["HELP_DESK_SERVICE_FIELD_ID"] -HELP_DESK_SERVICE_FIELD_VALUE = env["HELP_DESK_SERVICE_FIELD_VALUE"] +HELP_DESK_EMAIL = env.get("HELP_DESK_INTERFACE", "") +HELP_DESK_SERVICE_FIELD_ID = env.get("HELP_DESK_SERVICE_FIELD_ID", "") +HELP_DESK_SERVICE_FIELD_VALUE = env.get("HELP_DESK_SERVICE_FIELD_VALUE", "") # helpdesk abstraction HELP_DESK_INTERFACE = env.get("HELP_DESK_INTERFACE", "") From 09ecb61101648088ec4c8b4b599e94b8f248c130 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 27 Sep 2022 09:44:21 +0100 Subject: [PATCH 20/27] Lint migrations --- .../0005_rename_zendesk_reference_number_accessrequest.py | 8 ++++---- ...0005_rename_zendesk_ticket_id_datarequest_help_desk.py | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py b/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py index 630808e7fd..be1b8bee03 100644 --- a/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py +++ b/dataworkspace/dataworkspace/apps/request_access/migrations/0005_rename_zendesk_reference_number_accessrequest.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('request_access', '0004_alter_accessrequest_training_screenshot'), + ("request_access", "0004_alter_accessrequest_training_screenshot"), ] operations = [ migrations.RenameField( - model_name='accessrequest', - old_name='zendesk_reference_number', - new_name='help_desk_reference_number', + model_name="accessrequest", + old_name="zendesk_reference_number", + new_name="help_desk_reference_number", ), ] diff --git a/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py b/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py index b6fa7b4740..a0fc84a8d1 100644 --- a/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py +++ b/dataworkspace/dataworkspace/apps/request_data/migrations/0005_rename_zendesk_ticket_id_datarequest_help_desk.py @@ -6,13 +6,13 @@ class Migration(migrations.Migration): dependencies = [ - ('request_data', '0004_datarequest_data_licence'), + ("request_data", "0004_datarequest_data_licence"), ] operations = [ migrations.RenameField( - model_name='datarequest', - old_name='zendesk_ticket_id', - new_name='help_desk_ticket_id', + model_name="datarequest", + old_name="zendesk_ticket_id", + new_name="help_desk_ticket_id", ), ] From e47a46d96f0fccdf531c4f9939a9c86cdf6feb1a Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 27 Sep 2022 13:39:33 +0100 Subject: [PATCH 21/27] Add integration test settings --- Makefile | 2 +- infra/ecs_main_admin_container_definitions.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 0dde95418a..2fb5ee60e5 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ docker-test-unit: docker-build .PHONY: docker-test-integration docker-test-integration: docker-build - docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest test/ + docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest --ds=DJANGO_SETTINGS_MODULE=dataworkspace.settings.integration_tests test/ .PHONY: docker-test diff --git a/infra/ecs_main_admin_container_definitions.json b/infra/ecs_main_admin_container_definitions.json index 4a0921b76c..785b7d5c3e 100644 --- a/infra/ecs_main_admin_container_definitions.json +++ b/infra/ecs_main_admin_container_definitions.json @@ -837,7 +837,7 @@ "value": "${help_desk_service_field_id}" }, { - "name": "ZHELP_DESK_SERVICE_FIELD_VALUE", + "name": "HELP_DESK_SERVICE_FIELD_VALUE", "value": "${help_desk_service_field_value}" }, { From bde4840a15ab84f648d3a276436be72cee90e07a Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 27 Sep 2022 13:48:04 +0100 Subject: [PATCH 22/27] Try and pass setting as env var --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2fb5ee60e5..75eb23c6e3 100644 --- a/Makefile +++ b/Makefile @@ -37,8 +37,7 @@ docker-test-unit: docker-build .PHONY: docker-test-integration docker-test-integration: docker-build - docker-compose -f docker-compose-test.yml -p data-workspace-test run data-workspace-test pytest --ds=DJANGO_SETTINGS_MODULE=dataworkspace.settings.integration_tests test/ - + docker-compose -f docker-compose-test.yml -p data-workspace-test run -e DJANGO_SETTINGS_MODULE=dataworkspace.settings.integration_tests data-workspace-test pytest test/ .PHONY: docker-test docker-test: docker-test-integration docker-test-unit From 1f21f606ab75f581ad63e10f091864004ab6ba2b Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 4 Oct 2022 12:29:07 +0100 Subject: [PATCH 23/27] Debug CI --- .../settings/integration_tests.py | 18 ------------------ 1 file changed, 18 deletions(-) diff --git a/dataworkspace/dataworkspace/settings/integration_tests.py b/dataworkspace/dataworkspace/settings/integration_tests.py index b1cf96d43c..c75cdcc003 100644 --- a/dataworkspace/dataworkspace/settings/integration_tests.py +++ b/dataworkspace/dataworkspace/settings/integration_tests.py @@ -11,24 +11,6 @@ import requests -DEBUG = True - -LOGGING = { - "version": 1, - "disable_existing_loggers": False, - "formatters": { - "ecs": {"format": "%(asctime)s [%(levelname)s] [%(name)s] %(message)s"}, - "verbose": {"format": "%(asctime)s [%(levelname)s] [%(name)s] %(message)s"}, - }, - "handlers": {"dev": {"class": "logging.StreamHandler", "formatter": "verbose"}}, - "loggers": { - "app": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, - "test": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, - "dataworkspace": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, - "celery": {"handlers": ["dev"], "level": "INFO", "propagate": False}, - }, -} - class HelpDeskTest(HelpDeskBase): def __init__(self, *args, **kwargs) -> None: From 293ec69c1b7b445be04f297210bd67bc90003672 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Tue, 4 Oct 2022 16:44:33 +0100 Subject: [PATCH 24/27] Add logging back in --- .../dataworkspace/settings/integration_tests.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/dataworkspace/dataworkspace/settings/integration_tests.py b/dataworkspace/dataworkspace/settings/integration_tests.py index c75cdcc003..728b4107b5 100644 --- a/dataworkspace/dataworkspace/settings/integration_tests.py +++ b/dataworkspace/dataworkspace/settings/integration_tests.py @@ -12,6 +12,23 @@ import requests +LOGGING = { + "version": 1, + "disable_existing_loggers": False, + "formatters": { + "ecs": {"format": "%(asctime)s [%(levelname)s] [%(name)s] %(message)s"}, + "verbose": {"format": "%(asctime)s [%(levelname)s] [%(name)s] %(message)s"}, + }, + "handlers": {"dev": {"class": "logging.StreamHandler", "formatter": "verbose"}}, + "loggers": { + "app": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, + "test": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, + "dataworkspace": {"handlers": ["dev"], "level": "DEBUG", "propagate": True}, + "celery": {"handlers": ["dev"], "level": "INFO", "propagate": False}, + }, +} + + class HelpDeskTest(HelpDeskBase): def __init__(self, *args, **kwargs) -> None: self._ticket_id = 1234567890987654321 From 224006c3c4ff71259aaecf04cf4e5c76bdb95bc5 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 5 Oct 2022 09:55:31 +0100 Subject: [PATCH 25/27] Bump helpdesk version # --- requirements-dev.txt | 19 ++++--------------- requirements.txt | 17 ++--------------- 2 files changed, 6 insertions(+), 30 deletions(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 335d42ce69..a37ab2a12a 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -18,8 +18,6 @@ amqp==5.0.9 # kombu appdirs==1.4.4 # via pyppeteer -appnope==0.1.3 - # via ipython asgiref==3.3.4 # via # -r requirements.txt @@ -229,7 +227,7 @@ h11==0.13.0 # via wsproto hawk-server-asyncio==0.0.13 # via -r requirements.txt -helpdesk-client==0.1.6 +helpdesk-client==0.1.7 # via -r requirements.txt hiredis==1.1.0 # via @@ -249,9 +247,7 @@ ijson==3.1.3 importlib-metadata==4.11.3 # via pyppeteer iniconfig==1.1.1 - # via - # -r requirements.txt - # pytest + # via pytest ipdb==0.13.7 # via -r requirements-dev.in ipython==7.31.1 @@ -360,9 +356,7 @@ platformdirs==2.4.0 plotly==5.6.0 # via -r requirements.txt pluggy==1.0.0 - # via - # -r requirements.txt - # pytest + # via pytest prompt-toolkit==3.0.24 # via # -r requirements.txt @@ -382,9 +376,7 @@ psycopg2-binary==2.9.2 ptyprocess==0.7.0 # via pexpect py==1.11.0 - # via - # -r requirements.txt - # pytest + # via pytest pycodestyle==2.7.0 # via flake8 pycparser==2.20 @@ -429,8 +421,6 @@ pysocks==1.7.1 pytest==7.1.3 # via # -r requirements-dev.in - # -r requirements.txt - # helpdesk-client # pytest-cov # pytest-django # pytest-mock @@ -561,7 +551,6 @@ toml==0.10.2 # pylint tomli==2.0.1 # via - # -r requirements.txt # black # pytest tqdm==4.56.0 diff --git a/requirements.txt b/requirements.txt index c29c6114fa..f959d10afa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,7 +22,6 @@ attrs==20.2.0 # via # aiohttp # jsonschema - # pytest aws-requests-auth==0.4.3 # via -r requirements.in billiard==3.6.4.0 @@ -144,7 +143,7 @@ greenlet==1.1.0 # via gevent hawk-server-asyncio==0.0.13 # via -r requirements.in -helpdesk-client==0.1.6 +helpdesk-client==0.1.7 # via -r requirements.in hiredis==1.1.0 # via aioredis @@ -154,8 +153,6 @@ idna==2.10 # yarl ijson==3.1.3 # via tabulator -iniconfig==1.1.1 - # via pytest isodate==0.6.0 # via tableschema jdcal==1.4.1 @@ -191,17 +188,13 @@ oauthlib==3.1.0 openpyxl==3.0.6 # via tabulator packaging==20.4 - # via - # bleach - # pytest + # via bleach paste==3.4.3 # via -r requirements.in pglast==3.10 # via -r requirements.in plotly==5.6.0 # via -r requirements.in -pluggy==1.0.0 - # via pytest prompt-toolkit==3.0.24 # via click-repl psutil==5.7.2 @@ -212,8 +205,6 @@ psycogreen==1.0.2 # django-db-geventpool psycopg2-binary==2.9.2 # via -r requirements.in -py==1.11.0 - # via pytest pycparser==2.20 # via cffi pyjwt==2.4.0 @@ -224,8 +215,6 @@ pyparsing==2.4.7 # via packaging pyrsistent==0.17.3 # via jsonschema -pytest==7.1.3 - # via helpdesk-client python-dateutil==2.8.1 # via # botocore @@ -298,8 +287,6 @@ tenacity==6.2.0 # via # celery-redbeat # plotly -tomli==2.0.1 - # via pytest typing-extensions==4.0.0 # via aiohttp unicodecsv==0.14.1 From a0228df61322a57ca0453df25c43ccae36ade4ea Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 5 Oct 2022 10:40:48 +0100 Subject: [PATCH 26/27] Tidy up --- .../tests/request_access/test_views.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/dataworkspace/dataworkspace/tests/request_access/test_views.py b/dataworkspace/dataworkspace/tests/request_access/test_views.py index 9cfd2b9744..52e719b58d 100644 --- a/dataworkspace/dataworkspace/tests/request_access/test_views.py +++ b/dataworkspace/dataworkspace/tests/request_access/test_views.py @@ -95,7 +95,7 @@ def test_user_redirected_to_confirmation_page_after_form_submission( def test_helpdesk_ticket_created_after_form_submission( self, mock_upload_to_clamav, - helpdesk_create_ticket, + mock_helpdesk_create_ticket, client, user, metadata_db, @@ -105,7 +105,7 @@ class MockTicket: def id(self): return 1 - helpdesk_create_ticket.return_value = MockTicket() + mock_helpdesk_create_ticket.return_value = MockTicket() mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) @@ -138,8 +138,8 @@ def id(self): follow=True, ) - assert len(helpdesk_create_ticket.call_args_list) == 1 - call_args, _ = helpdesk_create_ticket.call_args_list[0] + assert len(mock_helpdesk_create_ticket.call_args_list) == 1 + call_args, _ = mock_helpdesk_create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" @@ -295,7 +295,7 @@ def test_user_redirected_to_summary_page_after_step_3_form_submission( def test_helpdesk_ticket_created_after_form_submission( self, mock_upload_to_clamav, - helpdesk_create_ticket, + mock_helpdesk_create_ticket, mock_boto, client, metadata_db, @@ -306,7 +306,7 @@ class MockTicket: def id(self): return 1 - helpdesk_create_ticket.return_value = MockTicket() + mock_helpdesk_create_ticket.return_value = MockTicket() mock_upload_to_clamav.return_value = ClamAVResponse({"malware": False}) @@ -329,8 +329,8 @@ def id(self): follow=True, ) - assert len(helpdesk_create_ticket.call_args_list) == 1 - call_args, _ = helpdesk_create_ticket.call_args_list[0] + assert len(mock_helpdesk_create_ticket.call_args_list) == 1 + call_args, _ = mock_helpdesk_create_ticket.call_args_list[0] ticket = call_args[0] assert ticket.subject == "Access Request for A master" From 620f91e90775cc902160408a7a55932f7b6d1054 Mon Sep 17 00:00:00 2001 From: Ross Miller Date: Wed, 5 Oct 2022 10:44:30 +0100 Subject: [PATCH 27/27] Tidy up --- test/stubs/help_desk/create-ticket.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/stubs/help_desk/create-ticket.json b/test/stubs/help_desk/create-ticket.json index 7dcfc115fb..0c7d3bddff 100644 --- a/test/stubs/help_desk/create-ticket.json +++ b/test/stubs/help_desk/create-ticket.json @@ -12,4 +12,4 @@ "ticket_id": 1234567890987654321, "author_id": 1 } -} \ No newline at end of file +}