diff --git a/Dockerfile.prod b/Dockerfile.prod new file mode 100644 index 0000000..6fa3e78 --- /dev/null +++ b/Dockerfile.prod @@ -0,0 +1,39 @@ +FROM python:3.10 + +ENV PYTHONUNBUFFERED 1 +ARG REQUEST_BROKER_DNS +ARG APPLICATION_PORT +RUN apt-get update +RUN apt-get install --yes apache2 apache2-dev +RUN apt-get install --yes postgresql +RUN apt-get -y install python3-pip +RUN apt-get -y install cron +RUN pip install --upgrade pip + +RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz \ + && tar xvfz 4.9.0.tar.gz \ + && cd mod_wsgi-4.9.0 \ + && ./configure --with-apxs=/usr/bin/apxs --with-python=/usr/local/bin/python \ + && make \ + && make install \ + && make clean + +ADD ./apache/000-request_broker.conf /etc/apache2/sites-available/000-request_broker.conf +RUN sed "s/ENV_REQUEST_BROKER_DNS/${REQUEST_BROKER_DNS}/" -i /etc/apache2/sites-available/000-request_broker.conf +RUN sed "s/ENV_REQUEST_BROKER_PORT/${APPLICATION_PORT}/" -i /etc/apache2/sites-available/000-request_broker.conf +ADD ./apache/wsgi.load /etc/apache2/mods-available/wsgi.load +RUN a2ensite 000-request_broker.conf +RUN a2enmod headers +RUN a2enmod rewrite +RUN a2enmod wsgi + +COPY . /var/www/request-broker +WORKDIR /var/www/request-broker +RUN pip install -r requirements.txt + +COPY --chmod=644 crontab /etc/cron.d/crontab +RUN crontab /etc/cron.d/crontab + +EXPOSE ${APPLICATION_PORT} + +ENTRYPOINT ["./entrypoint.prod.sh"] diff --git a/README.md b/README.md index 5c46519..05267cd 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Or, if you want to remove all data The request broker manages configuration by setting environment variables. These variables can be seen in `docker-compose.yml`. -Deployment using the `docker-compose.prod.yml` or `docker-compose.dev.yml` files requires the presence of an `.env.prod` or `.env.dev` file in the root directory of the application. The environment variables included in those files should match the variables in `docker-compose.yml`, although the values assigned to those variables may change. +Deployment using the `Dockerfile.prod` file is intended to bring up a production image (based on Apache/WSGI) which is ready to be proxied publicly by an apache, nginx, traefik or similar frontend. `Dockerfile.prod` expects two environment arguments to be available at build time: `REQUEST_BROKER_DNS` and `APPLICATION_PORT`. Apache will Listen on `${APPLICATION_PORT}` with a ServerName of `${REQUEST_BROKER_DNS}`. ## Services diff --git a/apache/000-request_broker.conf b/apache/000-request_broker.conf new file mode 100644 index 0000000..2aaceef --- /dev/null +++ b/apache/000-request_broker.conf @@ -0,0 +1,19 @@ +Listen ENV_REQUEST_BROKER_PORT + + ErrorLog /var/log/apache2/request_broker_error_log + CustomLog /var/log/apache2/request_broker_access_log combined + ServerName ENV_REQUEST_BROKER_DNS + DocumentRoot /var/www/html/ + Alias /static /var/www/request-broker/static + + Require all granted + + + WSGIProcessGroup request_broker + WSGIApplicationGroup %{GLOBAL} + Require all granted + + WSGIDaemonProcess request_broker home=/var/www/request-broker + WSGIProcessGroup request_broker + WSGIScriptAlias / /var/www/request-broker/request_broker/wsgi.py + diff --git a/apache/wsgi.load b/apache/wsgi.load new file mode 100644 index 0000000..d76d1d7 --- /dev/null +++ b/apache/wsgi.load @@ -0,0 +1 @@ +LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so diff --git a/crontab b/crontab new file mode 100644 index 0000000..6e644f3 --- /dev/null +++ b/crontab @@ -0,0 +1,2 @@ +*/5 * * * * /usr/local/bin/python3 -u /var/www/request-broker/manage.py runcrons >/proc/1/fd/1 2>/proc/1/fd/2 + diff --git a/entrypoint.prod.sh b/entrypoint.prod.sh new file mode 100755 index 0000000..74560b4 --- /dev/null +++ b/entrypoint.prod.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +./wait-for-it.sh db:${SQL_PORT} -- echo "Apply database migrations" +python manage.py migrate + +# Collect static files +echo "Collecting static files" +python manage.py collectstatic + +#Start cron +echo "Starting cron" +cron + +#Start server +echo "Starting server" +apache2ctl -D FOREGROUND diff --git a/entrypoint.sh b/entrypoint.sh index dfe20d4..e672c1b 100755 --- a/entrypoint.sh +++ b/entrypoint.sh @@ -8,7 +8,7 @@ if [ ! -f request_broker/config.py ]; then cp request_broker/config.py.example request_broker/config.py fi -./wait-for-it.sh db:5432 -- echo "Apply database migrations" +./wait-for-it.sh db:${SQL_PORT} -- echo "Apply database migrations" python manage.py migrate #Start server diff --git a/process_request/clients.py b/process_request/clients.py index 15bd7b9..5eac82c 100644 --- a/process_request/clients.py +++ b/process_request/clients.py @@ -1,7 +1,5 @@ from requests import Session -from request_broker import settings - def http_meth_factory(meth): """Utility method for producing HTTP proxy methods. @@ -27,10 +25,15 @@ def __init__(cls, name, parents, dct): class AeonAPIClient(metaclass=ProxyMethods): - def __init__(self, baseurl): + def __init__(self, baseurl, apikey): self.baseurl = baseurl self.session = Session() self.session.headers.update( {"Accept": "application/json", - "User-Agent": "AeonAPIClient/0.1", - "X-AEON-API-KEY": settings.AEON_API_KEY}) + "X-AEON-API-KEY": apikey}) + + def get_reading_rooms(self): + return self.get("ReadingRooms").json() + + def get_closures(self, reading_room_id): + return self.get("/".join(["ReadingRooms", str(reading_room_id), "Closures"])).json() diff --git a/process_request/helpers.py b/process_request/helpers.py index 6ee3f65..cd6f64f 100644 --- a/process_request/helpers.py +++ b/process_request/helpers.py @@ -8,6 +8,9 @@ from django.conf import settings from ordered_set import OrderedSet +from .clients import AeonAPIClient +from .models import ReadingRoomCache + CONFIDENCE_RATIO = 97 # Minimum confidence ratio to match against. OPEN_TEXT = ["Open for research", "Open for scholarly research"] CLOSED_TEXT = ["Restricted"] @@ -437,3 +440,15 @@ def get_formatted_resource_id(resource, client): Concatenates the resource id parts using the separator from the config """ return format_resource_id(resource, client, settings.RESOURCE_ID_SEPARATOR) + + +def refresh_reading_room_cache(): + """Gets reading room data from the Aeon API and stores it in the database as json + + Returns the newly saved object + """ + aeon = AeonAPIClient(baseurl=settings.AEON["baseurl"], apikey=settings.AEON["apikey"]) + reading_rooms = aeon.get_reading_rooms() + for reading_room in reading_rooms: + reading_room["closures"] = aeon.get_closures(reading_room["id"]) + return ReadingRoomCache.objects.update_or_create(defaults={'json': json.dumps(reading_rooms)})[0] diff --git a/process_request/migrations/0004_readingroomcache.py b/process_request/migrations/0004_readingroomcache.py new file mode 100644 index 0000000..6c484c7 --- /dev/null +++ b/process_request/migrations/0004_readingroomcache.py @@ -0,0 +1,21 @@ +# Generated by Django 4.0.6 on 2023-03-13 17:55 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('process_request', '0003_auto_20211106_1625'), + ] + + operations = [ + migrations.CreateModel( + name='ReadingRoomCache', + fields=[ + ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), + ('timestamp', models.DateTimeField(auto_now=True)), + ('json', models.TextField()), + ], + ), + ] diff --git a/process_request/models.py b/process_request/models.py index 89dd500..0815e45 100644 --- a/process_request/models.py +++ b/process_request/models.py @@ -1,4 +1,5 @@ from django.contrib.auth.models import AbstractUser +from django.db import models class User(AbstractUser): @@ -25,3 +26,8 @@ def __str__(self): Returns the full name and email of a user. """ return '{} <{}>'.format(self.full_name, self.email) + + +class ReadingRoomCache(models.Model): + timestamp = models.DateTimeField(auto_now=True) + json = models.TextField() diff --git a/process_request/views.py b/process_request/views.py index ab9b6ab..aab9059 100644 --- a/process_request/views.py +++ b/process_request/views.py @@ -1,15 +1,18 @@ import csv -from datetime import datetime +import threading +from datetime import datetime, timedelta from asnake.aspace import ASpace -from django.http import StreamingHttpResponse +from django.http import HttpResponse, StreamingHttpResponse from django.shortcuts import redirect +from django.utils import timezone from rest_framework.response import Response from rest_framework.views import APIView from request_broker import settings -from .helpers import resolve_ref_id +from .helpers import refresh_reading_room_cache, resolve_ref_id +from .models import ReadingRoomCache from .routines import AeonRequester, Mailer, Processor @@ -139,6 +142,24 @@ def get(self, request): return Response({"detail": str(e)}, status=500) +class AeonReadingRoomsView(APIView): + """Returns reading room information from Aeon""" + + def get(self, request): + try: + try: + cached_reading_rooms = ReadingRoomCache.objects.get() + cache_needs_refresh = cached_reading_rooms.timestamp + timedelta(minutes=settings.AEON["cache_duration"]) < timezone.now() + except ReadingRoomCache.DoesNotExist: + cached_reading_rooms = refresh_reading_room_cache() + cache_needs_refresh = False + if cache_needs_refresh: + threading.Thread(target=refresh_reading_room_cache).start() + return HttpResponse(cached_reading_rooms.json, content_type='application/json') + except Exception as e: + return Response({"detail": str(e)}, status=500) + + class PingView(APIView): """Checks if the application is able to process requests.""" diff --git a/request_broker/config.py.deploy b/request_broker/config.py.deploy index a1bfcbd..4f82f1c 100644 --- a/request_broker/config.py.deploy +++ b/request_broker/config.py.deploy @@ -23,4 +23,7 @@ DIMES_BASEURL = "${DIMES_BASEURL}" RESTRICTED_IN_CONTAINER = ${RESTRICTED_IN_CONTAINER} OFFSITE_BUILDINGS = ${OFFSITE_BUILDINGS} RESOURCE_ID_SEPARATOR = "${RESOURCE_ID_SEPARATOR}" +AEON_BASEURL = "${AEON_BASEURL}" +AEON_APIKEY = "${AEON_APIKEY}" +AEON_CACHE_DURATION = ${AEON_CACHE_DURATION} USE_LOCATION_TITLE = ${USE_LOCATION_TITLE} diff --git a/request_broker/config.py.example b/request_broker/config.py.example index c1d72bc..4494721 100644 --- a/request_broker/config.py.example +++ b/request_broker/config.py.example @@ -23,4 +23,7 @@ DIMES_BASEURL = "https://dimes.rockarch.org" # Base URL for DIMES application RESTRICTED_IN_CONTAINER = False # Fetch a list of restricted items in the same container as the requested item. OFFSITE_BUILDINGS = ["Armonk", "Greenrock"] # Names of offsite buildings, which will be added to locations (list of strings) RESOURCE_ID_SEPARATOR = ':' +AEON_BASEURL = "https://servername/aeon/api/" # base URL for an Aeon API instance +AEON_APIKEY = "00000000-0000-0000-0000-000000000000" # API key for the Aeon API +AEON_CACHE_DURATION = 7200 # cache duration in minutes USE_LOCATION_TITLE = False # Use the title field from a top container location diff --git a/request_broker/cron.py b/request_broker/cron.py new file mode 100644 index 0000000..940e2b3 --- /dev/null +++ b/request_broker/cron.py @@ -0,0 +1,12 @@ +from django_cron import CronJobBase, Schedule + +from process_request.helpers import refresh_reading_room_cache +from request_broker.settings import AEON + + +class RefreshReadingRoomCache(CronJobBase): + schedule = Schedule(run_every_mins=AEON["cache_duration"]) + code = 'request_broker.refresh_reading_room_cache' + + def do(self): + refresh_reading_room_cache() diff --git a/request_broker/settings.py b/request_broker/settings.py index 8a21c3c..30281af 100644 --- a/request_broker/settings.py +++ b/request_broker/settings.py @@ -39,6 +39,7 @@ 'django.contrib.sessions', 'django.contrib.messages', 'django.contrib.staticfiles', + 'django_cron', 'corsheaders', 'process_request', 'rest_framework', @@ -171,3 +172,13 @@ OFFSITE_BUILDINGS = getattr(config, 'OFFSITE_BUILDINGS', []) USE_LOCATION_TITLE = config.USE_LOCATION_TITLE RESOURCE_ID_SEPARATOR = config.RESOURCE_ID_SEPARATOR + +AEON = { + "baseurl": getattr(config, 'AEON_BASEURL', ''), + "apikey": getattr(config, 'AEON_APIKEY', ''), + "cache_duration": getattr(config, 'AEON_CACHE_DURATION', 7200), +} + +CRON_CLASSES = [ + "request_broker.cron.RefreshReadingRoomCache" +] diff --git a/request_broker/urls.py b/request_broker/urls.py index c3099cc..cfa9959 100644 --- a/request_broker/urls.py +++ b/request_broker/urls.py @@ -16,7 +16,8 @@ from django.contrib import admin from django.urls import path -from process_request.views import (DeliverDuplicationRequestView, +from process_request.views import (AeonReadingRoomsView, + DeliverDuplicationRequestView, DeliverReadingRoomRequestView, DownloadCSVView, LinkResolverView, MailerView, ParseBatchRequestView, @@ -31,5 +32,6 @@ path("api/process-request/parse-batch", ParseBatchRequestView.as_view(), name="parse-batch"), path("api/process-request/resolve", LinkResolverView.as_view(), name="resolve-request"), path("api/download-csv/", DownloadCSVView.as_view(), name="download-csv"), + path("api/reading-rooms/", AeonReadingRoomsView.as_view(), name="get-readingrooms"), path("api/status/", PingView.as_view(), name="ping") ] diff --git a/requirements.in b/requirements.in index 4792264..438ed7b 100644 --- a/requirements.in +++ b/requirements.in @@ -1,11 +1,12 @@ ArchivesSnake~=0.9 Django~=4.1 -django-cors-headers~=3.12 +django-cors-headers~=3.13 django-csp~=3.7 +django4-cron~=0.5 djangorestframework~=3.13 -inflect~=5.6 +inflect~=6.0.0 ordered-set~=4.1 psycopg2~=2.9 requests~=2.28 -shortuuid~=1.0 -vcrpy~=4.2 +shortuuid~=1.0.9 +vcrpy~=5.1 diff --git a/requirements.txt b/requirements.txt index 55946ac..e123d04 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,20 +19,26 @@ charset-normalizer==3.2.0 django==4.2.3 # via # -r requirements.in + # django-common-helpers # django-cors-headers # django-csp + # django4-cron # djangorestframework +django-common-helpers==0.9.2 + # via django4-cron django-cors-headers==3.14.0 # via -r requirements.in django-csp==3.7 # via -r requirements.in +django4-cron==0.5.1 + # via -r requirements.in djangorestframework==3.14.0 # via -r requirements.in idna==3.4 # via # requests # yarl -inflect==5.6.2 +inflect==6.0.5 # via -r requirements.in more-itertools==10.0.0 # via archivessnake @@ -42,6 +48,8 @@ ordered-set==4.1.0 # via -r requirements.in psycopg2==2.9.6 # via -r requirements.in +pydantic==1.10.12 + # via inflect pytz==2023.3 # via djangorestframework pyyaml==6.0.1 @@ -56,17 +64,17 @@ requests==2.31.0 # archivessnake shortuuid==1.0.11 # via -r requirements.in -six==1.16.0 - # via vcrpy sqlparse==0.4.4 # via django structlog==23.1.0 # via archivessnake typing-extensions==4.7.1 - # via asgiref + # via + # asgiref + # pydantic urllib3==2.0.4 # via requests -vcrpy==4.4.0 +vcrpy==5.1.0 # via -r requirements.in wrapt==1.15.0 # via vcrpy