Merge pull request #302 from RockefellerArchiveCenter/development

Add GET endpoint for reading room information

Dockerfile.prod

@@ -0,0 +1,39 @@
+FROM python:3.10
+
+ENV PYTHONUNBUFFERED 1
+ARG REQUEST_BROKER_DNS
+ARG APPLICATION_PORT
+RUN apt-get update
+RUN apt-get install --yes apache2 apache2-dev 
+RUN apt-get install --yes postgresql
+RUN apt-get -y install python3-pip
+RUN apt-get -y install cron
+RUN pip install --upgrade pip
+
+RUN wget https://github.com/GrahamDumpleton/mod_wsgi/archive/refs/tags/4.9.0.tar.gz \
+    && tar xvfz 4.9.0.tar.gz \
+    && cd mod_wsgi-4.9.0 \
+    && ./configure --with-apxs=/usr/bin/apxs --with-python=/usr/local/bin/python \
+    && make \
+    && make install \
+    && make clean
+
+ADD ./apache/000-request_broker.conf /etc/apache2/sites-available/000-request_broker.conf
+RUN sed "s/ENV_REQUEST_BROKER_DNS/${REQUEST_BROKER_DNS}/" -i /etc/apache2/sites-available/000-request_broker.conf
+RUN sed "s/ENV_REQUEST_BROKER_PORT/${APPLICATION_PORT}/" -i /etc/apache2/sites-available/000-request_broker.conf
+ADD ./apache/wsgi.load /etc/apache2/mods-available/wsgi.load
+RUN a2ensite 000-request_broker.conf
+RUN a2enmod headers
+RUN a2enmod rewrite
+RUN a2enmod wsgi
+
+COPY . /var/www/request-broker
+WORKDIR /var/www/request-broker
+RUN pip install -r requirements.txt
+
+COPY --chmod=644 crontab /etc/cron.d/crontab
+RUN crontab /etc/cron.d/crontab
+
+EXPOSE ${APPLICATION_PORT}
+
+ENTRYPOINT ["./entrypoint.prod.sh"]

README.md

@@ -33,7 +33,7 @@ Or, if you want to remove all data
 
 The request broker manages configuration by setting environment variables. These variables can be seen in `docker-compose.yml`.
 
-Deployment using the `docker-compose.prod.yml` or `docker-compose.dev.yml` files requires the presence of an `.env.prod` or `.env.dev` file in the root directory of the application. The environment variables included in those files should match the variables in `docker-compose.yml`, although the values assigned to those variables may change.
+Deployment using the `Dockerfile.prod` file is intended to bring up a production image (based on Apache/WSGI) which is ready to be proxied publicly by an apache, nginx, traefik or similar frontend.  `Dockerfile.prod` expects two environment arguments to be available at build time: `REQUEST_BROKER_DNS` and `APPLICATION_PORT`.  Apache will Listen on `${APPLICATION_PORT}` with a ServerName of `${REQUEST_BROKER_DNS}`.
 
 ## Services
 

apache/000-request_broker.conf

@@ -0,0 +1,19 @@
+Listen ENV_REQUEST_BROKER_PORT
+<VirtualHost *:ENV_REQUEST_BROKER_PORT>
+    ErrorLog /var/log/apache2/request_broker_error_log
+    CustomLog /var/log/apache2/request_broker_access_log combined
+    ServerName ENV_REQUEST_BROKER_DNS
+    DocumentRoot /var/www/html/
+    Alias /static /var/www/request-broker/static
+    <Directory /var/www/request-broker/static>
+        Require all granted
+    </Directory>
+    <Directory /var/www/request-broker/request_broker>
+        WSGIProcessGroup request_broker
+        WSGIApplicationGroup %{GLOBAL}
+        Require all granted
+    </Directory>
+    WSGIDaemonProcess request_broker home=/var/www/request-broker
+    WSGIProcessGroup request_broker
+    WSGIScriptAlias / /var/www/request-broker/request_broker/wsgi.py
+</VirtualHost>

apache/wsgi.load

@@ -0,0 +1 @@
+LoadModule wsgi_module /usr/lib/apache2/modules/mod_wsgi.so

crontab

@@ -0,0 +1,2 @@
+*/5 * * * * /usr/local/bin/python3 -u /var/www/request-broker/manage.py runcrons >/proc/1/fd/1 2>/proc/1/fd/2
+

entrypoint.prod.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+./wait-for-it.sh db:${SQL_PORT} -- echo "Apply database migrations"
+python manage.py migrate
+
+# Collect static files
+echo "Collecting static files"
+python manage.py collectstatic
+
+#Start cron
+echo "Starting cron"
+cron
+
+#Start server
+echo "Starting server"
+apache2ctl -D FOREGROUND

entrypoint.sh

@@ -8,7 +8,7 @@ if [ ! -f request_broker/config.py ]; then
     cp request_broker/config.py.example request_broker/config.py
 fi
 
-./wait-for-it.sh db:5432 -- echo "Apply database migrations"
+./wait-for-it.sh db:${SQL_PORT} -- echo "Apply database migrations"
 python manage.py migrate
 
 #Start server

process_request/clients.py

@@ -1,7 +1,5 @@
 from requests import Session
 
-from request_broker import settings
-
 
 def http_meth_factory(meth):
     """Utility method for producing HTTP proxy methods.
@@ -27,10 +25,15 @@ def __init__(cls, name, parents, dct):
 
 class AeonAPIClient(metaclass=ProxyMethods):
 
-    def __init__(self, baseurl):
+    def __init__(self, baseurl, apikey):
         self.baseurl = baseurl
         self.session = Session()
         self.session.headers.update(
             {"Accept": "application/json",
-             "User-Agent": "AeonAPIClient/0.1",
-             "X-AEON-API-KEY": settings.AEON_API_KEY})
+             "X-AEON-API-KEY": apikey})
+
+    def get_reading_rooms(self):
+        return self.get("ReadingRooms").json()
+
+    def get_closures(self, reading_room_id):
+        return self.get("/".join(["ReadingRooms", str(reading_room_id), "Closures"])).json()

process_request/helpers.py

@@ -8,6 +8,9 @@
 from django.conf import settings
 from ordered_set import OrderedSet
 
+from .clients import AeonAPIClient
+from .models import ReadingRoomCache
+
 CONFIDENCE_RATIO = 97  # Minimum confidence ratio to match against.
 OPEN_TEXT = ["Open for research", "Open for scholarly research"]
 CLOSED_TEXT = ["Restricted"]
@@ -437,3 +440,15 @@ def get_formatted_resource_id(resource, client):
     Concatenates the resource id parts using the separator from the config
     """
     return format_resource_id(resource, client, settings.RESOURCE_ID_SEPARATOR)
+
+
+def refresh_reading_room_cache():
+    """Gets reading room data from the Aeon API and stores it in the database as json
+
+    Returns the newly saved object
+    """
+    aeon = AeonAPIClient(baseurl=settings.AEON["baseurl"], apikey=settings.AEON["apikey"])
+    reading_rooms = aeon.get_reading_rooms()
+    for reading_room in reading_rooms:
+        reading_room["closures"] = aeon.get_closures(reading_room["id"])
+    return ReadingRoomCache.objects.update_or_create(defaults={'json': json.dumps(reading_rooms)})[0]

process_request/migrations/0004_readingroomcache.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.0.6 on 2023-03-13 17:55
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('process_request', '0003_auto_20211106_1625'),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name='ReadingRoomCache',
+            fields=[
+                ('id', models.BigAutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+                ('timestamp', models.DateTimeField(auto_now=True)),
+                ('json', models.TextField()),
+            ],
+        ),
+    ]

process_request/models.py

@@ -1,4 +1,5 @@
 from django.contrib.auth.models import AbstractUser
+from django.db import models
 
 
 class User(AbstractUser):
@@ -25,3 +26,8 @@ def __str__(self):
         Returns the full name and email of a user.
         """
         return '{} <{}>'.format(self.full_name, self.email)
+
+
+class ReadingRoomCache(models.Model):
+    timestamp = models.DateTimeField(auto_now=True)
+    json = models.TextField()

process_request/views.py

@@ -1,15 +1,18 @@
 import csv
-from datetime import datetime
+import threading
+from datetime import datetime, timedelta
 
 from asnake.aspace import ASpace
-from django.http import StreamingHttpResponse
+from django.http import HttpResponse, StreamingHttpResponse
 from django.shortcuts import redirect
+from django.utils import timezone
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
 from request_broker import settings
 
-from .helpers import resolve_ref_id
+from .helpers import refresh_reading_room_cache, resolve_ref_id
+from .models import ReadingRoomCache
 from .routines import AeonRequester, Mailer, Processor
 
 
@@ -139,6 +142,24 @@ def get(self, request):
             return Response({"detail": str(e)}, status=500)
 
 
+class AeonReadingRoomsView(APIView):
+    """Returns reading room information from Aeon"""
+
+    def get(self, request):
+        try:
+            try:
+                cached_reading_rooms = ReadingRoomCache.objects.get()
+                cache_needs_refresh = cached_reading_rooms.timestamp + timedelta(minutes=settings.AEON["cache_duration"]) < timezone.now()
+            except ReadingRoomCache.DoesNotExist:
+                cached_reading_rooms = refresh_reading_room_cache()
+                cache_needs_refresh = False
+            if cache_needs_refresh:
+                threading.Thread(target=refresh_reading_room_cache).start()
+            return HttpResponse(cached_reading_rooms.json, content_type='application/json')
+        except Exception as e:
+            return Response({"detail": str(e)}, status=500)
+
+
 class PingView(APIView):
     """Checks if the application is able to process requests."""
 

request_broker/config.py.deploy

@@ -23,4 +23,7 @@ DIMES_BASEURL = "${DIMES_BASEURL}"
 RESTRICTED_IN_CONTAINER = ${RESTRICTED_IN_CONTAINER}
 OFFSITE_BUILDINGS = ${OFFSITE_BUILDINGS}
 RESOURCE_ID_SEPARATOR = "${RESOURCE_ID_SEPARATOR}"
+AEON_BASEURL = "${AEON_BASEURL}"
+AEON_APIKEY = "${AEON_APIKEY}"
+AEON_CACHE_DURATION = ${AEON_CACHE_DURATION}
 USE_LOCATION_TITLE = ${USE_LOCATION_TITLE}

request_broker/config.py.example

@@ -23,4 +23,7 @@ DIMES_BASEURL = "https://dimes.rockarch.org" # Base URL for DIMES application
 RESTRICTED_IN_CONTAINER = False  # Fetch a list of restricted items in the same container as the requested item.
 OFFSITE_BUILDINGS = ["Armonk", "Greenrock"]  # Names of offsite buildings, which will be added to locations (list of strings)
 RESOURCE_ID_SEPARATOR = ':'
+AEON_BASEURL = "https://servername/aeon/api/" # base URL for an Aeon API instance
+AEON_APIKEY = "00000000-0000-0000-0000-000000000000" # API key for the Aeon API
+AEON_CACHE_DURATION = 7200 # cache duration in minutes
 USE_LOCATION_TITLE = False  # Use the title field from a top container location

request_broker/cron.py

@@ -0,0 +1,12 @@
+from django_cron import CronJobBase, Schedule
+
+from process_request.helpers import refresh_reading_room_cache
+from request_broker.settings import AEON
+
+
+class RefreshReadingRoomCache(CronJobBase):
+    schedule = Schedule(run_every_mins=AEON["cache_duration"])
+    code = 'request_broker.refresh_reading_room_cache'
+
+    def do(self):
+        refresh_reading_room_cache()

request_broker/settings.py

@@ -39,6 +39,7 @@
     'django.contrib.sessions',
     'django.contrib.messages',
     'django.contrib.staticfiles',
+    'django_cron',
     'corsheaders',
     'process_request',
     'rest_framework',
@@ -171,3 +172,13 @@
 OFFSITE_BUILDINGS = getattr(config, 'OFFSITE_BUILDINGS', [])
 USE_LOCATION_TITLE = config.USE_LOCATION_TITLE
 RESOURCE_ID_SEPARATOR = config.RESOURCE_ID_SEPARATOR
+
+AEON = {
+    "baseurl": getattr(config, 'AEON_BASEURL', ''),
+    "apikey": getattr(config, 'AEON_APIKEY', ''),
+    "cache_duration": getattr(config, 'AEON_CACHE_DURATION', 7200),
+}
+
+CRON_CLASSES = [
+    "request_broker.cron.RefreshReadingRoomCache"
+]

request_broker/urls.py

@@ -16,7 +16,8 @@
 from django.contrib import admin
 from django.urls import path
 
-from process_request.views import (DeliverDuplicationRequestView,
+from process_request.views import (AeonReadingRoomsView,
+                                   DeliverDuplicationRequestView,
                                    DeliverReadingRoomRequestView,
                                    DownloadCSVView, LinkResolverView,
                                    MailerView, ParseBatchRequestView,
@@ -31,5 +32,6 @@
     path("api/process-request/parse-batch", ParseBatchRequestView.as_view(), name="parse-batch"),
     path("api/process-request/resolve", LinkResolverView.as_view(), name="resolve-request"),
     path("api/download-csv/", DownloadCSVView.as_view(), name="download-csv"),
+    path("api/reading-rooms/", AeonReadingRoomsView.as_view(), name="get-readingrooms"),
     path("api/status/", PingView.as_view(), name="ping")
 ]

requirements.in

@@ -1,11 +1,12 @@
 ArchivesSnake~=0.9
 Django~=4.1
-django-cors-headers~=3.12
+django-cors-headers~=3.13
 django-csp~=3.7
+django4-cron~=0.5
 djangorestframework~=3.13
-inflect~=5.6
+inflect~=6.0.0
 ordered-set~=4.1
 psycopg2~=2.9
 requests~=2.28
-shortuuid~=1.0
-vcrpy~=4.2
+shortuuid~=1.0.9
+vcrpy~=5.1

requirements.txt

@@ -19,20 +19,26 @@ charset-normalizer==3.2.0
 django==4.2.3
     # via
     #   -r requirements.in
+    #   django-common-helpers
     #   django-cors-headers
     #   django-csp
+    #   django4-cron
     #   djangorestframework
+django-common-helpers==0.9.2
+    # via django4-cron
 django-cors-headers==3.14.0
     # via -r requirements.in
 django-csp==3.7
     # via -r requirements.in
+django4-cron==0.5.1
+    # via -r requirements.in
 djangorestframework==3.14.0
     # via -r requirements.in
 idna==3.4
     # via
     #   requests
     #   yarl
-inflect==5.6.2
+inflect==6.0.5
     # via -r requirements.in
 more-itertools==10.0.0
     # via archivessnake
@@ -42,6 +48,8 @@ ordered-set==4.1.0
     # via -r requirements.in
 psycopg2==2.9.6
     # via -r requirements.in
+pydantic==1.10.12
+    # via inflect
 pytz==2023.3
     # via djangorestframework
 pyyaml==6.0.1
@@ -56,17 +64,17 @@ requests==2.31.0
     #   archivessnake
 shortuuid==1.0.11
     # via -r requirements.in
-six==1.16.0
-    # via vcrpy
 sqlparse==0.4.4
     # via django
 structlog==23.1.0
     # via archivessnake
 typing-extensions==4.7.1
-    # via asgiref
+    # via
+    #   asgiref
+    #   pydantic
 urllib3==2.0.4
     # via requests
-vcrpy==4.4.0
+vcrpy==5.1.0
     # via -r requirements.in
 wrapt==1.15.0
     # via vcrpy