Merge pull request #53 from freelawproject/35-add-redis-and-task-queue

feat(redis): Add Redis and django-rq

.env.example

@@ -19,6 +19,10 @@ MASTODON_PRIVATE_KEY=0
 PACER_USERNAME=""
 PACER_PASSWORD=""
 
+# redis.py
+REDIS_HOST=""
+REDIS_PORT=""
+
 # sentry.py
 SENTRY_DSN=""
 SENTRY_SAMPLE_TRACE=1.0

bc/core/tasks.py

@@ -0,0 +1,7 @@
+from django_rq import job
+
+
+@job
+def fail_task() -> float:
+    division_by_zero = 1 / 0
+    return division_by_zero

bc/core/urls.py

@@ -1,8 +1,9 @@
 from django.urls import path
 
-from .views import health_check, sentry_fail
+from .views import health_check, rq_fail, sentry_fail
 
 urlpatterns = [
     path("monitoring/health-check/", health_check, name="health_check"),
     path("sentry/error/", sentry_fail, name="sentry_fail"),
+    path("sentry/rq-fail/", rq_fail, name="rq_fail"),
 ]

bc/core/views.py

@@ -2,6 +2,7 @@
 
 from django.http import HttpRequest, HttpResponse, JsonResponse
 
+from .tasks import fail_task
 from .utils import check_postgresql
 
 
@@ -25,3 +26,8 @@ def health_check(request: HttpRequest) -> JsonResponse:
         },
         status=status,
     )
+
+
+def rq_fail(request: HttpRequest) -> HttpResponse:
+    fail_task.delay()
+    return HttpResponse("Successfully failed RQ.")

bc/settings/__init__.py

@@ -4,4 +4,6 @@
 from .third_party.courtlistener import *
 from .third_party.mastodon import *
 from .third_party.pacer import *
+from .third_party.redis import *
+from .third_party.rq import *
 from .third_party.sentry import *

bc/settings/django.py

@@ -2,6 +2,8 @@
 
 import environ
 
+from .third_party.redis import REDIS_DATABASES, REDIS_HOST, REDIS_PORT
+
 env = environ.FileAwareEnv()
 
 DEVELOPMENT = env.bool("DEVELOPMENT", default=True)
@@ -30,6 +32,8 @@
     "bc.channel",
     "bc.subscription",
     "bc.web",
+    # other apps
+    "django_rq",
 ]
 
 MIDDLEWARE = [
@@ -82,6 +86,18 @@
     },
 }
 
+####################
+# Cache & Sessions #
+####################
+CACHES = {
+    "default": {
+        "BACKEND": "django.core.cache.backends.redis.RedisCache",
+        "LOCATION": f"{REDIS_HOST}:{REDIS_PORT}",
+        "OPTIONS": {"db": REDIS_DATABASES["CACHE"]},
+    },
+}
+
+SESSION_ENGINE = "django.contrib.sessions.backends.cache"
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.1/topics/i18n/

bc/settings/third_party/redis.py

@@ -0,0 +1,12 @@
+import environ
+
+env = environ.FileAwareEnv()
+
+
+REDIS_HOST = env("REDIS_HOST", default="redis://bc2-redis")
+REDIS_PORT = env("REDIS_PORT", default=6379)
+
+REDIS_DATABASES = {
+    "QUEUE": 0,
+    "CACHE": 1,
+}

bc/settings/third_party/rq.py

@@ -0,0 +1,10 @@
+from .redis import REDIS_DATABASES, REDIS_HOST, REDIS_PORT
+
+RQ_SHOW_ADMIN_LINK = True
+
+RQ_QUEUES = {
+    "default": {
+        "URL": f"{REDIS_HOST}:{REDIS_PORT}",
+        "DB": REDIS_DATABASES["QUEUE"],
+    },
+}

bc/settings/third_party/sentry.py

@@ -1,7 +1,7 @@
 import environ
 import sentry_sdk
 from sentry_sdk.integrations.django import DjangoIntegration
-from sentry_sdk.integrations.logging import ignore_logger
+from sentry_sdk.integrations.rq import RqIntegration
 
 env = environ.FileAwareEnv()
 SENTRY_DSN = env("SENTRY_DSN", default="")
@@ -13,6 +13,7 @@
         dsn=SENTRY_DSN,
         integrations=[
             DjangoIntegration(),
+            RqIntegration(),
         ],
         ignore_errors=[KeyboardInterrupt],
         traces_sample_rate=SENTRY_SAMPLE_TRACE,

bc/urls.py

@@ -7,4 +7,5 @@
     path("", include("bc.core.urls")),
     path("", include("bc.channel.urls")),
     path("", include("bc.subscription.urls")),
+    path("django-rq/", include("django_rq.urls")),
 ]

docker/bigcasesbot/docker-compose.yml

@@ -26,6 +26,32 @@ services:
     networks:
       - bc2_net_overlay
 
+  bc2-redis:
+    container_name: bc2-redis
+    image: redis:7-alpine
+    ports:
+      - "6379:6379"
+    networks:
+      - bc2_net_overlay
+
+
+  bc2-rq:
+    container_name: bc2-rq
+    image: ${DJANGO_DOCKER_IMAGE:-freelawproject/bigcases2:latest-rq-dev}
+    build:
+      dockerfile: docker/django/Dockerfile
+      context: ../../
+      target: rq
+      args:
+        - BUILD_ENV=dev
+    depends_on:
+      - bc2-postgresql
+      - bc2-doctor
+      - bc2-redis
+    networks:
+      - bc2_net_overlay
+
+
   bc2-django:
     container_name: bc2-django
     image: ${DJANGO_DOCKER_IMAGE:-freelawproject/bigcases2:latest-web-dev}
@@ -38,6 +64,7 @@ services:
     depends_on:
       - bc2-postgresql
       - bc2-doctor
+      - bc2-redis
     volumes:
       - ../..:/opt/bigcases2
     ports:

docker/django/Dockerfile

@@ -82,6 +82,11 @@ RUN mkdir /var/log/bigcases2 \
 
 WORKDIR /opt/bigcases2
 
+# freelawproject/bigcases2:latest-rq
+FROM python-base as rq
+
+CMD python /opt/bigcases2/manage.py rqworker default --with-scheduler
+
 #freelawproject/bigcases2:latest-web-dev
 FROM python-base as web-dev
 

docker/django/Makefile

@@ -13,6 +13,9 @@ REPO ?= freelawproject/bigcases2
 DOCKER_TAG_PROD = $(VERSION)-web-prod
 WEB_PROD ?= latest-web-prod
 WEB_DEV ?= latest-web-dev
+RQ_TAG = $(VERSION)-rq
+RQ_LATEST ?= latest-rq
+RQ_DEV ?= latest-rq-dev
 
 
 .PHONY: all image push multiarch_image multiarch_push x86_image x86_push
@@ -23,9 +26,11 @@ all: image
 
 development:
 	docker build --target web-dev -t $(REPO):$(WEB_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile .
+	docker build --target rq -t $(REPO):$(RQ_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile .
 
 image:
 	docker build --target web-prod -t $(REPO):$(DOCKER_TAG_PROD) -t $(REPO):$(WEB_PROD) --file docker/django/Dockerfile .
+	docker build --target rq -t $(REPO):$(RQ_TAG) -t $(REPO):$(RQ_LATEST) --file docker/django/Dockerfile .
 
 push: image
 	$(info Checking if valid architecture)

pyproject.toml

@@ -39,6 +39,8 @@ djangorestframework = "^3.14.0"
 django-environ = "^0.9.0"
 sentry-sdk = "^1.14.0"
 django-csp = "^3.7"
+redis = "^4.4.2"
+django-rq = "^2.6.0"
 gunicorn = "^20.1.0"
 
 [tool.poetry.dev-dependencies]