diff --git a/.env.example b/.env.example index da5df252..6c1ac3c7 100644 --- a/.env.example +++ b/.env.example @@ -19,6 +19,10 @@ MASTODON_PRIVATE_KEY=0 PACER_USERNAME="" PACER_PASSWORD="" +# redis.py +REDIS_HOST="" +REDIS_PORT="" + # sentry.py SENTRY_DSN="" SENTRY_SAMPLE_TRACE=1.0 diff --git a/bc/core/tasks.py b/bc/core/tasks.py new file mode 100644 index 00000000..d148a9ea --- /dev/null +++ b/bc/core/tasks.py @@ -0,0 +1,7 @@ +from django_rq import job + + +@job +def fail_task() -> float: + division_by_zero = 1 / 0 + return division_by_zero diff --git a/bc/core/urls.py b/bc/core/urls.py index 63024eec..f064e473 100644 --- a/bc/core/urls.py +++ b/bc/core/urls.py @@ -1,8 +1,9 @@ from django.urls import path -from .views import health_check, sentry_fail +from .views import health_check, rq_fail, sentry_fail urlpatterns = [ path("monitoring/health-check/", health_check, name="health_check"), path("sentry/error/", sentry_fail, name="sentry_fail"), + path("sentry/rq-fail/", rq_fail, name="rq_fail"), ] diff --git a/bc/core/views.py b/bc/core/views.py index 1287d3e8..b009f933 100644 --- a/bc/core/views.py +++ b/bc/core/views.py @@ -2,6 +2,7 @@ from django.http import HttpRequest, HttpResponse, JsonResponse +from .tasks import fail_task from .utils import check_postgresql @@ -25,3 +26,8 @@ def health_check(request: HttpRequest) -> JsonResponse: }, status=status, ) + + +def rq_fail(request: HttpRequest) -> HttpResponse: + fail_task.delay() + return HttpResponse("Successfully failed RQ.") diff --git a/bc/settings/__init__.py b/bc/settings/__init__.py index b5c43e15..6358d052 100644 --- a/bc/settings/__init__.py +++ b/bc/settings/__init__.py @@ -4,4 +4,6 @@ from .third_party.courtlistener import * from .third_party.mastodon import * from .third_party.pacer import * +from .third_party.redis import * +from .third_party.rq import * from .third_party.sentry import * diff --git a/bc/settings/django.py b/bc/settings/django.py index 5806619e..a8f9041c 100644 --- a/bc/settings/django.py +++ b/bc/settings/django.py @@ -2,6 +2,8 @@ import environ +from .third_party.redis import REDIS_DATABASES, REDIS_HOST, REDIS_PORT + env = environ.FileAwareEnv() DEVELOPMENT = env.bool("DEVELOPMENT", default=True) @@ -30,6 +32,8 @@ "bc.channel", "bc.subscription", "bc.web", + # other apps + "django_rq", ] MIDDLEWARE = [ @@ -82,6 +86,18 @@ }, } +#################### +# Cache & Sessions # +#################### +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": f"{REDIS_HOST}:{REDIS_PORT}", + "OPTIONS": {"db": REDIS_DATABASES["CACHE"]}, + }, +} + +SESSION_ENGINE = "django.contrib.sessions.backends.cache" # Internationalization # https://docs.djangoproject.com/en/4.1/topics/i18n/ diff --git a/bc/settings/third_party/redis.py b/bc/settings/third_party/redis.py new file mode 100644 index 00000000..bebb0a78 --- /dev/null +++ b/bc/settings/third_party/redis.py @@ -0,0 +1,12 @@ +import environ + +env = environ.FileAwareEnv() + + +REDIS_HOST = env("REDIS_HOST", default="redis://bc2-redis") +REDIS_PORT = env("REDIS_PORT", default=6379) + +REDIS_DATABASES = { + "QUEUE": 0, + "CACHE": 1, +} diff --git a/bc/settings/third_party/rq.py b/bc/settings/third_party/rq.py new file mode 100644 index 00000000..b9536589 --- /dev/null +++ b/bc/settings/third_party/rq.py @@ -0,0 +1,10 @@ +from .redis import REDIS_DATABASES, REDIS_HOST, REDIS_PORT + +RQ_SHOW_ADMIN_LINK = True + +RQ_QUEUES = { + "default": { + "URL": f"{REDIS_HOST}:{REDIS_PORT}", + "DB": REDIS_DATABASES["QUEUE"], + }, +} diff --git a/bc/settings/third_party/sentry.py b/bc/settings/third_party/sentry.py index b3c389cc..664045ff 100644 --- a/bc/settings/third_party/sentry.py +++ b/bc/settings/third_party/sentry.py @@ -1,7 +1,7 @@ import environ import sentry_sdk from sentry_sdk.integrations.django import DjangoIntegration -from sentry_sdk.integrations.logging import ignore_logger +from sentry_sdk.integrations.rq import RqIntegration env = environ.FileAwareEnv() SENTRY_DSN = env("SENTRY_DSN", default="") @@ -13,6 +13,7 @@ dsn=SENTRY_DSN, integrations=[ DjangoIntegration(), + RqIntegration(), ], ignore_errors=[KeyboardInterrupt], traces_sample_rate=SENTRY_SAMPLE_TRACE, diff --git a/bc/urls.py b/bc/urls.py index 03fb9ac5..5a403939 100644 --- a/bc/urls.py +++ b/bc/urls.py @@ -7,4 +7,5 @@ path("", include("bc.core.urls")), path("", include("bc.channel.urls")), path("", include("bc.subscription.urls")), + path("django-rq/", include("django_rq.urls")), ] diff --git a/docker/bigcasesbot/docker-compose.yml b/docker/bigcasesbot/docker-compose.yml index d6a4d5a8..bb9d1738 100644 --- a/docker/bigcasesbot/docker-compose.yml +++ b/docker/bigcasesbot/docker-compose.yml @@ -26,6 +26,32 @@ services: networks: - bc2_net_overlay + bc2-redis: + container_name: bc2-redis + image: redis:7-alpine + ports: + - "6379:6379" + networks: + - bc2_net_overlay + + + bc2-rq: + container_name: bc2-rq + image: ${DJANGO_DOCKER_IMAGE:-freelawproject/bigcases2:latest-rq-dev} + build: + dockerfile: docker/django/Dockerfile + context: ../../ + target: rq + args: + - BUILD_ENV=dev + depends_on: + - bc2-postgresql + - bc2-doctor + - bc2-redis + networks: + - bc2_net_overlay + + bc2-django: container_name: bc2-django image: ${DJANGO_DOCKER_IMAGE:-freelawproject/bigcases2:latest-web-dev} @@ -38,6 +64,7 @@ services: depends_on: - bc2-postgresql - bc2-doctor + - bc2-redis volumes: - ../..:/opt/bigcases2 ports: diff --git a/docker/django/Dockerfile b/docker/django/Dockerfile index 4cf468a6..f6332211 100644 --- a/docker/django/Dockerfile +++ b/docker/django/Dockerfile @@ -82,6 +82,11 @@ RUN mkdir /var/log/bigcases2 \ WORKDIR /opt/bigcases2 +# freelawproject/bigcases2:latest-rq +FROM python-base as rq + +CMD python /opt/bigcases2/manage.py rqworker default --with-scheduler + #freelawproject/bigcases2:latest-web-dev FROM python-base as web-dev diff --git a/docker/django/Makefile b/docker/django/Makefile index 8a8c13e7..d56b2b52 100644 --- a/docker/django/Makefile +++ b/docker/django/Makefile @@ -13,6 +13,9 @@ REPO ?= freelawproject/bigcases2 DOCKER_TAG_PROD = $(VERSION)-web-prod WEB_PROD ?= latest-web-prod WEB_DEV ?= latest-web-dev +RQ_TAG = $(VERSION)-rq +RQ_LATEST ?= latest-rq +RQ_DEV ?= latest-rq-dev .PHONY: all image push multiarch_image multiarch_push x86_image x86_push @@ -23,9 +26,11 @@ all: image development: docker build --target web-dev -t $(REPO):$(WEB_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile . + docker build --target rq -t $(REPO):$(RQ_DEV) --build-arg BUILD_ENV=dev --file docker/django/Dockerfile . image: docker build --target web-prod -t $(REPO):$(DOCKER_TAG_PROD) -t $(REPO):$(WEB_PROD) --file docker/django/Dockerfile . + docker build --target rq -t $(REPO):$(RQ_TAG) -t $(REPO):$(RQ_LATEST) --file docker/django/Dockerfile . push: image $(info Checking if valid architecture) diff --git a/poetry.lock b/poetry.lock index df65affa..1c0fb6c5 100644 --- a/poetry.lock +++ b/poetry.lock @@ -70,6 +70,18 @@ files = [ lazy-object-proxy = ">=1.4.0" wrapt = {version = ">=1.14,<2", markers = "python_version >= \"3.11\""} +[[package]] +name = "async-timeout" +version = "4.0.2" +description = "Timeout context manager for asyncio programs" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "async-timeout-4.0.2.tar.gz", hash = "sha256:2163e1640ddb52b7a8c80d0a67a08587e5d245cc9c553a74a847056bc2976b15"}, + {file = "async_timeout-4.0.2-py3-none-any.whl", hash = "sha256:8ca1e4fcf50d07413d66d1a5e416e42cfdf5851c981d679a09851a6853383b3c"}, +] + [[package]] name = "black" version = "23.1.0" @@ -336,7 +348,7 @@ files = [ name = "click" version = "8.1.3" description = "Composable command line interface toolkit" -category = "dev" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -351,7 +363,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" files = [ @@ -511,6 +523,27 @@ develop = ["coverage[toml] (>=5.0a4)", "furo (>=2021.8.17b43,<2021.9.0)", "pytes docs = ["furo (>=2021.8.17b43,<2021.9.0)", "sphinx (>=3.5.0)", "sphinx-notfound-page"] testing = ["coverage[toml] (>=5.0a4)", "pytest (>=4.6.11)"] +[[package]] +name = "django-rq" +version = "2.6.0" +description = "An app that provides django integration for RQ (Redis Queue)" +category = "main" +optional = false +python-versions = "*" +files = [ + {file = "django-rq-2.6.0.tar.gz", hash = "sha256:f03b1eb68afe218175989c14b1266c7b446d5152f629888a078d0022059b255e"}, + {file = "django_rq-2.6.0-py2.py3-none-any.whl", hash = "sha256:b1964ad656ae103d8b4714f8745e739586917b126f02c844791d9059940b01a0"}, +] + +[package.dependencies] +django = ">=2.0" +redis = ">=3" +rq = ">=1.2" + +[package.extras] +sentry = ["raven (>=6.1.0)"] +testing = ["mock (>=2.0.0)"] + [[package]] name = "django-stubs" version = "1.14.0" @@ -1182,6 +1215,25 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "redis" +version = "4.4.2" +description = "Python client for Redis database and key-value store" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "redis-4.4.2-py3-none-any.whl", hash = "sha256:e6206448e2f8a432871d07d432c13ed6c2abcf6b74edb436c99752b1371be387"}, + {file = "redis-4.4.2.tar.gz", hash = "sha256:a010f6cb7378065040a02839c3f75c7e0fb37a87116fb4a95be82a95552776c7"}, +] + +[package.dependencies] +async-timeout = ">=4.0.2" + +[package.extras] +hiredis = ["hiredis (>=1.0.0)"] +ocsp = ["cryptography (>=36.0.1)", "pyopenssl (==20.0.1)", "requests (>=2.26.0)"] + [[package]] name = "requests" version = "2.28.2" @@ -1204,6 +1256,22 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "rq" +version = "1.12.0" +description = "RQ is a simple, lightweight, library for creating background jobs, and processing them." +category = "main" +optional = false +python-versions = ">=3.5" +files = [ + {file = "rq-1.12.0-py2.py3-none-any.whl", hash = "sha256:b268947a94a1da7de3c5f3925a59db60bffdede782ca1f23da654bf985a83c7a"}, + {file = "rq-1.12.0.tar.gz", hash = "sha256:16ebbfa8891ece999485cb7d1e0559550ac576da43585138e6951db23654bbf6"}, +] + +[package.dependencies] +click = ">=5.0.0" +redis = ">=3.5.0" + [[package]] name = "sentry-sdk" version = "1.14.0" diff --git a/pyproject.toml b/pyproject.toml index d2f9b437..235cba61 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -39,6 +39,8 @@ djangorestframework = "^3.14.0" django-environ = "^0.9.0" sentry-sdk = "^1.14.0" django-csp = "^3.7" +redis = "^4.4.2" +django-rq = "^2.6.0" gunicorn = "^20.1.0" [tool.poetry.dev-dependencies]