Remove RabbitMQ and use Redis as Celery backend

TODO.md

@@ -13,7 +13,6 @@
 
 - Upgrade REDIS server on RADIS
 - Unfix pyright and its VS code extension
-- Move over to Celery with Redis backend and get rid of RabbitMQ
 - Replace sherlock on RADIS
 - Use DicomLogEntry during C-STORE
 - Allow to restart or cancel specific dicom task
@@ -116,7 +115,7 @@
 - Get rid of 7z archive feature. It think it was never used.
 - Allow to search multiple source servers with one query (maybe only in dicom explorer)
 - Bring everything behind Nginx as reverse proxy
-  -- Orthanc, Flower, Rabbit Management Console should then be directly behind Nginx (without Django-revproxy)
+  -- Orthanc and Flower should then be directly behind Nginx (without Django-revproxy)
   -- Use authentication module of nginx
   -- <http://nginx.org/en/docs/http/ngx_http_auth_request_module.html>
   -- <https://stackoverflow.com/a/70961666/166229>
@@ -144,7 +143,7 @@
 - Rewrite dicom_connector to use asyncio (wrap all pynetdicom calls in asyncio.to_thread)
   -- I don't think that this will gain any performance improvements, so maybe not worth it
 - Look out for a django-revproxy fix (see <https://github.com/jazzband/django-revproxy/issues/144>)
-  -- Flower and Rabbit Management Console are running behind a Django internal reverse proxy (django-revproxy) so that only admins can access them
+  -- Flower is running behind a Django internal reverse proxy (django-revproxy) so that only admins can access them
   -- Unfortunately the last released django-revproxy is broken with latest Django
   -- So we use the master branch here directly from Github (see pyproject.toml)
   -- Alternatively we could use <https://github.com/thomasw/djproxy>

adit/core/templates/core/admin_section.html

@@ -37,9 +37,6 @@ <h5>Admin Tools</h5>
         <li class="list-group-item">
             <a href="/flower/">Flower (Celery Monitoring)</a>
         </li>
-        <li class="list-group-item">
-            <a href="/rabbit/">RabbitMQ Management Console</a>
-        </li>
         <li class="list-group-item">
             <a href="/orthanc1/">Orthanc 1</a>
         </li>

adit/core/urls.py

@@ -6,7 +6,6 @@
     HomeView,
     Orthanc1ProxyView,
     Orthanc2ProxyView,
-    RabbitManagementProxyView,
     UpdatePreferencesView,
     admin_section,
 )
@@ -31,7 +30,6 @@
         HomeView.as_view(),
         name="home",
     ),
-    RabbitManagementProxyView.as_url(),
     FlowerProxyView.as_url(),
     Orthanc1ProxyView.as_url(),
     Orthanc2ProxyView.as_url(),

adit/core/views.py

@@ -441,15 +441,6 @@ def as_url(cls):
         return re_path(rf"^{cls.url_prefix}/(?P<path>.*)$", cls.as_view())  # type: ignore
 
 
-class RabbitManagementProxyView(AdminProxyView):
-    upstream = (
-        f"http://{settings.RABBIT_MANAGEMENT_HOST}:"
-        f"{settings.RABBIT_MANAGEMENT_PORT}"  # type: ignore
-    )
-    url_prefix = "rabbit"
-    rewrite = ((rf"^/{url_prefix}$", r"/"),)
-
-
 class FlowerProxyView(AdminProxyView):
     upstream = f"http://{settings.FLOWER_HOST}:{settings.FLOWER_PORT}"  # type: ignore
     url_prefix = "flower"

adit/settings/base.py

@@ -295,21 +295,14 @@
 # Channels
 ASGI_APPLICATION = "adit.asgi.application"
 
-# RabbitMQ is used as Celery message broker
-RABBITMQ_URL = env.str("RABBITMQ_URL", default="amqp://localhost")  # type: ignore
-
-# Rabbit Management console is integrated in ADIT by using an reverse
-# proxy (django-revproxy).This allows to use the authentication of ADIT.
-# But as RabbitMQ authentication can't be disabled we have to login
-# there with "guest" as username and password again.
-RABBIT_MANAGEMENT_HOST = env.str("RABBIT_MANAGEMENT_HOST", default="localhost")  # type: ignore
-RABBIT_MANAGEMENT_PORT = env.int("RABBIT_MANAGEMENT_PORT", default=15672)  # type: ignore
+# Redis is used as the Celery backend and for distributed locks
+REDIS_URL = env.str("REDIS_URL", default="redis://localhost:6379/0")  # type: ignore
 
 # Celery
 # see https://github.com/celery/celery/issues/5026 for how to name configs
 if USE_TZ:
     CELERY_TIMEZONE = TIME_ZONE
-CELERY_BROKER_URL = RABBITMQ_URL
+CELERY_BROKER_URL = REDIS_URL
 CELERY_WORKER_HIJACK_ROOT_LOGGER = False
 CELERY_IGNORE_RESULT = True
 CELERY_TASK_DEFAULT_QUEUE = "default_queue"
@@ -325,9 +318,6 @@
 FLOWER_HOST = env.str("FLOWER_HOST", default="localhost")  # type: ignore
 FLOWER_PORT = env.int("FLOWER_PORT", default=5555)  # type: ignore
 
-# Redis is used for distributed locks
-REDIS_URL = env.str("REDIS_URL", default="redis://localhost:6379/0")  # type: ignore
-
 # Orthanc servers are integrated in ADIT by using a reverse proxy (django-revproxy).
 ORTHANC1_HOST = env.str("ORTHANC1_HOST", default="localhost")  # type: ignore
 ORTHANC1_HTTP_PORT = env.int("ORTHANC1_HTTP_PORT", default=6501)  # type: ignore

compose/docker-compose.base.yml

@@ -5,16 +5,12 @@ x-app: &default-app
     - /mnt:/mnt
   depends_on:
     - postgres
-    - rabbit
     - redis
   environment:
     USE_DOCKER: 1
     DATABASE_URL: "psql://postgres:postgres@postgres:5432/postgres"
-    RABBITMQ_URL: "amqp://rabbit"
     REDIS_URL: "redis://redis:6379/0"
     DJANGO_STATIC_ROOT: "/var/www/adit/static/"
-    RABBIT_MANAGEMENT_HOST: "rabbit"
-    RABBIT_MANAGEMENT_PORT: "15672"
     FLOWER_HOST: "flower"
     FLOWER_PORT: "5555"
     ORTHANC1_HOST: "orthanc1"
@@ -65,25 +61,10 @@ services:
       - "5555"
     command: >
       bash -c "
-        wait-for-it -s rabbit:5672 -t 60 &&
-        celery --broker=amqp://rabbit/ flower --url_prefix=flower
+        wait-for-it -s redis:6379 -t 60 &&
+        celery --broker=redis://redis:6379/0 flower --url_prefix=flower
       "
 
-  # RabbitMQ authentication can't be disabled. So when we try to log into
-  # the management console we have to use "guest" as username and password.
-  # The real authentication happens by ADIT itself, because the management
-  # console is behind a ProxyView.
-  rabbit:
-    image: rabbitmq:3.12.2-management
-    expose:
-      - "5672"
-      - "15672"
-    configs:
-      - source: rabbit_config
-        target: /etc/rabbitmq/rabbitmq.conf
-    volumes:
-      - rabbit_data:/var/lib/rabbitmq
-
   redis:
     image: redis:7.2.3
     expose:
@@ -114,8 +95,6 @@ services:
       - orthanc2_data:/var/lib/orthanc/db
 
 configs:
-  rabbit_config:
-    file: ./rabbitmq/rabbitmq.conf
   orthanc1_config:
     file: ./orthanc/orthanc1.json
   orthanc2_config:
@@ -124,7 +103,6 @@ configs:
 volumes:
   adit_data:
   postgres_data:
-  rabbit_data:
   redis_data:
   flower_data:
   orthanc1_data:

compose/docker-compose.dev.yml

@@ -86,11 +86,6 @@ services:
       - full
       - extra
 
-  rabbit:
-    profiles:
-      - full
-      - extra
-
   redis:
     profiles:
       - full

compose/docker-compose.prod.yml

@@ -69,11 +69,6 @@ services:
     deploy:
       replicas: 1
 
-  rabbit:
-    restart: always
-    deploy:
-      replicas: 1
-
   redis:
     restart: always
     deploy:

pyproject.toml

@@ -9,7 +9,7 @@ license = "GPL-3.0-or-later"
 adrf = "^0.1.1"
 aiofiles = "^23.1.0"
 asyncinotify = "^4.0.1"
-celery = "^5.2.7"
+celery = {extras = ["redis"], version = "^5.3.5"}
 channels = "^4.0.0"
 channels-redis = "^4.0.0"
 cryptography = "^41.0.1"