feat: add Celery and Redis for periodic tasks

.github/workflows/ci-cd.yml

@@ -118,6 +118,7 @@ jobs:
           EMAIL_HOST_PASSWORD: ${{secrets.EMAIL_HOST_PASSWORD}}
           MANAGERS_EMAILS: ${{secrets.MANAGERS_EMAILS}}
           DEFAULT_FROM_EMAIL: ${{secrets.DEFAULT_FROM_EMAIL}}
+          BROKER_URL: ${{secrets.BROKER_URL}}
         with:
           yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }}
           folder-id: b1gpbt4jb9ls1d9364c8

docker-compose-deploy.yaml

@@ -1,5 +1,10 @@
 version: '3.8'
 services:
+
+  redis:
+    container_name: redis
+    image: redis:alpine
+
   app:
     restart: always
     volumes:
@@ -28,6 +33,7 @@ services:
       - DEFAULT_FROM_EMAIL={{env.DEFAULT_FROM_EMAIL}}
     depends_on:
       - db
+      - redis
 
   db:
     image: postgres:13-alpine
@@ -72,6 +78,35 @@ services:
     depends_on:
       - proxy
 
+  celeryworker:
+    container_name: celeryworker
+    image: cr.yandex/{{ env.CR_REGISTRY }}/{{ env.CR_REPOSITORY }}:app-{{ env.IMAGE_TAG }}
+    restart: on-failure
+    environment:
+      - BROKER_URL=${BROKER_URL}
+      - CELERY_BACKEND=${BROKER_URL}
+      - CELERY_BROKER=${BROKER_URL}
+    depends_on:
+      - app
+    command: 'celery -A ecomm1 worker -l DEBUG'
+    volumes:
+      - static:/vol/web
+
+  celerybeat:
+    container_name: celerybeat
+    image: cr.yandex/{{ env.CR_REGISTRY }}/{{ env.CR_REPOSITORY }}:app-{{ env.IMAGE_TAG }}
+    restart: on-failure
+    environment:
+      - BROKER_URL=${BROKER_URL}
+      - CELERY_BACKEND=${BROKER_URL}
+      - CELERY_BROKER=${BROKER_URL}
+    depends_on:
+      - app
+      - celeryworker
+    command: 'celery -A ecomm1 beat'
+    volumes:
+      - static:/vol/web
+
 volumes:
   postgres-data:
   static:

docker-compose.yaml

@@ -1,5 +1,10 @@
 version: '3.8'
 services:
+
+  redis:
+    container_name: redis
+    image: redis:alpine
+
   app:
     restart: always
     volumes:
@@ -9,7 +14,7 @@ services:
     container_name: app
     build:
       context: .
-    # environment:
+#    environment:
     #   - DEBUG=False
     #   - SECRET_KEY=os.environ.get('SECRET_KEY')
     #   - DB_ENGINE={{env.DB_ENGINE}}
@@ -22,6 +27,7 @@ services:
       - .env
     depends_on:
       - db
+      - redis
 
   db:
     image: postgres:13-alpine
@@ -33,14 +39,14 @@ services:
       - POSTGRES_USER=${DB_USER}
       - POSTGRES_PASSWORD=${DB_PASSWORD}
       - POSTGRES_DB=main_db
-#    env_file:
-#      - .env
+    env_file:
+      - .env
     ports:
       - "5432:5432"
 
   proxy:
-    build:
-      context: ./proxy
+    container_name: proxy
+    image: nginx:1.24.0-alpine
     restart: always
     depends_on:
       - app
@@ -49,6 +55,41 @@ services:
     volumes:
       - static:/vol/web
 
+  celeryworker:
+    container_name: celeryworker
+    build:
+      context: .
+    restart: on-failure
+    environment:
+      - BROKER_URL=${BROKER_URL}
+      - CELERY_BACKEND=${BROKER_URL}
+      - CELERY_BROKER=${BROKER_URL}
+    env_file:
+      - .env
+    depends_on:
+      - app
+    command: 'celery -A ecomm1 worker -l DEBUG'
+    volumes:
+      - static:/vol/web
+
+  celerybeat:
+    container_name: celerybeat
+    build:
+      context: .
+    restart: on-failure
+    environment:
+      - BROKER_URL=${BROKER_URL}
+      - CELERY_BACKEND=${BROKER_URL}
+      - CELERY_BROKER=${BROKER_URL}
+    env_file:
+      - .env
+    depends_on:
+      - app
+      - celeryworker
+    command: 'celery -A ecomm1 beat'
+    volumes:
+      - static:/vol/web
+
 volumes:
   postgres-data:
   static:

ecomm1/__init__.py

@@ -0,0 +1,3 @@
+from .celery import app as celery_app
+
+__all__ = ('celery_app',)

ecomm1/celery.py

@@ -0,0 +1,19 @@
+import os
+from celery import Celery
+from celery.schedules import crontab
+
+os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ecomm1.settings')
+
+app = Celery('store')
+
+app.config_from_object('django.conf:settings', namespace='CELERY')
+
+app.autodiscover_tasks()
+
+# celery beat tasks
+app.conf.beat_schedule = {
+    'text-every-minute': {
+        'task': 'store.tasks.make_goods_update',
+        'schedule': crontab(minute='0', hour='*/1'),
+    }
+}

ecomm1/settings.py

@@ -88,17 +88,17 @@
         'NAME': os.environ.get('DB_NAME', BASE_DIR / 'db.sqlite3'),
         'USER': os.environ.get('DB_USER'),
         'PASSWORD': os.environ.get('DB_PASSWORD'),
-        "HOST": os.environ.get("DB_HOST", "localhost"),
+        'HOST': os.environ.get('DB_HOST', 'localhost'),
         'PORT': '5432',
     }
 }
 
 # Internationalization
 # https://docs.djangoproject.com/en/4.1/topics/i18n/
 
-LANGUAGE_CODE = 'en-us'
+LANGUAGE_CODE = 'ru-ru'
 
-TIME_ZONE = 'UTC'
+TIME_ZONE = 'Europe/Moscow'
 
 USE_I18N = True
 
@@ -148,6 +148,22 @@
     EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD')
     DEFAULT_FROM_EMAIL = os.environ.get('DEFAULT_FROM_EMAIL')
 
+# Celery Configuration Options
+CELERY_TIMEZONE = 'Europe/Moscow'
+CELERY_TASK_TRACK_STARTED = True
+CELERY_TASK_TIME_LIMIT = 30 * 60
+BROKER_CONNECTION_RETRY = False
+BROKER_CONNECTION_RETRY_ON_STARTUP = True
+BROKER_CONNECTION_MAX_RETRIES = 5
+
+if DEBUG:
+    CELERY_BROKER_URL = 'redis://localhost:6379'
+else:
+    CELERY_BROKER = 'redis://redis:6379/0'
+    CELERY_BACKEND = 'redis://redis:6379/0'
+    BROKER_URL = 'redis://redis:6379/0'
+    CELERY_BROKER_URL = 'redis://redis:6379/0'
+
 LOGGING = {
     'version': 1,
     'disable_existing_loggers': False,

requirements.txt

@@ -1,8 +1,16 @@
+amqp==5.1.1
 asgiref==3.7.2
+async-timeout==4.0.3
+billiard==4.1.0
 boto3==1.28.2
 botocore==1.31.2
-certifi>=2023.07.22
+celery==5.3.1
+certifi==2023.7.22
 charset-normalizer==3.2.0
+click==8.1.6
+click-didyoumean==0.3.0
+click-plugins==1.1.1
+click-repl==0.3.0
 Django==4.2.3
 django-cleanup==7.0.0
 django-debug-toolbar==4.1.0
@@ -11,18 +19,24 @@ flake8==6.0.0
 gunicorn==20.1.0
 idna==3.4
 jmespath==1.0.1
+kombu==5.3.1
 mccabe==0.7.0
 Pillow==9.5.0
+prompt-toolkit==3.0.39
 psycopg2-binary==2.9.6
 pycodestyle==2.10.0
 pyflakes==3.0.1
 python-dateutil==2.8.2
 python-dotenv==1.0.0
 python-slugify==8.0.1
+redis==4.6.0
 requests==2.31.0
 s3transfer==0.6.1
 six==1.16.0
 sqlparse==0.4.4
 text-unidecode==1.3
 typing_extensions==4.6.2
+tzdata==2023.3
 urllib3==1.26.16
+vine==5.0.0
+wcwidth==0.2.6

store/tasks.py

@@ -0,0 +1,15 @@
+from celery import shared_task, Celery
+from threading import Thread
+import os
+from .update_products import FeedParser, ProductUpdater
+
+
+app = Celery()
+
+
+@shared_task
+def make_goods_update():
+    feed_url = os.environ.get('GOODS_FEED_URL')
+    feed_parser = FeedParser(feed_url)
+    product_updater = ProductUpdater(feed_parser)
+    Thread(target=product_updater.update_products, args=()).start()

store/update_products.py

@@ -114,7 +114,7 @@ def update_products(self):
 
         if not products_from_feed:
             # Если список товаров из фида пуст, выходим из метода
-            return 0, 0
+            return 0, 0, 0, 0
 
         updated_count = 0
         added_count = 0

store/views.py

@@ -166,10 +166,15 @@ def update_products_view(request):
     feed_url = os.environ.get('GOODS_FEED_URL')
     feed_parser = FeedParser(feed_url)
     product_updater = ProductUpdater(feed_parser)
+
     updated_count, photos_loaded_count, added_count, count_hidden_goods = product_updater.update_products()
-    message = f'Обновлено товаров: {updated_count}. Добавлено: {added_count}. Скрыто: {count_hidden_goods}.' \
-              f' Загружено фотографий: {photos_loaded_count}.'
-    messages.success(request, message)
+    if updated_count:
+        message = f'Обновлено товаров: {updated_count}. Добавлено: {added_count}. Скрыто: {count_hidden_goods}.' \
+                  f' Загружено фотографий: {photos_loaded_count}.'
+        messages.success(request, message)
+    else:
+        message = 'Произошла ошибка при обновлении товаров.'
+        messages.error(request, message)
 
     return HttpResponseRedirect(reverse('admin:store_product_changelist'))