From c46967a678b8f4d3a6a125ec2bbee5f5d995b6a9 Mon Sep 17 00:00:00 2001 From: lisin Date: Sun, 13 Aug 2023 21:40:57 -0700 Subject: [PATCH] feat: add Celery and Redis for periodic tasks --- .github/workflows/ci-cd.yml | 1 + docker-compose-deploy.yaml | 35 +++++++++++++++++++++++++ docker-compose.yaml | 51 +++++++++++++++++++++++++++++++++---- ecomm1/__init__.py | 3 +++ ecomm1/celery.py | 19 ++++++++++++++ ecomm1/settings.py | 22 +++++++++++++--- requirements.txt | 16 +++++++++++- store/tasks.py | 15 +++++++++++ store/update_products.py | 2 +- store/views.py | 11 +++++--- 10 files changed, 162 insertions(+), 13 deletions(-) create mode 100644 ecomm1/celery.py create mode 100644 store/tasks.py diff --git a/.github/workflows/ci-cd.yml b/.github/workflows/ci-cd.yml index e3ab7a5..e8ec49c 100644 --- a/.github/workflows/ci-cd.yml +++ b/.github/workflows/ci-cd.yml @@ -118,6 +118,7 @@ jobs: EMAIL_HOST_PASSWORD: ${{secrets.EMAIL_HOST_PASSWORD}} MANAGERS_EMAILS: ${{secrets.MANAGERS_EMAILS}} DEFAULT_FROM_EMAIL: ${{secrets.DEFAULT_FROM_EMAIL}} + BROKER_URL: ${{secrets.BROKER_URL}} with: yc-sa-json-credentials: ${{ secrets.YC_SA_JSON_CREDENTIALS }} folder-id: b1gpbt4jb9ls1d9364c8 diff --git a/docker-compose-deploy.yaml b/docker-compose-deploy.yaml index 97fac33..fff3050 100644 --- a/docker-compose-deploy.yaml +++ b/docker-compose-deploy.yaml @@ -1,5 +1,10 @@ version: '3.8' services: + + redis: + container_name: redis + image: redis:alpine + app: restart: always volumes: @@ -28,6 +33,7 @@ services: - DEFAULT_FROM_EMAIL={{env.DEFAULT_FROM_EMAIL}} depends_on: - db + - redis db: image: postgres:13-alpine @@ -72,6 +78,35 @@ services: depends_on: - proxy + celeryworker: + container_name: celeryworker + image: cr.yandex/{{ env.CR_REGISTRY }}/{{ env.CR_REPOSITORY }}:app-{{ env.IMAGE_TAG }} + restart: on-failure + environment: + - BROKER_URL=${BROKER_URL} + - CELERY_BACKEND=${BROKER_URL} + - CELERY_BROKER=${BROKER_URL} + depends_on: + - app + command: 'celery -A ecomm1 worker -l DEBUG' + volumes: + - static:/vol/web + + celerybeat: + container_name: celerybeat + image: cr.yandex/{{ env.CR_REGISTRY }}/{{ env.CR_REPOSITORY }}:app-{{ env.IMAGE_TAG }} + restart: on-failure + environment: + - BROKER_URL=${BROKER_URL} + - CELERY_BACKEND=${BROKER_URL} + - CELERY_BROKER=${BROKER_URL} + depends_on: + - app + - celeryworker + command: 'celery -A ecomm1 beat' + volumes: + - static:/vol/web + volumes: postgres-data: static: diff --git a/docker-compose.yaml b/docker-compose.yaml index cb60bb0..add1142 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -1,5 +1,10 @@ version: '3.8' services: + + redis: + container_name: redis + image: redis:alpine + app: restart: always volumes: @@ -9,7 +14,7 @@ services: container_name: app build: context: . - # environment: +# environment: # - DEBUG=False # - SECRET_KEY=os.environ.get('SECRET_KEY') # - DB_ENGINE={{env.DB_ENGINE}} @@ -22,6 +27,7 @@ services: - .env depends_on: - db + - redis db: image: postgres:13-alpine @@ -33,14 +39,14 @@ services: - POSTGRES_USER=${DB_USER} - POSTGRES_PASSWORD=${DB_PASSWORD} - POSTGRES_DB=main_db -# env_file: -# - .env + env_file: + - .env ports: - "5432:5432" proxy: - build: - context: ./proxy + container_name: proxy + image: nginx:1.24.0-alpine restart: always depends_on: - app @@ -49,6 +55,41 @@ services: volumes: - static:/vol/web + celeryworker: + container_name: celeryworker + build: + context: . + restart: on-failure + environment: + - BROKER_URL=${BROKER_URL} + - CELERY_BACKEND=${BROKER_URL} + - CELERY_BROKER=${BROKER_URL} + env_file: + - .env + depends_on: + - app + command: 'celery -A ecomm1 worker -l DEBUG' + volumes: + - static:/vol/web + + celerybeat: + container_name: celerybeat + build: + context: . + restart: on-failure + environment: + - BROKER_URL=${BROKER_URL} + - CELERY_BACKEND=${BROKER_URL} + - CELERY_BROKER=${BROKER_URL} + env_file: + - .env + depends_on: + - app + - celeryworker + command: 'celery -A ecomm1 beat' + volumes: + - static:/vol/web + volumes: postgres-data: static: diff --git a/ecomm1/__init__.py b/ecomm1/__init__.py index e69de29..fb989c4 100644 --- a/ecomm1/__init__.py +++ b/ecomm1/__init__.py @@ -0,0 +1,3 @@ +from .celery import app as celery_app + +__all__ = ('celery_app',) diff --git a/ecomm1/celery.py b/ecomm1/celery.py new file mode 100644 index 0000000..692b88f --- /dev/null +++ b/ecomm1/celery.py @@ -0,0 +1,19 @@ +import os +from celery import Celery +from celery.schedules import crontab + +os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'ecomm1.settings') + +app = Celery('store') + +app.config_from_object('django.conf:settings', namespace='CELERY') + +app.autodiscover_tasks() + +# celery beat tasks +app.conf.beat_schedule = { + 'text-every-minute': { + 'task': 'store.tasks.make_goods_update', + 'schedule': crontab(minute='0', hour='*/1'), + } +} diff --git a/ecomm1/settings.py b/ecomm1/settings.py index 2fb59e7..555c4d8 100644 --- a/ecomm1/settings.py +++ b/ecomm1/settings.py @@ -88,7 +88,7 @@ 'NAME': os.environ.get('DB_NAME', BASE_DIR / 'db.sqlite3'), 'USER': os.environ.get('DB_USER'), 'PASSWORD': os.environ.get('DB_PASSWORD'), - "HOST": os.environ.get("DB_HOST", "localhost"), + 'HOST': os.environ.get('DB_HOST', 'localhost'), 'PORT': '5432', } } @@ -96,9 +96,9 @@ # Internationalization # https://docs.djangoproject.com/en/4.1/topics/i18n/ -LANGUAGE_CODE = 'en-us' +LANGUAGE_CODE = 'ru-ru' -TIME_ZONE = 'UTC' +TIME_ZONE = 'Europe/Moscow' USE_I18N = True @@ -148,6 +148,22 @@ EMAIL_HOST_PASSWORD = os.environ.get('EMAIL_HOST_PASSWORD') DEFAULT_FROM_EMAIL = os.environ.get('DEFAULT_FROM_EMAIL') +# Celery Configuration Options +CELERY_TIMEZONE = 'Europe/Moscow' +CELERY_TASK_TRACK_STARTED = True +CELERY_TASK_TIME_LIMIT = 30 * 60 +BROKER_CONNECTION_RETRY = False +BROKER_CONNECTION_RETRY_ON_STARTUP = True +BROKER_CONNECTION_MAX_RETRIES = 5 + +if DEBUG: + CELERY_BROKER_URL = 'redis://localhost:6379' +else: + CELERY_BROKER = 'redis://redis:6379/0' + CELERY_BACKEND = 'redis://redis:6379/0' + BROKER_URL = 'redis://redis:6379/0' + CELERY_BROKER_URL = 'redis://redis:6379/0' + LOGGING = { 'version': 1, 'disable_existing_loggers': False, diff --git a/requirements.txt b/requirements.txt index 13f05d8..e3c4d7e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,16 @@ +amqp==5.1.1 asgiref==3.7.2 +async-timeout==4.0.3 +billiard==4.1.0 boto3==1.28.2 botocore==1.31.2 -certifi>=2023.07.22 +celery==5.3.1 +certifi==2023.7.22 charset-normalizer==3.2.0 +click==8.1.6 +click-didyoumean==0.3.0 +click-plugins==1.1.1 +click-repl==0.3.0 Django==4.2.3 django-cleanup==7.0.0 django-debug-toolbar==4.1.0 @@ -11,18 +19,24 @@ flake8==6.0.0 gunicorn==20.1.0 idna==3.4 jmespath==1.0.1 +kombu==5.3.1 mccabe==0.7.0 Pillow==9.5.0 +prompt-toolkit==3.0.39 psycopg2-binary==2.9.6 pycodestyle==2.10.0 pyflakes==3.0.1 python-dateutil==2.8.2 python-dotenv==1.0.0 python-slugify==8.0.1 +redis==4.6.0 requests==2.31.0 s3transfer==0.6.1 six==1.16.0 sqlparse==0.4.4 text-unidecode==1.3 typing_extensions==4.6.2 +tzdata==2023.3 urllib3==1.26.16 +vine==5.0.0 +wcwidth==0.2.6 diff --git a/store/tasks.py b/store/tasks.py new file mode 100644 index 0000000..1a27c4d --- /dev/null +++ b/store/tasks.py @@ -0,0 +1,15 @@ +from celery import shared_task, Celery +from threading import Thread +import os +from .update_products import FeedParser, ProductUpdater + + +app = Celery() + + +@shared_task +def make_goods_update(): + feed_url = os.environ.get('GOODS_FEED_URL') + feed_parser = FeedParser(feed_url) + product_updater = ProductUpdater(feed_parser) + Thread(target=product_updater.update_products, args=()).start() diff --git a/store/update_products.py b/store/update_products.py index 2447b7b..65306de 100644 --- a/store/update_products.py +++ b/store/update_products.py @@ -114,7 +114,7 @@ def update_products(self): if not products_from_feed: # Если список товаров из фида пуст, выходим из метода - return 0, 0 + return 0, 0, 0, 0 updated_count = 0 added_count = 0 diff --git a/store/views.py b/store/views.py index 15ccb35..d1f5708 100644 --- a/store/views.py +++ b/store/views.py @@ -166,10 +166,15 @@ def update_products_view(request): feed_url = os.environ.get('GOODS_FEED_URL') feed_parser = FeedParser(feed_url) product_updater = ProductUpdater(feed_parser) + updated_count, photos_loaded_count, added_count, count_hidden_goods = product_updater.update_products() - message = f'Обновлено товаров: {updated_count}. Добавлено: {added_count}. Скрыто: {count_hidden_goods}.' \ - f' Загружено фотографий: {photos_loaded_count}.' - messages.success(request, message) + if updated_count: + message = f'Обновлено товаров: {updated_count}. Добавлено: {added_count}. Скрыто: {count_hidden_goods}.' \ + f' Загружено фотографий: {photos_loaded_count}.' + messages.success(request, message) + else: + message = 'Произошла ошибка при обновлении товаров.' + messages.error(request, message) return HttpResponseRedirect(reverse('admin:store_product_changelist'))