set a quite high concurency per dataseries by default to keep 'old' b…

…ehaviour
neuroforgede · Nov 10, 2024 · 522f904 · 522f904
1 parent 339b12f
commit 522f904
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/skipper/skipper/environment_common.py b/skipper/skipper/environment_common.py
@@ -47,7 +47,9 @@
 # cooldown after requeueing a chunk
 # so we dont overload the system with requeues
 SKIPPER_CELERY_PERSIST_DATA_POINT_CHUNK_REQUEUE_COOLDOWN_MINUTES = int(os.environ.get('SKIPPER_CELERY_PERSIST_DATA_POINT_CHUNK_REQUEUE_COOLDOWN_MINUTES', 40))
-SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_PER_DATA_SERIES = int(os.environ.get('SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_PER_DATA_SERIES', '4'))
+# by default we keep something resembling the old behaviour
+# where we had effectively unbounded concurrency
+SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_PER_DATA_SERIES = int(os.environ.get('SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_PER_DATA_SERIES', '100'))
 SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_LOCK_TIMEOUT = int(os.environ.get('SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_LOCK_TIMEOUT', '180'))
 SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_MAX_RETRY_DELAY= int(os.environ.get('SKIPPER_CELERY_DATA_SERIES_BULK_ASYNC_CONCURRENCY_MAX_RETRY_DELAY', '60'))