From da3e946d321ecddf3466f8dc4b0a612b77bd047f Mon Sep 17 00:00:00 2001 From: Michael Benowitz Date: Thu, 10 Aug 2023 19:16:51 -0400 Subject: [PATCH 1/2] NOREF Limit requests handled by workers This update introduces two configuration settings for our gunicorn server: - `max_requests` Limits the number of total requests that can be handled by a single gunicorn worker. The gunicorn docs state this is intended to help limit effects of memory leaks - `max_requests_jitter` This randomizes the exact number of requests to prevent multiple workers from being restarted simultaneously The goal of this work is to hopefully limit memory consumption (which tends to run at 90% or greater in production) and to reduce the effect of `QueuePool` errors that result from long-running workers. At the current setting of 500 this should result in workers being restarted roughly every 45-60 minutes. This can be adjusted after review of effects on performance in QA (and production if no issues are raised in QA). --- docker/gunicorn.conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/gunicorn.conf.py b/docker/gunicorn.conf.py index 431b453a3..2ab00c2a8 100644 --- a/docker/gunicorn.conf.py +++ b/docker/gunicorn.conf.py @@ -21,6 +21,8 @@ bind = ["127.0.0.1:8000"] # listen on 8000, only on the loopback address workers = (2 * multiprocessing.cpu_count()) + 1 threads = 2 +max_requests = 2000 +max_requests_jitter = 100 pythonpath = ",".join([ str(VENV_ACTUAL), SIMPLIFIED_HOME, From 32605c5e8346b8c89e90377de7e6830968eef28c Mon Sep 17 00:00:00 2001 From: Michael Benowitz Date: Thu, 10 Aug 2023 19:26:14 -0400 Subject: [PATCH 2/2] NOREF Set to desired values These values (500 and 50) should give us the desired performance in production. --- docker/gunicorn.conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/gunicorn.conf.py b/docker/gunicorn.conf.py index 2ab00c2a8..97fbb4ec8 100644 --- a/docker/gunicorn.conf.py +++ b/docker/gunicorn.conf.py @@ -21,8 +21,8 @@ bind = ["127.0.0.1:8000"] # listen on 8000, only on the loopback address workers = (2 * multiprocessing.cpu_count()) + 1 threads = 2 -max_requests = 2000 -max_requests_jitter = 100 +max_requests = 500 +max_requests_jitter = 50 pythonpath = ",".join([ str(VENV_ACTUAL), SIMPLIFIED_HOME,