Merge pull request #58 from lsst/tickets/DM-36981

DM-36981: retry jobs by increasing memory with user defined task para…

doc/changes/DM-36981.feature.rst

@@ -0,0 +1 @@
+updates to map bps memory boosting atrributes to panda memory boosting attributes

python/lsst/ctrl/bps/panda/constants.py

@@ -34,6 +34,7 @@
 PANDA_DEFAULT_PRIORITY = 900
 PANDA_DEFAULT_MAX_JOBS_PER_TASK = 30000
 PANDA_DEFAULT_RSS = 1024
+PANDA_DEFAULT_RSS_MAX = 512000  # 500 * 1024
 PANDA_DEFAULT_MAX_WALLTIME = 90000
 PANDA_DEFAULT_MAX_ATTEMPTS = 3
 PANDA_DEFAULT_CORE_COUNT = 1

python/lsst/ctrl/bps/panda/utils.py

@@ -56,6 +56,7 @@
     PANDA_DEFAULT_PROCESSING_TYPE,
     PANDA_DEFAULT_PROD_SOURCE_LABEL,
     PANDA_DEFAULT_RSS,
+    PANDA_DEFAULT_RSS_MAX,
     PANDA_DEFAULT_TASK_TYPE,
     PANDA_DEFAULT_VO,
 )
@@ -248,6 +249,10 @@ def _make_doma_work(config, generic_workflow, gwjob, task_count, task_chunk):
         "fileDistributionEndPointDefault", opt={"curvals": cvals, "default": None}
     )
 
+    task_rss = gwjob.request_memory if gwjob.request_memory else PANDA_DEFAULT_RSS
+    task_rss_retry_step = task_rss * gwjob.memory_multiplier if gwjob.memory_mulitplier else 0
+    task_rss_retry_offset = 0 if task_rss_retry_step else task_rss
+
     # Assume input files are same across task
     local_pfns = {}
     direct_io_files = set()
@@ -314,7 +319,10 @@ def _make_doma_work(config, generic_workflow, gwjob, task_count, task_chunk):
             "type": "template",
         },
         encode_command_line=True,
-        task_rss=gwjob.request_memory if gwjob.request_memory else PANDA_DEFAULT_RSS,
+        task_rss=task_rss,
+        task_rss_retry_offset=task_rss_retry_offset,
+        task_rss_retry_step=task_rss_retry_step,
+        task_rss_max=gwjob.request_memory_max if gwjob.request_memory_max else PANDA_DEFAULT_RSS_MAX,
         task_cloud=gwjob.compute_cloud if gwjob.compute_cloud else PANDA_DEFAULT_CLOUD,
         task_site=site,
         task_priority=int(gwjob.priority) if gwjob.priority else PANDA_DEFAULT_PRIORITY,