diff --git a/doc/changes/DM-36981.feature.rst b/doc/changes/DM-36981.feature.rst new file mode 100644 index 0000000..161af76 --- /dev/null +++ b/doc/changes/DM-36981.feature.rst @@ -0,0 +1 @@ +updates to map bps memory boosting atrributes to panda memory boosting attributes diff --git a/python/lsst/ctrl/bps/panda/constants.py b/python/lsst/ctrl/bps/panda/constants.py index da97649..0fd65d5 100644 --- a/python/lsst/ctrl/bps/panda/constants.py +++ b/python/lsst/ctrl/bps/panda/constants.py @@ -34,6 +34,7 @@ PANDA_DEFAULT_PRIORITY = 900 PANDA_DEFAULT_MAX_JOBS_PER_TASK = 30000 PANDA_DEFAULT_RSS = 1024 +PANDA_DEFAULT_RSS_MAX = 512000 # 500 * 1024 PANDA_DEFAULT_MAX_WALLTIME = 90000 PANDA_DEFAULT_MAX_ATTEMPTS = 3 PANDA_DEFAULT_CORE_COUNT = 1 diff --git a/python/lsst/ctrl/bps/panda/utils.py b/python/lsst/ctrl/bps/panda/utils.py index cce6e53..cd1c78d 100644 --- a/python/lsst/ctrl/bps/panda/utils.py +++ b/python/lsst/ctrl/bps/panda/utils.py @@ -56,6 +56,7 @@ PANDA_DEFAULT_PROCESSING_TYPE, PANDA_DEFAULT_PROD_SOURCE_LABEL, PANDA_DEFAULT_RSS, + PANDA_DEFAULT_RSS_MAX, PANDA_DEFAULT_TASK_TYPE, PANDA_DEFAULT_VO, ) @@ -248,6 +249,10 @@ def _make_doma_work(config, generic_workflow, gwjob, task_count, task_chunk): "fileDistributionEndPointDefault", opt={"curvals": cvals, "default": None} ) + task_rss = gwjob.request_memory if gwjob.request_memory else PANDA_DEFAULT_RSS + task_rss_retry_step = task_rss * gwjob.memory_multiplier if gwjob.memory_mulitplier else 0 + task_rss_retry_offset = 0 if task_rss_retry_step else task_rss + # Assume input files are same across task local_pfns = {} direct_io_files = set() @@ -314,7 +319,10 @@ def _make_doma_work(config, generic_workflow, gwjob, task_count, task_chunk): "type": "template", }, encode_command_line=True, - task_rss=gwjob.request_memory if gwjob.request_memory else PANDA_DEFAULT_RSS, + task_rss=task_rss, + task_rss_retry_offset=task_rss_retry_offset, + task_rss_retry_step=task_rss_retry_step, + task_rss_max=gwjob.request_memory_max if gwjob.request_memory_max else PANDA_DEFAULT_RSS_MAX, task_cloud=gwjob.compute_cloud if gwjob.compute_cloud else PANDA_DEFAULT_CLOUD, task_site=site, task_priority=int(gwjob.priority) if gwjob.priority else PANDA_DEFAULT_PRIORITY,