From f64180cd2a34e8aebd3b330ea157e6de21483e04 Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 10:20:19 +0000 Subject: [PATCH 1/9] fix_resources --- bin/bootstrax | 41 ++++++++++++++++++++++++----------------- 1 file changed, 24 insertions(+), 17 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index 5a3f29ad2..cf8a1265a 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -79,13 +79,12 @@ parser.add_argument( help="Don't allow bootstrax to switch to a different target for special runs", ) parser.add_argument( - "--infer_mode", - action="store_true", - help=( - "Determine best number max-messages and cores for each run " - "automatically. Overrides --cores and --max_messages" - ), -) + '--fix_resources', action='store_true', + help="Don't allow bootstrax to switch to a different number of cores and max_messages for failures") +parser.add_argument( + '--infer_mode', action='store_true', + help="Determine best number max-messages and cores for each run " + "automatically. Overrides --cores and --max_messages") parser.add_argument( "--delete_live", action="store_true", @@ -100,14 +99,13 @@ parser.add_argument( ), ) parser.add_argument( - "--ignore_checks", - action="store_true", - help=( - "Do not use! This disables checks on e.g. the timestamps! Should only " - "be used if some run is very valuable but some checks are failing." - ), -) -parser.add_argument("--max_messages", type=int, default=10, help="number of max mailbox messages") + '--ignore_checks', action='store_true', + help="Do not use! This disables checks on e.g. the timestamps! Should only " + "be used if some run is very valuable but some checks are failing.") +parser.add_argument( + '--max_messages', type=int, default=10, + help="number of max mailbox messages") + actions = parser.add_mutually_exclusive_group() @@ -852,8 +850,17 @@ def infer_mode(rd): result[k] = result[k + "_old"] del df, benchmark, result["cores_old"], result["max_messages_old"] else: - result = dict(cores=args.cores, max_messages=args.max_messages, timeout=1000) - n_fails = rd["bootstrax"].get("n_failures", 0) + result = dict(cores=args.cores, + max_messages=args.max_messages, + timeout=1000) + + n_fails = rd['bootstrax'].get('n_failures', 0) + if args.fix_resources: + # If we are in a fix resource mode, we should not change the resources + # based on the number of failures. + n_fails = 0 + log.debug(f'Fixing resources, ignoring {n_fails} previous failures') + if n_fails: # Exponentially lower resources & increase timeout result = dict( From 38e5f2a72757e352ef208c51d8391dc72c44f7f6 Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 13:37:57 +0000 Subject: [PATCH 2/9] up to peak monitor --- bin/bootstrax | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index cf8a1265a..62cddd416 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -622,8 +622,15 @@ def infer_target(rd: dict) -> dict: elif "kr83m" in mode and (len(targets) or len(post_process)): # Override the first (highest level) plugin for Kr runs (could # also use source field, outcome is the same) - if "event_info" in targets or "event_info" in post_process: - targets = list(targets) + ["event_info_double"] + if 'event_info' in targets or 'event_info' in post_process: + targets = list(targets) + ['event_info_double'] + elif 'ambe' in mode: + # rates are very high, to ensure smooth operation let's just do this + # based on calibrations of Apr 2023 this is the only safe working solution + log.debug('ambe-mode') + targets = 'individual_peak_monitor' + post_process = list(['veto_intervals', 'peak_basics', 'peak_positions_mlp', 'peaklets']) + targets = strax.to_str_tuple(targets) post_process = strax.to_str_tuple(post_process) @@ -838,6 +845,18 @@ def infer_mode(rd): "timeout": [1200, 1200, None, None, None, None, None, None, None, None, None, 2400], } if data_rate and args.infer_mode: + + # Temporary solution + # It is a patch to try to process some ambe data without failing for memory + # If we are doing ambe -> consider the maximum rate + # so that we have 10 cores and 12 messages for new ebs + # and we have 8 cores and 6 messages for old ebs + # added by Carlo on 19 April 2023 + mode = str(rd.get('mode')) + if 'ambe' in mode: + data_rate = 550 + + df = pd.DataFrame(benchmark) if data_rate not in benchmark["mbs"]: df.loc[len(df.index)] = [data_rate, None, None, None, None, None] From fba01a320fbdcff98961f962ec027f1546b5a8fd Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 13:39:13 +0000 Subject: [PATCH 3/9] only raw records --- bin/bootstrax | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index 62cddd416..b566f092b 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -628,8 +628,8 @@ def infer_target(rd: dict) -> dict: # rates are very high, to ensure smooth operation let's just do this # based on calibrations of Apr 2023 this is the only safe working solution log.debug('ambe-mode') - targets = 'individual_peak_monitor' - post_process = list(['veto_intervals', 'peak_basics', 'peak_positions_mlp', 'peaklets']) + targets = 'raw_records' + post_process = list(['veto_intervals', ]) targets = strax.to_str_tuple(targets) From e44cb35ebccf5d09b1efd50ab97c455d0085dcfa Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 13:40:51 +0000 Subject: [PATCH 4/9] only peaklets --- bin/bootstrax | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index b566f092b..6c727ab03 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -628,8 +628,8 @@ def infer_target(rd: dict) -> dict: # rates are very high, to ensure smooth operation let's just do this # based on calibrations of Apr 2023 this is the only safe working solution log.debug('ambe-mode') - targets = 'raw_records' - post_process = list(['veto_intervals', ]) + targets = list(['peaklets', 'lone_hits']) + post_process = list(['events_mv', 'events_nv', 'veto_intervals']) targets = strax.to_str_tuple(targets) From 7b8a4a196b4df22d920385860a5cd224683799a4 Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 16:01:17 +0000 Subject: [PATCH 5/9] ambe do all --- bin/bootstrax | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index 6c727ab03..8f46e3095 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -628,8 +628,8 @@ def infer_target(rd: dict) -> dict: # rates are very high, to ensure smooth operation let's just do this # based on calibrations of Apr 2023 this is the only safe working solution log.debug('ambe-mode') - targets = list(['peaklets', 'lone_hits']) - post_process = list(['events_mv', 'events_nv', 'veto_intervals']) + targets = list(['individual_peak_monitor']) + post_process = list(['veto_intervals', 'peak_basics', 'peak_positions_mlp', 'peaklets', 'event_basics', 'events_nv']) targets = strax.to_str_tuple(targets) From 56a496a243b8aa8be3394464e9a42ae65c71908f Mon Sep 17 00:00:00 2001 From: cfuselli Date: Wed, 15 Nov 2023 21:33:53 +0000 Subject: [PATCH 6/9] new era --- bin/bootstrax | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/bin/bootstrax b/bin/bootstrax index 8f46e3095..404f6732e 100755 --- a/bin/bootstrax +++ b/bin/bootstrax @@ -628,9 +628,20 @@ def infer_target(rd: dict) -> dict: # rates are very high, to ensure smooth operation let's just do this # based on calibrations of Apr 2023 this is the only safe working solution log.debug('ambe-mode') - targets = list(['individual_peak_monitor']) - post_process = list(['veto_intervals', 'peak_basics', 'peak_positions_mlp', 'peaklets', 'event_basics', 'events_nv']) + # get the mode from the daq_db + # this is a new thing from Nov 2023 + # it overwrites the mode from the rundb + bootstrax_config_coll = daq_db['bootstrax_config'] + bootstrax_config = bootstrax_config_coll.find_one({'name': 'bootstrax_config'}) + + this_eb_ambe_mode = bootstrax_config['ambe_modes'].get(hostname[:3], 'default') + log.debug(f'Ambe mode for {hostname} is {this_eb_ambe_mode}') + + if this_eb_ambe_mode != 'default': + log.debug(f'Overwriting targets and post processing for {hostname} from daq_db') + targets = bootstrax_config['modes_definitions'][this_eb_ambe_mode]['targets'] + post_process = bootstrax_config['modes_definitions'][this_eb_ambe_mode]['post_process'] targets = strax.to_str_tuple(targets) post_process = strax.to_str_tuple(post_process) From 8a19b9d3d94fe42294c5b10cbe8266d98c2e2cd5 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:06:57 +0000 Subject: [PATCH 7/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- straxen/scripts/bootstrax.py | 62 ++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/straxen/scripts/bootstrax.py b/straxen/scripts/bootstrax.py index 5604a2a18..dee2358a2 100755 --- a/straxen/scripts/bootstrax.py +++ b/straxen/scripts/bootstrax.py @@ -79,12 +79,16 @@ help="Don't allow bootstrax to switch to a different target for special runs", ) parser.add_argument( - '--fix_resources', action='store_true', - help="Don't allow bootstrax to switch to a different number of cores and max_messages for failures") + "--fix_resources", + action="store_true", + help="Don't allow bootstrax to switch to a different number of cores and max_messages for failures", +) parser.add_argument( - '--infer_mode', action='store_true', + "--infer_mode", + action="store_true", help="Determine best number max-messages and cores for each run " - "automatically. Overrides --cores and --max_messages") + "automatically. Overrides --cores and --max_messages", +) parser.add_argument( "--delete_live", action="store_true", @@ -99,13 +103,12 @@ ), ) parser.add_argument( - '--ignore_checks', action='store_true', + "--ignore_checks", + action="store_true", help="Do not use! This disables checks on e.g. the timestamps! Should only " - "be used if some run is very valuable but some checks are failing.") -parser.add_argument( - '--max_messages', type=int, default=10, - help="number of max mailbox messages") - + "be used if some run is very valuable but some checks are failing.", +) +parser.add_argument("--max_messages", type=int, default=10, help="number of max mailbox messages") actions = parser.add_mutually_exclusive_group() @@ -622,26 +625,26 @@ def infer_target(rd: dict) -> dict: elif "kr83m" in mode and (len(targets) or len(post_process)): # Override the first (highest level) plugin for Kr runs (could # also use source field, outcome is the same) - if 'event_info' in targets or 'event_info' in post_process: - targets = list(targets) + ['event_info_double'] - elif 'ambe' in mode: + if "event_info" in targets or "event_info" in post_process: + targets = list(targets) + ["event_info_double"] + elif "ambe" in mode: # rates are very high, to ensure smooth operation let's just do this # based on calibrations of Apr 2023 this is the only safe working solution - log.debug('ambe-mode') + log.debug("ambe-mode") # get the mode from the daq_db # this is a new thing from Nov 2023 # it overwrites the mode from the rundb - bootstrax_config_coll = daq_db['bootstrax_config'] - bootstrax_config = bootstrax_config_coll.find_one({'name': 'bootstrax_config'}) + bootstrax_config_coll = daq_db["bootstrax_config"] + bootstrax_config = bootstrax_config_coll.find_one({"name": "bootstrax_config"}) - this_eb_ambe_mode = bootstrax_config['ambe_modes'].get(hostname[:3], 'default') - log.debug(f'Ambe mode for {hostname} is {this_eb_ambe_mode}') + this_eb_ambe_mode = bootstrax_config["ambe_modes"].get(hostname[:3], "default") + log.debug(f"Ambe mode for {hostname} is {this_eb_ambe_mode}") - if this_eb_ambe_mode != 'default': - log.debug(f'Overwriting targets and post processing for {hostname} from daq_db') - targets = bootstrax_config['modes_definitions'][this_eb_ambe_mode]['targets'] - post_process = bootstrax_config['modes_definitions'][this_eb_ambe_mode]['post_process'] + if this_eb_ambe_mode != "default": + log.debug(f"Overwriting targets and post processing for {hostname} from daq_db") + targets = bootstrax_config["modes_definitions"][this_eb_ambe_mode]["targets"] + post_process = bootstrax_config["modes_definitions"][this_eb_ambe_mode]["post_process"] targets = strax.to_str_tuple(targets) post_process = strax.to_str_tuple(post_process) @@ -862,16 +865,15 @@ def infer_mode(rd): if data_rate and args.infer_mode: # Temporary solution - # It is a patch to try to process some ambe data without failing for memory + # It is a patch to try to process some ambe data without failing for memory # If we are doing ambe -> consider the maximum rate # so that we have 10 cores and 12 messages for new ebs # and we have 8 cores and 6 messages for old ebs # added by Carlo on 19 April 2023 - mode = str(rd.get('mode')) - if 'ambe' in mode: + mode = str(rd.get("mode")) + if "ambe" in mode: data_rate = 550 - df = pd.DataFrame(benchmark) if data_rate not in benchmark["mbs"]: df.loc[len(df.index)] = [data_rate, None, None, None, None, None] @@ -884,16 +886,14 @@ def infer_mode(rd): result[k] = result[k + "_old"] del df, benchmark, result["cores_old"], result["max_messages_old"] else: - result = dict(cores=args.cores, - max_messages=args.max_messages, - timeout=1000) + result = dict(cores=args.cores, max_messages=args.max_messages, timeout=1000) - n_fails = rd['bootstrax'].get('n_failures', 0) + n_fails = rd["bootstrax"].get("n_failures", 0) if args.fix_resources: # If we are in a fix resource mode, we should not change the resources # based on the number of failures. n_fails = 0 - log.debug(f'Fixing resources, ignoring {n_fails} previous failures') + log.debug(f"Fixing resources, ignoring {n_fails} previous failures") if n_fails: # Exponentially lower resources & increase timeout From 127f481dbebc047c9326ea9777da1ace82b95223 Mon Sep 17 00:00:00 2001 From: cfuselli Date: Tue, 3 Dec 2024 13:17:31 +0000 Subject: [PATCH 8/9] long line --- straxen/scripts/bootstrax.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/straxen/scripts/bootstrax.py b/straxen/scripts/bootstrax.py index 5604a2a18..5e0c97f53 100755 --- a/straxen/scripts/bootstrax.py +++ b/straxen/scripts/bootstrax.py @@ -80,7 +80,7 @@ ) parser.add_argument( '--fix_resources', action='store_true', - help="Don't allow bootstrax to switch to a different number of cores and max_messages for failures") + help="Don't let bootstrax change number of cores/max_messages because of failures") parser.add_argument( '--infer_mode', action='store_true', help="Determine best number max-messages and cores for each run " From 493f980f7ad733208ee3bfc8a4ce42fe82e9140d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 3 Dec 2024 13:19:54 +0000 Subject: [PATCH 9/9] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- straxen/scripts/bootstrax.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/straxen/scripts/bootstrax.py b/straxen/scripts/bootstrax.py index 56c58000a..5b2fd6286 100755 --- a/straxen/scripts/bootstrax.py +++ b/straxen/scripts/bootstrax.py @@ -79,8 +79,10 @@ help="Don't allow bootstrax to switch to a different target for special runs", ) parser.add_argument( - '--fix_resources', action='store_true', - help="Don't let bootstrax change number of cores/max_messages because of failures") + "--fix_resources", + action="store_true", + help="Don't let bootstrax change number of cores/max_messages because of failures", +) parser.add_argument( "--infer_mode", action="store_true",