Skip to content

Commit

Permalink
Fix bootstrax timeouts (#1133)
Browse files Browse the repository at this point in the history
* Update bootstrax

* simple syntax is good syntax
  • Loading branch information
JoranAngevaare authored Jan 16, 2023
1 parent 68d56ba commit 39c563d
Showing 1 changed file with 8 additions and 6 deletions.
14 changes: 8 additions & 6 deletions bin/bootstrax
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,7 @@ timeouts = {
'retry_run': 60,
# Maximum time [s] for strax to complete a processing
# if exceeded, strax will be killed by bootstrax
'max_processing_time': 24*36000,
'max_processing_time': 48*3600,
# Max processing time [s] for bootstrax to finish processing after run ended
'max_processing_time_after_completion': 13*3600,
# Every this many [s] we should have at least one new file written to disk
Expand Down Expand Up @@ -1499,11 +1499,13 @@ def process_run(rd, send_heartbeats=args.production):
fail(f"Processing took longer than {timeouts['max_processing_time']} sec")

# Fail because we are taking a very long time to process despite the run having ended
if (get_end_time(rd) is not None
and t0 < now(-timeouts['max_processing_time_after_completion'])):
kill_process(strax_proc.pid)
fail(f"After the run ended, processing did not succeed in "
f"{timeouts['max_processing_time_after_completion']} sec.")
endtime = get_end_time(rd)
if (endtime is not None
and endtime < now(-timeouts['max_processing_time_after_completion'])
):
kill_process(strax_proc.pid)
fail(f"After the run ended, processing did not succeed in "
f"{timeouts['max_processing_time_after_completion']} sec.")

# Fail because for some reason, we are not writing any new files to disk.
if (t0 < now(-timeouts['max_no_write_time'])
Expand Down

0 comments on commit 39c563d

Please sign in to comment.