From 3b94b3fdb170215cda2a26ec5f25a2f1c89a7803 Mon Sep 17 00:00:00 2001 From: dachengx Date: Sun, 7 Jan 2024 09:38:27 -0600 Subject: [PATCH 1/3] Improve check of already made toydata and output --- alea/submitter.py | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/alea/submitter.py b/alea/submitter.py index 07d006d5..115c01da 100644 --- a/alea/submitter.py +++ b/alea/submitter.py @@ -361,15 +361,30 @@ def computation_tickets_generator(self): + " ".join(map(shlex.quote, script.split(" "))) ) + toydata_mode = i_args["toydata_mode"] + toydata_filename = i_args["toydata_filename"] + only_toydata = i_args["only_toydata"] output_filename = i_args["output_filename"] - if ( - (output_filename is not None) - and os.path.exists(output_filename) - and not self.resubmit - and self.computation != "threshold" + if (toydata_mode == "generate_and_store") and (toydata_filename is None): + raise ValueError( + "toydata_filename should be provided when toydata_mode is " + "generate_and_store." + ) + if (not only_toydata) and (output_filename is None): + raise ValueError( + "output_filename should be provided when only_toydata is False." + ) + + already_done = True + if self.resubmit or (self.computation == "threshold"): + already_done = False + if (toydata_mode == "generate_and_store") and ( + not os.path.exists(toydata_filename) ): - continue - else: + already_done = False + if (not only_toydata) and (not os.path.exists(output_filename)): + already_done = False + if not already_done: yield script, output_filename @staticmethod From 628c2eaa50fd339fdc0f27353923a4f00e53927a Mon Sep 17 00:00:00 2001 From: dachengx Date: Sun, 7 Jan 2024 09:42:44 -0600 Subject: [PATCH 2/3] Add comment --- alea/submitter.py | 1 + 1 file changed, 1 insertion(+) diff --git a/alea/submitter.py b/alea/submitter.py index 115c01da..8413222b 100644 --- a/alea/submitter.py +++ b/alea/submitter.py @@ -365,6 +365,7 @@ def computation_tickets_generator(self): toydata_filename = i_args["toydata_filename"] only_toydata = i_args["only_toydata"] output_filename = i_args["output_filename"] + # these check might need change if we support more modes if (toydata_mode == "generate_and_store") and (toydata_filename is None): raise ValueError( "toydata_filename should be provided when toydata_mode is " From 996907ea98ba51dc4105ef7a058364993ac367e3 Mon Sep 17 00:00:00 2001 From: dachengx Date: Sun, 7 Jan 2024 09:47:13 -0600 Subject: [PATCH 3/3] Split complex function --- alea/submitter.py | 51 +++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 26 deletions(-) diff --git a/alea/submitter.py b/alea/submitter.py index 8413222b..32abc5aa 100644 --- a/alea/submitter.py +++ b/alea/submitter.py @@ -361,32 +361,31 @@ def computation_tickets_generator(self): + " ".join(map(shlex.quote, script.split(" "))) ) - toydata_mode = i_args["toydata_mode"] - toydata_filename = i_args["toydata_filename"] - only_toydata = i_args["only_toydata"] - output_filename = i_args["output_filename"] - # these check might need change if we support more modes - if (toydata_mode == "generate_and_store") and (toydata_filename is None): - raise ValueError( - "toydata_filename should be provided when toydata_mode is " - "generate_and_store." - ) - if (not only_toydata) and (output_filename is None): - raise ValueError( - "output_filename should be provided when only_toydata is False." - ) - - already_done = True - if self.resubmit or (self.computation == "threshold"): - already_done = False - if (toydata_mode == "generate_and_store") and ( - not os.path.exists(toydata_filename) - ): - already_done = False - if (not only_toydata) and (not os.path.exists(output_filename)): - already_done = False - if not already_done: - yield script, output_filename + if not self.already_done(i_args): + yield script, i_args["output_filename"] + + def already_done(self, i_args: dict) -> bool: + """Check if the job is already done, considering the modes of toydata and output.""" + toydata_mode = i_args["toydata_mode"] + toydata_filename = i_args["toydata_filename"] + only_toydata = i_args["only_toydata"] + output_filename = i_args["output_filename"] + # these check might need change if we support more modes + if (toydata_mode == "generate_and_store") and (toydata_filename is None): + raise ValueError( + "toydata_filename should be provided when toydata_mode is generate_and_store." + ) + if (not only_toydata) and (output_filename is None): + raise ValueError("output_filename should be provided when only_toydata is False.") + + is_done = True + if self.resubmit or (self.computation == "threshold"): + is_done = False + if (toydata_mode == "generate_and_store") and (not os.path.exists(toydata_filename)): + is_done = False + if (not only_toydata) and (not os.path.exists(output_filename)): + is_done = False + return is_done @staticmethod def update_n_batch(runner_args):