diff --git a/cwltool/checker.py b/cwltool/checker.py index 541d59440..676245698 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -517,10 +517,13 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) - def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: - """Check if a step contains a `loop` directive with `all` outputMethod.""" + """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: - if source_step.get("loop") is not None and source_step.get("outputMethod") == "all": + if ( + source_step.get("loop") is not None + and source_step.get("outputMethod") == "all_iterations" + ): return True return False diff --git a/cwltool/errors.py b/cwltool/errors.py index 978078ec6..045b9b383 100644 --- a/cwltool/errors.py +++ b/cwltool/errors.py @@ -8,10 +8,8 @@ # flake8: noqa: F401 -from cwl_utils.errors import WorkflowException as WorkflowException - - from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException +from cwl_utils.errors import WorkflowException as WorkflowException class UnsupportedRequirement(WorkflowException): diff --git a/cwltool/schemas/v1.3.0-dev1/Workflow.yml b/cwltool/schemas/v1.3.0-dev1/Workflow.yml index f83575b2a..3e4045a43 100644 --- a/cwltool/schemas/v1.3.0-dev1/Workflow.yml +++ b/cwltool/schemas/v1.3.0-dev1/Workflow.yml @@ -497,8 +497,8 @@ $graph: docParent: "#LoopWorkflowStep" doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep). symbols: - - last - - all + - last_iteration + - all_iterations - name: AbstractWorkflowStep @@ -705,14 +705,14 @@ $graph: The `outputMethod` field describes how to deal with loop outputs after termination: - * **last** specifies that only the last computed element for each output - parameter should be propagated to the subsequent steps. This is the - default value. + * **last_iteration** specifies that only the last computed element for + each output parameter should be propagated to the subsequent steps. + This is the default value. - * **all** specifies that an array with all output values computed at the - end of each loop iteration should be propagated to the subsequent steps. - Elements in the array must be ordered according to the loop iterations - that produced them. + * **all_iterations** specifies that an array with all output values + computed at the end of each loop iteration should be propagated to + the subsequent steps. Elements in the array must be ordered according + to the loop iterations that produced them. Iterative execution in CWL is an optional feature and is not required to be implemented by all consumers of CWL documents. An implementation that @@ -734,9 +734,9 @@ $graph: mapPredicate: outputSource - name: outputMethod doc: | - If not specified, the default method is "last". + If not specified, the default method is "last_iteration". type: LoopOutputMethod? - default: last + default: last_iteration jsonldPredicate: "_id": "cwl:outputMethod" "_type": "@vocab" @@ -748,7 +748,8 @@ $graph: Only run the next iteration when the expression evaluates to `true`. If the first iteration evaluates to `false` the step is skipped. A skipped step produces a `null` on each output if the `outputMethod` - is set to `last`, and an empty array if the `outputMethod` is set to `all`. + is set to `last_iteration`, and an empty array if the `outputMethod` + is set to `all_iterations`. - name: Workflow diff --git a/cwltool/update.py b/cwltool/update.py index 7b801f5dc..4fd66b37a 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -11,12 +11,11 @@ cast, ) +from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader from schema_salad.sourceline import SourceLine -from ruamel.yaml.comments import CommentedMap, CommentedSeq - from .loghandler import _logger from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field @@ -51,7 +50,16 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: el["outputSource"] = source s["loop"] = r["loop"] if "outputMethod" in r: - s["outputMethod"] = r["outputMethod"] + if r["outputMethod"] == "all": + s["outputMethod"] = "all_iterations" + elif r["outputMethod"] == "last": + s["outputMethod"] = "last_iteration" + else: + raise SourceLine( + r, raise_type=ValidationException + ).makeError( # pragma: no cover + f"Invalid value {r['outputMethod']} for `outputMethod`." + ) cast( MutableSequence[CWLObjectType], s["requirements"], diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index 8402b2b4b..2e69ca70c 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -744,7 +744,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] _logger.info("[%s] will be skipped", step.name) if ( step.tool.get("loop") is not None - and step.tool.get("outputMethod", "last") == "all" + and step.tool.get("outputMethod", "last_iteration") == "all_iterations" ): callback({k["id"]: [] for k in outputparms}, "skipped") else: @@ -874,7 +874,7 @@ def _set_empty_output(self, outputMethod: str) -> None: for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) - if outputMethod == "all": + if outputMethod == "all_iterations": self.output_buffer[iid] = cast(MutableSequence[Optional[CWLOutputType]], []) else: self.output_buffer[iid] = None @@ -887,7 +887,7 @@ def job( ) -> JobsGeneratorType: """Generate a WorkflowJobStep job until the `when` condition evaluates to False.""" self.joborder = joborder - outputMethod = self.step.tool.get("outputMethod", "last") + outputMethod = self.step.tool.get("outputMethod", "last_iteration") callback = functools.partial( self.loop_callback, @@ -953,14 +953,14 @@ def loop_callback( self.iteration += 1 try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) - outputMethod = self.step.tool.get("outputMethod", "last") + outputMethod = self.step.tool.get("outputMethod", "last_iteration") state: Dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) if iid in jobout: state[iid] = WorkflowStateItem(i, jobout[iid], processStatus) - if outputMethod == "all": + if outputMethod == "all_iterations": if iid not in self.output_buffer: self.output_buffer[iid] = cast( MutableSequence[Optional[CWLOutputType]], [] diff --git a/tests/loop/all-output-loop-no-iteration.cwl b/tests/loop/all-output-loop-no-iteration.cwl index 04ec87a18..49189faff 100644 --- a/tests/loop/all-output-loop-no-iteration.cwl +++ b/tests/loop/all-output-loop-no-iteration.cwl @@ -25,4 +25,4 @@ steps: when: $(inputs.i1 < 1) loop: i1: o1 - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/all-output-loop.cwl b/tests/loop/all-output-loop.cwl index 157432e8f..001f50a7e 100644 --- a/tests/loop/all-output-loop.cwl +++ b/tests/loop/all-output-loop.cwl @@ -25,4 +25,4 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/default-value-loop.cwl b/tests/loop/default-value-loop.cwl index 04c64f0dd..08b032caf 100644 --- a/tests/loop/default-value-loop.cwl +++ b/tests/loop/default-value-loop.cwl @@ -44,4 +44,4 @@ steps: i1: outputSource: o1 default: 5 - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/invalid-loop-scatter.cwl b/tests/loop/invalid-loop-scatter.cwl index 02451351d..31c563414 100644 --- a/tests/loop/invalid-loop-scatter.cwl +++ b/tests/loop/invalid-loop-scatter.cwl @@ -26,7 +26,7 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-loop-when-exception.cwl b/tests/loop/invalid-loop-when-exception.cwl index 33f7795c4..0bedd197d 100644 --- a/tests/loop/invalid-loop-when-exception.cwl +++ b/tests/loop/invalid-loop-when-exception.cwl @@ -29,7 +29,7 @@ steps: } loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-loop-when-exception2.cwl b/tests/loop/invalid-loop-when-exception2.cwl index 8e80250c4..5d2d3a426 100644 --- a/tests/loop/invalid-loop-when-exception2.cwl +++ b/tests/loop/invalid-loop-when-exception2.cwl @@ -33,7 +33,7 @@ steps: } loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-multi-source-loop-no-requirement.cwl b/tests/loop/invalid-multi-source-loop-no-requirement.cwl index 811ba6c6e..2c1cf35ec 100644 --- a/tests/loop/invalid-multi-source-loop-no-requirement.cwl +++ b/tests/loop/invalid-multi-source-loop-no-requirement.cwl @@ -61,4 +61,4 @@ steps: i1: loopSource: [osmall, obig] pickValue: the_only_non_null - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/invalid-no-loopWhen.cwl b/tests/loop/invalid-no-loopWhen.cwl index 0a869be8e..8d4eb7589 100644 --- a/tests/loop/invalid-no-loopWhen.cwl +++ b/tests/loop/invalid-no-loopWhen.cwl @@ -25,7 +25,7 @@ steps: ${return {'o1': inputs.i1 + inputs.i2};} loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-non-boolean-loopWhen.cwl b/tests/loop/invalid-non-boolean-loopWhen.cwl index ea807ba3a..f7c16cd97 100644 --- a/tests/loop/invalid-non-boolean-loopWhen.cwl +++ b/tests/loop/invalid-non-boolean-loopWhen.cwl @@ -26,7 +26,7 @@ steps: when: $(inputs.i1) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-non-boolean-loopWhen2.cwl b/tests/loop/invalid-non-boolean-loopWhen2.cwl index ba13e38ca..021752b18 100644 --- a/tests/loop/invalid-non-boolean-loopWhen2.cwl +++ b/tests/loop/invalid-non-boolean-loopWhen2.cwl @@ -26,7 +26,7 @@ steps: when: '$(inputs.i1 == 1 ? true : "I am a string")' loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/invalid-value-from-loop-no-requirement.cwl b/tests/loop/invalid-value-from-loop-no-requirement.cwl index c232194bc..dd1b8e384 100644 --- a/tests/loop/invalid-value-from-loop-no-requirement.cwl +++ b/tests/loop/invalid-value-from-loop-no-requirement.cwl @@ -29,4 +29,4 @@ steps: loop: i1: valueFrom: $(inputs.i1 + 1) - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/loop-inside-loop-all.cwl b/tests/loop/loop-inside-loop-all.cwl index bc805c2f5..ef0a9798b 100644 --- a/tests/loop/loop-inside-loop-all.cwl +++ b/tests/loop/loop-inside-loop-all.cwl @@ -46,7 +46,7 @@ steps: when: $(inputs.i1 <= inputs.i2) loop: i1: o1 - outputMethod: all + outputMethod: all_iterations in: i1: i1 i2: i2 @@ -55,4 +55,4 @@ steps: loop: i2: valueFrom: $(inputs.i2 + 1) - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/loop-inside-loop.cwl b/tests/loop/loop-inside-loop.cwl index 9b9fd7367..df01d114c 100644 --- a/tests/loop/loop-inside-loop.cwl +++ b/tests/loop/loop-inside-loop.cwl @@ -42,7 +42,7 @@ steps: when: $(inputs.i1 <= inputs.i2) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 @@ -51,4 +51,4 @@ steps: loop: i2: valueFrom: $(inputs.i2 + 1) - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/loop-inside-scatter.cwl b/tests/loop/loop-inside-scatter.cwl index 3fc7fe257..fbb60c9cf 100644 --- a/tests/loop/loop-inside-scatter.cwl +++ b/tests/loop/loop-inside-scatter.cwl @@ -41,7 +41,7 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration in: i1: i1 i2: i2 diff --git a/tests/loop/multi-source-loop.cwl b/tests/loop/multi-source-loop.cwl index 8c2a271cf..504e1d4c2 100644 --- a/tests/loop/multi-source-loop.cwl +++ b/tests/loop/multi-source-loop.cwl @@ -62,4 +62,4 @@ steps: i1: outputSource: [osmall, obig] pickValue: the_only_non_null - outputMethod: all + outputMethod: all_iterations diff --git a/tests/loop/opt-var-loop.cwl b/tests/loop/opt-var-loop.cwl index d74f7c225..626c86788 100644 --- a/tests/loop/opt-var-loop.cwl +++ b/tests/loop/opt-var-loop.cwl @@ -27,4 +27,4 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/scatter-inside-loop.cwl b/tests/loop/scatter-inside-loop.cwl index ab857b0d0..f3d2370cc 100644 --- a/tests/loop/scatter-inside-loop.cwl +++ b/tests/loop/scatter-inside-loop.cwl @@ -46,4 +46,4 @@ steps: when: $(inputs.i1[0] < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/single-var-loop-no-iteration.cwl b/tests/loop/single-var-loop-no-iteration.cwl index a5678e735..a39fcf583 100644 --- a/tests/loop/single-var-loop-no-iteration.cwl +++ b/tests/loop/single-var-loop-no-iteration.cwl @@ -26,4 +26,4 @@ steps: when: $(inputs.i1 < 1) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/single-var-loop.cwl b/tests/loop/single-var-loop.cwl index ef3819ee1..3af765744 100644 --- a/tests/loop/single-var-loop.cwl +++ b/tests/loop/single-var-loop.cwl @@ -26,4 +26,4 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/two-vars-loop-2.cwl b/tests/loop/two-vars-loop-2.cwl index 7e50afe02..3542534b0 100644 --- a/tests/loop/two-vars-loop-2.cwl +++ b/tests/loop/two-vars-loop-2.cwl @@ -28,4 +28,4 @@ steps: when: $(inputs.i1 < 10) loop: i1: o1 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/two-vars-loop.cwl b/tests/loop/two-vars-loop.cwl index 15b659e4a..945bea8f1 100644 --- a/tests/loop/two-vars-loop.cwl +++ b/tests/loop/two-vars-loop.cwl @@ -30,4 +30,4 @@ steps: loop: i1: o1 i2: o2 - outputMethod: last + outputMethod: last_iteration diff --git a/tests/loop/value-from-loop.cwl b/tests/loop/value-from-loop.cwl index 545029bd1..457682cc3 100644 --- a/tests/loop/value-from-loop.cwl +++ b/tests/loop/value-from-loop.cwl @@ -30,4 +30,4 @@ steps: loop: i1: valueFrom: $(inputs.i1 + 1) - outputMethod: last + outputMethod: last_iteration diff --git a/tests/test_loop.py b/tests/test_loop.py index 269251627..bf908196d 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -143,7 +143,7 @@ def test_loop_two_variables_single_backpropagation() -> None: def test_loop_with_all_output_method() -> None: - """Test a loop case with outputMethod set to all.""" + """Test a loop case with outputMethod set to all_iterations.""" stream = StringIO() params = [ "--enable-dev", @@ -156,7 +156,7 @@ def test_loop_with_all_output_method() -> None: def test_loop_with_all_output_method_no_iteration() -> None: - """Test a loop case with outputMethod set to all and a false 'when' condition.""" + """Test a loop case with outputMethod set to all_iterations and a false 'when' condition.""" stream = StringIO() params = [ "--enable-dev", @@ -244,7 +244,7 @@ def test_nested_loops() -> None: def test_nested_loops_all() -> None: - """Test a workflow with two nested loops, both with outputMethod set to all.""" + """Test a workflow with two nested loops, both with outputMethod set to all_iterations.""" stream = StringIO() params = [ "--enable-dev", diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 0532f3cde..01ab7fe17 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -7,14 +7,14 @@ from ruamel.yaml.comments import CommentedMap from schema_salad.sourceline import cmap +from cwltool.builder import content_limit_respected_read from cwltool.command_line_tool import CommandLineTool from cwltool.context import LoadingContext, RuntimeContext +from cwltool.errors import WorkflowException from cwltool.main import main from cwltool.stdfsaccess import StdFsAccess from cwltool.update import INTERNAL_VERSION -from cwltool.utils import CWLObjectType, CONTENT_LIMIT, bytes2str_in_dicts -from cwltool.builder import content_limit_respected_read -from cwltool.errors import WorkflowException +from cwltool.utils import CONTENT_LIMIT, CWLObjectType, bytes2str_in_dicts from .util import needs_docker