Updated outputMethod values

Author: Iacopo Colonnelli

cwltool/checker.py

@@ -517,10 +517,13 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
 
 
 def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
-    """Check if a step contains a `loop` directive with `all` outputMethod."""
+    """Check if a step contains a `loop` directive with `all_iterations` outputMethod."""
     source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
     if source_step is not None:
-        if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
+        if (
+            source_step.get("loop") is not None
+            and source_step.get("outputMethod") == "all_iterations"
+        ):
             return True
     return False
 

cwltool/errors.py

@@ -8,10 +8,8 @@
 
 # flake8: noqa: F401
 
-from cwl_utils.errors import WorkflowException as WorkflowException
-
-
 from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException
+from cwl_utils.errors import WorkflowException as WorkflowException
 
 
 class UnsupportedRequirement(WorkflowException):

cwltool/schemas/v1.3.0-dev1/Workflow.yml

@@ -497,8 +497,8 @@ $graph:
   docParent: "#LoopWorkflowStep"
   doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep).
   symbols:
-    - last
-    - all
+    - last_iteration
+    - all_iterations
 
 
 - name: AbstractWorkflowStep
@@ -705,14 +705,14 @@ $graph:
     The `outputMethod` field describes how to deal with loop outputs after
     termination:
 
-      * **last** specifies that only the last computed element for each output
-        parameter should be propagated to the subsequent steps. This is the
-        default value.
+      * **last_iteration** specifies that only the last computed element for
+        each output parameter should be propagated to the subsequent steps.
+        This is the default value.
 
-      * **all** specifies that an array with all output values computed at the
-        end of each loop iteration should be propagated to the subsequent steps.
-        Elements in the array must be ordered according to the loop iterations
-        that produced them.
+      * **all_iterations** specifies that an array with all output values
+        computed at the end of each loop iteration should be propagated to
+        the subsequent steps. Elements in the array must be ordered according
+        to the loop iterations that produced them.
 
     Iterative execution in CWL is an optional feature and is not required
     to be implemented by all consumers of CWL documents. An implementation that
@@ -734,9 +734,9 @@ $graph:
         mapPredicate: outputSource
     - name: outputMethod
       doc: |
-        If not specified, the default method is "last".
+        If not specified, the default method is "last_iteration".
       type: LoopOutputMethod?
-      default: last
+      default: last_iteration
       jsonldPredicate:
         "_id": "cwl:outputMethod"
         "_type": "@vocab"
@@ -748,7 +748,8 @@ $graph:
         Only run the next iteration when the expression evaluates to `true`.
         If the first iteration evaluates to `false` the step is skipped.
         A skipped step produces a `null` on each output if the `outputMethod`
-        is set to `last`, and an empty array if the `outputMethod` is set to `all`.
+        is set to `last_iteration`, and an empty array if the `outputMethod`
+        is set to `all_iterations`.
 
 
 - name: Workflow

cwltool/update.py

@@ -11,12 +11,11 @@
     cast,
 )
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import SourceLine
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 from .loghandler import _logger
 from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field
 
@@ -51,7 +50,16 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None:
                                     el["outputSource"] = source
                             s["loop"] = r["loop"]
                         if "outputMethod" in r:
-                            s["outputMethod"] = r["outputMethod"]
+                            if r["outputMethod"] == "all":
+                                s["outputMethod"] = "all_iterations"
+                            elif r["outputMethod"] == "last":
+                                s["outputMethod"] = "last_iteration"
+                            else:
+                                raise SourceLine(
+                                    r, raise_type=ValidationException
+                                ).makeError(  # pragma: no cover
+                                    f"Invalid value {r['outputMethod']} for `outputMethod`."
+                                )
                         cast(
                             MutableSequence[CWLObjectType],
                             s["requirements"],

cwltool/workflow_job.py

@@ -744,7 +744,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType]
                     _logger.info("[%s] will be skipped", step.name)
                     if (
                         step.tool.get("loop") is not None
-                        and step.tool.get("outputMethod", "last") == "all"
+                        and step.tool.get("outputMethod", "last_iteration") == "all_iterations"
                     ):
                         callback({k["id"]: [] for k in outputparms}, "skipped")
                     else:
@@ -874,7 +874,7 @@ def _set_empty_output(self, outputMethod: str) -> None:
         for i in self.step.tool["outputs"]:
             if "id" in i:
                 iid = cast(str, i["id"])
-                if outputMethod == "all":
+                if outputMethod == "all_iterations":
                     self.output_buffer[iid] = cast(MutableSequence[Optional[CWLOutputType]], [])
                 else:
                     self.output_buffer[iid] = None
@@ -887,7 +887,7 @@ def job(
     ) -> JobsGeneratorType:
         """Generate a WorkflowJobStep job until the `when` condition evaluates to False."""
         self.joborder = joborder
-        outputMethod = self.step.tool.get("outputMethod", "last")
+        outputMethod = self.step.tool.get("outputMethod", "last_iteration")
 
         callback = functools.partial(
             self.loop_callback,
@@ -953,14 +953,14 @@ def loop_callback(
         self.iteration += 1
         try:
             loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", []))
-            outputMethod = self.step.tool.get("outputMethod", "last")
+            outputMethod = self.step.tool.get("outputMethod", "last_iteration")
             state: Dict[str, Optional[WorkflowStateItem]] = {}
             for i in self.step.tool["outputs"]:
                 if "id" in i:
                     iid = cast(str, i["id"])
                     if iid in jobout:
                         state[iid] = WorkflowStateItem(i, jobout[iid], processStatus)
-                        if outputMethod == "all":
+                        if outputMethod == "all_iterations":
                             if iid not in self.output_buffer:
                                 self.output_buffer[iid] = cast(
                                     MutableSequence[Optional[CWLOutputType]], []

tests/loop/all-output-loop-no-iteration.cwl

@@ -25,4 +25,4 @@ steps:
     when: $(inputs.i1 < 1)
     loop:
       i1: o1
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/all-output-loop.cwl

@@ -25,4 +25,4 @@ steps:
     when: $(inputs.i1 < 10)
     loop:
       i1: o1
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/default-value-loop.cwl

@@ -44,4 +44,4 @@ steps:
       i1:
         outputSource: o1
         default: 5
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/invalid-loop-scatter.cwl

@@ -26,7 +26,7 @@ steps:
     when: $(inputs.i1 < 10)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-loop-when-exception.cwl

@@ -29,7 +29,7 @@ steps:
       }
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-loop-when-exception2.cwl

@@ -33,7 +33,7 @@ steps:
       }
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-multi-source-loop-no-requirement.cwl

@@ -61,4 +61,4 @@ steps:
       i1:
         loopSource: [osmall, obig]
         pickValue: the_only_non_null
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/invalid-no-loopWhen.cwl

@@ -25,7 +25,7 @@ steps:
         ${return {'o1': inputs.i1 + inputs.i2};}
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-non-boolean-loopWhen.cwl

@@ -26,7 +26,7 @@ steps:
     when: $(inputs.i1)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-non-boolean-loopWhen2.cwl

@@ -26,7 +26,7 @@ steps:
     when: '$(inputs.i1 == 1 ? true : "I am a string")'
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/invalid-value-from-loop-no-requirement.cwl

@@ -29,4 +29,4 @@ steps:
     loop:
       i1:
         valueFrom: $(inputs.i1 + 1)
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/loop-inside-loop-all.cwl

@@ -46,7 +46,7 @@ steps:
           when: $(inputs.i1 <= inputs.i2)
           loop:
             i1: o1
-          outputMethod: all
+          outputMethod: all_iterations
     in:
       i1: i1
       i2: i2
@@ -55,4 +55,4 @@ steps:
     loop:
       i2:
         valueFrom: $(inputs.i2 + 1)
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/loop-inside-loop.cwl

@@ -42,7 +42,7 @@ steps:
           when: $(inputs.i1 <= inputs.i2)
           loop:
             i1: o1
-          outputMethod: last
+          outputMethod: last_iteration
     in:
       i1: i1
       i2: i2
@@ -51,4 +51,4 @@ steps:
     loop:
       i2:
         valueFrom: $(inputs.i2 + 1)
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/loop-inside-scatter.cwl

@@ -41,7 +41,7 @@ steps:
           when: $(inputs.i1 < 10)
           loop:
             i1: o1
-          outputMethod: last
+          outputMethod: last_iteration
     in:
       i1: i1
       i2: i2

tests/loop/multi-source-loop.cwl

@@ -62,4 +62,4 @@ steps:
       i1:
         outputSource: [osmall, obig]
         pickValue: the_only_non_null
-    outputMethod: all
+    outputMethod: all_iterations

tests/loop/opt-var-loop.cwl

@@ -27,4 +27,4 @@ steps:
     when: $(inputs.i1 < 10)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/scatter-inside-loop.cwl

@@ -46,4 +46,4 @@ steps:
     when: $(inputs.i1[0] < 10)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/single-var-loop-no-iteration.cwl

@@ -26,4 +26,4 @@ steps:
     when: $(inputs.i1 < 1)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/single-var-loop.cwl

@@ -26,4 +26,4 @@ steps:
     when: $(inputs.i1 < 10)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/two-vars-loop-2.cwl

@@ -28,4 +28,4 @@ steps:
     when: $(inputs.i1 < 10)
     loop:
       i1: o1
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/two-vars-loop.cwl

@@ -30,4 +30,4 @@ steps:
     loop:
       i1: o1
       i2: o2
-    outputMethod: last
+    outputMethod: last_iteration

tests/loop/value-from-loop.cwl

@@ -30,4 +30,4 @@ steps:
     loop:
       i1:
         valueFrom: $(inputs.i1 + 1)
-    outputMethod: last
+    outputMethod: last_iteration

tests/test_loop.py

@@ -143,7 +143,7 @@ def test_loop_two_variables_single_backpropagation() -> None:
 
 
 def test_loop_with_all_output_method() -> None:
-    """Test a loop case with outputMethod set to all."""
+    """Test a loop case with outputMethod set to all_iterations."""
     stream = StringIO()
     params = [
         "--enable-dev",
@@ -156,7 +156,7 @@ def test_loop_with_all_output_method() -> None:
 
 
 def test_loop_with_all_output_method_no_iteration() -> None:
-    """Test a loop case with outputMethod set to all and a false 'when' condition."""
+    """Test a loop case with outputMethod set to all_iterations and a false 'when' condition."""
     stream = StringIO()
     params = [
         "--enable-dev",
@@ -244,7 +244,7 @@ def test_nested_loops() -> None:
 
 
 def test_nested_loops_all() -> None:
-    """Test a workflow with two nested loops, both with outputMethod set to all."""
+    """Test a workflow with two nested loops, both with outputMethod set to all_iterations."""
     stream = StringIO()
     params = [
         "--enable-dev",

tests/test_path_checks.py

@@ -7,14 +7,14 @@
 from ruamel.yaml.comments import CommentedMap
 from schema_salad.sourceline import cmap
 
+from cwltool.builder import content_limit_respected_read
 from cwltool.command_line_tool import CommandLineTool
 from cwltool.context import LoadingContext, RuntimeContext
+from cwltool.errors import WorkflowException
 from cwltool.main import main
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.update import INTERNAL_VERSION
-from cwltool.utils import CWLObjectType, CONTENT_LIMIT, bytes2str_in_dicts
-from cwltool.builder import content_limit_respected_read
-from cwltool.errors import WorkflowException
+from cwltool.utils import CONTENT_LIMIT, CWLObjectType, bytes2str_in_dicts
 
 from .util import needs_docker