Merge pull request #2045 from common-workflow-language/main

Sync Updates to prov_data_input_output branch

Author: ElderMedic

.github/workflows/ci-tests.yml

@@ -32,7 +32,7 @@ jobs:
     strategy:
       matrix:
         py-ver-major: [3]
-        py-ver-minor: [8, 9, 10, 11, 12]
+        py-ver-minor: [8, 9, 10, 11, 12, 13]
         step: [lint, unit, bandit, mypy]
 
     env:

.gitignore

@@ -1,6 +1,8 @@
 # Generated during tests
 pytestdebug.log
 tmp/
+*.sif
+involucro
 
 # Python temps
 __pycache__/
@@ -59,4 +61,3 @@ cwltool/_version.py
 cwltool_deps
 docs/_build/
 docs/autoapi/
-

README.rst

@@ -1,5 +1,5 @@
 #############################################################################################
-``cwltool``: The reference reference implementation of the Common Workflow Language standards
+``cwltool``: The reference implementation of the Common Workflow Language standards
 #############################################################################################
 
 |Linux Status| |Coverage Status| |Docs Status|

cwltool/argparser.py

@@ -921,6 +921,11 @@ def add_argument(
             action = DirectoryAppendAction
         else:
             action = AppendAction
+            items = inptype["items"]
+            if items == "int" or items == "long":
+                atype = int
+            elif items == "double" or items == "float":
+                atype = float
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "enum":
         atype = str
     elif isinstance(inptype, MutableMapping) and inptype["type"] == "record":

cwltool/builder.py

@@ -282,7 +282,7 @@ def bind_input(
                 and "itemSeparator" not in binding
             ):
                 st["inputBinding"] = {}
-            for k in ("secondaryFiles", "format", "streamable"):
+            for k in ("secondaryFiles", "format", "streamable", "loadContents"):
                 if k in schema:
                     st[k] = schema[k]
             if value_from_expression:
@@ -349,7 +349,7 @@ def bind_input(
                         "type": schema["items"],
                         "inputBinding": b2,
                     }
-                    for k in ("secondaryFiles", "format", "streamable"):
+                    for k in ("secondaryFiles", "format", "streamable", "loadContents"):
                         if k in schema:
                             itemschema[k] = schema[k]
                     bindings.extend(

cwltool/checker.py

@@ -517,10 +517,13 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
 
 
 def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool:
-    """Check if a step contains a `loop` directive with `all` outputMethod."""
+    """Check if a step contains a `loop` directive with `all_iterations` outputMethod."""
     source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
     if source_step is not None:
-        if source_step.get("loop") is not None and source_step.get("outputMethod") == "all":
+        if (
+            source_step.get("loop") is not None
+            and source_step.get("outputMethod") == "all_iterations"
+        ):
             return True
     return False
 

cwltool/command_line_tool.py

@@ -806,7 +806,7 @@ def remove_dirname(d: CWLObjectType) -> None:
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> Generator[Union[JobBase, CallbackJob], None, None]:
         workReuse, _ = self.get_requirement("WorkReuse")

cwltool/errors.py

@@ -8,10 +8,8 @@
 
 # flake8: noqa: F401
 
-from cwl_utils.errors import WorkflowException as WorkflowException
-
-
 from cwl_utils.errors import GraphTargetMissingException as GraphTargetMissingException
+from cwl_utils.errors import WorkflowException as WorkflowException
 
 
 class UnsupportedRequirement(WorkflowException):

cwltool/process.py

@@ -1075,7 +1075,7 @@ def visit(self, op: Callable[[CommentedMap], None]) -> None:
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         pass

cwltool/procgenerator.py

@@ -30,7 +30,7 @@ def receive_output(self, jobout: Optional[CWLObjectType], processStatus: str) ->
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         try:
@@ -41,7 +41,7 @@ def job(
             while self.processStatus is None:
                 yield None
 
-            if self.processStatus != "success" and output_callbacks:
+            if self.processStatus != "success":
                 output_callbacks(self.jobout, self.processStatus)
                 return
 
@@ -89,7 +89,7 @@ def __init__(
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         return ProcessGeneratorJob(self).job(job_order, output_callbacks, runtimeContext)

cwltool/schemas/v1.3.0-dev1/Workflow.yml

@@ -497,8 +497,8 @@ $graph:
   docParent: "#LoopWorkflowStep"
   doc: The loop output method, as described in [workflow step loop](#LoopWorkflowStep).
   symbols:
-    - last
-    - all
+    - last_iteration
+    - all_iterations
 
 
 - name: AbstractWorkflowStep
@@ -705,14 +705,14 @@ $graph:
     The `outputMethod` field describes how to deal with loop outputs after
     termination:
 
-      * **last** specifies that only the last computed element for each output
-        parameter should be propagated to the subsequent steps. This is the
-        default value.
+      * **last_iteration** specifies that only the last computed element for
+        each output parameter should be propagated to the subsequent steps.
+        This is the default value.
 
-      * **all** specifies that an array with all output values computed at the
-        end of each loop iteration should be propagated to the subsequent steps.
-        Elements in the array must be ordered according to the loop iterations
-        that produced them.
+      * **all_iterations** specifies that an array with all output values
+        computed at the end of each loop iteration should be propagated to
+        the subsequent steps. Elements in the array must be ordered according
+        to the loop iterations that produced them.
 
     Iterative execution in CWL is an optional feature and is not required
     to be implemented by all consumers of CWL documents. An implementation that
@@ -734,9 +734,9 @@ $graph:
         mapPredicate: outputSource
     - name: outputMethod
       doc: |
-        If not specified, the default method is "last".
+        If not specified, the default method is "last_iteration".
       type: LoopOutputMethod?
-      default: last
+      default: last_iteration
       jsonldPredicate:
         "_id": "cwl:outputMethod"
         "_type": "@vocab"
@@ -748,7 +748,8 @@ $graph:
         Only run the next iteration when the expression evaluates to `true`.
         If the first iteration evaluates to `false` the step is skipped.
         A skipped step produces a `null` on each output if the `outputMethod`
-        is set to `last`, and an empty array if the `outputMethod` is set to `all`.
+        is set to `last_iteration`, and an empty array if the `outputMethod`
+        is set to `all_iterations`.
 
 
 - name: Workflow

cwltool/update.py

@@ -11,12 +11,11 @@
     cast,
 )
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import SourceLine
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-
 from .loghandler import _logger
 from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field
 
@@ -51,7 +50,16 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None:
                                     el["outputSource"] = source
                             s["loop"] = r["loop"]
                         if "outputMethod" in r:
-                            s["outputMethod"] = r["outputMethod"]
+                            if r["outputMethod"] == "all":
+                                s["outputMethod"] = "all_iterations"
+                            elif r["outputMethod"] == "last":
+                                s["outputMethod"] = "last_iteration"
+                            else:
+                                raise SourceLine(
+                                    r, raise_type=ValidationException
+                                ).makeError(  # pragma: no cover
+                                    f"Invalid value {r['outputMethod']} for `outputMethod`."
+                                )
                         cast(
                             MutableSequence[CWLObjectType],
                             s["requirements"],

cwltool/workflow.py

@@ -153,7 +153,7 @@ def make_workflow_step(
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         builder = self._init_job(job_order, runtimeContext)
@@ -420,7 +420,7 @@ def receive_output(
     def job(
         self,
         job_order: CWLObjectType,
-        output_callbacks: Optional[OutputCallbackType],
+        output_callbacks: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         """Initialize sub-workflow as a step in the parent profile."""

cwltool/workflow_job.py

@@ -65,7 +65,7 @@ def __init__(self, step: "WorkflowStep") -> None:
     def job(
         self,
         joborder: CWLObjectType,
-        output_callback: Optional[OutputCallbackType],
+        output_callback: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         runtimeContext = runtimeContext.copy()
@@ -584,7 +584,7 @@ def receive_output(
     def try_make_job(
         self,
         step: WorkflowJobStep,
-        final_output_callback: Optional[OutputCallbackType],
+        final_output_callback: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         container_engine = "docker"
@@ -744,7 +744,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType]
                     _logger.info("[%s] will be skipped", step.name)
                     if (
                         step.tool.get("loop") is not None
-                        and step.tool.get("outputMethod", "last") == "all"
+                        and step.tool.get("outputMethod", "last_iteration") == "all_iterations"
                     ):
                         callback({k["id"]: [] for k in outputparms}, "skipped")
                     else:
@@ -773,7 +773,7 @@ def run(
     def job(
         self,
         joborder: CWLObjectType,
-        output_callback: Optional[OutputCallbackType],
+        output_callback: OutputCallbackType,
         runtimeContext: RuntimeContext,
     ) -> JobsGeneratorType:
         self.state = {}
@@ -848,7 +848,7 @@ def job(
                 else:
                     yield None
 
-        if not self.did_callback and output_callback:
+        if not self.did_callback:
             # could have called earlier on line 336;
             self.do_output_callback(output_callback)
             # depends which one comes first. All steps are completed
@@ -874,7 +874,7 @@ def _set_empty_output(self, outputMethod: str) -> None:
         for i in self.step.tool["outputs"]:
             if "id" in i:
                 iid = cast(str, i["id"])
-                if outputMethod == "all":
+                if outputMethod == "all_iterations":
                     self.output_buffer[iid] = cast(MutableSequence[Optional[CWLOutputType]], [])
                 else:
                     self.output_buffer[iid] = None
@@ -887,7 +887,7 @@ def job(
     ) -> JobsGeneratorType:
         """Generate a WorkflowJobStep job until the `when` condition evaluates to False."""
         self.joborder = joborder
-        outputMethod = self.step.tool.get("outputMethod", "last")
+        outputMethod = self.step.tool.get("outputMethod", "last_iteration")
 
         callback = functools.partial(
             self.loop_callback,
@@ -953,14 +953,14 @@ def loop_callback(
         self.iteration += 1
         try:
             loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", []))
-            outputMethod = self.step.tool.get("outputMethod", "last")
+            outputMethod = self.step.tool.get("outputMethod", "last_iteration")
             state: Dict[str, Optional[WorkflowStateItem]] = {}
             for i in self.step.tool["outputs"]:
                 if "id" in i:
                     iid = cast(str, i["id"])
                     if iid in jobout:
                         state[iid] = WorkflowStateItem(i, jobout[iid], processStatus)
-                        if outputMethod == "all":
+                        if outputMethod == "all_iterations":
                             if iid not in self.output_buffer:
                                 self.output_buffer[iid] = cast(
                                     MutableSequence[Optional[CWLOutputType]], []

lint-requirements.txt

@@ -1,3 +1,3 @@
-flake8-bugbear<24.3
-black~=24.4
+flake8-bugbear<24.9
+black~=24.8
 codespell

mypy-requirements.txt

@@ -1,4 +1,4 @@
-mypy==1.10.1  # also update pyproject.toml
+mypy==1.11.2  # also update pyproject.toml
 ruamel.yaml>=0.16.0,<0.19
 cwl-utils>=0.32
 types-requests

pyproject.toml

@@ -2,7 +2,7 @@
 requires = [
     "setuptools>=45",
     "setuptools_scm[toml]>=8.0.4,<9",
-    "mypy==1.10.1",  # also update mypy-requirements.txt
+    "mypy==1.11.2",  # also update mypy-requirements.txt
     "types-requests",
     "types-psutil",
     "importlib_resources>=1.4;python_version<'3.9'",

setup.py

@@ -145,13 +145,13 @@
             "galaxy-util <24.2",
         ],
     },
-    python_requires=">=3.8, <4",
+    python_requires=">=3.8, <3.14",
     use_scm_version=True,
     setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"],
     test_suite="tests",
     tests_require=[
         "bagit >= 1.6.4, < 1.9",
-        "pytest >= 6.2, < 8.3",
+        "pytest >= 6.2, < 8.4",
         "mock >= 2.0.0",
         "pytest-mock >= 1.10.0",
         "pytest-httpserver",
@@ -176,6 +176,7 @@
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
         "Topic :: Scientific/Engineering",
         "Topic :: Scientific/Engineering :: Bio-Informatics",
         "Topic :: Scientific/Engineering :: Astronomy",

test-requirements.txt

@@ -1,5 +1,5 @@
 bagit>=1.6.4,<1.9
-pytest>= 6.2,< 8.3
+pytest>= 6.2,< 8.4
 pytest-xdist>=3.2.0 # for the worksteal scheduler
 psutil # enhances pytest-xdist to allow "-n logical"
 pytest-httpserver

tests/load_contents-1.txt

@@ -0,0 +1 @@
+1

tests/load_contents-2.txt

@@ -0,0 +1 @@
+2

tests/load_contents-array.cwl

@@ -0,0 +1,24 @@
+cwlVersion: "v1.2"
+class: CommandLineTool
+baseCommand: echo
+requirements:
+  InlineJavascriptRequirement: {}
+inputs:
+  files:
+    type:
+      type: array
+      items: File
+    loadContents: true
+    inputBinding:
+      valueFrom: |
+        ${
+          return JSON.stringify({
+            "data": inputs.files.map(item => parseInt(item.contents))
+          });
+        }
+outputs:
+  out:
+    type: File
+    outputBinding:
+      glob: "data.json"
+stdout: "data.json"