Skip to content

Commit 52710a1

Browse files
GlassOfWhiskeyGlassOfWhiskey
committed
Added loop support
1 parent e1a9100 commit 52710a1

File tree

6 files changed

+234
-75
lines changed

6 files changed

+234
-75
lines changed

‎cwltool/checker.py

Lines changed: 12 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -545,42 +545,37 @@ def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -
545545
def is_all_output_method_loop_step(
546546
param_to_step: Dict[str, CWLObjectType], parm_id: str
547547
) -> bool:
548-
"""Check if a step contains a http://commonwl.org/cwltool#Loop requirement with `all` outputMethod."""
548+
"""Check if a step contains a `loop` directive with `all` outputMethod."""
549549
source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id)
550550
if source_step is not None:
551-
for requirement in source_step.get("requirements", []):
552-
if (
553-
requirement["class"] == "http://commonwl.org/cwltool#Loop"
554-
and requirement.get("outputMethod") == "all"
555-
):
556-
return True
551+
if (
552+
source_step.get("loop") is not None
553+
and source_step.get("outputMethod") == "all"
554+
):
555+
return True
557556
return False
558557

559558

560559
def loop_checker(steps: Iterator[MutableMapping[str, Any]]) -> None:
561560
"""
562-
Check http://commonwl.org/cwltool#Loop requirement compatibility with other directives.
561+
Check `loop` compatibility with other directives.
563562
564563
:raises:
565-
ValidationException: If there is an incompatible combination between cwltool:loop and 'scatter' or 'when'.
564+
ValidationException: If there is an incompatible combination between `loop` and `scatter`.
566565
"""
567566
exceptions = []
568567
for step in steps:
569-
requirements = {
570-
**{h["class"]: h for h in step.get("hints", [])},
571-
**{r["class"]: r for r in step.get("requirements", [])},
572-
}
573-
if "http://commonwl.org/cwltool#Loop" in requirements:
574-
if "when" in step:
568+
if "loop" in step:
569+
if "when" not in step:
575570
exceptions.append(
576571
SourceLine(step, "id").makeError(
577-
"The `cwltool:Loop` clause is not compatible with the `when` directive."
572+
"The `when` clause is mandatory when the `loop` directive is defined."
578573
)
579574
)
580575
if "scatter" in step:
581576
exceptions.append(
582577
SourceLine(step, "id").makeError(
583-
"The `cwltool:Loop` clause is not compatible with the `scatter` directive."
578+
"The `loop` clause is not compatible with the `scatter` directive."
584579
)
585580
)
586581
if exceptions:

‎cwltool/extensions-v1.3.yml

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
$base: http://commonwl.org/cwltool#
2+
$namespaces:
3+
cwl: "https://w3id.org/cwl/cwl#"
4+
$graph:
5+
- $import: https://w3id.org/cwl/CommonWorkflowLanguage.yml
6+
7+
- name: Secrets
8+
type: record
9+
inVocab: false
10+
extends: cwl:ProcessRequirement
11+
fields:
12+
class:
13+
type: string
14+
doc: "Always 'Secrets'"
15+
jsonldPredicate:
16+
"_id": "@type"
17+
"_type": "@vocab"
18+
secrets:
19+
type: string[]
20+
doc: |
21+
List one or more input parameters that are sensitive (such as passwords)
22+
which will be deliberately obscured from logging.
23+
jsonldPredicate:
24+
"_type": "@id"
25+
refScope: 0
26+
27+
28+
- name: ProcessGenerator
29+
type: record
30+
inVocab: true
31+
extends: cwl:Process
32+
documentRoot: true
33+
fields:
34+
- name: class
35+
jsonldPredicate:
36+
"_id": "@type"
37+
"_type": "@vocab"
38+
type: string
39+
- name: run
40+
type: [string, cwl:Process]
41+
jsonldPredicate:
42+
_id: "cwl:run"
43+
_type: "@id"
44+
subscope: run
45+
doc: |
46+
Specifies the process to run.
47+
48+
- name: MPIRequirement
49+
type: record
50+
inVocab: false
51+
extends: cwl:ProcessRequirement
52+
doc: |
53+
Indicates that a process requires an MPI runtime.
54+
fields:
55+
- name: class
56+
type: string
57+
doc: "Always 'MPIRequirement'"
58+
jsonldPredicate:
59+
"_id": "@type"
60+
"_type": "@vocab"
61+
- name: processes
62+
type: [int, cwl:Expression]
63+
doc: |
64+
The number of MPI processes to start. If you give a string,
65+
this will be evaluated as a CWL Expression and it must
66+
evaluate to an integer.
67+
68+
- name: CUDARequirement
69+
type: record
70+
extends: cwl:ProcessRequirement
71+
inVocab: false
72+
doc: |
73+
Require support for NVIDA CUDA (GPU hardware acceleration).
74+
fields:
75+
class:
76+
type: string
77+
doc: 'cwltool:CUDARequirement'
78+
jsonldPredicate:
79+
_id: "@type"
80+
_type: "@vocab"
81+
cudaVersionMin:
82+
type: string
83+
doc: |
84+
Minimum CUDA version to run the software, in X.Y format. This
85+
corresponds to a CUDA SDK release. When running directly on
86+
the host (not in a container) the host must have a compatible
87+
CUDA SDK (matching the exact version, or, starting with CUDA
88+
11.3, matching major version). When run in a container, the
89+
container image should provide the CUDA runtime, and the host
90+
driver is injected into the container. In this case, because
91+
CUDA drivers are backwards compatible, it is possible to
92+
use an older SDK with a newer driver across major versions.
93+
94+
See https://docs.nvidia.com/deploy/cuda-compatibility/ for
95+
details.
96+
cudaComputeCapability:
97+
type:
98+
- 'string'
99+
- 'string[]'
100+
doc: |
101+
CUDA hardware capability required to run the software, in X.Y
102+
format.
103+
104+
* If this is a single value, it defines only the minimum
105+
compute capability. GPUs with higher capability are also
106+
accepted.
107+
108+
* If it is an array value, then only select GPUs with compute
109+
capabilities that explicitly appear in the array.
110+
cudaDeviceCountMin:
111+
type: ['null', int, cwl:Expression]
112+
default: 1
113+
doc: |
114+
Minimum number of GPU devices to request. If not specified,
115+
same as `cudaDeviceCountMax`. If neither are specified,
116+
default 1.
117+
cudaDeviceCountMax:
118+
type: ['null', int, cwl:Expression]
119+
doc: |
120+
Maximum number of GPU devices to request. If not specified,
121+
same as `cudaDeviceCountMin`.

‎cwltool/process.py

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1075,10 +1075,6 @@ def validate_hints(
10751075
sl = SourceLine(hints, i, ValidationException, debug)
10761076
with sl:
10771077
classname = cast(str, r["class"])
1078-
if classname == "http://commonwl.org/cwltool#Loop":
1079-
raise ValidationException(
1080-
"http://commonwl.org/cwltool#Loop is valid only under requirements."
1081-
)
10821078
avroname = classname
10831079
if classname in self.doc_loader.vocab:
10841080
avroname = avro_type_name(self.doc_loader.vocab[classname])

‎cwltool/update.py

Lines changed: 76 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,69 @@
2020
from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field
2121

2222

23+
def v1_2to1_3(
24+
doc: CommentedMap, loader: Loader, baseuri: str
25+
) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument
26+
"""Public updater for v1.2 to v1.3."""
27+
doc = copy.deepcopy(doc)
28+
29+
def rewrite_loop_requirements(t: CWLObjectType) -> None:
30+
if "steps" in t:
31+
for s in cast(MutableSequence[CWLObjectType], t["steps"]):
32+
if isinstance(s, MutableMapping):
33+
if "requirements" in s:
34+
for i, r in enumerate(
35+
list(
36+
cast(MutableSequence[CWLObjectType], s["requirements"])
37+
)
38+
):
39+
if isinstance(r, MutableMapping):
40+
cls = cast(str, r["class"])
41+
if cls == "http://commonwl.org/cwltool#Loop":
42+
if "when" in s:
43+
raise ValidationException(
44+
"The `cwltool:Loop` clause is not compatible with the `when` directive."
45+
)
46+
if "loopWhen" not in r:
47+
raise ValidationException(
48+
"The `loopWhen` clause is mandatory within the `cwltool:Loop` requirement."
49+
)
50+
s["when"] = r["loopWhen"]
51+
if "loop" in r:
52+
s["loop"] = r["loop"]
53+
if "outputMethod" in r:
54+
s["outputMethod"] = r["outputMethod"]
55+
cast(
56+
MutableSequence[CWLObjectType],
57+
s["requirements"],
58+
).pop(index=i)
59+
else:
60+
raise ValidationException(
61+
"requirements entries must be dictionaries: {} {}.".format(
62+
type(r), r
63+
)
64+
)
65+
if "hints" in s:
66+
for r in cast(MutableSequence[CWLObjectType], s["hints"]):
67+
if isinstance(r, MutableMapping):
68+
cls = cast(str, r["class"])
69+
if cls == "http://commonwl.org/cwltool#Loop":
70+
raise ValidationException(
71+
"http://commonwl.org/cwltool#Loop is valid only under requirements."
72+
)
73+
else:
74+
raise ValidationException(
75+
f"hints entries must be dictionaries: {type(r)} {r}."
76+
)
77+
else:
78+
raise ValidationException(
79+
f"steps entries must be dictionaries: {type(s)} {s}."
80+
)
81+
82+
visit_class(doc, "Workflow", rewrite_loop_requirements)
83+
return (doc, "v1.3")
84+
85+
2386
def v1_1to1_2(
2487
doc: CommentedMap, loader: Loader, baseuri: str
2588
) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument
@@ -206,6 +269,13 @@ def v1_2_0dev5to1_2(
206269
return (doc, "v1.2")
207270

208271

272+
def v1_3_0dev1to1_3(
273+
doc: CommentedMap, loader: Loader, baseuri: str
274+
) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument
275+
"""Public updater for v1.3.0-dev1 to v1.3."""
276+
return (doc, "v1.3")
277+
278+
209279
ORDERED_VERSIONS = [
210280
"v1.0",
211281
"v1.1.0-dev1",
@@ -216,12 +286,15 @@ def v1_2_0dev5to1_2(
216286
"v1.2.0-dev4",
217287
"v1.2.0-dev5",
218288
"v1.2",
289+
"v1.3.0-dev1",
290+
"v1.3",
219291
]
220292

221293
UPDATES = {
222294
"v1.0": v1_0to1_1,
223295
"v1.1": v1_1to1_2,
224-
"v1.2": None,
296+
"v1.2": v1_2to1_3,
297+
"v1.3": None,
225298
} # type: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]]
226299

227300
DEVUPDATES = {
@@ -231,13 +304,14 @@ def v1_2_0dev5to1_2(
231304
"v1.2.0-dev3": v1_2_0dev3todev4,
232305
"v1.2.0-dev4": v1_2_0dev4todev5,
233306
"v1.2.0-dev5": v1_2_0dev5to1_2,
307+
"v1.3.0-dev1": v1_3_0dev1to1_3,
234308
} # type: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]]
235309

236310

237311
ALLUPDATES = UPDATES.copy()
238312
ALLUPDATES.update(DEVUPDATES)
239313

240-
INTERNAL_VERSION = "v1.2"
314+
INTERNAL_VERSION = "v1.3"
241315

242316
ORIGINAL_CWLVERSION = "http://commonwl.org/cwltool#original_cwlVersion"
243317

‎cwltool/workflow.py

Lines changed: 1 addition & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,11 @@
66
from typing import (
77
Callable,
88
Dict,
9-
Iterable,
109
List,
1110
Mapping,
1211
MutableMapping,
1312
MutableSequence,
1413
Optional,
15-
Union,
1614
cast,
1715
)
1816
from uuid import UUID
@@ -31,7 +29,6 @@
3129
from .provenance_profile import ProvenanceProfile
3230
from .utils import (
3331
CWLObjectType,
34-
CWLOutputType,
3532
JobsGeneratorType,
3633
OutputCallbackType,
3734
StepType,
@@ -222,10 +219,7 @@ def __init__(
222219
if parent_req["class"] == step_req["class"]:
223220
found_in_step = True
224221
break
225-
if (
226-
not found_in_step
227-
and parent_req.get("class") != "http://commonwl.org/cwltool#Loop"
228-
):
222+
if not found_in_step:
229223
loadingContext.requirements.append(parent_req)
230224
loadingContext.requirements.extend(
231225
cast(
@@ -418,16 +412,6 @@ def __init__(
418412
else:
419413
self.parent_wf = self.prov_obj
420414

421-
def checkRequirements(
422-
self,
423-
rec: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType, None],
424-
supported_process_requirements: Iterable[str],
425-
) -> None:
426-
"""Check the presence of unsupported requirements."""
427-
supported_process_requirements = list(supported_process_requirements)
428-
supported_process_requirements.append("http://commonwl.org/cwltool#Loop")
429-
super().checkRequirements(rec, supported_process_requirements)
430-
431415
def receive_output(
432416
self,
433417
output_callback: OutputCallbackType,

0 commit comments

Comments
 (0)