Override identifiers are relative to workflow, other identifiers are relative

to job document.

Author: Peter Amstutz

cwltool/load_tool.py

@@ -38,9 +38,8 @@
     u"id": u"@id"
 }
 
-_cwldl = get_schema("v1.0")[0]
-overrides_ctx = copy.copy(_cwldl.ctx)
-overrides_ctx.update({
+
+overrides_ctx = {
     u"overrideTarget": {u"@type": u"@id"},
     u"cwltool": "http://commonwl.org/cwltool#",
     u"overrides": {
@@ -52,10 +51,13 @@
         "@id": "cwltool:override",
         "mapSubject": "class"
     }
-})
+}
 
 def resolve_tool_uri(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
-                    resolver=None,  # type: Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]
+                     resolver=None,  # type: Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]
+                     fetcher_constructor=None,
+                     # type: Callable[[Dict[Text, Text], requests.sessions.Session], Fetcher]
+                     document_loader=None  # type: Loader
 ):
     # type: (...) -> Tuple[Text, Text]
 
@@ -67,6 +69,8 @@ def resolve_tool_uri(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     elif os.path.exists(os.path.abspath(argsworkflow)):
         uri = file_uri(str(os.path.abspath(argsworkflow)))
     elif resolver:
+        if document_loader is None:
+            document_loader = Loader(jobloaderctx, fetcher_constructor=fetcher_constructor)  # type: ignore
         uri = resolver(document_loader, argsworkflow)
 
     if uri is None:
@@ -76,7 +80,7 @@ def resolve_tool_uri(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
         _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
 
     fileuri = urllib.parse.urldefrag(uri)[0]
-    return (uri, fileuri)
+    return uri, fileuri
 
 
 def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
@@ -92,7 +96,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap
     if isinstance(argsworkflow, string_types):
-        uri, fileuri = resolve_tool_uri(argsworkflow, resolver)
+        uri, fileuri = resolve_tool_uri(argsworkflow, resolver=resolver, document_loader=document_loader)
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
         uri = "#" + Text(id(argsworkflow))
@@ -190,7 +194,7 @@ def validate_document(document_loader,  # type: Loader
         del cast(dict, jobobj)["https://w3id.org/cwl/cwl#tool"]
 
         if "http://commonwl.org/cwltool#overrides" in jobobj:
-            overrides.extend(resolve_overrides(jobobj, uri))
+            overrides.extend(resolve_overrides(jobobj, uri, uri))
             del jobobj["http://commonwl.org/cwltool#overrides"]
 
         workflowobj = fetch_document(uri, fetcher_constructor=fetcher_constructor)[1]
@@ -329,15 +333,15 @@ def load_tool(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     return make_tool(document_loader, avsc_names, metadata, uri,
                      makeTool, kwargs if kwargs else {})
 
-def resolve_overrides(ov, baseurl):  # type: (CommentedMap, Text) -> List[Dict[Text, Any]]
+def resolve_overrides(ov, ov_uri, baseurl):  # type: (CommentedMap, Text) -> List[Dict[Text, Any]]
     ovloader = Loader(overrides_ctx)
-    del ov["id"]
-    del ov["name"]
     ret, _ = ovloader.resolve_all(ov, baseurl)
     if not isinstance(ret, CommentedMap):
         raise Exception("Expected CommentedMap, got %s" % type(ret))
+    cwl_docloader = get_schema("v1.0")[0]
+    cwl_docloader.resolve_all(ret, ov_uri)
     return ret["overrides"]
 
 def load_overrides(ov, base_url):  # type: (Text, Text) -> List[Dict[Text, Any]]
     ovloader = Loader(overrides_ctx)
-    return resolve_overrides(ovloader.fetch(ov), base_url)
+    return resolve_overrides(ovloader.fetch(ov), ov, base_url)

cwltool/main.py

@@ -544,7 +544,7 @@ def load_job_order(args,   # type: argparse.Namespace
         job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
 
     if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object:
-       overrides.extend(resolve_overrides(job_order_object, tool_file_uri))
+       overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file), tool_file_uri))
        del job_order_object["http://commonwl.org/cwltool#overrides"]
 
     if not job_order_object:
@@ -835,7 +835,9 @@ def main(argsl=None,  # type: List[str]
         else:
             use_standard_schema("v1.0")
 
-        uri, tool_file_uri = resolve_tool_uri(args.workflow, resolver)
+        uri, tool_file_uri = resolve_tool_uri(args.workflow,
+                                              resolver=resolver,
+                                              fetcher_constructor=fetcher_constructor)
 
         overrides = []  # type: List[Dict[Text, Any]]
 

tests/test_fetch.py

@@ -27,7 +27,7 @@ def fetch_text(self, url):  # type: (unicode) -> unicode
 outputs: []
 """
                 else:
-                    raise RuntimeError("Not foo.cwl")
+                    raise RuntimeError("Not foo.cwl, was %s" % url)
 
             def check_exists(self, url):  # type: (unicode) -> bool
                 if url == "baz:bar/foo.cwl":
@@ -46,7 +46,11 @@ def urljoin(self, base, url):
                     return urllib.parse.urljoin(base, url)
 
         def test_resolver(d, a):
-            return "baz:bar/" + a
+            if a.startswith("baz:bar/"):
+                return a
+            else:
+                return "baz:bar/" + a
+
 
         load_tool("foo.cwl", defaultMakeTool, resolver=test_resolver, fetcher_constructor=TestFetcher)