Make override identifiers are relative to the toplevel workflow uri.

Author: Peter Amstutz

README.rst

@@ -405,7 +405,9 @@ that should be applied.
 
 
 Overrides can be specified either on the command line, or as part of the job
-input document.
+input document.  Workflow steps are identified using the name of the workflow
+file followed by the step name as a document fragment identifier "#id".
+Override identifiers are relative to the toplevel workflow document.
 
 .. code:: bash
 
@@ -416,7 +418,7 @@ input document.
   input_parameter1: value1
   input_parameter2: value2
   cwltool:overrides:
-    echo.cwl:
+    workflow.cwl#step1:
       - class: EnvVarRequirement
         envDef:
           MESSAGE: override_value

cwltool/load_tool.py

@@ -54,6 +54,31 @@
     }
 })
 
+def resolve_tool_uri(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
+                    resolver=None,  # type: Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]
+):
+    # type: (...) -> Tuple[Text, Text]
+
+    uri = None  # type: Text
+    split = urllib.parse.urlsplit(argsworkflow)
+    # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that
+    if split.scheme and split.scheme in [u'http',u'https',u'file']:
+        uri = argsworkflow
+    elif os.path.exists(os.path.abspath(argsworkflow)):
+        uri = file_uri(str(os.path.abspath(argsworkflow)))
+    elif resolver:
+        uri = resolver(document_loader, argsworkflow)
+
+    if uri is None:
+        raise ValidationException("Not found: '%s'" % argsworkflow)
+
+    if argsworkflow != uri:
+        _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
+
+    fileuri = urllib.parse.urldefrag(uri)[0]
+    return (uri, fileuri)
+
+
 def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
                    resolver=None,  # type: Callable[[Loader, Union[Text, Dict[Text, Any]]], Text]
                    fetcher_constructor=None
@@ -67,22 +92,7 @@ def fetch_document(argsworkflow,  # type: Union[Text, Dict[Text, Any]]
     uri = None  # type: Text
     workflowobj = None  # type: CommentedMap
     if isinstance(argsworkflow, string_types):
-        split = urllib.parse.urlsplit(argsworkflow)
-        # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that
-        if split.scheme and split.scheme in [u'http',u'https',u'file']:
-            uri = argsworkflow
-        elif os.path.exists(os.path.abspath(argsworkflow)):
-            uri = file_uri(str(os.path.abspath(argsworkflow)))
-        elif resolver:
-            uri = resolver(document_loader, argsworkflow)
-
-        if uri is None:
-            raise ValidationException("Not found: '%s'" % argsworkflow)
-
-        if argsworkflow != uri:
-            _logger.info("Resolved '%s' to '%s'", argsworkflow, uri)
-
-        fileuri = urllib.parse.urldefrag(uri)[0]
+        uri, fileuri = resolve_tool_uri(argsworkflow, resolver)
         workflowobj = document_loader.fetch(fileuri)
     elif isinstance(argsworkflow, dict):
         uri = "#" + Text(id(argsworkflow))
@@ -326,6 +336,6 @@ def resolve_overrides(ov, baseurl):  # type: (CommentedMap, Text) -> List[Dict[T
         raise Exception("Expected CommentedMap, got %s" % type(ret))
     return ret["overrides"]
 
-def load_overrides(ov):  # type: (Text) -> List[Dict[Text, Any]]
+def load_overrides(ov, base_url):  # type: (Text, Text) -> List[Dict[Text, Any]]
     ovloader = Loader(overrides_ctx)
-    return resolve_overrides(ovloader.fetch(ov), ov)
+    return resolve_overrides(ovloader.fetch(ov), base_url)

cwltool/main.py

@@ -27,7 +27,7 @@
 from .builder import Builder
 from .cwlrdf import printdot, printrdf
 from .errors import UnsupportedRequirement, WorkflowException
-from .load_tool import (fetch_document, make_tool, validate_document,
+from .load_tool import (resolve_tool_uri, fetch_document, make_tool, validate_document,
                         jobloaderctx, resolve_overrides, load_overrides)
 from .mutation import MutationManager
 from .pack import pack
@@ -519,7 +519,8 @@ def generate_input_template(tool):
 def load_job_order(args,   # type: argparse.Namespace
                    stdin,  # type: IO[Any]
                    fetcher_constructor,  # Fetcher
-                   overrides  # type: List[Dict[Text, Any]]
+                   overrides,  # type: List[Dict[Text, Any]]
+                   tool_file_uri  # type: Text
 ):
     # type: (...) -> Tuple[Dict[Text, Any], Text, Loader]
 
@@ -543,8 +544,8 @@ def load_job_order(args,   # type: argparse.Namespace
         job_order_object, _ = loader.resolve_ref(job_order_file, checklinks=False)
 
     if job_order_object and "http://commonwl.org/cwltool#overrides" in job_order_object:
-        overrides.extend(resolve_overrides(job_order_object, file_uri(job_order_file)))
-        del job_order_object["http://commonwl.org/cwltool#overrides"]
+       overrides.extend(resolve_overrides(job_order_object, tool_file_uri))
+       del job_order_object["http://commonwl.org/cwltool#overrides"]
 
     if not job_order_object:
         input_basedir = args.basedir if args.basedir else os.getcwd()
@@ -834,18 +835,24 @@ def main(argsl=None,  # type: List[str]
         else:
             use_standard_schema("v1.0")
 
+        uri, tool_file_uri = resolve_tool_uri(args.workflow, resolver)[1]
+
         overrides = []  # type: List[Dict[Text, Any]]
 
         try:
-            job_order_object, input_basedir, jobloader = load_job_order(args, stdin, fetcher_constructor, overrides)
+            job_order_object, input_basedir, jobloader = load_job_order(args,
+                                                                        stdin,
+                                                                        fetcher_constructor,
+                                                                        overrides,
+                                                                        tool_file_uri)
         except Exception as e:
             _logger.error(Text(e), exc_info=args.debug)
 
         if args.overrides:
-            overrides.extend(load_overrides(file_uri(os.path.abspath(args.overrides))))
+            overrides.extend(load_overrides(file_uri(os.path.abspath(args.overrides)), tool_file_uri))
 
         try:
-            document_loader, workflowobj, uri = fetch_document(args.workflow, resolver=resolver,
+            document_loader, workflowobj, uri = fetch_document(uri, resolver=resolver,
                                                                fetcher_constructor=fetcher_constructor)
 
             if args.print_deps: