Implement Dependency hint to describe tool dependencies.

Author: jmchilton

cwltool/draft2tool.py

@@ -203,6 +203,7 @@ def rm_pending_output_callback(output_callback, jobcachepending,
         reffiles = set((f["path"] for f in builder.files))
 
         j = self.makeJobRunner()
+        j.tool_dependency_manager = self.tool_dependency_manager
         j.builder = builder
         j.joborder = builder.job
         j.stdin = None

cwltool/job.py

@@ -21,12 +21,16 @@
 from typing import Union, Iterable, Callable, Any, Mapping, IO, cast, Tuple
 from .pathmapper import PathMapper
 import functools
+try:
+    from galaxy.tools.deps.requirements import ToolRequirement
+except ImportError:
+    ToolRequirement = None
 
 _logger = logging.getLogger("cwltool")
 
 needs_shell_quoting_re = re.compile(r"""(^$|[\s|&;()<>\'"$@])""")
 
-FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "1") == "1"
+FORCE_SHELLED_POPEN = os.getenv("CWLTOOL_FORCE_SHELL_POPEN", "0") == "1"
 
 
 def deref_links(outputs):  # type: (Any) -> None
@@ -63,6 +67,7 @@ def __init__(self):  # type: () -> None
         self.tmpdir = None  # type: str
         self.environment = None  # type: Dict[str,str]
         self.generatefiles = None  # type: Dict[str,Union[Dict[str,str],str]]
+        self.dependency_manager = None  # type: DependencyManager
 
     def run(self, dry_run=False, pull_image=True, rm_container=True,
             rm_tmpdir=True, move_outputs=True, **kwargs):
@@ -191,6 +196,17 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             else:
                 stdout_path = None
 
+            prefix = None
+            job_dir = None
+            if self.tool_dependency_manager is not None:
+                dependencies = self._find_tool_dependencies()
+                job_dir = tempfile.mkdtemp(prefix="cwltooljob")
+                shell_commands = self.tool_dependency_manager.dependency_shell_commands(
+                    dependencies,
+                    job_directory=job_dir,
+                )
+                prefix = "\n".join(shell_commands)
+
             rcode = shelled_popen(
                 [str(x) for x in runtime + self.command_line],
                 stdin_path=stdin_path,
@@ -252,6 +268,23 @@ def run(self, dry_run=False, pull_image=True, rm_container=True,
             _logger.debug(u"[job %s] Removing empty output directory %s", self.name, self.outdir)
             shutil.rmtree(self.outdir, True)
 
+    def _find_tool_dependencies(self):
+        dependencies = []
+        for hint in self.hints:
+            hint_class = hint.get("class", "")
+            if not hint_class:
+                continue
+            base_name = hint["class"].rsplit("/", 1)[-1]
+            if base_name == "Dependency":
+                requirement_desc = {}
+                requirement_desc["type"] = "package"
+                name = hint["name"].rsplit("#", 1)[-1]
+                version = hint.get("version", "").rsplit("#", 1)[-1]
+                requirement_desc["name"] = name
+                requirement_desc["version"] = version or None
+                dependencies.append(ToolRequirement.from_dict(requirement_desc))
+        return dependencies
+
 
 SHELL_COMMAND_TEMPLATE = string.Template("""#!/bin/bash
 $prefix
@@ -308,6 +341,7 @@ def shelled_popen(commands,
                   stdout_path,
                   env,
                   cwd,
+                  job_dir=None,
                   prefix=None):
     if prefix is None and not FORCE_SHELLED_POPEN:
         if stdin_path is not None:
@@ -341,17 +375,19 @@ def shelled_popen(commands,
 
         return rcode
     else:
+        if job_dir is None:
+            job_dir = tempfile.mkdtemp(prefix="cwltooljob")
+
         template_kwds = dict(
             prefix=prefix or '',
         )
         job_script_contents = SHELL_COMMAND_TEMPLATE.substitute(
             **template_kwds
         )
-        job_dir = tempfile.mkdtemp(prefix="cwltooljob")
         job_description = dict(
             commands=commands,
             cwd=cwd,
-            env=env,
+            env=env.copy(),
             stdout_path=stdout_path,
             stdin_path=stdin_path,
         )

cwltool/main.py

@@ -25,13 +25,20 @@
 import rdflib
 import hashlib
 from typing import Union, Any, cast, Callable, Dict, Tuple, IO
+try:
+    from galaxy.tools import deps
+except ImportError:
+    deps = None
 
 _logger = logging.getLogger("cwltool")
 
 defaultStreamHandler = logging.StreamHandler()
 _logger.addHandler(defaultStreamHandler)
 _logger.setLevel(logging.INFO)
 
+if deps is not None:
+    deps.log = _logger
+
 
 def arg_parser():  # type: () -> argparse.ArgumentParser
     parser = argparse.ArgumentParser(description='Reference executor for Common Workflow Language')
@@ -129,6 +136,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
     exgroup.add_argument("--debug", action="store_true", help="Print even more logging")
 
+    parser.add_argument("--dependency-resolvers-configuration", help="Dependency resolver configuration file describing how to adapt 'Dependency' hints to current system.", default=None)
+    parser.add_argument("--dependencies-directory", help="Defaut root directory used by dependency resolvers configuration.", default=None)
+
     parser.add_argument("--tool-help", action="store_true", help="Print command line help for tool")
 
     parser.add_argument("--relative-deps", choices=['primary', 'cwd'], default="primary",
@@ -544,6 +554,12 @@ def main(argsl=None,
             if not hasattr(args, k):
                 setattr(args, k, v)
 
+        if deps is not None:
+            tool_dependencies_configuartion = DependenciesConfigruation(args)
+            tool_dependency_manager = deps.build_dependency_manager(tool_dependencies_configuartion)
+        else:
+            tool_dependency_manager = None
+
         if args.quiet:
             _logger.setLevel(logging.WARN)
         if args.debug:
@@ -588,8 +604,11 @@ def main(argsl=None,
                 printdot(uri, processobj, document_loader.ctx, stdout)
                 return 0
 
+            make_tool_kwargs = {
+                'tool_dependency_manager': tool_dependency_manager,
+            }
             tool = make_tool(document_loader, avsc_names, processobj, metadata,
-                             uri, makeTool, {})
+                             uri, makeTool, make_tool_kwargs)
         except (validate.ValidationException) as exc:
             _logger.error(u"Tool definition failed validation:\n%s", exc,
                           exc_info=(exc if args.debug else False))
@@ -676,5 +695,20 @@ def main(argsl=None,
         _logger.removeHandler(stderr_handler)
         _logger.addHandler(defaultStreamHandler)
 
+
+class DependenciesConfigruation(object):
+
+    def __init__(self, args):
+        conf_file = getattr(args, "dependency_resolvers_configuration", None)
+        tool_dependency_dir = getattr(args, "dependencies_directory", None)
+        if conf_file is not None and os.path.exists(conf_file):
+            self.use_tool_dependencies = True
+            if not tool_dependency_dir:
+                tool_dependency_dir = os.path.abspath(os.path.dirname(conf_file))
+            self.tool_dependency_dir = tool_dependency_dir
+            self.dependency_resolvers_config_file = conf_file
+        else:
+            self.use_tool_dependencies = False
+
 if __name__ == "__main__":
     sys.exit(main(sys.argv[1:]))

cwltool/process.py

@@ -295,7 +295,7 @@ def __init__(self, toolpath_object, **kwargs):
             avro.schema.make_avsc_object(self.outputs_record_schema, self.names)
         except avro.schema.SchemaParseException as e:
             raise validate.ValidationException(u"Got error `%s` while prcoessing outputs of %s:\n%s" % (str(e), self.tool["id"], json.dumps(self.outputs_record_schema, indent=4)))
-
+        self.tool_dependency_manager = kwargs.get("tool_dependency_manager", None)
 
     def _init_job(self, joborder, **kwargs):
         # type: (Dict[str, str], str, **Any) -> Builder