Refactor job script/dependency stuff to allow toil pickle-ing.

Does this look cleaner anyway? I suspect yes?

Author: jmchilton

cwltool/builder.py

@@ -42,7 +42,6 @@ def __init__(self):  # type: () -> None
         self.pathmapper = None  # type: PathMapper
         self.stagedir = None  # type: Text
         self.make_fs_access = None  # type: Type[StdFsAccess]
-        self.build_job_script = None  # type: Callable[[List[str]], Text]
         self.debug = False  # type: bool
         self.mutation_manager = None  # type: MutationManager
 
@@ -51,6 +50,15 @@ def __init__(self):  # type: () -> None
         self.loadListing = "deep_listing"  # type: Union[None, str]
 
         self.find_default_container = None  # type: Callable[[], Text]
+        self.job_script_provider = None  # type: Any
+
+    def build_job_script(self, commands):
+        # type: (List[str]) -> Text
+        build_job_script_method = getattr(self.job_script_provider, "build_job_script", None)  # type: Callable[[Builder, List[str]], Text]
+        if build_job_script_method:
+            return build_job_script_method(self, commands)
+        else:
+            return None
 
     def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
         # type: (Dict[Text, Any], Any, Union[int, List[int]], List[int]) -> List[Dict[Text, Any]]

cwltool/job.py

@@ -190,15 +190,19 @@ def _execute(self, runtime, env, rm_tmpdir=True, move_outputs="move"):
                     os.makedirs(dn)
                 stdout_path = absout
 
-            build_job_script = self.builder.build_job_script  # type: Callable[[List[str]], Text]
+            commands = [Text(x).encode('utf-8') for x in runtime + self.command_line]
+            job_script_contents = None  # type: Text
+            builder = getattr(self, "builder", None)  # type: Builder
+            if builder is not None:
+                job_script_contents = builder.build_job_script(commands)
             rcode = _job_popen(
-                [Text(x).encode('utf-8') for x in runtime + self.command_line],
+                commands,
                 stdin_path=stdin_path,
                 stdout_path=stdout_path,
                 stderr_path=stderr_path,
                 env=env,
                 cwd=self.outdir,
-                build_job_script=build_job_script,
+                job_script_contents=job_script_contents,
             )
 
             if self.successCodes and rcode in self.successCodes:
@@ -401,14 +405,9 @@ def _job_popen(
         env,  # type: Union[MutableMapping[Text, Text], MutableMapping[str, str]]
         cwd,  # type: Text
         job_dir=None,  # type: Text
-        build_job_script=None,  # type: Callable[[List[str]], Text]
+        job_script_contents=None,  # type: Text
 ):
     # type: (...) -> int
-
-    job_script_contents = None  # type: Text
-    if build_job_script:
-        job_script_contents = build_job_script(commands)
-
     if not job_script_contents and not FORCE_SHELLED_POPEN:
 
         stdin = None  # type: Union[IO[Any], int]

cwltool/main.py

@@ -728,10 +728,10 @@ def main(argsl=None,  # type: List[str]
 
             make_tool_kwds = vars(args)
 
-            build_job_script = None  # type: Callable[[Any, List[str]], Text]
+            job_script_provider = None  # type: Callable[[Any, List[str]], Text]
             if conf_file or use_conda_dependencies:
                 dependencies_configuration = DependenciesConfiguration(args)  # type: DependenciesConfiguration
-                make_tool_kwds["build_job_script"] = dependencies_configuration.build_job_script
+                make_tool_kwds["job_script_provider"] = dependencies_configuration
 
             make_tool_kwds["find_default_container"] = functools.partial(find_default_container, args)
 

cwltool/process.py

@@ -598,12 +598,7 @@ def _init_job(self, joborder, **kwargs):
 
         builder.resources = self.evalResources(builder, kwargs)
 
-        build_job_script = kwargs.get("build_job_script", None)  # type: Callable[[Builder, List[str]], Text]
-        curried_build_job_script = None  # type: Callable[[List[str]], Text]
-        if build_job_script:
-            curried_build_job_script = lambda commands: build_job_script(builder, commands)
-        builder.build_job_script = curried_build_job_script
-
+        builder.job_script_provider = kwargs.get("job_script_provider", None)
         return builder
 
     def evalResources(self, builder, kwargs):