Merge branch 'master' into fix/no-compute-checksum

Author: Anton Khodak

cwltool/builder.py

@@ -217,6 +217,9 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         prefix = binding.get("prefix")
         sep = binding.get("separate", True)
+        if prefix is None and not sep:
+            with SourceLine(binding, "separate", WorkflowException, _logger.isEnabledFor(logging.DEBUG)):
+                raise WorkflowException("'separate' option can not be specified without prefix")
 
         l = []  # type: List[Dict[Text,Text]]
         if isinstance(value, list):
@@ -249,8 +252,8 @@ def generate_arg(self, binding):  # type: (Dict[Text,Any]) -> List[Text]
 
         return [a for a in args if a is not None]
 
-    def do_eval(self, ex, context=None, pull_image=True, recursive=False):
-        # type: (Union[Dict[Text, Text], Text], Any, bool, bool) -> Any
+    def do_eval(self, ex, context=None, pull_image=True, recursive=False, strip_whitespace=True):
+        # type: (Union[Dict[Text, Text], Text], Any, bool, bool, bool) -> Any
         if recursive:
             if isinstance(ex, dict):
                 return {k: self.do_eval(v, context, pull_image, recursive) for k, v in iteritems(ex)}
@@ -265,4 +268,5 @@ def do_eval(self, ex, context=None, pull_image=True, recursive=False):
                                   timeout=self.timeout,
                                   debug=self.debug,
                                   js_console=self.js_console,
-                                  force_docker_pull=self.force_docker_pull)
+                                  force_docker_pull=self.force_docker_pull,
+                                  strip_whitespace=strip_whitespace)

cwltool/draft2tool.py

@@ -383,7 +383,7 @@ def rm_pending_output_callback(output_callbacks, jobcachepending,
             else:
                 for t in initialWorkdir["listing"]:
                     if "entry" in t:
-                        et = {u"entry": builder.do_eval(t["entry"])}
+                        et = {u"entry": builder.do_eval(t["entry"], strip_whitespace=False)}
                         if "entryname" in t:
                             et["entryname"] = builder.do_eval(t["entryname"])
                         else:

cwltool/expression.py

@@ -173,9 +173,10 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, force_docker_pull=Fals
 
 def interpolate(scan, rootvars,
                 timeout=None, fullJS=None, jslib="", force_docker_pull=False,
-                debug=False, js_console=False):
-    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool, bool) -> JSON
-    scan = scan.strip()
+                debug=False, js_console=False, strip_whitespace=True):
+    # type: (Text, Dict[Text, Any], int, bool, Union[str, Text], bool, bool, bool, bool) -> JSON
+    if strip_whitespace:
+        scan = scan.strip()
     parts = []
     w = scanner(scan)
     while w:
@@ -202,8 +203,9 @@ def interpolate(scan, rootvars,
 
 
 def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
-            context=None, pull_image=True, timeout=None, force_docker_pull=False, debug=False, js_console=False):
-    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool, bool) -> Any
+            context=None, pull_image=True, timeout=None, force_docker_pull=False,
+            debug=False, js_console=False, strip_whitespace=True):
+    # type: (Union[dict, AnyStr], Dict[Text, Union[Dict, List, Text]], List[Dict[Text, Any]], Text, Text, Dict[Text, Union[int, Text]], Any, bool, int, bool, bool, bool, bool) -> Any
 
     runtime = copy.copy(resources)
     runtime["tmpdir"] = docker_windows_path_adjust(tmpdir)
@@ -231,7 +233,8 @@ def do_eval(ex, jobinput, requirements, outdir, tmpdir, resources,
                                jslib=jslib,
                                force_docker_pull=force_docker_pull,
                                debug=debug,
-                               js_console=js_console)
+                               js_console=js_console,
+                               strip_whitespace=strip_whitespace)
 
         except Exception as e:
             raise WorkflowException("Expression evaluation error:\n%s" % e)

cwltool/load_tool.py

@@ -20,7 +20,7 @@
 from avro.schema import Names
 from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import ContextType, Fetcher, Loader, file_uri
-from schema_salad.sourceline import cmap
+from schema_salad.sourceline import cmap, SourceLine
 from schema_salad.validate import ValidationException
 
 from . import process, update
@@ -124,6 +124,9 @@ def _convert_stdstreams_to_files(workflowobj):
     if isinstance(workflowobj, dict):
         if workflowobj.get('class') == 'CommandLineTool':
             for out in workflowobj.get('outputs', []):
+                if type(out) is not CommentedMap:
+                    with SourceLine(workflowobj, "outputs", ValidationException, _logger.isEnabledFor(logging.DEBUG)):
+                        raise ValidationException("Output '%s' is not a valid OutputParameter." % out)
                 for streamtype in ['stdout', 'stderr']:
                     if out.get('type') == streamtype:
                         if 'outputBinding' in out:

tests/test_examples.py

@@ -7,6 +7,7 @@
 
 from io import StringIO
 
+from cwltool.errors import WorkflowException
 from cwltool.utils import onWindows
 
 try:
@@ -525,6 +526,20 @@ def test_lifting(self):
             echo = f.make(get_data("tests/test_bad_outputs_wf.cwl"))
             self.assertEqual(echo(inp="foo"), {"out": "foo\n"})
 
+    def test_malformed_outputs(self):
+        # check that tool validation fails if one of the outputs is not a valid CWL type
+        f = cwltool.factory.Factory()
+        with self.assertRaises(schema_salad.validate.ValidationException):
+            echo = f.make(get_data("tests/wf/malformed_outputs.cwl"))
+            echo()
+
+    def test_separate_without_prefix(self):
+        # check that setting 'separate = false' on an inputBinding without prefix fails the workflow
+        with self.assertRaises(WorkflowException):
+            f = cwltool.factory.Factory()
+            echo = f.make(get_data("tests/wf/separate_without_prefix.cwl"))
+            echo()
+
 
     def test_checker(self):
         # check that the static checker raises exception when a source type

tests/test_iwdr.py

@@ -0,0 +1,16 @@
+import unittest
+
+import cwltool
+import cwltool.factory
+from .util import get_data
+
+
+class TestInitialWorkDir(unittest.TestCase):
+
+    def test_newline_in_entry(self):
+        """
+        test that files in InitialWorkingDirectory are created with a newline character
+        """
+        f = cwltool.factory.Factory()
+        echo = f.make(get_data("tests/wf/iwdr-entry.cwl"))
+        self.assertEqual(echo(message="hello"), {"out": "CONFIGVAR=hello\n"})

tests/wf/iwdr-entry.cwl

@@ -0,0 +1,22 @@
+#!/usr/bin/env cwl-runner
+
+class: CommandLineTool
+cwlVersion: v1.0
+baseCommand: ["cat", "example.conf"]
+
+requirements:
+  InitialWorkDirRequirement:
+    listing:
+      - entryname: example.conf
+        entry: |
+          CONFIGVAR=$(inputs.message)
+
+inputs:
+  message: string
+outputs:
+  out:
+    type: string
+    outputBinding:
+      glob: example.conf
+      loadContents: true
+      outputEval: $(self[0].contents)

tests/wf/malformed_outputs.cwl

@@ -0,0 +1,8 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+
+inputs:
+  []
+outputs:
+  -

tests/wf/separate_without_prefix.cwl

@@ -0,0 +1,22 @@
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+
+inputs:
+  src:
+    type: string
+    default:
+      string
+    inputBinding:
+      position: 1
+      separate: false
+
+stdout: output.txt
+
+outputs:
+  output:
+    type: string
+    outputBinding:
+      glob: output.txt
+      loadContents: true
+      outputEval: $(self[0].contents)