[MRG] Working towards python3 compatible codebase

.gitignore

@@ -15,3 +15,18 @@ eggs/
 *~
 \#*\#
 .desktop
+
+# virtualenv
+venv/
+venv3/
+
+# pycharm
+.idea/
+
+# typshed repo
+typeshed/2and3/schema_salad
+typeshed/2and3/ruamel/yaml
+
+
+#mypy
+.mypy_cache/

Makefile

@@ -152,26 +152,26 @@ list-author-emails:
 	@git log --format='%aN,%aE' | sort -u | grep -v 'root'
 
 
-mypy: ${PYSOURCES}
-	rm -Rf typeshed/2.7/ruamel/yaml
+mypy2: ${PYSOURCES}
+	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-		typeshed/2.7/ruamel/yaml
-	rm -Rf typeshed/2.7/schema_salad
+		typeshed/2and3/ruamel/yaml
+	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
-		typeshed/2.7/schema_salad
-	MYPYPATH=typeshed/2.7 mypy --py2 --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		typeshed/2and3/schema_salad
+	MYPYPATH=$MYPYPATH:typeshed/2.7:typeshed/2and3 mypy --py2 --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores \
 		 cwltool
 
 mypy3: ${PYSOURCES}
-	rm -Rf typeshed/3/ruamel/yaml
+	rm -Rf typeshed/2and3/ruamel/yaml
 	ln -s $(shell python3 -c 'from __future__ import print_function; import ruamel.yaml; import os.path; print(os.path.dirname(ruamel.yaml.__file__))') \
-		typeshed/3/ruamel/yaml
-	rm -Rf typeshed/3/schema_salad
+		typeshed/2and3/ruamel/yaml
+	rm -Rf typeshed/2and3/schema_salad
 	ln -s $(shell python3 -c 'from __future__ import print_function; import schema_salad; import os.path; print(os.path.dirname(schema_salad.__file__))') \
-		typeshed/3/schema_salad
-	MYPYPATH=typeshed/3 mypy --disallow-untyped-calls \
-		 --warn-redundant-casts --warn-unused-ignores --fast-parser \
+		typeshed/2and3/schema_salad
+	MYPYPATH=$MYPYPATH:typeshed/3:typeshed/2and3 mypy --disallow-untyped-calls \
+		 --warn-redundant-casts --warn-unused-ignores \
 		 cwltool
 
 FORCE:

cwlref-runner/setup.py

@@ -1,5 +1,5 @@
 #!/usr/bin/env python
-
+from __future__ import absolute_import
 import os
 
 from setuptools import setup, find_packages

cwltool.py

@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+from __future__ import absolute_import
 """Convienance entry point for cwltool.
 
 This can be used instead of the recommended method of `./setup.py install`

cwltool/__init__.py

@@ -1 +1,2 @@
+from __future__ import absolute_import
 __author__ = '[email protected]'

cwltool/__main__.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import sys
 
 from . import main

cwltool/builder.py

@@ -1,6 +1,8 @@
+from __future__ import absolute_import
 import copy
-from typing import Any, Callable, Text, Type, Union
+from typing import Any, Callable, Dict, List, Text, Type, Union
 
+import six
 from six import iteritems, string_types
 
 import avro
@@ -15,6 +17,11 @@
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
 
+# if six.PY3:
+# AvroSchemaFromJSONData = avro.schema.SchemaFromJSONData
+# else:
+AvroSchemaFromJSONData = avro.schema.make_avsc_object
+
 CONTENT_LIMIT = 64 * 1024
 
 
@@ -86,7 +93,7 @@ def bind_input(self, schema, datum, lead_pos=None, tail_pos=None):
                 elif isinstance(t, dict) and "name" in t and self.names.has_name(t["name"], ""):
                     avsc = self.names.get_name(t["name"], "")
                 else:
-                    avsc = avro.schema.make_avsc_object(t, self.names)
+                    avsc = AvroSchemaFromJSONData(t, self.names)
                 if validate.validate(avsc, datum):
                     schema = copy.deepcopy(schema)
                     schema["type"] = t

cwltool/cwlrdf.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from typing import IO, Any, Dict, Text
 
 from rdflib import Graph

cwltool/docker.py

@@ -1,10 +1,11 @@
+from __future__ import absolute_import
 import logging
 import os
 import re
 import subprocess
 import sys
 import tempfile
-from typing import Text
+from typing import Dict, List, Text
 
 import requests
 
@@ -21,7 +22,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
         dockerRequirement["dockerImageId"] = dockerRequirement["dockerPull"]
 
     for ln in subprocess.check_output(
-            ["docker", "images", "--no-trunc", "--all"]).splitlines():
+            ["docker", "images", "--no-trunc", "--all"]).decode('utf-8').splitlines():
         try:
             m = re.match(r"^([^ ]+)\s+([^ ]+)\s+([^ ]+)", ln)
             sp = dockerRequirement["dockerImageId"].split(":")
@@ -46,7 +47,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             pass
 
     if not found and pull_image:
-        cmd = []  # type: List[str]
+        cmd = []  # type: List[Text]
         if "dockerPull" in dockerRequirement:
             cmd = ["docker", "pull", str(dockerRequirement["dockerPull"])]
             _logger.info(Text(cmd))
@@ -55,8 +56,8 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
                 found = True
         elif "dockerFile" in dockerRequirement:
             dockerfile_dir = str(tempfile.mkdtemp())
-            with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as df:
-                df.write(dockerRequirement["dockerFile"])
+            with open(os.path.join(dockerfile_dir, "Dockerfile"), str("w")) as df:
+                df.write(dockerRequirement["dockerFile"].encode('utf-8'))
             cmd = ["docker", "build", "--tag=%s" %
                    str(dockerRequirement["dockerImageId"]), dockerfile_dir]
             _logger.info(Text(cmd))
@@ -69,7 +70,7 @@ def get_image(dockerRequirement, pull_image, dry_run=False):
             if not dry_run:
                 if os.path.exists(dockerRequirement["dockerLoad"]):
                     _logger.info(u"Loading docker image from %s", dockerRequirement["dockerLoad"])
-                    with open(dockerRequirement["dockerLoad"], "rb") as f:
+                    with open(dockerRequirement["dockerLoad"], str("rb")) as f:
                         loadproc = subprocess.Popen(cmd, stdin=f, stdout=sys.stderr)
                 else:
                     loadproc = subprocess.Popen(cmd, stdin=subprocess.PIPE, stdout=sys.stderr)

cwltool/docker_uid.py

@@ -1,7 +1,8 @@
 from __future__ import print_function
+from __future__ import absolute_import
 
 import subprocess
-from typing import Text
+from typing import List, Text
 
 
 def docker_vm_uid():  # type: () -> int
@@ -91,6 +92,7 @@ def cmd_output_to_int(cmd):  # type: (List[Text]) -> int
         except ValueError:
             # ValueError is raised if int conversion fails
             return None
+    return None
 
 
 def boot2docker_uid():  # type: () -> int

cwltool/draft2tool.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 import copy
 import hashlib
 import json
@@ -7,7 +8,7 @@
 import shutil
 import tempfile
 from functools import partial
-from typing import Any, Callable, Dict, Generator, Optional, Text, Union, cast
+from typing import Any, Callable, Dict, Generator, List, Optional, Set, Text, Union, cast
 
 from six import string_types, u
 
@@ -28,6 +29,7 @@
                       normalizeFilesDirs, shortname, uniquename)
 from .stdfsaccess import StdFsAccess
 from .utils import aslist
+from six.moves import map
 
 ACCEPTLIST_EN_STRICT_RE = re.compile(r"^[a-zA-Z0-9._+-]+$")
 ACCEPTLIST_EN_RELAXED_RE = re.compile(r".*")  # Accept anything
@@ -60,7 +62,7 @@ def run(self, **kwargs):  # type: (**Any) -> None
                 normalizeFilesDirs(ev)
                 self.output_callback(ev, "success")
             except Exception as e:
-                _logger.warn(u"Failed to evaluate expression:\n%s",
+                _logger.warning(u"Failed to evaluate expression:\n%s",
                              e, exc_info=kwargs.get('debug'))
                 self.output_callback({}, "permanentFail")
 
@@ -224,7 +226,7 @@ def job(self,
             visit_class([cachebuilder.files, cachebuilder.bindings],
                        ("File", "Directory"), _check_adjust)
 
-            cmdline = flatten(map(cachebuilder.generate_arg, cachebuilder.bindings))
+            cmdline = flatten(list(map(cachebuilder.generate_arg, cachebuilder.bindings)))
             (docker_req, docker_is_req) = self.get_requirement("DockerRequirement")
             if docker_req and kwargs.get("use_container") is not False:
                 dockerimg = docker_req.get("dockerImageId") or docker_req.get("dockerPull")
@@ -253,7 +255,7 @@ def job(self,
                         keydict[r["class"]] = r
 
             keydictstr = json.dumps(keydict, separators=(',', ':'), sort_keys=True)
-            cachekey = hashlib.md5(keydictstr).hexdigest()
+            cachekey = hashlib.md5(keydictstr.encode('utf-8')).hexdigest()
 
             _logger.debug("[job %s] keydictstr is %s -> %s", jobname,
                           keydictstr, cachekey)
@@ -446,7 +448,7 @@ def register_reader(f):
                 cmd.extend(aslist(arg))
             j.command_line = ["/bin/sh", "-c", " ".join(cmd)]
         else:
-            j.command_line = flatten(map(builder.generate_arg, builder.bindings))
+            j.command_line = flatten(list(map(builder.generate_arg, builder.bindings)))
 
         j.pathmapper = builder.pathmapper
         j.collect_outputs = partial(
@@ -535,7 +537,7 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                                        "class": "File" if fs_access.isfile(g) else "Directory"}
                                       for g in fs_access.glob(fs_access.join(outdir, gb))])
                         except (OSError, IOError) as e:
-                            _logger.warn(Text(e))
+                            _logger.warning(Text(e))
                         except:
                             _logger.error("Unexpected error from fs_access", exc_info=True)
                             raise
@@ -547,14 +549,14 @@ def collect_output(self, schema, builder, outdir, fs_access, compute_checksum=Tr
                             get_listing(fs_access, files, (ll == "deep_listing"))
                     else:
                         with fs_access.open(files["location"], "rb") as f:
-                            contents = ""
+                            contents = b""
                             if binding.get("loadContents") or compute_checksum:
                                 contents = f.read(CONTENT_LIMIT)
                             if binding.get("loadContents"):
                                 files["contents"] = contents
                             if compute_checksum:
                                 checksum = hashlib.sha1()
-                                while contents != "":
+                                while contents != b"":
                                     checksum.update(contents)
                                     contents = f.read(1024 * 1024)
                                 files["checksum"] = "sha1$%s" % checksum.hexdigest()

cwltool/expression.py

@@ -1,29 +1,41 @@
+from __future__ import absolute_import
 import copy
 import json
 import logging
 import re
 from typing import Any, AnyStr, Dict, List, Text, Union
-
+import six
 from six import u
 
 from . import sandboxjs
 from .errors import WorkflowException
+from .utils import bytes2str_in_dicts
 
 _logger = logging.getLogger("cwltool")
 
 
 def jshead(engineConfig, rootvars):
     # type: (List[Text], Dict[Text, Any]) -> Text
+
+    # make sure all the byte strings are converted
+    # to str in `rootvars` dict.
+    # TODO: need to make sure the `rootvars dict`
+    # contains no bytes type in the first place.
+    if six.PY3:
+        rootvars = bytes2str_in_dicts(rootvars)  # type -> ignore
+
     return u"\n".join(engineConfig + [u"var %s = %s;" % (k, json.dumps(v, indent=4)) for k, v in rootvars.items()])
 
 
+# decode all raw strings to unicode
 seg_symbol = r"""\w+"""
 seg_single = r"""\['([^']|\\')+'\]"""
 seg_double = r"""\["([^"]|\\")+"\]"""
 seg_index = r"""\[[0-9]+\]"""
 segments = r"(\.%s|%s|%s|%s)" % (seg_symbol, seg_single, seg_double, seg_index)
-segment_re = re.compile(segments, flags=re.UNICODE)
-param_re = re.compile(r"\((%s)%s*\)$" % (seg_symbol, segments), flags=re.UNICODE)
+segment_re = re.compile(u(segments), flags=re.UNICODE)
+param_str = r"\((%s)%s*\)$" % (seg_symbol, segments)
+param_re = re.compile(u(param_str), flags=re.UNICODE)
 
 JSON = Union[Dict[Any, Any], List[Any], Text, int, float, bool, None]
 
@@ -110,7 +122,7 @@ def scanner(scan):  # type: (Text) -> List[int]
 def next_seg(remain, obj):  # type: (Text, Any) -> Any
     if remain:
         m = segment_re.match(remain)
-        key = None  # type: Union[str, int]
+        key = None  # type: Union[Text, int]
         if m.group(0)[0] == '.':
             key = m.group(0)[1:]
         elif m.group(0)[1] in ("'", '"'):

cwltool/factory.py

@@ -1,6 +1,7 @@
+from __future__ import absolute_import
 import os
 from typing import Callable as tCallable
-from typing import Any, Text, Tuple, Union
+from typing import Any, Dict, Text, Tuple, Union
 
 from . import load_tool, main, workflow
 from .process import Process

cwltool/flatten.py

@@ -1,3 +1,4 @@
+from __future__ import absolute_import
 from typing import Any, Callable, List, cast
 
 # http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html