Merge branch 'master' into patch-1

Author: mr-c

.dockerignore

@@ -0,0 +1,6 @@
+.tox/
+.eggs/
+.vscode/
+build/
+dist/
+*.Dockerfile

.gitattributes

@@ -0,0 +1 @@
+tests/wf/whale.txt test eol=lf

.travis.yml

@@ -1,4 +1,6 @@
-sudo: false
+sudo: true
+services:
+  - docker
 language: python
 cache: pip
 python:

Jenkinsfile

@@ -0,0 +1,26 @@
+pipeline {
+  agent {
+    node {
+      label 'windows'
+    }
+
+  }
+  stages {
+    stage('build') {
+      steps {
+        withPythonEnv(pythonInstallation: 'Windows-CPython-36') {
+          pybat(script: 'pip install .', returnStdout: true)
+          pybat 'jenkins.bat'
+        }
+
+      }
+    }
+  }
+  post {
+    always {
+      junit 'tests.xml'
+
+    }
+
+  }
+}

MANIFEST.in

@@ -1,3 +1,4 @@
+include MANIFEST.in
 include gittaggers.py Makefile cwltool.py
 include tests/*
 include tests/tmp1/tmp2/tmp3/.gitkeep

Makefile

@@ -15,7 +15,7 @@
 #
 # Contact: [email protected]
 
-# make pep8 to check for basic Python code compliance
+# make pycodestyle to check for basic Python code compliance
 # make autopep8 to fix most pep8 errors
 # make pylint to check Python code for enhanced compliance including naming
 #  and documentation
@@ -26,7 +26,7 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pydocstyle flake8 pytest isort mock
+DEVPKGS=pycodestyle diff_cover autopep8 pylint coverage pydocstyle flake8 pytest isort mock
 DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pydocstyle sloccount \
 	   python-flake8 python-mock shellcheck
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
@@ -71,15 +71,16 @@ clean: FORCE
 sort_imports:
 	isort ${MODULE}/*.py tests/*.py setup.py
 
-## pep8        : check Python code style
-pep8: $(PYSOURCES)
-	pep8 --exclude=_version.py  --show-source --show-pep8 $^ || true
+pep8: pycodestyle
+## pycodestyle        : check Python code style
+pycodestyle: $(PYSOURCES)
+	pycodestyle --exclude=_version.py  --show-source --show-pep8 $^ || true
 
-pep8_report.txt: $(PYSOURCES)
-	pep8 --exclude=_version.py $^ > pep8_report.txt || true
+pycodestyle_report.txt: $(PYSOURCES)
+	pycodestyle --exclude=_version.py $^ > $@ || true
 
-diff_pep8_report: pep8_report.txt
-	diff-quality --violations=pep8 pep8_report.txt
+diff_pycodestyle_report: pycodestyle_report.txt
+	diff-quality --violations=pycodestyle $^
 
 pep257: pydocstyle
 ## pydocstyle      : check Python code style
@@ -113,7 +114,7 @@ pylint_report.txt: ${PYSOURCES}
 diff_pylint_report: pylint_report.txt
 	diff-quality --violations=pylint pylint_report.txt
 
-.coverage: tests
+.coverage: testcov
 
 coverage: .coverage
 	coverage report
@@ -134,7 +135,11 @@ diff-cover.html: coverage-gcovr.xml coverage.xml
 		--html-report diff-cover.html
 
 ## test        : run the ${MODULE} test suite
-test: $(PYSOURCES)
+test: $(pysources)
+	python setup.py test
+
+## testcov     : run the ${MODULE} test suite and collect coverage
+testcov: $(pysources)
 	python setup.py test --addopts "--cov cwltool"
 
 sloccount.sc: ${PYSOURCES} Makefile

README.rst

@@ -2,16 +2,26 @@
 Common Workflow Language tool description reference implementation
 ==================================================================
 
-CWL conformance tests: |Build Status| Travis CI: |Unix Build Status|
+CWL conformance tests: |Conformance Status| |Linux Status| |Windows Status| |Coverage Status|
 
-.. |Unix Build Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/master.svg?label=unix%20build
+
+.. |Conformance Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
+   :target: https://ci.commonwl.org/job/cwltool-conformance/
+
+.. |Linux Status| image:: https://img.shields.io/travis/common-workflow-language/cwltool/master.svg?label=Linux%20builds
    :target: https://travis-ci.org/common-workflow-language/cwltool
 
+.. |Windows Status| image:: https://img.shields.io/appveyor/ci/mr-c/cwltool/master.svg?label=Windows%20builds
+   :target: https://ci.appveyor.com/project/mr-c/cwltool
+
+.. |Coverage Status| image:: https://img.shields.io/codecov/c/github/common-workflow-language/cwltool.svg
+  :target: https://codecov.io/gh/common-workflow-language/cwltool
+
 This is the reference implementation of the Common Workflow Language.  It is
 intended to feature complete and provide comprehensive validation of CWL
 files as well as provide other tools related to working with CWL.
 
-This is written and tested for Python ``2.7 and 3.x {x = 3, 4, 5, 6}``
+This is written and tested for `Python <https://www.python.org/>`_ ``2.7 and 3.x {x = 4, 5, 6}``
 
 The reference implementation consists of two packages.  The ``cwltool`` package
 is the primary Python module containing the reference implementation in the
@@ -78,6 +88,8 @@ List of all environment can be seen using:
 ``tox --listenvs``
 and running a specfic test env using:
 ``tox -e <env name>``
+and additionally run a specific test using this format:
+``tox -e py36-unit -- tests/test_examples.py::TestParamMatching``
 
 -  Running the entire suite of CWL conformance tests:
 
@@ -110,9 +122,6 @@ and ``--tmp-outdir-prefix`` to somewhere under ``/Users``::
 
     $ cwl-runner --tmp-outdir-prefix=/Users/username/project --tmpdir-prefix=/Users/username/project wc-tool.cwl wc-job.json
 
-.. |Build Status| image:: https://ci.commonwl.org/buildStatus/icon?job=cwltool-conformance
-   :target: https://ci.commonwl.org/job/cwltool-conformance/
-
 Using user-space replacements for Docker
 ----------------------------------------
 
@@ -132,8 +141,8 @@ Run `cwltool` just as you normally would, but with the new option, e.g. from the
 .. code:: bash
 
   cwltool --user-space-docker-cmd=udocker https://github.com/raw/common-workflow-language/common-workflow-language/master/v1.0/v1.0/test-cwl-out2.cwl https://github.com/common-workflow-language/common-workflow-language/blob/master/v1.0/v1.0/empty.json
-  
-or 
+
+or
 
 .. code:: bash
 

appveyor.yml

@@ -14,28 +14,45 @@ environment:
       PYTHON_VERSION: "2.7.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python34"
+      PYTHON_VERSION: "3.4.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python34-x64"
       PYTHON_VERSION: "3.4.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python35"
+      PYTHON_VERSION: "3.5.x"
+      PYTHON_ARCH: "32"
+
     - PYTHON: "C:\\Python35-x64"
       PYTHON_VERSION: "3.5.x"
       PYTHON_ARCH: "64"
 
+    - PYTHON: "C:\\Python36"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "32"
+
+    - PYTHON: "C:\\Python36-x64"
+      PYTHON_VERSION: "3.6.x"
+      PYTHON_ARCH: "64"
 
 install:
+  - "%PYTHON%\\python.exe -m pip install -U wheel setuptools pip pytest mock"
   # Get the latest stable version of Node.js or io.js
   - ps: Install-Product node $env:nodejs_version
-  - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
 
 build_script:
-  - "%CMD_IN_ENV% python -m pip install -U setuptools pip"
-  - "%CMD_IN_ENV% pip install pytest mock"
-  - "%CMD_IN_ENV% pip install ."
-
+  - "%PYTHON%\\python.exe -m pip install ."
 
 test_script:
-  - "%CMD_IN_ENV% py.test --verbose -p no:cacheprovider"
+  - "%PYTHON%\\python.exe -m pytest --verbose -p no:cacheprovider --junit-xml=tests.xml"
+
+after_test:
+  - ps: |
+      $wc = New-Object 'System.Net.WebClient'
+      $wc.UploadFile("https://ci.appveyor.com/api/testresults/junit/$($Env:APPVEYOR_JOB_ID)", (Resolve-Path .\tests.xml))
 
 branches:
   only:

build-cwl-docker.sh

@@ -1,4 +1,11 @@
 #!/bin/sh
 set -e
-docker build --file=cwltool_module.Dockerfile --tag=commonworkflowlanguage/cwltool_module .
+docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool-module --target module .
 docker build --file=cwltool.Dockerfile --tag=commonworkflowlanguage/cwltool .
+
+version=$(git describe --tags)
+echo $version | grep -vq '\-' >& /dev/null
+if [ $? -eq 0 ];then
+    docker tag commonworkflowlanguage/cwltool-module commonworkflowlanguage/cwltool-module:$version
+    docker tag commonworkflowlanguage/cwltool commonworkflowlanguage/cwltool:$version
+fi

cwl-docker.sh

@@ -1,3 +1,2 @@
 #!/bin/sh
-docker run --name=cwl-docker -v /var/lib/docker -i -t fedora-data true
-docker run --privileged -ti --volume=$PWD:$PWD -w=$PWD commonworkflowlanguage/cwltool $@
+exec docker run -v /var/run/docker.sock:/var/run/docker.sock -v /tmp:/tmp -v "$PWD":"$PWD" -w="$PWD" commonworkflowlanguage/cwltool "$@"

cwltool-in-docker.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env sh
+if [ ! -S /var/run/docker.sock ]; then
+  >&2 echo 'ERROR: cwltool cannot work inside a container without access to docker'
+  >&2 echo 'Launch the container with the option -v /var/run/docker.sock:/var/run/docker.sock'
+  exit 1
+elif [ "$PWD" = '/error' ]; then
+  >&2 echo 'ERROR: cwltool cannot work without access to the current path'
+  >&2 echo 'Launch the container with the options -v "$PWD":"$PWD" -w="$PWD"'
+  exit 1
+else
+  cwltool "$@"
+fi

cwltool.Dockerfile

@@ -1,21 +1,27 @@
-FROM commonworkflowlanguage/cwltool_module
-MAINTAINER [email protected]
-
-# Let's start with some basic stuff.
-RUN apt-get update -qq && apt-get install -qqy \
-    apt-transport-https \
-    ca-certificates \
-    curl \
-    lxc \
-    iptables \
-    python-setuptools
-
-# Install Docker from Docker Inc. repositories.
-RUN curl -sSL https://get.docker.com/ | sh
-
-# Install the magic wrapper.
-ADD ./wrapdocker /usr/local/bin/wrapdocker
-RUN chmod +x /usr/local/bin/wrapdocker
-
-VOLUME /var/lib/docker
-ENTRYPOINT ["wrapdocker", "cwltool"]
+FROM python:3.6-alpine as builder
+
+RUN apk add --no-cache git
+
+WORKDIR /cwltool
+COPY . .
+
+RUN python setup.py bdist_wheel --dist-dir=/wheels
+RUN pip wheel -r requirements.txt --wheel-dir=/wheels
+RUN pip install --no-index --no-warn-script-location --root=/pythonroot/ /wheels/*.whl
+
+FROM python:3.6-alpine as module
+LABEL maintainer [email protected]
+
+RUN apk add --no-cache docker nodejs
+COPY --from=builder /pythonroot/ /
+
+FROM python:3.6-alpine
+LABEL maintainer [email protected]
+
+RUN apk add --no-cache docker nodejs
+COPY --from=builder /pythonroot/ /
+COPY cwltool-in-docker.sh /cwltool-in-docker.sh
+
+WORKDIR /error
+
+ENTRYPOINT ["/cwltool-in-docker.sh"]

cwltool/argparser.py

@@ -214,6 +214,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     parser.add_argument("--custom-net", type=Text,
                         help="Will be passed to `docker run` as the '--net' "
                              "parameter. Implies '--enable-net'.")
+    parser.add_argument("--disable-validate", dest="do_validate",
+                        action="store_false", default=True,
+                        help=argparse.SUPPRESS)
 
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--enable-ga4gh-tool-registry", action="store_true", help="Enable resolution using GA4GH tool registry API",