Merge branch 'master' into requirement-overrides

Author: tetron

Makefile

@@ -26,8 +26,8 @@ MODULE=cwltool
 # `SHELL=bash` doesn't work for some, so don't use BASH-isms like
 # `[[` conditional expressions.
 PYSOURCES=$(wildcard ${MODULE}/**.py tests/*.py) setup.py
-DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest isort
-DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8
+DEVPKGS=pep8 diff_cover autopep8 pylint coverage pep257 flake8 pytest isort mock
+DEBDEVPKGS=pep8 python-autopep8 pylint python-coverage pep257 sloccount python-flake8 python-mock
 VERSION=1.0.$(shell date +%Y%m%d%H%M%S --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 mkfile_dir := $(dir $(abspath $(lastword $(MAKEFILE_LIST))))

cwltool/expression.py

@@ -144,6 +144,8 @@ def evaluator(ex, jslib, obj, fullJS=False, timeout=None, debug=False):
     # type: (Text, Text, Dict[Text, Any], bool, int, bool) -> JSON
     m = param_re.match(ex)
     if m:
+        if m.end(1)+1 == len(ex) and m.group(1) == "null":
+            return None
         try:
             return next_seg(m.group(0)[m.end(1) - m.start(0):-1], obj[m.group(1)])
         except Exception as w:

cwltool/load_tool.py

@@ -148,7 +148,8 @@ def validate_document(document_loader,  # type: Loader
                       enable_dev=False,  # type: bool
                       strict=True,  # type: bool
                       preprocess_only=False,  # type: bool
-                      fetcher_constructor=None
+                      fetcher_constructor=None,
+                      skip_schemas=None
                       # type: Callable[[Dict[unicode, unicode], requests.sessions.Session], Fetcher]
                       ):
     # type: (...) -> Tuple[Loader, Names, Union[Dict[Text, Any], List[Dict[Text, Any]]], Dict[Text, Any], Text]
@@ -195,9 +196,10 @@ def validate_document(document_loader,  # type: Loader
         raise avsc_names
 
     processobj = None  # type: Union[CommentedMap, CommentedSeq, unicode]
+
     document_loader = Loader(sch_document_loader.ctx, schemagraph=sch_document_loader.graph,
                              idx=document_loader.idx, cache=sch_document_loader.cache,
-                             fetcher_constructor=fetcher_constructor)
+                             fetcher_constructor=fetcher_constructor, skip_schemas=skip_schemas)
 
     _add_blank_ids(workflowobj)
 

cwltool/main.py

@@ -141,6 +141,9 @@ def arg_parser():  # type: () -> argparse.ArgumentParser
     exgroup.add_argument("--non-strict", action="store_false", help="Lenient validation (ignore unrecognized fields)",
                          default=True, dest="strict")
 
+    parser.add_argument("--skip-schemas", action="store_true",
+            help="Skip loading of schemas", default=True, dest="skip_schemas")
+
     exgroup = parser.add_mutually_exclusive_group()
     exgroup.add_argument("--verbose", action="store_true", help="Default logging")
     exgroup.add_argument("--quiet", action="store_true", help="Only print warnings and errors.")
@@ -641,6 +644,7 @@ def main(argsl=None,  # type: List[str]
                      'enable_dev': False,
                      'enable_ext': False,
                      'strict': True,
+                     'skip_schemas': False,
                      'rdf_serializer': None,
                      'basedir': None,
                      'tool_help': False,
@@ -709,7 +713,8 @@ def main(argsl=None,  # type: List[str]
                 = validate_document(document_loader, workflowobj, uri,
                                     enable_dev=args.enable_dev, strict=args.strict,
                                     preprocess_only=args.print_pre or args.pack,
-                                    fetcher_constructor=fetcher_constructor)
+                                    fetcher_constructor=fetcher_constructor,
+                                    skip_schemas=args.skip_schemas)
 
             if args.pack:
                 stdout.write(print_pack(document_loader, processobj, uri, metadata))

cwltool/schemas/README.md

@@ -1,18 +1,20 @@
 Common Workflow Language
 ========================
 
-[![GitHub stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+[![GitHub stars](https://img.shields.io/github/stars/common-workflow-language/common-workflow-language.svg)](https://github.com/common-workflow-language/common-workflow-language/stargazers) [![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/common-workflow-language/common-workflow-language?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) [**Support**](#Support)
 
 The Common Workflow Language (CWL) is a specification for describing analysis
-workflows and tools that are portable and scalable across a variety of software
-and hardware environments, from workstations to cluster, cloud, and high
-performance computing (HPC) environments.  CWL is designed to meet the needs of
-data-intensive science, such as Bioinformatics, Medical Imaging, Astronomy,
-Physics, and Chemistry.
+workflows and tools in a way that makes them portable and scalable across a
+variety of software and hardware environments, from workstations to cluster,
+cloud, and high performance computing (HPC) environments.  CWL is designed to
+meet the needs of data-intensive science, such as Bioinformatics, Medical
+Imaging, Astronomy, Physics, and Chemistry.
 
 CWL is developed by an informal, multi-vendor working group consisting of
-organizations and individuals interested enabling scientists to share data
-analysis workflows.  [The CWL project is on Github.](https://github.com/common-workflow-language/common-workflow-language)
+organizations and individuals aiming to enable scientists to share data
+analysis workflows.  [The CWL project is on Github](https://github.com/common-workflow-language/common-workflow-language) and we follow the [Open-Stand.org principles for collaborative open standards development](https://open-stand.org/about-us/principles/)
+
+[![Open Stand badge](http://standards.ieee.org/images/openstand/128x128-blue2.png)](https://open-stand.org/about-us/principles/).
 
 CWL builds on technologies such as [JSON-LD](http://json-ld.org)
 for data modeling and [Docker](http://docker.com) for portable runtime
@@ -40,17 +42,19 @@ Please cite https://dx.doi.org/10.6084/m9.figshare.3115156.v2
 
 ## Implementations
 
-|Software|Description|CWL support|Platform support|Platform Maturity|
-|--------|-----------|-----------|--------|--------|
-|[cwltool](https://github.com/common-workflow-language/cwltool)|Reference implementation of CWL|[![Build Status](https://ci.commonwl.org/job/cwltool-conformance/badge/icon)](http://ci.commonwl.org/job/cwltool-conformance/lastBuild/testReport/)|Linux, OS X, local execution only|Production|
-|[Arvados](https://arvados.org)|Distributed computing platform for data analysis on massive data sets. [Using CWL on Arvados](http://doc.arvados.org/user/cwl/cwl-runner.html)|[![Build Status](https://ci.commonwl.org/job/arvados-conformance/badge/icon)](http://ci.commonwl.org/job/arvados-conformance/lastBuild/testReport/)|AWS, GCP, Azure, Slurm|Production|
-|[Toil](https://github.com/BD2KGenomics/toil)|Toil is a workflow engine entirely written in Python.|[![Build Status](https://ci.commonwl.org/job/toil-conformance/badge/icon)](http://ci.commonwl.org/job/toil-conformance/lastBuild/testReport/)|AWS, GCP, Azure, OpenStack, Grid Engine, Mesos|Production|
-|[Rabix Bunny](https://github.com/rabix/bunny)|Reproducible Analyses for Bioinformatics|[![Build Status](https://ci.commonwl.org/buildStatus/icon?job=rabix-conformance)](https://ci.commonwl.org/job/rabix-conformance/)|Java|alpha|
-|[Apache Taverna](http://taverna.incubator.apache.org/)|Domain-independent Workflow Management System|[alpha](https://issues.apache.org/jira/browse/TAVERNA-900)|Java|Production|
-|[Galaxy](https://galaxyproject.org/)|Web-based platform for data intensive biomedical research.|[alpha](https://github.com/common-workflow-language/galaxy)|Python|Production|
-|[AWE](https://github.com/MG-RAST/AWE)|Workflow and resource management system for bioinformatics data analysis.|[alpha](https://github.com/wgerlach/AWE)|Go|Production|
-|[Funnel](https://github.com/bmeg/funnel)|Use Google Genomics Pipeline API with CWL|alpha|GCP|alpha|
-|[xenon-cwl-runner](https://github.com/NLeSC/xenon-cwl-runner)|Run CWL workflows using Xenon|alpha|[any Xenon backend](http://nlesc.github.io/Xenon/): local, ssh, SLURM, Torque, Grid Engine|Production|
+|Software|Description|CWL support|Platform support|
+|--------|-----------|-----------|--------|
+|[cwltool](https://github.com/common-workflow-language/cwltool)|Reference implementation of CWL|[![Build Status](https://ci.commonwl.org/job/cwltool-conformance/badge/icon)](http://ci.commonwl.org/job/cwltool-conformance/lastBuild/testReport/)|Linux, OS X, local execution only|
+|[Arvados](https://arvados.org)|Distributed computing platform for data analysis on massive data sets. [Using CWL on Arvados](http://doc.arvados.org/user/cwl/cwl-runner.html)|[![Build Status](https://ci.commonwl.org/job/arvados-conformance/badge/icon)](http://ci.commonwl.org/job/arvados-conformance/lastBuild/testReport/)|AWS, GCP, Azure, Slurm|
+|[Toil](https://github.com/BD2KGenomics/toil)|Toil is a workflow engine entirely written in Python.|[![Build Status](https://ci.commonwl.org/job/toil-conformance/badge/icon)](http://ci.commonwl.org/job/toil-conformance/lastBuild/testReport/)|AWS, Azure, GCP, Grid Engine, LSF, Mesos, OpenStack, Slurm|
+|[Rabix Bunny](https://github.com/rabix/bunny)|An open-source, Java-based implementation of Common Workflow Language with support for multiple drafts/versions. See [Rabix.io](http://rabix.io) for details.|[![Build Status](https://ci.commonwl.org/buildStatus/icon?job=rabix-conformance)](https://ci.commonwl.org/job/rabix-conformance/)|Linux, OS X, [GA4GH TES](https://github.com/ga4gh/task-execution-server) (experimental)|
+|[cwl-tes](https://github.com/common-workflow-language/cwl-tes)|CWL engine backended by the [GA4GH Task Execution API](https://github.com/ga4gh/task-execution-schemas)|[![Build Status](https://travis-ci.org/common-workflow-language/cwl-tes.svg?branch=master)](https://travis-ci.org/common-workflow-language/cwl-tes)|Local, GCP, AWS, HTCondor, Grid Engine, PBS/Torque, Slurm|
+|[Airflow](https://github.com/Barski-lab/incubator-airflow)|Airflow updated with CWL-runner feature (supported by BioWardrobe Team, CCHMC)|[![Build Status](https://ci.commonwl.org/buildStatus/icon?job=airflow-conformance)](https://ci.commonwl.org/job/airflow-conformance)|Linux, OS X|
+|[Consonance](https://github.com/Consonance/consonance)|orchestration tool for running SeqWare workflows and CWL tools|[pending](https://ci.commonwl.org/job/rabix-conformance/)|AWS, OpenStack, Azure|
+|[Apache Taverna](http://taverna.incubator.apache.org/)|Domain-independent Workflow Management System|[alpha](https://issues.apache.org/jira/browse/TAVERNA-900)|Java|
+|[Galaxy](https://galaxyproject.org/)|Web-based platform for data intensive biomedical research.|[alpha](https://github.com/common-workflow-language/galaxy)|Python|
+|[AWE](https://github.com/MG-RAST/AWE)|Workflow and resource management system for bioinformatics data analysis.|[alpha](https://github.com/wgerlach/AWE)|Go|
+|[Xenon](http://nlesc.github.io/Xenon/)|Run CWL workflows using Xenon|[alpha](https://github.com/NLeSC/xenon-cwl-runner)|[any Xenon backend](http://nlesc.github.io/Xenon/): local, ssh, SLURM, Torque, Grid Engine|
 
 
 ## Repositories of CWL Tools and Workflows
@@ -71,8 +75,20 @@ Please cite https://dx.doi.org/10.6084/m9.figshare.3115156.v2
 |[pypi2cwl](https://github.com/common-workflow-language/pypi2cwl)|Automatically run argparse2cwl on any package in PyPi|
 |[cwlavro](https://github.com/common-workflow-language/cwlavro)|Java classes for loading CWL documents|
 |[acd2cwl](https://github.com/common-workflow-language/acd2cwl)|CWL generator for ACD (EMBOSS) files |
-|[CWLviewer](https://github.com/MarkRobbo/CWLViewer)|A web application to view and share Common Workflow Language workflows|
+|[CWLviewer](https://view.commonwl.org/)|A web application to view and share Common Workflow Language workflows|
+|[cwl2zshcomp](https://github.com/kloetzl/cwl2zshcomp)|generates ZSH auto completions from CWL command line tool descriptions|
+|[scriptcwl](https://github.com/NLeSC/scriptcwl)|Create CWL workflows by writing a simple Python script|
+|[python-cwlgen](https://github.com/common-workflow-language/python-cwlgen)|Generation of CWL programmatically from Python.|
+
+## Projects the CWL community is participating in
 
+|Name|Details|
+|-------|-----------|
+|[GA4GH Task Execution API](https://github.com/ga4gh/task-execution-schemas/)|a minimal common API for submitting a single job to a remote execution endpoint. Many contributions from CWL project participants.|
+|[GA4GH Workflow Execution API](https://github.com/ga4gh/workflow-execution-schemas)|a minimal common API for submitting workflow requests to workflow execution systems in a standardized way. Many contributions from CWL project participants.|
+|[Bio-compute objects](https://hive.biochemistry.gwu.edu/htscsrs?pageid=biocompute)|"a step towards evaluation and validation of bio-medical scientific computations", CWL and researchobject.org participants are cooperating with this effort|
+
+<a name="Support"></a>
 ## Support, Community and Contributing
 
 The recommended place to ask a question about all things CWL is on
@@ -87,9 +103,6 @@ join the [mailing list common-workflow-language on Google Groups](https://groups
 [fork the repository](https://github.com/common-workflow-language/common-workflow-language)
 and send a pull request!
 
-Your CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
-with updates at http://mrc.commonwl.org.
-
 ### Code of Conduct
 
 The CWL Project is dedicated to providing a harassment-free experience for
@@ -114,11 +127,16 @@ Conduct](https://github.com/common-workflow-language/common-workflow-language/bl
 * [Institut Pasteur](http://www.pasteur.fr)
 * [Wellcome Trust Sanger Institute](https://www.sanger.ac.uk/)
 * [University of California Santa Cruz](https://cbse.soe.ucsc.edu/research/bioinfo)
-* [Harvard Chan School of Public Health](http://www.hsph.harvard.edu/)
+* [Harvard T.H. Chan School of Public Health](http://www.hsph.harvard.edu/)
 * [Cincinnati Children's Hospital Medical Center](http://www.cincinnatichildrens.org/)
 * [Broad Institute](https://www.broadinstitute.org)
-* [BioDatomics](http://www.biodatomics.com/)
-* [Wolfson Wohl Cancer Research Centre](http://www.gla.ac.uk/researchinstitutes/cancersciences/ics/facilities/wwcrc/)
+* [University of Melbourne Center for Cancer Research](https://umccr.github.io/)
+* [Netherlands eScience Center](https://www.esciencecenter.nl/)
+* [Texas Advanced Computing Center Life Science Computing Group](https://www.tacc.utexas.edu/life-sciences-computing) / [Agave Platform](https://agaveapi.co/)
+* [CyVerse](http://www.cyverse.org/)
+* [Institute for Systems Biology](https://www.systemsbiology.org/)
+* [ELIXIR Europe](https://www.elixir-europe.org/)
+* [BioExcel CoE](http://bioexcel.eu/)
 
 ## Individual Contributors
 
@@ -131,15 +149,24 @@ Conduct](https://github.com/common-workflow-language/common-workflow-language/bl
 * Michael R. Crusoe <[email protected]>
 * Roman Valls Guimerà <[email protected]>
 * Guillermo Carrasco Hernandez <[email protected]>
+* Kenzo-Hugo Hillion <[email protected]>
 * Sinisa Ivkovic <[email protected]>
+* Sehrish Kanwal <[email protected]>
 * Andrey Kartashov <[email protected]>
 * John Kern <[email protected]>
+* Farah Zaib Khan <[email protected]>
 * Dan Leehr <[email protected]>
 * Hervé Ménager <[email protected]>
 * Maxim Mikheev <[email protected]>
+* Michael Miller <[email protected]>
 * Tim Pierce <[email protected]>
 * Josh Randall <[email protected]>
+* Mark Robinson <[email protected]>
 * Janko Simonović <[email protected]>
-* Stian Soiland-Reyes <soiland-reyes@cs.manchester.ac.uk>
+* Stian Soiland-Reyes <[email protected]>
 * Luka Stojanovic <[email protected]>
 * Nebojša Tijanić <[email protected]>
+
+
+The CWL Community Engineer, Michael R. Crusoe, publishes a blog about his work
+with updates at http://mrc.commonwl.org.