diff --git a/.github/workflows/gh-pages.yaml b/.github/workflows/gh-pages.yaml index 3611b2ed..e3e47963 100644 --- a/.github/workflows/gh-pages.yaml +++ b/.github/workflows/gh-pages.yaml @@ -42,6 +42,9 @@ jobs: - name: Install dependencies run: python3 -m pip install -U -e .[all] + - name: pre-pull container images + run: make container-pull + - name: Build documentation run: make html diff --git a/.gitignore b/.gitignore index 045bc40e..e7239baf 100644 --- a/.gitignore +++ b/.gitignore @@ -12,3 +12,5 @@ _site _build/ *.egg-info/ + +src/_includes/cwl/**/output.txt diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 290267fb..70b76ed9 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -3,9 +3,10 @@ version: 2 # NOTE: If you modify this file to install a package with pip or apt, please # verify if we need the same package added to our CI. build: - os: ubuntu-20.04 + os: ubuntu-22.04 tools: python: "3.9" + nodejs: "16" apt_packages: - graphviz @@ -19,4 +20,3 @@ python: path: . extra_requirements: - all - diff --git a/Makefile b/Makefile index 7f787e87..9f6459ec 100644 --- a/Makefile +++ b/Makefile @@ -28,6 +28,7 @@ watch: clean --ignore='**venv' \ --ignore='**.github' \ --ignore='*.egg-info' \ + --ignore='**_includes/**/*.txt' \ --watch='cwl' ## unittest-examples : @@ -38,6 +39,9 @@ unittest-examples: check-json: python -m json.tool < src/.zenodo.json >> /dev/null && exit 0 || echo "NOT valid JSON"; exit 1 +container-pull: + for container in $$(git grep dockerPull $$(git ls-files *.cwl) | awk '-F: ' '{print $$3}'); do docker pull $${container}; done + .PHONY: help clean watch unittest-examples check-json Makefile # Catch-all target : route all unknown targets to Sphinx using the new diff --git a/src/_includes/cwl/03-input/whale.txt b/cwl/sphinx/__init__.py similarity index 100% rename from src/_includes/cwl/03-input/whale.txt rename to cwl/sphinx/__init__.py diff --git a/cwl/sphinx/runcmd.py b/cwl/sphinx/runcmd.py new file mode 100644 index 00000000..e9cb3016 --- /dev/null +++ b/cwl/sphinx/runcmd.py @@ -0,0 +1,183 @@ +import csv +import os +import re +import shlex +import subprocess +import sys + +from pathlib import Path + +from docutils.parsers.rst import directives +from sphinx.directives import code + +"""" +Patched version of https://github.com/sphinx-contrib/sphinxcontrib-runcmd +with default values to avoid having to re-type in every page. Also +prepends commands with a value (``$``), see https://github.com/invenia/sphinxcontrib-runcmd/issues/1. +Finally, it also checks if the command is ``cwltool``, and if then +tries to remove any paths from the command-line (not the logs). +""" + +__version__ = "0.2.0" + +# CONSTANTS +RE_SPLIT = re.compile(r"(?P.*)(?.*)") + + +# These classes were in the .util module of the original directive. +class _Singleton(type): + _instances = {} + + def __call__(cls, *args, **kwargs): + if cls not in cls._instances: + cls._instances[cls] = super(_Singleton, cls).__call__(*args, **kwargs) + return cls._instances[cls] + + +class Singleton(_Singleton("SingletonMeta", (object,), {})): + pass + + +class CMDCache(Singleton): + cache = {} + + def get(self, cmd, working_directory): + h = hash(cmd) + if h in self.cache: + return self.cache[h] + else: + result = run_command(cmd, working_directory) + self.cache[h] = result + return result + + +def run_command(command, working_directory): + true_cmd = shlex.split(command) + try: + # The subprocess Popen function takes a ``cwd`` argument that + # conveniently changes the working directory to run the command. + # + # We also patched the stderr to redirect to STDOUT, + # so that stderr and stdout appear in order, as you would see in + # a terminal. + # + # Finally, note that ``cwltool`` by default emits ANSI colors in the + # terminal, which are harder to be parsed and/or rendered in Sphinx. + # For that reason, we define --disable-color in the CWLTOOL_OPTIONS + # environment variable, which is used by ``cwltool``. + env = os.environ + cwl_options = set(env.get('CWLTOOL_OPTIONS', '').split(' ')) + cwl_options.add('--disable-color') + env['CWLTOOL_OPTIONS'] = ' '.join(cwl_options) + subp = subprocess.Popen( + true_cmd, + cwd=working_directory, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT + ) + except Exception as e: + out = "" + err = e + else: + out, err = subp.communicate() + encoding = sys.getfilesystemencoding() + out = out.decode(encoding, "replace").rstrip() + # The stderr is now combined with stdout. + # err = err.decode(encoding, "replace").rstrip() + + if err and err != "": + print("Error in runcmd: {}".format(err)) + out = "{}\n{}".format(out, err) + + return out + + +class RunCmdDirective(code.CodeBlock): + has_content = False + final_argument_whitespace = False + required_arguments = 1 + optional_arguments = 99 + + option_spec = { + # code.CodeBlock option_spec + "linenos": directives.flag, + "dedent": int, + "lineno-start": int, + "emphasize-lines": directives.unchanged_required, + "caption": directives.unchanged_required, + "class": directives.class_option, + "name": directives.unchanged, + # RunCmdDirective option_spec + "syntax": directives.unchanged, + "replace": directives.unchanged, + "prompt": directives.flag, + "dedent-output": int, + "working-directory": directives.unchanged + } + + def run(self): + # Grab a cache singleton instance + cache = CMDCache() + + # The examples in our User Guide are stored in ``src/_includes/cwl``. + # For convenience, instead of including that in every command, we + # allow the directive to receive a working directory, so that we + # change to that working directory before running the desired command. + # The working directory is omitted from the final output. + working_directory = self.options.get('working-directory', 'src/_includes/cwl/') + if working_directory == '': + # subprocess default value, so that we can disable it if needed. + working_directory = None + else: + # You can run Sphinx from the root directory, with `make watch` + # for instance, or from the src directory (RTD does that). + working_directory_path = Path(working_directory) + if not working_directory_path.exists() and str(working_directory_path).startswith('src/'): + working_directory = Path(working_directory[4:]) + + # Get the command output + command = " ".join(self.arguments) + output = cache.get(command, working_directory) + + # Grab our custom commands + syntax = self.options.get("syntax", "bash") + replace = self.options.get("replace", '') + reader = csv.reader([replace], delimiter=",", escapechar="\\") + # prompt = "prompt" in self.options + # We patched this so that the prompt is displayed by default, similar + # to how ``{code-block} console`` works. + prompt = True + dedent_output = self.options.get("dedent-output", 0) + + # Dedent the output if required + if dedent_output > 0: + output = "\n".join([x[dedent_output:] for x in output.split("\n")]) + + # Add the prompt to our output if required + if prompt: + output = "$ {}\n{}".format(command, output) + + # Do our "replace" syntax on the command output + for items in reader: + for regex in items: + if regex != "": + match = RE_SPLIT.match(regex) + p = match.group("pattern") + # Let's unescape the escape chars here as we don't need them to be + # escaped in the replacement at this point + r = match.group("replacement").replace("\\", "") + output = re.sub(p, r, output) + + # Set up our arguments to run the CodeBlock parent run function + self.arguments[0] = syntax + self.content = [output] + node = super(RunCmdDirective, self).run() + + return node + + +def setup(app): + app.add_directive("runcmd", RunCmdDirective) + + return {"version": __version__} diff --git a/setup.cfg b/setup.cfg index 2566cfb8..0d0a6f7f 100644 --- a/setup.cfg +++ b/setup.cfg @@ -32,11 +32,13 @@ packages = find_namespace: include_package_data = True python_requires = >=3.6 install_requires = + cwl_runner + cwl-utils==0.* myst-parser==0.* pydata-sphinx-theme==0.* sphinx==5.* sphinx-reredirects==0.1.* - cwl-utils==0.* + sphinxcontrib-runcmd==0.2.* [options.packages.find] include = cwl* @@ -47,6 +49,9 @@ test = cwltool==3.1.* watch = sphinx-autobuild==2021.3.* +rtd = + udocker all = %(test)s %(watch)s + %(rtd)s diff --git a/src/_includes/cwl/02-1st-example/1st-tool.cwl b/src/_includes/cwl/02-1st-example/1st-tool.cwl deleted file mode 100755 index 1d550fc2..00000000 --- a/src/_includes/cwl/02-1st-example/1st-tool.cwl +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: CommandLineTool -baseCommand: echo -inputs: - message: - type: string - inputBinding: - position: 1 -outputs: [] diff --git a/src/_includes/cwl/14-runtime/echo-job.yml b/src/_includes/cwl/14-runtime/echo-job.yml deleted file mode 100755 index 5329e737..00000000 --- a/src/_includes/cwl/14-runtime/echo-job.yml +++ /dev/null @@ -1 +0,0 @@ -message: Hello world! diff --git a/src/_includes/cwl/22-nested-workflows/1st-workflow.cwl b/src/_includes/cwl/22-nested-workflows/1st-workflow.cwl deleted file mode 100644 index 24b19bd2..00000000 --- a/src/_includes/cwl/22-nested-workflows/1st-workflow.cwl +++ /dev/null @@ -1,26 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: Workflow -inputs: - tarball: File - name_of_file_to_extract: string - -outputs: - compiled_class: - type: File - outputSource: compile/classfile - -steps: - untar: - run: tar-param.cwl - in: - tarfile: tarball - extractfile: name_of_file_to_extract - out: [extracted_file] - - compile: - run: arguments.cwl - in: - src: untar/extracted_file - out: [classfile] diff --git a/src/_includes/cwl/22-nested-workflows/arguments.cwl b/src/_includes/cwl/22-nested-workflows/arguments.cwl deleted file mode 100644 index c26d2642..00000000 --- a/src/_includes/cwl/22-nested-workflows/arguments.cwl +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: CommandLineTool -label: Example trivial wrapper for Java 9 compiler -hints: - DockerRequirement: - dockerPull: openjdk:9.0.1-11-slim -baseCommand: javac -arguments: ["-d", $(runtime.outdir)] -inputs: - src: - type: File - inputBinding: - position: 1 -outputs: - classfile: - type: File - outputBinding: - glob: "*.class" - diff --git a/src/_includes/cwl/22-nested-workflows/tar-param.cwl b/src/_includes/cwl/22-nested-workflows/tar-param.cwl deleted file mode 100644 index 87ad9e0f..00000000 --- a/src/_includes/cwl/22-nested-workflows/tar-param.cwl +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env cwl-runner - -cwlVersion: v1.0 -class: CommandLineTool -baseCommand: [tar, --extract] -inputs: - tarfile: - type: File - inputBinding: - prefix: --file - extractfile: - type: string - inputBinding: - position: 1 -outputs: - extracted_file: - type: File - outputBinding: - glob: $(inputs.extractfile) diff --git a/src/_includes/cwl/additional-arguments-and-parameters/Hello.class b/src/_includes/cwl/additional-arguments-and-parameters/Hello.class new file mode 100644 index 00000000..7ef1eca2 Binary files /dev/null and b/src/_includes/cwl/additional-arguments-and-parameters/Hello.class differ diff --git a/src/_includes/cwl/08-arguments/Hello.java b/src/_includes/cwl/additional-arguments-and-parameters/Hello.java similarity index 100% rename from src/_includes/cwl/08-arguments/Hello.java rename to src/_includes/cwl/additional-arguments-and-parameters/Hello.java diff --git a/src/_includes/cwl/08-arguments/arguments-job.yml b/src/_includes/cwl/additional-arguments-and-parameters/arguments-job.yml similarity index 100% rename from src/_includes/cwl/08-arguments/arguments-job.yml rename to src/_includes/cwl/additional-arguments-and-parameters/arguments-job.yml diff --git a/src/_includes/cwl/08-arguments/arguments.cwl b/src/_includes/cwl/additional-arguments-and-parameters/arguments.cwl similarity index 100% rename from src/_includes/cwl/08-arguments/arguments.cwl rename to src/_includes/cwl/additional-arguments-and-parameters/arguments.cwl diff --git a/src/_includes/cwl/conformance-test.yml b/src/_includes/cwl/conformance-test.yml index acdab96d..bb27f087 100644 --- a/src/_includes/cwl/conformance-test.yml +++ b/src/_includes/cwl/conformance-test.yml @@ -1,69 +1,64 @@ -# Section 2 -- doc: Test for section 2 - job: 02-1st-example/echo-job.yml - tool: hello_world.cwl - should_fail: false - output: {} - -# Section 3 -- doc: Test for section 3 - job: 03-input/inp-job.yml - tool: 03-input/inp.cwl - should_fail: false - output: {} - -# Section 4 -- doc: Test for section 4 - job: 04-output/tar-job.yml - tool: 04-output/tar.cwl +# Expressions +- doc: Test for Expressions section + job: expressions/empty.yml + tool: expressions/expression.cwl output: example_out: class: File - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 - basename: hello.txt + checksum: sha1$a739a6ff72d660d32111265e508ed2fc91f01a7c + basename: output.txt location: Any - size: 0 + size: 36 + +# Inputs +- doc: Test for Essential Inputs section + job: inputs/inp-job.yml + tool: inputs/inp.cwl + should_fail: false + output: {} -# Section 5 -- doc: Test for section 5 - job: 05-stdout/echo-job.yml - tool: 05-stdout/stdout.cwl +- doc: Test for Array Inputs section + job: inputs/array-inputs-job.yml + tool: inputs/array-inputs.cwl output: example_out: class: File - checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b + checksum: sha1$91038e29452bc77dcd21edef90a15075f3071540 basename: output.txt location: Any - size: 13 + size: 60 -# Section 6 -- doc: Test for section 6 - job: 06-params/tar-param-job.yml - tool: 06-params/tar-param.cwl +- doc: Test for Include and Exclusive Inputs section (1st example) + job: inputs/record-job1.yml + tool: inputs/record.cwl + should_fail: true + +- doc: Test for Include and Exclusive Inputs section (2nd example) + job: inputs/record-job2.yml + tool: inputs/record.cwl output: - extracted_file: + example_out: class: File - checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 - basename: goodbye.txt + checksum: sha1$329fe3b598fed0dfd40f511522eaf386edb2d077 + basename: output.txt location: Any - size: 0 + size: 23 -# Section 7 -- doc: Test for section 7 - job: 07-containers/docker-job.yml - tool: 07-containers/docker.cwl +- doc: Test for Include and Exclusive Inputs section (3rd example) + job: inputs/record-job3.yml + tool: inputs/record.cwl output: example_out: class: File - checksum: sha1$648a6a6ffffdaa0badb23b8baf90b6168dd16b3a + checksum: sha1$77f572b28e441240a5e30eb14f1d300bcc13a3b4 basename: output.txt location: Any - size: 12 + size: 22 -# Section 8 -- doc: Test for section 8 - job: 08-arguments/arguments-job.yml - tool: 08-arguments/arguments.cwl +# Additional Arguments and Parameters +- doc: Test for Additional Arguments and Parameters section + job: additional-arguments-and-parameters/arguments-job.yml + tool: additional-arguments-and-parameters/arguments.cwl output: classfile: class: File @@ -72,22 +67,44 @@ location: Any size: 184 -# Section 9 -- doc: Test for section 9 - job: 09-array-inputs/array-inputs-job.yml - tool: 09-array-inputs/array-inputs.cwl +# Parameter References +- doc: Test for Parameter References section + job: parameter-references/tar-param-job.yml + tool: parameter-references/tar-param.cwl + output: + extracted_file: + class: File + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + basename: goodbye.txt + location: Any + size: 0 + +# Outputs +- doc: Test for Outputs section + job: outputs/tar-job.yml + tool: outputs/tar.cwl output: example_out: class: File - checksum: sha1$91038e29452bc77dcd21edef90a15075f3071540 + checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709 + basename: hello.txt + location: Any + size: 0 + +- doc: Test for Outputs, Capturing Standard Output section + job: outputs/echo-job.yml + tool: outputs/stdout.cwl + output: + example_out: + class: File + checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b basename: output.txt location: Any - size: 60 + size: 13 -# Section 10 -- doc: Test for section 10 - job: 10-array-outputs/array-outputs-job.yml - tool: 10-array-outputs/array-outputs.cwl +- doc: Test for Outputs, Array Outputs section + job: outputs/array-outputs-job.yml + tool: outputs/array-outputs.cwl output: output: - class: File @@ -101,60 +118,76 @@ location: Any size: 0 -# Section 11 depends on side-effects -- doc: Test for section 11 (1st example) - job: 11-records/record-job1.yml - tool: 11-records/record.cwl - should_fail: true +# Custom Types +# NOTE: The checksum is always changed for every execution +- doc: Test for Custom Types section + job: custom-types/custom-types.yml + tool: custom-types/custom-types.cwl + output: + result: + class: File + basename: rich_sparse_otu_table.hdf5 + location: Any + size: 46992 -- doc: Test for section 11 (2nd example) - job: 11-records/record-job2.yml - tool: 11-records/record.cwl +# Workflows +- doc: Test for Workflows section + job: workflows/1st-workflow-job.yml + tool: workflows/1st-workflow.cwl output: - example_out: + compiled_class: class: File - checksum: sha1$329fe3b598fed0dfd40f511522eaf386edb2d077 - basename: output.txt + checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd + basename: Hello.class location: Any - size: 23 + size: 419 -- doc: Test for section 11 (3rd example) - job: 11-records/record-job3.yml - tool: 11-records/record.cwl +- doc: Test for Workflows, Nested Workflows section + tool: workflows/nestedworkflows.cwl output: - example_out: + classout: class: File - checksum: sha1$77f572b28e441240a5e30eb14f1d300bcc13a3b4 - basename: output.txt + checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd + basename: Hello.class location: Any - size: 22 + size: 419 -# Section 12 -- doc: Test for section 12 - job: 12-env/echo-job.yml - tool: 12-env/env.cwl +# TODO: The old section 23-scatter-workflow had workflows, but no conformance tests? +# Maybe add them here, they are now in workflows: +# - hello_world.cwl +# - hello_world_to_stdout.cwl +# - scatter-job.cwl +# - scatter-nested-workflow.cwl +# - scatter-two-steps.cwl +# - scatter-workflow.cwl +# - wc-tool.cwl + +# Environment Variables +- doc: Test for Environment Variables section + job: environment-variables/echo-job.yml + tool: environment-variables/env.cwl output: example_out: class: File basename: output.txt location: Any -# Section 13 -- doc: Test for section 13 - job: 13-expressions/empty.yml - tool: 13-expressions/expression.cwl +# Using Containers +- doc: Test for Using Containers section + job: using-containers/docker-job.yml + tool: using-containers/docker.cwl output: example_out: class: File - checksum: sha1$a739a6ff72d660d32111265e508ed2fc91f01a7c + checksum: sha1$648a6a6ffffdaa0badb23b8baf90b6168dd16b3a basename: output.txt location: Any - size: 36 + size: 12 -# Section 14 -- doc: Test for section 14 - job: 14-runtime/echo-job.yml - tool: 14-runtime/createfile.cwl +# Creating files at runtime +- doc: Test for Creating files at runtime section + job: creating-files-at-runtime/echo-job.yml + tool: creating-files-at-runtime/createfile.cwl output: example_out: class: File @@ -163,10 +196,10 @@ location: Any size: 25 -# Section 15 -- doc: Test for section 15 - job: 15-staging/arguments-job.yml - tool: 15-staging/linkfile.cwl +# Staging input files +- doc: Test for Staging input files section + job: staging-input-files/arguments-job.yml + tool: staging-input-files/linkfile.cwl output: classfile: class: File @@ -175,80 +208,45 @@ location: Any size: 184 -# Section 16 -# Note: The checksum and size is always changed for every executions -- doc: Test for section 16 - job: 16-file-formats/sample.yml - tool: 16-file-formats/metadata_example.cwl +# File formats +# NOTE: The checksum and size is always changed for every execution +- doc: Test for File formats section + job: file-formats/sample.yml + tool: file-formats/metadata_example.cwl output: report: class: File basename: output.txt location: Any -# Section 17 -# Note: The checksum and size is always changed for every executions -- doc: Test for section 17 (1st example) - job: 17-metadata/sample.yml - tool: 17-metadata/metadata_example2.cwl +# Metadata and authorship +# NOTE: The checksum and size is always changed for every executions +- doc: Test for Metadata and authorship section (1st example) + job: metadata-and-authorship/sample.yml + tool: metadata-and-authorship/metadata_example2.cwl output: report: class: File basename: output.txt location: Any -- doc: Test for section 17 (extended example) - job: 17-metadata/sample.yml - tool: 17-metadata/metadata_example3.cwl +- doc: Test for Metadata and authorship section (extended example) + job: metadata-and-authorship/sample.yml + tool: metadata-and-authorship/metadata_example3.cwl output: report: class: File basename: output.txt location: Any -# Section 19 -# Note: The checksum is always changed for every executions -- doc: Test for section 19 - job: 19-custom-types/custom-types.yml - tool: 19-custom-types/custom-types.cwl - output: - result: - class: File - basename: rich_sparse_otu_table.hdf5 - location: Any - size: 46992 - -# Section 20 +# Specifying software requirements # See: Issue #48 # - doc: Test for section 20 -# job: 20-software-requirements/custom-types.yml -# tool: 20-software-requirements/costom-types.cwl +# job: specifying-software-requirements/custom-types.yml +# tool: specifying-software-requirements/custom-types.cwl # output: # i5Annotations: # class: File # basename: test_proteins.i5_annotations # location: Any # size: Any # To be fixed - -# Section 21 -- doc: Test for section 21 - job: 21-1st-workflow/1st-workflow-job.yml - tool: 21-1st-workflow/1st-workflow.cwl - output: - compiled_class: - class: File - checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd - basename: Hello.class - location: Any - size: 419 - -# Section 22 -- doc: Test for section 22 - tool: 22-nested-workflows/nestedworkflows.cwl - output: - classout: - class: File - checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd - basename: Hello.class - location: Any - size: 419 diff --git a/src/_includes/cwl/14-runtime/createfile.cwl b/src/_includes/cwl/creating-files-at-runtime/createfile.cwl similarity index 100% rename from src/_includes/cwl/14-runtime/createfile.cwl rename to src/_includes/cwl/creating-files-at-runtime/createfile.cwl diff --git a/src/_includes/cwl/02-1st-example/echo-job.yml b/src/_includes/cwl/creating-files-at-runtime/echo-job.yml similarity index 100% rename from src/_includes/cwl/02-1st-example/echo-job.yml rename to src/_includes/cwl/creating-files-at-runtime/echo-job.yml diff --git a/src/_includes/cwl/19-custom-types/InterProScan-apps.yml b/src/_includes/cwl/custom-types/InterProScan-apps.yml similarity index 100% rename from src/_includes/cwl/19-custom-types/InterProScan-apps.yml rename to src/_includes/cwl/custom-types/InterProScan-apps.yml diff --git a/src/_includes/cwl/19-custom-types/biom-convert-table.yaml b/src/_includes/cwl/custom-types/biom-convert-table.yaml similarity index 100% rename from src/_includes/cwl/19-custom-types/biom-convert-table.yaml rename to src/_includes/cwl/custom-types/biom-convert-table.yaml diff --git a/src/_includes/cwl/19-custom-types/custom-types.cwl b/src/_includes/cwl/custom-types/custom-types.cwl similarity index 95% rename from src/_includes/cwl/19-custom-types/custom-types.cwl rename to src/_includes/cwl/custom-types/custom-types.cwl index fb70427a..707d5c2a 100644 --- a/src/_includes/cwl/19-custom-types/custom-types.cwl +++ b/src/_includes/cwl/custom-types/custom-types.cwl @@ -1,4 +1,4 @@ -#!/usr/bin/env cwl-runner +#!/usr/bin/env cwl-runner cwlVersion: v1.0 class: CommandLineTool @@ -43,7 +43,7 @@ inputs: baseCommand: [ biom, convert ] arguments: - - valueFrom: $(inputs.biom.nameroot).hdf5 + - valueFrom: $(inputs.biom.nameroot).hdf5 prefix: --output-fp - --to-hdf5 diff --git a/src/_includes/cwl/19-custom-types/custom-types.yml b/src/_includes/cwl/custom-types/custom-types.yml similarity index 100% rename from src/_includes/cwl/19-custom-types/custom-types.yml rename to src/_includes/cwl/custom-types/custom-types.yml diff --git a/src/_includes/cwl/19-custom-types/rich_sparse_otu_table.biom b/src/_includes/cwl/custom-types/rich_sparse_otu_table.biom similarity index 100% rename from src/_includes/cwl/19-custom-types/rich_sparse_otu_table.biom rename to src/_includes/cwl/custom-types/rich_sparse_otu_table.biom diff --git a/src/_includes/cwl/19-custom-types/test_proteins.fasta b/src/_includes/cwl/custom-types/test_proteins.fasta similarity index 100% rename from src/_includes/cwl/19-custom-types/test_proteins.fasta rename to src/_includes/cwl/custom-types/test_proteins.fasta diff --git a/src/_includes/cwl/echo.cwl b/src/_includes/cwl/echo.cwl new file mode 100644 index 00000000..6248ae84 --- /dev/null +++ b/src/_includes/cwl/echo.cwl @@ -0,0 +1,18 @@ +cwlVersion: v1.2 +class: CommandLineTool + +baseCommand: echo + +stdout: output.txt + +inputs: + message: + type: string + inputBinding: {} +outputs: + out: + type: string + outputBinding: + glob: output.txt + loadContents: true + outputEval: $(self[0].contents) diff --git a/src/_includes/cwl/05-stdout/echo-job.yml b/src/_includes/cwl/environment-variables/echo-job.yml similarity index 100% rename from src/_includes/cwl/05-stdout/echo-job.yml rename to src/_includes/cwl/environment-variables/echo-job.yml diff --git a/src/_includes/cwl/12-env/env.cwl b/src/_includes/cwl/environment-variables/env.cwl similarity index 100% rename from src/_includes/cwl/12-env/env.cwl rename to src/_includes/cwl/environment-variables/env.cwl diff --git a/src/_includes/cwl/13-expressions/empty.yml b/src/_includes/cwl/expressions/empty.yml similarity index 100% rename from src/_includes/cwl/13-expressions/empty.yml rename to src/_includes/cwl/expressions/empty.yml diff --git a/src/_includes/cwl/13-expressions/expression.cwl b/src/_includes/cwl/expressions/expression.cwl similarity index 100% rename from src/_includes/cwl/13-expressions/expression.cwl rename to src/_includes/cwl/expressions/expression.cwl diff --git a/src/_includes/cwl/16-file-formats/file-formats.bam b/src/_includes/cwl/file-formats/file-formats.bam similarity index 100% rename from src/_includes/cwl/16-file-formats/file-formats.bam rename to src/_includes/cwl/file-formats/file-formats.bam diff --git a/src/_includes/cwl/16-file-formats/metadata_example.cwl b/src/_includes/cwl/file-formats/metadata_example.cwl similarity index 100% rename from src/_includes/cwl/16-file-formats/metadata_example.cwl rename to src/_includes/cwl/file-formats/metadata_example.cwl diff --git a/src/_includes/cwl/16-file-formats/sample.yml b/src/_includes/cwl/file-formats/sample.yml similarity index 100% rename from src/_includes/cwl/16-file-formats/sample.yml rename to src/_includes/cwl/file-formats/sample.yml diff --git a/src/_includes/cwl/hello_world-job.json b/src/_includes/cwl/hello_world-job.json new file mode 100644 index 00000000..c74eb649 --- /dev/null +++ b/src/_includes/cwl/hello_world-job.json @@ -0,0 +1,3 @@ +{ + "message": "こんにちは世界" +} diff --git a/src/_includes/cwl/09-array-inputs/array-inputs-job.yml b/src/_includes/cwl/inputs/array-inputs-job.yml similarity index 100% rename from src/_includes/cwl/09-array-inputs/array-inputs-job.yml rename to src/_includes/cwl/inputs/array-inputs-job.yml diff --git a/src/_includes/cwl/09-array-inputs/array-inputs.cwl b/src/_includes/cwl/inputs/array-inputs.cwl similarity index 100% rename from src/_includes/cwl/09-array-inputs/array-inputs.cwl rename to src/_includes/cwl/inputs/array-inputs.cwl diff --git a/src/_includes/cwl/03-input/inp-job.yml b/src/_includes/cwl/inputs/inp-job.yml similarity index 100% rename from src/_includes/cwl/03-input/inp-job.yml rename to src/_includes/cwl/inputs/inp-job.yml diff --git a/src/_includes/cwl/03-input/inp.cwl b/src/_includes/cwl/inputs/inp.cwl similarity index 100% rename from src/_includes/cwl/03-input/inp.cwl rename to src/_includes/cwl/inputs/inp.cwl diff --git a/src/_includes/cwl/11-records/record-job1.yml b/src/_includes/cwl/inputs/record-job1.yml similarity index 100% rename from src/_includes/cwl/11-records/record-job1.yml rename to src/_includes/cwl/inputs/record-job1.yml diff --git a/src/_includes/cwl/11-records/record-job2.yml b/src/_includes/cwl/inputs/record-job2.yml similarity index 100% rename from src/_includes/cwl/11-records/record-job2.yml rename to src/_includes/cwl/inputs/record-job2.yml diff --git a/src/_includes/cwl/11-records/record-job3.yml b/src/_includes/cwl/inputs/record-job3.yml similarity index 100% rename from src/_includes/cwl/11-records/record-job3.yml rename to src/_includes/cwl/inputs/record-job3.yml diff --git a/src/_includes/cwl/11-records/record.cwl b/src/_includes/cwl/inputs/record.cwl similarity index 100% rename from src/_includes/cwl/11-records/record.cwl rename to src/_includes/cwl/inputs/record.cwl diff --git a/src/_includes/cwl/inputs/whale.txt b/src/_includes/cwl/inputs/whale.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/_includes/cwl/17-metadata/file-formats.bam b/src/_includes/cwl/metadata-and-authorship/file-formats.bam similarity index 100% rename from src/_includes/cwl/17-metadata/file-formats.bam rename to src/_includes/cwl/metadata-and-authorship/file-formats.bam diff --git a/src/_includes/cwl/17-metadata/metadata_example2.cwl b/src/_includes/cwl/metadata-and-authorship/metadata_example2.cwl similarity index 96% rename from src/_includes/cwl/17-metadata/metadata_example2.cwl rename to src/_includes/cwl/metadata-and-authorship/metadata_example2.cwl index b4921901..745c1779 100644 --- a/src/_includes/cwl/17-metadata/metadata_example2.cwl +++ b/src/_includes/cwl/metadata-and-authorship/metadata_example2.cwl @@ -38,7 +38,7 @@ s:contributor: s:citation: https://dx.doi.org/10.6084/m9.figshare.3115156.v2 s:codeRepository: https://github.com/common-workflow-language/common-workflow-language s:dateCreated: "2016-12-13" -s:license: https://spdx.org/licenses/Apache-2.0 +s:license: https://spdx.org/licenses/Apache-2.0 $namespaces: s: https://schema.org/ diff --git a/src/_includes/cwl/17-metadata/metadata_example3.cwl b/src/_includes/cwl/metadata-and-authorship/metadata_example3.cwl similarity index 96% rename from src/_includes/cwl/17-metadata/metadata_example3.cwl rename to src/_includes/cwl/metadata-and-authorship/metadata_example3.cwl index 99135075..09d31e4a 100644 --- a/src/_includes/cwl/17-metadata/metadata_example3.cwl +++ b/src/_includes/cwl/metadata-and-authorship/metadata_example3.cwl @@ -42,7 +42,7 @@ s:contributor: s:citation: https://dx.doi.org/10.6084/m9.figshare.3115156.v2 s:codeRepository: https://github.com/common-workflow-language/common-workflow-language s:dateCreated: "2016-12-13" -s:license: https://spdx.org/licenses/Apache-2.0 +s:license: https://spdx.org/licenses/Apache-2.0 s:keywords: edam:topic_0091 , edam:topic_0622 s:programmingLanguage: C diff --git a/src/_includes/cwl/17-metadata/sample.yml b/src/_includes/cwl/metadata-and-authorship/sample.yml similarity index 100% rename from src/_includes/cwl/17-metadata/sample.yml rename to src/_includes/cwl/metadata-and-authorship/sample.yml diff --git a/src/_includes/cwl/operations/echo.cwl b/src/_includes/cwl/operations/echo.cwl new file mode 100644 index 00000000..3d35dbb8 --- /dev/null +++ b/src/_includes/cwl/operations/echo.cwl @@ -0,0 +1,17 @@ +cwlVersion: v1.2 + +# What type of CWL process we have in this document. +class: CommandLineTool +# This CommandLineTool executes the linux "echo" command-line tool. +baseCommand: echo + +# The inputs for this process. +inputs: + message: + type: string + # A default value that can be overridden, e.g. --message "Hola mundo" + default: "Hello World" + # Bind this message value as an argument to "echo". + inputBinding: + position: 1 +outputs: [] diff --git a/src/_includes/cwl/operations/operations.cwl b/src/_includes/cwl/operations/operations.cwl new file mode 100644 index 00000000..43d7ef9e --- /dev/null +++ b/src/_includes/cwl/operations/operations.cwl @@ -0,0 +1,27 @@ +cwlVersion: v1.2 +class: Workflow + + +inputs: + message: string +outputs: [] + +steps: + echo: + run: ../echo.cwl + in: + message: message + out: [out] + # Here you know you want an operation that changes the case of + # the previous step, but you do not have an implementation yet. + uppercase: + run: + class: Operation + inputs: + message: string + outputs: + uppercase_message: string + in: + message: + source: echo/out + out: [uppercase_message] diff --git a/src/_includes/cwl/10-array-outputs/array-outputs-job.yml b/src/_includes/cwl/outputs/array-outputs-job.yml similarity index 100% rename from src/_includes/cwl/10-array-outputs/array-outputs-job.yml rename to src/_includes/cwl/outputs/array-outputs-job.yml diff --git a/src/_includes/cwl/10-array-outputs/array-outputs.cwl b/src/_includes/cwl/outputs/array-outputs.cwl similarity index 100% rename from src/_includes/cwl/10-array-outputs/array-outputs.cwl rename to src/_includes/cwl/outputs/array-outputs.cwl diff --git a/src/_includes/cwl/outputs/baz.txt b/src/_includes/cwl/outputs/baz.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/_includes/cwl/12-env/echo-job.yml b/src/_includes/cwl/outputs/echo-job.yml similarity index 100% rename from src/_includes/cwl/12-env/echo-job.yml rename to src/_includes/cwl/outputs/echo-job.yml diff --git a/src/_includes/cwl/outputs/foo.txt b/src/_includes/cwl/outputs/foo.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/_includes/cwl/04-output/hello.tar b/src/_includes/cwl/outputs/hello.tar similarity index 100% rename from src/_includes/cwl/04-output/hello.tar rename to src/_includes/cwl/outputs/hello.tar diff --git a/src/_includes/cwl/outputs/hello.txt b/src/_includes/cwl/outputs/hello.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/_includes/cwl/05-stdout/stdout.cwl b/src/_includes/cwl/outputs/stdout.cwl similarity index 100% rename from src/_includes/cwl/05-stdout/stdout.cwl rename to src/_includes/cwl/outputs/stdout.cwl diff --git a/src/_includes/cwl/04-output/tar-job.yml b/src/_includes/cwl/outputs/tar-job.yml similarity index 100% rename from src/_includes/cwl/04-output/tar-job.yml rename to src/_includes/cwl/outputs/tar-job.yml diff --git a/src/_includes/cwl/04-output/tar.cwl b/src/_includes/cwl/outputs/tar.cwl similarity index 100% rename from src/_includes/cwl/04-output/tar.cwl rename to src/_includes/cwl/outputs/tar.cwl diff --git a/src/_includes/cwl/parameter-references/goodbye.txt b/src/_includes/cwl/parameter-references/goodbye.txt new file mode 100644 index 00000000..e69de29b diff --git a/src/_includes/cwl/06-params/hello.tar b/src/_includes/cwl/parameter-references/hello.tar similarity index 100% rename from src/_includes/cwl/06-params/hello.tar rename to src/_includes/cwl/parameter-references/hello.tar diff --git a/src/_includes/cwl/06-params/tar-param-job.yml b/src/_includes/cwl/parameter-references/tar-param-job.yml similarity index 100% rename from src/_includes/cwl/06-params/tar-param-job.yml rename to src/_includes/cwl/parameter-references/tar-param-job.yml diff --git a/src/_includes/cwl/06-params/tar-param.cwl b/src/_includes/cwl/parameter-references/tar-param.cwl similarity index 100% rename from src/_includes/cwl/06-params/tar-param.cwl rename to src/_includes/cwl/parameter-references/tar-param.cwl diff --git a/src/_includes/cwl/20-software-requirements/InterProScan-apps.yml b/src/_includes/cwl/specifying-software-requirements/InterProScan-apps.yml similarity index 100% rename from src/_includes/cwl/20-software-requirements/InterProScan-apps.yml rename to src/_includes/cwl/specifying-software-requirements/InterProScan-apps.yml diff --git a/src/_includes/cwl/20-software-requirements/custom-types.cwl b/src/_includes/cwl/specifying-software-requirements/custom-types.cwl similarity index 100% rename from src/_includes/cwl/20-software-requirements/custom-types.cwl rename to src/_includes/cwl/specifying-software-requirements/custom-types.cwl diff --git a/src/_includes/cwl/20-software-requirements/custom-types.yml b/src/_includes/cwl/specifying-software-requirements/custom-types.yml similarity index 100% rename from src/_includes/cwl/20-software-requirements/custom-types.yml rename to src/_includes/cwl/specifying-software-requirements/custom-types.yml diff --git a/src/_includes/cwl/20-software-requirements/test_proteins.fasta b/src/_includes/cwl/specifying-software-requirements/test_proteins.fasta similarity index 100% rename from src/_includes/cwl/20-software-requirements/test_proteins.fasta rename to src/_includes/cwl/specifying-software-requirements/test_proteins.fasta diff --git a/src/_includes/cwl/staging-input-files/Hello.class b/src/_includes/cwl/staging-input-files/Hello.class new file mode 100644 index 00000000..7ef1eca2 Binary files /dev/null and b/src/_includes/cwl/staging-input-files/Hello.class differ diff --git a/src/_includes/cwl/15-staging/Hello.java b/src/_includes/cwl/staging-input-files/Hello.java similarity index 100% rename from src/_includes/cwl/15-staging/Hello.java rename to src/_includes/cwl/staging-input-files/Hello.java diff --git a/src/_includes/cwl/15-staging/arguments-job.yml b/src/_includes/cwl/staging-input-files/arguments-job.yml similarity index 100% rename from src/_includes/cwl/15-staging/arguments-job.yml rename to src/_includes/cwl/staging-input-files/arguments-job.yml diff --git a/src/_includes/cwl/15-staging/linkfile.cwl b/src/_includes/cwl/staging-input-files/linkfile.cwl similarity index 100% rename from src/_includes/cwl/15-staging/linkfile.cwl rename to src/_includes/cwl/staging-input-files/linkfile.cwl diff --git a/src/_includes/cwl/simplest_cwl.cwl b/src/_includes/cwl/true.cwl similarity index 100% rename from src/_includes/cwl/simplest_cwl.cwl rename to src/_includes/cwl/true.cwl diff --git a/src/_includes/cwl/simplest_cwl_shebang.cwl b/src/_includes/cwl/true_shebang.cwl old mode 100644 new mode 100755 similarity index 100% rename from src/_includes/cwl/simplest_cwl_shebang.cwl rename to src/_includes/cwl/true_shebang.cwl diff --git a/src/_includes/cwl/07-containers/docker-job.yml b/src/_includes/cwl/using-containers/docker-job.yml similarity index 100% rename from src/_includes/cwl/07-containers/docker-job.yml rename to src/_includes/cwl/using-containers/docker-job.yml diff --git a/src/_includes/cwl/07-containers/docker.cwl b/src/_includes/cwl/using-containers/docker.cwl similarity index 100% rename from src/_includes/cwl/07-containers/docker.cwl rename to src/_includes/cwl/using-containers/docker.cwl diff --git a/src/_includes/cwl/07-containers/hello.js b/src/_includes/cwl/using-containers/hello.js similarity index 100% rename from src/_includes/cwl/07-containers/hello.js rename to src/_includes/cwl/using-containers/hello.js diff --git a/src/_includes/cwl/21-1st-workflow/1st-workflow-job.yml b/src/_includes/cwl/workflows/1st-workflow-job.yml similarity index 100% rename from src/_includes/cwl/21-1st-workflow/1st-workflow-job.yml rename to src/_includes/cwl/workflows/1st-workflow-job.yml diff --git a/src/_includes/cwl/21-1st-workflow/1st-workflow.cwl b/src/_includes/cwl/workflows/1st-workflow.cwl similarity index 100% rename from src/_includes/cwl/21-1st-workflow/1st-workflow.cwl rename to src/_includes/cwl/workflows/1st-workflow.cwl diff --git a/src/_includes/cwl/workflows/Hello.class b/src/_includes/cwl/workflows/Hello.class new file mode 100644 index 00000000..839e858a Binary files /dev/null and b/src/_includes/cwl/workflows/Hello.class differ diff --git a/src/_includes/cwl/21-1st-workflow/arguments.cwl b/src/_includes/cwl/workflows/arguments.cwl similarity index 100% rename from src/_includes/cwl/21-1st-workflow/arguments.cwl rename to src/_includes/cwl/workflows/arguments.cwl diff --git a/src/_includes/cwl/21-1st-workflow/hello.tar b/src/_includes/cwl/workflows/hello.tar similarity index 100% rename from src/_includes/cwl/21-1st-workflow/hello.tar rename to src/_includes/cwl/workflows/hello.tar diff --git a/src/_includes/cwl/23-scatter-workflow/hello_world.cwl b/src/_includes/cwl/workflows/hello_world.cwl similarity index 94% rename from src/_includes/cwl/23-scatter-workflow/hello_world.cwl rename to src/_includes/cwl/workflows/hello_world.cwl index 4ebbdf38..48e2f792 100755 --- a/src/_includes/cwl/23-scatter-workflow/hello_world.cwl +++ b/src/_includes/cwl/workflows/hello_world.cwl @@ -8,6 +8,6 @@ inputs: type: string inputBinding: position: 1 -outputs: +outputs: echo_out: type: stdout diff --git a/src/_includes/cwl/23-scatter-workflow/hello_world_to_stdout.cwl b/src/_includes/cwl/workflows/hello_world_to_stdout.cwl similarity index 100% rename from src/_includes/cwl/23-scatter-workflow/hello_world_to_stdout.cwl rename to src/_includes/cwl/workflows/hello_world_to_stdout.cwl diff --git a/src/_includes/cwl/22-nested-workflows/nestedworkflows.cwl b/src/_includes/cwl/workflows/nestedworkflows.cwl similarity index 100% rename from src/_includes/cwl/22-nested-workflows/nestedworkflows.cwl rename to src/_includes/cwl/workflows/nestedworkflows.cwl diff --git a/src/_includes/cwl/23-scatter-workflow/scatter-job.yml b/src/_includes/cwl/workflows/scatter-job.yml similarity index 100% rename from src/_includes/cwl/23-scatter-workflow/scatter-job.yml rename to src/_includes/cwl/workflows/scatter-job.yml diff --git a/src/_includes/cwl/23-scatter-workflow/scatter-nested-workflow.cwl b/src/_includes/cwl/workflows/scatter-nested-workflow.cwl similarity index 90% rename from src/_includes/cwl/23-scatter-workflow/scatter-nested-workflow.cwl rename to src/_includes/cwl/workflows/scatter-nested-workflow.cwl index 95bbb0f5..9b2f6aa6 100644 --- a/src/_includes/cwl/23-scatter-workflow/scatter-nested-workflow.cwl +++ b/src/_includes/cwl/workflows/scatter-nested-workflow.cwl @@ -8,13 +8,13 @@ requirements: SubworkflowFeatureRequirement: {} inputs: - message_array: string[] + message_array: string[] steps: subworkflow: - run: + run: class: Workflow - inputs: + inputs: message: string outputs: [] steps: @@ -29,7 +29,7 @@ steps: input_file: echo/echo_out out: [] scatter: message - in: + in: message: message_array out: [] outputs: [] diff --git a/src/_includes/cwl/23-scatter-workflow/scatter-two-steps.cwl b/src/_includes/cwl/workflows/scatter-two-steps.cwl similarity index 93% rename from src/_includes/cwl/23-scatter-workflow/scatter-two-steps.cwl rename to src/_includes/cwl/workflows/scatter-two-steps.cwl index 347f6fff..57b9ca2e 100644 --- a/src/_includes/cwl/23-scatter-workflow/scatter-two-steps.cwl +++ b/src/_includes/cwl/workflows/scatter-two-steps.cwl @@ -7,7 +7,7 @@ requirements: ScatterFeatureRequirement: {} inputs: - message_array: string[] + message_array: string[] steps: echo: diff --git a/src/_includes/cwl/23-scatter-workflow/scatter-workflow.cwl b/src/_includes/cwl/workflows/scatter-workflow.cwl similarity index 89% rename from src/_includes/cwl/23-scatter-workflow/scatter-workflow.cwl rename to src/_includes/cwl/workflows/scatter-workflow.cwl index 80f941eb..80a334f3 100644 --- a/src/_includes/cwl/23-scatter-workflow/scatter-workflow.cwl +++ b/src/_includes/cwl/workflows/scatter-workflow.cwl @@ -7,7 +7,7 @@ requirements: ScatterFeatureRequirement: {} inputs: - message_array: string[] + message_array: string[] steps: echo: diff --git a/src/_includes/cwl/21-1st-workflow/tar-param.cwl b/src/_includes/cwl/workflows/tar-param.cwl similarity index 100% rename from src/_includes/cwl/21-1st-workflow/tar-param.cwl rename to src/_includes/cwl/workflows/tar-param.cwl diff --git a/src/_includes/cwl/23-scatter-workflow/wc-tool.cwl b/src/_includes/cwl/workflows/wc-tool.cwl similarity index 100% rename from src/_includes/cwl/23-scatter-workflow/wc-tool.cwl rename to src/_includes/cwl/workflows/wc-tool.cwl diff --git a/src/_includes/expression-tool.md b/src/_includes/expression-tool.md deleted file mode 100644 index 68094ac8..00000000 --- a/src/_includes/expression-tool.md +++ /dev/null @@ -1,68 +0,0 @@ -An expression tool is a type of Process that can be run by itself or -as a Workflow step. It executes a pure JavaScript expression. It is -meant to be used as a way to isolate complex JavaScript expressions -that need to operate on input data and produce some result as output. - -Similar to the command-line tool it requires `inputs` and `outputs`. -But instead of `baseCommand`, it requires an `expression` attribute. - -% TODO: Fix the missing link the graph below. We cannot have -% it here as this file is included in two other files. -% Sphinx prohibits it for the case where this could lead -% to duplicate anchors in a page (e.g. single-html). -% :name: expression-tool-graph - -```{graphviz} -:caption: CWL expression tool. -:align: center - -digraph G { - compound=true; - rankdir="LR"; - fontname="Verdana"; - fontsize="10"; - graph [splines=ortho]; - - node [fontname="Verdana", fontsize="10", shape=box]; - edge [fontname="Verdana", fontsize="10"]; - - subgraph cluster_0 { - expression[style="filled" label="JavaScript"]; - label="expression"; - fill=gray; - } - - inputs -> expression [lhead=cluster_0]; - expression -> outputs [ltail=cluster_0]; -} -``` - -% TODO: Fix the missing link the code below. We cannot have -% it here as this file is included in two other files. -% Sphinx prohibits it for the case where this could lead -% to duplicate anchors in a page (e.g. single-html). -% :name: uppercase.cwl - -```{code-block} cwl -:caption: "`uppercase.cwl`" -cwlVersion: v1.2 -class: ExpressionTool - -requirements: - InlineJavascriptRequirement: {} - -inputs: - message: string -outputs: - uppercase_message: string - -expression: | - ${ return {"uppercase_message": inputs.message.toUpperCase()}; } -``` - -```{note} - -We had to use an `InlineJavascriptRequirement` as our expression -contains a JavaScript call in `.toUpperCase()`. This means to tools -using the expression tool that JavaScript is a requirement. -``` diff --git a/src/conf.py b/src/conf.py index 7b3e2183..a17b63a9 100644 --- a/src/conf.py +++ b/src/conf.py @@ -39,7 +39,8 @@ extensions = [ 'myst_parser', 'sphinx.ext.graphviz', - 'sphinx_reredirects' + 'sphinx_reredirects', + 'cwl.sphinx.runcmd' ] # myst-parser settings @@ -72,13 +73,13 @@ # CONTRIBUTING.md is referenced in the footer, but not linked via Sphinx # aio.md is also referenced in one page, but not directly via Sphinx, hence the exclusions here. exclude_patterns = [ - 'Thumbs.db', - '.DS_Store', - '.git', + '**/Thumbs.db', + '**/.DS_Store', + '**/.git', '.idea', '.github', - '_build', - '_includes', + '**/_build', + '**/_includes', 'cwl', 'venv', 'README.md', diff --git a/src/introduction/prerequisites.md b/src/introduction/prerequisites.md index b93ad0b8..574611fb 100644 --- a/src/introduction/prerequisites.md +++ b/src/introduction/prerequisites.md @@ -61,7 +61,7 @@ for other ways to install `cwltool` with `apt` and `conda`. Let's use a simple workflow `true.cwl` with `cwltool`. -```{literalinclude} /_includes/cwl/simplest_cwl.cwl +```{literalinclude} /_includes/cwl/true.cwl :language: cwl :caption: "`true.cwl`" :name: true.cwl @@ -75,29 +75,16 @@ to pass `--validate` to the `cwltool` command: % TODO: maybe figure out a way to avoid /home/kinow/ etc. in the documentation % to avoid multiple user-names/directory-locations varying in the docs. -```{code-block} console +```{runcmd} cwltool --validate true.cwl :name: validating-truecwl-with-cwltool :caption: Validating `true.cwl` with `cwltool`. - -$ cwltool --validate true.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220406080846 -INFO Resolved 'true.cwl' to 'file:///tmp/true.cwl' -true.cwl is valid CWL. ``` You can run the CWL workflow now that you know it is valid: -```{code-block} console +```{runcmd} cwltool true.cwl :name: running-true.cwl-with-cwltool :caption: Running `true.cwl` with `cwltool`. - -$ cwltool true.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220406080846 -INFO Resolved 'true.cwl' to 'file:///tmp/true.cwl' -INFO [job true.cwl] /tmp/f8xlh1pl$ true -INFO [job true.cwl] completed success -{} -INFO Final process status is success ``` ### cwl-runner Python module @@ -126,56 +113,42 @@ Now you can validate and run your workflow with `cwl-runner` executable, which will invoke `cwltool`. You should have the same results and output as in the previous section. -```{code-block} console +```{runcmd} cwl-runner --validate true.cwl :name: validating-true.cwl-with-cwl-runner :caption: Validating `true.cwl` with `cwl-runner`. - -$ cwl-runner --validate true.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwl-runner 3.1.20220406080846 -INFO Resolved 'true.cwl' to 'file:///tmp/true.cwl' -true.cwl is valid CWL. ``` -```{code-block} console +```{runcmd} cwl-runner true.cwl :name: running-true.cwl-with-cwl-runner :caption: Running `true.cwl` with `cwl-runner`. - -$ cwl-runner true.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwl-runner 3.1.20220406080846 -INFO Resolved 'true.cwl' to 'file:///tmp/true.cwl' -INFO [job true.cwl] /tmp/7a2gf1nh$ true -INFO [job true.cwl] completed success -{} -INFO Final process status is success ``` Another way to execute `cwl-runner` is invoking the file directly. For that, -the first thing you need is to modify the `true.cwl` workflow and include -a special first line, a *shebang*: +the first thing you need to copy `true.cwl` workflow into a new file +`true_shebang.cwl` and include a special first line, a *shebang*: -```{literalinclude} /_includes/cwl/simplest_cwl_shebang.cwl +```{literalinclude} /_includes/cwl/true_shebang.cwl :language: cwl -:name: cwltool-with-a-shebang -:caption: "`cwltool` with a shebang" +:name: "true_shebang.cwl" +:caption: "`true_shebang.cwl`" ``` -Now, after you make the file `true.cwl` executable with `chmod u+x`, -you can execute it directly in the command-line and the program -specified in the shebang (`cwl-runner`) will be used to execute the -rest of the file. +Now you can make the file `true_shebang.cwl` executable with `chmod u+x`. ```{code-block} console -:name: making-true.cwl-executable-and-running-it -:caption: Making `true.cwl` executable and running it. +:name: making-true.cwl-executable +:caption: Making `true.cwl` executable. $ chmod u+x true.cwl -$ ./true.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwl-runner 3.1.20220406080846 -INFO Resolved './true.cwl' to 'file:///tmp/true.cwl' -INFO [job true.cwl] /tmp/jz7ups99$ true -INFO [job true.cwl] completed success -{} -INFO Final process status is success +``` + +And finally you can execute it directly in the command-line and the program +specified in the shebang (`cwl-runner`) will be used to execute the +rest of the file. + +```{runcmd} ./true_shebang.cwl +:name: running-true_shebang.cwl-with-a-shebang +:caption: Running `true_shebang.cwl` with a shebang. ``` ```{note} diff --git a/src/introduction/quick-start.md b/src/introduction/quick-start.md index f31610d5..60a6d965 100644 --- a/src/introduction/quick-start.md +++ b/src/introduction/quick-start.md @@ -55,66 +55,34 @@ The usage of the `cwltool` command-line executable is basically `cwltool [OPTIONS] [INPUTS_OBJECT]`. You can run the `hello_world.cwl` workflow without specifying any option: -```{code-block} console +```{runcmd} cwltool hello_world.cwl :name: running-hello_world.cwl-with-cwltool :caption: Running `hello_world.cwl` with `cwltool`. - -$ cwltool hello_world.cwl -INFO /tmp/venv/bin/cwltool 3.1.20220628170238 -INFO Resolved 'hello_world.cwl' to 'file:///tmp/hello_world.cwl' -INFO [job hello_world.cwl] /tmp/yn0e8xu6$ echo \ - 'Hello World' -Hello World -INFO [job hello_world.cwl] completed success -{} -INFO Final process status is success ``` Or you can override the default value of the input parameter `message`, similar to how you would change the argument of the `echo` base command: -```{code-block} console +```{runcmd} cwltool hello_world.cwl --message="Hola mundo" :name: running-hello_world.cwl-with-cwltool-passing-an-input-parameter :caption: Running `hello_world.cwl` with `cwltool` passing an input parameter. - -$ cwltool hello_world.cwl --message="Hola mundo" -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220406080846 -INFO Resolved '/tmp/hello_world.cwl' to 'file:///tmp/hello_world.cwl' -INFO [job hello_world.cwl] /tmp/ua5vt9hl$ echo \ - 'Hola mundo' -Hola mundo -INFO [job hello_world.cwl] completed success -{} -INFO Final process status is success ``` Another way of passing values to your workflow input parameters is via an *Inputs Object*. This is a file containing the input fields with the corresponding values. This file can be written in JSON or YAML. For example: -```{code-block} json +```{literalinclude} /_includes/cwl/hello_world-job.json +:language: json :name: hello_world-job.json :caption: "`hello_world-job.json`" -{ - "message": "こんにちは世界" -} ``` -

hello_world-job.json

You can use this Inputs Object file now to execute the “Hello World” workflow: -```{code-block} console +```{runcmd} cwltool hello_world.cwl hello_world-job.json :name: passing-an-inputs-object-file-to-cwltool :caption: Passing an Inputs Object file to `cwltool`. -$ cwltool hello_world.cwl hello_world-job.json -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220406080846 -INFO Resolved '/tmp/hello_world.cwl' to 'file:///tmp/hello_world.cwl' -INFO [job hello_world.cwl] /tmp/c5uchknw$ echo \ - こんにちは世界 -こんにちは世界 -INFO [job hello_world.cwl] completed success -{} -INFO Final process status is success ``` ```{note} diff --git a/src/topics/additional-arguments-and-parameters.md b/src/topics/additional-arguments-and-parameters.md index 1589e256..fb39ae06 100644 --- a/src/topics/additional-arguments-and-parameters.md +++ b/src/topics/additional-arguments-and-parameters.md @@ -10,13 +10,13 @@ directories in which they appear) may be read-only, so we need to instruct "javac" to write the class file to the designated output directory instead. -```{literalinclude} /_includes/cwl/08-arguments/arguments.cwl +```{literalinclude} /_includes/cwl/additional-arguments-and-parameters/arguments.cwl :language: cwl :caption: "`arguments.cwl`" :name: arguments.cwl ``` -```{literalinclude} /_includes/cwl/08-arguments/arguments-job.yml +```{literalinclude} /_includes/cwl/additional-arguments-and-parameters/arguments-job.yml :language: yaml :caption: "`arguments-job.yml`" ``` @@ -27,36 +27,10 @@ Next, create a sample Java file to use with the command-line tool. $ echo "public class Hello {}" > Hello.java ``` -And now invoke `cwl-runner` providing the tool description and the input object on the command line: +And now invoke `cwltool` providing the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner arguments.cwl arguments-job.yml -[job arguments.cwl] /tmp/tmpwYALo1$ docker \ - run \ - -i \ - --volume=/home/peter/work/common-workflow-language/v1.0/examples/Hello.java:/var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java:ro \ - --volume=/tmp/tmpwYALo1:/var/spool/cwl:rw \ - --volume=/tmp/tmpptIAJ8:/tmp:rw \ - --workdir=/var/spool/cwl \ - --read-only=true \ - --user=1001 \ - --rm \ - --env=TMPDIR=/tmp \ - --env=HOME=/var/spool/cwl \ - openjdk:9.0.1-11-slim \ - javac \ - -d \ - /var/spool/cwl \ - /var/lib/cwl/stg8939ac04-7443-4990-a518-1855b2322141/Hello.java -Final process status is success -{ - "classfile": { - "size": 416, - "location": "/home/example/Hello.class", - "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", - "class": "File" - } -} +```{runcmd} cwltool arguments.cwl arguments-job.yml +:working-directory: src/_includes/cwl/additional-arguments-and-parameters ``` Here we use the `arguments` field to add an additional argument to the diff --git a/src/topics/command-line-tool.md b/src/topics/command-line-tool.md index e70ffbb3..2977e3a1 100644 --- a/src/topics/command-line-tool.md +++ b/src/topics/command-line-tool.md @@ -47,26 +47,9 @@ digraph G { % to duplicate anchors in a page (e.g. single-html). % :name: echo.cwl -```{code-block} cwl +```{literalinclude} /_includes/cwl/echo.cwl +:language: cwl :caption: "`echo.cwl`" -cwlVersion: v1.2 -class: CommandLineTool - -baseCommand: echo - -stdout: output.txt - -inputs: - message: - type: string - inputBinding: {} -outputs: - out: - type: string - outputBinding: - glob: output.txt - loadContents: true - outputEval: $(self[0].contents) ``` ```{note} diff --git a/src/topics/creating-files-at-runtime.md b/src/topics/creating-files-at-runtime.md index 88a93717..ae6583c4 100644 --- a/src/topics/creating-files-at-runtime.md +++ b/src/topics/creating-files-at-runtime.md @@ -6,7 +6,7 @@ rather than the command line parameters, or need a small wrapper shell script. To generate such files we can use the `InitialWorkDirRequirement`. -```{literalinclude} /_includes/cwl/14-runtime/createfile.cwl +```{literalinclude} /_includes/cwl/creating-files-at-runtime/createfile.cwl :language: cwl :caption: "`createfile.cwl`" :name: createfile.cwl @@ -26,7 +26,7 @@ to be evaluated by the shell script instead of the CWL engine. To test the above CWL tool use this job to provide the input value `message`: -```{literalinclude} /_includes/cwl/14-runtime/echo-job.yml +```{literalinclude} /_includes/cwl/creating-files-at-runtime/echo-job.yml :language: yaml :caption: "`echo-job.yml`" :name: echo-job.yml @@ -51,26 +51,13 @@ which is YAML quoting syntax, and means that you are using a multiline string See the [YAML Guide](../topics/yaml-guide.md#maps) for more about the formatting. ``` -Now invoke `cwl-runner` with the tool description and the input object on the +Now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner createfile.cwl echo-job.yml -[job createfile.cwl] /private/tmp/docker_tmphrqxxcdl$ sh \ - example.sh > /private/tmp/docker_tmphrqxxcdl/output.txt -Could not collect memory usage, job ended before monitoring began. -[job createfile.cwl] completed success -{ - "example_out": { - "location": "file:///home/example/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$9045abe4bd04dd8ccfe50c6ff61820b784b64aa7", - "size": 25, - "path": "/home/example/output.txt" - } -} -Final process status is success -$ cat output.txt -Message is: Hello world! +```{runcmd} cwltool createfile.cwl echo-job.yml +:working-directory: src/_includes/cwl/creating-files-at-runtime/ +``` + +```{runcmd} cat output.txt +:working-directory: src/_includes/cwl/creating-files-at-runtime/ ``` diff --git a/src/topics/custom-types.md b/src/topics/custom-types.md index 3a4fbdcd..dd20d79a 100644 --- a/src/topics/custom-types.md +++ b/src/topics/custom-types.md @@ -9,22 +9,22 @@ the CWL description directly. The example below is a CWL description of the [biom convert format][biom] tool for converting a standard biom table file to hd5 format. -```{literalinclude} /_includes/cwl/19-custom-types/custom-types.cwl +```{literalinclude} /_includes/cwl/custom-types/custom-types.cwl :language: cwl :caption: "`custom-types.cwl`" :name: custom-types.cwl ``` -```{literalinclude} /_includes/cwl/19-custom-types/custom-types.yml +```{literalinclude} /_includes/cwl/custom-types/custom-types.yml :language: yaml :caption: "`custom-types.yml`" :name: custom-types.yml ``` -___Note:___ To follow the example below, you need to download the example input file, *rich_sparse_otu_table.biom*. The file is available from [https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/19-custom-types/rich_sparse_otu_table.biom](https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/19-custom-types/rich_sparse_otu_table.biom) and can be downloaded e.g. via `wget`: +___Note:___ To follow the example below, you need to download the example input file, *rich_sparse_otu_table.biom*. The file is available from [https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/custom-types/rich_sparse_otu_table.biom](https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/custom-types/rich_sparse_otu_table.biom) and can be downloaded e.g. via `wget`: -```bash -wget https://raw.githubusercontent.com/common-workflow-language/user_guide/main/_includes/cwl/19-custom-types/rich_sparse_otu_table.biom +```{code-block} console +$ wget https://raw.githubusercontent.com/common-workflow-language/user_guide/main/src/_includes/cwl/custom-types/rich_sparse_otu_table.biom ``` On line 29, in `inputs:table_type`, a list of allowable table options to be used in the @@ -52,7 +52,7 @@ tells the tool to create an OTU table in hd5 format. The contents of the YAML file describing the custom type are given below: -```{literalinclude} /_includes/cwl/19-custom-types/biom-convert-table.yaml +```{literalinclude} /_includes/cwl/custom-types/biom-convert-table.yaml :language: yaml :caption: "`biom-convert-table.yaml`" :name: biom-convert-table.yaml diff --git a/src/topics/environment-variables.md b/src/topics/environment-variables.md index b23df87b..a2228029 100644 --- a/src/topics/environment-variables.md +++ b/src/topics/environment-variables.md @@ -4,38 +4,24 @@ Tools run in a restricted environment and do not inherit most environment variables from the parent process. You can set environment variables for the tool using `EnvVarRequirement`. -```{literalinclude} /_includes/cwl/12-env/env.cwl +```{literalinclude} /_includes/cwl/environment-variables/env.cwl :language: cwl :caption: "`env.cwl`" :name: env.cwl ``` -```{literalinclude} /_includes/cwl/12-env/echo-job.yml +```{literalinclude} /_includes/cwl/environment-variables/echo-job.yml :language: yaml :caption: "`echo-job.yml`" ``` -Now invoke `cwl-runner` with the tool description and the input object on the +Now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner env.cwl echo-job.yml -[job env.cwl] /home/example$ env > /home/example/output.txt -[job env.cwl] completed success -{ - "example_out": { - "location": "file:///home/example/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$1ca16a840b14807b2fd3323022c476b06a150e2f", - "size": 94, - "path": "/home/example/output.txt" - } -} -Final process status is success -$ cat output.txt -HELLO=Hello world! -PATH=/bin:/usr/bin:/usr/local/bin -HOME=/home/example -TMPDIR=/tmp/tmp63Obpk +```{runcmd} cwltool env.cwl echo-job.yml +:working-directory: src/_includes/cwl/environment-variables/ +``` + +```{runcmd} cat output.txt +:working-directory: src/_includes/cwl/environment-variables/ ``` diff --git a/src/topics/expressions.md b/src/topics/expressions.md index 29cf6467..31b3ee9e 100644 --- a/src/topics/expressions.md +++ b/src/topics/expressions.md @@ -12,7 +12,7 @@ When manipulating file names, extensions, paths etc, consider whether one of the etc, could be used instead. See the [list of best practices](best-practices.md). -```{literalinclude} /_includes/cwl/13-expressions/expression.cwl +```{literalinclude} /_includes/cwl/expressions/expression.cwl :language: cwl :caption: "`expression.cwl`" :name: expression.cwl @@ -21,7 +21,7 @@ See the [list of best practices](best-practices.md). As this tool does not require any `inputs` we can run it with an (almost) empty job file: -```{literalinclude} /_includes/cwl/13-expressions/empty.yml +```{literalinclude} /_includes/cwl/expressions/empty.yml :language: yaml :caption: "`empty.yml`" :name: empty.cwl @@ -33,41 +33,14 @@ represented simply by a set of empty brackets. We can then run `expression.cwl`: -```{code-block} console +```{runcmd} cwltool expression.cwl empty.yml :name: running-expression.cwl :caption: Running `expression.cwl` +:working-directory: src/_includes/cwl/expressions/ +``` -$ cwl-runner expression.cwl empty.yml -[job expression.cwl] /home/example$ echo \ - -A \ - 2 \ - -B \ - baz \ - -C \ - 10 \ - 9 \ - 8 \ - 7 \ - 6 \ - 5 \ - 4 \ - 3 \ - 2 \ - 1 > /home/example/output.txt -[job expression.cwl] completed success -{ - "example_out": { - "location": "file:///home/example/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$a739a6ff72d660d32111265e508ed2fc91f01a7c", - "size": 36, - "path": "/home/example/output.txt" - } -} -Final process status is success -$ cat output.txt --A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1 +```{runcmd} cat output.txt +:working-directory: src/_includes/cwl/expressions/ ``` Note that requirements can be provided with the map syntax, as in the example above: diff --git a/src/topics/file-formats.md b/src/topics/file-formats.md index f09a6c48..e32440ed 100644 --- a/src/topics/file-formats.md +++ b/src/topics/file-formats.md @@ -17,7 +17,7 @@ document in greater detail, so don't worry about these for now. Note that for added value `cwltool` can do some basic reasoning based on file formats and warn you if there seem to be some obvious mismatches. -```{literalinclude} /_includes/cwl/16-file-formats/metadata_example.cwl +```{literalinclude} /_includes/cwl/file-formats/metadata_example.cwl :language: cwl :caption: "`metadata_example.cwl`" :name: metadata_example.cwl @@ -25,8 +25,8 @@ formats and warn you if there seem to be some obvious mismatches. The equivalent of this CWL description in command line format is: -```bash -wc -l /path/to/aligned_sequences.ext > output.txt +```{code-block} console +$ wc -l /path/to/aligned_sequences.ext > output.txt ``` ## Sample Parameter Files @@ -36,41 +36,24 @@ checking in working examples of parameter files for your tool. This allows others to quickly work with your tool, starting from a "known good" parameterization. -```{literalinclude} /_includes/cwl/16-file-formats/sample.yml +```{literalinclude} /_includes/cwl/file-formats/sample.yml :language: yaml :caption: "`sample.yml`" :name: sample.yml ``` -___Note:___ To follow the example below, you need to download the example input file, *file-formats.bam*. The file is available from [https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/16-file-formats/file-formats.bam -](https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/16-file-formats/file-formats.bam) and can be downloaded e.g. via `wget`: +___Note:___ To follow the example below, you need to download the example input file, *file-formats.bam*. The file is available from [https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam +](https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam) and can be downloaded e.g. via `wget`: -```bash -wget https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/16-file-formats/file-formats.bam +```{code-block} +$ wget https://github.com/common-workflow-language/user_guide/raw/main/_includes/cwl/file-formats/file-formats.bam ``` -Now invoke `cwl-runner` with the tool description and the input object on the +Now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwltool metadata_example.cwl sample.yml -/usr/local/bin/cwltool 1.0.20161114152756 -Resolved 'metadata_example.cwl' to 'file:///media/large_volume/testing/cwl_tutorial2/metadata_example.cwl' -[job metadata_example.cwl] /tmp/tmpNWyAd6$ /bin/sh \ - -c \ - 'wc' '-l' '/tmp/tmpBf6m9u/stge293ac74-3d42-45c9-b506-dd35ea3e6eea/file-formats.bam' > /tmp/tmpNWyAd6/output.txt -Final process status is success -{ - "report": { - "format": "http://edamontology.org/format_1964", - "checksum": "sha1$49dc5004959ba9f1d07b8c00da9c46dd802cbe79", - "basename": "output.txt", - "location": "file:///media/large_volume/testing/cwl_tutorial2/output.txt", - "path": "/media/large_volume/testing/cwl_tutorial2/output.txt", - "class": "File", - "size": 80 - } -} +```{runcmd} cwltool metadata_example.cwl sample.yml +:working-directory: src/_includes/cwl/file-formats/ ``` [IANA]: https://www.iana.org/assignments/media-types/media-types.xhtml diff --git a/src/topics/inputs.md b/src/topics/inputs.md index aba85daa..0ab20d05 100644 --- a/src/topics/inputs.md +++ b/src/topics/inputs.md @@ -13,20 +13,17 @@ special types *File*, *Directory* and *Any*. The following example demonstrates some input parameters with different types and appearing on the command line in different ways. -First, create a file called inp.cwl, containing the following: +First, create a file called `inp.cwl`, containing the following: - -```{literalinclude} /_includes/cwl/03-input/inp.cwl +```{literalinclude} /_includes/cwl/inputs/inp.cwl :language: cwl :caption: "`inp.cwl`" :name: inp.cwl ``` -Create a file called inp-job.yml: - -*inp-job.yml* +Create a file called `inp-job.yml`: -```{literalinclude} /_includes/cwl/03-input/inp-job.yml +```{literalinclude} /_includes/cwl/inputs/inp-job.yml :language: yaml :caption: "`inp-job.yml`" :name: inp-job.yml @@ -35,21 +32,18 @@ Create a file called inp-job.yml: Notice that "example_file", as a `File` type, must be provided as an object with the fields `class: File` and `path`. -Next, create a whale.txt using [touch] by typing `touch whale.txt` on the command line and then invoke `cwl-runner` with the tool description and the input object on the command line, using the command `cwl-runner inp.cwl inp-job.yml`. The following boxed text describes these two commands and the expected output from the command line: +Next, create a whale.txt using [touch] by typing `touch whale.txt` on the command line. ```{code-block} console $ touch whale.txt -$ cwl-runner inp.cwl inp-job.yml -[job inp.cwl] /tmp/tmpzrSnfX$ echo \ - -f \ - -i42 \ - --example-string \ - hello \ - --file=/tmp/tmpRBSHIG/stg979b6d24-d50a-47e3-9e9e-90097eed2cbc/whale.txt --f -i42 --example-string hello --file=/tmp/tmpRBSHIG/stg979b6d24-d50a-47e3-9e9e-90097eed2cbc/whale.txt -[job inp.cwl] completed success -{} -Final process status is success +``` + +Now invoke `cwltool` with the tool description and the input object on the command line, +using the command `cwltool inp.cwl inp-job.yml`. The following boxed text describes these +two commands and the expected output from the command line: + +```{runcmd} cwltool inp.cwl inp-job.yml +:working-directory: src/_includes/cwl/inputs ```` ```{tip} @@ -145,44 +139,26 @@ line. There are two ways to specify an array parameter. First is to provide that may appear in the array. Alternatively, brackets `[]` may be added after the type name to indicate that input parameter is array of that type. -```{literalinclude} /_includes/cwl/09-array-inputs/array-inputs.cwl +```{literalinclude} /_includes/cwl/inputs/array-inputs.cwl :language: cwl :caption: "`array-inputs.cwl`" :name: array-inputs.cwl ``` -```{literalinclude} /_includes/cwl/09-array-inputs/array-inputs-job.yml +```{literalinclude} /_includes/cwl/inputs/array-inputs-job.yml :language: yaml :caption: "`array-inputs-job.yml`" :name: array-inputs-job.yml ``` -Now invoke `cwl-runner` providing the tool description and the input object +Now invoke `cwltool` providing the tool description and the input object on the command line: +```{runcmd} cwltool array-inputs.cwl array-inputs-job.yml +:working-directory: src/_includes/cwl/inputs/ +``` + ```{code-block} console -$ cwl-runner array-inputs.cwl array-inputs-job.yml -[job array-inputs.cwl] /home/examples$ echo \ - -A \ - one \ - two \ - three \ - -B=four \ - -B=five \ - -B=six \ - -C=seven,eight,nine > /home/examples/output.txt -[job array-inputs.cwl] completed success -{ - "example_out": { - "location": "file:///home/examples/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$91038e29452bc77dcd21edef90a15075f3071540", - "size": 60, - "path": "/home/examples/output.txt" - } -} -Final process status is success $ cat output.txt -A one two three -B=four -B=five -B=six -C=seven,eight,nine ``` @@ -207,56 +183,35 @@ together (they are dependent) or several arguments that cannot be provided together (they are exclusive). You can use records and type unions to group parameters together to describe these two conditions. -```{literalinclude} /_includes/cwl/11-records/record.cwl +```{literalinclude} /_includes/cwl/inputs/record.cwl :language: cwl :caption: "`record.cwl`" :name: record.cwl ``` -```{literalinclude} /_includes/cwl/11-records/record-job1.yml +```{literalinclude} /_includes/cwl/inputs/record-job1.yml :language: yaml :caption: "`record-job1.yml`" :name: record-job1.yml ``` -```{code-block} console -$ cwl-runner record.cwl record-job1.yml -Workflow error, try again with --debug for more information: -Invalid job input record: -record-job1.yml:1:1: the `dependent_parameters` field is not valid because - missing required field `itemB` +```{runcmd} cwltool record.cwl record-job1.yml +:working-directory: src/_includes/cwl/inputs/ ``` In the first example, you can't provide `itemA` without also providing `itemB`. -```{literalinclude} /_includes/cwl/11-records/record-job2.yml +```{literalinclude} /_includes/cwl/inputs/record-job2.yml :language: yaml :caption: "`record-job2.yml`" :name: record-job2.yml ``` -```cwl -$ cwl-runner record.cwl record-job2.yml -record-job2.yml:6:3: invalid field `itemD`, expected one of: 'itemC' -[job record.cwl] /home/example$ echo \ - -A \ - one \ - -B \ - two \ - -C \ - three > /home/example/output.txt -[job record.cwl] completed success -{ - "example_out": { - "location": "file:///home/example/11-records/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$329fe3b598fed0dfd40f511522eaf386edb2d077", - "size": 23, - "path": "/home/example/output.txt" - } -} -Final process status is success +```{runcmd} cwltool record.cwl record-job2.yml +:working-directory: src/_includes/cwl/inputs +```` + +```{code-block} console $ cat output.txt -A one -B two -C three ``` @@ -264,33 +219,17 @@ $ cat output.txt In the second example, `itemC` and `itemD` are exclusive, so only `itemC` is added to the command line and `itemD` is ignored. -```{literalinclude} /_includes/cwl/11-records/record-job3.yml +```{literalinclude} /_includes/cwl/inputs/record-job3.yml :language: yaml :caption: "`record-job3.yml`" :name: record-job3.yml ``` +```{runcmd} cwltool record.cwl record-job3.yml +:working-directory: src/_includes/cwl/inputs +```` + ```{code-block} console -$ cwl-runner record.cwl record-job3.yml -[job record.cwl] /home/example$ echo \ - -A \ - one \ - -B \ - two \ - -D \ - four > /home/example/output.txt -[job record.cwl] completed success -{ - "example_out": { - "location": "file:///home/example/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$77f572b28e441240a5e30eb14f1d300bcc13a3b4", - "size": 22, - "path": "/home/example/output.txt" - } -} -Final process status is success $ cat output.txt -A one -B two -D four ``` diff --git a/src/topics/metadata-and-authorship.md b/src/topics/metadata-and-authorship.md index 9f101a49..24e63fc8 100644 --- a/src/topics/metadata-and-authorship.md +++ b/src/topics/metadata-and-authorship.md @@ -13,7 +13,7 @@ Otherwise, one must use full URLs: `format: http://edamontology.org/format_2572` For all developers, we recommend the following minimal metadata for your tool and workflows. This example includes metadata allowing others to cite your tool. -```{literalinclude} /_includes/cwl/17-metadata/metadata_example2.cwl +```{literalinclude} /_includes/cwl/metadata-and-authorship/metadata_example2.cwl :language: cwl :caption: "`metadata_example2.cwl`" :name: metadata_example2.cwl @@ -21,8 +21,8 @@ and workflows. This example includes metadata allowing others to cite your tool. The equivalent of this CWL description in command line format is: -```bash -wc -l /path/to/aligned_sequences.ext > output.txt +```{code-block} +$ wc -l /path/to/aligned_sequences.ext > output.txt ``` ## Extended Example @@ -32,7 +32,7 @@ with a much larger amount of metadata. This example includes EDAM ontology tags as keywords (allowing the grouping of related tools), hints at hardware requirements in order to use the tool, and a few more metadata fields. -```{literalinclude} /_includes/cwl/17-metadata/metadata_example3.cwl +```{literalinclude} /_includes/cwl/metadata-and-authorship/metadata_example3.cwl :language: cwl :caption: "`metadata_example3.cwl`" :name: metadata_example3.cwl diff --git a/src/topics/operations.md b/src/topics/operations.md index 5303dda9..19f8c817 100644 --- a/src/topics/operations.md +++ b/src/topics/operations.md @@ -7,71 +7,18 @@ but it does not provide enough information to be executed. You can create operations to visualize a workflow during development, before you are ready to submit the workflow to a CWL runner: -```{code-block} cwl +```{literalinclude} /_includes/cwl/operations/operations.cwl +:language: cwl :name: operations.cwl :caption: "`operations.cwl`" - -cwlVersion: v1.2 -class: Workflow - - -inputs: - message: string -outputs: [] - -steps: - echo: - run: echo.cwl - in: - message: message - out: [out] - # Here you know you want an operation that changes the case of - # the previous step, but you do not have an implementation yet. - uppercase: - run: - class: Operation - inputs: - message: string - outputs: - out: string - in: - message: - source: echo/out - out: [uppercase_message] ``` The `uppercase` step of the workflow is an operation. It can be used like use a command line tool or an expression. You can also plot it with the CWL Viewer or `cwltool`: -```{code-block} console -$ cwltool --print-dot operations.cwl -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220628170238 -INFO Resolved 'operations.cwl' to 'file:///tmp/operations.cwl' -digraph G { -bgcolor="#eeeeee"; -clusterrank=local; -labeljust=right; -labelloc=bottom; -"echo" [fillcolor=lightgoldenrodyellow, label=echo, shape=record, style=filled]; -"uppercase" [fillcolor=lightgoldenrodyellow, label=uppercase, shape=record, style=dashed]; -"echo" -> "uppercase"; -subgraph cluster_inputs { -label="Workflow Inputs"; -rank=same; -style=dashed; -"message" [fillcolor="#94DDF4", label=message, shape=record, style=filled]; -} - -"message" -> "echo"; -subgraph cluster_outputs { -label="Workflow Outputs"; -labelloc=b; -rank=same; -style=dashed; -} - -} +```{runcmd} cwltool --print-dot operations.cwl +:working-directory: src/_includes/cwl/operations/ ``` The output of the command above can be rendered with a Graphviz renderer. The following @@ -108,15 +55,10 @@ style=dashed; If you try running it with `cwltool` the command will fail, since `cwltool` does not have enough information to know how to execute it: -```{code-block} console +```{runcmd} cwltool operations.cwl --message Hello +:working-directory: src/_includes/cwl/operations/ :name: operations-output-error-cwltool :caption: "`cwltool` does not know how to run operations" - -$ cwltool operations.cwl --message Hello -INFO /home/kinow/Development/python/workspace/user_guide/venv/bin/cwltool 3.1.20220628170238 -INFO Resolved 'operations.cwl' to 'file:///tmp/operations.cwl' -ERROR Workflow error, try again with --debug for more information: -operations.cwl:19:7: Workflow has unrunnable abstract Operation ``` ```{note} diff --git a/src/topics/outputs.md b/src/topics/outputs.md index c2316789..accf5caf 100644 --- a/src/topics/outputs.md +++ b/src/topics/outputs.md @@ -23,13 +23,13 @@ Instead of a single string we can use an _array of strings_. The first element any subsequent elements are mandatory command line arguments ``` -```{literalinclude} /_includes/cwl/04-output/tar.cwl +```{literalinclude} /_includes/cwl/outputs/tar.cwl :language: cwl :caption: "`tar.cwl`" :name: tar.cwl ``` -```{literalinclude} /_includes/cwl/04-output/tar-job.yml +```{literalinclude} /_includes/cwl/outputs/tar-job.yml :language: yaml :caption: "`tar-job.yml`" :name: tar-job.yml @@ -42,25 +42,10 @@ $ touch hello.txt && tar -cvf hello.tar hello.txt hello.txt ``` -And now invoke `cwl-runner` with the tool description and the input object on the command line: +And now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner tar.cwl tar-job.yml -[job tar.cwl] /tmp/tmpqOeawQ$ tar \ - --extract --file \ - /tmp/tmpGDk8Y1/stg80bbad20-494d-47af-8075-dffc32df03a3/hello.tar -[job tar.cwl] completed success -{ - "example_out": { - "location": "file:///home/me/cwl/user_guide/hello.txt", - "basename": "hello.txt", - "class": "File", - "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", - "size": 0, - "path": "/home/me/cwl/user_guide/hello.txt" - } -} -Final process status is success +```{runcmd} cwltool tar.cwl tar-job.yml +:working-directory: src/_includes/cwl/outputs/ ``` The field `outputBinding` describes how to set the value of each @@ -83,77 +68,45 @@ To capture a tool's standard output stream, add the `stdout` field with the name of the file where the output stream should go. Then add `type: stdout` on the corresponding output parameter. -```{literalinclude} /_includes/cwl/05-stdout/stdout.cwl +```{literalinclude} /_includes/cwl/outputs/stdout.cwl :language: cwl :caption: "`stdout.cwl`" :name: stdout.cwl ``` -```{literalinclude} /_includes/cwl/05-stdout/echo-job.yml +```{literalinclude} /_includes/cwl/outputs/echo-job.yml :language: yaml :caption: "`echo-job.yml`" ``` -Now invoke `cwl-runner` providing the tool description and the input object +Now invoke `cwltool` providing the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner stdout.cwl echo-job.yml -[job stdout.cwl] /tmp/tmpE0gTz7$ echo \ - 'Hello world!' > /tmp/tmpE0gTz7/output.txt -[job stdout.cwl] completed success -{ - "example_out": { - "location": "file:///home/me/cwl/user_guide/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$47a013e660d408619d894b20806b1d5086aab03b", - "size": 13, - "path": "/home/me/cwl/user_guide/output.txt" - } -} -Final process status is success +```{runcmd} cwltool stdout.cwl echo-job.yml +:working-directory: src/_includes/cwl/outputs/ ``` ## Array Outputs You can also capture multiple output files into an array of files using `glob`. -```{literalinclude} /_includes/cwl/10-array-outputs/array-outputs.cwl +```{literalinclude} /_includes/cwl/outputs/array-outputs.cwl :language: cwl :caption: "`array-outputs.cwl`" :name: array-outputs.cwl ``` -```{literalinclude} /_includes/cwl/10-array-outputs/array-outputs-job.yml +```{literalinclude} /_includes/cwl/outputs/array-outputs-job.yml :language: yaml :caption: "`array-outputs-job.yml`" :name: array-outputs-job.yml ``` -Now invoke `cwl-runner` providing the tool description and the input object +Now invoke `cwltool` providing the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner array-outputs.cwl array-outputs-job.yml -[job 140190876078160] /home/example$ touch foo.txt bar.dat baz.txt -Final process status is success -{ - "output": [ - { - "size": 0, - "location": "foo.txt", - "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", - "class": "File" - }, - { - "size": 0, - "location": "baz.txt", - "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", - "class": "File" - } - ] -} +```{runcmd} cwltool array-outputs.cwl array-outputs-job.yml +:working-directory: src/_includes/cwl/outputs/ ``` As described in the [YAML Guide](yaml-guide.md#arrays), diff --git a/src/topics/parameter-references.md b/src/topics/parameter-references.md index 3f957890..c007b37f 100644 --- a/src/topics/parameter-references.md +++ b/src/topics/parameter-references.md @@ -10,40 +10,27 @@ this example, you will see how to reference the value of input parameters dynamically from other fields, which will allow us to then specify the name of the file to extract. -```{literalinclude} /_includes/cwl/06-params/tar-param.cwl +```{literalinclude} /_includes/cwl/parameter-references/tar-param.cwl :language: cwl :caption: "`tar-param.cwl`" :name: tar-param.cwl ``` -```{literalinclude} /_includes/cwl/06-params/tar-param-job.yml +```{literalinclude} /_includes/cwl/parameter-references/tar-param-job.yml :language: yaml :caption: "`tar-param-job.yml`" :name: tar-param-job.yml ``` -Create your input files and invoke `cwl-runner` with the tool description and the +Create your input files and invoke `cwltool` with the tool description and the input object on the command line: ```{code-block} console $ rm hello.tar || true && touch goodbye.txt && tar -cvf hello.tar goodbye.txt -$ cwl-runner tar-param.cwl tar-param-job.yml -[job tar-param.cwl] /tmp/tmpwH4ouT$ tar \ - --extract --file \ - /tmp/tmpREYiEt/stgd7764383-99c9-4848-af51-7c2d6e5527d9/hello.tar \ - goodbye.txt -[job tar-param.cwl] completed success -{ - "extracted_file": { - "location": "file:///home/me/cwl/user_guide/goodbye.txt", - "basename": "goodbye.txt", - "class": "File", - "checksum": "sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709", - "size": 0, - "path": "/home/me/cwl/user_guide/goodbye.txt" - } -} -Final process status is success +``` + +```{runcmd} cwltool tar-param.cwl tar-param-job.yml +:working-directory: src/_includes/cwl/parameter-references ``` Certain fields permit parameter references which are enclosed in `$(...)`. diff --git a/src/topics/specifying-software-requirements.md b/src/topics/specifying-software-requirements.md index db644c18..2215bad6 100644 --- a/src/topics/specifying-software-requirements.md +++ b/src/topics/specifying-software-requirements.md @@ -6,7 +6,7 @@ make it easier for others to use your descriptions, you can include a This may also help to avoid confusion about which version of a tool the description was written for. -```{literalinclude} /_includes/cwl/20-software-requirements/custom-types.cwl +```{literalinclude} /_includes/cwl/specifying-software-requirements/custom-types.cwl :language: cwl ``` diff --git a/src/topics/staging-input-files.md b/src/topics/staging-input-files.md index a48bcdcb..3f081c3a 100644 --- a/src/topics/staging-input-files.md +++ b/src/topics/staging-input-files.md @@ -6,30 +6,20 @@ write its output files alongside the input file in the same directory. You use In this example, we use a JavaScript expression to extract the base name of the input file from its leading directory path. -```{literalinclude} /_includes/cwl/15-staging/linkfile.cwl +```{literalinclude} /_includes/cwl/staging-input-files/linkfile.cwl :language: cwl :caption: "`linkfile.cwl`" :name: linkfile.cwl ``` -```{literalinclude} /_includes/cwl/15-staging/arguments-job.yml +```{literalinclude} /_includes/cwl/staging-input-files/arguments-job.yml :language: yaml :caption: "`arguments-job.yml`" ``` -Now invoke `cwl-runner` with the tool description and the input object on the +Now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner linkfile.cwl arguments-job.yml -[job 139928309171664] /home/example$ docker run -i --volume=/home/example/Hello.java:/var/lib/cwl/job557617295_examples/Hello.java:ro --volume=/home/example:/var/spool/cwl:rw --volume=/tmp/tmpmNbApw:/tmp:rw --workdir=/var/spool/cwl --read-only=true --net=none --user=1001 --rm --env=TMPDIR=/tmp openjdk:9.0.1-11-slim javac Hello.java -Final process status is success -{ -"classfile": { - "size": 416, - "location": "/home/example/Hello.class", - "checksum": "sha1$2f7ac33c1f3aac3f1fec7b936b6562422c85b38a", - "class": "File" - } -} +```{runcmd} cwltool linkfile.cwl arguments-job.yml +:working-directory: src/_includes/cwl/staging-input-files/ ``` diff --git a/src/topics/using-containers.md b/src/topics/using-containers.md index cc208a34..aff7d4ab 100644 --- a/src/topics/using-containers.md +++ b/src/topics/using-containers.md @@ -18,13 +18,13 @@ input files to reflect the location where they appear inside the container. This example runs a simple Node.js script inside a Docker container which will then print "Hello World" to the standard output. -```{literalinclude} /_includes/cwl/07-containers/docker.cwl +```{literalinclude} /_includes/cwl/using-containers/docker.cwl :language: cwl :caption: "`docker.cwl`" :name: docker.cwl ``` -```{literalinclude} /_includes/cwl/07-containers/docker-job.yml +```{literalinclude} /_includes/cwl/using-containers/docker-job.yml :language: yaml :caption: "`docker-job.yml`" :name: docker-job.yml @@ -49,41 +49,19 @@ the name of the container image (you can even specify the tag, which is good ide best practises when using containers for reproducible research). In this case we have used a container called `node:slim`. -Provide a "hello.js" and invoke `cwl-runner` providing the tool description and the +Provide a "hello.js" and invoke `cwltool` providing the tool description and the input object on the command line: ```{code-block} console $ echo "console.log(\"Hello World\");" > hello.js -$ cwl-runner docker.cwl docker-job.yml -[job docker.cwl] /tmp/tmpgugLND$ docker \ - run \ - -i \ - --volume=/tmp/tmpgugLND:/var/spool/cwl:rw \ - --volume=/tmp/tmpSs5JoN:/tmp:rw \ - --volume=/home/me/cwl/user_guide/hello.js:/var/lib/cwl/job369354770_examples/hello.js:ro \ - --workdir=/var/spool/cwl \ - --read-only=true \ - --user=1000 \ - --rm \ - --env=TMPDIR=/tmp \ - --env=HOME=/var/spool/cwl \ - node:slim \ - node \ - /var/lib/cwl/job369354770_examples/hello.js > /tmp/tmpgugLND/output.txt -[job docker.cwl] completed success -{ - "example_out": { - "location": "file:///home/me/cwl/user_guide/output.txt", - "basename": "output.txt", - "class": "File", - "checksum": "sha1$648a6a6ffffdaa0badb23b8baf90b6168dd16b3a", - "size": 12, - "path": "/home/me/cwl/user_guide/output.txt" - } -} -Final process status is success -$ cat output.txt -Hello World +``` + +```{runcmd} cwltool docker.cwl docker-job.yml +:working-directory: src/_includes/cwl/using-containers/ +``` + +```{runcmd} cat output.txt +:working-directory: src/_includes/cwl/using-containers/ ``` Notice the CWL runner has constructed a Docker command line to run the diff --git a/src/topics/workflows.md b/src/topics/workflows.md index a962a633..b3991f7f 100644 --- a/src/topics/workflows.md +++ b/src/topics/workflows.md @@ -167,19 +167,22 @@ of this user guide in more detail. This workflow extracts a java source file from a tar file and then compiles it. -```{literalinclude} /_includes/cwl/21-1st-workflow/1st-workflow.cwl +```{literalinclude} /_includes/cwl/workflows/1st-workflow.cwl :language: cwl :caption: "`1st-workflow.cwl`" :name: 1st-workflow.cwl ``` +% TODO: The link below is for a previous commit with the workflow above. Ideally, I think we should either use `cwltool` +% or Python to add the graph here. Or, maybe re-create the graph for the latest main version? + ```{admonition} Visualization of 1st-workflow.cwl -[![Visualization of 1st-workflow.cwl](https://view.commonwl.org/graph/svg/github.com/common-workflow-language/user_guide/blob/main/_includes/cwl/21-1st-workflow/1st-workflow.cwl)](https://view.commonwl.org/workflows/github.com/common-workflow-language/user_guide/blob/main/_includes/cwl/21-1st-workflow/1st-workflow.cwl) +[![Visualization of 1st-workflow.cwl](https://view.commonwl.org/graph/png/github.com/common-workflow-language/user_guide/blob/a29e7eae0006660946fc705a310b37a21a7e1edc/_includes/cwl/21-1st-workflow/1st-workflow.cwl)](https://view.commonwl.org/graph/png/github.com/common-workflow-language/user_guide/blob/a29e7eae0006660946fc705a310b37a21a7e1edc/_includes/cwl/21-1st-workflow/1st-workflow.cwl) ``` Use a YAML or a JSON object in a separate file to describe the input of a run: -```{literalinclude} /_includes/cwl/21-1st-workflow/1st-workflow-job.yml +```{literalinclude} /_includes/cwl/workflows/1st-workflow-job.yml :language: yaml :caption: "`1st-workflow-job.yml`" :name: 1st-workflow-job.yml @@ -192,36 +195,11 @@ $ echo "public class Hello {}" > Hello.java && tar -cvf hello.tar Hello.java Hello.java ``` -Now invoke `cwl-runner` with the tool description and the input object on the +Now invoke `cwltool` with the tool description and the input object on the command line: -```{code-block} console -$ cwl-runner 1st-workflow.cwl 1st-workflow-job.yml -INFO /usr/bin/cwl-runner 3.1.20220802125926 -INFO Resolved '1st-workflow.cwl' to 'file:///home/example/1st-workflow.cwl' -INFO [workflow ] start -INFO [workflow ] starting step untar -INFO [step untar] start -INFO [job untar] /tmp/rzlj4nsg$ tar --extract --file /tmp/1yikucse/stg4148c64f-2489-449c-bef7-12d059758b13/hello.tar Hello.java -INFO [job untar] completed success -INFO [step untar] completed success -INFO [workflow ] starting step compile -INFO [step compile] start -INFO [job compile] /tmp/r0kmwijd$ docker run -i --mount=type=bind,source=/tmp/r0kmwijd,target=/zurNsS --mount=type=bind,source=/tmp/bza6s5oh,target=/tmp --mount=type=bind,source=/tmp/rzlj4nsg/Hello.java,target=/var/lib/cwl/stg37272664-b5b2-4bbd-b254-1ef521ebbe3d/Hello.java,readonly --workdir=/zurNsS --read-only=true --user=1000:1000 --rm --cidfile=/tmp/4d8t2lj2/20220825125241-159808.cid --env=TMPDIR=/tmp --env=HOME=/zurNsS openjdk:9.0.1-11-slim javac -d /zurNsS /var/lib/cwl/stg37272664-b5b2-4bbd-b254-1ef521ebbe3d/Hello.java -INFO [job compile] Max memory used: 28MiB -INFO [job compile] completed success -INFO [step compile] completed success -INFO [workflow ] completed success -{ - "compiled_class": { - "location": "file:///home/example/Hello.class", - "basename": "Hello.class", - "class": "File", - "checksum": "sha1$fdb876b40ad9ebc7fee873212e02d5940588642e", - "size": 184, - "path": "/home/example/Hello.class" - } -} +```{runcmd} cwltool 1st-workflow.cwl 1st-workflow-job.yml +:working-directory: src/_includes/cwl/workflows/ ``` What's going on here? Let's break it down: @@ -276,7 +254,7 @@ instead the order is determined by the dependencies between steps (using another may run in parallel. The first step, `untar` runs `tar-param.cwl` (described previously in -[Parameter References](parameter-references.md). +[Parameter References](parameter-references.md)). This tool has two input parameters, `tarfile` and `extractfile` and one output parameter `extracted_file`. @@ -319,7 +297,7 @@ requirements: Here's an example workflow that uses our `1st-workflow.cwl` as a nested workflow: -```{literalinclude} /_includes/cwl/22-nested-workflows/nestedworkflows.cwl +```{literalinclude} /_includes/cwl/workflows/nestedworkflows.cwl :language: cwl :caption: "`nestedworkflows.cwl`" :name: nestedworkflows.cwl @@ -333,11 +311,11 @@ the `compile` step in orange; `compile` is another workflow, diagrammed on the right. In purple we see the fixed string `"Hello.java"` being supplied as the `name_of_file_to_extract`. -Visualization of nestedworkflows.cwl -Visualization of 1st-workflow.cwl ``` @@ -434,7 +412,7 @@ The most common reason a new user might want to use scatter is to perform the sa different samples. Let's start with a simple workflow that calls our first example (`hello_world.cwl`) and takes an array of strings as input to the workflow: -```{literalinclude} /_includes/cwl/23-scatter-workflow/scatter-workflow.cwl +```{literalinclude} /_includes/cwl/workflows/scatter-workflow.cwl :language: cwl :caption: "`scatter-workflow.cwl`" :name: scatter-workflow.cwl @@ -471,43 +449,18 @@ as well! Using the following input file: -```{literalinclude} /_includes/cwl/23-scatter-workflow/scatter-job.yml +```{literalinclude} /_includes/cwl/workflows/scatter-job.yml :language: yaml :caption: "`scatter-job.yml`" :name: scatter-job.yml ``` As a reminder, [`hello_world.cwl`](../introduction/quick-start.md) simply calls the command -`echo` on a message. If we invoke `cwl-runner scatter-workflow.cwl scatter-job.yml` on the +`echo` on a message. If we invoke `cwltool scatter-workflow.cwl scatter-job.yml` on the command line: -```{code-block} console -$ cwl-runner scatter-workflow.cwl scatter-job.yml -[workflow scatter-workflow.cwl] start -[step echo] start -[job echo] /tmp/tmp0hqmg400$ echo \ - 'Hello world!' -Hello world! -[job echo] completed success -[step echo] start -[job echo_2] /tmp/tmpu65_m1zw$ echo \ - 'Hola mundo!' -Hola mundo! -[job echo_2] completed success -[step echo] start -[job echo_3] /tmp/tmp5cs7a2wh$ echo \ - 'Bonjour le monde!' -Bonjour le monde! -[job echo_3] completed success -[step echo] start -[job echo_4] /tmp/tmp301wo7p8$ echo \ - 'Hallo welt!' -Hallo welt! -[job echo_4] completed success -[step echo] completed success -[workflow scatter-workflow.cwl] completed success -{} -Final process status is success +```{runcmd} cwltool scatter-workflow.cwl scatter-job.yml +:working-directory: src/_includes/cwl/workflows/ ``` You can see that the workflow calls echo multiple times on each element of our @@ -527,7 +480,7 @@ outputs: And add a second step that uses `wc` to count the characters in each file. See the tool below: -```{literalinclude} /_includes/cwl/23-scatter-workflow/wc-tool.cwl +```{literalinclude} /_includes/cwl/workflows/wc-tool.cwl :language: cwl :caption: "`wc-tool.cwl`" :name: wc-tool.cwl @@ -535,7 +488,7 @@ below: Now, how do we incorporate scatter? Remember the scatter field is under each step: -```{literalinclude} /_includes/cwl/23-scatter-workflow/scatter-two-steps.cwl +```{literalinclude} /_includes/cwl/workflows/scatter-two-steps.cwl :language: cwl :caption: "`scatter-two-steps.cwl`" :name: scatter-two-steps.cwl @@ -557,7 +510,7 @@ Ok, so how do we scatter on steps that can proceed independent of other samples? [Nested Workflows](#nested-workflows), that we can make an entire workflow a single step in another workflow! Convert our two-step workflow to a single step subworkflow: -```{literalinclude} /_includes/cwl/23-scatter-workflow/scatter-nested-workflow.cwl +```{literalinclude} /_includes/cwl/workflows/scatter-nested-workflow.cwl :language: cwl :caption: "`scatter-nested-workflow.cwl`" :name: scatter-nested-workflow.cwl @@ -617,7 +570,7 @@ class: Workflow cwlVersion: v1.2 ``` -The first step of the worklfow (step1) contains two input properties and will execute foo.cwl when the conditions are met. The new property `when` is where the condition validation takes place. In this case only when `in1` from the workflow contains a value `< 1` this step will be executed. +The first step of the workflow (step1) contains two input properties and will execute foo.cwl when the conditions are met. The new property `when` is where the condition validation takes place. In this case only when `in1` from the workflow contains a value `< 1` this step will be executed. ```cwl steps: @@ -631,7 +584,7 @@ steps: out: [out1] ``` -Using the following command `cwltool cond-wf-003.1.cwl --val 0` the value will pass the first conditional step and will therefor be executed and is shown in the log by `INFO [step step1] start` whereas the second step is skipped as indicated by `INFO [step step2] will be skipped`. +Using the following command `cwltool cond-wf-003.1.cwl --val 0` the value will pass the first conditional step and will therefore be executed and is shown in the log by `INFO [step step1] start` whereas the second step is skipped as indicated by `INFO [step step2] will be skipped`. ```{code-block} console INFO [workflow ] start @@ -691,8 +644,8 @@ WARNING Final process status is permanentFail % TODO % - Scatter % - ScatterMethod https://github.com/common-workflow-language/user_guide/issues/29 -% - Also in the **episode 23** of the current User Guide - https://www.commonwl.org/user_guide/23-scatter-workflow/index.html +% - Also in the **episode 23** of the current User Guide - https://www.commonwl.org/user_guide/workflows/index.html % - Subworkflows/nested workflows -% - Covered in the **episode 22** from the current User Guide - https://www.commonwl.org/user_guide/22-nested-workflows/index.html +% - Covered in the **episode 22** from the current User Guide - https://www.commonwl.org/user_guide/workflows/index.html % - Conditionals https://github.com/common-workflow-language/user_guide/issues/191 & https://github.com/common-workflow-language/user_guide/issues/188 % - Also in the **episode 24** of the current User Guide - https://www.commonwl.org/user_guide/24_conditional-workflow/index.html