WIP Migrate sections to use runcmd where possible

Author: kinow

Makefile

@@ -28,6 +28,7 @@ watch: clean
 			--ignore='**venv' \
 			--ignore='**.github' \
 			--ignore='*.egg-info' \
+			--ignore='**_includes/**/*.txt' \
 			--watch='cwl'
 
 ## unittest-examples	:

cwl/sphinx/runcmd.py

@@ -4,6 +4,8 @@
 import subprocess
 import sys
 
+from pathlib import Path
+
 from docutils.parsers.rst import directives
 from sphinx.directives import code
 
@@ -116,7 +118,12 @@ def run(self):
         # change to that working directory before running the desired command.
         # The working directory is omitted from the final output.
         # TODO: PATCHED
-        working_directory = self.options.get('working-directory', 'src/_includes/cwl/')
+        includes_dir = Path('src/_includes/cwl/')
+        if not includes_dir.exists():
+            # You can run Sphinx from the root directory, with `make watch`
+            # for instance, or from the src directory (RTD does that).
+            includes_dir = Path('_includes/cwl')
+        working_directory = self.options.get('working-directory', includes_dir)
         if working_directory == '':
             # subprocess default value, so that we can disable it if needed.
             working_directory = None

src/_includes/cwl/05-stdout/echo-job.yml

@@ -1,69 +1,64 @@
-# Section 2
-- doc: Test for section 2
-  job: 02-1st-example/echo-job.yml
-  tool: hello_world.cwl
-  should_fail: false
-  output: {}
-
-# Section 3
-- doc: Test for section 3
-  job: 03-input/inp-job.yml
-  tool: 03-input/inp.cwl
-  should_fail: false
-  output: {}
-
-# Section 4
-- doc: Test for section 4
-  job: 04-output/tar-job.yml
-  tool: 04-output/tar.cwl
+# Expressions
+- doc: Test for Expressions section
+  job: expressions/empty.yml
+  tool: expressions/expression.cwl
   output:
     example_out:
       class: File
-      checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709
-      basename: hello.txt
+      checksum: sha1$a739a6ff72d660d32111265e508ed2fc91f01a7c
+      basename: output.txt
       location: Any
-      size: 0
+      size: 36
 
-# Section 5
-- doc: Test for section 5
-  job: 05-stdout/echo-job.yml
-  tool: 05-stdout/stdout.cwl
+# Inputs
+- doc: Test for Essential Inputs section
+  job: inputs/inp-job.yml
+  tool: inputs/inp.cwl
+  should_fail: false
+  output: {}
+
+- doc: Test for Array Inputs section
+  job: inputs/array-inputs-job.yml
+  tool: inputs/array-inputs.cwl
   output:
     example_out:
       class: File
-      checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b
+      checksum: sha1$91038e29452bc77dcd21edef90a15075f3071540
       basename: output.txt
       location: Any
-      size: 13
+      size: 60
+
+- doc: Test for Include and Exclusive Inputs section (1st example)
+  job: inputs/record-job1.yml
+  tool: inputs/record.cwl
+  should_fail: true
 
-# Section 6
-- doc: Test for section 6
-  job: 06-params/tar-param-job.yml
-  tool: 06-params/tar-param.cwl
+- doc: Test for Include and Exclusive Inputs section (2nd example)
+  job: inputs/record-job2.yml
+  tool: inputs/record.cwl
   output:
-    extracted_file:
+    example_out:
       class: File
-      checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709
-      basename: goodbye.txt
+      checksum: sha1$329fe3b598fed0dfd40f511522eaf386edb2d077
+      basename: output.txt
       location: Any
-      size: 0
+      size: 23
 
-# Section 7
-- doc: Test for section 7
-  job: 07-containers/docker-job.yml
-  tool: 07-containers/docker.cwl
+- doc: Test for Include and Exclusive Inputs section (3rd example)
+  job: inputs/record-job3.yml
+  tool: inputs/record.cwl
   output:
     example_out:
       class: File
-      checksum: sha1$648a6a6ffffdaa0badb23b8baf90b6168dd16b3a
+      checksum: sha1$77f572b28e441240a5e30eb14f1d300bcc13a3b4
       basename: output.txt
       location: Any
-      size: 12
+      size: 22
 
-# Section 8
-- doc: Test for section 8
-  job: 08-arguments/arguments-job.yml
-  tool: 08-arguments/arguments.cwl
+# Additional Arguments and Parameters
+- doc: Test for Additional Arguments and Parameters section
+  job: additional-arguments-and-parameters/arguments-job.yml
+  tool: additional-arguments-and-parameters/arguments.cwl
   output:
     classfile:
       class: File
@@ -72,22 +67,44 @@
       location: Any
       size: 184
 
-# Section 9
-- doc: Test for section 9
-  job: 09-array-inputs/array-inputs-job.yml
-  tool: 09-array-inputs/array-inputs.cwl
+# Parameter References
+- doc: Test for Parameter References section
+  job: parameter-references/tar-param-job.yml
+  tool: parameter-references/tar-param.cwl
+  output:
+    extracted_file:
+      class: File
+      checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709
+      basename: goodbye.txt
+      location: Any
+      size: 0
+
+# Outputs
+- doc: Test for Outputs section
+  job: outputs/tar-job.yml
+  tool: outputs/tar.cwl
   output:
     example_out:
       class: File
-      checksum: sha1$91038e29452bc77dcd21edef90a15075f3071540
+      checksum: sha1$da39a3ee5e6b4b0d3255bfef95601890afd80709
+      basename: hello.txt
+      location: Any
+      size: 0
+
+- doc: Test for Outputs, Capturing Standard Output section
+  job: outputs/echo-job.yml
+  tool: outputs/stdout.cwl
+  output:
+    example_out:
+      class: File
+      checksum: sha1$47a013e660d408619d894b20806b1d5086aab03b
       basename: output.txt
       location: Any
-      size: 60
+      size: 13
 
-# Section 10
-- doc: Test for section 10
-  job: 10-array-outputs/array-outputs-job.yml
-  tool: 10-array-outputs/array-outputs.cwl
+- doc: Test for Outputs, Array Outputs section
+  job: outputs/array-outputs-job.yml
+  tool: outputs/array-outputs.cwl
   output:
     output:
       - class: File
@@ -101,33 +118,39 @@
         location: Any
         size: 0
 
-# Section 11 depends on side-effects
-- doc: Test for section 11 (1st example)
-  job: 11-records/record-job1.yml
-  tool: 11-records/record.cwl
-  should_fail: true
-
-- doc: Test for section 11 (2nd example)
-  job: 11-records/record-job2.yml
-  tool: 11-records/record.cwl
+# Workflows
+- doc: Test for Workflows section
+  job: workflows/1st-workflow-job.yml
+  tool: workflows/1st-workflow.cwl
   output:
-    example_out:
+    compiled_class:
       class: File
-      checksum: sha1$329fe3b598fed0dfd40f511522eaf386edb2d077
-      basename: output.txt
+      checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd
+      basename: Hello.class
       location: Any
-      size: 23
+      size: 419
 
-- doc: Test for section 11 (3rd example)
-  job: 11-records/record-job3.yml
-  tool: 11-records/record.cwl
+# TODO: The old section 23-scatter-workflow had workflows, but no conformance tests?
+#       Maybe add them here, they are now in workflows:
+#       - hello_world.cwl
+#       - hello_world_to_stdout.cwl
+#       - scatter-job.cwl
+#       - scatter-nested-workflow.cwl
+#       - scatter-two-steps.cwl
+#       - scatter-workflow.cwl
+#       - wc-tool.cwl
+
+# Section 7
+- doc: Test for section 7
+  job: 07-containers/docker-job.yml
+  tool: 07-containers/docker.cwl
   output:
     example_out:
       class: File
-      checksum: sha1$77f572b28e441240a5e30eb14f1d300bcc13a3b4
+      checksum: sha1$648a6a6ffffdaa0badb23b8baf90b6168dd16b3a
       basename: output.txt
       location: Any
-      size: 22
+      size: 12
 
 # Section 12
 - doc: Test for section 12
@@ -139,18 +162,6 @@
       basename: output.txt
       location: Any
 
-# Section 13
-- doc: Test for section 13
-  job: 13-expressions/empty.yml
-  tool: 13-expressions/expression.cwl
-  output:
-    example_out:
-      class: File
-      checksum: sha1$a739a6ff72d660d32111265e508ed2fc91f01a7c
-      basename: output.txt
-      location: Any
-      size: 36
-
 # Section 14
 - doc: Test for section 14
   job: 14-runtime/echo-job.yml
@@ -222,26 +233,14 @@
 # See: Issue #48
 # - doc: Test for section 20
 #   job: 20-software-requirements/custom-types.yml
-#   tool: 20-software-requirements/costom-types.cwl
+#   tool: 20-software-requirements/custom-types.cwl
 #   output:
 #     i5Annotations:
 #       class: File
 #       basename: test_proteins.i5_annotations
 #       location: Any
 #       size: Any # To be fixed
 
-# Section 21
-- doc: Test for section 21
-  job: 21-1st-workflow/1st-workflow-job.yml
-  tool: 21-1st-workflow/1st-workflow.cwl
-  output:
-    compiled_class:
-      class: File
-      checksum: sha1$39e3219327347c05aa3e82236f83aa6d77fe6bfd
-      basename: Hello.class
-      location: Any
-      size: 419
-
 # Section 22
 - doc: Test for section 22
   tool: 22-nested-workflows/nestedworkflows.cwl

src/_includes/cwl/additional-arguments-and-parameters/Hello.class

@@ -0,0 +1 @@
+-A 2 -B baz -C 10 9 8 7 6 5 4 3 2 1

src/_includes/cwl/08-arguments/Hello.java renamed to src/_includes/cwl/additional-arguments-and-parameters/Hello.java

@@ -0,0 +1,3 @@
+{
+  "message": "こんにちは世界"
+}

src/_includes/cwl/08-arguments/arguments-job.yml renamed to src/_includes/cwl/additional-arguments-and-parameters/arguments-job.yml

@@ -0,0 +1 @@
+-A one -B two -D four

src/_includes/cwl/08-arguments/arguments.cwl renamed to src/_includes/cwl/additional-arguments-and-parameters/arguments.cwl

@@ -0,0 +1 @@
+Hello world!

src/_includes/cwl/conformance-test.yml

@@ -0,0 +1,13 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: CommandLineTool
+baseCommand: echo
+inputs:
+  message:
+    type: string
+    inputBinding:
+      position: 1
+outputs: 
+  echo_out:
+    type: stdout

src/_includes/cwl/13-expressions/empty.yml renamed to src/_includes/cwl/expressions/empty.yml

@@ -0,0 +1,5 @@
+message_array: 
+  - Hello world!
+  - Hola mundo!
+  - Bonjour le monde!
+  - Hallo welt!

src/_includes/cwl/13-expressions/expression.cwl renamed to src/_includes/cwl/expressions/expression.cwl

@@ -0,0 +1,35 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+
+requirements:
+ ScatterFeatureRequirement: {}
+ SubworkflowFeatureRequirement: {}
+
+inputs:
+  message_array: string[] 
+
+steps:
+  subworkflow:
+    run: 
+      class: Workflow
+      inputs: 
+        message: string
+      outputs: []
+      steps:
+        echo:
+          run: hello_world_to_stdout.cwl
+          in:
+            message: message
+          out: [echo_out]
+        wc:
+          run: wc-tool.cwl
+          in:
+            input_file: echo/echo_out
+          out: []
+    scatter: message
+    in: 
+      message: message_array
+    out: []
+outputs: []

src/_includes/cwl/expressions/output.txt

@@ -0,0 +1,26 @@
+#!/usr/bin/env cwl-runner
+
+cwlVersion: v1.0
+class: Workflow
+
+requirements:
+ ScatterFeatureRequirement: {}
+
+inputs:
+  message_array: string[] 
+
+steps:
+  echo:
+    run: hello_world_to_stdout.cwl
+    scatter: message
+    in:
+      message: message_array
+    out: [echo_out]
+  wc:
+    run: wc-tool.cwl
+    scatter: input_file
+    in:
+      input_file: echo/echo_out
+    out: []
+
+outputs: []