support podman even when not installed as "docker" alias

Author: mr-c

cwltool/builder.py

@@ -17,14 +17,15 @@
 from cwl_utils import expression
 from cwl_utils.file_formats import check_format
 from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names, Schema, make_avsc_object
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine
 from schema_salad.utils import convert_to_dict, json_dumps
 from schema_salad.validate import validate
 from typing_extensions import TYPE_CHECKING, Type  # pylint: disable=unused-import
 
+from ruamel.yaml.comments import CommentedMap
+
 from .errors import WorkflowException
 from .loghandler import _logger
 from .mutation import MutationManager

cwltool/command_line_tool.py

@@ -30,7 +30,6 @@
 )
 
 import shellescape
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Schema
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import file_uri, uri_file_path
@@ -39,14 +38,16 @@
 from schema_salad.validate import validate_ex
 from typing_extensions import TYPE_CHECKING, Type
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .builder import (
     INPUT_OBJ_VOCAB,
     Builder,
     content_limit_respected_read_bytes,
     substitute,
 )
 from .context import LoadingContext, RuntimeContext, getdefault
-from .docker import DockerCommandLineJob
+from .docker import DockerCommandLineJob, PodmanCommandLineJob
 from .errors import UnsupportedRequirement, WorkflowException
 from .flatten import flatten
 from .job import CommandLineJob, JobBase
@@ -460,6 +461,8 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]:
                         raise UnsupportedRequirement(
                             "Both Docker and MPI have been hinted - don't know what to do"
                         )
+            if runtimeContext.podman:
+                return PodmanCommandLineJob
             return DockerCommandLineJob
         if dockerRequired:
             raise UnsupportedRequirement(

cwltool/context.py

@@ -17,13 +17,14 @@
     Union,
 )
 
-# move to a regular typing import when Python 3.3-3.6 is no longer supported
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.avro.schema import Names
 from schema_salad.ref_resolver import Loader
 from schema_salad.utils import FetcherCallableType
 from typing_extensions import TYPE_CHECKING
 
+# move to a regular typing import when Python 3.3-3.6 is no longer supported
+from ruamel.yaml.comments import CommentedMap
+
 from .builder import Builder
 from .mpi import MpiConfig
 from .mutation import MutationManager

cwltool/cwlrdf.py

@@ -4,10 +4,11 @@
 
 from rdflib import Graph
 from rdflib.query import ResultRow
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.jsonld_context import makerdf
 from schema_salad.utils import ContextType
 
+from ruamel.yaml.comments import CommentedMap
+
 from .cwlviewer import CWLViewer
 from .process import Process
 

cwltool/docker.py

@@ -90,9 +90,10 @@ def __init__(
     ) -> None:
         """Initialize a command line builder using the Docker software container engine."""
         super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+        self.docker_exec = "docker"
 
-    @staticmethod
     def get_image(
+        self,
         docker_requirement: Dict[str, str],
         pull_image: bool,
         force_pull: bool,
@@ -117,7 +118,7 @@ def get_image(
 
         for line in (
             subprocess.check_output(  # nosec
-                ["docker", "images", "--no-trunc", "--all"]
+                [self.docker_exec, "images", "--no-trunc", "--all"]
             )
             .decode("utf-8")
             .splitlines()
@@ -151,7 +152,7 @@ def get_image(
         if (force_pull or not found) and pull_image:
             cmd = []  # type: List[str]
             if "dockerPull" in docker_requirement:
-                cmd = ["docker", "pull", str(docker_requirement["dockerPull"])]
+                cmd = [self.docker_exec, "pull", str(docker_requirement["dockerPull"])]
                 _logger.info(str(cmd))
                 subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
                 found = True
@@ -160,7 +161,7 @@ def get_image(
                 with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile:
                     dfile.write(docker_requirement["dockerFile"])
                 cmd = [
-                    "docker",
+                    self.docker_exec,
                     "build",
                     "--tag=%s" % str(docker_requirement["dockerImageId"]),
                     dockerfile_dir,
@@ -169,7 +170,7 @@ def get_image(
                 subprocess.check_call(cmd, stdout=sys.stderr)  # nosec
                 found = True
             elif "dockerLoad" in docker_requirement:
-                cmd = ["docker", "load"]
+                cmd = [self.docker_exec, "load"]
                 _logger.info(str(cmd))
                 if os.path.exists(docker_requirement["dockerLoad"]):
                     _logger.info(
@@ -203,7 +204,7 @@ def get_image(
                 found = True
             elif "dockerImport" in docker_requirement:
                 cmd = [
-                    "docker",
+                    self.docker_exec,
                     "import",
                     str(docker_requirement["dockerImport"]),
                     str(docker_requirement["dockerImageId"]),
@@ -225,8 +226,8 @@ def get_from_requirements(
         force_pull: bool,
         tmp_outdir_prefix: str,
     ) -> Optional[str]:
-        if not shutil.which("docker"):
-            raise WorkflowException("docker executable is not available")
+        if not shutil.which(self.docker_exec):
+            raise WorkflowException(f"{self.docker_exec} executable is not available")
 
         if self.get_image(
             cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix
@@ -341,10 +342,10 @@ def create_runtime(
                     runtime = [user_space_docker_cmd, "--quiet", "run", "--nobanner"]
             else:
                 runtime = [user_space_docker_cmd, "run"]
-        elif runtimeContext.podman:
-            runtime = ["podman", "run", "-i", "--userns=keep-id"]
         else:
-            runtime = ["docker", "run", "-i"]
+            runtime = [self.docker_exec, "run", "-i"]
+        if runtimeContext.podman:
+            runtime.append("--userns=keep-id")
         self.append_volume(
             runtime, os.path.realpath(self.outdir), self.builder.outdir, writable=True
         )
@@ -460,3 +461,20 @@ def create_runtime(
                 )
 
         return runtime, cidfile_path
+
+
+class PodmanCommandLineJob(DockerCommandLineJob):
+    """Runs a CommandLineJob in a software container using the podman engine."""
+
+    def __init__(
+        self,
+        builder: Builder,
+        joborder: CWLObjectType,
+        make_path_mapper: Callable[..., PathMapper],
+        requirements: List[CWLObjectType],
+        hints: List[CWLObjectType],
+        name: str,
+    ) -> None:
+        """Initialize a command line builder using the Podman software container engine."""
+        super().__init__(builder, joborder, make_path_mapper, requirements, hints, name)
+        self.docker_exec = "podman"

cwltool/load_tool.py

@@ -21,7 +21,6 @@
 )
 
 from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri
 from schema_salad.schema import validate_doc
@@ -34,6 +33,8 @@
     json_dumps,
 )
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from . import CWL_CONTENT_TYPES, process, update
 from .context import LoadingContext
 from .errors import GraphTargetMissingException

cwltool/main.py

@@ -36,14 +36,15 @@
 import argcomplete
 import coloredlogs
 import pkg_resources  # part of setuptools
-import ruamel.yaml
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
-from ruamel.yaml.main import YAML
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader, file_uri, uri_file_path
 from schema_salad.sourceline import cmap, strip_dup_lineno
 from schema_salad.utils import ContextType, FetcherCallableType, json_dumps, yaml_no_ts
 
+import ruamel.yaml
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+from ruamel.yaml.main import YAML
+
 from . import CWL_CONTENT_TYPES, workflow
 from .argparser import arg_parser, generate_parser, get_default_args
 from .context import LoadingContext, RuntimeContext, getdefault

cwltool/pack.py

@@ -14,10 +14,11 @@
     cast,
 )
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.ref_resolver import Loader, SubLoader
 from schema_salad.utils import ResolveType
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .context import LoadingContext
 from .load_tool import fetch_document, resolve_and_validate_document
 from .process import shortname, uniquename

cwltool/process.py

@@ -35,7 +35,6 @@
 from mypy_extensions import mypyc_attr
 from pkg_resources import resource_stream
 from rdflib import Graph
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import (
     Names,
     Schema,
@@ -50,6 +49,8 @@
 from schema_salad.validate import avro_type_name, validate_ex
 from typing_extensions import TYPE_CHECKING
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .builder import INPUT_OBJ_VOCAB, Builder
 from .context import LoadingContext, RuntimeContext, getdefault
 from .errors import UnsupportedRequirement, WorkflowException

cwltool/procgenerator.py

@@ -1,10 +1,11 @@
 import copy
 from typing import Dict, Optional, Tuple, cast
 
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import indent
 
+from ruamel.yaml.comments import CommentedMap
+
 from .context import LoadingContext, RuntimeContext
 from .errors import WorkflowException
 from .load_tool import load_tool

cwltool/update.py

@@ -11,11 +11,12 @@
     cast,
 )
 
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 from schema_salad.ref_resolver import Loader
 from schema_salad.sourceline import SourceLine
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .loghandler import _logger
 from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field
 

cwltool/validate_js.py

@@ -19,7 +19,6 @@
 from cwl_utils.expression import scanner as scan_expression
 from cwl_utils.sandboxjs import code_fragment_to_js, exec_js_process
 from pkg_resources import resource_stream
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import (
     ArraySchema,
     EnumSchema,
@@ -31,6 +30,8 @@
 from schema_salad.utils import json_dumps
 from schema_salad.validate import validate_ex
 
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
+
 from .errors import WorkflowException
 from .loghandler import _logger
 

cwltool/workflow.py

@@ -17,10 +17,11 @@
 )
 from uuid import UUID
 
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.exceptions import ValidationException
 from schema_salad.sourceline import SourceLine, indent
 
+from ruamel.yaml.comments import CommentedMap
+
 from . import command_line_tool, context, procgenerator
 from .checker import circular_dependency_checker, loop_checker, static_checker
 from .context import LoadingContext, RuntimeContext, getdefault

tests/test_anon_types.py

@@ -1,11 +1,11 @@
 from typing import cast
 
 import pytest
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.sourceline import cmap
 
 from cwltool.command_line_tool import CommandLineTool
 from cwltool.context import LoadingContext
+from ruamel.yaml.comments import CommentedMap
 
 snippet = cast(
     CommentedMap,

tests/test_examples.py

@@ -15,7 +15,6 @@
 import pytest
 from cwl_utils.errors import JavascriptException
 from cwl_utils.sandboxjs import param_re
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.exceptions import ValidationException
 
 import cwltool.checker
@@ -29,6 +28,7 @@
 from cwltool.main import main
 from cwltool.process import CWL_IANA
 from cwltool.utils import CWLObjectType, dedup
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from .util import get_data, get_main_output, needs_docker, working_directory
 

tests/test_mpi.py

@@ -8,7 +8,6 @@
 
 import pkg_resources
 import pytest
-from ruamel.yaml.comments import CommentedMap, CommentedSeq
 from schema_salad.avro.schema import Names
 from schema_salad.utils import yaml_no_ts
 
@@ -19,6 +18,7 @@
 from cwltool.context import LoadingContext, RuntimeContext
 from cwltool.main import main
 from cwltool.mpi import MpiConfig, MPIRequirementName
+from ruamel.yaml.comments import CommentedMap, CommentedSeq
 
 from .util import get_data, working_directory
 

tests/test_path_checks.py

@@ -4,7 +4,6 @@
 from typing import IO, Any, List, cast
 
 import pytest
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.sourceline import cmap
 
 from cwltool.command_line_tool import CommandLineTool
@@ -13,6 +12,7 @@
 from cwltool.stdfsaccess import StdFsAccess
 from cwltool.update import INTERNAL_VERSION
 from cwltool.utils import CWLObjectType
+from ruamel.yaml.comments import CommentedMap
 
 from .util import needs_docker
 

tests/test_streaming.py

@@ -4,7 +4,6 @@
 from typing import cast
 
 import pytest
-from ruamel.yaml.comments import CommentedMap
 from schema_salad.sourceline import cmap
 
 from cwltool.command_line_tool import CommandLineTool
@@ -13,6 +12,7 @@
 from cwltool.job import JobBase
 from cwltool.update import INTERNAL_VERSION, ORIGINAL_CWLVERSION
 from cwltool.utils import CWLObjectType
+from ruamel.yaml.comments import CommentedMap
 
 from .util import get_data