diff --git a/.github/workflows/ci-tests.yml b/.github/workflows/ci-tests.yml index 7de5ad329..2095b53b6 100644 --- a/.github/workflows/ci-tests.yml +++ b/.github/workflows/ci-tests.yml @@ -32,7 +32,7 @@ jobs: strategy: matrix: py-ver-major: [3] - py-ver-minor: [8, 9, 10, 11, 12, 13] + py-ver-minor: [9, 10, 11, 12, 13] step: [lint, unit, bandit, mypy] env: diff --git a/Makefile b/Makefile index 1b08f4290..5b9dd214e 100644 --- a/Makefile +++ b/Makefile @@ -24,7 +24,7 @@ MODULE=cwltool # `SHELL=bash` doesn't work for some, so don't use BASH-isms like # `[[` conditional expressions. -PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py) setup.py +PYSOURCES=$(wildcard ${MODULE}/**.py cwltool/cwlprov/*.py tests/*.py tests/cwl-conformance/*.py) setup.py DEVPKGS=diff_cover pylint pep257 pydocstyle 'tox<4' tox-pyenv auto-walrus \ isort wheel autoflake pyupgrade bandit -rlint-requirements.txt\ -rtest-requirements.txt -rmypy-requirements.txt -rdocs/requirements.txt @@ -190,7 +190,7 @@ shellcheck: FORCE cwltool-in-docker.sh pyupgrade: $(PYSOURCES) - pyupgrade --exit-zero-even-if-changed --py38-plus $^ + pyupgrade --exit-zero-even-if-changed --py39-plus $^ auto-walrus $^ release-test: FORCE diff --git a/cwltool/argparser.py b/cwltool/argparser.py index efced5386..7b3125d94 100644 --- a/cwltool/argparser.py +++ b/cwltool/argparser.py @@ -3,19 +3,8 @@ import argparse import os import urllib -from typing import ( - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Type, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sequence +from typing import Any, Callable, Optional, Union, cast from .loghandler import _logger from .process import Process, shortname @@ -718,7 +707,7 @@ def arg_parser() -> argparse.ArgumentParser: return parser -def get_default_args() -> Dict[str, Any]: +def get_default_args() -> dict[str, Any]: """Get default values of cwltool's command line options.""" ap = arg_parser() args = ap.parse_args([]) @@ -732,7 +721,7 @@ class FSAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -770,7 +759,7 @@ class FSAppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, @@ -827,7 +816,7 @@ class AppendAction(argparse.Action): def __init__( self, - option_strings: List[str], + option_strings: list[str], dest: str, nargs: Any = None, **kwargs: Any, @@ -859,7 +848,7 @@ def add_argument( toolparser: argparse.ArgumentParser, name: str, inptype: Any, - records: List[str], + records: list[str], description: str = "", default: Any = None, input_required: bool = True, @@ -888,9 +877,9 @@ def add_argument( return None ahelp = description.replace("%", "%%") - action: Optional[Union[Type[argparse.Action], str]] = None + action: Optional[Union[type[argparse.Action], str]] = None atype: Optional[Any] = None - typekw: Dict[str, Any] = {} + typekw: dict[str, Any] = {} if inptype == "File": action = FileAction @@ -962,8 +951,8 @@ def add_argument( def generate_parser( toolparser: argparse.ArgumentParser, tool: Process, - namemap: Dict[str, str], - records: List[str], + namemap: dict[str, str], + records: list[str], input_required: bool = True, urljoin: Callable[[str, str], str] = urllib.parse.urljoin, base_uri: str = "", diff --git a/cwltool/builder.py b/cwltool/builder.py index 2ba1e6543..e1de5b857 100644 --- a/cwltool/builder.py +++ b/cwltool/builder.py @@ -3,21 +3,9 @@ import copy import logging import math +from collections.abc import MutableMapping, MutableSequence from decimal import Decimal -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Type, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from cwl_utils.file_formats import check_format @@ -55,7 +43,7 @@ ) from .pathmapper import PathMapper -INPUT_OBJ_VOCAB: Dict[str, str] = { +INPUT_OBJ_VOCAB: dict[str, str] = { "Any": "https://w3id.org/cwl/salad#Any", "File": "https://w3id.org/cwl/cwl#File", "Directory": "https://w3id.org/cwl/cwl#Directory", @@ -107,16 +95,16 @@ class Builder(HasReqsHints): def __init__( self, job: CWLObjectType, - files: List[CWLObjectType], - bindings: List[CWLObjectType], + files: list[CWLObjectType], + bindings: list[CWLObjectType], schemaDefs: MutableMapping[str, CWLObjectType], names: Names, - requirements: List[CWLObjectType], - hints: List[CWLObjectType], - resources: Dict[str, Union[int, float]], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], + resources: dict[str, Union[int, float]], mutation_manager: Optional[MutationManager], formatgraph: Optional[Graph], - make_fs_access: Type[StdFsAccess], + make_fs_access: type[StdFsAccess], fs_access: StdFsAccess, job_script_provider: Optional[DependenciesConfiguration], timeout: float, @@ -172,7 +160,8 @@ def __init__( self.find_default_container: Optional[Callable[[], str]] = None self.container_engine = container_engine - def build_job_script(self, commands: List[str]) -> Optional[str]: + def build_job_script(self, commands: list[str]) -> Optional[str]: + """Use the job_script_provider to turn the commands into a job script.""" if self.job_script_provider is not None: return self.job_script_provider.build_job_script(self, commands) return None @@ -180,11 +169,11 @@ def build_job_script(self, commands: List[str]) -> Optional[str]: def bind_input( self, schema: CWLObjectType, - datum: Union[CWLObjectType, List[CWLObjectType]], + datum: Union[CWLObjectType, list[CWLObjectType]], discover_secondaryFiles: bool, - lead_pos: Optional[Union[int, List[int]]] = None, - tail_pos: Optional[Union[str, List[int]]] = None, - ) -> List[MutableMapping[str, Union[str, List[int]]]]: + lead_pos: Optional[Union[int, list[int]]] = None, + tail_pos: Optional[Union[str, list[int]]] = None, + ) -> list[MutableMapping[str, Union[str, list[int]]]]: """ Bind an input object to the command line. @@ -200,8 +189,8 @@ def bind_input( if lead_pos is None: lead_pos = [] - bindings: List[MutableMapping[str, Union[str, List[int]]]] = [] - binding: Union[MutableMapping[str, Union[str, List[int]]], CommentedMap] = {} + bindings: list[MutableMapping[str, Union[str, list[int]]]] = [] + binding: Union[MutableMapping[str, Union[str, list[int]]], CommentedMap] = {} value_from_expression = False if "inputBinding" in schema and isinstance(schema["inputBinding"], MutableMapping): binding = CommentedMap(schema["inputBinding"].items()) @@ -324,7 +313,7 @@ def bind_input( if schema["type"] == "record": datum = cast(CWLObjectType, datum) - for f in cast(List[CWLObjectType], schema["fields"]): + for f in cast(list[CWLObjectType], schema["fields"]): name = cast(str, f["name"]) if name in datum and datum[name] is not None: bindings.extend( @@ -372,7 +361,7 @@ def _capture_files(f: CWLObjectType) -> CWLObjectType: self.files.append(datum) loadContents_sourceline: Union[ - None, MutableMapping[str, Union[str, List[int]]], CWLObjectType + None, MutableMapping[str, Union[str, list[int]]], CWLObjectType ] = None if binding and binding.get("loadContents"): loadContents_sourceline = binding @@ -513,7 +502,7 @@ def addsf( if "format" in schema: eval_format: Any = self.do_eval(schema["format"]) if isinstance(eval_format, str): - evaluated_format: Union[str, List[str]] = eval_format + evaluated_format: Union[str, list[str]] = eval_format elif isinstance(eval_format, MutableSequence): for index, entry in enumerate(eval_format): message = None @@ -541,7 +530,7 @@ def addsf( raise SourceLine( schema["format"], index, WorkflowException, debug ).makeError(message) - evaluated_format = cast(List[str], eval_format) + evaluated_format = cast(list[str], eval_format) else: raise SourceLine(schema, "format", WorkflowException, debug).makeError( "An expression in the 'format' field must " @@ -586,8 +575,8 @@ def addsf( # Position to front of the sort key if binding: for bi in bindings: - bi["position"] = cast(List[int], binding["position"]) + cast( - List[int], bi["position"] + bi["position"] = cast(list[int], binding["position"]) + cast( + list[int], bi["position"] ) bindings.append(binding) @@ -618,7 +607,8 @@ def tostr(self, value: Union[MutableMapping[str, str], Any]) -> str: else: return str(value) - def generate_arg(self, binding: CWLObjectType) -> List[str]: + def generate_arg(self, binding: CWLObjectType) -> list[str]: + """Convert an input binding to a list of command line arguments.""" value = binding.get("datum") debug = _logger.isEnabledFor(logging.DEBUG) if "valueFrom" in binding: @@ -648,7 +638,7 @@ def generate_arg(self, binding: CWLObjectType) -> List[str]: argl = [itemSeparator.join([self.tostr(v) for v in value])] elif binding.get("valueFrom"): value = [self.tostr(v) for v in value] - return cast(List[str], ([prefix] if prefix else [])) + cast(List[str], value) + return cast(list[str], ([prefix] if prefix else [])) + cast(list[str], value) elif prefix and value: return [prefix] else: diff --git a/cwltool/checker.py b/cwltool/checker.py index 676245698..17cba77ba 100644 --- a/cwltool/checker.py +++ b/cwltool/checker.py @@ -1,19 +1,8 @@ """Static checking of CWL workflow connectivity.""" from collections import namedtuple -from typing import ( - Any, - Dict, - Iterator, - List, - Literal, - MutableMapping, - MutableSequence, - Optional, - Sized, - Union, - cast, -) +from collections.abc import Iterator, MutableMapping, MutableSequence, Sized +from typing import Any, Literal, Optional, Union, cast from schema_salad.exceptions import ValidationException from schema_salad.sourceline import SourceLine, bullets, strip_dup_lineno @@ -25,8 +14,7 @@ from .utils import CWLObjectType, CWLOutputType, SinkType, aslist -def _get_type(tp): - # type: (Any) -> Any +def _get_type(tp: Any) -> Any: if isinstance(tp, MutableMapping): if tp.get("type") not in ("array", "record", "enum"): return tp["type"] @@ -98,10 +86,10 @@ def can_assign_src_to_sink(src: SinkType, sink: Optional[SinkType], strict: bool if src["type"] == "record" and sink["type"] == "record": return _compare_records(src, sink, strict) if src["type"] == "File" and sink["type"] == "File": - for sinksf in cast(List[CWLObjectType], sink.get("secondaryFiles", [])): + for sinksf in cast(list[CWLObjectType], sink.get("secondaryFiles", [])): if not [ 1 - for srcsf in cast(List[CWLObjectType], src.get("secondaryFiles", [])) + for srcsf in cast(list[CWLObjectType], src.get("secondaryFiles", [])) if sinksf == srcsf ]: if strict: @@ -167,7 +155,8 @@ def _rec_fields(rec: MutableMapping[str, Any]) -> MutableMapping[str, Any]: return True -def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: +def missing_subset(fullset: list[Any], subset: list[Any]) -> list[Any]: + """Calculate the items missing from the fullset given the subset.""" missing = [] for i in subset: if i not in fullset: @@ -176,11 +165,11 @@ def missing_subset(fullset: List[Any], subset: List[Any]) -> List[Any]: def static_checker( - workflow_inputs: List[CWLObjectType], + workflow_inputs: list[CWLObjectType], workflow_outputs: MutableSequence[CWLObjectType], step_inputs: MutableSequence[CWLObjectType], - step_outputs: List[CWLObjectType], - param_to_step: Dict[str, CWLObjectType], + step_outputs: list[CWLObjectType], + param_to_step: dict[str, CWLObjectType], ) -> None: """ Check if all source and sink types of a workflow are compatible before run time. @@ -191,7 +180,7 @@ def static_checker( # sink parameters: step_inputs and workflow_outputs # make a dictionary of source parameters, indexed by the "id" field - src_dict: Dict[str, CWLObjectType] = {} + src_dict: dict[str, CWLObjectType] = {} for param in workflow_inputs + step_outputs: src_dict[cast(str, param["id"])] = param @@ -245,7 +234,7 @@ def static_checker( ", ".join( shortname(cast(str, s["id"])) for s in cast( - List[Dict[str, Union[str, bool]]], + list[dict[str, Union[str, bool]]], param_to_step[sink["id"]]["inputs"], ) if not s.get("not_connected") @@ -328,11 +317,11 @@ def static_checker( def check_all_types( - src_dict: Dict[str, CWLObjectType], + src_dict: dict[str, CWLObjectType], sinks: MutableSequence[CWLObjectType], sourceField: Union[Literal["source"], Literal["outputSource"]], - param_to_step: Dict[str, CWLObjectType], -) -> Dict[str, List[SrcSink]]: + param_to_step: dict[str, CWLObjectType], +) -> dict[str, list[SrcSink]]: """ Given a list of sinks, check if their types match with the types of their sources. @@ -340,7 +329,7 @@ def check_all_types( (from :py:func:`check_types`) :raises ValidationException: if a sourceField is missing """ - validation = {"warning": [], "exception": []} # type: Dict[str, List[SrcSink]] + validation: dict[str, list[SrcSink]] = {"warning": [], "exception": []} for sink in sinks: if sourceField in sink: valueFrom = cast(Optional[str], sink.get("valueFrom")) @@ -351,18 +340,18 @@ def check_all_types( extra_message = "pickValue is: %s" % pickValue if isinstance(sink[sourceField], MutableSequence): - linkMerge = cast( + linkMerge: Optional[str] = cast( Optional[str], sink.get( "linkMerge", ("merge_nested" if len(cast(Sized, sink[sourceField])) > 1 else None), ), - ) # type: Optional[str] + ) if pickValue in ["first_non_null", "the_only_non_null"]: linkMerge = None - srcs_of_sink = [] # type: List[CWLObjectType] + srcs_of_sink: list[CWLObjectType] = [] for parm_id in cast(MutableSequence[str], sink[sourceField]): srcs_of_sink += [src_dict[parm_id]] if is_conditional_step(param_to_step, parm_id) and pickValue is None: @@ -404,10 +393,10 @@ def check_all_types( snk_typ = sink["type"] if "null" not in src_typ: - src_typ = ["null"] + cast(List[Any], src_typ) + src_typ = ["null"] + cast(list[Any], src_typ) if "null" not in cast( - Union[List[str], CWLObjectType], snk_typ + Union[list[str], CWLObjectType], snk_typ ): # Given our type names this works even if not a list validation["warning"].append( SrcSink( @@ -440,7 +429,7 @@ def check_all_types( return validation -def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: +def circular_dependency_checker(step_inputs: list[CWLObjectType]) -> None: """ Check if a workflow has circular dependency. @@ -448,8 +437,8 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: """ adjacency = get_dependency_tree(step_inputs) vertices = adjacency.keys() - processed: List[str] = [] - cycles: List[List[str]] = [] + processed: list[str] = [] + cycles: list[list[str]] = [] for vertex in vertices: if vertex not in processed: traversal_path = [vertex] @@ -461,7 +450,7 @@ def circular_dependency_checker(step_inputs: List[CWLObjectType]) -> None: raise ValidationException(exception_msg) -def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str]]: +def get_dependency_tree(step_inputs: list[CWLObjectType]) -> dict[str, list[str]]: """Get the dependency tree in the form of adjacency list.""" adjacency = {} # adjacency list of the dependency tree for step_input in step_inputs: @@ -482,10 +471,10 @@ def get_dependency_tree(step_inputs: List[CWLObjectType]) -> Dict[str, List[str] def processDFS( - adjacency: Dict[str, List[str]], - traversal_path: List[str], - processed: List[str], - cycles: List[List[str]], + adjacency: dict[str, list[str]], + traversal_path: list[str], + processed: list[str], + cycles: list[list[str]], ) -> None: """Perform depth first search.""" tip = traversal_path[-1] @@ -509,14 +498,15 @@ def get_step_id(field_id: str) -> str: return step_id -def is_conditional_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_conditional_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: + """Return True if the step given by the parm_id is a conditional step.""" if (source_step := param_to_step.get(parm_id)) is not None: if source_step.get("when") is not None: return True return False -def is_all_output_method_loop_step(param_to_step: Dict[str, CWLObjectType], parm_id: str) -> bool: +def is_all_output_method_loop_step(param_to_step: dict[str, CWLObjectType], parm_id: str) -> bool: """Check if a step contains a `loop` directive with `all_iterations` outputMethod.""" source_step: Optional[MutableMapping[str, Any]] = param_to_step.get(parm_id) if source_step is not None: diff --git a/cwltool/command_line_tool.py b/cwltool/command_line_tool.py index eb0b1a4f5..e201fb12b 100644 --- a/cwltool/command_line_tool.py +++ b/cwltool/command_line_tool.py @@ -12,25 +12,11 @@ import threading import urllib import urllib.parse +from collections.abc import Generator, Mapping, MutableMapping, MutableSequence from enum import Enum from functools import cmp_to_key, partial -from typing import ( - TYPE_CHECKING, - Any, - Dict, - Generator, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Pattern, - Set, - TextIO, - Type, - Union, - cast, -) +from re import Pattern +from typing import TYPE_CHECKING, Any, Optional, TextIO, Union, cast from mypy_extensions import mypyc_attr from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -163,8 +149,8 @@ def __init__( builder: Builder, script: str, output_callback: Optional[OutputCallbackType], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], outdir: Optional[str] = None, tmpdir: Optional[str] = None, ) -> None: @@ -241,7 +227,8 @@ def job( raise WorkflowException("Abstract operation cannot be executed.") -def remove_path(f): # type: (CWLObjectType) -> None +def remove_path(f: CWLObjectType) -> None: + """Remove any 'path' property, if present.""" if "path" in f: del f["path"] @@ -334,7 +321,7 @@ def __init__( self.output_callback = output_callback self.cachebuilder = cachebuilder self.outdir = jobcache - self.prov_obj = None # type: Optional[ProvenanceProfile] + self.prov_obj: Optional[ProvenanceProfile] = None def run( self, @@ -392,7 +379,7 @@ def check_valid_locations(fs_access: StdFsAccess, ob: CWLObjectType) -> None: raise ValidationException("Does not exist or is not a Directory: '%s'" % location) -OutputPortsType = Dict[str, Optional[CWLOutputType]] +OutputPortsType = dict[str, Optional[CWLOutputType]] class ParameterOutputWorkflowException(WorkflowException): @@ -411,13 +398,14 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext """Initialize this CommandLineTool.""" super().__init__(toolpath_object, loadingContext) self.prov_obj = loadingContext.prov_obj - self.path_check_mode = ( + self.path_check_mode: PathCheckingMode = ( PathCheckingMode.RELAXED if loadingContext.relax_path_checks else PathCheckingMode.STRICT - ) # type: PathCheckingMode + ) - def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: + def make_job_runner(self, runtimeContext: RuntimeContext) -> type[JobBase]: + """Return the correct CommandLineJob class given the container settings.""" dockerReq, dockerRequired = self.get_requirement("DockerRequirement") mpiReq, mpiRequired = self.get_requirement(MPIRequirementName) @@ -477,7 +465,7 @@ def make_job_runner(self, runtimeContext: RuntimeContext) -> Type[JobBase]: @staticmethod def make_path_mapper( - reffiles: List[CWLObjectType], + reffiles: list[CWLObjectType], stagedir: str, runtimeContext: RuntimeContext, separateDirs: bool, @@ -499,9 +487,9 @@ def updatePathmap(self, outdir: str, pathmap: PathMapper, fn: CWLObjectType) -> ("Writable" if fn.get("writable") else "") + cast(str, fn["class"]), False, ) - for sf in cast(List[CWLObjectType], fn.get("secondaryFiles", [])): + for sf in cast(list[CWLObjectType], fn.get("secondaryFiles", [])): self.updatePathmap(outdir, pathmap, sf) - for ls in cast(List[CWLObjectType], fn.get("listing", [])): + for ls in cast(list[CWLObjectType], fn.get("listing", [])): self.updatePathmap(os.path.join(outdir, cast(str, fn["basename"])), pathmap, ls) def _initialworkdir(self, j: JobBase, builder: Builder) -> None: @@ -517,7 +505,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: cwl_version ) < ORDERED_VERSIONS.index("v1.1.0-dev1") - ls = [] # type: List[CWLObjectType] + ls: list[CWLObjectType] = [] if isinstance(initialWorkdir["listing"], str): # "listing" is just a string (must be an expression) so # just evaluate it and use the result as if it was in @@ -587,7 +575,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: raise SourceLine(initialWorkdir, "listing", WorkflowException, debug).makeError( message ) - ls = cast(List[CWLObjectType], ls_evaluated) + ls = cast(list[CWLObjectType], ls_evaluated) else: # "listing" is an array of either expressions or Dirent so # evaluate each item @@ -634,10 +622,10 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: for e in entry: ec = cast(CWLObjectType, e) ec["writable"] = t.get("writable", False) - ls.extend(cast(List[CWLObjectType], entry)) + ls.extend(cast(list[CWLObjectType], entry)) continue - et = {} # type: CWLObjectType + et: CWLObjectType = {} if isinstance(entry, Mapping) and entry.get("class") in ( "File", "Directory", @@ -686,7 +674,7 @@ def _initialworkdir(self, j: JobBase, builder: Builder) -> None: if not initwd_item: continue if isinstance(initwd_item, MutableSequence): - ls.extend(cast(List[CWLObjectType], initwd_item)) + ls.extend(cast(list[CWLObjectType], initwd_item)) else: ls.append(cast(CWLObjectType, initwd_item)) @@ -850,9 +838,9 @@ def job( cmdline = ["docker", "run", dockerimg] + cmdline # not really run using docker, just for hashing purposes - keydict = { + keydict: dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] = { "cmdline": cmdline - } # type: Dict[str, Union[MutableSequence[Union[str, int]], CWLObjectType]] + } for shortcut in ["stdin", "stdout", "stderr"]: if shortcut in self.tool: @@ -1071,8 +1059,8 @@ def update_status_output_callback( j.inplace_update = cast(bool, inplaceUpdateReq["inplaceUpdate"]) normalizeFilesDirs(j.generatefiles) - readers = {} # type: Dict[str, CWLObjectType] - muts = set() # type: Set[str] + readers: dict[str, CWLObjectType] = {} + muts: set[str] = set() if builder.mutation_manager is not None: @@ -1103,7 +1091,7 @@ def register_reader(f: CWLObjectType) -> None: timelimit, _ = self.get_requirement("ToolTimeLimit") if timelimit is not None: with SourceLine(timelimit, "timelimit", ValidationException, debug): - limit_field = cast(Dict[str, Union[str, int]], timelimit)["timelimit"] + limit_field = cast(dict[str, Union[str, int]], timelimit)["timelimit"] if isinstance(limit_field, str): timelimit_eval = builder.do_eval(limit_field) if timelimit_eval and not isinstance(timelimit_eval, int): @@ -1142,7 +1130,7 @@ def register_reader(f: CWLObjectType) -> None: required_env = {} evr, _ = self.get_requirement("EnvVarRequirement") if evr is not None: - for eindex, t3 in enumerate(cast(List[Dict[str, str]], evr["envDef"])): + for eindex, t3 in enumerate(cast(list[dict[str, str]], evr["envDef"])): env_value_field = t3["envValue"] if "${" in env_value_field or "$(" in env_value_field: env_value_eval = builder.do_eval(env_value_field) @@ -1160,7 +1148,7 @@ def register_reader(f: CWLObjectType) -> None: shellcmd, _ = self.get_requirement("ShellCommandRequirement") if shellcmd is not None: - cmd = [] # type: List[str] + cmd: list[str] = [] for b in builder.bindings: arg = builder.generate_arg(b) if b.get("shellQuote", True): @@ -1201,7 +1189,7 @@ def register_reader(f: CWLObjectType) -> None: def collect_output_ports( self, - ports: Union[CommentedSeq, Set[CWLObjectType]], + ports: Union[CommentedSeq, set[CWLObjectType]], builder: Builder, outdir: str, rcode: int, @@ -1209,7 +1197,7 @@ def collect_output_ports( jobname: str = "", readers: Optional[MutableMapping[str, CWLObjectType]] = None, ) -> OutputPortsType: - ret = {} # type: OutputPortsType + ret: OutputPortsType = {} debug = _logger.isEnabledFor(logging.DEBUG) cwl_version = self.metadata.get(ORIGINAL_CWLVERSION, None) if cwl_version != "v1.0": @@ -1284,16 +1272,16 @@ def collect_output( fs_access: StdFsAccess, compute_checksum: bool = True, ) -> Optional[CWLOutputType]: - r = [] # type: List[CWLOutputType] + r: list[CWLOutputType] = [] empty_and_optional = False debug = _logger.isEnabledFor(logging.DEBUG) result: Optional[CWLOutputType] = None if "outputBinding" in schema: binding = cast( - MutableMapping[str, Union[bool, str, List[str]]], + MutableMapping[str, Union[bool, str, list[str]]], schema["outputBinding"], ) - globpatterns = [] # type: List[str] + globpatterns: list[str] = [] revmap = partial(revmap_file, builder, outdir) @@ -1359,7 +1347,7 @@ def collect_output( _logger.error("Unexpected error from fs_access", exc_info=True) raise - for files in cast(List[Dict[str, Optional[CWLOutputType]]], r): + for files in cast(list[dict[str, Optional[CWLOutputType]]], r): rfile = files.copy() revmap(rfile) if files["class"] == "Directory": @@ -1515,7 +1503,7 @@ def collect_output( and schema["type"]["type"] == "record" ): out = {} - for field in cast(List[CWLObjectType], schema["type"]["fields"]): + for field in cast(list[CWLObjectType], schema["type"]["fields"]): out[shortname(cast(str, field["name"]))] = self.collect_output( field, builder, outdir, fs_access, compute_checksum=compute_checksum ) diff --git a/cwltool/context.py b/cwltool/context.py index 283936d61..237a90968 100644 --- a/cwltool/context.py +++ b/cwltool/context.py @@ -5,20 +5,8 @@ import shutil import tempfile import threading -from typing import ( - IO, - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - List, - Literal, - Optional, - TextIO, - Tuple, - Union, -) +from collections.abc import Iterable +from typing import IO, TYPE_CHECKING, Any, Callable, Literal, Optional, TextIO, Union from ruamel.yaml.comments import CommentedMap from schema_salad.avro.schema import Names @@ -46,7 +34,7 @@ class ContextBase: """Shared kwargs based initializer for :py:class:`RuntimeContext` and :py:class:`LoadingContext`.""" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize.""" if kwargs: for k, v in kwargs.items(): @@ -87,13 +75,13 @@ def set_log_dir(outdir: str, log_dir: str, subdir_name: str) -> str: class LoadingContext(ContextBase): - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the LoadingContext from the kwargs.""" self.debug: bool = False self.metadata: CWLObjectType = {} - self.requirements: Optional[List[CWLObjectType]] = None - self.hints: Optional[List[CWLObjectType]] = None - self.overrides_list: List[CWLObjectType] = [] + self.requirements: Optional[list[CWLObjectType]] = None + self.hints: Optional[list[CWLObjectType]] = None + self.overrides_list: list[CWLObjectType] = [] self.loader: Optional[Loader] = None self.avsc_names: Optional[Names] = None self.disable_js_validation: bool = False @@ -117,7 +105,7 @@ def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: self.singularity: bool = False self.podman: bool = False self.eval_timeout: float = 60 - self.codegen_idx: Dict[str, Tuple[Any, "LoadingOptions"]] = {} + self.codegen_idx: dict[str, tuple[Any, "LoadingOptions"]] = {} self.fast_parser = False self.skip_resolve_all = False self.skip_schemas = False @@ -136,11 +124,11 @@ class RuntimeContext(ContextBase): tmp_outdir_prefix: str = "" stagedir: str = "" - def __init__(self, kwargs: Optional[Dict[str, Any]] = None) -> None: + def __init__(self, kwargs: Optional[dict[str, Any]] = None) -> None: """Initialize the RuntimeContext from the kwargs.""" select_resources_callable = Callable[ - [Dict[str, Union[int, float]], RuntimeContext], - Dict[str, Union[int, float]], + [dict[str, Union[int, float]], RuntimeContext], + dict[str, Union[int, float]], ] self.user_space_docker_cmd: Optional[str] = None self.secret_store: Optional["SecretStore"] = None diff --git a/cwltool/cuda.py b/cwltool/cuda.py index 719bfd867..1394ec239 100644 --- a/cwltool/cuda.py +++ b/cwltool/cuda.py @@ -2,13 +2,12 @@ import subprocess # nosec import xml.dom.minidom # nosec -from typing import Tuple from .loghandler import _logger from .utils import CWLObjectType -def cuda_version_and_device_count() -> Tuple[str, int]: +def cuda_version_and_device_count() -> tuple[str, int]: """Determine the CUDA version and number of attached CUDA GPUs.""" try: out = subprocess.check_output(["nvidia-smi", "-q", "-x"]) # nosec diff --git a/cwltool/cwlprov/__init__.py b/cwltool/cwlprov/__init__.py index b8ff8d14d..a09a57c34 100644 --- a/cwltool/cwlprov/__init__.py +++ b/cwltool/cwlprov/__init__.py @@ -6,10 +6,10 @@ import re import uuid from getpass import getuser -from typing import IO, Any, Callable, Dict, List, Optional, Tuple, TypedDict, Union +from typing import IO, Any, Callable, Optional, TypedDict, Union -def _whoami() -> Tuple[str, str]: +def _whoami() -> tuple[str, str]: """Return the current operating system account as (username, fullname).""" username = getuser() try: @@ -106,8 +106,8 @@ def _valid_orcid(orcid: Optional[str]) -> str: { "uri": str, "about": str, - "content": Optional[Union[str, List[str]]], - "oa:motivatedBy": Dict[str, str], + "content": Optional[Union[str, list[str]]], + "oa:motivatedBy": dict[str, str], }, ) @@ -116,11 +116,11 @@ class Aggregate(TypedDict, total=False): """RO Aggregate class.""" uri: Optional[str] - bundledAs: Optional[Dict[str, Any]] + bundledAs: Optional[dict[str, Any]] mediatype: Optional[str] - conformsTo: Optional[Union[str, List[str]]] + conformsTo: Optional[Union[str, list[str]]] createdOn: Optional[str] - createdBy: Optional[Dict[str, str]] + createdBy: Optional[dict[str, str]] # Aggregate.bundledAs is actually type Aggregate, but cyclic definitions are not supported diff --git a/cwltool/cwlprov/provenance_profile.py b/cwltool/cwlprov/provenance_profile.py index ce8d63ad4..d4dfd6cb4 100644 --- a/cwltool/cwlprov/provenance_profile.py +++ b/cwltool/cwlprov/provenance_profile.py @@ -3,22 +3,11 @@ import logging import urllib import uuid +from collections.abc import MutableMapping, MutableSequence, Sequence from io import BytesIO from pathlib import PurePath, PurePosixPath from socket import getfqdn -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Sequence, - Tuple, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Optional, Union, cast from prov.identifier import Identifier, QualifiedName from prov.model import PROV, PROV_LABEL, PROV_TYPE, PROV_VALUE, ProvDocument, ProvEntity @@ -117,7 +106,7 @@ def __str__(self) -> str: """Represent this Provenvance profile as a string.""" return f"ProvenanceProfile <{self.workflow_run_uri}> in <{self.research_object}>" - def generate_prov_doc(self) -> Tuple[str, ProvDocument]: + def generate_prov_doc(self) -> tuple[str, ProvDocument]: """Add basic namespaces.""" def host_provenance(document: ProvDocument) -> None: @@ -177,7 +166,7 @@ def host_provenance(document: ProvDocument) -> None: # by a user account, as cwltool is a command line tool account = self.document.agent(ACCOUNT_UUID) if self.orcid or self.full_name: - person: Dict[Union[str, Identifier], Any] = { + person: dict[Union[str, Identifier], Any] = { PROV_TYPE: PROV["Person"], "prov:type": SCHEMA["Person"], } @@ -291,7 +280,8 @@ def record_process_end( self.generate_output_prov(outputs, process_run_id, process_name) self.document.wasEndedBy(process_run_id, None, self.workflow_run_uri, when) - def declare_file(self, value: CWLObjectType) -> Tuple[ProvEntity, ProvEntity, str]: + def declare_file(self, value: CWLObjectType) -> tuple[ProvEntity, ProvEntity, str]: + """Construct a FileEntity for the given CWL File object.""" if value["class"] != "File": raise ValueError("Must have class:File: %s" % value) # Need to determine file hash aka RO filename @@ -399,10 +389,10 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: # dir_bundle.identifier, {PROV["type"]: ORE["ResourceMap"], # ORE["describes"]: coll_b.identifier}) - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [ + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [ (ORE["isDescribedBy"], dir_bundle.identifier) ] - coll_b_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_b_attribs: list[tuple[Union[str, Identifier], Any]] = [] # FIXME: .listing might not be populated yet - hopefully # a later call to this method will sort that @@ -469,7 +459,7 @@ def declare_directory(self, value: CWLObjectType) -> ProvEntity: self.research_object.add_uri(coll.identifier.uri) return coll - def declare_string(self, value: str) -> Tuple[ProvEntity, str]: + def declare_string(self, value: str) -> tuple[ProvEntity, str]: """Save as string in UTF-8.""" byte_s = BytesIO(str(value).encode(ENCODING)) data_file = self.research_object.add_data_file(byte_s, content_type=TEXT_PLAIN) @@ -518,7 +508,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: # Already processed this value, but it might not be in this PROV entities = self.document.get_record(value["@id"]) if entities: - return cast(List[ProvEntity], entities)[0] + return cast(list[ProvEntity], entities)[0] # else, unknown in PROV, re-add below as if it's fresh # Base case - we found a File we need to update @@ -549,7 +539,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: coll.add_asserted_type(CWLPROV[value["class"]]) # Let's iterate and recurse - coll_attribs: List[Tuple[Union[str, Identifier], Any]] = [] + coll_attribs: list[tuple[Union[str, Identifier], Any]] = [] for key, val in value.items(): v_ent = self.declare_artefact(val) self.document.membership(coll, v_ent) @@ -601,7 +591,7 @@ def declare_artefact(self, value: Any) -> ProvEntity: def used_artefacts( self, - job_order: Union[CWLObjectType, List[CWLObjectType]], + job_order: Union[CWLObjectType, list[CWLObjectType]], process_run_id: str, name: Optional[str] = None, ) -> None: @@ -704,7 +694,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) """Add http://www.w3.org/TR/prov-aq/ relations to nested PROV files.""" # NOTE: The below will only work if the corresponding metadata/provenance arcp URI # is a pre-registered namespace in the PROV Document - attribs: List[Tuple[Union[str, Identifier], Any]] = [ + attribs: list[tuple[Union[str, Identifier], Any]] = [ (PROV["has_provenance"], prov_id) for prov_id in prov_ids ] self.document.activity(activity, other_attributes=attribs) @@ -713,7 +703,7 @@ def activity_has_provenance(self, activity: str, prov_ids: Sequence[Identifier]) uris = [i.uri for i in prov_ids] self.research_object.add_annotation(activity, uris, PROV["has_provenance"].uri) - def finalize_prov_profile(self, name: Optional[str]) -> List[QualifiedName]: + def finalize_prov_profile(self, name: Optional[str]) -> list[QualifiedName]: """Transfer the provenance related files to the RO.""" # NOTE: Relative posix path if name is None: diff --git a/cwltool/cwlprov/ro.py b/cwltool/cwlprov/ro.py index 7c6eaf5d6..ac60afc92 100644 --- a/cwltool/cwlprov/ro.py +++ b/cwltool/cwlprov/ro.py @@ -7,20 +7,9 @@ import tempfile import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from pathlib import Path, PurePosixPath -from typing import ( - IO, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import IO, Any, Optional, Union, cast import prov.model as provM from prov.model import PROV, ProvDocument @@ -75,12 +64,12 @@ def __init__( self.folder = create_tmp_dir(temp_prefix_ro) self.closed = False # map of filename "data/de/alsdklkas": 12398123 bytes - self.bagged_size: Dict[str, int] = {} - self.tagfiles: Set[str] = set() - self._file_provenance: Dict[str, Aggregate] = {} - self._external_aggregates: List[Aggregate] = [] - self.annotations: List[Annotation] = [] - self._content_types: Dict[str, str] = {} + self.bagged_size: dict[str, int] = {} + self.tagfiles: set[str] = set() + self._file_provenance: dict[str, Aggregate] = {} + self._external_aggregates: list[Aggregate] = [] + self.annotations: list[Annotation] = [] + self._content_types: dict[str, str] = {} self.fsaccess = fsaccess # These should be replaced by generate_prov_doc when workflow/run IDs are known: self.engine_uuid = f"urn:uuid:{uuid.uuid4()}" @@ -202,14 +191,14 @@ def add_tagfile(self, path: str, timestamp: Optional[datetime.datetime] = None) "conformsTo": None, } - def _ro_aggregates(self) -> List[Aggregate]: + def _ro_aggregates(self) -> list[Aggregate]: """Gather dictionary of files to be added to the manifest.""" def guess_mediatype( rel_path: str, - ) -> Tuple[Optional[str], Optional[Union[str, List[str]]]]: + ) -> tuple[Optional[str], Optional[Union[str, list[str]]]]: """Return the mediatypes.""" - media_types: Dict[Union[str, None], str] = { + media_types: dict[Union[str, None], str] = { # Adapted from # https://w3id.org/bundle/2014-11-05/#media-types "txt": TEXT_PLAIN, @@ -223,12 +212,12 @@ def guess_mediatype( "provn": 'text/provenance-notation; charset="UTF-8"', "nt": "application/n-triples", } - conforms_to: Dict[Union[str, None], str] = { + conforms_to: dict[Union[str, None], str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "cwl": "https://w3id.org/cwl/", } - prov_conforms_to: Dict[str, str] = { + prov_conforms_to: dict[str, str] = { "provn": "http://www.w3.org/TR/2013/REC-prov-n-20130430/", "rdf": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", "ttl": "http://www.w3.org/TR/2013/REC-prov-o-20130430/", @@ -244,7 +233,7 @@ def guess_mediatype( extension = None mediatype: Optional[str] = media_types.get(extension, None) - conformsTo: Optional[Union[str, List[str]]] = conforms_to.get(extension, None) + conformsTo: Optional[Union[str, list[str]]] = conforms_to.get(extension, None) # TODO: Open CWL file to read its declared "cwlVersion", e.g. # cwlVersion = "v1.0" @@ -261,7 +250,7 @@ def guess_mediatype( conformsTo = prov_conforms_to[extension] return (mediatype, conformsTo) - aggregates: List[Aggregate] = [] + aggregates: list[Aggregate] = [] for path in self.bagged_size.keys(): temp_path = PurePosixPath(path) folder = temp_path.parent @@ -291,7 +280,7 @@ def guess_mediatype( bundledAs.update(self._file_provenance[path]) else: aggregate_dict["bundledAs"] = cast( - Optional[Dict[str, Any]], self._file_provenance[path] + Optional[dict[str, Any]], self._file_provenance[path] ) else: # Probably made outside wf run, part of job object? @@ -343,7 +332,7 @@ def add_uri(self, uri: str, timestamp: Optional[datetime.datetime] = None) -> Ag return aggr def add_annotation( - self, about: str, content: List[str], motivated_by: str = "oa:describing" + self, about: str, content: list[str], motivated_by: str = "oa:describing" ) -> str: """Cheap URI relativize for current directory and /.""" self.self_check() @@ -359,9 +348,9 @@ def add_annotation( self.annotations.append(ann) return uri - def _ro_annotations(self) -> List[Annotation]: + def _ro_annotations(self) -> list[Annotation]: """Append base RO and provenance annotations to the list of annotations.""" - annotations: List[Annotation] = [] + annotations: list[Annotation] = [] annotations.append( { "uri": uuid.uuid4().urn, @@ -511,7 +500,7 @@ def add_data_file( def _self_made( self, timestamp: Optional[datetime.datetime] = None - ) -> Tuple[str, Dict[str, str]]: # createdOn, createdBy + ) -> tuple[str, dict[str, str]]: # createdOn, createdBy if timestamp is None: timestamp = datetime.datetime.now() return ( @@ -519,7 +508,7 @@ def _self_made( {"uri": self.engine_uuid, "name": self.cwltool_version}, ) - def add_to_manifest(self, rel_path: str, checksums: Dict[str, str]) -> None: + def add_to_manifest(self, rel_path: str, checksums: dict[str, str]) -> None: """Add files to the research object manifest.""" self.self_check() if PurePosixPath(rel_path).is_absolute(): diff --git a/cwltool/cwlprov/writablebagfile.py b/cwltool/cwlprov/writablebagfile.py index d5ff3c731..06d7d0bf7 100644 --- a/cwltool/cwlprov/writablebagfile.py +++ b/cwltool/cwlprov/writablebagfile.py @@ -8,10 +8,11 @@ import uuid from array import array from collections import OrderedDict +from collections.abc import MutableMapping from io import FileIO, TextIOWrapper from mmap import mmap from pathlib import Path, PurePosixPath -from typing import Any, BinaryIO, Dict, MutableMapping, Optional, Union, cast +from typing import Any, BinaryIO, Optional, Union, cast from schema_salad.utils import json_dumps @@ -246,7 +247,7 @@ def create_job( relativised_input_objecttemp: CWLObjectType = {} research_object._relativise_files(copied) - def jdefault(o: Any) -> Dict[Any, Any]: + def jdefault(o: Any) -> dict[Any, Any]: return dict(o) if is_output: diff --git a/cwltool/cwlrdf.py b/cwltool/cwlrdf.py index dbe9e2f97..126f0c780 100644 --- a/cwltool/cwlrdf.py +++ b/cwltool/cwlrdf.py @@ -1,6 +1,7 @@ import urllib from codecs import StreamWriter -from typing import IO, Any, Dict, Iterator, Optional, TextIO, Union, cast +from collections.abc import Iterator +from typing import IO, Any, Optional, TextIO, Union, cast from rdflib import Graph from rdflib.query import ResultRow @@ -117,7 +118,7 @@ def dot_with_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: def dot_without_parameters(g: Graph, stdout: Union[TextIO, StreamWriter]) -> None: - dotname: Dict[str, str] = {} + dotname: dict[str, str] = {} clusternode = {} stdout.write("compound=true\n") diff --git a/cwltool/cwlviewer.py b/cwltool/cwlviewer.py index e544a568e..769343964 100644 --- a/cwltool/cwlviewer.py +++ b/cwltool/cwlviewer.py @@ -1,7 +1,8 @@ """Visualize a CWL workflow.""" +from collections.abc import Iterator from pathlib import Path -from typing import Iterator, List, cast +from typing import cast from urllib.parse import urlparse import pydot @@ -154,7 +155,7 @@ def _get_root_graph_uri(self) -> rdflib.term.Identifier: with open(_get_root_query_path) as f: get_root_query = f.read() root = cast( - List[rdflib.query.ResultRow], + list[rdflib.query.ResultRow], list( self._rdf_graph.query( get_root_query, diff --git a/cwltool/docker.py b/cwltool/docker.py index d0f628b15..b03ae635c 100644 --- a/cwltool/docker.py +++ b/cwltool/docker.py @@ -9,8 +9,9 @@ import subprocess # nosec import sys import threading +from collections.abc import MutableMapping from io import StringIO # pylint: disable=redefined-builtin -from typing import Callable, Dict, List, MutableMapping, Optional, Set, Tuple, cast +from typing import Callable, Optional, cast import requests @@ -23,13 +24,13 @@ from .pathmapper import MapperEnt, PathMapper from .utils import CWLObjectType, create_tmp_dir, ensure_writable -_IMAGES: Set[str] = set() +_IMAGES: set[str] = set() _IMAGES_LOCK = threading.Lock() -__docker_machine_mounts: Optional[List[str]] = None +__docker_machine_mounts: Optional[list[str]] = None __docker_machine_mounts_lock = threading.Lock() -def _get_docker_machine_mounts() -> List[str]: +def _get_docker_machine_mounts() -> list[str]: global __docker_machine_mounts if __docker_machine_mounts is None: with __docker_machine_mounts_lock: @@ -83,9 +84,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Docker software container engine.""" @@ -94,7 +95,7 @@ def __init__( def get_image( self, - docker_requirement: Dict[str, str], + docker_requirement: dict[str, str], pull_image: bool, force_pull: bool, tmp_outdir_prefix: str, @@ -127,7 +128,7 @@ def get_image( except (OSError, subprocess.CalledProcessError, UnicodeError): pass - cmd: List[str] = [] + cmd: list[str] = [] if "dockerFile" in docker_requirement: dockerfile_dir = create_tmp_dir(tmp_outdir_prefix) with open(os.path.join(dockerfile_dir, "Dockerfile"), "w") as dfile: @@ -204,13 +205,13 @@ def get_from_requirements( if not shutil.which(self.docker_exec): raise WorkflowException(f"{self.docker_exec} executable is not available") - if self.get_image(cast(Dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): + if self.get_image(cast(dict[str, str], r), pull_image, force_pull, tmp_outdir_prefix): return cast(Optional[str], r["dockerImageId"]) raise WorkflowException("Docker image %s not found" % r["dockerImageId"]) @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, @@ -233,7 +234,7 @@ def append_volume( os.makedirs(source) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" if not volume.resolved.startswith("_:"): @@ -242,7 +243,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -266,7 +267,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -295,7 +296,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: # spec currently says "HOME must be set to the designated output # directory." but spec might change to designated temp directory. # runtime.append("--env=HOME=/tmp") @@ -306,7 +307,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtimeContext: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False user_space_docker_cmd = runtimeContext.user_space_docker_cmd if user_space_docker_cmd: @@ -445,9 +446,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize a command line builder using the Podman software container engine.""" diff --git a/cwltool/docker_id.py b/cwltool/docker_id.py index bb436b2cb..90484b686 100644 --- a/cwltool/docker_id.py +++ b/cwltool/docker_id.py @@ -1,10 +1,10 @@ """Helper functions for docker.""" import subprocess # nosec -from typing import List, Optional, Tuple +from typing import Optional -def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: +def docker_vm_id() -> tuple[Optional[int], Optional[int]]: """ Return the User ID and Group ID of the default docker user inside the VM. @@ -21,7 +21,7 @@ def docker_vm_id() -> Tuple[Optional[int], Optional[int]]: return (None, None) -def check_output_and_strip(cmd: List[str]) -> Optional[str]: +def check_output_and_strip(cmd: list[str]) -> Optional[str]: """ Pass a command list to :py:func:`subprocess.check_output`. @@ -48,7 +48,7 @@ def docker_machine_name() -> Optional[str]: return check_output_and_strip(["docker-machine", "active"]) -def cmd_output_matches(check_cmd: List[str], expected_status: str) -> bool: +def cmd_output_matches(check_cmd: list[str], expected_status: str) -> bool: """ Run a command and compares output to expected. @@ -80,7 +80,7 @@ def docker_machine_running() -> bool: return cmd_output_matches(["docker-machine", "status", machine_name], "Running") -def cmd_output_to_int(cmd: List[str]) -> Optional[int]: +def cmd_output_to_int(cmd: list[str]) -> Optional[int]: """ Run the provided command and returns the integer value of the result. @@ -97,7 +97,7 @@ def cmd_output_to_int(cmd: List[str]) -> Optional[int]: return None -def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: +def boot2docker_id() -> tuple[Optional[int], Optional[int]]: """ Get the UID and GID of the docker user inside a running boot2docker vm. @@ -108,7 +108,7 @@ def boot2docker_id() -> Tuple[Optional[int], Optional[int]]: return (uid, gid) -def docker_machine_id() -> Tuple[Optional[int], Optional[int]]: +def docker_machine_id() -> tuple[Optional[int], Optional[int]]: """ Ask docker-machine for active machine and gets the UID of the docker user. diff --git a/cwltool/env_to_stdout.py b/cwltool/env_to_stdout.py index 33b832479..0309fe08f 100644 --- a/cwltool/env_to_stdout.py +++ b/cwltool/env_to_stdout.py @@ -11,10 +11,9 @@ """ import os -from typing import Dict -def deserialize_env(data: str) -> Dict[str, str]: +def deserialize_env(data: str) -> dict[str, str]: """Deserialize the output of `env -0` to dictionary.""" result = {} for item in data.strip("\0").split("\0"): diff --git a/cwltool/executors.py b/cwltool/executors.py index bfc87f9c7..6070462ab 100644 --- a/cwltool/executors.py +++ b/cwltool/executors.py @@ -7,18 +7,9 @@ import os import threading from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, MutableSequence from threading import Lock -from typing import ( - Dict, - Iterable, - List, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from typing import Optional, Union, cast import psutil from mypy_extensions import mypyc_attr @@ -50,8 +41,8 @@ class JobExecutor(metaclass=ABCMeta): def __init__(self) -> None: """Initialize.""" self.final_output: MutableSequence[Optional[CWLObjectType]] = [] - self.final_status: List[str] = [] - self.output_dirs: Set[str] = set() + self.final_status: list[str] = [] + self.output_dirs: set[str] = set() def __call__( self, @@ -59,7 +50,7 @@ def __call__( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return self.execute(process, job_order_object, runtime_context, logger) def output_callback(self, out: Optional[CWLObjectType], process_status: str) -> None: @@ -83,7 +74,7 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: logging.Logger = _logger, - ) -> Tuple[Union[Optional[CWLObjectType]], str]: + ) -> tuple[Union[Optional[CWLObjectType]], str]: """Execute the process.""" self.final_output = [] @@ -112,7 +103,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: runtime_context.toplevel = True runtime_context.workflow_eval_lock = threading.Condition(threading.RLock()) - job_reqs: Optional[List[CWLObjectType]] = None + job_reqs: Optional[list[CWLObjectType]] = None if "https://w3id.org/cwl/cwl#requirements" in job_order_object: if process.metadata.get(ORIGINAL_CWLVERSION) == "v1.0": raise WorkflowException( @@ -121,7 +112,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - List[CWLObjectType], + list[CWLObjectType], job_order_object["https://w3id.org/cwl/cwl#requirements"], ) elif "cwl:defaults" in process.metadata and "https://w3id.org/cwl/cwl#requirements" in cast( @@ -134,7 +125,7 @@ def check_for_abstract_op(tool: CWLObjectType) -> None: "can set the cwlVersion to v1.1" ) job_reqs = cast( - Optional[List[CWLObjectType]], + Optional[list[CWLObjectType]], cast(CWLObjectType, process.metadata["cwl:defaults"])[ "https://w3id.org/cwl/cwl#requirements" ], @@ -277,8 +268,8 @@ class MultithreadedJobExecutor(JobExecutor): def __init__(self) -> None: """Initialize.""" super().__init__() - self.exceptions: List[WorkflowException] = [] - self.pending_jobs: List[JobsType] = [] + self.exceptions: list[WorkflowException] = [] + self.pending_jobs: list[JobsType] = [] self.pending_jobs_lock = threading.Lock() self.max_ram = int(psutil.virtual_memory().available / 2**20) @@ -289,10 +280,10 @@ def __init__(self) -> None: self.allocated_cuda: int = 0 def select_resources( - self, request: Dict[str, Union[int, float]], runtime_context: RuntimeContext - ) -> Dict[str, Union[int, float]]: # pylint: disable=unused-argument + self, request: dict[str, Union[int, float]], runtime_context: RuntimeContext + ) -> dict[str, Union[int, float]]: # pylint: disable=unused-argument """Naïve check for available cpu cores and memory.""" - result: Dict[str, Union[int, float]] = {} + result: dict[str, Union[int, float]] = {} maxrsc = {"cores": self.max_cores, "ram": self.max_ram} resources_types = {"cores", "ram"} if "cudaDeviceCountMin" in request or "cudaDeviceCountMax" in request: @@ -491,5 +482,5 @@ def execute( job_order_object: CWLObjectType, runtime_context: RuntimeContext, logger: Optional[logging.Logger] = None, - ) -> Tuple[Optional[CWLObjectType], str]: + ) -> tuple[Optional[CWLObjectType], str]: return {}, "success" diff --git a/cwltool/factory.py b/cwltool/factory.py index 85d7344e6..eaf98e3cf 100644 --- a/cwltool/factory.py +++ b/cwltool/factory.py @@ -1,5 +1,5 @@ import os -from typing import Any, Dict, Optional, Union +from typing import Any, Optional, Union from . import load_tool from .context import LoadingContext, RuntimeContext @@ -62,7 +62,7 @@ def __init__( else: self.loading_context = loading_context - def make(self, cwl: Union[str, Dict[str, Any]]) -> Callable: + def make(self, cwl: Union[str, dict[str, Any]]) -> Callable: """Instantiate a CWL object from a CWl document.""" load = load_tool.load_tool(cwl, self.loading_context) if isinstance(load, int): diff --git a/cwltool/flatten.py b/cwltool/flatten.py index 420d90d04..3c057ebbe 100644 --- a/cwltool/flatten.py +++ b/cwltool/flatten.py @@ -1,12 +1,17 @@ -from typing import Any, Callable, List, cast +""" +Our version of the popular flatten() method. -# http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +http://rightfootin.blogspot.com/2006/09/more-on-python-flatten.html +""" +from typing import Any, Callable, cast -def flatten(thing, ltypes=(list, tuple)): - # type: (Any, Any) -> List[Any] + +def flatten(thing: Any) -> list[Any]: + """Flatten a list without recursion problems.""" if thing is None: return [] + ltypes = (list, tuple) if not isinstance(thing, ltypes): return [thing] @@ -22,4 +27,4 @@ def flatten(thing, ltypes=(list, tuple)): else: thing_list[i : i + 1] = thing_list[i] i += 1 - return cast(Callable[[Any], List[Any]], ltype)(thing_list) + return cast(Callable[[Any], list[Any]], ltype)(thing_list) diff --git a/cwltool/job.py b/cwltool/job.py index 1731a5350..b360be25f 100644 --- a/cwltool/job.py +++ b/cwltool/job.py @@ -16,24 +16,10 @@ import time import uuid from abc import ABCMeta, abstractmethod +from collections.abc import Iterable, Mapping, MutableMapping, MutableSequence +from re import Match from threading import Timer -from typing import ( - IO, - TYPE_CHECKING, - Callable, - Dict, - Iterable, - List, - Mapping, - Match, - MutableMapping, - MutableSequence, - Optional, - TextIO, - Tuple, - Union, - cast, -) +from typing import IO, TYPE_CHECKING, Callable, Optional, TextIO, Union, cast import psutil from prov.model import PROV @@ -122,9 +108,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Initialize the job object.""" @@ -140,7 +126,7 @@ def __init__( self.requirements = requirements self.hints = hints self.name = name - self.command_line: List[str] = [] + self.command_line: list[str] = [] self.pathmapper = PathMapper([], "", "") self.make_path_mapper = make_path_mapper self.generatemapper: Optional[PathMapper] = None @@ -228,7 +214,7 @@ def is_streamable(file: str) -> bool: def _execute( self, - runtime: List[str], + runtime: list[str], env: MutableMapping[str, str], runtimeContext: RuntimeContext, monitor_function: Optional[Callable[["subprocess.Popen[str]"], None]] = None, @@ -321,7 +307,7 @@ def stderr_stdout_log_path( commands = [str(x) for x in runtime + self.command_line] if runtimeContext.secret_store is not None: commands = cast( - List[str], + list[str], runtimeContext.secret_store.retrieve(cast(CWLOutputType, commands)), ) env = cast( @@ -456,7 +442,7 @@ def stderr_stdout_log_path( shutil.rmtree(self.tmpdir, True) @abstractmethod - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: """Variables required by the CWL spec (HOME, TMPDIR, etc). Note that with containers, the paths will (likely) be those from @@ -481,7 +467,7 @@ def prepare_environment( applied (in that order). """ # Start empty - env: Dict[str, str] = {} + env: dict[str, str] = {} # Preserve any env vars if runtimeContext.preserve_entire_environment: @@ -589,7 +575,7 @@ def run( self._execute([], self.environment, runtimeContext, monitor_function) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: env = {} env["HOME"] = self.outdir env["TMPDIR"] = self.tmpdir @@ -623,24 +609,24 @@ def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext, - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the list of commands to run the selected container engine.""" @staticmethod @abstractmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" @abstractmethod def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: """Append volume a file/dir mapping to the runtime option list.""" @abstractmethod def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -650,7 +636,7 @@ def add_writable_file_volume( @abstractmethod def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -674,7 +660,7 @@ def _preserve_environment_on_containers_warning( def create_file_and_add_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], secret_store: Optional[SecretStore], @@ -706,7 +692,7 @@ def create_file_and_add_volume( def add_volumes( self, pathmapper: PathMapper, - runtime: List[str], + runtime: list[str], tmpdir_prefix: str, secret_store: Optional[SecretStore] = None, any_path_okay: bool = False, @@ -918,7 +904,7 @@ def docker_monitor( def _job_popen( - commands: List[str], + commands: list[str], stdin_path: Optional[str], stdout_path: Optional[str], stderr_path: Optional[str], diff --git a/cwltool/load_tool.py b/cwltool/load_tool.py index d6352f918..4d7f3a930 100644 --- a/cwltool/load_tool.py +++ b/cwltool/load_tool.py @@ -7,18 +7,9 @@ import re import urllib import uuid +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Any, Optional, Union, cast from cwl_utils.parser import cwl_v1_2, cwl_v1_2_utils from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -93,7 +84,7 @@ def resolve_tool_uri( resolver: Optional[ResolverType] = None, fetcher_constructor: Optional[FetcherCallableType] = None, document_loader: Optional[Loader] = None, -) -> Tuple[str, str]: +) -> tuple[str, str]: uri = None # type: Optional[str] split = urllib.parse.urlsplit(argsworkflow) # In case of Windows path, urlsplit misjudge Drive letters as scheme, here we are skipping that @@ -117,7 +108,7 @@ def resolve_tool_uri( def fetch_document( argsworkflow: Union[str, CWLObjectType], loadingContext: Optional[LoadingContext] = None, -) -> Tuple[LoadingContext, CommentedMap, str]: +) -> tuple[LoadingContext, CommentedMap, str]: """Retrieve a CWL document.""" if loadingContext is None: loadingContext = LoadingContext() @@ -144,7 +135,7 @@ def fetch_document( return loadingContext, workflowobj, uri if isinstance(argsworkflow, MutableMapping): uri = cast(str, argsworkflow["id"]) if argsworkflow.get("id") else "_:" + str(uuid.uuid4()) - workflowobj = cast(CommentedMap, cmap(cast(Dict[str, Any], argsworkflow), fn=uri)) + workflowobj = cast(CommentedMap, cmap(cast(dict[str, Any], argsworkflow), fn=uri)) loadingContext.loader.idx[uri] = workflowobj return loadingContext, workflowobj, uri raise ValidationException("Must be URI or object: '%s'" % argsworkflow) @@ -306,7 +297,7 @@ def fast_parser( uri: str, loadingContext: LoadingContext, fetcher: Fetcher, -) -> Tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: +) -> tuple[Union[CommentedMap, CommentedSeq], CommentedMap]: lopt = cwl_v1_2.LoadingOptions(idx=loadingContext.codegen_idx, fileuri=fileuri, fetcher=fetcher) if uri not in loadingContext.codegen_idx: @@ -326,7 +317,7 @@ def fast_parser( processobj = cwl_v1_2.save(objects, relative_uris=False) - metadata: Dict[str, Any] = {} + metadata: dict[str, Any] = {} metadata["id"] = loadopt.fileuri if loadopt.namespaces: @@ -353,7 +344,7 @@ def fast_parser( objects, loadopt = loadingContext.codegen_idx[nofrag] fileobj = cmap( cast( - Union[int, float, str, Dict[str, Any], List[Any], None], + Union[int, float, str, dict[str, Any], list[Any], None], cwl_v1_2.save(objects, relative_uris=False), ) ) @@ -370,7 +361,7 @@ def fast_parser( return cast( Union[CommentedMap, CommentedSeq], - cmap(cast(Union[Dict[str, Any], List[Any]], processobj)), + cmap(cast(Union[dict[str, Any], list[Any]], processobj)), ), cast(CommentedMap, cmap(metadata)) @@ -379,7 +370,7 @@ def resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str]: +) -> tuple[LoadingContext, str]: """Validate a CWL document.""" if not loadingContext.loader: raise ValueError("loadingContext must have a loader.") @@ -394,7 +385,7 @@ def resolve_and_validate_document( if "cwl:tool" in workflowobj: jobobj, _ = loader.resolve_all(workflowobj, uri) uri = urllib.parse.urljoin(uri, workflowobj["https://w3id.org/cwl/cwl#tool"]) - del cast(Dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] + del cast(dict[str, Any], jobobj)["https://w3id.org/cwl/cwl#tool"] workflowobj = fetch_document(uri, loadingContext)[1] @@ -624,17 +615,18 @@ def resolve_overrides( ov: IdxResultType, ov_uri: str, baseurl: str, -) -> List[CWLObjectType]: +) -> list[CWLObjectType]: ovloader = Loader(overrides_ctx) ret, _ = ovloader.resolve_all(ov, baseurl) if not isinstance(ret, CommentedMap): raise Exception("Expected CommentedMap, got %s" % type(ret)) cwl_docloader = get_schema("v1.0")[0] cwl_docloader.resolve_all(ret, ov_uri) - return cast(List[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) + return cast(list[CWLObjectType], ret["http://commonwl.org/cwltool#overrides"]) -def load_overrides(ov: str, base_url: str) -> List[CWLObjectType]: +def load_overrides(ov: str, base_url: str) -> list[CWLObjectType]: + """Load and resolve any overrides.""" ovloader = Loader(overrides_ctx) return resolve_overrides(ovloader.fetch(ov), ov, base_url) @@ -644,7 +636,7 @@ def recursive_resolve_and_validate_document( workflowobj: Union[CommentedMap, CommentedSeq], uri: str, preprocess_only: bool = False, -) -> Tuple[LoadingContext, str, Process]: +) -> tuple[LoadingContext, str, Process]: """Validate a CWL document, checking that a tool object can be built.""" loadingContext, uri = resolve_and_validate_document( loadingContext, diff --git a/cwltool/main.py b/cwltool/main.py index 30f299f09..99928d0bd 100755 --- a/cwltool/main.py +++ b/cwltool/main.py @@ -15,21 +15,8 @@ import urllib import warnings from codecs import getwriter -from typing import ( - IO, - Any, - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence, Sized +from typing import IO, Any, Callable, Optional, Union, cast import argcomplete import coloredlogs @@ -185,11 +172,11 @@ def append_word_to_default_user_agent(word: str) -> None: def generate_example_input( inptype: Optional[CWLOutputType], default: Optional[CWLOutputType], -) -> Tuple[Any, str]: +) -> tuple[Any, str]: """Convert a single input schema into an example.""" example = None comment = "" - defaults = { + defaults: CWLObjectType = { "null": "null", "Any": "null", "boolean": False, @@ -202,7 +189,7 @@ def generate_example_input( "Directory": ruamel.yaml.comments.CommentedMap( [("class", "Directory"), ("path", "a/directory/path")] ), - } # type: CWLObjectType + } if isinstance(inptype, MutableSequence): optional = False if "null" in inptype: @@ -244,7 +231,7 @@ def generate_example_input( if default is not None: example = default elif inptype["type"] == "enum": - symbols = cast(List[str], inptype["symbols"]) + symbols = cast(list[str], inptype["symbols"]) if default is not None: example = default elif "default" in inptype: @@ -260,7 +247,7 @@ def generate_example_input( comment = '"{}" record type.'.format(inptype["name"]) else: comment = "Anonymous record type." - for field in cast(List[CWLObjectType], inptype["fields"]): + for field in cast(list[CWLObjectType], inptype["fields"]): value, f_comment = generate_example_input(field["type"], None) example.insert(0, shortname(cast(str, field["name"])), value, f_comment) elif "default" in inptype: @@ -343,7 +330,7 @@ def generate_input_template(tool: Process) -> CWLObjectType: """Generate an example input object for the given CWL process.""" template = ruamel.yaml.comments.CommentedMap() for inp in cast( - List[MutableMapping[str, str]], + list[MutableMapping[str, str]], realize_input_schema(tool.tool["inputs"], tool.schemaDefs), ): name = shortname(inp["id"]) @@ -356,9 +343,9 @@ def load_job_order( args: argparse.Namespace, stdin: IO[Any], fetcher_constructor: Optional[FetcherCallableType], - overrides_list: List[CWLObjectType], + overrides_list: list[CWLObjectType], tool_file_uri: str, -) -> Tuple[Optional[CWLObjectType], str, Loader]: +) -> tuple[Optional[CWLObjectType], str, Loader]: job_order_object = None job_order_file = None @@ -423,8 +410,8 @@ def init_job_order( ) -> CWLObjectType: secrets_req, _ = process.get_requirement("http://commonwl.org/cwltool#Secrets") if job_order_object is None: - namemap = {} # type: Dict[str, str] - records = [] # type: List[str] + namemap: dict[str, str] = {} + records: list[str] = [] toolparser = generate_parser( argparse.ArgumentParser(prog=args.workflow), process, @@ -463,7 +450,7 @@ def init_job_order( if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -486,7 +473,7 @@ def path_to_loc(p: CWLObjectType) -> None: p["location"] = p["path"] del p["path"] - ns = {} # type: ContextType + ns: ContextType = {} ns.update(cast(ContextType, job_order_object.get("$namespaces", {}))) ns.update(cast(ContextType, process.metadata.get("$namespaces", {}))) ld = Loader(ns) @@ -532,7 +519,7 @@ def expand_formats(p: CWLObjectType) -> None: if secret_store and secrets_req: secret_store.store( - [shortname(sc) for sc in cast(List[str], secrets_req["secrets"])], + [shortname(sc) for sc in cast(list[str], secrets_req["secrets"])], job_order_object, ) @@ -583,7 +570,7 @@ def prov_deps( def remove_non_cwl(deps: CWLObjectType) -> None: if "secondaryFiles" in deps: - sec_files = cast(List[CWLObjectType], deps["secondaryFiles"]) + sec_files = cast(list[CWLObjectType], deps["secondaryFiles"]) for index, entry in enumerate(sec_files): if not ("format" in entry and entry["format"] == CWL_IANA): del sec_files[index] @@ -602,11 +589,11 @@ def find_deps( nestdirs: bool = True, ) -> CWLObjectType: """Find the dependencies of the CWL document.""" - deps = { + deps: CWLObjectType = { "class": "File", "location": uri, "format": CWL_IANA, - } # type: CWLObjectType + } def loadref(base: str, uri: str) -> Union[CommentedMap, CommentedSeq, str, None]: return document_loader.fetch(document_loader.fetcher.urljoin(base, uri)) @@ -638,7 +625,8 @@ def print_pack( return json_dumps(target, indent=4, default=str) -def supported_cwl_versions(enable_dev: bool) -> List[str]: +def supported_cwl_versions(enable_dev: bool) -> list[str]: + """Return a list of currently supported CWL versions.""" # ALLUPDATES and UPDATES are dicts if enable_dev: versions = list(ALLUPDATES) @@ -692,8 +680,8 @@ def formatTime(self, record: logging.LogRecord, datefmt: Optional[str] = None) - def setup_provenance( args: argparse.Namespace, runtimeContext: RuntimeContext, - argsl: Optional[List[str]] = None, -) -> Tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: + argsl: Optional[list[str]] = None, +) -> tuple[ProvOut, "logging.StreamHandler[ProvOut]"]: if not args.compute_checksum: _logger.error("--provenance incompatible with --no-compute-checksum") raise ArgumentException() @@ -940,7 +928,7 @@ def print_targets( _logger.info("%s steps targets:", prefix[:-1]) for t in tool.tool["steps"]: print(f" {prefix}{shortname(t['id'])}", file=stdout) - run: Union[str, Process, Dict[str, Any]] = t["run"] + run: Union[str, Process, dict[str, Any]] = t["run"] if isinstance(run, str): process = make_tool(run, loading_context) elif isinstance(run, dict): @@ -951,7 +939,7 @@ def print_targets( def main( - argsl: Optional[List[str]] = None, + argsl: Optional[list[str]] = None, args: Optional[argparse.Namespace] = None, job_order_object: Optional[CWLObjectType] = None, stdin: IO[Any] = sys.stdin, @@ -998,7 +986,7 @@ def main( if args is None: if argsl is None: argsl = sys.argv[1:] - addl = [] # type: List[str] + addl: list[str] = [] if "CWLTOOL_OPTIONS" in os.environ: c_opts = os.environ["CWLTOOL_OPTIONS"].split(" ") addl = [x for x in c_opts if x != ""] @@ -1250,7 +1238,7 @@ def main( if args.parallel: temp_executor = MultithreadedJobExecutor() runtimeContext.select_resources = temp_executor.select_resources - real_executor = temp_executor # type: JobExecutor + real_executor: JobExecutor = temp_executor else: real_executor = SingleJobExecutor() else: @@ -1260,7 +1248,7 @@ def main( runtimeContext.basedir = input_basedir if isinstance(tool, ProcessGenerator): - tfjob_order = {} # type: CWLObjectType + tfjob_order: CWLObjectType = {} if loadingContext.jobdefaults: tfjob_order.update(loadingContext.jobdefaults) if job_order_object: diff --git a/cwltool/mpi.py b/cwltool/mpi.py index 2cc1122c6..a7bdcbe03 100644 --- a/cwltool/mpi.py +++ b/cwltool/mpi.py @@ -3,7 +3,8 @@ import inspect import os import re -from typing import List, Mapping, MutableMapping, Optional, Type, TypeVar, Union +from collections.abc import Mapping, MutableMapping +from typing import Optional, TypeVar, Union from schema_salad.utils import yaml_no_ts @@ -18,9 +19,9 @@ def __init__( runner: str = "mpirun", nproc_flag: str = "-n", default_nproc: Union[int, str] = 1, - extra_flags: Optional[List[str]] = None, - env_pass: Optional[List[str]] = None, - env_pass_regex: Optional[List[str]] = None, + extra_flags: Optional[list[str]] = None, + env_pass: Optional[list[str]] = None, + env_pass_regex: Optional[list[str]] = None, env_set: Optional[Mapping[str, str]] = None, ) -> None: """ @@ -46,7 +47,7 @@ def __init__( self.env_set = env_set or {} @classmethod - def load(cls: Type[MpiConfigT], config_file_name: str) -> MpiConfigT: + def load(cls: type[MpiConfigT], config_file_name: str) -> MpiConfigT: """Create the MpiConfig object from the contents of a YAML file. The file must contain exactly one object, whose attributes must diff --git a/cwltool/mutation.py b/cwltool/mutation.py index 077b92cb7..9f58a86cf 100644 --- a/cwltool/mutation.py +++ b/cwltool/mutation.py @@ -1,5 +1,5 @@ from collections import namedtuple -from typing import Dict, cast +from typing import cast from .errors import WorkflowException from .utils import CWLObjectType @@ -20,7 +20,7 @@ class MutationManager: def __init__(self) -> None: """Initialize.""" - self.generations: Dict[str, MutationState] = {} + self.generations: dict[str, MutationState] = {} def register_reader(self, stepname: str, obj: CWLObjectType) -> None: loc = cast(str, obj["location"]) diff --git a/cwltool/pack.py b/cwltool/pack.py index c9fbc4e04..d3705d5e4 100644 --- a/cwltool/pack.py +++ b/cwltool/pack.py @@ -2,17 +2,8 @@ import copy import urllib -from typing import ( - Any, - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Set, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.ref_resolver import Loader, SubLoader @@ -30,7 +21,7 @@ def find_run( d: Union[CWLObjectType, ResolveType], loadref: LoadRefType, - runs: Set[str], + runs: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -46,7 +37,7 @@ def find_run( def find_ids( d: Union[CWLObjectType, CWLOutputType, MutableSequence[CWLObjectType], None], - ids: Set[str], + ids: set[str], ) -> None: if isinstance(d, MutableSequence): for s in d: @@ -59,7 +50,8 @@ def find_ids( find_ids(cast(CWLOutputType, s2), ids) -def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> None: +def replace_refs(d: Any, rewrite: dict[str, str], stem: str, newstem: str) -> None: + """Replace references with the actual value.""" if isinstance(d, MutableSequence): for s, v in enumerate(d): if isinstance(v, str): @@ -88,7 +80,7 @@ def replace_refs(d: Any, rewrite: Dict[str, str], stem: str, newstem: str) -> No def import_embed( d: Union[MutableSequence[CWLObjectType], CWLObjectType, CWLOutputType], - seen: Set[str], + seen: set[str], ) -> None: if isinstance(d, MutableSequence): for v in d: @@ -114,7 +106,7 @@ def import_embed( def pack( loadingContext: LoadingContext, uri: str, - rewrite_out: Optional[Dict[str, str]] = None, + rewrite_out: Optional[dict[str, str]] = None, loader: Optional[Loader] = None, ) -> CWLObjectType: # The workflow document we have in memory right now may have been @@ -153,7 +145,7 @@ def pack( document_loader.idx[po["id"]] = CommentedMap(po.items()) document_loader.idx[metadata["id"]] = CommentedMap(metadata.items()) - found_versions = {cast(str, loadingContext.metadata["cwlVersion"])} # type: Set[str] + found_versions: set[str] = {cast(str, loadingContext.metadata["cwlVersion"])} def loadref(base: Optional[str], lr_uri: str) -> ResolveType: lr_loadingContext = loadingContext.copy() @@ -167,15 +159,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: raise Exception("loader should not be None") return lr_loadingContext.loader.resolve_ref(lr_uri, base_url=base)[0] - input_ids: Set[str] = set() - output_ids: Set[str] = set() + input_ids: set[str] = set() + output_ids: set[str] = set() if isinstance(processobj, MutableSequence): mainobj = processobj[0] else: mainobj = processobj - find_ids(cast(Dict[str, Any], mainobj)["inputs"], input_ids) - find_ids(cast(Dict[str, Any], mainobj)["outputs"], output_ids) + find_ids(cast(dict[str, Any], mainobj)["inputs"], input_ids) + find_ids(cast(dict[str, Any], mainobj)["outputs"], output_ids) runs = {uri} find_run(processobj, loadref, runs) @@ -190,15 +182,15 @@ def loadref(base: Optional[str], lr_uri: str) -> ResolveType: for f in runs: find_ids(document_loader.resolve_ref(f)[0], input_ids) - input_names: Set[str] = set() - output_names: Set[str] = set() + input_names: set[str] = set() + output_names: set[str] = set() - rewrite_inputs: Dict[str, str] = {} - rewrite_outputs: Dict[str, str] = {} + rewrite_inputs: dict[str, str] = {} + rewrite_outputs: dict[str, str] = {} mainpath, _ = urllib.parse.urldefrag(uri) - def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) -> None: + def rewrite_id(r: str, mainuri: str, rewrite: dict[str, str], names: set[str]) -> None: if r == mainuri: rewrite[r] = "#main" elif r.startswith(mainuri) and r[len(mainuri)] in ("#", "/"): @@ -225,7 +217,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - packed = CommentedMap((("$graph", CommentedSeq()), ("cwlVersion", update_to_version))) namespaces = metadata.get("$namespaces", None) - schemas: Set[str] = set() + schemas: set[str] = set() if "$schemas" in metadata: for each_schema in metadata["$schemas"]: schemas.add(each_schema) @@ -261,7 +253,7 @@ def rewrite_id(r: str, mainuri: str, rewrite: Dict[str, str], names: Set[str]) - "Operation", ): continue - dc = cast(Dict[str, Any], copy.deepcopy(dcr)) + dc = cast(dict[str, Any], copy.deepcopy(dcr)) v = rewrite_inputs[r] dc["id"] = v for n in ("name", "cwlVersion", "$namespaces", "$schemas"): diff --git a/cwltool/pathmapper.py b/cwltool/pathmapper.py index 0a06eb47b..10cb7a733 100644 --- a/cwltool/pathmapper.py +++ b/cwltool/pathmapper.py @@ -4,17 +4,8 @@ import stat import urllib import uuid -from typing import ( - Dict, - ItemsView, - Iterable, - Iterator, - KeysView, - List, - Optional, - Tuple, - cast, -) +from collections.abc import ItemsView, Iterable, Iterator, KeysView +from typing import Optional, cast from mypy_extensions import mypyc_attr from schema_salad.exceptions import ValidationException @@ -92,20 +83,20 @@ class PathMapper: def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, separateDirs: bool = True, ) -> None: """Initialize the PathMapper.""" - self._pathmap: Dict[str, MapperEnt] = {} + self._pathmap: dict[str, MapperEnt] = {} self.stagedir = stagedir self.separateDirs = separateDirs self.setup(dedup(referenced_files), basedir) def visitlisting( self, - listing: List[CWLObjectType], + listing: list[CWLObjectType], stagedir: str, basedir: str, copy: bool = False, @@ -147,7 +138,7 @@ def visit( if location.startswith("file://"): staged = False self.visitlisting( - cast(List[CWLObjectType], obj.get("listing", [])), + cast(list[CWLObjectType], obj.get("listing", [])), tgt, basedir, copy=copy, @@ -189,16 +180,19 @@ def visit( deref, tgt, "WritableFile" if copy else "File", staged ) self.visitlisting( - cast(List[CWLObjectType], obj.get("secondaryFiles", [])), + cast(list[CWLObjectType], obj.get("secondaryFiles", [])), stagedir, basedir, copy=copy, staged=staged, ) - def setup(self, referenced_files: List[CWLObjectType], basedir: str) -> None: - # Go through each file and set the target to its own directory along - # with any secondary files. + def setup(self, referenced_files: list[CWLObjectType], basedir: str) -> None: + """ + For each file, set the target to its own directory. + + Also processes secondary files into that same directory. + """ stagedir = self.stagedir for fob in referenced_files: if self.separateDirs: @@ -246,7 +240,7 @@ def parents(path: str) -> Iterable[str]: def reversemap( self, target: str, - ) -> Optional[Tuple[str, str]]: + ) -> Optional[tuple[str, str]]: """Find the (source, resolved_path) for the given target, if any.""" for k, v in self._pathmap.items(): if v[1] == target: diff --git a/cwltool/process.py b/cwltool/process.py index bde035118..fe5f84764 100644 --- a/cwltool/process.py +++ b/cwltool/process.py @@ -13,25 +13,9 @@ import textwrap import urllib.parse import uuid +from collections.abc import Iterable, Iterator, MutableMapping, MutableSequence, Sized from os import scandir -from typing import ( - TYPE_CHECKING, - Any, - Callable, - Dict, - Iterable, - Iterator, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Type, - Union, - cast, -) +from typing import TYPE_CHECKING, Any, Callable, Optional, Union, cast from cwl_utils import expression from mypy_extensions import mypyc_attr @@ -161,14 +145,14 @@ def filter(self, record: logging.LogRecord) -> bool: "vocab_res_proc.yml", ) -SCHEMA_CACHE: Dict[ - str, Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] +SCHEMA_CACHE: dict[ + str, tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader] ] = {} SCHEMA_FILE: Optional[CWLObjectType] = None SCHEMA_DIR: Optional[CWLObjectType] = None SCHEMA_ANY: Optional[CWLObjectType] = None -custom_schemas: Dict[str, Tuple[str, str]] = {} +custom_schemas: dict[str, tuple[str, str]] = {} def use_standard_schema(version: str) -> None: @@ -186,11 +170,11 @@ def use_custom_schema(version: str, name: str, text: str) -> None: def get_schema( version: str, -) -> Tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: +) -> tuple[Loader, Union[Names, SchemaParseException], CWLObjectType, Loader]: if version in SCHEMA_CACHE: return SCHEMA_CACHE[version] - cache: Dict[str, Union[str, Graph, bool]] = {} + cache: dict[str, Union[str, Graph, bool]] = {} version = version.split("#")[-1] if ".dev" in version: version = ".".join(version.split(".")[:-1]) @@ -244,7 +228,7 @@ def stage_files( :raises WorkflowException: if there is a file staging conflict """ items = pathmapper.items() if not symlink else pathmapper.items_exclude_children() - targets: Dict[str, MapperEnt] = {} + targets: dict[str, MapperEnt] = {} for key, entry in list(items): if "File" not in entry.type: continue @@ -309,11 +293,11 @@ def stage_files( def relocateOutputs( outputObj: CWLObjectType, destination_path: str, - source_directories: Set[str], + source_directories: set[str], action: str, fs_access: StdFsAccess, compute_checksum: bool = True, - path_mapper: Type[PathMapper] = PathMapper, + path_mapper: type[PathMapper] = PathMapper, ) -> CWLObjectType: adjustDirObjs(outputObj, functools.partial(get_listing, fs_access, recursive=True)) @@ -414,7 +398,7 @@ def add_sizes(fsaccess: StdFsAccess, obj: CWLObjectType) -> None: def fill_in_defaults( - inputs: List[CWLObjectType], + inputs: list[CWLObjectType], job: CWLObjectType, fsaccess: StdFsAccess, ) -> None: @@ -578,7 +562,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext self.tool["id"] = "_:" + str(uuid.uuid4()) self.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.tool["id"]).get( "requirements", [] ), @@ -617,7 +601,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext avroize_type(cast(MutableSequence[CWLOutputType], sdtypes)) av = make_valid_avro( sdtypes, - {cast(str, t["name"]): cast(Dict[str, Any], t) for t in sdtypes}, + {cast(str, t["name"]): cast(dict[str, Any], t) for t in sdtypes}, set(), vocab=INPUT_OBJ_VOCAB, ) @@ -655,9 +639,9 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext c["type"] = avroize_type(c["type"], c["name"]) if key == "inputs": - cast(List[CWLObjectType], self.inputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.inputs_record_schema["fields"]).append(c) elif key == "outputs": - cast(List[CWLObjectType], self.outputs_record_schema["fields"]).append(c) + cast(list[CWLObjectType], self.outputs_record_schema["fields"]).append(c) with SourceLine(toolpath_object, "inputs", ValidationException, debug): self.inputs_record_schema = cast( @@ -681,7 +665,7 @@ def __init__(self, toolpath_object: CommentedMap, loadingContext: LoadingContext if toolpath_object.get("class") is not None and not getdefault( loadingContext.disable_js_validation, False ): - validate_js_options: Optional[Dict[str, Union[List[str], str, int]]] = None + validate_js_options: Optional[dict[str, Union[list[str], str, int]]] = None if loadingContext.js_hint_options_file is not None: try: with open(loadingContext.js_hint_options_file) as options_file: @@ -784,7 +768,7 @@ def _init_job(self, joborder: CWLObjectType, runtime_context: RuntimeContext) -> v = job[k] dircount = [0] - def inc(d: List[int]) -> None: + def inc(d: list[int]) -> None: d[0] += 1 visit_class(v, ("Directory",), lambda x: inc(dircount)) # noqa: B023 @@ -820,7 +804,7 @@ def inc(d: List[int]) -> None: except (ValidationException, WorkflowException) as err: raise WorkflowException("Invalid job input record:\n" + str(err)) from err - files: List[CWLObjectType] = [] + files: list[CWLObjectType] = [] bindings = CommentedSeq() outdir = "" tmpdir = "" @@ -947,7 +931,7 @@ def inc(d: List[int]) -> None: def evalResources( self, builder: Builder, runtimeContext: RuntimeContext - ) -> Dict[str, Union[int, float]]: + ) -> dict[str, Union[int, float]]: resourceReq, _ = self.get_requirement("ResourceRequirement") if resourceReq is None: resourceReq = {} @@ -957,7 +941,7 @@ def evalResources( ram = 1024 else: ram = 256 - request: Dict[str, Union[int, float, str]] = { + request: dict[str, Union[int, float, str]] = { "coresMin": 1, "coresMax": 1, "ramMin": ram, @@ -1005,7 +989,7 @@ def evalResources( request[a + "Min"] = mn request[a + "Max"] = cast(Union[int, float], mx) - request_evaluated = cast(Dict[str, Union[int, float]], request) + request_evaluated = cast(dict[str, Union[int, float]], request) if runtimeContext.select_resources is not None: # Call select resources hook return runtimeContext.select_resources(request_evaluated, runtimeContext) @@ -1038,7 +1022,7 @@ def checkRequirements( f"Unsupported requirement {entry['class']}." ) - def validate_hints(self, avsc_names: Names, hints: List[CWLObjectType], strict: bool) -> None: + def validate_hints(self, avsc_names: Names, hints: list[CWLObjectType], strict: bool) -> None: """Process the hints field.""" if self.doc_loader is None: return @@ -1085,10 +1069,11 @@ def __str__(self) -> str: return f"{type(self).__name__}: {self.tool['id']}" -_names: Set[str] = set() +_names: set[str] = set() -def uniquename(stem: str, names: Optional[Set[str]] = None) -> str: +def uniquename(stem: str, names: Optional[set[str]] = None) -> str: + """Construct a thread-unique name using the given stem as a prefix.""" global _names if names is None: names = _names @@ -1123,8 +1108,8 @@ def nestdir(base: str, deps: CWLObjectType) -> CWLObjectType: def mergedirs( listing: MutableSequence[CWLObjectType], ) -> MutableSequence[CWLObjectType]: - r: List[CWLObjectType] = [] - ents: Dict[str, CWLObjectType] = {} + r: list[CWLObjectType] = [] + ents: dict[str, CWLObjectType] = {} for e in listing: basename = cast(str, e["basename"]) if basename not in ents: @@ -1138,14 +1123,14 @@ def mergedirs( if e.get("listing"): # name already in entries # merge it into the existing listing - cast(List[CWLObjectType], ents[basename].setdefault("listing", [])).extend( - cast(List[CWLObjectType], e["listing"]) + cast(list[CWLObjectType], ents[basename].setdefault("listing", [])).extend( + cast(list[CWLObjectType], e["listing"]) ) for e in ents.values(): if e["class"] == "Directory" and "listing" in e: e["listing"] = cast( MutableSequence[CWLOutputType], - mergedirs(cast(List[CWLObjectType], e["listing"])), + mergedirs(cast(list[CWLObjectType], e["listing"])), ) r.extend(ents.values()) return r @@ -1157,8 +1142,8 @@ def mergedirs( def scandeps( base: str, doc: Union[CWLObjectType, MutableSequence[CWLObjectType]], - reffields: Set[str], - urlfields: Set[str], + reffields: set[str], + urlfields: set[str], loadref: Callable[[str, str], Union[CommentedMap, CommentedSeq, str, None]], urljoin: Callable[[str, str], str] = urllib.parse.urljoin, nestdirs: bool = True, diff --git a/cwltool/procgenerator.py b/cwltool/procgenerator.py index 34c1e650f..9839ce5d4 100644 --- a/cwltool/procgenerator.py +++ b/cwltool/procgenerator.py @@ -1,5 +1,5 @@ import copy -from typing import Dict, Optional, Tuple, cast +from typing import Optional, cast from ruamel.yaml.comments import CommentedMap from schema_salad.exceptions import ValidationException @@ -99,12 +99,12 @@ def result( job_order: CWLObjectType, jobout: CWLObjectType, runtimeContext: RuntimeContext, - ) -> Tuple[Process, CWLObjectType]: + ) -> tuple[Process, CWLObjectType]: try: loadingContext = self.loadingContext.copy() loadingContext.metadata = {} embedded_tool = load_tool( - cast(Dict[str, str], jobout["runProcess"])["location"], loadingContext + cast(dict[str, str], jobout["runProcess"])["location"], loadingContext ) except ValidationException as vexc: if runtimeContext.debug: diff --git a/cwltool/run_job.py b/cwltool/run_job.py index 307872f7a..5a81ce20c 100644 --- a/cwltool/run_job.py +++ b/cwltool/run_job.py @@ -4,10 +4,10 @@ import os import subprocess # nosec import sys -from typing import BinaryIO, Dict, List, Optional, TextIO, Union +from typing import BinaryIO, Optional, TextIO, Union -def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[str, str]: +def handle_software_environment(cwl_env: dict[str, str], script: str) -> dict[str, str]: """Update the provided environment dict by running the script.""" exec_env = cwl_env.copy() exec_env["_CWLTOOL"] = "1" @@ -29,7 +29,7 @@ def handle_software_environment(cwl_env: Dict[str, str], script: str) -> Dict[st return env -def main(argv: List[str]) -> int: +def main(argv: list[str]) -> int: """ Read in the configuration JSON and execute the commands. diff --git a/cwltool/secrets.py b/cwltool/secrets.py index f35f24c37..c73e0108c 100644 --- a/cwltool/secrets.py +++ b/cwltool/secrets.py @@ -1,7 +1,8 @@ """Minimal in memory storage of secrets.""" import uuid -from typing import Dict, List, MutableMapping, MutableSequence, Optional, cast +from collections.abc import MutableMapping, MutableSequence +from typing import Optional, cast from .utils import CWLObjectType, CWLOutputType @@ -11,7 +12,7 @@ class SecretStore: def __init__(self) -> None: """Initialize the secret store.""" - self.secrets: Dict[str, str] = {} + self.secrets: dict[str, str] = {} def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: """ @@ -28,7 +29,7 @@ def add(self, value: Optional[CWLOutputType]) -> Optional[CWLOutputType]: return placeholder return value - def store(self, secrets: List[str], job: CWLObjectType) -> None: + def store(self, secrets: list[str], job: CWLObjectType) -> None: """Sanitize the job object of any of the given secrets.""" for j in job: if j in secrets: diff --git a/cwltool/singularity.py b/cwltool/singularity.py index c43183ac7..0029d3950 100644 --- a/cwltool/singularity.py +++ b/cwltool/singularity.py @@ -6,8 +6,9 @@ import re import shutil import sys +from collections.abc import MutableMapping from subprocess import check_call, check_output # nosec -from typing import Callable, Dict, List, MutableMapping, Optional, Tuple, cast +from typing import Callable, Optional, cast from schema_salad.sourceline import SourceLine from spython.main import Client @@ -28,13 +29,13 @@ # This is a list containing major and minor versions as integer. # (The number of minor version digits can vary among different distributions, # therefore we need a list here.) -_SINGULARITY_VERSION: Optional[List[int]] = None +_SINGULARITY_VERSION: Optional[list[int]] = None # Cached flavor / distribution of singularity # Can be singularity, singularity-ce or apptainer _SINGULARITY_FLAVOR: str = "" -def get_version() -> Tuple[List[int], str]: +def get_version() -> tuple[list[int], str]: """ Parse the output of 'singularity --version' to determine the flavor and version. @@ -131,9 +132,9 @@ def __init__( self, builder: Builder, joborder: CWLObjectType, - make_path_mapper: Callable[[List[CWLObjectType], str, RuntimeContext, bool], PathMapper], - requirements: List[CWLObjectType], - hints: List[CWLObjectType], + make_path_mapper: Callable[[list[CWLObjectType], str, RuntimeContext, bool], PathMapper], + requirements: list[CWLObjectType], + hints: list[CWLObjectType], name: str, ) -> None: """Builder for invoking the Singularty software container engine.""" @@ -141,7 +142,7 @@ def __init__( @staticmethod def get_image( - dockerRequirement: Dict[str, str], + dockerRequirement: dict[str, str], pull_image: bool, tmp_outdir_prefix: str, force_pull: bool = False, @@ -247,7 +248,7 @@ def get_image( dockerRequirement["dockerImageId"] = path found = True if (force_pull or not found) and pull_image: - cmd = [] # type: List[str] + cmd: list[str] = [] if "dockerPull" in dockerRequirement: if cache_folder: env = os.environ.copy() @@ -338,7 +339,7 @@ def get_from_requirements( if not bool(shutil.which("singularity")): raise WorkflowException("singularity executable is not available") - if not self.get_image(cast(Dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): + if not self.get_image(cast(dict[str, str], r), pull_image, tmp_outdir_prefix, force_pull): raise WorkflowException("Container image {} not found".format(r["dockerImageId"])) if "CWL_SINGULARITY_CACHE" in os.environ: @@ -350,7 +351,7 @@ def get_from_requirements( return os.path.abspath(img_path) @staticmethod - def append_volume(runtime: List[str], source: str, target: str, writable: bool = False) -> None: + def append_volume(runtime: list[str], source: str, target: str, writable: bool = False) -> None: """Add binding arguments to the runtime list.""" if is_version_3_9_or_newer(): DockerCommandLineJob.append_volume(runtime, source, target, writable, skip_mkdirs=True) @@ -364,7 +365,7 @@ def append_volume(runtime: List[str], source: str, target: str, writable: bool = runtime.append(vol) def add_file_or_directory_volume( - self, runtime: List[str], volume: MapperEnt, host_outdir_tgt: Optional[str] + self, runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str] ) -> None: if not volume.resolved.startswith("_:"): if host_outdir_tgt is not None and not is_version_3_4_or_newer(): @@ -380,7 +381,7 @@ def add_file_or_directory_volume( def add_writable_file_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -417,7 +418,7 @@ def add_writable_file_volume( def add_writable_directory_volume( self, - runtime: List[str], + runtime: list[str], volume: MapperEnt, host_outdir_tgt: Optional[str], tmpdir_prefix: str, @@ -452,7 +453,7 @@ def add_writable_directory_volume( shutil.copytree(volume.resolved, host_outdir_tgt) ensure_writable(host_outdir_tgt or new_dir) - def _required_env(self) -> Dict[str, str]: + def _required_env(self) -> dict[str, str]: return { "TMPDIR": self.CONTAINER_TMPDIR, "HOME": self.builder.outdir, @@ -460,7 +461,7 @@ def _required_env(self) -> Dict[str, str]: def create_runtime( self, env: MutableMapping[str, str], runtime_context: RuntimeContext - ) -> Tuple[List[str], Optional[str]]: + ) -> tuple[list[str], Optional[str]]: """Return the Singularity runtime list of commands and options.""" any_path_okay = self.builder.get_requirement("DockerRequirement")[1] or False runtime = [ diff --git a/cwltool/software_requirements.py b/cwltool/software_requirements.py index ec99bda05..6ad84da4b 100644 --- a/cwltool/software_requirements.py +++ b/cwltool/software_requirements.py @@ -10,17 +10,8 @@ import argparse import os import string -from typing import ( - TYPE_CHECKING, - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import TYPE_CHECKING, Any, Optional, Union, cast from .utils import HasReqsHints @@ -79,7 +70,8 @@ def __init__(self, args: argparse.Namespace) -> None: if self.tool_dependency_dir and not os.path.exists(self.tool_dependency_dir): os.makedirs(self.tool_dependency_dir) - def build_job_script(self, builder: "Builder", command: List[str]) -> str: + def build_job_script(self, builder: "Builder", command: list[str]) -> str: + """Use the galaxy-tool-util library to construct a build script.""" ensure_galaxy_lib_available() resolution_config_dict = { "use": self.use_tool_dependencies, @@ -103,14 +95,14 @@ def build_job_script(self, builder: "Builder", command: List[str]) -> str: ) ) - template_kwds: Dict[str, str] = dict(handle_dependencies=handle_dependencies) + template_kwds: dict[str, str] = dict(handle_dependencies=handle_dependencies) job_script = COMMAND_WITH_DEPENDENCIES_TEMPLATE.substitute(template_kwds) return job_script def get_dependencies(builder: HasReqsHints) -> ToolRequirements: (software_requirement, _) = builder.get_requirement("SoftwareRequirement") - dependencies: List[Union["ToolRequirement", Dict[str, Any]]] = [] + dependencies: list[Union["ToolRequirement", dict[str, Any]]] = [] if software_requirement and software_requirement.get("packages"): packages = cast( MutableSequence[MutableMapping[str, Union[str, MutableSequence[str]]]], diff --git a/cwltool/stdfsaccess.py b/cwltool/stdfsaccess.py index 069289111..c58257f63 100644 --- a/cwltool/stdfsaccess.py +++ b/cwltool/stdfsaccess.py @@ -3,7 +3,7 @@ import glob import os import urllib -from typing import IO, Any, List +from typing import IO, Any from schema_salad.ref_resolver import file_uri, uri_file_path @@ -31,7 +31,8 @@ def __init__(self, basedir: str) -> None: def _abs(self, p: str) -> str: return abspath(p, self.basedir) - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: + """Return a possibly empty list of absolute URI paths that match pathname.""" return [file_uri(str(self._abs(line))) for line in glob.glob(self._abs(pattern))] def open(self, fn: str, mode: str) -> IO[Any]: @@ -49,7 +50,8 @@ def isfile(self, fn: str) -> bool: def isdir(self, fn: str) -> bool: return os.path.isdir(self._abs(fn)) - def listdir(self, fn: str) -> List[str]: + def listdir(self, fn: str) -> list[str]: + """Return a list containing the absolute path URLs of the entries in the directory given by path.""" return [abspath(urllib.parse.quote(entry), fn) for entry in os.listdir(self._abs(fn))] def join(self, path, *paths): # type: (str, *str) -> str diff --git a/cwltool/subgraph.py b/cwltool/subgraph.py index f6df7e69f..204e987a8 100644 --- a/cwltool/subgraph.py +++ b/cwltool/subgraph.py @@ -1,18 +1,7 @@ import urllib from collections import namedtuple -from typing import ( - Any, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - Set, - Tuple, - Union, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -33,7 +22,7 @@ def subgraph_visit( current: str, nodes: MutableMapping[str, Node], - visited: Set[str], + visited: set[str], direction: str, ) -> None: if current in visited: @@ -48,7 +37,13 @@ def subgraph_visit( subgraph_visit(c, nodes, visited, direction) -def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: +def declare_node(nodes: dict[str, Node], nodeid: str, tp: Optional[str]) -> Node: + """ + Record the given nodeid in the graph. + + If the nodeid is already present, but its type is unset, set it. + :returns: The Node tuple (even if already present in the graph). + """ if nodeid in nodes: n = nodes[nodeid] if n.type is None: @@ -59,8 +54,8 @@ def declare_node(nodes: Dict[str, Node], nodeid: str, tp: Optional[str]) -> Node def find_step( - steps: List[WorkflowStep], stepid: str, loading_context: LoadingContext -) -> Tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: + steps: list[WorkflowStep], stepid: str, loading_context: LoadingContext +) -> tuple[Optional[CWLObjectType], Optional[WorkflowStep]]: """Find the step (raw dictionary and WorkflowStep) for a given step id.""" for st in steps: st_tool_id = st.tool["id"] @@ -114,7 +109,7 @@ def get_subgraph( if tool.tool["class"] != "Workflow": raise Exception("Can only extract subgraph from workflow") - nodes: Dict[str, Node] = {} + nodes: dict[str, Node] = {} for inp in tool.tool["inputs"]: declare_node(nodes, inp["id"], INPUT) @@ -149,7 +144,7 @@ def get_subgraph( nodes[out].up.append(st["id"]) # Find all the downstream nodes from the starting points - visited_down: Set[str] = set() + visited_down: set[str] = set() for r in roots: if nodes[r].type == OUTPUT: subgraph_visit(r, nodes, visited_down, UP) @@ -157,8 +152,8 @@ def get_subgraph( subgraph_visit(r, nodes, visited_down, DOWN) # Now make sure all the nodes are connected to upstream inputs - visited: Set[str] = set() - rewire: Dict[str, Tuple[str, CWLObjectType]] = {} + visited: set[str] = set() + rewire: dict[str, tuple[str, CWLObjectType]] = {} for v in visited_down: visited.add(v) if nodes[v].type in (STEP, OUTPUT): @@ -221,7 +216,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C extracted["inputs"] = CommentedSeq() extracted["outputs"] = CommentedSeq() - for in_port in cast(List[CWLObjectType], step["in"]): + for in_port in cast(list[CWLObjectType], step["in"]): name = "#" + cast(str, in_port["id"]).split("#")[-1].split("/")[-1] inp: CWLObjectType = {"id": name, "type": "Any"} if "default" in in_port: @@ -231,7 +226,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C if "linkMerge" in in_port: del in_port["linkMerge"] - for outport in cast(List[Union[str, Mapping[str, Any]]], step["out"]): + for outport in cast(list[Union[str, Mapping[str, Any]]], step["out"]): if isinstance(outport, Mapping): outport_id = cast(str, outport["id"]) else: @@ -256,7 +251,7 @@ def get_step(tool: Workflow, step_id: str, loading_context: LoadingContext) -> C def get_process( tool: Workflow, step_id: str, loading_context: LoadingContext -) -> Tuple[Any, WorkflowStep]: +) -> tuple[Any, WorkflowStep]: """Find the underlying Process for a given Workflow step id.""" if loading_context.loader is None: raise Exception("loading_context.loader cannot be None") diff --git a/cwltool/udocker.py b/cwltool/udocker.py index 6598d6a7c..ea3fc78ca 100644 --- a/cwltool/udocker.py +++ b/cwltool/udocker.py @@ -1,7 +1,5 @@ """Enables Docker software containers via the udocker runtime.""" -from typing import List - from .docker import DockerCommandLineJob @@ -10,7 +8,7 @@ class UDockerCommandLineJob(DockerCommandLineJob): @staticmethod def append_volume( - runtime: List[str], + runtime: list[str], source: str, target: str, writable: bool = False, diff --git a/cwltool/update.py b/cwltool/update.py index 4fd66b37a..67e1f4257 100644 --- a/cwltool/update.py +++ b/cwltool/update.py @@ -1,15 +1,7 @@ import copy +from collections.abc import MutableMapping, MutableSequence from functools import partial -from typing import ( - Callable, - Dict, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from typing import Callable, Optional, Union, cast from ruamel.yaml.comments import CommentedMap, CommentedSeq from schema_salad.exceptions import ValidationException @@ -20,7 +12,7 @@ from .utils import CWLObjectType, CWLOutputType, aslist, visit_class, visit_field -def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> Tuple[CommentedMap, str]: +def v1_2to1_3dev1(doc: CommentedMap, loader: Loader, baseuri: str) -> tuple[CommentedMap, str]: """Public updater for v1.2 to v1.3.0-dev1.""" doc = copy.deepcopy(doc) @@ -78,7 +70,7 @@ def rewrite_loop_requirements(t: CWLObjectType) -> None: def v1_1to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1 to v1.2.""" doc = copy.deepcopy(doc) @@ -94,7 +86,7 @@ def v1_1to1_2( def v1_0to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.0 to v1.1.""" doc = copy.deepcopy(doc) @@ -195,21 +187,21 @@ def fix_inputBinding(t: CWLObjectType) -> None: def v1_1_0dev1to1_1( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.1.0-dev1 to v1.1.""" return (doc, "v1.1") def v1_2_0dev1todev2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev1 to v1.2.0-dev2.""" return (doc, "v1.2.0-dev2") def v1_2_0dev2todev3( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev2 to v1.2.0-dev3.""" doc = copy.deepcopy(doc) @@ -232,21 +224,21 @@ def update_pickvalue(t: CWLObjectType) -> None: def v1_2_0dev3todev4( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev3 to v1.2.0-dev4.""" return (doc, "v1.2.0-dev4") def v1_2_0dev4todev5( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev4 to v1.2.0-dev5.""" return (doc, "v1.2.0-dev5") def v1_2_0dev5to1_2( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Public updater for v1.2.0-dev5 to v1.2.""" return (doc, "v1.2") @@ -264,13 +256,13 @@ def v1_2_0dev5to1_2( "v1.3.0-dev1", ] -UPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +UPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.0": v1_0to1_1, "v1.1": v1_1to1_2, "v1.2": v1_2to1_3dev1, } -DEVUPDATES: Dict[str, Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]]] = { +DEVUPDATES: dict[str, Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]]] = { "v1.1.0-dev1": v1_1_0dev1to1_1, "v1.2.0-dev1": v1_2_0dev1todev2, "v1.2.0-dev2": v1_2_0dev2todev3, @@ -291,7 +283,7 @@ def v1_2_0dev5to1_2( def identity( doc: CommentedMap, loader: Loader, baseuri: str -) -> Tuple[CommentedMap, str]: # pylint: disable=unused-argument +) -> tuple[CommentedMap, str]: # pylint: disable=unused-argument """Do-nothing, CWL document upgrade function.""" return (doc, cast(str, doc["cwlVersion"])) @@ -300,7 +292,7 @@ def checkversion( doc: Union[CommentedSeq, CommentedMap], metadata: CommentedMap, enable_dev: bool, -) -> Tuple[CommentedMap, str]: +) -> tuple[CommentedMap, str]: """Check the validity of the version of the give CWL document. Returns the document and the validated version string. @@ -365,7 +357,7 @@ def update( (cdoc, version) = checkversion(doc, metadata, enable_dev) originalversion = copy.copy(version) - nextupdate: Optional[Callable[[CommentedMap, Loader, str], Tuple[CommentedMap, str]]] = identity + nextupdate: Optional[Callable[[CommentedMap, Loader, str], tuple[CommentedMap, str]]] = identity while version != update_to and nextupdate: (cdoc, version) = nextupdate(cdoc, loader, baseuri) diff --git a/cwltool/utils.py b/cwltool/utils.py index c8620994a..e460842a9 100644 --- a/cwltool/utils.py +++ b/cwltool/utils.py @@ -19,9 +19,11 @@ import tempfile import urllib import uuid +from collections.abc import Generator, Iterable, MutableMapping, MutableSequence from datetime import datetime from email.utils import parsedate_to_datetime from functools import partial +from importlib.resources import as_file, files from itertools import zip_longest from pathlib import Path, PurePosixPath from tempfile import NamedTemporaryFile @@ -31,17 +33,9 @@ Any, Callable, Deque, - Dict, - Generator, - Iterable, - List, Literal, - MutableMapping, - MutableSequence, NamedTuple, Optional, - Set, - Tuple, TypedDict, Union, cast, @@ -54,11 +48,6 @@ from schema_salad.exceptions import ValidationException from schema_salad.ref_resolver import Loader -if sys.version_info >= (3, 9): - from importlib.resources import as_file, files -else: - from importlib_resources import as_file, files - if TYPE_CHECKING: from .command_line_tool import CallbackJob, ExpressionJob from .job import CommandLineJob, JobBase @@ -92,13 +81,13 @@ OutputCallbackType = Callable[[Optional[CWLObjectType], str], None] ResolverType = Callable[["Loader", str], Optional[str]] DestinationsType = MutableMapping[str, Optional[CWLOutputType]] -ScatterDestinationsType = MutableMapping[str, List[Optional[CWLOutputType]]] +ScatterDestinationsType = MutableMapping[str, list[Optional[CWLOutputType]]] ScatterOutputCallbackType = Callable[[Optional[ScatterDestinationsType], str], None] SinkType = Union[CWLOutputType, CWLObjectType] DirectoryType = TypedDict( - "DirectoryType", {"class": str, "listing": List[CWLObjectType], "basename": str} + "DirectoryType", {"class": str, "listing": list[CWLObjectType], "basename": str} ) -JSONType = Union[Dict[str, "JSONType"], List["JSONType"], str, int, float, bool, None] +JSONType = Union[dict[str, "JSONType"], list["JSONType"], str, int, float, bool, None] class WorkflowStateItem(NamedTuple): @@ -109,7 +98,7 @@ class WorkflowStateItem(NamedTuple): success: str -ParametersType = List[CWLObjectType] +ParametersType = list[CWLObjectType] StepType = CWLObjectType # WorkflowStep LoadListingType = Union[Literal["no_listing"], Literal["shallow_listing"], Literal["deep_listing"]] @@ -143,7 +132,7 @@ def copytree_with_merge(src: str, dst: str) -> None: shutil.copy2(spath, dpath) -def cmp_like_py2(dict1: Dict[str, Any], dict2: Dict[str, Any]) -> int: +def cmp_like_py2(dict1: dict[str, Any], dict2: dict[str, Any]) -> int: """ Compare in the same manner as Python2. @@ -259,20 +248,20 @@ def adjustDirObjs(rec: Any, op: Union[Callable[..., Any], "partial[Any]"]) -> No visit_class(rec, ("Directory",), op) -def dedup(listing: List[CWLObjectType]) -> List[CWLObjectType]: +def dedup(listing: list[CWLObjectType]) -> list[CWLObjectType]: marksub = set() - def mark(d: Dict[str, str]) -> None: + def mark(d: dict[str, str]) -> None: marksub.add(d["location"]) for entry in listing: if entry["class"] == "Directory": - for e in cast(List[CWLObjectType], entry.get("listing", [])): + for e in cast(list[CWLObjectType], entry.get("listing", [])): adjustFileObjs(e, mark) adjustDirObjs(e, mark) dd = [] - markdup: Set[str] = set() + markdup: set[str] = set() for r in listing: if r["location"] not in marksub and r["location"] not in markdup: dd.append(r) @@ -284,14 +273,14 @@ def mark(d: Dict[str, str]) -> None: def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = True) -> None: """Expand, recursively, any 'listing' fields in a Directory.""" if rec.get("class") != "Directory": - finddirs: List[CWLObjectType] = [] + finddirs: list[CWLObjectType] = [] visit_class(rec, ("Directory",), finddirs.append) for f in finddirs: get_listing(fs_access, f, recursive=recursive) return if "listing" in rec: return - listing: List[CWLOutputType] = [] + listing: list[CWLOutputType] = [] loc = cast(str, rec["location"]) for ld in fs_access.listdir(loc): parse = urllib.parse.urlparse(ld) @@ -310,7 +299,7 @@ def get_listing(fs_access: "StdFsAccess", rec: CWLObjectType, recursive: bool = rec["listing"] = listing -def trim_listing(obj: Dict[str, Any]) -> None: +def trim_listing(obj: dict[str, Any]) -> None: """ Remove 'listing' field from Directory objects that are file references. @@ -322,7 +311,7 @@ def trim_listing(obj: Dict[str, Any]) -> None: del obj["listing"] -def downloadHttpFile(httpurl: str) -> Tuple[str, Optional[datetime]]: +def downloadHttpFile(httpurl: str) -> tuple[str, Optional[datetime]]: """ Download a remote file, possibly using a locally cached copy. @@ -414,7 +403,7 @@ def normalizeFilesDirs( ] ] ) -> None: - def addLocation(d: Dict[str, Any]) -> None: + def addLocation(d: dict[str, Any]) -> None: if "location" not in d: if d["class"] == "File" and ("contents" not in d): raise ValidationException( @@ -484,10 +473,10 @@ class HasReqsHints: def __init__(self) -> None: """Initialize this reqs decorator.""" - self.requirements: List[CWLObjectType] = [] - self.hints: List[CWLObjectType] = [] + self.requirements: list[CWLObjectType] = [] + self.hints: list[CWLObjectType] = [] - def get_requirement(self, feature: str) -> Tuple[Optional[CWLObjectType], Optional[bool]]: + def get_requirement(self, feature: str) -> tuple[Optional[CWLObjectType], Optional[bool]]: """Retrieve the named feature from the requirements field, or the hints field.""" for item in reversed(self.requirements): if item["class"] == feature: diff --git a/cwltool/validate_js.py b/cwltool/validate_js.py index de4adaa14..b43b7ef0d 100644 --- a/cwltool/validate_js.py +++ b/cwltool/validate_js.py @@ -3,17 +3,8 @@ import json import logging from collections import namedtuple -from typing import ( - Any, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence +from typing import Any, Optional, Union, cast from cwl_utils.errors import SubstitutionError from cwl_utils.expression import scanner as scan_expression @@ -63,7 +54,7 @@ def get_expressions( tool: Union[CommentedMap, str, CommentedSeq], schema: Optional[Union[Schema, ArraySchema]], source_line: Optional[SourceLine] = None, -) -> List[Tuple[str, Optional[SourceLine]]]: +) -> list[tuple[str, Optional[SourceLine]]]: debug = _logger.isEnabledFor(logging.DEBUG) if is_expression(tool, schema): return [(cast(str, tool), source_line)] @@ -124,8 +115,8 @@ def get_expressions( def jshint_js( js_text: str, - globals: Optional[List[str]] = None, - options: Optional[Dict[str, Union[List[str], str, int]]] = None, + globals: Optional[list[str]] = None, + options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> JSHintJSReturn: @@ -177,7 +168,7 @@ def dump_jshint_error() -> None: except ValueError: dump_jshint_error() - jshint_errors: List[str] = [] + jshint_errors: list[str] = [] js_text_lines = js_text.split("\n") @@ -193,7 +184,7 @@ def dump_jshint_error() -> None: return JSHintJSReturn(jshint_errors, jshint_json.get("globals", [])) -def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[SourceLine]) -> None: +def print_js_hint_messages(js_hint_messages: list[str], source_line: Optional[SourceLine]) -> None: """Log the message from JSHint, using the line number.""" if source_line is not None: for js_hint_message in js_hint_messages: @@ -203,7 +194,7 @@ def print_js_hint_messages(js_hint_messages: List[str], source_line: Optional[So def validate_js_expressions( tool: CommentedMap, schema: Schema, - jshint_options: Optional[Dict[str, Union[List[str], str, int]]] = None, + jshint_options: Optional[dict[str, Union[list[str], str, int]]] = None, container_engine: str = "docker", eval_timeout: float = 60, ) -> None: diff --git a/cwltool/workflow.py b/cwltool/workflow.py index 982ec7e70..3bf32251f 100644 --- a/cwltool/workflow.py +++ b/cwltool/workflow.py @@ -3,16 +3,8 @@ import functools import logging import random -from typing import ( - Callable, - Dict, - List, - Mapping, - MutableMapping, - MutableSequence, - Optional, - cast, -) +from collections.abc import Mapping, MutableMapping, MutableSequence +from typing import Callable, Optional, cast from uuid import UUID from mypy_extensions import mypyc_attr @@ -98,7 +90,7 @@ def __init__( loadingContext.requirements = self.requirements loadingContext.hints = self.hints - self.steps: List[WorkflowStep] = [] + self.steps: list[WorkflowStep] = [] validation_errors = [] for index, step in enumerate(self.tool.get("steps", [])): try: @@ -119,9 +111,9 @@ def __init__( workflow_inputs = self.tool["inputs"] workflow_outputs = self.tool["outputs"] - step_inputs: List[CWLObjectType] = [] - step_outputs: List[CWLObjectType] = [] - param_to_step: Dict[str, CWLObjectType] = {} + step_inputs: list[CWLObjectType] = [] + step_outputs: list[CWLObjectType] = [] + param_to_step: dict[str, CWLObjectType] = {} for step in self.steps: step_inputs.extend(step.tool["inputs"]) step_outputs.extend(step.tool["outputs"]) @@ -220,7 +212,7 @@ def __init__( loadingContext.requirements.append(parent_req) loadingContext.requirements.extend( cast( - List[CWLObjectType], + list[CWLObjectType], get_overrides(getdefault(loadingContext.overrides_list, []), self.id).get( "requirements", [] ), diff --git a/cwltool/workflow_job.py b/cwltool/workflow_job.py index d144128e6..6cd0b2e7c 100644 --- a/cwltool/workflow_job.py +++ b/cwltool/workflow_job.py @@ -3,19 +3,8 @@ import functools import logging import threading -from typing import ( - TYPE_CHECKING, - Dict, - List, - MutableMapping, - MutableSequence, - Optional, - Set, - Sized, - Tuple, - Union, - cast, -) +from collections.abc import MutableMapping, MutableSequence, Sized +from typing import TYPE_CHECKING, Optional, Union, cast from cwl_utils import expression from schema_salad.sourceline import SourceLine @@ -89,11 +78,11 @@ def __init__( ) -> None: """Initialize.""" self.dest = dest - self._completed: Set[int] = set() + self._completed: set[int] = set() self.processStatus = "success" self.total = total self.output_callback = output_callback - self.steps: List[Optional[JobsGeneratorType]] = [] + self.steps: list[Optional[JobsGeneratorType]] = [] @property def completed(self) -> int: @@ -123,7 +112,7 @@ def receive_scatter_output(self, index: int, jobout: CWLObjectType, processStatu def setTotal( self, total: int, - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], ) -> None: """ Set the total number of expected outputs along with the steps. @@ -137,7 +126,7 @@ def setTotal( def parallel_steps( - steps: List[Optional[JobsGeneratorType]], + steps: list[Optional[JobsGeneratorType]], rc: ReceiveScatterOutput, runtimeContext: RuntimeContext, ) -> JobsGeneratorType: @@ -187,7 +176,7 @@ def nested_crossproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) assert sjob is not None # nosec @@ -254,11 +243,11 @@ def _flat_crossproduct_scatter( callback: ReceiveScatterOutput, startindex: int, runtimeContext: RuntimeContext, -) -> Tuple[List[Optional[JobsGeneratorType]], int]: +) -> tuple[list[Optional[JobsGeneratorType]], int]: """Inner loop.""" scatter_key = scatter_keys[0] jobl = len(cast(Sized, joborder[scatter_key])) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] put = startindex for index in range(0, jobl): sjob: Optional[CWLObjectType] = copy.copy(joborder) @@ -309,7 +298,7 @@ def dotproduct_scatter( rc = ReceiveScatterOutput(output_callback, output, jobl) - steps: List[Optional[JobsGeneratorType]] = [] + steps: list[Optional[JobsGeneratorType]] = [] for index in range(0, jobl): sjobo: Optional[CWLObjectType] = copy.copy(joborder) assert sjobo is not None # nosec @@ -357,7 +346,7 @@ def match_types( elif linkMerge: if iid not in inputobj: inputobj[iid] = [] - sourceTypes = cast(List[Optional[CWLOutputType]], inputobj[iid]) + sourceTypes = cast(list[Optional[CWLOutputType]], inputobj[iid]) if linkMerge == "merge_nested": sourceTypes.append(src.value) elif linkMerge == "merge_flattened": @@ -380,7 +369,7 @@ def match_types( def object_from_state( - state: Dict[str, Optional[WorkflowStateItem]], + state: dict[str, Optional[WorkflowStateItem]], params: ParametersType, frag_only: bool, supportsMultipleInput: bool, @@ -487,7 +476,7 @@ def __init__(self, workflow: "Workflow", runtimeContext: RuntimeContext) -> None self.prov_obj = workflow.provenance_object self.parent_wf = workflow.parent_wf self.steps = [WorkflowJobStep(s) for s in workflow.steps] - self.state: Dict[str, Optional[WorkflowStateItem]] = {} + self.state: dict[str, Optional[WorkflowStateItem]] = {} self.processStatus = "" self.did_callback = False self.made_progress: Optional[bool] = None @@ -554,7 +543,7 @@ def do_output_callback(self, final_output_callback: OutputCallbackType) -> None: def receive_output( self, step: WorkflowJobStep, - outputparms: List[CWLObjectType], + outputparms: list[CWLObjectType], final_output_callback: OutputCallbackType, jobout: CWLObjectType, processStatus: str, @@ -701,7 +690,7 @@ def valueFromFunc(k: str, v: Optional[CWLOutputType]) -> Optional[CWLOutputType] return psio if "scatter" in step.tool: - scatter = cast(List[str], aslist(step.tool["scatter"])) + scatter = cast(list[str], aslist(step.tool["scatter"])) method = step.tool.get("scatterMethod") if method is None and len(scatter) != 1: raise WorkflowException( @@ -961,7 +950,7 @@ def loop_callback( try: loop = cast(MutableSequence[CWLObjectType], self.step.tool.get("loop", [])) outputMethod = self.step.tool.get("outputMethod", "last_iteration") - state: Dict[str, Optional[WorkflowStateItem]] = {} + state: dict[str, Optional[WorkflowStateItem]] = {} for i in self.step.tool["outputs"]: if "id" in i: iid = cast(str, i["id"]) diff --git a/mypy-requirements.txt b/mypy-requirements.txt index a7d0dacb8..ccd7737f4 100644 --- a/mypy-requirements.txt +++ b/mypy-requirements.txt @@ -1,6 +1,7 @@ mypy==1.11.2 # also update pyproject.toml ruamel.yaml>=0.16.0,<0.19 cwl-utils>=0.32 +cwltest types-requests types-setuptools types-psutil diff --git a/pyproject.toml b/pyproject.toml index 05a2b82f7..4f3f91c31 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -19,4 +19,4 @@ write_to = "cwltool/_version.py" [tool.black] line-length = 100 -target-version = [ "py38" ] +target-version = [ "py39" ] diff --git a/setup.py b/setup.py index fa39a378b..40c3fd8d4 100644 --- a/setup.py +++ b/setup.py @@ -129,7 +129,6 @@ "prov == 1.5.1", "mypy-extensions", "psutil >= 5.6.6", - "importlib_resources>=1.4;python_version<'3.9'", "coloredlogs", "pydot >= 1.4.1, <3", "argcomplete >= 1.12.0", @@ -143,7 +142,7 @@ "galaxy-util <24.2", ], }, - python_requires=">=3.8, <3.14", + python_requires=">=3.9, <3.14", use_scm_version=True, setup_requires=PYTEST_RUNNER + ["setuptools_scm>=8.0.4,<9"], test_suite="tests", @@ -169,7 +168,6 @@ "Operating System :: POSIX", "Operating System :: POSIX :: Linux", "Programming Language :: Python :: 3", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", diff --git a/tests/cwl-conformance/cwltool-conftest.py b/tests/cwl-conformance/cwltool-conftest.py index 3e2b83990..c87cf0ef7 100644 --- a/tests/cwl-conformance/cwltool-conftest.py +++ b/tests/cwl-conformance/cwltool-conftest.py @@ -6,20 +6,20 @@ import json from io import StringIO -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Optional from cwltest import utils def pytest_cwl_execute_test( config: utils.CWLTestConfig, processfile: str, jobfile: Optional[str] -) -> Tuple[int, Optional[Dict[str, Any]]]: +) -> tuple[int, Optional[dict[str, Any]]]: """Use the CWL reference runner (cwltool) to execute tests.""" from cwltool import main from cwltool.errors import WorkflowException stdout = StringIO() - argsl: List[str] = [f"--outdir={config.outdir}"] + argsl: list[str] = [f"--outdir={config.outdir}"] if config.runner_quiet: argsl.append("--quiet") elif config.verbose: diff --git a/tests/test_dependencies.py b/tests/test_dependencies.py index ae18a41ae..b903c04d6 100644 --- a/tests/test_dependencies.py +++ b/tests/test_dependencies.py @@ -6,7 +6,7 @@ from pathlib import Path from shutil import which from types import ModuleType -from typing import Optional, Tuple +from typing import Optional import pytest @@ -56,7 +56,7 @@ def test_biocontainers_resolution(tmp_path: Path) -> None: @pytest.fixture(scope="session") -def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: +def bioconda_setup(request: pytest.FixtureRequest) -> tuple[Optional[int], str]: """ Caches the conda environment created for seqtk_seq.cwl. @@ -108,7 +108,7 @@ def bioconda_setup(request: pytest.FixtureRequest) -> Tuple[Optional[int], str]: @pytest.mark.skipif(not deps, reason="galaxy-tool-util is not installed") -def test_bioconda(bioconda_setup: Tuple[Optional[int], str]) -> None: +def test_bioconda(bioconda_setup: tuple[Optional[int], str]) -> None: error_code, stderr = bioconda_setup assert error_code == 0, stderr diff --git a/tests/test_environment.py b/tests/test_environment.py index ba87041b3..a4bfd1ac3 100644 --- a/tests/test_environment.py +++ b/tests/test_environment.py @@ -2,8 +2,9 @@ import os from abc import ABC, abstractmethod +from collections.abc import Mapping from pathlib import Path -from typing import Any, Callable, Dict, List, Mapping, Union +from typing import Any, Callable, Union import pytest @@ -17,7 +18,7 @@ # TODO: maybe add regex? Env = Mapping[str, str] CheckerTypes = Union[None, str, Callable[[str, Env], bool]] -EnvChecks = Dict[str, CheckerTypes] +EnvChecks = dict[str, CheckerTypes] def assert_envvar_matches(check: CheckerTypes, k: str, env: Mapping[str, str]) -> None: @@ -66,7 +67,7 @@ def checks(tmp_prefix: str) -> EnvChecks: """Return a mapping from environment variable names to how to check for correctness.""" # Any flags to pass to cwltool to force use of the correct container - flags: List[str] + flags: list[str] # Does the env tool (maybe in our container) accept a `-0` flag? env_accepts_null: bool diff --git a/tests/test_examples.py b/tests/test_examples.py index 4d479e313..23d17dcb2 100644 --- a/tests/test_examples.py +++ b/tests/test_examples.py @@ -9,7 +9,7 @@ import urllib.parse from io import StringIO from pathlib import Path -from typing import Any, Dict, List, Union, cast +from typing import Any, Union, cast import cwl_utils.expression as expr import pydot @@ -69,7 +69,7 @@ def test_expression_match(expression: str, expected: bool) -> None: assert (match is not None) == expected -interpolate_input = { +interpolate_input: dict[str, Any] = { "foo": { "bar": {"baz": "zab1"}, "b ar": {"baz": 2}, @@ -77,7 +77,7 @@ def test_expression_match(expression: str, expected: bool) -> None: 'b"ar': {"baz": None}, }, "lst": ["A", "B"], -} # type: Dict[str, Any] +} interpolate_parameters = [ ("$(foo)", interpolate_input["foo"]), @@ -410,7 +410,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -473,7 +473,7 @@ def loadref( assert scanned_deps == expected_deps scanned_deps2 = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( cast(str, obj["id"]), obj, @@ -515,7 +515,7 @@ def loadref( raise Exception("test case can't load things") scanned_deps = cast( - List[Dict[str, Any]], + list[dict[str, Any]], cwltool.process.scandeps( "", obj, @@ -576,7 +576,7 @@ def test_scandeps_defaults_with_secondaryfiles() -> None: def test_dedupe() -> None: - not_deduped = [ + not_deduped: list[CWLObjectType] = [ {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/a"}, {"class": "File", "location": "file:///example/d"}, @@ -585,7 +585,7 @@ def test_dedupe() -> None: "location": "file:///example/c", "listing": [{"class": "File", "location": "file:///example/d"}], }, - ] # type: List[CWLObjectType] + ] expected = [ {"class": "File", "location": "file:///example/a"}, @@ -649,7 +649,7 @@ def test_dedupe() -> None: @pytest.mark.parametrize("name, source, sink, expected", source_to_sink) def test_compare_types( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink) == expected, name @@ -675,7 +675,7 @@ def test_compare_types( @pytest.mark.parametrize("name, source, sink, expected", source_to_sink_strict) def test_compare_types_strict( - name: str, source: Dict[str, Any], sink: Dict[str, Any], expected: bool + name: str, source: dict[str, Any], sink: dict[str, Any], expected: bool ) -> None: assert can_assign_src_to_sink(source, sink, strict=True) == expected, name @@ -1682,7 +1682,7 @@ def test_arguments_self() -> None: else: factory.runtime_context.use_container = False check = factory.make(get_data("tests/wf/paramref_arguments_self.cwl")) - outputs = cast(Dict[str, Any], check()) + outputs = cast(dict[str, Any], check()) assert "self_review" in outputs assert len(outputs) == 1 assert outputs["self_review"]["checksum"] == "sha1$724ba28f4a9a1b472057ff99511ed393a45552e1" diff --git a/tests/test_fetch.py b/tests/test_fetch.py index e55491d90..962b7d7e5 100644 --- a/tests/test_fetch.py +++ b/tests/test_fetch.py @@ -1,6 +1,6 @@ import os from pathlib import Path -from typing import Any, List, Optional +from typing import Any, Optional from urllib.parse import urljoin, urlsplit import pytest @@ -25,7 +25,7 @@ def __init__( ) -> None: """Create a Fetcher that provides a fixed result for testing purposes.""" - def fetch_text(self, url: str, content_types: Optional[List[str]] = None) -> str: + def fetch_text(self, url: str, content_types: Optional[list[str]] = None) -> str: if url == "baz:bar/foo.cwl": return """ cwlVersion: v1.0 diff --git a/tests/test_http_input.py b/tests/test_http_input.py index 6b4d9b479..e80260ff9 100644 --- a/tests/test_http_input.py +++ b/tests/test_http_input.py @@ -1,10 +1,7 @@ import os -import sys from datetime import datetime from pathlib import Path -from typing import List -import pytest from pytest_httpserver import HTTPServer from cwltool.pathmapper import PathMapper @@ -15,7 +12,7 @@ def test_http_path_mapping(tmp_path: Path) -> None: input_file_path = ( "https://raw.githubusercontent.com/common-workflow-language/cwltool/main/tests/2.fasta" ) - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": "https://raw.githubusercontent.com/common-workflow-language/" @@ -34,7 +31,6 @@ def test_http_path_mapping(tmp_path: Path) -> None: assert ">Sequence 561 BP; 135 A; 106 C; 98 G; 222 T; 0 other;" in contents -@pytest.mark.skipif(sys.version_info < (3, 7), reason="timesout on CI") def test_modification_date(tmp_path: Path) -> None: """Local copies of remote files should preserve last modification date.""" # Initialize the server @@ -58,7 +54,7 @@ def test_modification_date(tmp_path: Path) -> None: ) location = httpserver.url_for(f"/{remote_file_name}") - base_file: List[CWLObjectType] = [ + base_file: list[CWLObjectType] = [ { "class": "File", "location": location, diff --git a/tests/test_js_sandbox.py b/tests/test_js_sandbox.py index f4839e8a0..2c5df6339 100644 --- a/tests/test_js_sandbox.py +++ b/tests/test_js_sandbox.py @@ -5,7 +5,7 @@ import shutil import threading from pathlib import Path -from typing import Any, List +from typing import Any import pytest from cwl_utils import sandboxjs @@ -48,8 +48,8 @@ def test_value_from_two_concatenated_expressions() -> None: def hide_nodejs(temp_dir: Path) -> str: """Generate a new PATH that hides node{js,}.""" - paths: List[str] = os.environ.get("PATH", "").split(":") - names: List[str] = [] + paths: list[str] = os.environ.get("PATH", "").split(":") + names: list[str] = [] for name in ("nodejs", "node"): path = shutil.which(name) if path: diff --git a/tests/test_loop.py b/tests/test_loop.py index bf908196d..e8a043611 100644 --- a/tests/test_loop.py +++ b/tests/test_loop.py @@ -1,8 +1,8 @@ """Test the 1.3 loop feature.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_loop_ext.py b/tests/test_loop_ext.py index 499dd17b4..1769d64ad 100644 --- a/tests/test_loop_ext.py +++ b/tests/test_loop_ext.py @@ -1,8 +1,8 @@ """Test the prototype cwltool:Loop extension.""" import json +from collections.abc import MutableMapping, MutableSequence from io import StringIO -from typing import MutableMapping, MutableSequence from cwltool.main import main diff --git a/tests/test_mpi.py b/tests/test_mpi.py index 643907a39..92f0e353c 100644 --- a/tests/test_mpi.py +++ b/tests/test_mpi.py @@ -3,9 +3,10 @@ import json import os.path import sys +from collections.abc import Generator, MutableMapping from io import StringIO from pathlib import Path -from typing import Any, Generator, List, MutableMapping, Optional, Tuple +from typing import Any, Optional import pytest from ruamel.yaml.comments import CommentedMap, CommentedSeq @@ -75,12 +76,12 @@ def __init__(self): else: self.indata = sys.stdin.read().encode(sys.stdin.encoding) - def run_once(self, args: List[str]): + def run_once(self, args: list[str]): subprocess.run( args, input=self.indata, stdout=sys.stdout, stderr=sys.stderr ).check_returncode() - def run_many(self, n: int, args: List[str]): + def run_many(self, n: int, args: list[str]): for i in range(n): self.run_once(args) @@ -122,7 +123,7 @@ def make_processes_input(np: int, tmp_path: Path) -> Path: return input_file -def cwltool_args(fake_mpi_conf: str) -> List[str]: +def cwltool_args(fake_mpi_conf: str) -> list[str]: return ["--enable-ext", "--enable-dev", "--mpi-config-file", fake_mpi_conf] @@ -296,10 +297,10 @@ def schema_ext11() -> Generator[Names, None, None]: def mk_tool( schema: Names, - opts: List[str], - reqs: Optional[List[CommentedMap]] = None, - hints: Optional[List[CommentedMap]] = None, -) -> Tuple[LoadingContext, RuntimeContext, CommentedMap]: + opts: list[str], + reqs: Optional[list[CommentedMap]] = None, + hints: Optional[list[CommentedMap]] = None, +) -> tuple[LoadingContext, RuntimeContext, CommentedMap]: tool = basetool.copy() if reqs is not None: diff --git a/tests/test_override.py b/tests/test_override.py index 980c853bb..93c836c84 100644 --- a/tests/test_override.py +++ b/tests/test_override.py @@ -1,6 +1,5 @@ import json from io import StringIO -from typing import Dict, List import pytest @@ -76,7 +75,7 @@ @needs_docker @pytest.mark.parametrize("parameters,result", override_parameters) -def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: +def test_overrides(parameters: list[str], result: dict[str, str]) -> None: sio = StringIO() assert main(parameters, stdout=sio) == 0 @@ -119,7 +118,7 @@ def test_overrides(parameters: List[str], result: Dict[str, str]) -> None: @needs_docker @pytest.mark.parametrize("parameters,expected_error", failing_override_parameters) -def test_overrides_fails(parameters: List[str], expected_error: str) -> None: +def test_overrides_fails(parameters: list[str], expected_error: str) -> None: sio = StringIO() assert main(parameters, stderr=sio) == 1 diff --git a/tests/test_pack.py b/tests/test_pack.py index 1d38e35e8..a65996f8f 100644 --- a/tests/test_pack.py +++ b/tests/test_pack.py @@ -5,7 +5,6 @@ from functools import partial from io import StringIO from pathlib import Path -from typing import Dict import pytest from schema_salad.utils import yaml_no_ts @@ -95,7 +94,7 @@ def test_pack_fragment() -> None: def test_pack_rewrites() -> None: - rewrites: Dict[str, str] = {} + rewrites: dict[str, str] = {} loadingContext, workflowobj, uri = fetch_document(get_data("tests/wf/default-wf5.cwl")) loadingContext.do_update = False diff --git a/tests/test_path_checks.py b/tests/test_path_checks.py index 01ab7fe17..096de9942 100644 --- a/tests/test_path_checks.py +++ b/tests/test_path_checks.py @@ -1,7 +1,7 @@ import urllib.parse from io import BytesIO from pathlib import Path -from typing import IO, Any, List, cast +from typing import IO, Any, cast import pytest from ruamel.yaml.comments import CommentedMap @@ -112,7 +112,7 @@ def test_unicode_in_output_files(tmp_path: Path, filename: str) -> None: class StubFsAccess(StdFsAccess): """Stub fs access object that doesn't rely on the filesystem.""" - def glob(self, pattern: str) -> List[str]: + def glob(self, pattern: str) -> list[str]: """glob.""" return [pattern] diff --git a/tests/test_pathmapper.py b/tests/test_pathmapper.py index b7cf2f6a1..4ffac24bd 100644 --- a/tests/test_pathmapper.py +++ b/tests/test_pathmapper.py @@ -1,5 +1,3 @@ -from typing import List, Tuple - import pytest from cwltool.pathmapper import PathMapper @@ -10,7 +8,7 @@ def test_subclass() -> None: class SubPathMapper(PathMapper): def __init__( self, - referenced_files: List[CWLObjectType], + referenced_files: list[CWLObjectType], basedir: str, stagedir: str, new: str, @@ -81,7 +79,7 @@ def test_normalizeFilesDirs(name: str, file_dir: CWLObjectType, expected: CWLObj @pytest.mark.parametrize("filename,expected", basename_generation_parameters) -def test_basename_field_generation(filename: str, expected: Tuple[str, str]) -> None: +def test_basename_field_generation(filename: str, expected: tuple[str, str]) -> None: nameroot, nameext = expected expected2 = { "class": "File", diff --git a/tests/test_provenance.py b/tests/test_provenance.py index 83eb61c22..e8d8416be 100644 --- a/tests/test_provenance.py +++ b/tests/test_provenance.py @@ -3,8 +3,9 @@ import pickle import sys import urllib +from collections.abc import Generator from pathlib import Path -from typing import IO, Any, Generator, cast +from typing import IO, Any, cast import arcp import bagit diff --git a/tests/test_relocate.py b/tests/test_relocate.py index 81877c776..692e995fa 100644 --- a/tests/test_relocate.py +++ b/tests/test_relocate.py @@ -1,18 +1,13 @@ import json import os import shutil -import sys +from io import StringIO from pathlib import Path from cwltool.main import main from .util import get_data, needs_docker -if sys.version_info[0] < 3: - from StringIO import StringIO -else: - from io import StringIO - @needs_docker def test_for_910(tmp_path: Path) -> None: diff --git a/tests/test_secrets.py b/tests/test_secrets.py index bd90bee78..a8c0b67af 100644 --- a/tests/test_secrets.py +++ b/tests/test_secrets.py @@ -1,7 +1,7 @@ import shutil import tempfile from io import StringIO -from typing import Callable, Dict, List, Tuple, Union +from typing import Callable, Union import pytest @@ -13,7 +13,7 @@ @pytest.fixture -def secrets() -> Tuple[SecretStore, CWLObjectType]: +def secrets() -> tuple[SecretStore, CWLObjectType]: """Fixture to return a secret store.""" sec_store = SecretStore() job: CWLObjectType = {"foo": "bar", "baz": "quux"} @@ -22,7 +22,7 @@ def secrets() -> Tuple[SecretStore, CWLObjectType]: return sec_store, job -def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: +def test_obscuring(secrets: tuple[SecretStore, CWLObjectType]) -> None: """Basic test of secret store.""" storage, obscured = secrets assert obscured["foo"] != "bar" @@ -41,8 +41,8 @@ def test_obscuring(secrets: Tuple[SecretStore, CWLObjectType]) -> None: @pytest.mark.parametrize("factory,expected", obscured_factories_expected) def test_secrets( factory: Callable[[str], CWLObjectType], - expected: Union[str, List[str], Dict[str, str]], - secrets: Tuple[SecretStore, CWLObjectType], + expected: Union[str, list[str], dict[str, str]], + secrets: tuple[SecretStore, CWLObjectType], ) -> None: storage, obscured = secrets obs = obscured["foo"] diff --git a/tests/test_tmpdir.py b/tests/test_tmpdir.py index 73fe240d0..18a588cf8 100644 --- a/tests/test_tmpdir.py +++ b/tests/test_tmpdir.py @@ -6,7 +6,7 @@ import subprocess import sys from pathlib import Path -from typing import List, cast +from typing import cast import pytest from ruamel.yaml.comments import CommentedMap @@ -318,7 +318,7 @@ def test_docker_tmpdir_prefix(tmp_path: Path) -> None: "docker", ) job = DockerCommandLineJob(builder, {}, CommandLineTool.make_path_mapper, [], [], "") - runtime: List[str] = [] + runtime: list[str] = [] volume_writable_file = MapperEnt( resolved=get_data("tests/2.fastq"), target="foo", type=None, staged=None diff --git a/tests/test_toolargparse.py b/tests/test_toolargparse.py index 11ce5e3db..2e50fe722 100644 --- a/tests/test_toolargparse.py +++ b/tests/test_toolargparse.py @@ -1,7 +1,7 @@ import argparse from io import StringIO from pathlib import Path -from typing import Callable, List +from typing import Callable import pytest @@ -296,7 +296,7 @@ def test_argparser_without_doc() -> None: ), ], ) -def test_argparse_append_with_default(job_order: List[str], expected_values: List[str]) -> None: +def test_argparse_append_with_default(job_order: list[str], expected_values: list[str]) -> None: """ Confirm that the appended arguments must not include the default. diff --git a/tests/util.py b/tests/util.py index 0547cfa9a..44d2f108c 100644 --- a/tests/util.py +++ b/tests/util.py @@ -8,9 +8,10 @@ import shutil import subprocess import sys +from collections.abc import Generator, Mapping from contextlib import ExitStack from pathlib import Path -from typing import Dict, Generator, List, Mapping, Optional, Tuple, Union +from typing import Optional, Union import pytest @@ -83,11 +84,11 @@ def env_accepts_null() -> bool: def get_main_output( - args: List[str], + args: list[str], replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, -) -> Tuple[Optional[int], str, str]: +) -> tuple[Optional[int], str, str]: """Run cwltool main. args: the command line args to call it with @@ -127,13 +128,13 @@ def get_main_output( def get_tool_env( tmp_path: Path, - flag_args: List[str], + flag_args: list[str], inputs_file: Optional[str] = None, replacement_env: Optional[Mapping[str, str]] = None, extra_env: Optional[Mapping[str, str]] = None, monkeypatch: Optional[pytest.MonkeyPatch] = None, runtime_env_accepts_null: Optional[bool] = None, -) -> Dict[str, str]: +) -> dict[str, str]: """Get the env vars for a tool's invocation.""" # GNU env accepts the -0 option to end each variable's # printing with "\0". No such luck on BSD-ish. diff --git a/tox.ini b/tox.ini index c75d0cc47..2a5a431b9 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,9 @@ [tox] envlist = - py3{8,9,10,11,12,13}-lint - py3{8,9,10,11,12,13}-unit - py3{8,9,10,11,12,13}-bandit - py3{8,9,10,11,12,13}-mypy + py3{9,10,11,12,13}-lint + py3{9,10,11,12,13}-unit + py3{9,10,11,12,13}-bandit + py3{9,10,11,12,13}-mypy py312-lintreadme py312-shellcheck py312-pydocstyle @@ -16,7 +16,6 @@ testpaths = tests [gh-actions] python = - 3.8: py38 3.9: py39 3.10: py310 3.11: py311 @@ -25,13 +24,13 @@ python = [testenv] skipsdist = - py3{8,9,10,11,12,13}-!{unit,mypy,lintreadme} = True + py3{9,10,11,12,13}-!{unit,mypy,lintreadme} = True description = - py3{8,9,10,11,12,13}-unit: Run the unit tests - py3{8,9,10,11,12,13}-lint: Lint the Python code - py3{8,9,10,11,12,13}-bandit: Search for common security issues - py3{8,9,10,11,12,13}-mypy: Check for type safety + py3{9,10,11,12,13}-unit: Run the unit tests + py3{9,10,11,12,13}-lint: Lint the Python code + py3{9,10,11,12,13}-bandit: Search for common security issues + py3{9,10,11,12,13}-mypy: Check for type safety py312-pydocstyle: docstring style checker py312-shellcheck: syntax check for shell scripts py312-lintreadme: Lint the README.rst→.md conversion @@ -44,14 +43,14 @@ passenv = SINGULARITY_FAKEROOT extras = - py3{8,9,10,11,12,13}-unit: deps + py3{9,10,11,12,13}-unit: deps deps = - py3{8,9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt - py3{8,9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt - py3{8,9,10,11,12,13}-lint: -rlint-requirements.txt - py3{8,9,10,11,12,13}-bandit: bandit - py3{8,9,10,11,12,13}-mypy: -rmypy-requirements.txt + py3{9,10,11,12,13}-{unit,lint,bandit,mypy}: -rrequirements.txt + py3{9,10,11,12,13}-{unit,mypy}: -rtest-requirements.txt + py3{9,10,11,12,13}-lint: -rlint-requirements.txt + py3{9,10,11,12,13}-bandit: bandit + py3{9,10,11,12,13}-mypy: -rmypy-requirements.txt py312-pydocstyle: pydocstyle py312-pydocstyle: diff-cover py312-lintreadme: twine @@ -63,20 +62,20 @@ setenv = HOME = {envtmpdir} commands_pre = - py3{8,9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel + py3{9,10,11,12,13}-unit: python -m pip install -U pip setuptools wheel py312-lintreadme: python -m build --outdir {distdir} commands = - py3{8,9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} - py3{8,9,10,11,12,13}-bandit: bandit -r cwltool - py3{8,9,10,11,12,13}-lint: make flake8 format-check codespell-check - py3{8,9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} - py3{8,9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-unit: make coverage-report coverage.xml PYTEST_EXTRA={posargs} + py3{9,10,11,12,13}-bandit: bandit -r cwltool + py3{9,10,11,12,13}-lint: make flake8 format-check codespell-check + py3{9,10,11,12,13}-mypy: make mypy PYTEST_EXTRA={posargs} + py3{9,10,11,12}-mypy: make mypyc PYTEST_EXTRA={posargs} py312-shellcheck: make shellcheck py312-pydocstyle: make diff_pydocstyle_report py312-lintreadme: twine check {distdir}/* skip_install = - py3{8,9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true + py3{9,10,11,12,13}-{bandit,lint,mypy,shellcheck,pydocstyle,lintreadme}: true allowlist_externals = make