diff --git a/.travis.yml b/.travis.yml index dc415602..0f6811fe 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,6 @@ dist: bionic # VirtualEnv is too old on xenial matrix: include: - - python: "3.5" - python: "3.6" - python: "3.7" - python: "3.8" @@ -23,9 +22,6 @@ matrix: # Testing on macOS/Darwin tends to be much slower so only test the bare minimum # - # Minimum testing version is 3.6, since the 3.5 binaries from python.org fail - # with TLS error when trying to install `tox`. - # # When changing any version here also update the relevant checksum below with # the values found on the https://python.org/ website. - os: osx diff --git a/ipfshttpclient/client/base.py b/ipfshttpclient/client/base.py index a8d91d7f..ccdf88a8 100644 --- a/ipfshttpclient/client/base.py +++ b/ipfshttpclient/client/base.py @@ -9,11 +9,14 @@ from .. import multipart, http, utils +#XXX: Review if this is still necessary after requiring Python 3.8. if ty.TYPE_CHECKING: import typing_extensions as ty_ext else: ty_ext = utils +# Ensure that proper types show up when generating documentation without hard-depending on +# the cid package. if "cid" in sys.modules: import cid # type: ignore[import] cid_t = ty.Union[str, cid.CIDv0, cid.CIDv1] @@ -29,7 +32,7 @@ json_primitive_t = utils.json_primitive_t json_value_t = utils.json_value_t -# The following would be much more useful once GH/python/mypy#4441 is implemented… +#XXX: The following would be more useful with https://github.com/python/mypy/issues/4441 if ty.TYPE_CHECKING: # Lame workaround for type checkers CommonArgs = ty.Union[bool, http.auth_t, http.cookies_t, http.reqdata_sync_t, @@ -48,7 +51,7 @@ else: CommonArgs = ty.Dict[str, ty.Any] -# work around GH/mypy/mypy#731: no recursive structural types yet +#XXX: work around https://github.com/python/mypy/issues/731: no recursive structural types yet response_item_t = ty.Union[ json_primitive_t, "ResponseBase", diff --git a/ipfshttpclient/exceptions.py b/ipfshttpclient/exceptions.py index 594fc1c4..db234a7c 100644 --- a/ipfshttpclient/exceptions.py +++ b/ipfshttpclient/exceptions.py @@ -1,5 +1,5 @@ """ -The class hierachy for exceptions is:: +The class hierarchy for exceptions is: Error ├── VersionMismatch @@ -161,4 +161,4 @@ class ConnectionError(CommunicationError): class TimeoutError(CommunicationError): """Raised when the daemon didn't respond in time.""" - __slots__ = () \ No newline at end of file + __slots__ = () diff --git a/ipfshttpclient/filescanner.py b/ipfshttpclient/filescanner.py index 9fd132c2..9c6e9f3d 100644 --- a/ipfshttpclient/filescanner.py +++ b/ipfshttpclient/filescanner.py @@ -21,32 +21,24 @@ re_pattern_t = re_pattern_type = type(re.compile("")) -if hasattr(enum, "auto"): #PY36+ - enum_auto = enum.auto -elif not ty.TYPE_CHECKING: #PY35 - _counter = 0 - - def enum_auto() -> int: - global _counter - _counter += 1 - return _counter - - -O_DIRECTORY = getattr(os, "O_DIRECTORY", 0) # type: int +# Windows does not have os.O_DIRECTORY +O_DIRECTORY: int = getattr(os, "O_DIRECTORY", 0) -HAVE_FWALK = hasattr(os, "fwalk") # type: bool -HAVE_FWALK_BYTES = HAVE_FWALK and sys.version_info >= (3, 7) # type: bool +# Neither Windows nor MacOS have os.fwalk even through Python 3.9 +HAVE_FWALK: bool = hasattr(os, "fwalk") +HAVE_FWALK_BYTES = HAVE_FWALK and sys.version_info >= (3, 7) class Matcher(ty.Generic[ty.AnyStr], metaclass=abc.ABCMeta): """Represents a type that can match on file paths and decide whether they should be included in some file scanning/adding operation""" __slots__ = ("is_binary",) - #is_binary: bool - + + is_binary: bool + def __init__(self, is_binary: bool = False) -> None: - self.is_binary = is_binary # type: bool + self.is_binary = is_binary @abc.abstractmethod def should_descend(self, path: ty.AnyStr) -> bool: @@ -111,8 +103,8 @@ def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> utils.Literal_False return False -MATCH_ALL = MatchAll() # type: MatchAll[str] -MATCH_NONE = MatchNone() # type: MatchNone[str] +MATCH_ALL: MatchAll[str] = MatchAll() +MATCH_NONE: MatchNone[str] = MatchNone() class GlobMatcher(Matcher[ty.AnyStr], ty.Generic[ty.AnyStr]): @@ -136,11 +128,13 @@ class emulates. If your are accustomed the globing on real Unix shells pasting it into a real shell works this may be why. """ __slots__ = ("period_special", "_sep", "_pat", "_dir_only") - #period_special: bool - #_sep: ty.AnyStr - #_pat: ty.List[ty.Optional[re_pattern_t[ty.AnyStr]]] - #_dir_only: bool - + + period_special: bool + + _sep: ty.AnyStr + _pat: "ty.List[ty.Optional[re_pattern_t[ty.AnyStr]]]" + _dir_only: bool + def __init__(self, pat: ty.AnyStr, *, period_special: bool = True): """ Arguments @@ -153,14 +147,14 @@ def __init__(self, pat: ty.AnyStr, *, period_special: bool = True): shells allow one to disable this behaviour """ super().__init__(isinstance(pat, bytes)) - - self.period_special = period_special # type: bool - - self._sep = utils.maybe_fsencode(os.path.sep, pat) # type: ty.AnyStr - dblstar = utils.maybe_fsencode("**", pat) # type: ty.AnyStr - dot = utils.maybe_fsencode(".", pat) # type: ty.AnyStr - pat_ndot = utils.maybe_fsencode(r"(?![.])", pat) # type: ty.AnyStr - + + self.period_special = period_special + + self._sep = utils.maybe_fsencode(os.path.sep, pat) + dblstar = utils.maybe_fsencode("**", pat) + dot = utils.maybe_fsencode(".", pat) + pat_ndot = utils.maybe_fsencode(r"(?![.])", pat) + # Normalize path separator if os.path.altsep: pat = pat.replace(utils.maybe_fsencode(os.path.altsep, pat), self._sep) @@ -174,9 +168,9 @@ def __init__(self, pat: ty.AnyStr, *, period_special: bool = True): # (TBH, I find it hard to see how that is useful, but everybody does it # and it keeps things consistent overall – something to only match files # would be nice however.) - self._dir_only = pat.endswith(self._sep) # type: bool - - self._pat = [] # type: ty.List[ty.Optional[re_pattern_t[ty.AnyStr]]] + self._dir_only = pat.endswith(self._sep) + + self._pat = [] for label in pat.split(self._sep): # Skip over useless path components if len(label) < 1 or label == dot: @@ -193,7 +187,6 @@ def __init__(self, pat: ty.AnyStr, *, period_special: bool = True): "an issue if you need this!".format(os.fsdecode(label)) ) else: - #re_expr: ty.AnyStr if not isinstance(label, bytes): re_expr = fnmatch.translate(label) else: @@ -202,8 +195,7 @@ def __init__(self, pat: ty.AnyStr, *, period_special: bool = True): if period_special and not label.startswith(dot): re_expr = pat_ndot + re_expr self._pat.append(re.compile(re_expr)) - - + def should_descend(self, path: ty.AnyStr) -> bool: for idx, label in enumerate(path.split(self._sep)): # Always descend into any directory below a recursive pattern as we @@ -227,8 +219,7 @@ def should_descend(self, path: ty.AnyStr) -> bool: # The given path matched part of this pattern, so we should include this # directory to go further return True - - + def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool: # A final slash means “only match directories” if self._dir_only and not is_dir: @@ -236,8 +227,7 @@ def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool: labels = path.split(self._sep) # type: ty.List[ty.AnyStr] return self._match(labels, idx_pat=0, idx_path=0, is_dir=is_dir) - - + def _match(self, labels: ty.List[ty.AnyStr], *, idx_pat: int, idx_path: int, is_dir: bool) -> bool: while idx_pat < len(self._pat): @@ -308,32 +298,34 @@ class ReMatcher(Matcher[ty.AnyStr], ty.Generic[ty.AnyStr]): own matcher with a proper :meth:`Matcher.should_descend` method. """ __slots__ = ("_pat",) - #_pat: re_pattern_t[ty.AnyStr] - + + _pat: "re_pattern_t[ty.AnyStr]" + def __init__(self, pat: ty.Union[ty.AnyStr, "re_pattern_t[ty.AnyStr]"]): - self._pat = re.compile(pat) # type: re_pattern_t[ty.AnyStr] - + self._pat = re.compile(pat) + super().__init__(not (self._pat.flags & re.UNICODE)) def should_descend(self, path: ty.AnyStr) -> bool: return True def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool: - suffix = utils.maybe_fsencode(os.path.sep, path) if is_dir else type(path)() # type: ty.AnyStr + suffix: ty.AnyStr = utils.maybe_fsencode(os.path.sep, path) if is_dir else type(path)() return bool(self._pat.match(path + suffix)) class MetaMatcher(Matcher[ty.AnyStr], ty.Generic[ty.AnyStr]): """Match files and directories by delegating to other matchers""" __slots__ = ("_children",) - #_children: ty.List[Matcher[ty.AnyStr]] - + + _children: ty.List[Matcher[ty.AnyStr]] + def __init__(self, children: ty.List[Matcher[ty.AnyStr]]): assert len(children) > 0 super().__init__(children[0].is_binary) - - self._children = children # type: ty.List[Matcher[ty.AnyStr]] - + + self._children = children + def should_descend(self, path: ty.AnyStr) -> bool: return any(m.should_descend(path) for m in self._children) @@ -342,7 +334,7 @@ def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool: class NoRecusionAdapterMatcher(Matcher[ty.AnyStr], ty.Generic[ty.AnyStr]): - """Matcher adapter that will prevent any recusion + """Matcher adapter that will prevent any recursion Takes a subordinate matcher, but tells the scanner to never descend into any child directory and to never store files from such a directory. This is an @@ -350,13 +342,14 @@ class NoRecusionAdapterMatcher(Matcher[ty.AnyStr], ty.Generic[ty.AnyStr]): scanner and hence provides ``recursive=False`` semantics. """ __slots__ = ("_child",) - #_child: Matcher[ty.AnyStr] - + + _child: Matcher[ty.AnyStr] + def __init__(self, child: Matcher[ty.AnyStr]): super().__init__(child.is_binary) - - self._child = child # type: Matcher[ty.AnyStr] - + + self._child = child + def should_descend(self, path: ty.AnyStr) -> bool: return False @@ -367,8 +360,9 @@ def should_report(self, path: ty.AnyStr, *, is_dir: bool) -> bool: if ty.TYPE_CHECKING: _match_spec_t = ty.Union[ty.AnyStr, re_pattern_t[ty.AnyStr], Matcher[ty.AnyStr]] -else: # Using `re_pattern_t` here like in the type checking case makes - # sphinx_autodoc_typehints explode # noqa: E114 +else: + # Using `re_pattern_t` here like in the type checking case makes + # sphinx_autodoc_typehints explode _match_spec_t = ty.Union[ty.AnyStr, re_pattern_t, Matcher[ty.AnyStr]] match_spec_t = ty.Union[ ty.Iterable[_match_spec_t[ty.AnyStr]], @@ -388,6 +382,7 @@ def matcher_from_spec(spec: None, *, recursive: bool = ...) -> Matcher[str]: ... + def matcher_from_spec(spec: match_spec_t[ty.AnyStr], *, # type: ignore[misc] # noqa: E302 period_special: bool = True, recursive: bool = True) -> Matcher[ty.AnyStr]: @@ -404,16 +399,20 @@ def matcher_from_spec(spec: match_spec_t[ty.AnyStr], *, # type: ignore[misc] # elif isinstance(spec, (str, bytes)): return GlobMatcher(spec, period_special=period_special) elif isinstance(spec, collections.abc.Iterable) and not isinstance(spec, Matcher): - spec = ty.cast(ty.Iterable[_match_spec_t[ty.AnyStr]], spec) # type: ignore[redundant-cast] - - matchers = [matcher_from_spec(s, # type: ignore[arg-type] # mypy bug - recursive=recursive, period_special=period_special) for s in spec] + matchers: ty.List[Matcher[ty.AnyStr]] = [ + matcher_from_spec( + s, # type: ignore[arg-type] + recursive=recursive, + period_special=period_special) + for s in spec + ] + if len(matchers) == 0: # Edge case: Empty list of matchers - return MATCH_NONE # type: ignore[return-value] + return MatchNone() elif len(matchers) == 1: # Edge case: List of exactly one matcher - return matchers[0] # type: ignore[return-value] # same mypy bug + return matchers[0] else: # Actual list of matchers (plural) - return MetaMatcher(matchers) # type: ignore[arg-type] # same mypy bug + return MetaMatcher(matchers) else: return spec @@ -422,8 +421,8 @@ def matcher_from_spec(spec: match_spec_t[ty.AnyStr], *, # type: ignore[misc] # from .filescanner_ty import FSNodeType, FSNodeEntry else: class FSNodeType(enum.Enum): - FILE = enum_auto() - DIRECTORY = enum_auto() + FILE = enum.auto() + DIRECTORY = enum.auto() FSNodeEntry = ty.NamedTuple("FSNodeEntry", [ ("type", FSNodeType), @@ -436,10 +435,10 @@ class FSNodeType(enum.Enum): class walk(ty.Generator[FSNodeEntry, ty.Any, None], ty.Generic[ty.AnyStr]): __slots__ = ("_generator", "_close_fd") - #_generator: ty.Generator[FSNodeEntry, ty.Any, None] - #_close_fd: ty.Optional[int] - - + + _generator: ty.Generator[FSNodeEntry, None, None] + _close_fd: ty.Optional[int] + def __init__( self, directory: ty.Union[ty.AnyStr, utils.PathLike[ty.AnyStr], int], @@ -482,43 +481,50 @@ def __init__( :class:`NoRecusionAdapterMatcher` and hence prevent the scanner from doing any recursion. """ - self._close_fd = None # type: ty.Optional[int] - + self._close_fd = None + # Create matcher object - matcher = matcher_from_spec( - match_spec, recursive=recursive, period_special=period_special - ) # type: Matcher[ty.AnyStr] # type: ignore[assignment] - + matcher = matcher_from_spec( # type: ignore[type-var] + match_spec, # type: ignore[arg-type] + recursive=recursive, + period_special=period_special + ) + # Convert directory path to string … if isinstance(directory, int): if not HAVE_FWALK: raise NotImplementedError("Passing a file descriptor as directory is " "not supported on this platform") - + self._generator = self._walk( - directory, None, matcher, follow_symlinks, intermediate_dirs - ) # type: ty.Generator[FSNodeEntry, ty.Any, None] + directory, + None, + matcher, # type: ignore[arg-type] + follow_symlinks, + intermediate_dirs + ) else: - #directory_str: ty.AnyStr - if hasattr(os, "fspath"): #PY36+ - directory_str = os.fspath(directory) - elif not ty.TYPE_CHECKING: #PY35 - directory_str = utils.convert_path(directory) - + directory_str = os.fspath(directory) + # Best-effort ensure that target directory exists if it is accessed by path os.stat(directory_str) # … and possibly open it as a FD if this is supported by the platform # # Note: `os.fwalk` support for binary paths was only added in 3.7+. - directory_str_or_fd = directory_str # type: ty.Union[ty.AnyStr, int] + directory_str_or_fd: ty.Union[ty.AnyStr, int] = directory_str if HAVE_FWALK and (not isinstance(directory_str, bytes) or HAVE_FWALK_BYTES): - self._close_fd = directory_str_or_fd = os.open(directory_str, os.O_RDONLY | O_DIRECTORY) - + fd = os.open(directory_str, os.O_RDONLY | O_DIRECTORY) + self._close_fd = directory_str_or_fd = fd + self._generator = self._walk( - directory_str_or_fd, directory_str, matcher, follow_symlinks, intermediate_dirs + directory_str_or_fd, + directory_str, + matcher, # type: ignore[arg-type] + follow_symlinks, + intermediate_dirs ) - + def __iter__(self) -> 'walk[ty.AnyStr]': return self @@ -548,7 +554,8 @@ def throw(self, typ: ty.Union[ty.Type[BaseException], BaseException], tb: ty.Optional[types.TracebackType] = None) -> FSNodeEntry: try: if isinstance(typ, type): - return self._generator.throw(typ, val, tb) + bt = ty.cast(ty.Type[BaseException], typ) # type: ignore[redundant-cast] + return self._generator.throw(bt, val, tb) else: assert val is None return self._generator.throw(typ, val, tb) @@ -595,8 +602,8 @@ def _walk( while directory.endswith(sep): directory = directory[:-len(sep)] prefix = (directory if not isinstance(directory, int) else dot) + sep - - reported_directories = set() # type: ty.Set[ty.AnyStr] + + reported_directories: ty.Set[ty.AnyStr] = set() # Always report the top-level directory even if nothing therein is matched reported_directories.add(utils.maybe_fsencode("", sep)) @@ -616,14 +623,19 @@ def _walk( try: for result in walk_iter: dirpath, dirnames, filenames = result[0:3] - dirfd = result[3] if len(result) > 3 else None # type:ty.Optional[int] # type: ignore[misc] - + + if len(result) <= 3: + dirfd: ty.Optional[int] = None + else: + # mypy wrongly believes this will produce an index-out-of-range exception. + dirfd = result[3] # type: ignore[misc] + # Remove the directory prefix from the received path _, _, dirpath = dirpath.partition(prefix) # Keep track of reported intermediaries, so that we only check for # these at most once per directory base - intermediates_reported = False # type: bool + intermediates_reported = False for filename, is_dir in self._join_dirs_and_files(list(dirnames), filenames): filepath = os.path.join(dirpath, filename) @@ -682,6 +694,6 @@ def _walk( if HAVE_FWALK: # pragma: no cover - supports_fd = frozenset({walk}) # type: ty.FrozenSet[ty.Callable[..., ty.Any]] + supports_fd: ty.FrozenSet[ty.Callable[..., ty.Any]] = frozenset({walk}) else: # pragma: no cover supports_fd = frozenset() diff --git a/ipfshttpclient/http_common.py b/ipfshttpclient/http_common.py index ff44983c..c74a7b11 100644 --- a/ipfshttpclient/http_common.py +++ b/ipfshttpclient/http_common.py @@ -245,10 +245,11 @@ def close(self) -> None: def multiaddr_to_url_data(addr: addr_t, base: str # type: ignore[no-any-unimported] ) -> ty.Tuple[str, ty.Optional[str], socket.AddressFamily, bool]: try: - addr = multiaddr.Multiaddr(addr) + multi_addr = multiaddr.Multiaddr(addr) except multiaddr.exceptions.ParseError as error: raise exceptions.AddressError(addr) from error - addr_iter = iter(addr.items()) + + addr_iter = iter(multi_addr.items()) # Parse the `host`, `family`, `port` & `secure` values from the given # multiaddr, raising on unsupported `addr` values @@ -258,7 +259,8 @@ def multiaddr_to_url_data(addr: addr_t, base: str # type: ignore[no-any-unimpor family = socket.AF_UNSPEC host_numeric = proto.code in (P_IP4, P_IP6) - uds_path = None # type: ty.Optional[str] + uds_path: ty.Optional[str] = None + if proto.code in (P_IP4, P_DNS4): family = socket.AF_INET elif proto.code in (P_IP6, P_DNS6): @@ -578,7 +580,7 @@ def request( # type: ignore[misc] args Positional parameters to be sent along with the HTTP request opts - Query string paramters to be sent along with the HTTP request + Query string parameters to be sent along with the HTTP request offline Whether to request to daemon to handle this request in “offline-mode” return_result @@ -701,7 +703,7 @@ def download( Set this to :py:`math.inf` to disable timeouts entirely. """ - opts2 = dict(opts.items()) # type: ty.Dict[str, str] + opts2: ty.Dict[str, str] = dict(opts.items()) opts2["archive"] = "true" opts2["compress"] = "true" if compress else "false" diff --git a/ipfshttpclient/multipart.py b/ipfshttpclient/multipart.py index c6a99403..f06f95b4 100644 --- a/ipfshttpclient/multipart.py +++ b/ipfshttpclient/multipart.py @@ -122,10 +122,12 @@ class StreamBase(metaclass=abc.ABCMeta): The maximum size that any single file chunk may have in bytes """ __slots__ = ("chunk_size", "name", "_boundary", "_headers") - #chunk_size: int - #name: str - #_boundry: str - #_headers: ty.Dict[str, str] + + chunk_size: int + name: str + + _boundry: str + _headers: ty.Dict[str, str] def __init__(self, name: str, chunk_size: int = default_chunk_size) -> None: self.chunk_size = chunk_size @@ -371,9 +373,10 @@ class DirectoryStream(StreamBase, StreamFileMixin, ty.Generic[ty.AnyStr]): shells allow one to disable this behaviour """ __slots__ = ("abspath", "follow_symlinks", "scanner") - #abspath: ty.Optional[ty.AnyStr] - #follow_symlinks: bool - #scanner: filescanner.walk[ty.AnyStr] + + abspath: ty.Optional[ty.AnyStr] + follow_symlinks: bool + scanner: filescanner.walk[ty.AnyStr] def __init__(self, directory: ty.Union[ty.AnyStr, utils.PathLike[ty.AnyStr], int], *, chunk_size: int = default_chunk_size, @@ -388,18 +391,21 @@ def __init__(self, directory: ty.Union[ty.AnyStr, utils.PathLike[ty.AnyStr], int # Create file scanner from parameters self.scanner = filescanner.walk( - directory, patterns, follow_symlinks=follow_symlinks, - period_special=period_special, recursive=recursive - ) # type: filescanner.walk[ty.AnyStr] - + directory, + patterns, + follow_symlinks=follow_symlinks, + period_special=period_special, + recursive=recursive + ) + # Figure out the absolute path of the directory added - self.abspath = None # type: ty.Optional[ty.AnyStr] + self.abspath = None if not isinstance(directory, int): self.abspath = os.path.abspath(utils.convert_path(directory)) # Figure out basename of the containing directory # (normpath is an acceptable approximation here) - basename = "_" # type: ty.Union[str, bytes] + basename = "_" if not isinstance(directory, int): basename = os.fsdecode(os.path.basename(os.path.normpath(directory))) super().__init__(os.fsdecode(basename), chunk_size=chunk_size) @@ -420,13 +426,13 @@ def _body(self) -> gen_bytes_t: stat_data = os.stat(path, follow_symlinks=self.follow_symlinks) if not stat.S_ISREG(stat_data.st_mode): continue - - absolute_path = None # type: ty.Optional[str] + + absolute_path: ty.Optional[str] = None if self.abspath is not None: absolute_path = os.fsdecode(os.path.join(self.abspath, relpath)) if parentfd is None: - f_path_or_desc = path # type: ty.Union[ty.AnyStr, int] + f_path_or_desc: ty.Union[ty.AnyStr, int] = path else: f_path_or_desc = os.open(name, os.O_RDONLY | os.O_CLOEXEC, dir_fd=parentfd) # Stream file to client @@ -459,14 +465,15 @@ class BytesFileStream(FilesStream): The maximum size of a single data chunk """ __slots__ = ("data",) - #data: ty.Iterable[bytes] + + data: ty.Iterable[bytes] def __init__(self, data: ty.Union[bytes, gen_bytes_t], name: str = "bytes", *, chunk_size: int = default_chunk_size) -> None: super().__init__([], name=name, chunk_size=chunk_size) if not isinstance(data, bytes): - self.data = data # type: ty.Iterable[bytes] + self.data = data else: self.data = (data,) diff --git a/ipfshttpclient/utils.py b/ipfshttpclient/utils.py index bbb886a7..82303a68 100644 --- a/ipfshttpclient/utils.py +++ b/ipfshttpclient/utils.py @@ -2,7 +2,6 @@ """ import mimetypes import os -import pathlib import sys import typing as ty from functools import wraps @@ -12,6 +11,8 @@ else: ty_ext = ty +T = ty.TypeVar("T") + if sys.version_info >= (3, 8): #PY38+ Literal = ty_ext.Literal Protocol = ty_ext.Protocol @@ -19,7 +20,7 @@ Literal_True = ty.Literal[True] Literal_False = ty.Literal[False] else: #PY37- - class Literal(ty.Generic[ty.T]): + class Literal(ty.Generic[T]): ... class Protocol: @@ -27,78 +28,50 @@ class Protocol: Literal_True = Literal_False = bool -if sys.version_info >= (3, 6): #PY36+ - # `os.PathLike` only has a type param while type checking - if ty.TYPE_CHECKING: - PathLike = os.PathLike - PathLike_str = os.PathLike[str] - PathLike_bytes = os.PathLike[bytes] - else: - class PathLike(Protocol, ty.Generic[ty.AnyStr]): - def __fspath__(self) -> ty.AnyStr: - ... - - PathLike_str = PathLike_bytes = os.PathLike - - path_str_t = ty.Union[str, PathLike_str] - path_bytes_t = ty.Union[bytes, PathLike_bytes] - path_t = ty.Union[path_str_t, path_bytes_t] - AnyPath = ty.TypeVar("AnyPath", str, PathLike_str, bytes, PathLike_bytes) - - path_types = (str, bytes, os.PathLike,) - path_obj_types = (os.PathLike,) - - @ty.overload - def convert_path(path: ty.AnyStr) -> ty.AnyStr: - ... - - @ty.overload - def convert_path(path: PathLike_str) -> PathLike_str: - ... - - @ty.overload - def convert_path(path: PathLike_bytes) -> PathLike_bytes: - ... - - @ty.overload - def convert_path(path: AnyPath) -> AnyPath: - ... - - def convert_path(path: AnyPath) -> AnyPath: - # Not needed since all system APIs also accept an `os.PathLike` - return path -else: #PY35 - class PathLike(pathlib.PurePath, ty.Generic[ty.AnyStr]): - ... - - path_str_t = ty.Union[str, pathlib.PurePath] - path_bytes_t = ty.Union[bytes] - path_t = ty.Union[path_str_t, path_bytes_t] - AnyPath = ty.TypeVar("AnyPath", str, pathlib.PurePath, bytes) - - path_types = (str, bytes, pathlib.PurePath,) - path_obj_types = (pathlib.PurePath,) - - # Independently maintained forward-port of `pathlib` for Py27 and others - try: - import pathlib2 - path_types += (pathlib2.PurePath,) - path_obj_types += (pathlib2.PurePath,) - except ImportError: - pass - - @ty.overload - def convert_path(path: path_str_t) -> str: - ... - - @ty.overload - def convert_path(path: path_bytes_t) -> bytes: - ... - - def convert_path(path: path_t) -> ty.Union[str, bytes]: - # `pathlib`'s PathLike objects need to be treated specially and - # converted to strings when interacting with system APIs - return str(path) if isinstance(path, path_obj_types) else path +# `os.PathLike` only has a type param while type checking +if ty.TYPE_CHECKING: + PathLike = os.PathLike + PathLike_str = os.PathLike[str] + PathLike_bytes = os.PathLike[bytes] +else: + class PathLike(Protocol, ty.Generic[ty.AnyStr]): + def __fspath__(self) -> ty.AnyStr: + ... + + PathLike_str = PathLike_bytes = os.PathLike + +path_str_t = ty.Union[str, PathLike_str] +path_bytes_t = ty.Union[bytes, PathLike_bytes] +path_t = ty.Union[path_str_t, path_bytes_t] +AnyPath = ty.TypeVar("AnyPath", str, PathLike_str, bytes, PathLike_bytes) + +path_types = (str, bytes, os.PathLike,) +path_obj_types = (os.PathLike,) + + +@ty.overload +def convert_path(path: ty.AnyStr) -> ty.AnyStr: + ... + + +@ty.overload +def convert_path(path: PathLike_str) -> PathLike_str: + ... + + +@ty.overload +def convert_path(path: PathLike_bytes) -> PathLike_bytes: + ... + + +@ty.overload +def convert_path(path: AnyPath) -> AnyPath: + ... + + +def convert_path(path: AnyPath) -> AnyPath: + # Not needed since all system APIs also accept an `os.PathLike` + return path # work around GH/mypy/mypy#731: no recursive structural types yet @@ -201,8 +174,7 @@ def clean_files(files: ty.Union[clean_file_t, ty.Iterable[clean_file_t]]) \ yield clean_file(ty.cast(clean_file_t, files)) -T = ty.TypeVar("T") -F = ty.TypeVar("F", bound=ty.Callable[..., ty.Dict[str, T]]) +F = ty.TypeVar("F", bound=ty.Callable[..., ty.Dict[str, ty.Any]]) class return_field(ty.Generic[T]): @@ -240,4 +212,4 @@ def wrapper(*args: ty.Any, **kwargs: ty.Any) -> T: """ res = cmd(*args, **kwargs) # type: ty.Dict[str, T] return res[self.field] - return wrapper \ No newline at end of file + return wrapper diff --git a/pyproject.toml b/pyproject.toml index ebb0f8a0..27509e62 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,9 +16,9 @@ description-file = "README.md" # a critical bug (https://bugs.python.org/issue34921) in 3.7.0 to 3.7.1. So the # compatible versions below reflect the range of Python versions with working # `typing.NoReturn` function signature support. (Also, many other `typing` module -# items were only introduced post-release of Python 3.5 and 3.6 and version -# restrictions on these versions ensure that those are all available as well.) -requires-python = ">=3.5.4,!=3.6.0,!=3.6.1,!=3.7.0,!=3.7.1" +# items were only introduced post-release in 3.6 and version restrictions on these +# versions ensure that those are all available as well.) +requires-python = ">=3.6.2,!=3.7.0,!=3.7.1" requires = [ "multiaddr (>=0.0.7)", "requests (>=2.11)" @@ -43,7 +43,6 @@ classifiers = [ # Specify the Python versions you support here. In particular, ensure # that you indicate whether you support Python 2, Python 3 or both. "Programming Language :: Python :: 3 :: Only", - "Programming Language :: Python :: 3.5", "Programming Language :: Python :: 3.6", "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8" diff --git a/test/functional/test_files.py b/test/functional/test_files.py index 9b7b4d5e..bdcd97db 100644 --- a/test/functional/test_files.py +++ b/test/functional/test_files.py @@ -12,7 +12,8 @@ import conftest -O_DIRECTORY = getattr(os, "O_DIRECTORY", 0) # type: int +# Windows does not have os.O_DIRECTORY +O_DIRECTORY: int = getattr(os, "O_DIRECTORY", 0) ### test_add_multiple_from_list @@ -263,7 +264,7 @@ def test_add_filepattern_from_dirname(client, cleanup_pins): reason="No point in disabling os.fwalk if it isn't actually supported") def test_add_filepattern_from_dirname_nofwalk(client, cleanup_pins, monkeypatch): monkeypatch.setattr(ipfshttpclient.filescanner, "HAVE_FWALK", False) - + res = client.add(FAKE_DIR_PATH, pattern=FAKE_DIR_FNPATTERN1) assert conftest.sort_by_key(res) == conftest.sort_by_key(FAKE_DIR_FNPATTERN1_HASH) @@ -271,7 +272,7 @@ def test_add_filepattern_from_dirname_nofwalk(client, cleanup_pins, monkeypatch) @pytest.mark.skipif(not ipfshttpclient.filescanner.HAVE_FWALK, reason="Passing directory as file descriptor requires os.fwalk") def test_add_filepattern_from_dirfd(client, cleanup_pins): - fd = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) # type: int + fd: int = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) try: res = client.add(fd, pattern=FAKE_DIR_FNPATTERN1) finally: @@ -288,7 +289,7 @@ def test_add_filepattern_from_dirname_recursive(client, cleanup_pins): reason="No point in disabling os.fwalk if it isn't actually supported") def test_add_filepattern_from_dirname_recursive_nofwalk(client, cleanup_pins, monkeypatch): monkeypatch.setattr(ipfshttpclient.filescanner, "HAVE_FWALK", False) - + res = client.add(FAKE_DIR_PATH, pattern=FAKE_DIR_FNPATTERN1, recursive=True) assert conftest.sort_by_key(res) == conftest.sort_by_key(FAKE_DIR_FNPATTERN1_RECURSIVE_HASH) @@ -297,19 +298,22 @@ def test_add_filepattern_from_dirname_recursive_nofwalk(client, cleanup_pins, mo reason="Opening directory FDs does not work on Windows") def test_add_filepattern_from_dirfd_recursive_nofwalk(client, cleanup_pins, monkeypatch): monkeypatch.setattr(ipfshttpclient.filescanner, "HAVE_FWALK", False) - - with pytest.raises(NotImplementedError): - fd = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) # type: int - try: + + assert os.path.isdir(FAKE_DIR_PATH) + + # On Windows, this line will fail with PermissionError: [Errno 13] Permission denied + fd: int = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) + try: + with pytest.raises(NotImplementedError): client.add(fd, pattern=FAKE_DIR_FNPATTERN1, recursive=True) - finally: - os.close(fd) + finally: + os.close(fd) @pytest.mark.skipif(not ipfshttpclient.filescanner.HAVE_FWALK, reason="Passing directory as file descriptor requires os.fwalk") def test_add_filepattern_from_dirfd_recursive(client, cleanup_pins): - fd = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) # type: int + fd: int = os.open(str(FAKE_DIR_PATH), os.O_RDONLY | O_DIRECTORY) try: res = client.add(fd, pattern=FAKE_DIR_FNPATTERN1, recursive=True) finally: @@ -327,7 +331,7 @@ def test_add_filepattern_from_dirname_recursive_binary(client, cleanup_pins): reason="No point in disabling os.fwalk if it isn't actually supported") def test_add_filepattern_from_dirname_recursive_nofwalk_binary(client, cleanup_pins, monkeypatch): monkeypatch.setattr(ipfshttpclient.filescanner, "HAVE_FWALK", False) - + res = client.add(os.fsencode(str(FAKE_DIR_PATH)), pattern=os.fsencode(FAKE_DIR_FNPATTERN1), recursive=True) assert conftest.sort_by_key(res) == conftest.sort_by_key(FAKE_DIR_FNPATTERN1_RECURSIVE_HASH) diff --git a/test/unit/test_encoding.py b/test/unit/test_encoding.py index 9f0940a3..7b28b57f 100644 --- a/test/unit/test_encoding.py +++ b/test/unit/test_encoding.py @@ -2,10 +2,11 @@ import json import pytest +import typing as ty import ipfshttpclient.encoding import ipfshttpclient.exceptions - +import ipfshttpclient.utils @pytest.fixture @@ -72,19 +73,35 @@ def test_json_parse_incomplete(json_encoder): def test_json_encode(json_encoder): """Tests serialization of an object into a JSON formatted UTF-8 string.""" - data = {'key': 'value with Ünicøde characters ☺'} + + data = ty.cast( + ipfshttpclient.utils.json_dict_t, + {'key': 'value with Ünicøde characters ☺'} + ) + assert json_encoder.encode(data) == \ b'{"key":"value with \xc3\x9cnic\xc3\xb8de characters \xe2\x98\xba"}' + def test_json_encode_invalid_surrogate(json_encoder): """Tests serialization of an object into a JSON formatted UTF-8 string.""" - data = {'key': 'value with Ünicøde characters and disallowed surrgate: \uDC00'} + + data = ty.cast( + ipfshttpclient.utils.json_dict_t, + {'key': 'value with Ünicøde characters and disallowed surrgate: \uDC00'} + ) with pytest.raises(ipfshttpclient.exceptions.EncodingError): json_encoder.encode(data) + def test_json_encode_invalid_type(json_encoder): """Tests serialization of an object into a JSON formatted UTF-8 string.""" - data = {'key': b'value that is not JSON encodable'} + + data = ty.cast( + ipfshttpclient.utils.json_dict_t, + {'key': b'value that is not JSON encodable'} + ) + with pytest.raises(ipfshttpclient.exceptions.EncodingError): json_encoder.encode(data) diff --git a/test/unit/test_multipart.py b/test/unit/test_multipart.py index 957b5058..0723ac08 100644 --- a/test/unit/test_multipart.py +++ b/test/unit/test_multipart.py @@ -13,6 +13,7 @@ import io import os import re +import typing as ty import unittest import urllib.parse @@ -216,14 +217,17 @@ def do_test__gen_file(self, name, file_location, abspath): def test__gen_file(self): self.do_test__gen_file("functional/fake_dir/fsdfgh", file_location=None, abspath=False) + def test__gen_file_relative(self): filepath = "functional/fake_dir/fsdfgh" self.do_test__gen_file(filepath, filepath, abspath=False) + def test__gen_file_absolute(self): filepath = "/functional/fake_dir/fsdfgh" self.do_test__gen_file(filepath, filepath, abspath=True) - def do_test__gen_file_start(self, name, file_location, abspath): + @staticmethod + def do_test__gen_file_start(name: str, file_location: ty.Optional[str], abspath: bool): """Test the _gen_file_start function against sample output.""" generator = StreamFileMixinSub(name) @@ -238,6 +242,7 @@ def do_test__gen_file_start(self, name, file_location, abspath): def test__gen_file_start(self): self.do_test__gen_file_start("test_name", file_location=None, abspath=False) + def test__gen_file_start_with_filepath(self): name = "test_name" self.do_test__gen_file_start(name, os.path.join(os.path.sep, name), abspath=True) diff --git a/tox.ini b/tox.ini index 2146c598..ccdb42f0 100644 --- a/tox.ini +++ b/tox.ini @@ -34,14 +34,6 @@ setenv = PYTHONPATH = -[testenv:py35] -# Fix missing dependencies when checking Python 3.5 -deps = - {[testenv]deps} - multiaddr (>=0.0.7) - requests (>=2.11) - - [testenv:py3-httpx] deps-exclusive = httpx (~= 0.14.0) @@ -107,7 +99,7 @@ exclude = .git,.tox,+junk,coverage,dist,doc,*egg,build,tools,test/unit,docs,*__i # E722: Using bare except for cleanup-on-error is fine # (see bug report at https://github.com/PyCQA/pycodestyle/issues/703) # W292: No newline at end of file -# W391: Blank line at end of file (sometimes trigged instead of the above!?) +# W391: Blank line at end of file (sometimes triggered instead of the above!?) # F403: `from import *` used; unable to detect undefined names ←– Probably should be fixed… # F811: PyFlakes bug: `@ty.overload` annotation is not detected to mean `@typing.overload` # (see bug report at https://github.com/PyCQA/pyflakes/issues/561)