diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index 5d68c53a..f4bd16d4 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -7,13 +7,13 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.7, 3.8, 3.9, "3.10", "3.11-dev"] os: [ubuntu-latest, windows-latest] steps: - - uses: actions/checkout@v2 + - uses: actions/checkout@v3 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 + uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/noxfile.py b/noxfile.py index cee93c14..7be17355 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1,3 +1,5 @@ +import sys + import nox from pathlib import Path @@ -17,7 +19,7 @@ def black(session): @nox.session() def lint(session): session.install("flake8") - session.run(*"flake8".split()) + session.run("flake8", "upath") @nox.session() @@ -27,6 +29,12 @@ def install(session): @nox.session() def smoke(session): + if (3, 10) < sys.version_info <= (3, 11, 0, "final"): + # workaround for missing aiohttp wheels for py3.11 + session.install("aiohttp", "--no-binary", "aiohttp", env={ + "AIOHTTP_NO_EXTENSIONS": "1" + }) + session.install( "pytest", "adlfs", diff --git a/setup.py b/setup.py index e8d9bdb8..485fed85 100644 --- a/setup.py +++ b/setup.py @@ -16,5 +16,5 @@ description="pathlib api extended to use fsspec backends", long_description=long_description, long_description_content_type="text/markdown", - license="MIT" + license="MIT", ) diff --git a/upath/core.py b/upath/core.py index 058643d4..34af7243 100644 --- a/upath/core.py +++ b/upath/core.py @@ -29,7 +29,7 @@ def __init__(self, parsed_url: ParseResult, **kwargs): def _format_path(self, path: "UPath") -> str: return path.path - def open(self, path, mode='r', *args, **kwargs): + def open(self, path, mode="r", *args, **kwargs): return self._fs.open(self._format_path(path), mode, *args, **kwargs) def stat(self, path, **kwargs): @@ -185,6 +185,9 @@ def parent(self): def stat(self): return self._accessor.stat(self) + def samefile(self, other_path): + raise NotImplementedError + def iterdir(self): """Iterate over the files in this directory. Does not yield any result for the special paths '.' and '..'. @@ -223,6 +226,10 @@ def relative_to(self, *other): output._kwargs = self._kwargs return output + def _scandir(self): + # provided in Python3.11 but not required in fsspec glob implementation + raise NotImplementedError + def glob(self, pattern): path_pattern = self.joinpath(pattern) for name in self._accessor.glob(self, path_pattern): @@ -230,11 +237,25 @@ def glob(self, pattern): name = name.split(self._flavour.sep) yield self._make_child(name) + def rglob(self, pattern): + path_pattern = self.joinpath("**", pattern) + for name in self._accessor.glob(self, path_pattern): + name = self._sub_path(name) + name = name.split(self._flavour.sep) + yield self._make_child(name) + def _sub_path(self, name): # only want the path name with iterdir sp = self.path return re.sub(f"^({sp}|{sp[1:]})/", "", name) + def absolute(self): + # fsspec paths are always absolute + return self + + def resolve(self, strict=False): + raise NotImplementedError + def exists(self): """ Whether this path exists. @@ -290,6 +311,9 @@ def is_block_device(self): def is_char_device(self): return False + def is_absolute(self): + return True + def unlink(self, missing_ok=False): if not self.exists(): if not missing_ok: @@ -308,13 +332,25 @@ def rmdir(self, recursive=True): raise NotDirectoryError self._accessor.rm(self, recursive=recursive) - def chmod(self, mod): + def chmod(self, mode, *, follow_symlinks=True): raise NotImplementedError def rename(self, target): # can be implemented, but may be tricky raise NotImplementedError + def replace(self, target): + raise NotImplementedError + + def symlink_to(self, target, target_is_directory=False): + raise NotImplementedError + + def hardlink_to(self, target): + raise NotImplementedError + + def link_to(self, target): + raise NotImplementedError + def cwd(self): raise NotImplementedError @@ -342,6 +378,28 @@ def readlink(self): def touch(self, truncate=True, **kwargs): self._accessor.touch(self, truncate=truncate, **kwargs) + def mkdir(self, mode=0o777, parents=False, exist_ok=False): + """ + Create a new directory at this given path. + """ + if parents: + self._accessor.mkdir( + self, + create_parents=True, + exist_ok=exist_ok, + mode=mode, + ) + else: + try: + self._accessor.mkdir( + self, + create_parents=False, + mode=mode, + ) + except FileExistsError: + if not exist_ok or not self.is_dir(): + raise + @classmethod def _from_parts(cls, args, url=None, **kwargs): obj = object.__new__(cls) @@ -426,7 +484,7 @@ def with_suffix(self, suffix): f = self._flavour if f.sep in suffix or f.altsep and f.altsep in suffix: raise ValueError("Invalid suffix %r" % (suffix,)) - if suffix and not suffix.startswith('.') or suffix == '.': + if suffix and not suffix.startswith(".") or suffix == ".": raise ValueError("Invalid suffix %r" % (suffix)) name = self.name if not name: @@ -435,17 +493,24 @@ def with_suffix(self, suffix): if not old_suffix: name = name + suffix else: - name = name[:-len(old_suffix)] + suffix - return self._from_parsed_parts(self._drv, self._root, - self._parts[:-1] + [name], url=self._url) + name = name[: -len(old_suffix)] + suffix + return self._from_parsed_parts( + self._drv, self._root, self._parts[:-1] + [name], url=self._url + ) def with_name(self, name): """Return a new path with the file name changed.""" if not self.name: raise ValueError("%r has an empty name" % (self,)) drv, root, parts = self._flavour.parse_parts((name,)) - if (not name or name[-1] in [self._flavour.sep, self._flavour.altsep] - or drv or root or len(parts) != 1): + if ( + not name + or name[-1] in [self._flavour.sep, self._flavour.altsep] + or drv + or root + or len(parts) != 1 + ): raise ValueError("Invalid name %r" % (name)) - return self._from_parsed_parts(self._drv, self._root, - self._parts[:-1] + [name], url=self._url) + return self._from_parsed_parts( + self._drv, self._root, self._parts[:-1] + [name], url=self._url + ) diff --git a/upath/implementations/cloud.py b/upath/implementations/cloud.py index c482e18a..f4ac26bc 100644 --- a/upath/implementations/cloud.py +++ b/upath/implementations/cloud.py @@ -9,6 +9,15 @@ def _format_path(self, path): """ return f"{path._url.netloc}/{path.path.lstrip('/')}" + def mkdir(self, path, create_parents=True, **kwargs): + if ( + not create_parents + and not kwargs.get("exist_ok", False) + and self._fs.exists(self._format_path(path)) + ): + raise FileExistsError + return super().mkdir(path, create_parents=create_parents, **kwargs) + # project is not part of the path, but is part of the credentials class CloudPath(upath.core.UPath): diff --git a/upath/implementations/hdfs.py b/upath/implementations/hdfs.py index 84b29bd5..eeda435b 100644 --- a/upath/implementations/hdfs.py +++ b/upath/implementations/hdfs.py @@ -10,6 +10,15 @@ def touch(self, path, **kwargs): kwargs.pop("truncate", None) super().touch(path, **kwargs) + def mkdir(self, path, create_parents=True, **kwargs): + pth = self._format_path(path) + if create_parents: + return self._fs.makedirs(pth, **kwargs) + else: + if not kwargs.get("exist_ok", False) and self._fs.exists(pth): + raise FileExistsError + return self._fs.mkdir(pth, create_parents=create_parents, **kwargs) + class HDFSPath(upath.core.UPath): _default_accessor = _HDFSAccessor diff --git a/upath/tests/cases.py b/upath/tests/cases.py index ffd90117..0ddb22cf 100644 --- a/upath/tests/cases.py +++ b/upath/tests/cases.py @@ -9,6 +9,8 @@ class BaseTests: + SUPPORTS_EMPTY_DIRS = True + def test_cwd(self): with pytest.raises(NotImplementedError): self.path.cwd() @@ -124,8 +126,25 @@ def test_lstat(self): def test_mkdir(self): new_dir = self.path.joinpath("new_dir") new_dir.mkdir() + if not self.SUPPORTS_EMPTY_DIRS: + new_dir.joinpath(".file").touch() assert new_dir.exists() + def test_mkdir_exists_ok_true(self): + new_dir = self.path.joinpath("new_dir_may_exists") + new_dir.mkdir() + if not self.SUPPORTS_EMPTY_DIRS: + new_dir.joinpath(".file").touch() + new_dir.mkdir(exist_ok=True) + + def test_mkdir_exists_ok_false(self): + new_dir = self.path.joinpath("new_dir_may_not_exists") + new_dir.mkdir() + if not self.SUPPORTS_EMPTY_DIRS: + new_dir.joinpath(".file").touch() + with pytest.raises(FileExistsError): + new_dir.mkdir(exist_ok=False) + def test_open(self): pass diff --git a/upath/tests/implementations/test_azure.py b/upath/tests/implementations/test_azure.py index f5fd293f..f4c33307 100644 --- a/upath/tests/implementations/test_azure.py +++ b/upath/tests/implementations/test_azure.py @@ -10,6 +10,8 @@ @skip_on_windows @pytest.mark.usefixtures("path") class TestAzurePath(BaseTests): + SUPPORTS_EMPTY_DIRS = False + @pytest.fixture(autouse=True, scope="function") def path(self, azurite_credentials, azure_fixture): account_name, connection_string = azurite_credentials @@ -24,14 +26,8 @@ def path(self, azurite_credentials, azure_fixture): def test_is_AzurePath(self): assert isinstance(self.path, AzurePath) - def test_mkdir(self): - new_dir = self.path / "new_dir" - new_dir.mkdir() - (new_dir / "test.txt").touch() - assert new_dir.exists() - def test_rmdir(self): - new_dir = self.path / "new_dir" + new_dir = self.path / "new_dir_rmdir" new_dir.mkdir() path = new_dir / "test.txt" path.write_text("hello") diff --git a/upath/tests/implementations/test_gcs.py b/upath/tests/implementations/test_gcs.py index e14ecb5f..99e189ed 100644 --- a/upath/tests/implementations/test_gcs.py +++ b/upath/tests/implementations/test_gcs.py @@ -10,6 +10,8 @@ @skip_on_windows @pytest.mark.usefixtures("path") class TestGCSPath(BaseTests): + SUPPORTS_EMPTY_DIRS = False + @pytest.fixture(autouse=True, scope="function") def path(self, gcs_fixture): path, endpoint_url = gcs_fixture @@ -19,11 +21,6 @@ def path(self, gcs_fixture): def test_is_GCSPath(self): assert isinstance(self.path, GCSPath) - def test_mkdir(self): - new_dir = self.path.joinpath("new_dir") - new_dir.joinpath("test.txt").touch() - assert new_dir.exists() - def test_rmdir(self): dirname = "rmdir_test" mock_dir = self.path.joinpath(dirname) diff --git a/upath/tests/implementations/test_http.py b/upath/tests/implementations/test_http.py index 8c7a217c..ed370145 100644 --- a/upath/tests/implementations/test_http.py +++ b/upath/tests/implementations/test_http.py @@ -34,15 +34,27 @@ def path(self, http_fixture): def test_work_at_root(self): assert "folder" in (f.name for f in self.path.parent.iterdir()) + @pytest.mark.skip def test_mkdir(self): pass + @pytest.mark.skip + def test_mkdir_exists_ok_false(self): + pass + + @pytest.mark.skip + def test_mkdir_exists_ok_true(self): + pass + + @pytest.mark.skip def test_touch_unlink(self): pass + @pytest.mark.skip def test_write_bytes(self, pathlib_base): pass + @pytest.mark.skip def test_write_text(self, pathlib_base): pass diff --git a/upath/tests/implementations/test_s3.py b/upath/tests/implementations/test_s3.py index e24360eb..8e10bcf0 100644 --- a/upath/tests/implementations/test_s3.py +++ b/upath/tests/implementations/test_s3.py @@ -9,6 +9,8 @@ class TestUPathS3(BaseTests): + SUPPORTS_EMPTY_DIRS = False + @pytest.fixture(autouse=True) def path(self, s3_fixture): path, anon, s3so = s3_fixture @@ -23,14 +25,6 @@ def test_chmod(self): # todo pass - def test_mkdir(self): - new_dir = self.path.joinpath("new_dir") - # new_dir.mkdir() - # mkdir doesn't really do anything. A directory only exists in s3 - # if some file or something is written to it - new_dir.joinpath("test.txt").touch() - assert new_dir.exists() - def test_rmdir(self): dirname = "rmdir_test" mock_dir = self.path.joinpath(dirname)