From 79527056aec24a04b0063c09ffe217b29f6c0141 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 8 Nov 2019 17:30:10 -0500 Subject: [PATCH 1/6] Add `_fromfile` and `_tofile` methods To make it easier to sub out different ways of reading and writing data, create to methods to handle reading and writing directly. This way users don't need to reimplement all of the same logic that `__getitem__` and `__setitem__` are doing and can focus on simply reading and writing. Can be useful for trying memmaping for example. --- zarr/storage.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/zarr/storage.py b/zarr/storage.py index b2ed796580..f8b0381222 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -746,6 +746,14 @@ def __init__(self, path, normalize_keys=False): def _normalize_key(self, key): return key.lower() if self.normalize_keys else key + def _fromfile(self, fn): + with open(filepath, 'rb') as f: + return f.read() + + def _tofile(self, a, fn): + with open(fn, mode='wb') as f: + return f.write(a) + def __getitem__(self, key): key = self._normalize_key(key) filepath = os.path.join(self.path, key) From 58440f8e6faff4a5e690c3d02d623ca55faba2e9 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Fri, 8 Nov 2019 17:30:38 -0500 Subject: [PATCH 2/6] Use _fromfile/_tofile in __getitem__/__set item__ --- zarr/storage.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/zarr/storage.py b/zarr/storage.py index f8b0381222..7d0443eb0f 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -758,8 +758,7 @@ def __getitem__(self, key): key = self._normalize_key(key) filepath = os.path.join(self.path, key) if os.path.isfile(filepath): - with open(filepath, 'rb') as f: - return f.read() + return self._fromfile(filepath) else: raise KeyError(key) @@ -792,8 +791,7 @@ def __setitem__(self, key, value): temp_name = file_name + '.' + uuid.uuid4().hex + '.partial' temp_path = os.path.join(dir_path, temp_name) try: - with open(temp_path, mode='wb') as f: - f.write(value) + self._tofile(value, temp_path) # move temporary file into place replace(temp_path, file_path) From fd7dda53d6d707d2abb0689e65f4b8e38e982188 Mon Sep 17 00:00:00 2001 From: jakirkham Date: Fri, 8 Nov 2019 17:56:59 -0500 Subject: [PATCH 3/6] Update zarr/storage.py Co-Authored-By: James Bourbeau --- zarr/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/storage.py b/zarr/storage.py index 7d0443eb0f..459e36a75a 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -747,7 +747,7 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def _fromfile(self, fn): - with open(filepath, 'rb') as f: + with open(fn, 'rb') as f: return f.read() def _tofile(self, a, fn): From 3ab2811c3fc2c6bb15afcb221dbce4465d4d11a2 Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 11 Nov 2019 01:12:11 -0500 Subject: [PATCH 4/6] Add docstrings to `_fromfile` and `_tofile` --- zarr/storage.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/zarr/storage.py b/zarr/storage.py index 459e36a75a..60fc0a7c65 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -747,10 +747,37 @@ def _normalize_key(self, key): return key.lower() if self.normalize_keys else key def _fromfile(self, fn): + """ Read data from a file + + Parameters + ---------- + fn: str + Filepath to open and read from. + + Notes + ----- + Subclasses should overload this method to specify any custom + file reading logic. + """ with open(fn, 'rb') as f: return f.read() def _tofile(self, a, fn): + """ Write data to a file + + Parameters + ---------- + a: array-like + Data to write into the file. + + fn: str + Filepath to open and write to. + + Notes + ----- + Subclasses should overload this method to specify any custom + file writing logic. + """ with open(fn, mode='wb') as f: return f.write(a) From ead6ad781f84e74c7ea1bc499b866220e8e30ace Mon Sep 17 00:00:00 2001 From: John Kirkham Date: Mon, 11 Nov 2019 12:05:18 -0500 Subject: [PATCH 5/6] No need to return in `_tofile` --- zarr/storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zarr/storage.py b/zarr/storage.py index 60fc0a7c65..5a33940ad5 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -779,7 +779,7 @@ def _tofile(self, a, fn): file writing logic. """ with open(fn, mode='wb') as f: - return f.write(a) + f.write(a) def __getitem__(self, key): key = self._normalize_key(key) From 83236028fb5e9a5bc0127dcb8efd51c927195eca Mon Sep 17 00:00:00 2001 From: James Bourbeau Date: Mon, 11 Nov 2019 20:31:12 -0600 Subject: [PATCH 6/6] Adds release notes entry --- docs/release.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/release.rst b/docs/release.rst index 0aad67ef8a..42297edc3e 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -40,6 +40,9 @@ Upcoming Release * Use ``math.ceil`` for scalars. By :user:`John Kirkham `; :issue:`500`. +* Refactor out ``_tofile``/``_fromfile`` from ``DirectoryStore``. + By :user:`John Kirkham `; :issue:`503`. + .. _release_2.3.2: 2.3.2