-
-
Notifications
You must be signed in to change notification settings - Fork 335
Default to RemoteStore for fsspec URIs #2198
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from all commits
f81364b
64b9371
72fb559
205807e
69b6cda
0ea04af
c8535af
fe089fc
4db6386
940084c
41b2be5
8bbc508
1610371
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -11,6 +11,8 @@ | |
from zarr.store.local import LocalStore | ||
from zarr.store.memory import MemoryStore | ||
|
||
# from zarr.store.remote import RemoteStore | ||
|
||
if TYPE_CHECKING: | ||
from zarr.core.buffer import BufferPrototype | ||
from zarr.core.common import AccessModeLiteral | ||
|
@@ -75,30 +77,69 @@ def __eq__(self, other: Any) -> bool: | |
|
||
|
||
async def make_store_path( | ||
store_like: StoreLike | None, *, mode: AccessModeLiteral | None = None | ||
store_like: StoreLike | None, | ||
*, | ||
mode: AccessModeLiteral | None = None, | ||
storage_options: dict[str, Any] | None = None, | ||
) -> StorePath: | ||
from zarr.store.remote import RemoteStore # circular import | ||
|
||
used_storage_options = False | ||
|
||
if isinstance(store_like, StorePath): | ||
if mode is not None: | ||
assert AccessMode.from_literal(mode) == store_like.store.mode | ||
return store_like | ||
result = store_like | ||
elif isinstance(store_like, Store): | ||
if mode is not None: | ||
assert AccessMode.from_literal(mode) == store_like.mode | ||
await store_like._ensure_open() | ||
return StorePath(store_like) | ||
result = StorePath(store_like) | ||
elif store_like is None: | ||
if mode is None: | ||
mode = "w" # exception to the default mode = 'r' | ||
return StorePath(await MemoryStore.open(mode=mode)) | ||
result = StorePath(await MemoryStore.open(mode=mode)) | ||
elif isinstance(store_like, Path): | ||
return StorePath(await LocalStore.open(root=store_like, mode=mode or "r")) | ||
result = StorePath(await LocalStore.open(root=store_like, mode=mode or "r")) | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. we can pass There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. could be missing something, but I don't think that'll work. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I see what you mean. I was thinking |
||
elif isinstance(store_like, str): | ||
return StorePath(await LocalStore.open(root=Path(store_like), mode=mode or "r")) | ||
storage_options = storage_options or {} | ||
|
||
if _is_fsspec_uri(store_like): | ||
used_storage_options = True | ||
result = StorePath( | ||
RemoteStore.from_url(store_like, storage_options=storage_options, mode=mode or "r") | ||
) | ||
else: | ||
result = StorePath(await LocalStore.open(root=Path(store_like), mode=mode or "r")) | ||
elif isinstance(store_like, dict): | ||
# We deliberate only consider dict[str, Buffer] here, and not arbitrary mutable mappings. | ||
# By only allowing dictionaries, which are in-memory, we know that MemoryStore appropriate. | ||
return StorePath(await MemoryStore.open(store_dict=store_like, mode=mode)) | ||
raise TypeError | ||
result = StorePath(await MemoryStore.open(store_dict=store_like, mode=mode)) | ||
else: | ||
msg = f"Unsupported type for store_like: '{type(store_like).__name__}'" # type: ignore[unreachable] | ||
raise TypeError(msg) | ||
|
||
if storage_options and not used_storage_options: | ||
msg = "'storage_options' was provided but unused. 'storage_options' is only used for fsspec filesystem stores." | ||
raise TypeError(msg) | ||
|
||
return result | ||
|
||
|
||
def _is_fsspec_uri(uri: str) -> bool: | ||
""" | ||
Check if a URI looks like a non-local fsspec URI. | ||
|
||
Examples | ||
-------- | ||
>>> _is_fsspec_uri("s3://bucket") | ||
True | ||
>>> _is_fsspec_uri("my-directory") | ||
False | ||
>>> _is_fsspec_uri("local://my-directory") | ||
False | ||
""" | ||
return "://" in uri or "::" in uri and "local://" not in uri | ||
|
||
|
||
async def ensure_no_existing_node(store_path: StorePath, zarr_format: ZarrFormat) -> None: | ||
|
Uh oh!
There was an error while loading. Please reload this page.