From 367e22cf3be884d5c80552708e36c4bd384dbbde Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Wed, 9 Dec 2020 15:47:40 +0100 Subject: [PATCH 1/3] add encodings["preferred_chunks"], used in open_dataset instead of encodings["chunks"] --- xarray/backends/zarr.py | 1 + xarray/core/dataset.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index abf0c7372eb..77abdb2ef5b 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -311,6 +311,7 @@ def open_store_variable(self, name, zarr_array): attributes = dict(attributes) encoding = { "chunks": zarr_array.chunks, + "preferred_chunks": zarr_array.chunks, "compressor": zarr_array.compressor, "filters": zarr_array.filters, } diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 0a3c969c617..42b43f802db 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -390,8 +390,8 @@ def _get_chunk(var, chunks): if isinstance(chunks, int) or (chunks == "auto"): chunks = dict.fromkeys(var.dims, chunks) - preferred_chunks_list = var.encoding.get("chunks", {}) - preferred_chunks = dict(zip(var.dims, var.encoding.get("chunks", {}))) + preferred_chunks_list = var.encoding.get("preferred_chunks", {}) + preferred_chunks = dict(zip(var.dims, preferred_chunks_list)) chunks_list = [ chunks.get(dim, None) or preferred_chunks.get(dim, None) for dim in var.dims From 46dbf4012f5d079edf3d88ead8b0c40726996fa8 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Thu, 10 Dec 2020 15:25:16 +0100 Subject: [PATCH 2/3] modify preferred_chunks: now it's a dictionary not a list --- xarray/backends/zarr.py | 2 +- xarray/core/dataset.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/xarray/backends/zarr.py b/xarray/backends/zarr.py index 77abdb2ef5b..82f79efaaee 100644 --- a/xarray/backends/zarr.py +++ b/xarray/backends/zarr.py @@ -311,7 +311,7 @@ def open_store_variable(self, name, zarr_array): attributes = dict(attributes) encoding = { "chunks": zarr_array.chunks, - "preferred_chunks": zarr_array.chunks, + "preferred_chunks": dict(zip(dimensions, zarr_array.chunks)), "compressor": zarr_array.compressor, "filters": zarr_array.filters, } diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 42b43f802db..91a90950aa6 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -390,8 +390,8 @@ def _get_chunk(var, chunks): if isinstance(chunks, int) or (chunks == "auto"): chunks = dict.fromkeys(var.dims, chunks) - preferred_chunks_list = var.encoding.get("preferred_chunks", {}) - preferred_chunks = dict(zip(var.dims, preferred_chunks_list)) + preferred_chunks = var.encoding.get("preferred_chunks", {}) + preferred_chunks_list = [preferred_chunks.get(dim, None) for dim in var.dims] chunks_list = [ chunks.get(dim, None) or preferred_chunks.get(dim, None) for dim in var.dims From b926946774f622d40f7ed86b0384ea7c8f5b7ef8 Mon Sep 17 00:00:00 2001 From: Aureliana Barghini Date: Fri, 11 Dec 2020 10:42:37 +0100 Subject: [PATCH 3/3] fix if preferred_chunks is not defined --- xarray/core/dataset.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/xarray/core/dataset.py b/xarray/core/dataset.py index 91a90950aa6..cc8c14dda68 100644 --- a/xarray/core/dataset.py +++ b/xarray/core/dataset.py @@ -391,7 +391,9 @@ def _get_chunk(var, chunks): chunks = dict.fromkeys(var.dims, chunks) preferred_chunks = var.encoding.get("preferred_chunks", {}) - preferred_chunks_list = [preferred_chunks.get(dim, None) for dim in var.dims] + preferred_chunks_list = [ + preferred_chunks.get(dim, shape) for dim, shape in zip(var.dims, var.shape) + ] chunks_list = [ chunks.get(dim, None) or preferred_chunks.get(dim, None) for dim in var.dims