Skip to content

Commit

Permalink
Fix the detection of the last chunk
Browse files Browse the repository at this point in the history
  • Loading branch information
josephnowak committed Sep 30, 2024
1 parent 23a864a commit 1825af3
Show file tree
Hide file tree
Showing 2 changed files with 14 additions and 5 deletions.
9 changes: 4 additions & 5 deletions xarray/backends/zarr.py
Original file line number Diff line number Diff line change
Expand Up @@ -229,15 +229,14 @@ def _determine_zarr_chunks(
raise ValueError(base_error)

if not allow_partial_chunks:
chunk_start = sum(dchunks[:-1]) + region_start
if chunk_start % zchunk:
region_stop = interval.stop if interval.stop else size

if region_start % zchunk:
# The last chunk which can also be the only one is a partial chunk
# if it is not aligned at the beginning
raise ValueError(base_error)

region_stop = interval.stop if interval.stop else size

if size - region_stop + 1 < zchunk:
if np.ceil(region_stop / zchunk) == np.ceil(size / zchunk):
# If the region is covering the last chunk then check
# if the reminder with the default chunk size
# is equal to the size of the last chunk
Expand Down
10 changes: 10 additions & 0 deletions xarray/tests/test_backends.py
Original file line number Diff line number Diff line change
Expand Up @@ -6256,3 +6256,13 @@ def test_zarr_safe_chunk_region(tmp_path):

with pytest.raises(ValueError):
arr.isel(a=slice(5, -1)).chunk(a=5).to_zarr(store, region="auto", mode="r+")

# Test if the code is detecting the last chunk correctly
data = np.random.RandomState(0).randn(2920, 25, 53)
ds = xr.Dataset({'temperature': (('time', 'lat', 'lon'), data)})
chunks = {'time': 1000, 'lat': 25, 'lon': 53}
ds.chunk(chunks).to_zarr(store, compute=False)
region = {'time': slice(1000, 2000, 1)}
chunk = ds.isel(region)
chunk = chunk.chunk()
chunk.chunk().to_zarr(store, region=region)

0 comments on commit 1825af3

Please sign in to comment.