open_virtual_dataset with dmr++

virtualizarr/manifests/manifest.py

@@ -71,8 +71,8 @@ class ChunkManifest:
     """
 
     _paths: np.ndarray[Any, np.dtypes.StringDType]  # type: ignore[name-defined]
-    _offsets: np.ndarray[Any, np.dtype[np.int32]]
-    _lengths: np.ndarray[Any, np.dtype[np.int32]]
+    _offsets: np.ndarray[Any, np.dtype[np.int64]]
+    _lengths: np.ndarray[Any, np.dtype[np.int64]]
 
     def __init__(self, entries: dict) -> None:
         """
@@ -100,8 +100,8 @@ def __init__(self, entries: dict) -> None:
 
         # Initializing to empty implies that entries with path='' are treated as missing chunks
         paths = np.empty(shape=shape, dtype=np.dtypes.StringDType())  # type: ignore[attr-defined]
-        offsets = np.empty(shape=shape, dtype=np.dtype("int32"))
-        lengths = np.empty(shape=shape, dtype=np.dtype("int32"))
+        offsets = np.empty(shape=shape, dtype=np.dtype("int64"))
+        lengths = np.empty(shape=shape, dtype=np.dtype("int64"))
 
         # populate the arrays
         for key, entry in entries.items():
@@ -128,8 +128,8 @@ def __init__(self, entries: dict) -> None:
     def from_arrays(
         cls,
         paths: np.ndarray[Any, np.dtype[np.dtypes.StringDType]],  # type: ignore[name-defined]
-        offsets: np.ndarray[Any, np.dtype[np.int32]],
-        lengths: np.ndarray[Any, np.dtype[np.int32]],
+        offsets: np.ndarray[Any, np.dtype[np.int64]],
+        lengths: np.ndarray[Any, np.dtype[np.int64]],
     ) -> "ChunkManifest":
         """
         Create manifest directly from numpy arrays containing the path and byte range information.
@@ -161,11 +161,11 @@ def from_arrays(
             raise ValueError(
                 f"paths array must have a numpy variable-length string dtype, but got dtype {paths.dtype}"
             )
-        if offsets.dtype != np.dtype("int32"):
+        if offsets.dtype != np.dtype("int64"):
             raise ValueError(
                 f"offsets array must have 32-bit integer dtype, but got dtype {offsets.dtype}"
             )
-        if lengths.dtype != np.dtype("int32"):
+        if lengths.dtype != np.dtype("int64"):
             raise ValueError(
                 f"lengths array must have 32-bit integer dtype, but got dtype {lengths.dtype}"
             )