Active storage

Changelog.rst

@@ -3,6 +3,12 @@ version NEXT
 
 **2024-??-??**
 
+* Allow access to netCDF-4 files in S3 object stores
+  (https://github.com/NCAS-CMS/cf-python/issues/712)
+* New class `cf.H5netcdfArray`
+* New class `cf.NetCDF4Array`
+* New class `cf.CFAH5netcdfArray`
+* New class `cf.CFANetCDF4Array`
 * Added spherical regridding to discrete sampling geometry destination
   grids (https://github.com/NCAS-CMS/cf-python/issues/716)
 * Added 3-d spherical regridding to `cf.Field.regrids`, and the option
@@ -15,6 +21,11 @@ version NEXT
 * Fix bug in `cf.read` when reading UM files that caused LBPROC value
   131072 (Mean over an ensemble of parallel runs) to be ignored
   (https://github.com/NCAS-CMS/cf-python/issues/737)
+* New dependency: ``h5netcdf>=1.3.0``
+* New dependency: ``h5py>=3.10.0``
+* New dependency: ``s3fs>=2024.2.0``
+* Changed dependency: ``1.11.2.0<=cfdm<1.11.3.0``
+* Changed dependency: ``cfunits>=3.3.7``
 
 ----
 
@@ -75,6 +86,8 @@ version 3.16.0
 * Changed dependency: ``1.11.0.0<=cfdm<1.11.1.0``
 * New dependency: ``scipy>=1.10.0``
 
+----
+
 version 3.15.4
 --------------
 
@@ -213,7 +226,11 @@ version 3.14.1
 
 ----
 
+<<<<<<< HEAD
+version 3.14.0 (*first Dask version*)
+=======
 version 3.14.0 (*first Dask release*)
+>>>>>>> cafeee513aa48788978d65a96d57212d25549a55
 -------------------------------------
 
 **2023-01-31**
@@ -248,7 +265,11 @@ version 3.14.0 (*first Dask release*)
 
 ----
 
+<<<<<<< HEAD
+version 3.13.1 (*last LAMA version*)
+=======
 version 3.13.1 (*last LAMA release*)
+>>>>>>> cafeee513aa48788978d65a96d57212d25549a55
 ------------------------------------
 
 **2022-10-17**

cf/__init__.py

@@ -13,10 +13,17 @@
 
 * read field constructs from netCDF, CDL, PP and UM datasets,
 
+* read field constructs and domain constructs from netCDF, CDL, PP and
+  UM datasets with a choice of netCDF backends,
+
+* read files from OPeNDAP servers and S3 object stores,
+
 * create new field constructs in memory,
 
 * write and append field constructs to netCDF datasets on disk,
 
+* read, write, and manipulate UGRID mesh topologies,
+
 * read, write, and create coordinates defined by geometry cells,
 
 * read netCDF and CDL datasets containing hierarchical groups,
@@ -74,8 +81,8 @@
 """
 
 __Conventions__ = "CF-1.11"
-__date__ = "2023-03-01"
-__version__ = "3.16.1"
+__date__ = "2024-??-??"
+__version__ = "3.17.0"
 
 _requires = (
     "numpy",
@@ -191,16 +198,16 @@
     )
 
 # Check the version of cfunits
-_minimum_vn = "3.3.6"
+_minimum_vn = "3.3.7"
 if Version(cfunits.__version__) < Version(_minimum_vn):
     raise RuntimeError(
         f"Bad cfunits version: cf requires cfunits>={_minimum_vn}. "
         f"Got {cfunits.__version__} at {cfunits.__file__}"
     )
 
 # Check the version of cfdm
-_minimum_vn = "1.11.1.0"
-_maximum_vn = "1.11.2.0"
+_minimum_vn = "1.11.2.0"
+_maximum_vn = "1.11.3.0"
 _cfdm_version = Version(cfdm.__version__)
 if not Version(_minimum_vn) <= _cfdm_version < Version(_maximum_vn):
     raise RuntimeError(
@@ -274,15 +281,19 @@
 from .data.array import (
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFANetCDFArray,
+    CFAH5netcdfArray,
+    CFANetCDF4Array,
     FullArray,
     GatheredArray,
+    H5netcdfArray,
     NetCDFArray,
+    NetCDF4Array,
     PointTopologyArray,
     RaggedContiguousArray,
     RaggedIndexedArray,
     RaggedIndexedContiguousArray,
     SubsampledArray,
+    UMArray,
 )
 
 from .data.fragment import (

cf/aggregate.py

@@ -3207,7 +3207,7 @@ def aggregate(
                     #
                     #   0.0012 , 0.019 , 0.55 , 2.1
                     #
-                    # compared with current method timings of
+                    # compared with new timings of
                     #
                     #   0.00035, 0.0012, 0.013, 0.064
                     # ------------------------------------------------
@@ -4817,7 +4817,7 @@ def _aggregate_2_fields(
 
         # Ensure that the axis orders are the same in both fields
         transpose_axes1 = [dim0_name_map[axis0] for axis0 in data_axes0]
-        if transpose_axes1 != data_axes1:
+        if transpose_axes1 != list(data_axes1):
             parent1.transpose(transpose_axes1, inplace=True)
 
         construct_type = parent0.construct_type

cf/cellmethod.py

@@ -56,7 +56,7 @@ class CellMethod(cfdm.CellMethod):
     def __new__(cls, *args, **kwargs):
         """This must be overridden in subclasses.
 
-        .. versionadded:: (cfdm) 3.7.0
+        .. versionadded:: 3.7.0
 
         """
         instance = super().__new__(cls)

cf/cfimplementation.py

@@ -29,9 +29,11 @@
 from .data.array import (
     BoundsFromNodesArray,
     CellConnectivityArray,
-    CFANetCDFArray,
+    CFAH5netcdfArray,
+    CFANetCDF4Array,
     GatheredArray,
-    NetCDFArray,
+    H5netcdfArray,
+    NetCDF4Array,
     PointTopologyArray,
     RaggedContiguousArray,
     RaggedIndexedArray,
@@ -85,65 +87,39 @@ def set_construct(self, parent, construct, axes=None, copy=True, **kwargs):
             parent, construct, axes=axes, copy=copy, **kwargs
         )
 
-    def initialise_CFANetCDFArray(
-        self,
-        filename=None,
-        address=None,
-        dtype=None,
-        mask=True,
-        units=False,
-        calendar=False,
-        instructions=None,
-        substitutions=None,
-        term=None,
-        x=None,
-        **kwargs,
-    ):
-        """Return a `CFANetCDFArray` instance.
+    def initialise_CFANetCDF4Array(self, **kwargs):
+        """Return a `CFANetCDF4Array` instance.
 
         :Parameters:
 
-            filename: `str`
-
-            address: (sequence of) `str` or `int`
-
-            dytpe: `numpy.dtype`
-
-            mask: `bool`, optional
+            kwargs: optional
+                Initialisation parameters to pass to the new instance.
 
-            units: `str` or `None`, optional
+        :Returns:
 
-            calendar: `str` or `None`, optional
+            `CFANetCDF4Array`
 
-            instructions: `str`, optional
+        """
+        cls = self.get_class("CFANetCDF4Array")
+        return cls(**kwargs)
 
-            substitutions: `dict`, optional
+    def initialise_CFAH5netcdfArray(self, **kwargs):
+        """Return a `CFAH5netcdfArray` instance.
 
-            term: `str`, optional
+        .. versionadded:: NEXTVERSION
 
-            x: `dict`, optional
+        :Parameters:
 
             kwargs: optional
-                Ignored.
+                Initialisation parameters to pass to the new instance.
 
         :Returns:
 
-            `CFANetCDFArray`
+            `CFAH5netcdfArray`
 
         """
-        cls = self.get_class("CFANetCDFArray")
-        return cls(
-            filename=filename,
-            address=address,
-            dtype=dtype,
-            mask=mask,
-            units=units,
-            calendar=calendar,
-            instructions=instructions,
-            substitutions=substitutions,
-            term=term,
-            x=x,
-        )
+        cls = self.get_class("CFAH5netcdfArray")
+        return cls(**kwargs)
 
 
 _implementation = CFImplementation(
@@ -152,7 +128,8 @@ def initialise_CFANetCDFArray(
     CellConnectivity=CellConnectivity,
     CellMeasure=CellMeasure,
     CellMethod=CellMethod,
-    CFANetCDFArray=CFANetCDFArray,
+    CFAH5netcdfArray=CFAH5netcdfArray,
+    CFANetCDF4Array=CFANetCDF4Array,
     CoordinateReference=CoordinateReference,
     DimensionCoordinate=DimensionCoordinate,
     Domain=Domain,
@@ -175,7 +152,8 @@ def initialise_CFANetCDFArray(
     BoundsFromNodesArray=BoundsFromNodesArray,
     CellConnectivityArray=CellConnectivityArray,
     GatheredArray=GatheredArray,
-    NetCDFArray=NetCDFArray,
+    H5netcdfArray=H5netcdfArray,
+    NetCDF4Array=NetCDF4Array,
     PointTopologyArray=PointTopologyArray,
     RaggedContiguousArray=RaggedContiguousArray,
     RaggedIndexedArray=RaggedIndexedArray,
@@ -209,7 +187,8 @@ def implementation():
      'CellConnectivityArray': cf.data.array.cellconnectivityarray.CellConnectivityArray,
      'CellMeasure': cf.cellmeasure.CellMeasure,
      'CellMethod': cf.cellmethod.CellMethod,
-     'CFANetCDFArray': cf.data.array.cfanetcdfarray.CFANetCDFArray,
+     'CFAH5netcdfArray': cf.data.array.cfah5netcdfarray.CFAH5netcdfArray,
+     'CFANetCDF4Array': cf.data.array.cfanetcdf4array.CFANetCDF4Array,
      'CoordinateReference': cf.coordinatereference.CoordinateReference,
      'DimensionCoordinate': cf.dimensioncoordinate.DimensionCoordinate,
      'Domain': cf.domain.Domain,
@@ -230,7 +209,8 @@ def implementation():
      'PartNodeCountProperties': cf.partnodecountproperties.PartNodeCountProperties,
      'Data': cf.data.data.Data,
      'GatheredArray': cf.data.array.gatheredarray.GatheredArray,
-     'NetCDFArray': cf.data.array.netcdfarray.NetCDFArray,
+     'H5netcdfArray': cf.data.array.h5netcdfarray.H5netcdfArray,
+     'NetCDF4Array': cf.data.array.netcdf4array.NetCDF4Array,
      'PointTopologyArray': <class 'cf.data.array.pointtopologyarray.PointTopologyArray'>,
      'RaggedContiguousArray': cf.data.array.raggedcontiguousarray.RaggedContiguousArray,
      'RaggedIndexedArray': cf.data.array.raggedindexedarray.RaggedIndexedArray,

cf/constants.py

@@ -63,6 +63,9 @@
     "LOG_LEVEL": logging.getLevelName(logging.getLogger().level),
     "BOUNDS_COMBINATION_MODE": "AND",
     "CHUNKSIZE": parse_bytes(_CHUNKSIZE),
+    "active_storage": False,
+    "active_storage_url": None,
+    "netcdf_lock": True,
 }
 
 masked = np.ma.masked

cf/data/array/__init__.py

@@ -1,9 +1,12 @@
 from .boundsfromnodesarray import BoundsFromNodesArray
 from .cellconnectivityarray import CellConnectivityArray
-from .cfanetcdfarray import CFANetCDFArray
+from .cfah5netcdfarray import CFAH5netcdfArray
+from .cfanetcdf4array import CFANetCDF4Array
 from .fullarray import FullArray
 from .gatheredarray import GatheredArray
+from .h5netcdfarray import H5netcdfArray
 from .netcdfarray import NetCDFArray
+from .netcdf4array import NetCDF4Array
 from .pointtopologyarray import PointTopologyArray
 from .raggedcontiguousarray import RaggedContiguousArray
 from .raggedindexedarray import RaggedIndexedArray