Update ECMWF forecast dataset to handle new resolution (#285)

* Update ECMWF forecast dataset to handle new resolution * Add Unit Test for ECMWF dataset * Update dataset container
microsoft · Mar 15, 2024 · fe4bd1c · fe4bd1c
1 parent 93803b6
commit fe4bd1c
Show file tree

Hide file tree

Showing 6 changed files with 44 additions and 6 deletions.
diff --git a/datasets/ecmwf-forecast/Dockerfile b/datasets/ecmwf-forecast/Dockerfile
@@ -21,15 +21,15 @@ RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10
 # See https://github.com/mapbox/rasterio/issues/1289
 ENV CURL_CA_BUNDLE=/etc/ssl/certs/ca-certificates.crt
 
-# Install Python 3.8
+# Install Python 3.10
 RUN curl -L -O "https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-$(uname)-$(uname -m).sh" \
      && bash "Mambaforge-$(uname)-$(uname -m).sh" -b -p /opt/conda \
      && rm -rf "Mambaforge-$(uname)-$(uname -m).sh"
 
 ENV PATH /opt/conda/bin:$PATH
 ENV LD_LIBRARY_PATH /opt/conda/lib/:$LD_LIBRARY_PATH
 
-RUN mamba install -y -c conda-forge python=3.8 gdal=3.3.3 pip setuptools cython numpy==1.21.5
+RUN mamba install -y -c conda-forge python=3.10 gdal=3.3.3 pip setuptools cython numpy==1.21.5
 
 RUN python -m pip install --upgrade pip
 

diff --git a/datasets/ecmwf-forecast/README.md b/datasets/ecmwf-forecast/README.md
@@ -7,6 +7,17 @@
 - Item creation is fast (the data is not touched), so a single chunk file for daily data is fine — no need to limit `chunk_length`.
 
 ## Dockerfile
+Build and publish a new container image with:
+```shell
+tag="20240314.1"
+registry="pccomponents"
+image="$registry.azurecr.io/pctasks-ecmwf-forecast:$tag"
+az acr login -n $registry
+docker build -t $image -f datasets/ecmwf-forecast/Dockerfile .
+docker push $image
+```
+
+Or:
 
 ```shell
 az acr build -r {the registry} --subscription {the subscription} -t pctasks-ecmwf-forecast:latest -t pctasks-ecmwf-forecast:{date}.{count} -f datasets/ecmwf-forecast/Dockerfile .

diff --git a/datasets/ecmwf-forecast/dataset.yaml b/datasets/ecmwf-forecast/dataset.yaml
@@ -1,5 +1,5 @@
 id: ecmwf_forecast
-image: ${{ args.registry }}/pctasks-ecmwf-forecast:20230614.1
+image: ${{ args.registry }}/pctasks-ecmwf-forecast:20240314.1
 
 args:
 - registry

diff --git a/datasets/ecmwf-forecast/ecmwf_forecast.py b/datasets/ecmwf-forecast/ecmwf_forecast.py
@@ -14,9 +14,16 @@ def create_item(
         cls, asset_uri: str, storage_factory: StorageFactory
     ) -> Union[List[pystac.Item], WaitTaskResult]:
         asset_storage, asset_path = storage_factory.get_storage_for_file(asset_uri)
-
+        # Starting March 2024, the ECMWF forecasts data will be available at 0.25 resolution
+        resolution = None
+        if "0p25" in asset_path.split("/"):
+            resolution = "0.25"
+        if "0p4-beta" in asset_path.split("/"):
+            resolution = "0.40"
         grib2_href = asset_storage.get_url(asset_path)
         index_href = grib2_href.rsplit(".", 1)[0] + ".index"
-        item = stac.create_item([grib2_href, index_href], split_by_step=True)
+        item = stac.create_item(
+            [grib2_href, index_href], split_by_step=True, resolution=resolution
+        )
 
         return [item]
diff --git a/datasets/ecmwf-forecast/requirements.txt b/datasets/ecmwf-forecast/requirements.txt
@@ -1 +1 @@
-git+https://github.com/stactools-packages/ecmwf-forecast@1c9726e6a4f38e1c15648df6d45a5c8432ce92a9
+git+https://github.com/stactools-packages/ecmwf-forecast@0.2.0
diff --git a/datasets/ecmwf-forecast/test_ecmwf_forecast.py b/datasets/ecmwf-forecast/test_ecmwf_forecast.py
@@ -0,0 +1,20 @@
+import pytest
+from ecmwf_forecast import EcmwfCollection
+from pctasks.core.storage import StorageFactory
+
+
+@pytest.mark.parametrize(
+    "href",
+    [
+        "blob://ai4edataeuwest/ecmwf/20240314/00z/ifs/0p4-beta/enfo/20240314000000-0h-enfo-ef.grib2",
+        "blob://ai4edataeuwest/ecmwf/20240314/00z/ifs/0p25/waef/20240314000000-0h-waef-ef.grib2",
+    ],
+)
+def test_ecmwf(href: str) -> None:
+    storage_factory = StorageFactory()
+    (item,) = EcmwfCollection.create_item(href, storage_factory)
+    assert "ecmwf:resolution" in item.properties
+    if "/0p4-beta/" in href:
+        assert item.properties["ecmwf:resolution"] == "0.40"
+    if "/0p25/" in href:
+        assert item.properties["ecmwf:resolution"] == "0.25"