Update preprocessing notebook

docs/examples.rst

@@ -5,7 +5,7 @@ Examples
 .. toctree::
 
   Platform tutorial <examples/ewatercycle_api_notebook.ipynb>
-  Pre-processing <examples/preprocessing.ipynb>
+  Generate forcing <examples/generate_forcing.ipynb>
   LISFLOOD <examples/lisflood.ipynb>
   MARRMoT M01 <examples/MarrmotM01.ipynb>
   MARRMoT M14 <examples/MarrmotM14.ipynb>

ewatercycle/forcing/_hype.py

@@ -5,7 +5,7 @@
 
 from esmvalcore.experimental import get_recipe
 
-from ..util import get_time
+from ..util import get_time, to_absolute_path
 from ._default import DefaultForcing
 from .datasets import DATASETS
 
@@ -40,7 +40,7 @@ def generate(  # type: ignore
 
         for preproc_name in preproc_names:
             recipe.data['preprocessors'][preproc_name]['extract_shape'][
-                'shapefile'] = shape
+                'shapefile'] = to_absolute_path(shape)
 
         recipe.data['datasets'] = [DATASETS[dataset]]
 
@@ -67,7 +67,8 @@ def generate(  # type: ignore
         # instantiate forcing object based on generated data
         return HypeForcing(directory=directory,
                            start_time=str(startyear),
-                           end_time=str(endyear))
+                           end_time=str(endyear),
+                           shape=shape)
 
     def plot(self):
         raise NotImplementedError('Dont know how to plot')

ewatercycle/forcing/_lisflood.py

@@ -1,14 +1,15 @@
 """Forcing related functionality for lisflood"""
 
-from pathlib import Path
 from typing import Optional
+import logging
 
 from esmvalcore.experimental import get_recipe
 
-from ..util import data_files_from_recipe_output, get_extents, get_time
+from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path
 from ._default import DefaultForcing
 from .datasets import DATASETS
 
+logger = logging.getLogger(__name__)
 
 class LisfloodForcing(DefaultForcing):
     """Container for lisflood forcing data."""
@@ -53,11 +54,12 @@ def generate(  # type: ignore
         end_time: str,
         shape: str,
         extract_region: dict = None,
+        run_lisvap: bool = False,
     ) -> 'LisfloodForcing':
         """
             extract_region (dict): Region specification, dictionary must contain `start_longitude`,
                 `end_longitude`, `start_latitude`, `end_latitude`
-
+            run_lisvap (bool): if lisvap should be run. Default is False. Running lisvap is not supported yet.
             TODO add regrid options so forcing can be generated for parameter set
             TODO that is not on a 0.1x0.1 grid
         """
@@ -69,7 +71,7 @@ def generate(  # type: ignore
         preproc_names = ('general', 'daily_water', 'daily_temperature',
                          'daily_radiation', 'daily_windspeed')
 
-        basin = Path(shape).stem
+        basin = to_absolute_path(shape).stem
         for preproc_name in preproc_names:
             recipe.data['preprocessors'][preproc_name]['extract_shape'][
                 'shapefile'] = shape
@@ -103,16 +105,22 @@ def generate(  # type: ignore
         # TODO forcing_files['e0'] = ...
 
         # instantiate forcing object based on generated data
-        return LisfloodForcing(
-            directory=directory,
-            start_time=start_time,
-            end_time=end_time,
-            PrefixPrecipitation=forcing_files["pr"],
-            PrefixTavg=forcing_files["tas"],
-            PrefixE0=forcing_files['e0'],
-            PrefixES0=forcing_files['es0'],
-            PrefixET0=forcing_files['et0'],
-        )
+        if run_lisvap:
+            raise NotImplementedError('Dont know how to run LISVAP.')
+        else:
+            message = (
+                f"Parameter `run_lisvap` is set to False. No forcing data will be generator for 'e0', 'es0' and 'et0'. "
+                f"However, the recipe creates LISVAP input data that can be found in {directory}.")
+            logger.warning("%s", message)
+            return LisfloodForcing(
+                directory=directory,
+                start_time=start_time,
+                end_time=end_time,
+                shape=shape,
+                PrefixPrecipitation=forcing_files["pr"],
+                PrefixTavg=forcing_files["tas"],
+            )
+
 
     def plot(self):
         raise NotImplementedError('Dont know how to plot')

ewatercycle/forcing/_marrmot.py

@@ -5,7 +5,7 @@
 
 from esmvalcore.experimental import get_recipe
 
-from ..util import get_time
+from ..util import get_time, to_absolute_path
 from ._default import DefaultForcing
 from .datasets import DATASETS
 
@@ -45,7 +45,7 @@ def generate(  # type: ignore
         recipe = get_recipe(recipe_name)
 
         # model-specific updates to the recipe
-        basin = Path(shape).stem
+        basin = to_absolute_path(shape).stem
         recipe.data['preprocessors']['daily']['extract_shape'][
             'shapefile'] = shape
         recipe.data['diagnostics']['diagnostic_daily']['scripts']['script'][
@@ -75,6 +75,7 @@ def generate(  # type: ignore
         return MarrmotForcing(directory=directory,
                               start_time=start_time,
                               end_time=end_time,
+                              shape=shape,
                               forcing_file=forcing_file.name)
 
     def plot(self):

ewatercycle/forcing/_pcrglobwb.py

@@ -5,7 +5,7 @@
 
 from esmvalcore.experimental import get_recipe
 
-from ..util import data_files_from_recipe_output, get_extents, get_time
+from ..util import data_files_from_recipe_output, get_extents, get_time, to_absolute_path
 from ._default import DefaultForcing
 from .datasets import DATASETS
 
@@ -66,7 +66,7 @@ def generate(  # type: ignore
                 "additional_datasets"
             ] = [DATASETS[dataset]]
 
-        basin = Path(shape).stem
+        basin = to_absolute_path(shape).stem
         recipe.data["diagnostics"]["diagnostic_daily"]["scripts"]["script"][
             "basin"
         ] = basin
@@ -91,11 +91,11 @@ def generate(  # type: ignore
 
         var_names_climatology = "pr_climatology", "tas_climatology"
 
-        startyear_climatology = get_time(start_time_climatology)
+        startyear_climatology = get_time(start_time_climatology).year
         for var_name in var_names_climatology:
             variables[var_name]["start_year"] = startyear_climatology
 
-        endyear_climatology = get_time(end_time_climatology)
+        endyear_climatology = get_time(end_time_climatology).year
         for var_name in var_names_climatology:
             variables[var_name]["end_year"] = endyear_climatology
 
@@ -109,6 +109,7 @@ def generate(  # type: ignore
             directory=directory,
             start_time=start_time,
             end_time=end_time,
+            shape=shape,
             precipitationNC=forcing_files["pr"],
             temperatureNC=forcing_files["tas"],
         )

ewatercycle/forcing/_wflow.py

@@ -4,7 +4,7 @@
 
 from esmvalcore.experimental import get_recipe
 
-from ..util import get_extents, get_time
+from ..util import get_extents, get_time, to_absolute_path
 from ._default import DefaultForcing
 from .datasets import DATASETS
 
@@ -61,7 +61,7 @@ def generate(  # type: ignore
         recipe_name = "hydrology/recipe_wflow.yml"
         recipe = get_recipe(recipe_name)
 
-        basin = Path(shape).stem
+        basin = to_absolute_path(shape).stem
         recipe.data['diagnostics']['wflow_daily']['scripts']['script'][
             'basin'] = basin
 
@@ -100,6 +100,7 @@ def generate(  # type: ignore
         return WflowForcing(directory=directory,
                             start_time=start_time,
                             end_time=end_time,
+                            shape=shape,
                             netcdfinput=forcing_file.name)
 
     def __str__(self):

ewatercycle/util.py

@@ -81,7 +81,7 @@ def get_extents(shapefile: Any, pad=0) -> Dict[str, float]:
     Returns:
         Dict with `start_longitude`, `start_latitude`, `end_longitude`, `end_latitude`
     """
-    shape = fiona.open(shapefile)
+    shape = fiona.open(to_absolute_path(shapefile))
     x0, y0, x1, y1 = [geometry.shape(p["geometry"]).bounds for p in shape][0]
     x0 = round((x0 - pad), 1)
     y0 = round((y0 - pad), 1)

tests/conftest.py

@@ -42,8 +42,7 @@ def sample_parameterset(yaml_config_url):
 
 @pytest.fixture
 def sample_shape():
-    return str(Path(__file__).parent / 'models' / 'data' / 'Rhine' / 'Rhine.shp')
-
+    return str(Path(__file__).parents[1] / 'docs' / 'examples' / 'data' / 'Rhine' / 'Rhine.shp')
 
 @pytest.fixture
 def sample_marrmot_forcing_file():

tests/forcing/test_lisflood.py

@@ -37,14 +37,11 @@ def create_netcdf(var_name, filename):
 def mock_recipe_run(monkeypatch, tmp_path):
     """Overload the `run` method on esmvalcore Recipe's."""
     data = {}
-
+    # TODO add lisvap input files once implemented, see issue #96
     class MockTaskOutput:
         data_files = (
             create_netcdf('pr', tmp_path / 'lisflood_pr.nc'),
             create_netcdf('tas', tmp_path / 'lisflood_tas.nc'),
-            create_netcdf('e0', tmp_path / 'lisflood_e0.nc'),
-            create_netcdf('es0', tmp_path / 'lisflood_es0.nc'),
-            create_netcdf('et0', tmp_path / 'lisflood_et0.nc'),
         )
 
     def mock_run(self):
@@ -258,15 +255,13 @@ def reference_recipe(self):
             }
         }
 
-    def test_result(self, forcing, tmp_path):
+    def test_result(self, forcing, tmp_path, sample_shape):
         expected = LisfloodForcing(directory=str(tmp_path),
                                    start_time='1989-01-02T00:00:00Z',
                                    end_time='1999-01-02T00:00:00Z',
+                                   shape=str(sample_shape),
                                    PrefixPrecipitation='lisflood_pr.nc',
-                                   PrefixTavg='lisflood_tas.nc',
-                                   PrefixE0='lisflood_e0.nc',
-                                   PrefixES0='lisflood_es0.nc',
-                                   PrefixET0='lisflood_et0.nc')
+                                   PrefixTavg='lisflood_tas.nc')
         assert forcing == expected
 
     def test_recipe_configured(self, forcing, mock_recipe_run,
@@ -286,5 +281,7 @@ def test_recipe_configured(self, forcing, mock_recipe_run,
 
     def test_saved_yaml(self, forcing, tmp_path):
         saved_forcing = load(tmp_path)
+        # shape should is not included in the yaml file
+        forcing.shape = None
 
         assert forcing == saved_forcing

tests/forcing/test_marrmot.py

@@ -100,11 +100,12 @@ def reference_recipe(self):
                                                           }}}
         }
 
-    def test_result(self, forcing, tmp_path):
+    def test_result(self, forcing, tmp_path, sample_shape):
         expected = MarrmotForcing(
             directory=str(tmp_path),
             start_time='1989-01-02T00:00:00Z',
             end_time='1999-01-02T00:00:00Z',
+            shape = str(sample_shape),
             forcing_file='marrmot.mat'
         )
         assert forcing == expected
@@ -123,6 +124,8 @@ def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe, sam
 
     def test_saved_yaml(self, forcing, tmp_path):
         saved_forcing = load(tmp_path)
+        # shape should is not included in the yaml file
+        forcing.shape = None
 
         assert forcing == saved_forcing
 

tests/forcing/test_pcrglobwb.py

@@ -69,11 +69,12 @@ def forcing(self, mock_recipe_run, sample_shape):
             )
         )
 
-    def test_result(self, forcing, tmp_path):
+    def test_result(self, forcing, tmp_path, sample_shape):
         expected = PCRGlobWBForcing(
             directory=str(tmp_path),
             start_time='1989-01-02T00:00:00Z',
             end_time='1999-01-02T00:00:00Z',
+            shape=str(sample_shape),
             precipitationNC='pcrglobwb_pr.nc',
             temperatureNC='pcrglobwb_tas.nc'
         )

tests/forcing/test_wflow.py

@@ -103,11 +103,12 @@ def forcing(self, mock_recipe_run, sample_shape):
             )
         )
 
-    def test_result(self, forcing, tmp_path):
+    def test_result(self, forcing, tmp_path, sample_shape):
         expected = WflowForcing(
             directory=str(tmp_path),
             start_time='1989-01-02T00:00:00Z',
             end_time='1999-01-02T00:00:00Z',
+            shape = str(sample_shape),
             netcdfinput='wflow_forcing.nc'
         )
         assert forcing == expected
@@ -117,5 +118,7 @@ def test_recipe_configured(self, forcing, mock_recipe_run, reference_recipe):
 
     def test_saved_yaml(self, forcing, tmp_path):
         saved_forcing = load(tmp_path)
+        # shape should is not included in the yaml file
+        forcing.shape = None
 
         assert forcing == saved_forcing