Add parameterset catalogue (#118)

* initial idea for parameterset catalogue * Refactor catalogue into parameter_sets module * Added some tests * Use the fixture * More tests * Even more tests * Make target_model optional * More todos * less todos * Implemented download_example_parameter_sets() + other bits and pieces * formatting * dont write ewatercycle_config to yaml output * write to previous config; add todo's * Refactor download example parameter sets * Config no longer has `.` in keys, removed dead code/test * Added CFG.save_to_file() * Move example parameter set to own files * Format with black * Added docstrings + made ParameterSet public because it is used in api notebook * Added tests * Formatting * Replace model specific parameter set with generic one * Added notebook for download_example_parameter_sets() * Moved MaskMap from parameter set to setup Fixes #121 * Added ParameterSet.supported_model_versions + AbstractModel._check_parameter_set() Needed a lot of refactoring to please mypy and to have all info together for _check_parameter_set() * Replace SubVersionCopier with get_parameter_set() * Replace `Path.is_relative_to()` with Py37 compatible implementation * Correct sphinx links * Remove unused imports They should be part fix for #124 * Use Lisflood.setup{MaskMap) in notebook * example_parameter_sets() return a dict instead of iterator * Fallback to default value when key is missing in config * Make MarrmotM14 a non-generic class * Flatten test hierarchy * Correct nesting * Raise exceptions in available_parameter_sets and return tuple * Update ewatercycle/models/abstract.py Co-authored-by: Peter Kalverla <[email protected]> * Check model.version against model.available_versions * List class var before object vars * Remove duplicate bmi in docs * Fix tests * Update ewatercycle/models/abstract.py Co-authored-by: Peter Kalverla <[email protected]> * Replace id with name * Add skip_existing arg to download * Revert "Fallback to default value when key is missing in config" This reverts commit f776fce. * Fix test * Fix mypy warning Co-authored-by: Stefan Verhoeven <[email protected]> Co-authored-by: Stefan Verhoeven <[email protected]>
eWaterCycle · Jul 2, 2021 · 911f449 · 911f449
1 parent da87e2d
commit 911f449
Show file tree

Hide file tree

Showing 33 changed files with 1,551 additions and 561 deletions.
diff --git a/.gitignore b/.gitignore
@@ -24,3 +24,6 @@ docs/apidocs/
 .scannerwork/
 
 .ipynb_checkpoints
+
+docs/examples/parameter-sets
+docs/examples/ewatercycle.yaml
diff --git a/docs/examples.rst b/docs/examples.rst
@@ -12,3 +12,4 @@ Examples
   PCRGlobWB <examples/pcrglobwb.ipynb>
   Wflow <examples/wflow.ipynb>
   Irrigation experiment <examples/Irrigation.ipynb>
+  System setup <examples/system_setup.ipynb>
diff --git a/docs/examples/ewatercycle_api_notebook.ipynb b/docs/examples/ewatercycle_api_notebook.ipynb
@@ -324,7 +324,7 @@
     "parameter_set.doi\n",
     "# 'https://doi.org/10.1000/182'\n",
     "\n",
-    "parameter_set.id\n",
+    "parameter_set.name\n",
     "# 'wflow-30-min-global'\n",
     "\n",
     "parameter_set.supported_model_versions\n",
@@ -576,9 +576,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.4"
+   "version": "3.9.5"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}
diff --git a/docs/examples/lisflood.ipynb b/docs/examples/lisflood.ipynb
@@ -34,7 +34,7 @@
     "from ewatercycle import CFG\n",
     "import ewatercycle.models\n",
     "from ewatercycle.forcing import load_foreign\n",
-    "from ewatercycle.models.lisflood import LisfloodParameterSet"
+    "from ewatercycle.parameter_sets import ParameterSet\n"
    ]
   },
   {
@@ -70,11 +70,14 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "parameterset = LisfloodParameterSet(\n",
-    "    PathRoot='/projects/0/wtrcycle/comparison/lisflood_input/Lisflood01degree_masked',\n",
-    "    MaskMap='/projects/0/wtrcycle/comparison/recipes_auxiliary_datasets/LISFLOOD/model_mask.nc',\n",
-    "    config_template='/projects/0/wtrcycle/comparison/lisflood_input/settings_templates/settings_lisflood.xml',\n",
-    ")"
+    "parameterset = ParameterSet(\n",
+    "    name='Lisflood01degree_masked',\n",
+    "    directory='/projects/0/wtrcycle/comparison/lisflood_input/Lisflood01degree_masked',\n",
+    "    config='/projects/0/wtrcycle/comparison/lisflood_input/settings_templates/settings_lisflood.xml',\n",
+    "    supported_model_versions={'20.10'},\n",
+    "    target_model='lisflood'\n",
+    ")\n",
+    "mask_map = '/projects/0/wtrcycle/comparison/recipes_auxiliary_datasets/LISFLOOD/model_mask.nc'"
    ]
   },
   {
@@ -202,7 +205,9 @@
    ],
    "source": [
     "# setup model\n",
-    "config_file, config_dir = model.setup(IrrigationEfficiency='0.8', end_time='1990-10-10T00:00:00Z')"
+    "config_file, config_dir = model.setup(IrrigationEfficiency='0.8', \n",
+    "                                      end_time='1990-10-10T00:00:00Z'\n",
+    "                                      MaskMap=mask_map)"
    ]
   },
   {
@@ -851,9 +856,9 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.9.2"
+   "version": "3.9.5"
   }
  },
  "nbformat": 4,
  "nbformat_minor": 5
-}
+}
diff --git a/docs/examples/pcrglobwb.ipynb b/docs/examples/pcrglobwb.ipynb
diff --git a/docs/examples/system_setup.ipynb b/docs/examples/system_setup.ipynb
@@ -0,0 +1,239 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# System setup\n",
+    "\n",
+    "To use ewatercycle package you need to setup the system with software and data.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 2,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import logging\n",
+    "logging.basicConfig(level=logging.INFO)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 3,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from ewatercycle import CFG\n",
+    "from ewatercycle.parameter_sets import download_example_parameter_sets\n",
+    "from ewatercycle.parameter_sets import available_parameter_sets\n",
+    "from ewatercycle.parameter_sets import get_parameter_set"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Configuration\n",
+    "\n",
+    "The ewatercycle package simplifies the API by reading some of the directories and other configurations from a configuration file.\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 4,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:ewatercycle.config._config_object:Config written to /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/ewatercycle.yaml\n"
+     ]
+    },
+    {
+     "data": {
+      "text/plain": [
+       "PosixPath('/home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/ewatercycle.yaml')"
+      ]
+     },
+     "execution_count": 4,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "CFG['parameterset_dir'] = './parameter-sets'\n",
+    "CFG['ewatercycle_config'] = './ewatercycle.yaml'\n",
+    "CFG.save_to_file()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 5,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "CFG.load_from_file('./ewatercycle.yaml')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Download example parameter sets\n",
+    "\n",
+    "To quickly run the models it is advised to setup a example parameter sets for each model. "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 6,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "INFO:ewatercycle.parameter_sets._example:Downloading example parameter set wflow_rhine_sbm_nc to /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets/wflow_rhine_sbm_nc...\n",
+      "INFO:ewatercycle.parameter_sets._example:Download complete.\n",
+      "INFO:ewatercycle.parameter_sets._example:Adding parameterset wflow_rhine_sbm_nc to ewatercycle.CFG... \n",
+      "INFO:ewatercycle.parameter_sets._example:Downloading example parameter set pcrglobwb_example_case to /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets/pcrglobwb_example_case...\n",
+      "INFO:ewatercycle.parameter_sets._example:Download complete.\n",
+      "INFO:ewatercycle.parameter_sets._example:Adding parameterset pcrglobwb_example_case to ewatercycle.CFG... \n",
+      "INFO:ewatercycle.parameter_sets._example:Downloading example parameter set lisflood_fraser to /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets/lisflood_fraser...\n",
+      "INFO:ewatercycle.parameter_sets._example:Download complete.\n",
+      "INFO:ewatercycle.parameter_sets._example:Adding parameterset lisflood_fraser to ewatercycle.CFG... \n",
+      "INFO:ewatercycle.parameter_sets:3 example parameter sets were downloaded\n",
+      "INFO:ewatercycle.config._config_object:Config written to /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/ewatercycle.yaml\n",
+      "INFO:ewatercycle.parameter_sets:Saved parameter sets to configuration file /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/ewatercycle.yaml\n"
+     ]
+    }
+   ],
+   "source": [
+    "download_example_parameter_sets()"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "Example parameter sets have been downloaded and added to the configuration file."
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "container_engine: null\n",
+      "esmvaltool_config: None\n",
+      "grdc_location: None\n",
+      "output_dir: None\n",
+      "parameter_sets:\n",
+      "  lisflood_fraser:\n",
+      "    config: lisflood_fraser/settings_lat_lon-Run.xml\n",
+      "    directory: lisflood_fraser\n",
+      "    doi: N/A\n",
+      "    supported_model_versions: !!set {'20.10': null}\n",
+      "    target_model: lisflood\n",
+      "  pcrglobwb_example_case:\n",
+      "    config: pcrglobwb_example_case/setup_natural_test.ini\n",
+      "    directory: pcrglobwb_example_case\n",
+      "    doi: N/A\n",
+      "    supported_model_versions: !!set {setters: null}\n",
+      "    target_model: pcrglobwb\n",
+      "  wflow_rhine_sbm_nc:\n",
+      "    config: wflow_rhine_sbm_nc/wflow_sbm_NC.ini\n",
+      "    directory: wflow_rhine_sbm_nc\n",
+      "    doi: N/A\n",
+      "    supported_model_versions: !!set {2020.1.1: null}\n",
+      "    target_model: wflow\n",
+      "parameterset_dir: /home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets\n",
+      "singularity_dir: None\n"
+     ]
+    }
+   ],
+   "source": [
+    "!cat ./ewatercycle.yaml"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 10,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "('wflow_rhine_sbm_nc', 'pcrglobwb_example_case', 'lisflood_fraser')"
+      ]
+     },
+     "execution_count": 10,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "available_parameter_sets()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 9,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "text/plain": [
+       "ParameterSet(name='pcrglobwb_example_case', directory=PosixPath('/home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets/pcrglobwb_example_case'), config=PosixPath('/home/verhoes/git/eWaterCycle/ewatercycle/docs/examples/parameter-sets/pcrglobwb_example_case/setup_natural_test.ini'), doi='N/A', target_model='pcrglobwb', supported_model_versions={'setters'})"
+      ]
+     },
+     "execution_count": 9,
+     "metadata": {},
+     "output_type": "execute_result"
+    }
+   ],
+   "source": [
+    "parameter_set = get_parameter_set('pcrglobwb_example_case')\n",
+    "parameter_set"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "The `parameter_set` variable can be passed to a model class constructor."
+   ]
+  }
+ ],
+ "metadata": {
+  "interpreter": {
+   "hash": "4f63c3f017d58640bc2174e5f1b6c1610e3d96c1a3fe90d1d439f265cee739e3"
+  },
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.5"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
+}