Make path consistent (#143)

docs/conf.py

@@ -32,7 +32,7 @@
 # Add any Sphinx extension module names here, as strings. They can be
 # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
 # ones.
-extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'nbsphinx']
+extensions = ['sphinx.ext.autodoc', 'sphinx.ext.napoleon', 'nbsphinx', 'sphinx.ext.intersphinx']
 
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']
@@ -56,9 +56,9 @@
 # built documents.
 #
 # The short X.Y version.
-from ewatercycle.version import __version__ as version
 # The full version, including alpha/beta/rc tags.
-release = version
+# TODO update version
+release = '1.0.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -240,3 +240,22 @@ def setup(app):
     ('pcrglobwb', 'params_style'),
     ('wflow', 'params_style'),
 ]
+
+intersphinx_mapping = {
+    'cf_units': ('https://scitools.org.uk/cf-units/docs/latest/', None),
+    'esmvalcore':
+    (f'https://docs.esmvaltool.org/projects/esmvalcore/en/latest/',
+     None),
+    'esmvaltool': (f'https://docs.esmvaltool.org/en/latest/', None),
+    'grpc4bmi': (f'https://grpc4bmi.readthedocs.io/en/latest/', None),
+    'iris': ('https://scitools-iris.readthedocs.io/en/latest/', None),
+    'lime': ('https://lime-ml.readthedocs.io/en/latest/', None),
+    'basic_modeling_interface': ('https://bmi.readthedocs.io/en/latest/', None),
+    'matplotlib': ('https://matplotlib.org/', None),
+    'numpy': ('https://numpy.org/doc/stable/', None),
+    'pandas': ('https://pandas.pydata.org/pandas-docs/dev', None),
+    'python': ('https://docs.python.org/3/', None),
+    'scipy': ('https://docs.scipy.org/doc/scipy/reference/', None),
+    'seaborn': ('https://seaborn.pydata.org/', None),
+    'sklearn': ('https://scikit-learn.org/stable', None),
+}

ewatercycle/config/_config_object.py

@@ -11,6 +11,8 @@
 from ._validators import _validators
 from ._validated_config import ValidatedConfig
 
+from ewatercycle.util import to_absolute_path
+
 logger = getLogger(__name__)
 
 
@@ -35,7 +37,6 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config':
             Name of the config file, must be yaml format
         """
         new = cls()
-
         mapping = read_config_file(filename)
         mapping['ewatercycle_config'] = filename
 
@@ -45,19 +46,17 @@ def _load_user_config(cls, filename: Union[os.PathLike, str]) -> 'Config':
         return new
 
     @classmethod
-    def _load_default_config(cls, filename: Union[os.PathLike,
-                                                  str]) -> 'Config':
+    def _load_default_config(cls, filename: Union[os.PathLike, str]) -> 'Config':
         """Load the default configuration."""
         new = cls()
-
         mapping = read_config_file(filename)
         new.update(mapping)
 
         return new
 
     def load_from_file(self, filename: Union[os.PathLike, str]) -> None:
         """Load user configuration from the given file."""
-        path = Path(filename).expanduser()
+        path = to_absolute_path(str(filename))
         if not path.exists():
             raise FileNotFoundError(f'Cannot find: `{filename}')
 
@@ -119,7 +118,7 @@ def save_to_file(self, config_file: Optional[Union[os.PathLike, str]] = None):
 
 def read_config_file(config_file: Union[os.PathLike, str]) -> dict:
     """Read config user file and store settings in a dictionary."""
-    config_file = Path(config_file)
+    config_file = to_absolute_path(str(config_file))
     if not config_file.exists():
         raise IOError(f'Config file `{config_file}` does not exist.')
 

ewatercycle/forcing/__init__.py

@@ -6,6 +6,8 @@
 from ._default import DefaultForcing, FORCING_YAML
 from . import _hype, _lisflood, _marrmot, _pcrglobwb, _wflow
 
+from ewatercycle.util import to_absolute_path
+
 FORCING_CLASSES: Dict[str, Type[DefaultForcing]] = {
     "hype": _hype.HypeForcing,
     "lisflood": _lisflood.LisfloodForcing,
@@ -25,17 +27,18 @@ def load(directory: str) -> DefaultForcing:
     Returns: Forcing object
     """
     yaml = YAML()
-    source = Path(directory) / FORCING_YAML
+    source = to_absolute_path(directory)
     # TODO give nicer error
     yaml.register_class(DefaultForcing)
     for forcing_cls in FORCING_CLASSES.values():
         yaml.register_class(forcing_cls)
-    content = source.read_text()
     # Set directory in yaml string to parent of yaml file
     # Because in DefaultForcing.save the directory was removed
-    abs_dir = str(source.parent.expanduser().resolve())
-    content += f'directory: {abs_dir}\n'
-    forcing_info = yaml.load(content)
+    forcing_info = yaml.load(source / FORCING_YAML)
+    forcing_info.directory = source
+    if forcing_info.shape:
+        forcing_info.shape = to_absolute_path(forcing_info.shape, parent = source)
+
     return forcing_info
 
 

ewatercycle/forcing/_default.py

@@ -2,9 +2,14 @@
 from copy import copy
 from pathlib import Path
 from typing import Optional
+import logging
 
 from ruamel.yaml import YAML
 
+from ewatercycle.util import to_absolute_path
+
+logger = logging.getLogger(__name__)
+
 FORCING_YAML = 'ewatercycle_forcing.yaml'
 
 
@@ -26,8 +31,18 @@ def __init__(self,
                  shape: Optional[str] = None):
         self.start_time = start_time
         self.end_time = end_time
-        self.directory = directory
-        self.shape = shape
+        self.directory = to_absolute_path(directory)
+        self.shape = to_absolute_path(shape) if shape is not None else shape
+
+    def __str__(self):
+        """Nice formatting of forcing object."""
+        return "\n".join(
+            [
+                "eWaterCycle forcing",
+                "-------------------",
+            ]
+            + [f"{k}={v!s}" for k, v in self.__dict__.items()]
+        )
 
     @classmethod
     def generate(
@@ -45,11 +60,19 @@ def save(self):
         """Export forcing data for later use."""
         yaml = YAML()
         yaml.register_class(self.__class__)
-        target = Path(self.directory) / FORCING_YAML
+        target = self.directory / FORCING_YAML
         # We want to make the yaml and its parent movable,
-        # so the directory should not be included in the yaml file
+        # so the directory and shape should not be included in the yaml file
         clone = copy(self)
         del clone.directory
+
+        if clone.shape:
+            try:
+                clone.shape = str(clone.shape.relative_to(self.directory))
+            except ValueError:
+                clone.shape = None
+                logger.info(f"Shapefile {self.shape} is not in forcing directory {self.directory}. So, it won't be saved in {target}.")
+
         with open(target, 'w') as f:
             yaml.dump(clone, f)
         return target

ewatercycle/models/abstract.py

@@ -1,4 +1,5 @@
 import logging
+import textwrap
 from abc import ABCMeta, abstractmethod
 from datetime import datetime
 from typing import Tuple, Iterable, Any, TypeVar, Generic, Optional, ClassVar, Set
@@ -42,6 +43,20 @@ def __del__(self):
         except AttributeError:
             pass
 
+    def __str__(self):
+        """Nice formatting of model object."""
+        return "\n".join(
+            [
+                f"eWaterCycle {self.__class__.__name__}",
+                "-------------------",
+                f"Version = {self.version}",
+                "Parameter set = ",
+                textwrap.indent(str(self.parameter_set), '  '),
+                "Forcing = ",
+                textwrap.indent(str(self.forcing), '  '),
+            ]
+        )
+
     @abstractmethod
     def setup(self, *args, **kwargs) -> Tuple[str, str]:
         """Performs model setup.

ewatercycle/models/lisflood.py

@@ -14,7 +14,7 @@
 from ewatercycle.models.abstract import AbstractModel
 from ewatercycle.parameter_sets import ParameterSet
 from ewatercycle.parametersetdb.config import AbstractConfig
-from ewatercycle.util import get_time, find_closest_point
+from ewatercycle.util import get_time, find_closest_point, to_absolute_path
 
 
 class Lisflood(AbstractModel[LisfloodForcing]):
@@ -84,7 +84,7 @@ def setup(self,  # type: ignore
         """
 
         # TODO forcing can be a part of parameter_set
-        cfg_dir_as_path = Path(cfg_dir) if cfg_dir else None
+        cfg_dir_as_path = to_absolute_path(cfg_dir) if cfg_dir else None
         cfg_dir_as_path = _generate_workdir(cfg_dir_as_path)
         config_file = self._create_lisflood_config(cfg_dir_as_path, start_time, end_time, IrrigationEfficiency, MaskMap)
 
@@ -94,7 +94,7 @@ def setup(self,  # type: ignore
                     str(self.forcing_dir)
                 ]
         if MaskMap is not None:
-            mask_map = Path(MaskMap).expanduser().resolve()
+            mask_map = to_absolute_path(MaskMap)
             try:
                 mask_map.relative_to(self.parameter_set.directory)
             except ValueError:
@@ -128,7 +128,7 @@ def _check_forcing(self, forcing):
         # if not warn users to run reindex_forcings
         if isinstance(forcing, LisfloodForcing):
             self.forcing = forcing
-            self.forcing_dir = Path(forcing.directory).expanduser().resolve()
+            self.forcing_dir = to_absolute_path(forcing.directory)
             # convert date_strings to datetime objects
             self._start = get_time(forcing.start_time)
             self._end = get_time(forcing.end_time)
@@ -168,7 +168,7 @@ def _create_lisflood_config(self, cfg_dir: Path, start_time_iso: str = None, end
         if IrrigationEfficiency is not None:
             settings['IrrigationEfficiency'] = IrrigationEfficiency
         if MaskMap is not None:
-            mask_map = Path(MaskMap).expanduser().resolve()
+            mask_map = to_absolute_path(MaskMap)
             settings['MaskMap'] = str(mask_map.with_suffix(''))
 
         for textvar in self.cfg.config.iter("textvar"):
@@ -308,7 +308,7 @@ def _generate_workdir(cfg_dir: Path = None) -> Path:
         scratch_dir = CFG['output_dir']
         # TODO this timestamp isnot safe for parallel processing
         timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
-        cfg_dir = Path(scratch_dir) / f'lisflood_{timestamp}'
+        cfg_dir = to_absolute_path(f'lisflood_{timestamp}', parent=Path(scratch_dir))
     cfg_dir.mkdir(parents=True, exist_ok=True)
     return cfg_dir
 

ewatercycle/models/marrmot.py

@@ -14,7 +14,7 @@
 from ewatercycle import CFG
 from ewatercycle.forcing._marrmot import MarrmotForcing
 from ewatercycle.models.abstract import AbstractModel
-from ewatercycle.util import get_time
+from ewatercycle.util import get_time, to_absolute_path
 
 logger = logging.getLogger(__name__)
 
@@ -38,7 +38,7 @@ def _generate_cfg_dir(cfg_dir: Path = None) -> Path:
         scratch_dir = CFG['output_dir']
         # TODO this timestamp isnot safe for parallel processing
         timestamp = datetime.datetime.now(datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
-        cfg_dir = Path(scratch_dir) / f'marrmot_{timestamp}'
+        cfg_dir = to_absolute_path(f'marrmot_{timestamp}', parent=Path(scratch_dir))
     cfg_dir.mkdir(parents=True, exist_ok=True)
     return cfg_dir
 
@@ -63,7 +63,7 @@ class MarrmotM01(AbstractModel[MarrmotForcing]):
 
     def __init__(self, version: str, forcing: MarrmotForcing):
         """Construct MarrmotM01 with initial values. """
-        super().__init__(version)
+        super().__init__(version, forcing=forcing)
         self._parameters = [1000.0]
         self.store_ini = [900.0]
         self.solver = Solver()
@@ -114,7 +114,7 @@ def setup(self,  # type: ignore
             self.store_ini = [initial_soil_moisture_storage]
         if solver:
             self.solver = solver
-        cfg_dir_as_path = Path(cfg_dir) if cfg_dir else None
+        cfg_dir_as_path = to_absolute_path(cfg_dir) if cfg_dir else None
 
         cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path)
         config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time)
@@ -141,7 +141,7 @@ def setup(self,  # type: ignore
     def _check_forcing(self, forcing):
         """"Check forcing argument and get path, start and end time of forcing data."""
         if isinstance(forcing, MarrmotForcing):
-            forcing_dir = Path(forcing.directory).expanduser().resolve()
+            forcing_dir = to_absolute_path(forcing.directory)
             self.forcing_file = str(forcing_dir / forcing.forcing_file)
             # convert date_strings to datetime objects
             self.forcing_start_time = get_time(forcing.start_time)
@@ -288,7 +288,7 @@ class MarrmotM14(AbstractModel[MarrmotForcing]):
 
     def __init__(self, version: str, forcing: MarrmotForcing):
         """Construct MarrmotM14 with initial values. """
-        super().__init__(version)
+        super().__init__(version, forcing=forcing)
         self._parameters = [1000.0, 0.5, 0.5, 100.0, 0.5, 4.25, 2.5]
         self.store_ini = [900.0, 900.0]
         self.solver = Solver()
@@ -358,7 +358,7 @@ def setup(self,  # type: ignore
             self.store_ini[1] = initial_saturated_zone_storage
         if solver:
             self.solver = solver
-        cfg_dir_as_path = Path(cfg_dir) if cfg_dir else None
+        cfg_dir_as_path = to_absolute_path(cfg_dir) if cfg_dir else None
 
         cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path)
         config_file = self._create_marrmot_config(cfg_dir_as_path, start_time, end_time)
@@ -385,7 +385,7 @@ def setup(self,  # type: ignore
     def _check_forcing(self, forcing):
         """"Check forcing argument and get path, start and end time of forcing data."""
         if isinstance(forcing, MarrmotForcing):
-            forcing_dir = Path(forcing.directory).expanduser().resolve()
+            forcing_dir = to_absolute_path(forcing.directory)
             self.forcing_file = str(forcing_dir / forcing.forcing_file)
             # convert date_strings to datetime objects
             self.forcing_start_time = get_time(forcing.start_time)