Merge pull request #128 from EcoExtreML/python-compliant

Support Python and Octave compliant model

.vscode/launch.json

@@ -0,0 +1,35 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "type": "OctaveDebugger",
+            "request": "launch",
+            "name": "Octave: Execute selected m-file",
+            "program": "${file}",
+            "octave": "octave.bat --no-gui --interactive --silent",
+            "autoTerminate": true,
+            "sourceFolder": "${workspaceFolder}/src"
+        },
+        {
+            "type": "OctaveDebugger",
+            "request": "launch",
+            "name": "Octave: Debug STEMMUS-SCOPE",
+            "program": "debug_Octave.m",
+            "octave": "octave.bat --no-gui --interactive --silent",
+            "autoTerminate": true,
+            "sourceFolder": "${workspaceFolder}/src"
+        },
+        {
+            "type": "OctaveDebugger",
+            "request": "launch",
+            "name": "Octave: Interactive",
+            "program": "",
+            "octave": "octave.bat --no-gui --interactive --silent",
+            "autoTerminate": false,
+            "sourceFolder": "${workspaceFolder}"
+        },
+    ]
+}

CONTRIBUTING.md

@@ -1,15 +1,17 @@
 # Contributing guidelines
 
-We welcome any kind of contributions to our software, from simple
-comment or question to a full [pull
-request](https://help.github.com/articles/about-pull-requests/). Please
+This repository includes the MATLAB source code of the STEMMUS-SCOPE model. We welcome any
+kind of contributions to our software, from simple comments or questions to a full
+[pull request](https://help.github.com/articles/about-pull-requests/). Please
 read and follow our contributing guidelines.
 
 ## Contributing via GitHub
 
-Note when we want to work with `STEMMUS_SCOPE` repository on a new computer for
-the first time, we need to configure a few things following steps 1 to 4 below.
+If you want to work with the `STEMMUS_SCOPE` repository for the first time, or on a new computer, 
+you need to configure a few things following steps 1 through 5 below.
 
+<details>
+  <summary>Steps 1 to 5 </summary>
 ### 1. Enable two-factor authentication
 
 It is strongly recommended using two-factor authentication. Here is the link of
@@ -132,10 +134,24 @@ git clone [email protected]:EcoExtreML/STEMMUS_SCOPE.git
 Now a new GitHub folder `STEMMUS_SCOPE` is created in your `HOME` directory.
 
 > In this command, we clone the repository using `ssh` option. As we set the ssh
-connection in [**Step 2**](#2-set-ssh-connection), this command here does not ask for our user name and
-password.
+connection in [**Step 2**](#2-set-ssh-connection), this command here does not
+ask for our user name and password.
 
 ### 5. Collaborate using GitHub
 
 To know about the most common Git commands, follow the guides
-[here](https://hackmd.io/B4v6KwsBRzG-akLDF8e5pg).
+[here](https://hackmd.io/B4v6KwsBRzG-akLDF8e5pg).
+</details>
+
+## Development of the MATLAB source of STEMMUS_SCOPE model
+
+To contribute to the STEMMUS_SCOPE model, you need access to the model source code that is stored in the repository [STEMMUS_SCOPE](https://github.com/EcoExtreML/STEMMUS_SCOPE). You also need a MATLAB license. MATLAB `2021a` is installed on
+[Snellius]((https://servicedesk.surfsara.nl/wiki/display/WIKI/Snellius)) and [CRIB](https://crib.utwente.nl/), see [this instruction](https://pystemmusscope.readthedocs.io/en/latest/contributing_link.html#development-of-stemmus-scope-model).
+
+## Create an executable file of STEMMUS_SCOPE
+
+See the [exe readme](./exe/README.md).
+
+## Follow MATLAB style guidelines
+
+Please follow the style introduced in [MATLAB Guidelines 2.0, Richard Johnson](http://cnl.sogang.ac.kr/cnlab/lectures/programming/matlab/Richard_Johnson-MatlabStyle2_book.pdf).

Octave_instructions.md

@@ -0,0 +1,48 @@
+# Using STEMMUS-SCOPE with GNU Octave
+The downloads can be found here
+https://octave.org/download
+
+TODO: installation of octave on linux
+After installation, launch octave and install the following Octave packages:
+`pkg install -forge netcdf`
+`pkg install -forge statistics`
+
+For off-line installation, first, download the packages [netcdf](https://octave.sourceforge.io/netcdf/index.html), [io](https://octave.sourceforge.io/io/index.html) and [statistics](https://octave.sourceforge.io/statistics/index.html). Then, launch octave and run:
+
+`pkg install netcdf-1.0.16.tar.gz`
+`pkg install io-2.6.4.tar.gz`
+`pkg install statistics-1.4.3.tar.gz`
+
+### VS Code setup
+Add Octave to path, e.g. for (64-bit) Windows add the following folders:
+`C:\Program Files\GNU Octave\Octave-7.1.0\mingw64\bin`
+`C:\Program Files\GNU Octave\Octave-7.1.0\usr\bin`
+
+Add the extensions
+`Octave Debugger` by Paulo Silva https://marketplace.visualstudio.com/items?itemName=paulosilva.vsc-octave-debugger
+This allows to run Octave in the VS Code debugger.
+The debugger configurations are included in `/.vscode/launch.json`
+
+`Octave Hacking` by Andrew Janke https://marketplace.visualstudio.com/items?itemName=apjanke.octave-hacking
+This adds syntax highlighting and formatting.
+
+### Running STEMMUS-SCOPE in Octave
+It is possible to run STEMMUS-SCOPE from the command line with the following setup:
+`octave.bat --no-gui --interactive --silent --eval "STEMMUS_SCOPE_exe('path_to_config_file')"`
+
+On a Unix system, use `octave` instead of `octave.bat`.
+### Developing STEMMUS-SCOPE in Octave
+Open the `run_Octave.m` file, either in VS Code or the Octave GUI.
+
+#### Octave GUI
+Set the workspace to the `STEMMUS_SCOPE/src` folder, and open the `run_Octave.m` file.
+Here you can set the config file that should be used, and then run the file.
+
+#### VS Code
+While having the `STEMMUS_SCOPE` folder as the workspace, open the debugger and select `Octave: Debug STEMMUS-SCOPE`.
+Start the debugger to run (and debug) the model.
+
+In the `run_Octave.m` file you can set the config file that should be used.
+
+
+

README.md

@@ -1,8 +1,9 @@
 # STEMMUS_SCOPE
 
-`STEMMUS_SCOPE` serves as an integrated code of SCOPE and STEMMUS.
+Integrated code of SCOPE and STEMMUS.
 
-SCOPE is a radiative transfer and energy balance model, and STEMMUS model is a two-phase mass and heat transfer model. For more information about the coupling between these two models, please check [this reference](https://gmd.copernicus.org/articles/14/1379/2021/).
+SCOPE is a radiative transfer and energy balance model, and STEMMUS model is a two-phase mass and heat transfer model. For more information about the coupling between these two models, please check [this reference](https://gmd.copernicus.org/articles/14/1379/2021/). Before running the model, you need to prepare input data and a configuration file. This can be done using the python package
+[PyStemmusScope](https://pystemmusscope.readthedocs.io).  
 
 <img width="500" alt="Logo" src=./docs/assets/imgs/coupling_scheme.png>
 (by Zeng & Su, 2021)
@@ -41,6 +42,3 @@ have a look at the [contribution guidelines](CONTRIBUTING.md).
 ## How to cite us
 [![RSD](https://img.shields.io/badge/rsd-ecoextreml-00a3e3.svg)](https://research-software-directory.org/projects/ecoextreml)
 <!-- [![DOI](https://zenodo.org/badge/DOI/<replace-with-created-DOI>.svg)](https://doi.org/<replace-with-created-DOI>) -->
-
-<!--TODO: add links to zenodo. -->
-More information will follow soon.

config_file_crib.txt

@@ -1,7 +1,14 @@
+WorkDir=/home/jovyan/private/
 SoilPropertyPath=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/SoilProperty/
-InputPath=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/
-OutputPath=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/output/
 ForcingPath=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/Plumber2_data/
 ForcingFileName=AU-DaS_2010-2017_OzFlux_Met.nc
+directional=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/directional/
+fluspect_parameters=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/fluspect_parameters/
+leafangles=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/leafangles/
+radiationdata=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/radiationdata/
+soil_spectrum=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/soil_spectrum/
+input_data=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/input_data.xlsx
 InitialConditionPath=/data/shared/EcoExtreML/STEMMUS_SCOPEv1.0.0/input/Initial_condition/
-DurationSize=NA
+NumberOfTimeSteps=NA
+InputPath=
+OutputPath=

config_file_template.txt

@@ -0,0 +1,14 @@
+WorkDir=/path_to_working_directory/
+SoilPropertyPath=/path_to_soil_property_data/
+ForcingPath=/path_to_forcing_data/
+ForcingFileName=AU-DaS_2010-2017_OzFlux_Met.nc
+directional=/path_to_directional_data/
+fluspect_parameters=/path_to_fluspect_parameters_data/
+leafangles=/path_to_leafangles_data/
+radiationdata=/path_to_radiation_data/
+soil_spectrum=/path_to_soil_spectra_data/
+InitialConditionPath=/path_to_soil_initial_condition_data/
+input_data=/path_to_input_data.xlsx_file/
+NumberOfTimeSteps=NA
+InputPath=will_be_created_under_WorkDir
+OutputPath=will_be_created_under_WorkDir

pull_request_template.md

@@ -5,9 +5,9 @@ Please add a description of the changes proposed in the pull request.
 Please re-generate exe file if matlab codes are changed. About how to create an exe file, see [exe readme](https://github.com/EcoExtreML/STEMMUS_SCOPE/blob/main/exe/README.md).
 
 ### Checklist
-- [ ] Add a refernce to related issues.
+- [ ] Add a reference to related issues.
 - [ ] @mentions of the person or team responsible for reviewing proposed changes.
 - [ ] This pull request has a descriptive title.
-- [ ] Code is written according to the [guidelines].(http://cnl.sogang.ac.kr/cnlab/lectures/programming/matlab/Richard_Johnson-MatlabStyle2_book.pdf) 
+- [ ] Code is written according to the [guidelines](http://cnl.sogang.ac.kr/cnlab/lectures/programming/matlab/Richard_Johnson-MatlabStyle2_book.pdf).
 - [ ] Documentation is available.
 - [ ] Model runs successfully.

run_model_on_snellius/README.md

@@ -0,0 +1,9 @@
+## Run STEMMUS_SCOPE on Snellius
+
+This folder contains files that are needed for running `STEMMUS_SCOPE` on Snellius:
+
+- `run_stemmus_scope_snellius.sh` is the batch script for job submission
+- `run_model.py` is the python script for executing the workflow
+- `config_file_snellius.txt` is the model config file
+
+Before submitting the job via `sbatch run_stemmus_scope_snellius.sh`, make sure that conda environment `pystemmusscope` is created, see the [environment file](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/environment.yml). Also, set the `WorkDir` and `NumberOfTimeSteps` in the model config file e.g. `config_file_snellius.txt`.

run_model_on_snellius/config_file_snellius.txt

@@ -0,0 +1,14 @@
+WorkDir=/scratch-shared/ecoextreml/stemmus_scope/
+SoilPropertyPath=/projects/0/einf2480/model_parameters/soil_property/
+ForcingPath=/projects/0/einf2480/forcing/plumber2_data/
+ForcingFileName=FI-Hyy_1996-2014_FLUXNET2015_Met.nc
+directional=/projects/0/einf2480/model_parameters/vegetation_property/directional/
+fluspect_parameters=/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/
+leafangles=/projects/0/einf2480/model_parameters/vegetation_property/leafangles/
+radiationdata=/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/
+soil_spectrum=/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/
+input_data=/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx
+InitialConditionPath=/projects/0/einf2480/soil_initialcondition/
+NumberOfTimeSteps=17520
+InputPath=
+OutputPath=

run_model_on_snellius/run_model.py

@@ -0,0 +1,89 @@
+"""Run STEMMUS_SCOPE on Snellius.
+This script is used in a job submission file to loop over forcing files and run
+StemmusScope model on a HPC cluster using slurm workload manager. In order to
+run the model, PyStemmusScope should be installed, see
+https://pystemmusscope.readthedocs.io/.
+"""
+
+import argparse
+from pathlib import Path
+from PyStemmusScope import StemmusScope
+from PyStemmusScope import save
+
+
+def run_model(ncfile_index, job_id):
+    """Workflow executer.
+
+    Run StemmusScope model with PyStemmusScope.
+    """
+    path_to_config_file = "./config_file_snellius.txt"
+    path_to_exe_file = "./exe/STEMMUS_SCOPE"
+
+    # create an instance of the model
+    model = StemmusScope(config_file=path_to_config_file, exe_file=path_to_exe_file)
+
+    # get the forcing file
+    forcing_filenames_list = [
+        file.name for file in Path(model.config["ForcingPath"]).iterdir()
+    ]
+    # note that index starts from 1 from bash input, so ncfile_index-1
+    nc_file = forcing_filenames_list[ncfile_index - 1]
+    station_name = nc_file.split("_")[0]
+
+    # feed model with the correct forcing file
+    config_path = model.setup(ForcingFileName=nc_file)
+
+    # run model
+    model_log = model.run()
+
+    # save output in netcdf format
+    required_netcdf_variables = "../utils/csv_to_nc/required_netcdf_variables.csv"
+    netcdf_file_name = save.to_netcdf(config_path, required_netcdf_variables)
+
+    slurm_log_msg = [
+        f"Input path is {model.config['InputPath']}",
+        f"Output path is {model.config['OutputPath']}",
+        f"model log is {model_log}",
+        f"model outputs are in {netcdf_file_name}",
+    ]
+
+    slurm_file_name = (
+        Path(model.config["OutputPath"])
+        / f"slurm_{job_id}_{ncfile_index}_{station_name}.out"
+    )
+
+    # create slurm log
+    slurm_log(slurm_file_name, slurm_log_msg)
+
+
+def slurm_log(slurm_file_name, slurm_log_msg):
+    """Create slurm log file for the submitted job."""
+    with open(slurm_file_name, "w+", encoding="utf-8") as file:
+        for line in slurm_log_msg:
+            file.write(line)
+            file.write("\n")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument(
+        "-n",
+        "--ncfile_index",
+        required=True,
+        type=int,
+        default=0,
+        help="index of netCDF file",
+    )
+    parser.add_argument(
+        "-j",
+        "--job_id",
+        required=True,
+        type=int,
+        default=0,
+        help="slurm job id from snellius",
+    )
+    # get arguments
+    args = parser.parse_args()
+
+    run_model(args.ncfile_index, args.job_id)

run_model_on_snellius/run_stemmus_scope_snellius.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+# This is a batch script for Snellius at Surf
+# usage: cd STEMMUS_SCOPE/run_model_on_snellius; sbatch run_stemmus_scope_snellius.sh
+
+# SLURM settings
+#SBATCH -J stemmus_scope
+#SBATCH -t 00:10:00
+#SBATCH --nodes=1
+#SBATCH --ntasks=32
+#SBATCH -p thin
+#SBATCH --output=./slurm_%j.out
+#SBATCH --error=./slurm_%j.out
+
+### 1. Load module needed to run the model (no need for license)
+module load 2021
+
+### This is for Matlab
+module load MCR/R2021a.3
+
+### python environment pystemmusscope is needed
+### see https://pystemmusscope.readthedocs.io
+. ~/mamba/bin/activate pystemmusscope
+
+### 2. Some security: stop script on error and undefined variables
+set -euo pipefail
+
+### 3. Run parallel loop
+# number of cores on a shared node
+ncores=32 
+
+# number of forcing files
+nfiles=170
+
+# some indices for loop
+k=0
+i=0
+
+for k in `seq 0 5`; do
+  for j in `seq 1 $ncores` ; do
+  (
+    i=$(( ncores * k + j ))
+    if [[ $i -le $nfiles ]]; then
+      python run_model.py -n $i -j ${SLURM_JOB_ID}
+    fi
+  )&
+  done
+  wait
+done