Merge PR NOAA-EMC#2204 branch into PR NOAA-EMC#2214 branch

Merge remote-tracking branch 'upstream/feature/setup_expt_oceanres' into feature/clean_out_rotdir * upstream/feature/setup_expt_oceanres: Update docs/doxygen/mainpage.h update default resdetocean update documentation instances of resdet and resens; add resdetocean update yamls for ci tests fix pynorm error add option to provide an ocean resolution during setup_expt.py
KateFriedman-NOAA · Jan 11, 2024 · 731076e · 731076e
2 parents 236f1a5 + 15845c3
commit 731076e
Show file tree

Hide file tree

Showing 15 changed files with 80 additions and 64 deletions.
diff --git a/ci/cases/pr/C48_ATM.yaml b/ci/cases/pr/C48_ATM.yaml
@@ -5,7 +5,7 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 48
+  resdetatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312

diff --git a/ci/cases/pr/C48_S2SW.yaml b/ci/cases/pr/C48_S2SW.yaml
@@ -5,7 +5,8 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SW
-  resdet: 48
+  resdetatmos: 48
+  resdetocean: 5.0
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312

diff --git a/ci/cases/pr/C48_S2SWA_gefs.yaml b/ci/cases/pr/C48_S2SWA_gefs.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 48
-  resens: 48
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
   nens: 2
   gfs_cyc: 1
   start: cold

diff --git a/ci/cases/pr/C96C48_hybatmDA.yaml b/ci/cases/pr/C96C48_hybatmDA.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
-  resens: 48
+  resdetatmos: 96
+  resdetocean: 5.0
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48

diff --git a/ci/cases/pr/C96_atm3DVar.yaml b/ci/cases/pr/C96_atm3DVar.yaml
@@ -5,10 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
+  resdetatmos: 96
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: ${ICSDIR_ROOT}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv ]}/C96C48
   idate: 2021122018
   edate: 2021122106
   nens: 0

diff --git a/ci/cases/weekly/C384C192_hybatmda.yaml b/ci/cases/weekly/C384C192_hybatmda.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192

diff --git a/ci/cases/weekly/C384_S2SWA.yaml b/ci/cases/weekly/C384_S2SWA.yaml
@@ -5,7 +5,8 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 384
+  resdetatmos: 384
+  resdetocean: 0.5
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2016070100

diff --git a/ci/cases/weekly/C384_atm3DVar.yaml b/ci/cases/weekly/C384_atm3DVar.yaml
@@ -5,8 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192

diff --git a/ci/platforms/gefs_ci_defaults.yaml b/ci/platforms/gefs_ci_defaults.yaml
@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
diff --git a/ci/platforms/gfs_defaults_ci.yaml b/ci/platforms/gfs_defaults_ci.yaml
@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}
diff --git a/docs/doxygen/mainpage.h b/docs/doxygen/mainpage.h
@@ -23,7 +23,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
     $> setup_expt.py -h
     usage: setup_expt.py [-h] --pslot PSLOT
                      [--configdir CONFIGDIR] [--idate IDATE] [--icsdir ICSDIR]
-                     [--resdet RESDET] [--resens RESENS] [--comroot COMROOT]
+                     [--resdetatmos RESDET] [--resensatmos RESENS] [--comroot COMROOT]
                      [--expdir EXPDIR] [--nens NENS] [--cdump CDUMP]
 
     Setup files and directories to start a GFS parallel. Create EXPDIR, copy
@@ -40,9 +40,11 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
                         (default: 2016100100)
         --icsdir     full path to initial condition directory
                         (default: /scratch4/NCEPDEV/da/noscrub/Rahul.Mahajan/ICS)
-        --resdet     resolution of the deterministic model forecast
+        --resdetatmos atmosphere resolution of the deterministic model forecast
                         (default: 384)
-        --resens     resolution of the ensemble model forecast
+        --resdetocean ocean resolution of the deterministic model forecast
+                        (default: 0. [determined automatically based on atmosphere resolution])
+        --resensatmos     resolution of the ensemble model forecast
                         (default: 192)
         --comroot    full path to COMROOT, where ROTDIR (COMROOT+PSLOT) will be created
                         (default: None)
@@ -53,7 +55,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
         --cdump      CDUMP to start the experiment
                         (default: gdas)
 
-The above script creates directories <b>\c EXPDIR</b> and <b>\c ROTDIR</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdet</b> and the ensemble <b>\c --resens</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
+The above script creates directories <b>\c EXPDIR</b> and <b>\c ROTDIR</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdetatmos</b> and the ensemble <b>\c --resensatmos</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
 
 Sample initial conditions for a few resolutions are available at:<br>
 <b>Hera:</b> TODO: /path/here/for/initial/conditions<br>

diff --git a/docs/source/setup.rst b/docs/source/setup.rst
@@ -28,7 +28,7 @@ The following command examples include variables for reference but users should
 ::
 
    cd workflow
-   ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdet $RESDET]
+   ./setup_expt.py gfs forecast-only --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC] [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN]
      [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR]
 
 where:
@@ -50,7 +50,8 @@ where:
    * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete
    * ``$PSLOT`` is the name of your experiment [default: test]
    * ``$CONFIGDIR`` is the path to the ``/config`` folder under the copy of the system you're using [default: $TOP_OF_CLONE/parm/config/]
-   * ``$RESDET`` is the FV3 resolution (i.e. 768 for C768) [default: 384]
+   * ``$RESDETATMOS`` is the resolution of the atmosphere component of the system (i.e. 768 for C768) [default: 384]
+   * ``$RESDETOCEAN`` is the resolution of the ocean component of the system (i.e. 0.25 for 1/4 degree) [default: 0.; determined based on atmosphere resolution]
    * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles)
    * ``$COMROOT`` is the path to your experiment output directory. Your ``ROTDIR`` (rotating com directory) will be created using ``COMROOT`` and ``PSLOT``. [default: $HOME (but do not use default due to limited space in home directories normally, provide a path to a larger scratch space)]
    * ``$EXPDIR`` is the path to your experiment directory where your configs will be placed and where you will find your workflow monitoring files (i.e. rocoto database and xml file). DO NOT include PSLOT folder at end of path, it will be built for you. [default: $HOME]
@@ -62,21 +63,21 @@ Atm-only:
 ::
 
    cd workflow
-   ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdet 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
+   ./setup_expt.py gfs forecast-only --pslot test --idate 2020010100 --edate 2020010118 --resdetatmos 384 --gfs_cyc 4 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
 
 Coupled:
 
 ::
 
    cd workflow
-   ./setup_expt.py gfs forecast-only --app S2SW --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdet 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
+   ./setup_expt.py gfs forecast-only --app S2SW --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdetatmos 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
 
 Coupled with aerosols:
 
 ::
 
    cd workflow
-   ./setup_expt.py gfs forecast-only --app S2SWA --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdet 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
+   ./setup_expt.py gfs forecast-only --app S2SWA --pslot coupled_test --idate 2013040100 --edate 2013040100 --resdetatmos 384 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir
 
 ****************************************
 Step 2: Set user and experiment settings
@@ -134,13 +135,13 @@ Scripts that will be used:
 Step 1) Run experiment generator script
 ***************************************
 
-The following command examples include variables for reference but users should not use environmental variables but explicit values to submit the commands. Exporting variables like EXPDIR to your environment causes an error when the python scripts run. Please explicitly include the argument inputs when running both setup scripts:
+The following command examples include variables for reference but users should not use environment variables but explicit values to submit the commands. Exporting variables like EXPDIR to your environment causes an error when the python scripts run. Please explicitly include the argument inputs when running both setup scripts:
 
 ::
 
    cd workflow
    ./setup_expt.py gfs cycled --idate $IDATE --edate $EDATE [--app $APP] [--start $START] [--gfs_cyc $GFS_CYC]
-     [--resdet $RESDET] [--resens $RESENS] [--nens $NENS] [--cdump $CDUMP]
+     [--resdetatmos $RESDETATMOS] [--resdetocean $RESDETOCEAN] [--resensatmos $RESENSATMOS] [--nens $NENS] [--cdump $CDUMP]
      [--pslot $PSLOT] [--configdir $CONFIGDIR] [--comroot $COMROOT] [--expdir $EXPDIR] [--icsdir $ICSDIR]
 
 where:
@@ -161,8 +162,9 @@ where:
    * ``$EDATE`` is the ending date of your run (YYYYMMDDCC) and is the last cycle that will complete
    * ``$START`` is the start type (warm or cold [default])
    * ``$GFS_CYC`` is the forecast frequency (0 = none, 1 = 00z only [default], 2 = 00z & 12z, 4 = all cycles)
-   * ``$RESDET`` is the FV3 resolution of the deterministic forecast [default: 384]
-   * ``$RESENS`` is the FV3 resolution of the ensemble (EnKF) forecast [default: 192]
+   * ``$RESDETATMOS`` is the resolution of the atmosphere component of the deterministic forecast [default: 384]
+   * ``$RESDETOCEAN`` is the resolution of the ocean component of the deterministic forecast [default: 0.; determined based on atmosphere resolution]
+   * ``$RESENSATMOS`` is the resolution of the atmosphere component of the ensemble forecast [default: 192]
    * ``$NENS`` is the number of ensemble members [default: 20]
    * ``$CDUMP`` is the starting phase [default: gdas]
    * ``$PSLOT`` is the name of your experiment [default: test]
@@ -178,13 +180,13 @@ Example:
 ::
 
    cd workflow
-   ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdet 384 --resens 192 --nens 80 --gfs_cyc 4
+   ./setup_expt.py gfs cycled --pslot test --configdir /home/Joe.Schmo/git/global-workflow/parm/config --idate 2020010100 --edate 2020010118 --comroot /some_large_disk_area/Joe.Schmo/comroot --expdir /some_safe_disk_area/Joe.Schmo/expdir --resdetatmos 384 --resensatmos 192 --nens 80 --gfs_cyc 4
 
 Example ``setup_expt.py`` on Orion:
 
 ::
 
-   Orion-login-3$ ./setup_expt.py gfs cycled --pslot test --idate 2022010118 --edate 2022010200 --resdet 192 --resens 96 --nens 80 --comroot /work/noaa/stmp/jschmo/comroot --expdir /work/noaa/global/jschmo/expdir
+   Orion-login-3$ ./setup_expt.py gfs cycled --pslot test --idate 2022010118 --edate 2022010200 --resdetatmos 192 --resensatmos 96 --nens 80 --comroot /work/noaa/stmp/jschmo/comroot --expdir /work/noaa/global/jschmo/expdir
    EDITED:  /work/noaa/global/jschmo/expdir/test/config.base as per user input.
    EDITED:  /work/noaa/global/jschmo/expdir/test/config.aeroanl as per user input.
    EDITED:  /work/noaa/global/jschmo/expdir/test/config.ocnanal as per user input.
@@ -195,7 +197,7 @@ What happens if I run ``setup_expt.py`` again for an experiment that already exi
 
 ::
 
-   Orion-login-3$ ./setup_expt.py gfs cycled --pslot test --idate 2022010118 --edate 2022010200 --resdet 192 --resens 96 --nens 80 --comroot /work/noaa/stmp/jschmo/comroot --expdir /work/noaa/global/jschmo/expdir
+   Orion-login-3$ ./setup_expt.py gfs cycled --pslot test --idate 2022010118 --edate 2022010200 --resdetatmos 192 --resensatmos 96 --nens 80 --comroot /work/noaa/stmp/jschmo/comroot --expdir /work/noaa/global/jschmo/expdir
 
    directory already exists in /work/noaa/stmp/jschmo/comroot/test
 

diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn
@@ -154,35 +154,27 @@ export OPS_RES="C768" # Do not change
 export LEVS=128
 export CASE="@CASECTL@"  # CASE is required in GEFS to determine ocean/ice/wave resolutions
 export CASE_ENS="@CASEENS@"
-# TODO: This should not depend on $CASE or $CASE_ENS
-# These are the currently available grid-combinations
+export OCNRES="@OCNRES@"
+export ICERES="${OCNRES}"
+# These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export OCNRES=500
         export waveGRD='glo_500'
         ;;
-    "C96")
-        export OCNRES=100
-        export waveGRD='glo_200'
-        ;;
-    "C192")
-        export OCNRES=050
+    "C96" | "C192")
         export waveGRD='glo_200'
         ;;
     "C384")
-        export OCNRES=025
         export waveGRD='glo_025'
         ;;
-    "C768")
-        export OCNRES=025
-        export waveGRD='mx025'
+    "C768" | "C1152")
+           export waveGRD='mx025'
         ;;
     *)
-        export OCNRES=025
-        export waveGRD='glo_025'
+        echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"
+        exit 1
         ;;
 esac
-export ICERES=${OCNRES}
 
 case "${APP}" in
   ATM)

diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
@@ -177,35 +177,27 @@ export OPS_RES="C768" # Do not change # TODO: Why is this needed and where is it
 export LEVS=128
 export CASE="@CASECTL@"
 export CASE_ENS="@CASEENS@"
-# TODO: This should not depend on $CASE or $CASE_ENS
-# These are the currently available grid-combinations
+export OCNRES="@OCNRES@"
+export ICERES="${OCNRES}"
+# These are the currently recommended grid-combinations
 case "${CASE}" in
     "C48")
-        export OCNRES=500
         export waveGRD='glo_500'
         ;;
-    "C96")
-        export OCNRES=500
-        export waveGRD='glo_200'
-        ;;
-    "C192")
-        export OCNRES=050
+    "C96" | "C192")
         export waveGRD='glo_200'
         ;;
     "C384")
-        export OCNRES=025
         export waveGRD='glo_025'
         ;;
     "C768" | "C1152")
-        export OCNRES=025
         export waveGRD='mx025'
         ;;
     *)
         echo "FATAL ERROR: Unrecognized CASE ${CASE}, ABORT!"
         exit 1
         ;;
 esac
-export ICERES=${OCNRES}
 
 case "${APP}" in
   ATM)

diff --git a/workflow/setup_expt.py b/workflow/setup_expt.py
@@ -309,7 +309,8 @@ def edit_baseconfig(host, inputs, yaml_dict):
         "@PSLOT@": inputs.pslot,
         "@SDATE@": datetime_to_YMDH(inputs.idate),
         "@EDATE@": datetime_to_YMDH(inputs.edate),
-        "@CASECTL@": f'C{inputs.resdet}',
+        "@CASECTL@": f'C{inputs.resdetatmos}',
+        "@OCNRES@": f"{int(100.*inputs.resdetocean):03d}",
         "@EXPDIR@": inputs.expdir,
         "@COMROOT@": inputs.comroot,
         "@EXP_WARM_START@": is_warm_start,
@@ -322,7 +323,7 @@ def edit_baseconfig(host, inputs, yaml_dict):
     extend_dict = dict()
     if getattr(inputs, 'nens', 0) > 0:
         extend_dict = {
-            "@CASEENS@": f'C{inputs.resens}',
+            "@CASEENS@": f'C{inputs.resensatmos}',
             "@NMEM_ENS@": inputs.nens,
         }
         tmpl_dict = dict(tmpl_dict, **extend_dict)
@@ -387,8 +388,10 @@ def input_args(*argv):
     def _common_args(parser):
         parser.add_argument('--pslot', help='parallel experiment name',
                             type=str, required=False, default='test')
-        parser.add_argument('--resdet', help='resolution of the deterministic model forecast',
+        parser.add_argument('--resdetatmos', help='atmosphere resolution of the deterministic model forecast',
                             type=int, required=False, default=384)
+        parser.add_argument('--resdetocean', help='ocean resolution of the deterministic model forecast',
+                            type=float, required=False, default=0.0)  # 0.0 (or lower) means determine from resdetatmos (limited combinations will be available)
         parser.add_argument('--comroot', help='full path to COMROOT',
                             type=str, required=False, default=os.getenv('HOME'))
         parser.add_argument('--expdir', help='full path to EXPDIR',
@@ -418,7 +421,7 @@ def _gfs_cycled_args(parser):
         return parser
 
     def _gfs_or_gefs_ensemble_args(parser):
-        parser.add_argument('--resens', help='resolution of the ensemble model forecast',
+        parser.add_argument('--resensatmos', help='atmosphere resolution of the ensemble model forecast',
                             type=int, required=False, default=192)
         parser.add_argument('--nens', help='number of ensemble members',
                             type=int, required=False, default=20)
@@ -512,7 +515,7 @@ def query_and_clean(dirname):
 def validate_user_request(host, inputs):
     supp_res = host.info['SUPPORTED_RESOLUTIONS']
     machine = host.machine
-    for attr in ['resdet', 'resens']:
+    for attr in ['resdetatmos', 'resensatmos']:
         try:
             expt_res = f'C{getattr(inputs, attr)}'
         except AttributeError:
@@ -521,13 +524,32 @@ def validate_user_request(host, inputs):
             raise NotImplementedError(f"Supported resolutions on {machine} are:\n{', '.join(supp_res)}")
 
 
+def get_ocean_resolution(resdetatmos):
+    """
+    Method to determine the ocean resolution based on the atmosphere resolution
+    Limited options are going to be available
+    """
+    atmos_to_ocean_map = {
+        1152: 0.25, 768: 0.25, 384: 0.25,
+        192: 1.0,
+        96: 5.0, 48: 5.0}
+    try:
+        return atmos_to_ocean_map[resdetatmos]
+    except KeyError:
+        raise KeyError(f"Ocean resolution for {resdetatmos} is not implemented")
+
+
 def main(*argv):
 
     user_inputs = input_args(*argv)
     host = Host()
 
     validate_user_request(host, user_inputs)
 
+    # Determine ocean resolution if not provided
+    if user_inputs.resdetocean <= 0:
+        user_inputs.resdetocean = get_ocean_resolution(user_inputs.resdetatmos)
+
     rotdir = os.path.join(user_inputs.comroot, user_inputs.pslot)
     expdir = os.path.join(user_inputs.expdir, user_inputs.pslot)