Merge branch 'NOAA-EMC:develop' into develop

.github/workflows/docs.yaml

@@ -17,7 +17,7 @@ on:
 jobs:
 
   documentation:
-    
+
     permissions:
       pull-requests: 'write'
 
@@ -56,7 +56,8 @@ jobs:
         path: artifact/doc_warnings.log
         if-no-files-found: ignore
 
-    - name: Comment ReadDocs
+    - name: Comment ReadDocs Link in PR
+      if: github.event_name == 'pull_request'
       uses: actions/github-script@v6
       with:
         script: |
@@ -68,5 +69,5 @@ jobs:
            issue_number: context.issue.number,
            owner: context.repo.owner,
            repo: context.repo.repo,
-           body: message 
-          })     
+           body: message
+          })

.gitignore

@@ -91,6 +91,11 @@ parm/post/postxconfig-NT-GFS.txt
 parm/post/postxconfig-NT-gefs-aerosol.txt
 parm/post/postxconfig-NT-gefs-chem.txt
 parm/ufs/noahmptable.tbl
+parm/ufs/model_configure.IN
+parm/ufs/MOM_input_*.IN
+parm/ufs/MOM6_data_table.IN
+parm/ufs/ice_in.IN
+parm/ufs/ufs.configure.*.IN
 parm/wafs
 
 # Ignore sorc and logs folders from externals

ci/cases/pr/C48_ATM.yaml

@@ -5,9 +5,9 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C48_S2SW.yaml

@@ -5,9 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SW
-  resdet: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 48
+  resdetocean: 5.0
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C48_S2SWA_gefs.yaml

@@ -5,13 +5,14 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 48
-  resens: 48
+  resdetatmos: 48
+  resdetocean: 5.0
+  resensatmos: 48
   nens: 2
   gfs_cyc: 1
   start: cold
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2021032312
   edate: 2021032312
-  yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gefs_ci_defaults.yaml

ci/cases/pr/C96C48_hybatmDA.yaml

@@ -5,17 +5,15 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
-  resens: 48
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 96
+  resdetocean: 5.0
+  resensatmos: 48
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
   idate: 2021122018
   edate: 2021122106
   nens: 2
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96_atm3DVar.yaml

@@ -5,16 +5,13 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 96
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: ${ICSDIR_ROOT}/C96C48
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
   idate: 2021122018
   edate: 2021122106
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
-
-skip_ci_on_hosts:
-  - hercules
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/pr/C96_atmsnowDA.yaml

@@ -0,0 +1,21 @@
+experiment:
+  system: gfs
+  mode: cycled
+
+arguments:
+  pslot: {{ 'pslot' | getenv }}
+  app: ATM
+  resdetatmos: 96
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
+  expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
+  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48
+  idate: 2021122012
+  edate: 2021122100
+  nens: 0
+  gfs_cyc: 1
+  start: cold
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/atmsnowDA_defaults_ci.yaml
+
+skip_ci_on_hosts:
+  - orion
+  - hercules

ci/cases/weekly/C384C192_hybatmda.yaml

@@ -5,14 +5,15 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 2
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/weekly/C384_S2SWA.yaml

@@ -5,9 +5,10 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: S2SWA
-  resdet: 384
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.25
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   idate: 2016070100
   edate: 2016070100
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/weekly/C384_atm3DVar.yaml

@@ -5,14 +5,15 @@ experiment:
 arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATM
-  resdet: 384
-  resens: 192
-  comrot: {{ 'RUNTESTS' | getenv }}/COMROT
+  resdetatmos: 384
+  resdetocean: 0.25
+  resensatmos: 192
+  comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C384C192
   idate: 2023040118
   edate: 2023040200
   nens: 0
   gfs_cyc: 1
   start: cold
-  yaml: {{ HOMEgfs }}/ci/platforms/gfs_defaults_ci.yaml
+  yaml: {{ HOMEgfs }}/ci/cases/yamls/gfs_defaults_ci.yaml

ci/cases/yamls/atmsnowDA_defaults_ci.yaml

@@ -0,0 +1,6 @@
+defaults:
+  !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
+base:
+  DOIAU: "NO"
+  DO_JEDILANDDA: "YES"
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

ci/platforms/gefs_ci_defaults.yaml → ci/cases/yamls/gefs_ci_defaults.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gefs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

ci/platforms/gfs_defaults_ci.yaml → ci/cases/yamls/gfs_defaults_ci.yaml

@@ -1,4 +1,4 @@
 defaults:
   !INC {{ HOMEgfs }}/parm/config/gfs/yaml/defaults.yaml
 base:
-  ACCOUNT: ${SLURM_ACCOUNT}
+  ACCOUNT: {{ 'SLURM_ACCOUNT' | getenv }}

ci/scripts/check_ci.sh

@@ -22,7 +22,7 @@ REPO_URL="https://github.com/NOAA-EMC/global-workflow.git"
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;
@@ -149,7 +149,7 @@ for pr in ${pr_list}; do
     if [[ "${num_done}" -eq  "${num_cycles}" ]]; then
       #Remove Experment cases that completed successfully
       rm -Rf "${pslot_dir}"
-      rm -Rf "${pr_dir}/RUNTESTS/COMROT/${pslot}"
+      rm -Rf "${pr_dir}/RUNTESTS/COMROOT/${pslot}"
       rm -f "${output_ci_single}"
       # echo "\`\`\`" > "${output_ci_single}"
       DATE=$(date +'%D %r')

ci/scripts/driver.sh

@@ -34,7 +34,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;
@@ -173,7 +173,7 @@ for pr in ${pr_list}; do
   # we need to exit this instance of the driver script
   #################################################################
   if [[ ${ci_status} -ne 0 ]]; then
-     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "{pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
+     build_PID_check=$("${ROOT_DIR}/ci/scripts/pr_list_database.py" --display "${pr}" --dbfile "${pr_list_dbfile}" | awk '{print $4}' | cut -d":" -f1) || true
      if [[ "${build_PID_check}" -ne "$$" ]]; then
         echo "Driver build PID: ${build_PID_check} no longer running this build ... exiting"
         exit 0

ci/scripts/driver_weekly.sh

@@ -38,7 +38,7 @@ export PS4='+ $(basename ${BASH_SOURCE[0]})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
     echo "Running Automated Testing on ${MACHINE_ID}"
     source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
     ;;

ci/scripts/run-check_ci.sh

@@ -12,12 +12,12 @@ pslot=${2:-${pslot:-?}}        # Name of the experiment being tested by this scr
 
 # TEST_DIR contains 2 directories;
 # 1. HOMEgfs: clone of the global-workflow
-# 2. RUNTESTS: A directory containing EXPDIR and COMROT for experiments
+# 2. RUNTESTS: A directory containing EXPDIR and COMROOT for experiments
 # # e.g. $> tree ./TEST_DIR
 # ./TEST_DIR
 # ├── HOMEgfs
 # └── RUNTESTS
-#     ├── COMROT
+#     ├── COMROOT
 #     │   └── ${pslot}
 #     └── EXPDIR
 #         └── ${pslot}

ci/scripts/run_ci.sh

@@ -20,7 +20,7 @@ export PS4='+ $(basename ${BASH_SOURCE})[${LINENO}]'
 
 source "${ROOT_DIR}/ush/detect_machine.sh"
 case ${MACHINE_ID} in
-  hera | orion)
+  hera | orion | hercules)
    echo "Running Automated Testing on ${MACHINE_ID}"
    source "${ROOT_DIR}/ci/platforms/config.${MACHINE_ID}"
    ;;

docs/doxygen/mainpage.h

@@ -12,7 +12,7 @@
 With the FV3GFS system, a Rocoto driven workflow is being used. This workflow builds on the experiences and work of Kate.Howard and Terry.McGuinness for the GSM, but strips down a lot of complications arising from the use of a cumbersome <b>\c para_config</b>. The <b>\c para_config</b> no longer exists in the workflow and instead a number of <b> \c config </b> files are added, one each for an individual task e.g. <b>\p config.anal</b> contains the analysis specific information. A base config called as <b>\c config.base</b> contains
 information related to the machine, super-structure, etc. The idea behind splitting the <b>\p para_config</b> into individual, smaller and managable configs is to provide a means to run any chosen task without the overhead of the full cycling framework. All the configs are located under <b>\c fv3gfs/config</b>
 
-Additionally, the structure of the <b>\c COMROT</b> directory is now modified to look like operations. This enables the use of the workflow much closer to the operational environment, with the exception of the workflow manager.
+Additionally, the structure of the <b>\c ROTDIR</b> directory is now modified to look like operations. This enables the use of the workflow much closer to the operational environment, with the exception of the workflow manager.
 
 This is a very much a work in progress and any issues should be reported back and are greatly appreciated.
 
@@ -23,12 +23,12 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
     $> setup_expt.py -h
     usage: setup_expt.py [-h] --pslot PSLOT
                      [--configdir CONFIGDIR] [--idate IDATE] [--icsdir ICSDIR]
-                     [--resdet RESDET] [--resens RESENS] [--comrot COMROT]
+                     [--resdetatmos RESDET] [--resensatmos RESENS] [--comroot COMROOT]
                      [--expdir EXPDIR] [--nens NENS] [--cdump CDUMP]
 
     Setup files and directories to start a GFS parallel. Create EXPDIR, copy
-    config files Create COMROT experiment directory structure, link initial
-    condition files from $ICSDIR to $COMROT
+    config files Create ROTDIR experiment directory structure, link initial
+    condition files from $ICSDIR to $ROTDIR
 
     optional arguments:
         -h, --help   show this help message and exit
@@ -40,11 +40,13 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
                         (default: 2016100100)
         --icsdir     full path to initial condition directory
                         (default: /scratch4/NCEPDEV/da/noscrub/Rahul.Mahajan/ICS)
-        --resdet     resolution of the deterministic model forecast
+        --resdetatmos atmosphere resolution of the deterministic model forecast
                         (default: 384)
-        --resens     resolution of the ensemble model forecast
+        --resdetocean ocean resolution of the deterministic model forecast
+                        (default: 0. [determined automatically based on atmosphere resolution])
+        --resensatmos     resolution of the ensemble model forecast
                         (default: 192)
-        --comrot     full path to COMROT
+        --comroot    full path to COMROOT, where ROTDIR (COMROOT+PSLOT) will be created
                         (default: None)
         --expdir     full path to EXPDIR
                         (default: None)
@@ -53,7 +55,7 @@ To setup an experiment, a python script <b>\c setup_expt.py</b> (located in <b>\
         --cdump      CDUMP to start the experiment
                         (default: gdas)
 
-The above script creates directories <b>\c EXPDIR</b> and <b>\c COMROT</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdet</b> and the ensemble <b>\c --resens</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
+The above script creates directories <b>\c EXPDIR</b> and <b>\c ROTDIR</b>. It will make links for initial conditions from a location provided via the <b>\c --icsdir</b> argument for a chosen resolution for the control <b>\c --resdetatmos</b> and the ensemble <b>\c --resensatmos</b>. Experiment name is controlled by the input argument <b>\c --pslot</b>. The script will ask user input in case any of the directories already exist. It will copy experiment configuration files into the <b>\c EXPDIR</b> from <b>\c CONFIGDIR</b>.
 
 Sample initial conditions for a few resolutions are available at:<br>
 <b>Hera:</b> TODO: /path/here/for/initial/conditions<br>

docs/source/errors_faq.rst

@@ -36,10 +36,10 @@ Example::
 **Solution:** set TERM to "xterm" (bash: export TERM=xterm ; csh/tcsh: setenv TERM xterm)
 
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
-Issue: Directory name change for EnKF folder in COMROT
+Issue: Directory name change for EnKF folder in ROTDIR
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 
-**Issue:** The EnKF COMROT folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
+**Issue:** The EnKF ROTDIR folders were renamed during the GFS v15 development process to remove the period between "enkf" and "gdas": enkf.gdas.$PDY → enkfgdas.$PDY
 
 **Fix:** Older tarballs on HPSS will have the older directory name with the period between 'enkf' and 'gdas'. Make sure to rename folder to 'enkfgdas.$PDY' after obtaining. Only an issue for the initial cycle.
 

docs/source/index.rst

@@ -10,7 +10,7 @@ Status
 ======
 
 * State of develop (HEAD) branch: GFSv17+ development
-* State of operations (dev/gfs.v16 branch): GFS v16.3.12 `tag: [gfs.v16.3.12] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.12>`_
+* State of operations (dev/gfs.v16 branch): GFS v16.3.13 `tag: [gfs.v16.3.13] <https://github.com/NOAA-EMC/global-workflow/releases/tag/gfs.v16.3.13>`_
 
 =============
 Code managers