Merge branch 'develop' into feature/coupled_ic

ci/cases/C96C48_hybatmDA.yaml

@@ -10,7 +10,7 @@ arguments:
   expdir: ${RUNTESTS}/${pslot}/EXPDIR
   icsdir: ${ICSDIR_ROOT}/C96C48
   idate: 2021122018
-  edate: 2021122200
+  edate: 2021122106
   nens: 2
   gfs_cyc: 1
   start: cold

ci/cases/C96_atm3DVar.yaml

@@ -9,7 +9,7 @@ arguments:
   expdir: ${RUNTESTS}/${pslot}/EXPDIR
   icsdir: ${ICSDIR_ROOT}/C96C48
   idate: 2021122018
-  edate: 2021122100
+  edate: 2021122106
   nens: 0
   gfs_cyc: 1
   start: cold

docs/source/clone.rst

@@ -31,6 +31,19 @@ For cycled (w/ data assimilation):
    ./build_all.sh
    ./link_workflow.sh
 
+For coupled cycling (include new UFSDA):
+
+[Currently only available on Hera and Orion]
+
+::
+
+   git clone https://github.com/NOAA-EMC/global-workflow.git
+   cd global-workflow/sorc
+   ./checkout.sh -gu
+   ./build_all.sh
+   ./link_workflow.sh
+
+
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
 Clone workflow and component repositories
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
@@ -104,6 +117,14 @@ Or with the ``-g`` switch to include data assimilation (GSI) for cycling:
    cd sorc
    ./checkout.sh -g
 
+Or also with the ``-u`` swtich to include coupled DA (via UFSDA):
+[Currently only available on Hera and Orion]
+
+::
+
+   cd sorc
+   ./checkout.sh -gu
+
 Each component cloned via checkout.sh will have a log (``/sorc/logs/checkout-COMPONENT.log``). Check the screen output and logs for clone errors.
 
 ^^^^^^^^^^^^^^^^

docs/source/setup.rst

@@ -2,69 +2,13 @@
 Experiment Setup
 ================
 
- Global workflow uses a set of scripts to help configure and set up the drivers (also referred to as Workflow Manager) that run the end-to-end system. While currently we use a `ROCOTO <https://github.com/christopherwharrop/rocoto/wiki/documentation>`__ based system and that is documented here, an `ecFlow <https://www.ecmwf.int/en/learning/training/introduction-ecmwf-job-scheduler-ecflow>`__ based systm is also under development and will be introduced to the Global Workflow when it is mature. To run the setup scripts, you need to make sure to have a copy of ``python3`` with ``numpy`` available. The easiest way to guarantee this is to load python from the `official hpc-stack installation <https://github.com/NOAA-EMC/hpc-stack/wiki/Official-Installations>`_ for the machine you are on:
+ Global workflow uses a set of scripts to help configure and set up the drivers (also referred to as Workflow Manager) that run the end-to-end system. While currently we use a `ROCOTO <https://github.com/christopherwharrop/rocoto/wiki/documentation>`__ based system and that is documented here, an `ecFlow <https://www.ecmwf.int/en/learning/training/introduction-ecmwf-job-scheduler-ecflow>`__ based systm is also under development and will be introduced to the Global Workflow when it is mature. To run the setup scripts, you need to have rocoto and a python3 environment with several specific libraries. The easiest way to guarantee this is to source the following script, which will load the necessary modules for your machine:
 
-.. list-table:: Python Module Load Commands
-   :widths: 25 120
-   :header-rows: 1
+ ::
 
-   * - **MACHINE**
-     - **COMMAND(S)**
-   * - Hera
-     - ::
+   # Note: this will wipe your existing lmod environment
+   source workflow/gw_setup.sh
 
-           module use -a /contrib/anaconda/modulefiles
-           module load anaconda/anaconda3-5.3.1
-   * - Orion
-     - ::
-
-           module load python/3.7.5
-   * - WCOSS2
-     - ::
-
-           module load python/3.8.6
-   * - S4
-     - ::
-
-           module load miniconda/3.8-s4
-
-   * - Jet
-     - ::
-
-           module use /mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles
-           module load miniconda3/4.12.0
-           conda activate ufswm
-
-If running with Rocoto make sure to have a Rocoto module loaded before running setup scripts:
-
-.. list-table:: ROCOTO Module Load Commands
-   :widths: 25 120
-   :header-rows: 1
-
-   * - **MACHINE**
-     - **COMMAND(S)**
-   * - Hera
-     - ::
-
-           module load rocoto/1.3.3
-   * - Orion
-     - ::
-
-           module load contrib
-           module load rocoto/1.3.3
-   * - WCOSS2
-     - ::
-
-           module use /apps/ops/test/nco/modulefiles/
-           module load core/rocoto/1.3.5
-   * - S4
-     - ::
-
-           module load rocoto/1.3.4
-   * - Jet
-     - ::
-
-           module load rocoto/1.3.3
 
 ^^^^^^^^^^^^^^^^^^^^^^^^
 Forecast-only experiment

jobs/rocoto/ocnpost.sh

@@ -56,7 +56,7 @@ for fhr in ${fhrlst}; do
   # shellcheck disable=
   declare -x VDATE
   cd "${DATA}" || exit 2
-  if (( fhr > 0 )); then
+  if (( 10#${fhr} > 0 )); then
     # TODO: This portion calls NCL scripts that are deprecated (see Issue #923)
     if [[ "${MAKE_OCN_GRIB:-YES}" == "YES" ]]; then
       export MOM6REGRID=${MOM6REGRID:-${HOMEgfs}}

modulefiles/module_gwsetup.jet.lua

@@ -0,0 +1,22 @@
+help([[
+Load environment to run GFS workflow setup scripts on Jet
+]])
+
+load(pathJoin("rocoto", "1.3.3"))
+
+if (mode() == "unload") then
+    -- `execute` delays commands until last, but we need conda deactivated
+    --   before unloading miniconda. `print` (bizarrely) still executes the
+    --   command, but does it immediately. The semicolon is necessary 
+    --   because otherwise other commands get tacked onto the same line.
+    print("conda deactivate;")
+end
+
+-- Temporary until official hpc-stack is updated
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles")
+load(pathJoin("miniconda3", "4.12.0"))
+if (mode() == "load") then
+    execute{cmd="conda activate ufswm", modeA={"load"}}
+end
+
+whatis("Description: GFS run setup environment")

modulefiles/module_gwsetup.orion.lua

@@ -2,13 +2,12 @@ help([[
 Load environment to run GFS workflow ci scripts on Orion
 ]])
 
--- Temporary until official hpc-stack is updated
-
 prepend_path("MODULEPATH", "/apps/modulefiles/core")
 load(pathJoin("contrib","0.1"))
 load(pathJoin("rocoto","1.3.3"))
 load(pathJoin("git","2.28.0"))
 
+-- Temporary until official hpc-stack is updated
 prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")
 load(pathJoin("hpc", "1.2.0"))
 load(pathJoin("hpc-miniconda3", "4.6.14"))

modulefiles/module_gwsetup.s4.lua

@@ -0,0 +1,9 @@
+help([[
+Load environment to run GFS workflow setup scripts on S4
+]])
+
+load(pathJoin("miniconda", "3.8-s4"))
+load(pathJoin("rocoto","1.3.5"))
+load(pathJoin("git","2.30.0"))
+
+whatis("Description: GFS run setup environment")

modulefiles/module_gwsetup.wcoss2.lua

@@ -0,0 +1,10 @@
+help([[
+Load environment to run GFS workflow ci scripts on WCOSS2
+]])
+
+load(pathJoin("git","2.29.0"))
+
+prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core")
+load(pathJoin("rocoto","1.3.5"))
+
+whatis("Description: GFS run ci top-level sripts environment")

parm/config/gfs/config.base.emc.dyn

@@ -76,7 +76,7 @@ export MODE="@MODE@" # cycled/forecast-only
 # Build paths relative to $HOMEgfs
 export FIXgsi="${HOMEgfs}/fix/gsi"
 export HOMEpost="${HOMEgfs}"
-export HOMEobsproc="${BASE_GIT}/obsproc/v${obsproc_run_ver:-1.1.2}"
+export HOMEobsproc="${BASE_GIT:-}/obsproc/v${obsproc_run_ver:-1.1.2}"
 
 # CONVENIENT utility scripts and other environment parameters
 export NCP="/bin/cp -p"
@@ -86,8 +86,8 @@ export VERBOSE="YES"
 export KEEPDATA="NO"
 export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
 export CHGRP_CMD="@CHGRP_CMD@"
-export NCDUMP="$NETCDF/bin/ncdump"
-export NCLEN="$HOMEgfs/ush/getncdimlen"
+export NCDUMP="${NETCDF:-}/bin/ncdump"
+export NCLEN="${HOMEgfs}/ush/getncdimlen"
 
 # Machine environment, jobs, and other utility scripts
 export BASE_ENV="${HOMEgfs}/env"
@@ -134,7 +134,7 @@ export SENDCOM=${SENDCOM:-"YES"}
 export SENDSDM=${SENDSDM:-"NO"}
 export SENDDBN_NTC=${SENDDBN_NTC:-"NO"}
 export SENDDBN=${SENDDBN:-"NO"}
-export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
+export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn}
 
 # APP settings
 export APP=@APP@
@@ -227,11 +227,11 @@ case "${APP}" in
         export confignamevarfornems="${confignamevarfornems}_outerwave"
     fi
 
-    source ${EXPDIR}/config.defaults.s2sw
+    source "${EXPDIR}/config.defaults.s2sw"
 
     ;;
   *)
-    echo "Unrecognized APP: ${1}"
+    echo "Unrecognized APP: '${APP}'"
     exit 1
     ;;
 esac
@@ -262,7 +262,7 @@ export FHMAX_GFS_00=${FHMAX_GFS_00:-120}
 export FHMAX_GFS_06=${FHMAX_GFS_06:-120}
 export FHMAX_GFS_12=${FHMAX_GFS_12:-120}
 export FHMAX_GFS_18=${FHMAX_GFS_18:-120}
-export FHMAX_GFS=$(eval echo \${FHMAX_GFS_${cyc}})
+current_fhmax_var=FHMAX_GFS_${cyc}; declare -x FHMAX_GFS=${!current_fhmax_var}
 
 export FHOUT_GFS=${FHOUT_GFS:-3}
 export FHMAX_HF_GFS=${FHMAX_HF_GFS:-0}
@@ -275,7 +275,10 @@ fi
 export ILPOST=1           # gempak output frequency up to F120
 
 # GFS restart interval in hours
-export restart_interval_gfs=0
+export restart_interval_gfs=12
+# NOTE: Do not set this to zero.  Instead set it to $FHMAX_GFS
+# TODO: Remove this variable from config.base and reference from config.fcst
+# TODO: rework logic in config.wave and push it to parsing_nameslist_WW3.sh where it is actually used
 
 export QUILTING=".true."
 export OUTPUT_GRID="gaussian_grid"
@@ -285,7 +288,7 @@ export WRITE_NSFLIP=".true."
 # IAU related parameters
 export DOIAU="YES"        # Enable 4DIAU for control with 3 increments
 export IAUFHRS="3,6,9"
-export IAU_FHROT=$(echo ${IAUFHRS} | cut -c1)
+export IAU_FHROT=${IAUFHRS%%,*}
 export IAU_DELTHRS=6
 export IAU_OFFSET=6
 export DOIAU_ENKF=${DOIAU:-"YES"}   # Enable 4DIAU for EnKF ensemble
@@ -325,7 +328,7 @@ if [[ ${DOHYBVAR} = "YES" ]]; then
     export FHMAX_ENKF=9
     export FHMAX_ENKF_GFS=120
     export FHOUT_ENKF_GFS=3
-    if [ $l4densvar = ".true." ]; then
+    if [[ ${l4densvar} = ".true." ]]; then
         export FHOUT=1
         export FHOUT_ENKF=1
     else