Merge remote-tracking branch 'origin/develop' into fix/early_run_configs

* origin/develop: Fix gdas build on Gaea and add Gaea to available CI list (NOAA-EMC#2857) Support ATM forecast only on Google (NOAA-EMC#2832) Add GEFS C48 support on AWS (NOAA-EMC#2818) Update omega calculation (NOAA-EMC#2751) Add snow DA update and recentering for the EnKF forecasts (NOAA-EMC#2690)
DavidHuber-NOAA · Aug 28, 2024 · 473d47b · 473d47b
2 parents 15c2883 + 935ac64
commit 473d47b
Show file tree

Hide file tree

Showing 48 changed files with 1,059 additions and 50 deletions.
diff --git a/.gitignore b/.gitignore
@@ -200,6 +200,7 @@ ush/month_name.sh
 ush/imsfv3_scf2ioda.py
 ush/atparse.bash
 ush/run_bufr2ioda.py
+ush/bufr2ioda_insitu*
 
 # version files
 versions/build.ver

diff --git a/ci/Jenkinsfile b/ci/Jenkinsfile
@@ -78,7 +78,7 @@ pipeline {
                     Machine = machine[0].toUpperCase() + machine.substring(1)
                     echo "Getting Common Workspace for ${Machine}"
                     ws("${custom_workspace[machine]}/${env.CHANGE_ID}") {
-                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
+                        properties([parameters([[$class: 'NodeParameterDefinition', allowedSlaves: ['built-in', 'Hercules-EMC', 'Hera-EMC', 'Orion-EMC', 'Gaea'], defaultSlaves: ['built-in'], name: '', nodeEligibility: [$class: 'AllNodeEligibility'], triggerIfResult: 'allCases']])])
                         GH = sh(script: "which gh || echo '~/bin/gh'", returnStdout: true).trim()
                         CUSTOM_WORKSPACE = "${WORKSPACE}"
                         sh(script: "mkdir -p ${CUSTOM_WORKSPACE}/RUNTESTS;rm -Rf ${CUSTOM_WORKSPACE}/RUNTESTS/*")
@@ -267,7 +267,7 @@ pipeline {
                 }
             }
         }
-            
+
 
         stage( '5. FINALIZE' ) {
             agent { label NodeName[machine].toLowerCase() }
@@ -297,6 +297,6 @@ pipeline {
                     }
                 }
             }
-        }   
+        }
     }
 }
diff --git a/ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml b/ci/cases/pr/C96_atmaerosnowDA.yaml → ci/cases/pr/C96C48_hybatmaerosnowDA.yaml
@@ -6,12 +6,13 @@ arguments:
   pslot: {{ 'pslot' | getenv }}
   app: ATMA
   resdetatmos: 96
+  resensatmos: 48
   comroot: {{ 'RUNTESTS' | getenv }}/COMROOT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
   icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C96C48/20240610
   idate: 2021122012
   edate: 2021122100
-  nens: 0
+  nens: 2
   gfs_cyc: 1
   start: cold
   yaml: {{ HOMEgfs }}/ci/cases/yamls/atmaerosnowDA_defaults_ci.yaml

diff --git a/ci/cases/pr/C96C48_ufs_hybatmDA.yaml b/ci/cases/pr/C96C48_ufs_hybatmDA.yaml
@@ -22,4 +22,5 @@ skip_ci_on_hosts:
   - gaea
   - orion
   - hercules
+  - wcoss2
 
diff --git a/env/AWSPW.env b/env/AWSPW.env
@@ -27,7 +27,7 @@ if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-
     NTHREADS1=${threads_per_task:-1}
     [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
     [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
-    APRUN="${launcher} -n ${ntasks}"
+    export APRUN="${launcher} -n ${ntasks}"
 else
     echo "ERROR config.resources must be sourced before sourcing AWSPW.env"
     exit 2
@@ -43,6 +43,13 @@ if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
     export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
     unset nnodes ufs_ntasks
 
+elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll" ]] || [[ "${step}" = "wavepostpnt" ]]; then
+
+    export CFP_MP="YES"
+    if [[ "${step}" = "waveprep" ]]; then export MP_PULSE=0 ; fi
+    export wavempexec=${launcher}
+    export wave_mpmd=${mpmd_opt}
+
 elif [[ "${step}" = "post" ]]; then
 
     export NTHREADS_NP=${NTHREADS1}
@@ -52,6 +59,15 @@ elif [[ "${step}" = "post" ]]; then
     [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
     export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
 
+elif [[ "${step}" = "atmos_products" ]]; then
+
+    export USE_CFP="YES"  # Use MPMD for downstream product generation on Hera
+
+elif [[ "${step}" = "oceanice_products" ]]; then
+
+    export NTHREADS_OCNICEPOST=${NTHREADS1}
+    export APRUN_OCNICEPOST="${launcher} -n 1 --cpus-per-task=${NTHREADS_OCNICEPOST}"
+
 elif [[ "${step}" = "ecen" ]]; then
 
     export NTHREADS_ECEN=${NTHREADSmax}

diff --git a/env/GOOGLEPW.env b/env/GOOGLEPW.env
@@ -0,0 +1,55 @@
+#! /usr/bin/env bash
+
+if [[ $# -ne 1 ]]; then
+
+    echo "Must specify an input argument to set runtime environment variables!"
+    exit 1
+
+fi
+
+step=$1
+
+export launcher="srun -l --export=ALL"
+export mpmd_opt="--multi-prog --output=mpmd.%j.%t.out"
+
+# Configure MPI environment
+export OMP_STACKSIZE=2048000
+export NTHSTACK=1024000000
+
+ulimit -s unlimited
+ulimit -a
+
+# Calculate common variables
+# Check first if the dependent variables are set
+if [[ -n "${ntasks:-}" && -n "${max_tasks_per_node:-}" && -n "${tasks_per_node:-}" ]]; then
+    max_threads_per_task=$((max_tasks_per_node / tasks_per_node))
+    NTHREADSmax=${threads_per_task:-${max_threads_per_task}}
+    NTHREADS1=${threads_per_task:-1}
+    [[ ${NTHREADSmax} -gt ${max_threads_per_task} ]] && NTHREADSmax=${max_threads_per_task}
+    [[ ${NTHREADS1} -gt ${max_threads_per_task} ]] && NTHREADS1=${max_threads_per_task}
+    APRUN="${launcher} -n ${ntasks}"
+else
+    echo "ERROR config.resources must be sourced before sourcing GOOGLEPW.env"
+    exit 2
+fi
+
+if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
+
+    export launcher="srun --mpi=pmi2 -l"
+
+    (( nnodes = (ntasks+tasks_per_node-1)/tasks_per_node ))
+    (( ufs_ntasks = nnodes*tasks_per_node ))
+    # With ESMF threading, the model wants to use the full node
+    export APRUN_UFS="${launcher} -n ${ufs_ntasks}"
+    unset nnodes ufs_ntasks
+
+elif [[ "${step}" = "post" ]]; then
+
+    export NTHREADS_NP=${NTHREADS1}
+    export APRUN_NP="${APRUN}"
+
+    export NTHREADS_DWN=${threads_per_task_dwn:-1}
+    [[ ${NTHREADS_DWN} -gt ${max_threads_per_task} ]] && export NTHREADS_DWN=${max_threads_per_task}
+    export APRUN_DWN="${launcher} -n ${ntasks_dwn}"
+
+fi
diff --git a/env/HERA.env b/env/HERA.env
@@ -80,7 +80,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN} --cpus-per-task=${NTHREADS_ATMENSANLFV3INC}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -106,6 +106,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "marinebmat" ]]; then
 
     export APRUNCFP="${launcher} -n \$ncmd --multi-prog"

diff --git a/env/HERCULES.env b/env/HERCULES.env
@@ -105,6 +105,14 @@ case ${step} in
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
  ;;
+ "esnowrecen")
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+ ;;
+
  "marinebmat")
 
     export APRUNCFP="${launcher} -n \$ncmd ${mpmd_opt}"

diff --git a/env/JET.env b/env/JET.env
@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/ORION.env b/env/ORION.env
@@ -97,6 +97,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/S4.env b/env/S4.env
@@ -68,7 +68,7 @@ elif [[ "${step}" = "atmensanlletkf" ]]; then
 elif [[ "${step}" = "atmensanlfv3inc" ]]; then
 
     export NTHREADS_ATMENSANLFV3INC=${NTHREADSmax}
-    export APRUN_ATMENSANLFV3INC="${APRUN}"    
+    export APRUN_ATMENSANLFV3INC="${APRUN}"
 
 elif [[ "${step}" = "aeroanlrun" ]]; then
 
@@ -89,6 +89,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN} --cpus-per-task=${NTHREADS_ESNOWRECEN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/env/WCOSS2.env b/env/WCOSS2.env
@@ -82,6 +82,13 @@ elif [[ "${step}" = "snowanl" ]]; then
 
     export APRUN_APPLY_INCR="${launcher} -n 6"
 
+elif [[ "${step}" = "esnowrecen" ]]; then
+
+    export NTHREADS_ESNOWRECEN=${NTHREADSmax}
+    export APRUN_ESNOWRECEN="${APRUN}"
+
+    export APRUN_APPLY_INCR="${launcher} -n 6"
+
 elif [[ "${step}" = "atmanlfv3inc" ]]; then
 
     export NTHREADS_ATMANLFV3INC=${NTHREADSmax}

diff --git a/jobs/JGDAS_ENKF_SNOW_RECENTER b/jobs/JGDAS_ENKF_SNOW_RECENTER
@@ -0,0 +1,59 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "esnowrecen" -c "base esnowrecen"
+
+##############################################
+# Set variables used in the script
+##############################################
+# Ignore possible spelling error (nothing is misspelled)
+# shellcheck disable=SC2153
+GDUMP="gdas"
+export GDUMP
+
+##############################################
+# Begin JOB SPECIFIC work
+##############################################
+# Generate COM variables from templates
+YMD=${PDY} HH=${cyc} declare_from_tmpl -rx \
+    COMIN_OBS:COM_OBS_TMPL \
+    COMOUT_ATMOS_ANALYSIS:COM_ATMOS_ANALYSIS_TMPL \
+    COMOUT_CONF:COM_CONF_TMPL
+MEMDIR="ensstat" YMD=${PDY} HH=${cyc} declare_from_tmpl \
+    COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+
+mkdir -p "${COMOUT_SNOW_ANALYSIS}" "${COMOUT_CONF}"
+
+for imem in $(seq 1 "${NMEM_ENS}"); do
+    memchar="mem$(printf %03i "${imem}")"
+    MEMDIR=${memchar} YMD=${PDY} HH=${cyc} declare_from_tmpl \
+        COMOUT_SNOW_ANALYSIS:COM_SNOW_ANALYSIS_TMPL
+    mkdir -p "${COMOUT_SNOW_ANALYSIS}"
+done
+
+###############################################################
+# Run relevant script
+
+EXSCRIPT=${SNOWANLPY:-${SCRgfs}/exgdas_enkf_snow_recenter.py}
+${EXSCRIPT}
+status=$?
+(( status != 0 )) && exit "${status}"
+
+##############################################
+# End JOB SPECIFIC work
+##############################################
+
+##############################################
+# Final processing
+##############################################
+if [[ -e "${pgmout}" ]] ; then
+  cat "${pgmout}"
+fi
+
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ "${KEEPDATA}" = "NO" ]] && rm -rf "${DATA}"
+
+exit 0
diff --git a/jobs/JGLOBAL_PREP_SNOW_OBS b/jobs/JGLOBAL_PREP_SNOW_OBS
@@ -41,4 +41,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0
diff --git a/jobs/JGLOBAL_SNOW_ANALYSIS b/jobs/JGLOBAL_SNOW_ANALYSIS
@@ -44,4 +44,10 @@ if [[ -e "${pgmout}" ]] ; then
   cat "${pgmout}"
 fi
 
+##########################################
+# Remove the Temporary working directory
+##########################################
+cd "${DATAROOT}" || exit 1
+[[ ${KEEPDATA} = "NO" ]] && rm -rf "${DATA}"
+
 exit 0
diff --git a/jobs/rocoto/esnowrecen.sh b/jobs/rocoto/esnowrecen.sh
@@ -0,0 +1,18 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source UFSDA workflow modules
+. "${HOMEgfs}/ush/load_ufsda_modules.sh"
+status=$?
+[[ ${status} -ne 0 ]] && exit "${status}"
+
+export job="esnowrecen"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+"${HOMEgfs}/jobs/JGDAS_ENKF_SNOW_RECENTER"
+status=$?
+exit "${status}"
diff --git a/modulefiles/module_gwsetup.gaea.lua b/modulefiles/module_gwsetup.gaea.lua
@@ -15,5 +15,6 @@ load(pathJoin("python", python_ver))
 load("py-jinja2")
 load("py-pyyaml")
 load("py-numpy")
+load("git-lfs")
 
 whatis("Description: GFS run setup environment")
diff --git a/parm/config/gefs/config.base b/parm/config/gefs/config.base
@@ -345,4 +345,10 @@ export DELETE_COM_IN_ARCHIVE_JOB="YES"   # NO=retain ROTDIR.  YES default in arc
 # Number of regional collectives to create soundings for
 export NUM_SND_COLLECTIVES=${NUM_SND_COLLECTIVES:-9}
 
+# The tracker, genesis, and METplus jobs are not supported on AWS yet
+# TODO: we should place these in workflow/hosts/awspw.yaml as part of AWS setup, not for general.
+if [[ "${machine}" == "AWSPW" ]]; then
+   export DO_WAVE="NO"
+fi
+
 echo "END: config.base"
diff --git a/parm/config/gefs/config.fcst b/parm/config/gefs/config.fcst
@@ -193,6 +193,7 @@ case ${imp_physics} in
         export dt_inner=$((DELTIM/2))
         export sedi_semi=.true.
         if [[ "${sedi_semi}" == .true. ]]; then export dt_inner=${DELTIM} ; fi
+        if [[ dt_inner -gt 300 ]]; then export  dt_inner=300 ; fi
         export decfl=10
 
         export hord_mt_nh_nonmono=5

diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
@@ -41,7 +41,7 @@ case ${machine} in
     ;;
   "AWSPW")
     export PARTITION_BATCH="compute"
-    max_tasks_per_node=40
+    max_tasks_per_node=36
     ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"