Merge branch 'develop' of https://github.com/NOAA-EMC/global-workflow …

…into develop_mdtoy_hr3

ci/cases/pr/C48_S2SA_gefs.yaml

@@ -9,9 +9,9 @@ arguments:
   resens: 48
   nens: 2
   gfs_cyc: 1
+  start: cold
   comrot: {{ 'RUNTESTS' | getenv }}/COMROT
   expdir: {{ 'RUNTESTS' | getenv }}/EXPDIR
-  icsdir: {{ 'ICSDIR_ROOT' | getenv }}/C48C48mx500
   idate: 2021032312
   edate: 2021032312
   yaml: {{ HOMEgfs }}/ci/platforms/gefs_ci_defaults.yaml

env/ORION.env

@@ -132,20 +132,20 @@ elif [[ "${step}" = "anal" ]] || [[ "${step}" = "analcalc" ]]; then
 
     export NTHREADS_GSI=${nth_anal:-${nth_max}}
     [[ ${NTHREADS_GSI} -gt ${nth_max} ]] && export NTHREADS_GSI=${nth_max}
-    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}}"
+    export APRUN_GSI="${launcher} -n ${npe_gsi:-${npe_anal}} --cpus-per-task=${NTHREADS_GSI}"
 
     export NTHREADS_CALCINC=${nth_calcinc:-1}
     [[ ${NTHREADS_CALCINC} -gt ${nth_max} ]] && export NTHREADS_CALCINC=${nth_max}
-    export APRUN_CALCINC="${launcher} \$ncmd"
+    export APRUN_CALCINC="${launcher} \$ncmd --cpus-per-task=${NTHREADS_CALCINC}"
 
     export NTHREADS_CYCLE=${nth_cycle:-12}
     [[ ${NTHREADS_CYCLE} -gt ${npe_node_max} ]] && export NTHREADS_CYCLE=${npe_node_max}
     npe_cycle=${ntiles:-6}
-    export APRUN_CYCLE="${launcher} -n ${npe_cycle}"
+    export APRUN_CYCLE="${launcher} -n ${npe_cycle} --cpus-per-task=${NTHREADS_CYCLE}"
 
     export NTHREADS_GAUSFCANL=1
     npe_gausfcanl=${npe_gausfcanl:-1}
-    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl}"
+    export APRUN_GAUSFCANL="${launcher} -n ${npe_gausfcanl} --cpus-per-task=${NTHREADS_GAUSFCANL}"
 
 elif [[ "${step}" = "sfcanl" ]]; then
     nth_max=$((npe_node_max / npe_node_sfcanl))

jobs/JGFS_ATMOS_CYCLONE_GENESIS

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis" -c "base genesis"
 
 
 ##############################################

jobs/JGFS_ATMOS_CYCLONE_TRACKER

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "tracker" -c "base tracker"
 
 
 export COMPONENT="atmos"

jobs/JGFS_ATMOS_FSU_GENESIS

@@ -1,7 +1,7 @@
 #! /usr/bin/env bash
 
 source "${HOMEgfs}/ush/preamble.sh"
-source "${HOMEgfs}/ush/jjob_header.sh" -e "vrfy" -c "base vrfy"
+source "${HOMEgfs}/ush/jjob_header.sh" -e "genesis_fsu" -c "base genesis_fsu"
 
 export COMPONENT="atmos"
 

jobs/JGLOBAL_FORECAST

@@ -46,7 +46,7 @@ declare -rx gcyc="${GDATE:8:2}"
 
 # Construct COM variables from templates (see config.com)
 YMD=${PDY} HH=${cyc} generate_com -rx COM_ATMOS_RESTART COM_ATMOS_INPUT COM_ATMOS_ANALYSIS \
-  COM_ATMOS_HISTORY COM_ATMOS_MASTER COM_TOP
+  COM_ATMOS_HISTORY COM_ATMOS_MASTER COM_TOP COM_CONF
 
 RUN=${rCDUMP} YMD="${gPDY}" HH="${gcyc}" generate_com -rx \
   COM_ATMOS_RESTART_PREV:COM_ATMOS_RESTART_TMPL

jobs/rocoto/genesis.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="genesis"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_GENESIS"
+status=$?
+
+exit "${status}"

jobs/rocoto/genesis_fsu.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="genesis_fsu"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS"
+status=$?
+
+exit "${status}"

jobs/rocoto/post.sh

@@ -8,9 +8,26 @@ source "${HOMEgfs}/ush/preamble.sh"
 ###############################################################
 
 # Source FV3GFS workflow modules
-. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-status=$?
-[[ ${status} -ne 0 ]] && exit ${status}
+# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+# status=$?
+# [[ ${status} -ne 0 ]] && exit ${status}
+# Temporarily load modules from UPP
+source "${HOMEgfs}/ush/detect_machine.sh"
+source "${HOMEgfs}/ush/module-setup.sh"
+module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
+module load "${MACHINE_ID}"
+module load prod_util
+if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
+    module load cray-pals
+    module load cfp
+else
+    # shellcheck disable=SC2154
+    export UTILROOT="${prod_util_ROOT}"
+fi
+module load grib-util
+module load wgrib2
+export WGRIB2=wgrib2
+# End hack
 
 export job="post"
 export jobid="${job}.$$"

jobs/rocoto/tracker.sh

@@ -0,0 +1,20 @@
+#! /usr/bin/env bash
+
+source "${HOMEgfs}/ush/preamble.sh"
+
+###############################################################
+# Source FV3GFS workflow modules
+source "${HOMEgfs}/ush/load_fv3gfs_modules.sh"
+status=$?
+(( status != 0 )) && exit "${status}"
+
+export job="tracker"
+export jobid="${job}.$$"
+
+###############################################################
+# Execute the JJOB
+
+"${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER"
+status=$?
+
+exit "${status}"

jobs/rocoto/vrfy.sh

@@ -52,34 +52,6 @@ if [[ "${RUNMOS}" == "YES" && "${CDUMP}" == "gfs" ]]; then
 fi
 
 
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE TRACK VERIFICATION ==============="
-if [[ ${VRFYTRAK} = "YES" ]]; then
-
-    COMINsyn=${COMINsyn:-$(compath.py "${envir}/com/gfs/${gfs_ver}")/syndat}
-    export COMINsyn
-
-    ${TRACKERSH}
-fi
-
-
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION ==============="
-if [[ ${VRFYGENESIS} = "YES" && "${CDUMP}" = "gfs" ]]; then
-    ${GENESISSH}
-fi
-
-
-################################################################################
-echo
-echo "=============== START TO RUN CYCLONE GENESIS VERIFICATION (FSU) ==============="
-if [[ ${VRFYFSU} = "YES" && "${CDUMP}" = "gfs" ]]; then
-    ${GENESISFSU}
-fi
-
-
 ###############################################################
 # Force Exit out cleanly
 cd "${DATAROOT}"

parm/config/gefs/config.base.emc.dyn

@@ -282,11 +282,16 @@ export NMEM_ENS=@NMEM_ENS@
 export ENSMEM="000"
 export MEMDIR="mem${ENSMEM}"
 
+export DOIAU="NO"  # While we are not doing IAU, we may want to warm start w/ IAU in the future
 # Check if cycle is cold starting
 if [[ "${EXP_WARM_START}" = ".false." ]]; then
-    export IAU_FHROT=0
+  export IAU_FHROT=0
 else
+  if [[ "${DOIAU}" = "YES" ]]; then
     export IAU_FHROT=3
+  else
+    export IAU_FHROT=0
+  fi
 fi
 
 # turned on nsst in anal and/or fcst steps, and turn off rtgsst

parm/config/gfs/config.base.emc.dyn

@@ -57,6 +57,9 @@ export DO_BUFRSND="NO"     # BUFR sounding products
 export DO_GEMPAK="NO"      # GEMPAK products
 export DO_AWIPS="NO"       # AWIPS products
 export DO_VRFY="YES"       # VRFY step
+export DO_TRACKER="YES"    # Hurricane track verification
+export DO_GENESIS="YES"    # Cyclone genesis verification
+export DO_GENESIS_FSU="NO" # Cyclone genesis verification (FSU)
 export DO_VERFOZN="YES"    # Ozone data assimilation monitoring
 export DO_VERFRAD="YES"    # Radiance data assimilation monitoring
 export DO_VMINMON="YES"    # GSI minimization monitoring

parm/config/gfs/config.genesis

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.genesis ##########
+echo "BEGIN: config.genesis"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" genesis
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.genesis"

parm/config/gfs/config.genesis_fsu

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.genesis_fsu ##########
+echo "BEGIN: config.genesis_fsu"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" genesis_fsu
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.genesis_fsu"

parm/config/gfs/config.resources

@@ -15,6 +15,7 @@ if [[ $# -ne 1 ]]; then
     echo "landanl"
     echo "aeroanlinit aeroanlrun aeroanlfinal"
     echo "anal sfcanl analcalc analdiag fcst post echgres"
+    echo "tracker genesis genesis_fsu"
     echo "verfozn verfrad vminmon vrfy fit2obs metp arch cleanup"
     echo "eobs ediag eomg eupd ecen esfc efcs epos earc"
     echo "init_chem mom6ic ocnpost"
@@ -749,19 +750,39 @@ elif [[ ${step} = "vminmon" ]]; then
     export npe_node_vminmon_gfs=1
     export memory_vminmon="1G"
 
+elif [[ ${step} = "tracker" ]]; then
+
+    export wtime_tracker="00:10:00"
+    export npe_tracker=1
+    export nth_tracker=1
+    export npe_node_tracker=1
+    export memory_tracker="4G"
+
+elif [[ ${step} = "genesis" ]]; then
+
+    export wtime_genesis="00:25:00"
+    export npe_genesis=1
+    export nth_genesis=1
+    export npe_node_genesis=1
+    export memory_genesis="4G"
+
+elif [[ ${step} = "genesis_fsu" ]]; then
+
+    export wtime_genesis_fsu="00:10:00"
+    export npe_genesis_fsu=1
+    export nth_genesis_fsu=1
+    export npe_node_genesis_fsu=1
+    export memory_genesis_fsu="4G"
+
 elif [[ ${step} = "vrfy" ]]; then
 
-    export wtime_vrfy="03:00:00"
-    export wtime_vrfy_gfs="06:00:00"
-    export npe_vrfy=3
+    export wtime_vrfy="00:10:00"
+    export wtime_vrfy_gfs="00:10:00"
+    export npe_vrfy=1
     export nth_vrfy=1
     export npe_node_vrfy=1
     export npe_vrfy_gfs=1
     export npe_node_vrfy_gfs=1
-    if [[ ${machine} == "HERA" ]]; then
-      export memory_vrfy="16384M"
-    fi
-    export is_exclusive=True
 
 elif [[ "${step}" = "fit2obs" ]]; then
 

parm/config/gfs/config.stage_ic

@@ -18,7 +18,7 @@ case "${CASE}" in
     export CPL_ATMIC=GEFS-NoahMP-aerosols-p8c_refactored
     export CPL_ICEIC=CPC_refactored
     export CPL_OCNIC=CPC3Dvar_refactored
-    export CPL_WAVIC=GEFSwave20210528v2_refactored
+    export CPL_WAVIC=workflow_C384_refactored
     ;;
   "C768")
     export CPL_ATMIC=HR2_refactored

parm/config/gfs/config.tracker

@@ -0,0 +1,12 @@
+#! /usr/bin/env bash
+
+########## config.tracker ##########
+echo "BEGIN: config.tracker"
+
+# Get task specific resources
+. "${EXPDIR}/config.resources" tracker
+
+# Get tropcy settings
+. "${EXPDIR}/config.tropcy"
+
+echo "END: config.tracker"

parm/config/gfs/config.tropcy

@@ -0,0 +1,15 @@
+#! /usr/bin/env bash
+
+########## config.tropcy ##########
+echo "BEGIN: config.tropcy"
+
+# Tracker/genesis package location
+export HOMEens_tracker=${BASE_GIT}/TC_tracker/${ens_tracker_ver}
+
+export SENDCOM="YES" # Needed by tracker scripts still
+
+export FHOUT_CYCLONE=6
+FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 ))
+export FHMAX_CYCLONE
+
+echo "END: config.tropcy"

parm/config/gfs/config.vrfy

@@ -10,45 +10,8 @@ echo "BEGIN: config.vrfy"
 
 export CDFNL="gdas"               # Scores verification against GDAS/GFS analysis
 export MKPGB4PRCP="YES"           # Make 0.25-deg pgb files in ARCDIR for precip verification
-export VRFYTRAK="YES"             # Hurricane track verification
-export VRFYGENESIS="YES"          # Cyclone genesis verification
-export VRFYFSU="NO"               # Cyclone genesis verification (FSU)
 export RUNMOS="NO"                # whether to run entire MOS package
 
-#-------------------------------------------------
-# Cyclone genesis and cyclone track verification
-#-------------------------------------------------
-
-export SENDCOM="YES" # Needed by tracker/genesis scripts still
-
-export HOMEens_tracker=$BASE_GIT/TC_tracker/${tracker_ver}
-
-if [[ "${VRFYTRAK}" = "YES" ]]; then
-
-    export TRACKERSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_TRACKER"
-    COMINsyn=${COMINsyn:-$(compath.py "${envir}"/com/gfs/"${gfs_ver}")/syndat}
-    export COMINsyn
-    if [[ "${RUN}" = "gdas" ]]; then
-        export FHOUT_CYCLONE=3
-        export FHMAX_CYCLONE=${FHMAX}
-    else
-        export FHOUT_CYCLONE=6
-        FHMAX_CYCLONE=$(( FHMAX_GFS<240 ? FHMAX_GFS : 240 ))
-        export FHMAX_CYCLONE
-    fi
-fi
-
-
-if [[ "${VRFYGENESIS}" == "YES" && "${RUN}" == "gfs" ]]; then
-
-    export GENESISSH="${HOMEgfs}/jobs/JGFS_ATMOS_CYCLONE_GENESIS"
-fi
-
-if [[ "${VRFYFSU}" == "YES" && "${RUN}" == "gfs" ]]; then
-
-    export GENESISFSU="${HOMEgfs}/jobs/JGFS_ATMOS_FSU_GENESIS"
-fi
-
 if [[ "${RUNMOS}" == "YES" && "${RUN}" == "gfs" ]]; then
 
     if [[ "${machine}" = "HERA" ]] ; then