From 8f0185a6e4fbe823f67f41e48ca7393be5591725 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 12:10:08 -0500
Subject: [PATCH 01/46] Add run-specific resource variables

---
 parm/config/gfs/config.resources | 211 ++++++++++---------------------
 workflow/rocoto/tasks.py         |  39 +++---
 2 files changed, 94 insertions(+), 156 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 0972f74f9c..ee5958e026 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -77,11 +77,7 @@ case ${step} in
     export npe_prep=4
     export npe_node_prep=2
     export nth_prep=1
-    if [[ "${machine}" == "WCOSS2" ]]; then
-      export is_exclusive=True
-    else
-      export memory_prep="40GB"
-    fi
+    export memory_prep="40GB"
     ;;
 
   "prepsnowobs")
@@ -117,27 +113,27 @@ case ${step} in
     ;;
 
   "waveprep")
-    export wtime_waveprep="00:10:00"
-    export npe_waveprep=5
+    export wtime_waveprep_gdas="00:10:00"
+    export npe_waveprep_gdas=5
     export npe_waveprep_gfs=65
-    export nth_waveprep=1
+    export nth_waveprep_gdas=1
     export nth_waveprep_gfs=1
-    export npe_node_waveprep=$(( npe_node_max / nth_waveprep ))
+    export npe_node_waveprep_gdas=$(( npe_node_max / nth_waveprep_gdas ))
     export npe_node_waveprep_gfs=$(( npe_node_max / nth_waveprep_gfs ))
-    export NTASKS=${npe_waveprep}
+    export NTASKS_gdas=${npe_waveprep_gdas}
     export NTASKS_gfs=${npe_waveprep_gfs}
-    export memory_waveprep="100GB"
+    export memory_waveprep_gdas="100GB"
     export memory_waveprep_gfs="150GB"
     ;;
 
   "wavepostsbs")
-    export wtime_wavepostsbs="00:20:00"
+    export wtime_wavepostsbs_gdas="00:20:00"
     export wtime_wavepostsbs_gfs="03:00:00"
     export npe_wavepostsbs=8
     export nth_wavepostsbs=1
     export npe_node_wavepostsbs=$(( npe_node_max / nth_wavepostsbs ))
     export NTASKS=${npe_wavepostsbs}
-    export memory_wavepostsbs="10GB"
+    export memory_wavepostsbs_gdas="10GB"
     export memory_wavepostsbs_gfs="10GB"
     ;;
 
@@ -229,11 +225,12 @@ case ${step} in
     export layout_y=${layout_y_atmanl}
 
     export wtime_atmanlvar="00:30:00"
-    export npe_atmanlvar=$(( layout_x * layout_y * 6 ))
+    export npe_atmanlvar_gdas=$(( layout_x * layout_y * 6 ))
     export npe_atmanlvar_gfs=$(( layout_x * layout_y * 6 ))
-    export nth_atmanlvar=1
-    export nth_atmanlvar_gfs=${nth_atmanlvar}
-    export npe_node_atmanlvar=$(( npe_node_max / nth_atmanlvar ))
+    export nth_atmanlvar_gdas=1
+    export nth_atmanlvar_gfs=${nth_atmanlvar_gdas}
+    export npe_node_atmanlvar_gdas=$(( npe_node_max / nth_atmanlvar_gdas ))
+    export npe_node_atmanlvar_gfs=$(( npe_node_max / nth_atmanlvar_gfs ))
     export memory_atmanlvar="96GB"
     export is_exclusive=True
     ;;
@@ -243,11 +240,12 @@ case ${step} in
     export layout_y=${layout_y_atmanl}
 
     export wtime_atmanlfv3inc="00:30:00"
-    export npe_atmanlfv3inc=$(( layout_x * layout_y * 6 ))
+    export npe_atmanlfv3inc_gdas=$(( layout_x * layout_y * 6 ))
     export npe_atmanlfv3inc_gfs=$(( layout_x * layout_y * 6 ))
-    export nth_atmanlfv3inc=1
-    export nth_atmanlfv3inc_gfs=${nth_atmanlfv3inc}
-    export npe_node_atmanlfv3inc=$(( npe_node_max / nth_atmanlfv3inc ))
+    export nth_atmanlfv3inc_gdas=1
+    export nth_atmanlfv3inc_gfs=${nth_atmanlfv3inc_gdas}
+    export npe_node_atmanlfv3inc_gdas=$(( npe_node_max / nth_atmanlfv3inc_gdas ))
+    export npe_node_atmanlfv3inc_gfs=$(( npe_node_max / nth_atmanlfv3inc_gfs ))
     export memory_atmanlfv3inc="96GB"
     export is_exclusive=True
     ;;
@@ -351,11 +349,12 @@ case ${step} in
     export layout_y
 
     export wtime_aeroanlrun="00:30:00"
-    export npe_aeroanlrun=$(( layout_x * layout_y * 6 ))
+    export npe_aeroanlrun_gdas=$(( layout_x * layout_y * 6 ))
     export npe_aeroanlrun_gfs=$(( layout_x * layout_y * 6 ))
-    export nth_aeroanlrun=1
+    export nth_aeroanlrun_gdas=1
     export nth_aeroanlrun_gfs=1
-    export npe_node_aeroanlrun=$(( npe_node_max / nth_aeroanlrun ))
+    export npe_node_aeroanlrun_gdas=$(( npe_node_max / nth_aeroanlrun_gdas ))
+    export npe_node_aeroanlrun_gfs=$(( npe_node_max / nth_aeroanlrun_gfs ))
     export is_exclusive=True
     ;;
 
@@ -525,56 +524,22 @@ case ${step} in
     ;;
 
   "anal")
-    export wtime_anal="01:20:00"
+    export wtime_anal_gdas="01:20:00"
     export wtime_anal_gfs="01:00:00"
-    export npe_anal=780
-    export nth_anal=5
+    export npe_anal_gdas=780
+    export nth_anal_gdas=5
     export npe_anal_gfs=825
     export nth_anal_gfs=5
-    if [[ "${machine}" == "WCOSS2" ]]; then
-      export nth_anal=8
-      export nth_anal_gfs=8
-    fi
     case ${CASE} in
       "C384")
-        export npe_anal=160
+        export npe_anal_gdas=160
         export npe_anal_gfs=160
-        export nth_anal=10
+        export nth_anal_gdas=10
         export nth_anal_gfs=10
-        if [[ ${machine} = "HERA" ]]; then
-          export npe_anal=270
-          export npe_anal_gfs=270
-          export nth_anal=8
-          export nth_anal_gfs=8
-        fi	  
-        if [[ ${machine} = "S4" ]]; then
-          #On the S4-s4 partition, this is accomplished by increasing the task
-          #count to a multiple of 32
-          if [[ ${PARTITION_BATCH} = "s4" ]]; then
-            export npe_anal=416
-            export npe_anal_gfs=416
-          fi
-          #S4 is small, so run this task with just 1 thread
-          export nth_anal=1
-          export nth_anal_gfs=1
-          export wtime_anal="02:00:00"
-        fi
         ;;
       "C192" | "C96" | "C48")
-        export npe_anal=84
+        export npe_anal_gdas=84
         export npe_anal_gfs=84
-        if [[ ${machine} == "S4" ]]; then
-          export nth_anal=4
-          export nth_anal_gfs=4
-          #Adjust job count for S4
-          if [[ ${PARTITION_BATCH} == "s4" ]]; then
-            export npe_anal=88
-            export npe_anal_gfs=88
-          elif [[ ${PARTITION_BATCH} == "ivy" ]]; then
-            export npe_anal=90
-            export npe_anal_gfs=90
-          fi
-        fi
         ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
@@ -592,9 +557,9 @@ case ${step} in
     export npe_analcalc=127
     export ntasks="${npe_analcalc}"
     export nth_analcalc=1
-    export nth_echgres=4
-    export nth_echgres_gfs=12
     export npe_node_analcalc=$(( npe_node_max / nth_analcalc ))
+    export nth_echgres_gdas=4
+    export nth_echgres_gfs=12
     export is_exclusive=True
     export memory_analcalc="48GB"
     ;;
@@ -635,6 +600,15 @@ case ${step} in
         ntasks_quilt=${ntasks_quilt_gfs}
         nthreads_fv3=${nthreads_fv3_gfs}
         nthreads_ufs=${nthreads_ufs_gfs}
+      elif [[ "${_CDUMP}" =~ "gdas" ]]; then
+        export layout_x=${layout_x_gdas}
+        export layout_y=${layout_y_gdas}
+        export WRITE_GROUP=${WRITE_GROUP_GDAS}
+        export WRTTASK_PER_GROUP_PER_THREAD=${WRTTASK_PER_GROUP_PER_THREAD_GDAS}
+        ntasks_fv3=${ntasks_fv3_gdas}
+        ntasks_quilt=${ntasks_quilt_gdas}
+        nthreads_fv3=${nthreads_fv3_gdas}
+        nthreads_ufs=${nthreads_ufs_gdas}
       fi
 
       # Determine if using ESMF-managed threading or traditional threading
@@ -729,29 +703,28 @@ case ${step} in
 
       echo "Total PETS for ${_CDUMP} = ${NTASKS_TOT}"
 
-      if [[ "${_CDUMP}" =~ "gfs" ]]; then
-        declare -x "npe_${step}_gfs"="${NTASKS_TOT}"
-        declare -x "nth_${step}_gfs"="${UFS_THREADS}"
-        declare -x "npe_node_${step}_gfs"="${npe_node_max}"
-      else
-        declare -x "npe_${step}"="${NTASKS_TOT}"
-        declare -x "nth_${step}"="${UFS_THREADS}"
-        declare -x "npe_node_${step}"="${npe_node_max}"
-      fi
+      declare -x "npe_${step}_${_CDUMP}"="${NTASKS_TOT}"
+      declare -x "nth_${step}_${_CDUMP}"="${UFS_THREADS}"
+      declare -x "npe_node_${step}_${_CDUMP}"="${npe_node_max}"
 
     done
 
     case "${CASE}" in
       "C48" | "C96" | "C192")
-        declare -x "wtime_${step}"="00:20:00"
+        declare -x "wtime_${step}_gdas"="00:20:00"
+        declare -x "wtime_${step}_enkfgdas"="00:20:00"
         declare -x "wtime_${step}_gfs"="03:00:00"
+        declare -x "wtime_${step}_enkfgfs"="00:20:00"
         ;;
       "C384")
-        declare -x "wtime_${step}"="00:30:00"
+        declare -x "wtime_${step}_gdas"="00:30:00"
+        declare -x "wtime_${step}_enkfgdas"="00:30:00"
         declare -x "wtime_${step}_gfs"="06:00:00"
+        declare -x "wtime_${step}_enkfgfs"="00:30:00"
         ;;
       "C768" | "C1152")
-        declare -x "wtime_${step}"="00:30:00"
+        # Not valid resolutions for ensembles
+        declare -x "wtime_${step}_gdas"="00:30:00"
         declare -x "wtime_${step}_gfs"="06:00:00"
         ;;
       *)
@@ -784,14 +757,12 @@ case ${step} in
       "C768")
         export npe_upp=120
         export memory_upp="96GB"
-        if [[ ${machine} == "WCOSS2" ]]; then export memory_upp="480GB" ; fi
       ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
         exit 4
       ;;
     esac
-    if [[ ${machine} == "JET" ]]; then unset memory_upp ; fi
     export npe_node_upp=${npe_upp}
 
     export nth_upp=1
@@ -808,10 +779,6 @@ case ${step} in
     export npe_atmos_products=24
     export nth_atmos_products=1
     export npe_node_atmos_products="${npe_atmos_products}"
-    export wtime_atmos_products_gfs="${wtime_atmos_products}"
-    export npe_atmos_products_gfs="${npe_atmos_products}"
-    export nth_atmos_products_gfs="${nth_atmos_products}"
-    export npe_node_atmos_products_gfs="${npe_node_atmos_products}"
     export is_exclusive=True
     ;;
 
@@ -836,10 +803,6 @@ case ${step} in
     export npe_vminmon=1
     export nth_vminmon=1
     export npe_node_vminmon=1
-    export wtime_vminmon_gfs="00:05:00"
-    export npe_vminmon_gfs=1
-    export nth_vminmon_gfs=1
-    export npe_node_vminmon_gfs=1
     export memory_vminmon="1G"
     ;;
 
@@ -873,17 +836,14 @@ case ${step} in
     export nth_fit2obs=1
     export npe_node_fit2obs=1
     export memory_fit2obs="20G"
-    if [[ ${machine} == "WCOSS2" ]]; then export npe_node_fit2obs=3 ; fi
     ;;
 
   "metp")
     export nth_metp=1
-    export wtime_metp="03:00:00"
+    export wtime_metp_gdas="03:00:00"
+    export wtime_metp_gfs="06:00:00"
     export npe_metp=4
     export npe_node_metp=4
-    export wtime_metp_gfs="06:00:00"
-    export npe_metp_gfs=4
-    export npe_node_metp_gfs=4
     export is_exclusive=True
     ;;
 
@@ -892,9 +852,6 @@ case ${step} in
     export npe_echgres=3
     export nth_echgres=${npe_node_max}
     export npe_node_echgres=1
-    if [[ "${machine}" == "WCOSS2" ]]; then
-      export memory_echgres="200GB"
-    fi
     ;;
 
   "init")
@@ -925,9 +882,6 @@ case ${step} in
     declare -x "npe_node_${step}"="1"
     declare -x "nth_${step}"="1"
     declare -x "memory_${step}"="4096M"
-    if [[ "${machine}" == "WCOSS2" ]]; then
-      declare -x "memory_${step}"="50GB"
-    fi
     ;;
 
   "cleanup")
@@ -962,11 +916,12 @@ case ${step} in
     export layout_y=${layout_y_atmensanl}
 
     export wtime_atmensanlletkf="00:30:00"
-    export npe_atmensanlletkf=$(( layout_x * layout_y * 6 ))
-    export npe_atmensanlletkf_gfs=$(( layout_x * layout_y * 6 ))
-    export nth_atmensanlletkf=1
-    export nth_atmensanlletkf_gfs=${nth_atmensanlletkf}
-    export npe_node_atmensanlletkf=$(( npe_node_max / nth_atmensanlletkf ))
+    export npe_atmensanlletkf_enkfgdas=$(( layout_x * layout_y * 6 ))
+    export npe_atmensanlletkf_enkfgfs=$(( layout_x * layout_y * 6 ))
+    export nth_atmensanlletkf_enkfgdas=1
+    export nth_atmensanlletkf_enkfgfs=${nth_atmensanlletkf_enkfgdas}
+    export npe_node_atmensanlletkf_gdas=$(( npe_node_max / nth_atmensanlletkf_enkfgdas ))
+    export npe_node_atmensanlletkf_gfs=$(( npe_node_max / nth_atmensanlletkf_enkfgfs ))
     export memory_atmensanlletkf="96GB"
     export is_exclusive=True
     ;;
@@ -976,11 +931,12 @@ case ${step} in
     export layout_y=${layout_y_atmensanl}
 
     export wtime_atmensanlfv3inc="00:30:00"
-    export npe_atmensanlfv3inc=$(( layout_x * layout_y * 6 ))
-    export npe_atmensanlfv3inc_gfs=$(( layout_x * layout_y * 6 ))
-    export nth_atmensanlfv3inc=1
-    export nth_atmensanlfv3inc_gfs=${nth_atmensanlfv3inc}
-    export npe_node_atmensanlfv3inc=$(( npe_node_max / nth_atmensanlfv3inc ))
+    export npe_atmensanlfv3inc_enkfgdas=$(( layout_x * layout_y * 6 ))
+    export npe_atmensanlfv3inc_enkfgfs=$(( layout_x * layout_y * 6 ))
+    export nth_atmensanlfv3inc_enkfgdas=1
+    export nth_atmensanlfv3inc_enkfgfs=${nth_atmensanlfv3inc_enkfgdas}
+    export npe_node_atmensanlfv3inc_enkfgdas=$(( npe_node_max / nth_atmensanlfv3inc_enkfgdas ))
+    export npe_node_atmensanlfv3inc_enkfgfs=$(( npe_node_max / nth_atmensanlfv3inc_enkfgfs ))
     export memory_atmensanlfv3inc="96GB"
     export is_exclusive=True
     ;;
@@ -1008,19 +964,10 @@ case ${step} in
     export npe_eomg=${npe_eobs}
     export nth_eobs=2
     export nth_eomg=${nth_eobs}
+    # NOTE The number of tasks and cores used must be the same for eobs
+    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
     export npe_node_eobs=$(( npe_node_max / nth_eobs ))
     export is_exclusive=True
-    # The number of tasks and cores used must be the same for eobs
-    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
-    # For S4, this is accomplished by running 10 tasks/node
-    if [[ ${machine} = "S4" ]]; then
-      export npe_node_eobs=10
-    elif [[ ${machine} = "HERCULES" ]]; then
-      # For Hercules, this is only an issue at C384; use 20 tasks/node
-      if [[ ${CASE} = "C384" ]]; then
-        export npe_node_eobs=20
-      fi
-    fi
     export npe_node_eomg=${npe_node_eobs}
     ;;
 
@@ -1038,30 +985,14 @@ case ${step} in
       "C768")
         export npe_eupd=480
         export nth_eupd=6
-        if [[ "${machine}" == "WCOSS2" ]]; then
-          export npe_eupd=315
-          export nth_eupd=14
-        fi
         ;;
       "C384")
         export npe_eupd=270
         export nth_eupd=8
-        if [[ "${machine}" == "WCOSS2" ]]; then
-          export npe_eupd=315
-          export nth_eupd=14
-        elif [[ "${machine}" == "S4" ]]; then
-           export npe_eupd=160
-           export nth_eupd=2
-        elif [[ "${machine}" == "HERA" ]]; then
-          export npe_eupd=80
-        fi
         ;;
       "C192" | "C96" | "C48")
         export npe_eupd=42
         export nth_eupd=2
-        if [[ "${machine}" == "HERA" || "${machine}" == "JET" ]]; then
-          export nth_eupd=4
-        fi
         ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
@@ -1076,7 +1007,6 @@ case ${step} in
     export wtime_ecen="00:10:00"
     export npe_ecen=80
     export nth_ecen=4
-    if [[ "${machine}" == "HERA" ]]; then export nth_ecen=6; fi
     if [[ ${CASE} == "C384" || ${CASE} == "C192" || ${CASE} == "C96" || ${CASE} == "C48" ]]; then
       export nth_ecen=2
     fi
@@ -1093,7 +1023,6 @@ case ${step} in
     export npe_node_esfc=$(( npe_node_max / nth_esfc ))
     export nth_cycle=${nth_esfc}
     export npe_node_cycle=$(( npe_node_max / nth_cycle ))
-    if [[ ${machine} != "JET" ]]; then export memory_esfc="80G" ; fi
     ;;
 
   "epos")
@@ -1136,12 +1065,12 @@ case ${step} in
 
   "gempak")
     export wtime_gempak="03:00:00"
-    export npe_gempak=2
+    export npe_gempak_gdas=2
     export npe_gempak_gfs=28
-    export npe_node_gempak=2
+    export npe_node_gempak_gdas=2
     export npe_node_gempak_gfs=28
     export nth_gempak=1
-    export memory_gempak="4GB"
+    export memory_gempak_gdas="4GB"
     export memory_gempak_gfs="2GB"
     ;;
 
diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 52d5466d10..909b20f435 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -140,8 +140,8 @@ def _get_forecast_hours(cdump, config, component='atmos') -> List[str]:
         # Get a list of all forecast hours
         fhrs = []
         if cdump in ['gdas']:
-            fhmax = local_config['FHMAX']
-            fhout = local_config['FHOUT']
+            fhmax = local_config['FHMAX_GDAS']
+            fhout = local_config['FHOUT_GDAS']
             fhrs = list(range(fhmin, fhmax + fhout, fhout))
         elif cdump in ['gfs', 'gefs']:
             fhmax = local_config['FHMAX_GFS']
@@ -171,25 +171,34 @@ def get_resource(self, task_name):
 
         account = task_config['ACCOUNT_SERVICE'] if task_name in Tasks.SERVICE_TASKS else task_config['ACCOUNT']
 
-        walltime = task_config[f'wtime_{task_name}']
-        if self.cdump in ['gfs'] and f'wtime_{task_name}_gfs' in task_config.keys():
-            walltime = task_config[f'wtime_{task_name}_gfs']
+        if f'wtime_{task_name}_{self.cdump}' in task_config:
+            walltime = task_config[f'wtime_{task_name}_{self.cdump}']
+        else:
+            walltime = task_config[f'wtime_{task_name}']
 
-        cores = task_config[f'npe_{task_name}']
-        if self.cdump in ['gfs'] and f'npe_{task_name}_gfs' in task_config.keys():
-            cores = task_config[f'npe_{task_name}_gfs']
+        if f'npe_{task_name}_{self.cdump}' in task_config:
+            cores = task_config[f'npe_{task_name}_{self.cdump}']
+        else:
+            cores = task_config[f'npe_{task_name}']
 
-        ppn = task_config[f'npe_node_{task_name}']
-        if self.cdump in ['gfs'] and f'npe_node_{task_name}_gfs' in task_config.keys():
-            ppn = task_config[f'npe_node_{task_name}_gfs']
+        if f'npe_node_{task_name}_{self.cdump}' in task_config:
+            ppn = task_config[f'npe_node_{task_name}_{self.cdump}']
+        else:
+            ppn = task_config[f'npe_node_{task_name}']
 
         nodes = int(np.ceil(float(cores) / float(ppn)))
 
-        threads = task_config[f'nth_{task_name}']
-        if self.cdump in ['gfs'] and f'nth_{task_name}_gfs' in task_config.keys():
-            threads = task_config[f'nth_{task_name}_gfs']
+        if f'nth_{task_name}_{self.cdump}' in task_config:
+            threads = task_config[f'nth_{task_name}_{self.cdump}']
+        else:
+            threads = task_config[f'nth_{task_name}']
+
+        if f'memory_{task_name}_{self.cdump}' in task_config:
+            memory = task_config[f'memory_{task_name}_{self.cdump}']
+        else:
+            # Memory is not required
+            memory = task_config.get(f'memory_{task_name}', None)
 
-        memory = task_config.get(f'memory_{task_name}', None)
         if scheduler in ['pbspro']:
             if task_config.get('prepost', False):
                 memory += ':prepost=true'

From 420edf84cb8e622fb4bf4fefe520ef2d5714221b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 12:10:36 -0500
Subject: [PATCH 02/46] Add machine-specific resource files

---
 parm/config/gfs/config.resources.hera     | 36 ++++++++++++++
 parm/config/gfs/config.resources.hercules | 17 +++++++
 parm/config/gfs/config.resources.jet      | 54 +++++++++++++++++++++
 parm/config/gfs/config.resources.s4       | 57 +++++++++++++++++++++++
 parm/config/gfs/config.resources.wcoss2   | 51 ++++++++++++++++++++
 5 files changed, 215 insertions(+)
 create mode 100644 parm/config/gfs/config.resources.hera
 create mode 100644 parm/config/gfs/config.resources.hercules
 create mode 100644 parm/config/gfs/config.resources.jet
 create mode 100644 parm/config/gfs/config.resources.s4
 create mode 100644 parm/config/gfs/config.resources.wcoss2

diff --git a/parm/config/gfs/config.resources.hera b/parm/config/gfs/config.resources.hera
new file mode 100644
index 0000000000..ad1cddb8d2
--- /dev/null
+++ b/parm/config/gfs/config.resources.hera
@@ -0,0 +1,36 @@
+#!/usr/bin/bash
+
+# Hera-specific job resources
+
+case ${step} in
+  "anal")
+    if "${CASE}" == "C384"; then
+      export npe_anal_gdas=270
+      export npe_anal_gfs=270
+      export nth_anal_gdas=8
+      export nth_anal_gfs=8
+    fi
+    ;;
+
+  "eupd")
+    case ${CASE} in
+      "C384")
+        export npe_eupd=80
+        ;;
+      "C192" | "C96" | "C48")
+        export nth_eupd=4
+        ;;
+      *)
+        ;;
+    esac
+    export npe_node_eupd=$(( npe_node_max / nth_eupd ))
+    ;;
+
+  "ecen")
+    if [[ "${CASE}" == "C768" ]]; then export nth_ecen=6; fi
+    export npe_node_ecen=$(( npe_node_max / nth_ecen ))
+    ;;
+
+  *)
+    ;;
+esac
diff --git a/parm/config/gfs/config.resources.hercules b/parm/config/gfs/config.resources.hercules
new file mode 100644
index 0000000000..401f5db3b0
--- /dev/null
+++ b/parm/config/gfs/config.resources.hercules
@@ -0,0 +1,17 @@
+#!/usr/bin/bash
+
+# Hercules-specific job resources
+
+case ${step} in
+  "eobs" | "eomg")
+    # The number of tasks and cores used must be the same for eobs
+    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
+    # For Hercules, this is only an issue at C384; use 20 tasks/node
+    if [[ ${CASE} = "C384" ]]; then
+      export npe_node_eobs=20
+    fi
+    export npe_node_eomg=${npe_node_eobs}
+    ;;
+  *)
+    ;;
+esac
diff --git a/parm/config/gfs/config.resources.jet b/parm/config/gfs/config.resources.jet
new file mode 100644
index 0000000000..64c1f4a240
--- /dev/null
+++ b/parm/config/gfs/config.resources.jet
@@ -0,0 +1,54 @@
+#!/usr/bin/bash
+
+# Jet-specific job resources
+
+case ${PARTITION_BATCH} in
+  "xjet") node_mem="61GB";;
+  "vjet") node_mem="61GB";;
+  "sjet") node_mem="29GB";;
+  "kjet") node_mem="93GB";;
+  *)
+    echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for Jet"
+    exit 3
+esac
+
+case ${step} in
+  "anal")
+    if "${CASE}" == "C384"; then
+      export npe_anal_gdas=270
+      export npe_anal_gfs=270
+      export nth_anal_gdas=8
+      export nth_anal_gfs=8
+    fi
+    ;;
+
+  "eupd")
+    case ${CASE} in
+      "C384")
+        export npe_eupd=80
+        ;;
+      "C192" | "C96" | "C48")
+        export nth_eupd=4
+        ;;
+      *)
+        ;;
+    esac
+    export npe_node_eupd=$(( npe_node_max / nth_eupd ))
+    ;;
+
+  "ecen")
+    if [[ "${CASE}" == "C768" ]]; then export nth_ecen=6; fi
+    export npe_node_ecen=$(( npe_node_max / nth_ecen ))
+    ;;
+
+  "upp")
+    export memory_upp="${node_mem}"
+    ;;
+
+  "esfc")
+    export memory_esfc="${node_mem}"
+    ;;
+
+  *)
+    ;;
+esac
diff --git a/parm/config/gfs/config.resources.s4 b/parm/config/gfs/config.resources.s4
new file mode 100644
index 0000000000..e686dc4d40
--- /dev/null
+++ b/parm/config/gfs/config.resources.s4
@@ -0,0 +1,57 @@
+#!/usr/bin/bash
+
+# S4-specific job resources
+
+case ${step} in
+  "anal")
+    case ${CASE} in
+      "C384")
+        #Some of the intermediate data can be lost if the number of tasks
+        #per node does not match the number of reserved cores/node.
+        #On the S4-s4 partition, this is accomplished by increasing the task
+        #count to a multiple of 32
+        if [[ ${PARTITION_BATCH} = "s4" ]]; then
+          export npe_anal_gdas=416
+          export npe_anal_gfs=416
+        fi
+        #S4 is small, so run this task with just 1 thread
+        export nth_anal_gdas=1
+        export nth_anal_gfs=1
+        export wtime_anal_gdas="02:00:00"
+        export wtime_anal_gfs="02:00:00"
+        ;;
+      "C192" | "C96" | "C48")
+        export nth_anal_gdas=4
+        export nth_anal_gfs=4
+        if [[ ${PARTITION_BATCH} == "s4" ]]; then
+          export npe_anal_gdas=88
+          export npe_anal_gfs=88
+        elif [[ ${PARTITION_BATCH} == "ivy" ]]; then
+          export npe_anal_gdas=90
+          export npe_anal_gfs=90
+        fi
+        ;;
+      *)
+        ;;
+    esac
+    ;;
+
+  "eobs" | "eomg")
+    # The number of tasks and cores used must be the same for eobs
+    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
+    # For S4, this is accomplished by running 10 tasks/node
+    export npe_node_eobs=10
+    export npe_node_eomg=${npe_node_eobs}
+    ;;
+
+  "eupd")
+    if [[ "$CASE" == "C384" ]]; then
+      export npe_eupd=160
+      export nth_eupd=2
+    fi
+    export npe_node_eupd=$(( npe_node_max / nth_eupd ))
+    ;;
+
+  *)
+    ;;
+esac
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
new file mode 100644
index 0000000000..d39fb97fac
--- /dev/null
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -0,0 +1,51 @@
+#!/usr/bin/bash
+
+# WCOSS2-specific job resources
+
+case ${step} in
+  "prep")
+    export is_exclusive=True
+    ;;
+
+  "anal")
+    export nth_anal_gdas=8
+    export nth_anal_gfs=8
+    ;;
+  "upp")
+    if [[ "${CASE}" == "C768" ]]; then
+        then export memory_upp="480GB"
+    fi
+    ;;
+
+  "fit2obs")
+    export npe_node_fit2obs=3
+    ;;
+
+  "echgres")
+    export memory_echgres="200GB"
+    ;;
+
+  "arch" | "earc" | "getic")
+    declare -x "memory_${step}"="50GB"
+    ;;
+
+  "eupd")
+    case ${CASE} in
+      "C768" | "C384")
+        export npe_eupd=315
+        export nth_eupd=14
+        ;;
+      "C384")
+        export npe_eupd=315
+        export nth_eupd=14
+        ;;
+      *)
+        ;;
+    esac
+    export npe_node_eupd=$(( npe_node_max / nth_eupd ))
+    ;;
+
+  *)
+    ;;
+
+esac

From 8376ee6db62246cb96a72e4a10b9455bd87a50e4 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 12:40:31 -0500
Subject: [PATCH 03/46] Force npe_node_eobs to be set properly #2092

---
 parm/config/gfs/config.resources        |  7 +++++++
 parm/config/gfs/config.resources.gaea   | 21 +++++++++++++++++++++
 parm/config/gfs/config.resources.jet    |  9 +++++++++
 parm/config/gfs/config.resources.s4     |  3 +--
 parm/config/gfs/config.resources.wcoss2 | 10 ++++++++++
 5 files changed, 48 insertions(+), 2 deletions(-)
 create mode 100644 parm/config/gfs/config.resources.gaea

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index ee5958e026..40fed0b124 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -969,6 +969,13 @@ case ${step} in
     export npe_node_eobs=$(( npe_node_max / nth_eobs ))
     export is_exclusive=True
     export npe_node_eomg=${npe_node_eobs}
+    # Unset npe_node_eobs if it is not a multiple of npe_node_max
+    # to prevent dropping data on the floor.  This should be set int
+    # config.resources.{machine} instead.  This will result in an error at
+    # experiment setup time if not set in config.resources.{machine}.
+    if [[ $(( npe_node_max % npe_node_eobs )) != 0 ]]; then
+      unset npe_node_max
+    fi
     ;;
 
   "ediag")
diff --git a/parm/config/gfs/config.resources.gaea b/parm/config/gfs/config.resources.gaea
new file mode 100644
index 0000000000..427748bb8c
--- /dev/null
+++ b/parm/config/gfs/config.resources.gaea
@@ -0,0 +1,21 @@
+#!/usr/bin/bash
+
+# Gaea-specific job resources
+
+case ${step} in
+  "eobs")
+    # The number of tasks and cores used must be the same for eobs
+    # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
+    case ${CASE} in
+      "C768" | "C384")
+        export npe_node_eobs=50
+        ;;
+      *)
+        export npe_node_eobs=40
+        ;;
+    esac
+
+  *)
+    ;;
+
+esac
diff --git a/parm/config/gfs/config.resources.jet b/parm/config/gfs/config.resources.jet
index 64c1f4a240..1855ad3552 100644
--- a/parm/config/gfs/config.resources.jet
+++ b/parm/config/gfs/config.resources.jet
@@ -22,6 +22,15 @@ case ${step} in
     fi
     ;;
 
+  "eobs")
+    if [[ "${PARTITION_BATCH}" == "xjet" ]]; then
+      # The number of tasks and cores used must be the same for eobs
+      # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
+      # This would also be an issues for vjet and sjet if anyone runs on those nodes.
+      export npe_node_eobs=10
+    fi
+    ;;
+
   "eupd")
     case ${CASE} in
       "C384")
diff --git a/parm/config/gfs/config.resources.s4 b/parm/config/gfs/config.resources.s4
index e686dc4d40..46d3e183f7 100644
--- a/parm/config/gfs/config.resources.s4
+++ b/parm/config/gfs/config.resources.s4
@@ -36,12 +36,11 @@ case ${step} in
     esac
     ;;
 
-  "eobs" | "eomg")
+  "eobs")
     # The number of tasks and cores used must be the same for eobs
     # See https://github.com/NOAA-EMC/global-workflow/issues/2092 for details
     # For S4, this is accomplished by running 10 tasks/node
     export npe_node_eobs=10
-    export npe_node_eomg=${npe_node_eobs}
     ;;
 
   "eupd")
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
index d39fb97fac..d270e05447 100644
--- a/parm/config/gfs/config.resources.wcoss2
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -45,6 +45,16 @@ case ${step} in
     export npe_node_eupd=$(( npe_node_max / nth_eupd ))
     ;;
 
+  "eobs")
+    case ${CASE} in
+      "C768" | "C384")
+        export npe_node_eobs=50
+        ;;
+      *)
+        export npe_node_eobs=40
+        ;;
+    esac
+
   *)
     ;;
 

From aacecb0510373f6a3435b413399ddb7b4afdf94e Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 12:41:37 -0500
Subject: [PATCH 04/46] Use run-specific variables

---
 parm/config/gfs/config.anal | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal
index 5c20a198de..1a54b686d5 100644
--- a/parm/config/gfs/config.anal
+++ b/parm/config/gfs/config.anal
@@ -19,11 +19,12 @@ if [[ "${CDUMP}" = "gfs" ]] ; then
   export DIAG_TARBALL="YES"
 fi
 
-export npe_gsi=${npe_anal}
-
 if [[ "${CDUMP}" == "gfs" ]] ; then
   export npe_gsi=${npe_anal_gfs}
   export nth_anal=${nth_anal_gfs}
+elif [[ "${CDUMP}" == "gdas" ]]; then
+  export npe_gsi=${npe_anal_gdas}
+  export nth_anal=${nth_anal_gdas}
 fi
 
 # Set parameters specific to L127

From cf0e397e7602ea4ec9d6a75f557aace6c67b0ff7 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 12:41:48 -0500
Subject: [PATCH 05/46] Fix tabbing

---
 parm/config/gfs/config.ufs | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
index 9f6c47ce72..bc3599bc75 100644
--- a/parm/config/gfs/config.ufs
+++ b/parm/config/gfs/config.ufs
@@ -96,8 +96,8 @@ case "${fv3_res}" in
         export nthreads_fv3_gfs=1
         export nthreads_ufs=1
         export nthreads_ufs_gfs=1
-	      export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
-	      export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+        export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
+        export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="40.0,1.77,1.0,1.0"   # settings for GSL drag suite
         export knob_ugwp_tauamp=6.0e-3      # setting for UGWPv1 non-stationary GWD
         export k_split=1
@@ -138,8 +138,8 @@ case "${fv3_res}" in
           export nthreads_fv3_gfs=1
           export nthreads_ufs=1
           export nthreads_ufs_gfs=1
-	        export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
-	        export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
+          export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
+          export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
           export cdmbgwd_gsl="20.0,2.5,1.0,1.0"   # settings for GSL drag suite
           export knob_ugwp_tauamp=3.0e-3      # setting for UGWPv1 non-stationary GWD
           export k_split=1

From 463c93161b7d944918be3f9ffe427040be9056f1 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:08:06 -0500
Subject: [PATCH 06/46] Make nth_anal independent of run

---
 parm/config/gfs/config.resources        | 6 ++----
 parm/config/gfs/config.resources.hera   | 3 +--
 parm/config/gfs/config.resources.jet    | 3 +--
 parm/config/gfs/config.resources.s4     | 6 ++----
 parm/config/gfs/config.resources.wcoss2 | 4 ++--
 5 files changed, 8 insertions(+), 14 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 40fed0b124..d8883112f7 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -527,15 +527,13 @@ case ${step} in
     export wtime_anal_gdas="01:20:00"
     export wtime_anal_gfs="01:00:00"
     export npe_anal_gdas=780
-    export nth_anal_gdas=5
+    export nth_anal=5
     export npe_anal_gfs=825
-    export nth_anal_gfs=5
     case ${CASE} in
       "C384")
         export npe_anal_gdas=160
         export npe_anal_gfs=160
-        export nth_anal_gdas=10
-        export nth_anal_gfs=10
+        export nth_anal=10
         ;;
       "C192" | "C96" | "C48")
         export npe_anal_gdas=84
diff --git a/parm/config/gfs/config.resources.hera b/parm/config/gfs/config.resources.hera
index ad1cddb8d2..fd1c396a73 100644
--- a/parm/config/gfs/config.resources.hera
+++ b/parm/config/gfs/config.resources.hera
@@ -7,8 +7,7 @@ case ${step} in
     if "${CASE}" == "C384"; then
       export npe_anal_gdas=270
       export npe_anal_gfs=270
-      export nth_anal_gdas=8
-      export nth_anal_gfs=8
+      export nth_anal=8
     fi
     ;;
 
diff --git a/parm/config/gfs/config.resources.jet b/parm/config/gfs/config.resources.jet
index 1855ad3552..94a91a24f9 100644
--- a/parm/config/gfs/config.resources.jet
+++ b/parm/config/gfs/config.resources.jet
@@ -17,8 +17,7 @@ case ${step} in
     if "${CASE}" == "C384"; then
       export npe_anal_gdas=270
       export npe_anal_gfs=270
-      export nth_anal_gdas=8
-      export nth_anal_gfs=8
+      export nth_anal=8
     fi
     ;;
 
diff --git a/parm/config/gfs/config.resources.s4 b/parm/config/gfs/config.resources.s4
index 46d3e183f7..1da59aebac 100644
--- a/parm/config/gfs/config.resources.s4
+++ b/parm/config/gfs/config.resources.s4
@@ -15,14 +15,12 @@ case ${step} in
           export npe_anal_gfs=416
         fi
         #S4 is small, so run this task with just 1 thread
-        export nth_anal_gdas=1
-        export nth_anal_gfs=1
+        export nth_anal=1
         export wtime_anal_gdas="02:00:00"
         export wtime_anal_gfs="02:00:00"
         ;;
       "C192" | "C96" | "C48")
-        export nth_anal_gdas=4
-        export nth_anal_gfs=4
+        export nth_anal=4
         if [[ ${PARTITION_BATCH} == "s4" ]]; then
           export npe_anal_gdas=88
           export npe_anal_gfs=88
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
index d270e05447..4c75e3ac96 100644
--- a/parm/config/gfs/config.resources.wcoss2
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -8,9 +8,9 @@ case ${step} in
     ;;
 
   "anal")
-    export nth_anal_gdas=8
-    export nth_anal_gfs=8
+    export nth_anal=8
     ;;
+
   "upp")
     if [[ "${CASE}" == "C768" ]]; then
         then export memory_upp="480GB"

From a9ebe3b100be38f3cfc7dce3993a200445ec65a5 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:14:52 -0500
Subject: [PATCH 07/46] Fix gdas FHMAX/OUT variable

---
 workflow/rocoto/tasks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/workflow/rocoto/tasks.py b/workflow/rocoto/tasks.py
index 909b20f435..d8ecf930ba 100644
--- a/workflow/rocoto/tasks.py
+++ b/workflow/rocoto/tasks.py
@@ -140,8 +140,8 @@ def _get_forecast_hours(cdump, config, component='atmos') -> List[str]:
         # Get a list of all forecast hours
         fhrs = []
         if cdump in ['gdas']:
-            fhmax = local_config['FHMAX_GDAS']
-            fhout = local_config['FHOUT_GDAS']
+            fhmax = local_config['FHMAX']
+            fhout = local_config['FHOUT']
             fhrs = list(range(fhmin, fhmax + fhout, fhout))
         elif cdump in ['gfs', 'gefs']:
             fhmax = local_config['FHMAX_GFS']

From e5084f97bde97d74f09ac5dd9f8e1def954a5d7c Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:15:29 -0500
Subject: [PATCH 08/46] Correct enkf task variable name

---
 parm/config/gfs/config.resources | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index d8883112f7..143e180758 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -918,8 +918,8 @@ case ${step} in
     export npe_atmensanlletkf_enkfgfs=$(( layout_x * layout_y * 6 ))
     export nth_atmensanlletkf_enkfgdas=1
     export nth_atmensanlletkf_enkfgfs=${nth_atmensanlletkf_enkfgdas}
-    export npe_node_atmensanlletkf_gdas=$(( npe_node_max / nth_atmensanlletkf_enkfgdas ))
-    export npe_node_atmensanlletkf_gfs=$(( npe_node_max / nth_atmensanlletkf_enkfgfs ))
+    export npe_node_atmensanlletkf_enkfgdas=$(( npe_node_max / nth_atmensanlletkf_enkfgdas ))
+    export npe_node_atmensanlletkf_enkfgfs=$(( npe_node_max / nth_atmensanlletkf_enkfgfs ))
     export memory_atmensanlletkf="96GB"
     export is_exclusive=True
     ;;

From 6bed0d14bc7552d5120b49b58fab7508471a0a09 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:29:48 -0500
Subject: [PATCH 09/46] Fixed shellcheck issues

---
 parm/config/gfs/config.resources.gaea   | 1 +
 parm/config/gfs/config.resources.hera   | 2 +-
 parm/config/gfs/config.resources.s4     | 2 +-
 parm/config/gfs/config.resources.wcoss2 | 2 +-
 4 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/parm/config/gfs/config.resources.gaea b/parm/config/gfs/config.resources.gaea
index 427748bb8c..7d4cdcf241 100644
--- a/parm/config/gfs/config.resources.gaea
+++ b/parm/config/gfs/config.resources.gaea
@@ -14,6 +14,7 @@ case ${step} in
         export npe_node_eobs=40
         ;;
     esac
+    ;;
 
   *)
     ;;
diff --git a/parm/config/gfs/config.resources.hera b/parm/config/gfs/config.resources.hera
index fd1c396a73..c844f15503 100644
--- a/parm/config/gfs/config.resources.hera
+++ b/parm/config/gfs/config.resources.hera
@@ -4,7 +4,7 @@
 
 case ${step} in
   "anal")
-    if "${CASE}" == "C384"; then
+    if [[ "${CASE}" == "C384" ]]; then
       export npe_anal_gdas=270
       export npe_anal_gfs=270
       export nth_anal=8
diff --git a/parm/config/gfs/config.resources.s4 b/parm/config/gfs/config.resources.s4
index 1da59aebac..a119bf627b 100644
--- a/parm/config/gfs/config.resources.s4
+++ b/parm/config/gfs/config.resources.s4
@@ -42,7 +42,7 @@ case ${step} in
     ;;
 
   "eupd")
-    if [[ "$CASE" == "C384" ]]; then
+    if [[ "${CASE}" == "C384" ]]; then
       export npe_eupd=160
       export nth_eupd=2
     fi
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
index 4c75e3ac96..350edc44c1 100644
--- a/parm/config/gfs/config.resources.wcoss2
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -13,7 +13,7 @@ case ${step} in
 
   "upp")
     if [[ "${CASE}" == "C768" ]]; then
-        then export memory_upp="480GB"
+        export memory_upp="480GB"
     fi
     ;;
 

From b716671351c36339a12e658104d17f05e7f9fcea Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:32:51 -0500
Subject: [PATCH 10/46] Fix more shellcheck issues

---
 parm/config/gfs/config.resources.jet    | 2 +-
 parm/config/gfs/config.resources.wcoss2 | 5 +++--
 2 files changed, 4 insertions(+), 3 deletions(-)

diff --git a/parm/config/gfs/config.resources.jet b/parm/config/gfs/config.resources.jet
index 94a91a24f9..c392d105ec 100644
--- a/parm/config/gfs/config.resources.jet
+++ b/parm/config/gfs/config.resources.jet
@@ -14,7 +14,7 @@ esac
 
 case ${step} in
   "anal")
-    if "${CASE}" == "C384"; then
+    if [[ "${CASE}" == "C384" ]]; then
       export npe_anal_gdas=270
       export npe_anal_gfs=270
       export nth_anal=8
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
index 350edc44c1..1d0842a83e 100644
--- a/parm/config/gfs/config.resources.wcoss2
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -40,7 +40,7 @@ case ${step} in
         export nth_eupd=14
         ;;
       *)
-        ;;
+      ;;
     esac
     export npe_node_eupd=$(( npe_node_max / nth_eupd ))
     ;;
@@ -54,8 +54,9 @@ case ${step} in
         export npe_node_eobs=40
         ;;
     esac
+    ;;
 
   *)
-    ;;
+  ;;
 
 esac

From e2e2421f5dc7ee65fc39d62aefd4f1f418490ec6 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 10 Jun 2024 13:34:26 -0500
Subject: [PATCH 11/46] Remove unused case

---
 parm/config/gfs/config.resources.wcoss2 | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.wcoss2
index 1d0842a83e..089fdeb24f 100644
--- a/parm/config/gfs/config.resources.wcoss2
+++ b/parm/config/gfs/config.resources.wcoss2
@@ -35,10 +35,6 @@ case ${step} in
         export npe_eupd=315
         export nth_eupd=14
         ;;
-      "C384")
-        export npe_eupd=315
-        export nth_eupd=14
-        ;;
       *)
       ;;
     esac

From 7fd5f0e2b5776cb42eb45a155b65b088b1a8322d Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 11 Jun 2024 11:35:17 -0500
Subject: [PATCH 12/46] Make additional fcst, efcs, anal variables RUN-specific

---
 env/AWSPW.env                    |  27 ++-----
 env/GAEA.env                     |  12 ++-
 env/HERA.env                     |  17 ++---
 env/HERCULES.env                 |  26 ++-----
 env/JET.env                      |  12 ++-
 env/ORION.env                    |  26 ++-----
 env/S4.env                       |  12 ++-
 env/WCOSS2.env                   |  23 +++---
 parm/config/gfs/config.anal      |   6 +-
 parm/config/gfs/config.resources |   3 +
 parm/config/gfs/config.ufs       | 126 ++++++++++++++++---------------
 11 files changed, 120 insertions(+), 170 deletions(-)

diff --git a/env/AWSPW.env b/env/AWSPW.env
index 7d81000f5c..00eea066b9 100755
--- a/env/AWSPW.env
+++ b/env/AWSPW.env
@@ -26,13 +26,11 @@ ulimit -a
 
 if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
@@ -109,21 +107,6 @@ elif [[ "${step}" = "awips" ]]; then
     [[ ${NTHREADS_AWIPS} -gt ${nth_max} ]] && export NTHREADS_AWIPS=${nth_max}
     export APRUN_AWIPSCFP="${launcher} -n ${npe_awips} ${mpmd_opt}"
 
-elif [[ "${step}" = "gempak" ]]; then
-
-    export CFP_MP="YES"
-
-    if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
-        npe_node_gempak=${npe_node_gempak_gfs}
-    fi
-
-    nth_max=$((npe_node_max / npe_node_gempak))
-
-    export NTHREADS_GEMPAK=${nth_gempak:-1}
-    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
-
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
diff --git a/env/GAEA.env b/env/GAEA.env
index c19fecc934..18980601be 100755
--- a/env/GAEA.env
+++ b/env/GAEA.env
@@ -19,13 +19,11 @@ ulimit -a
 
 if [[ "${step}" = "fcst" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
diff --git a/env/HERA.env b/env/HERA.env
index ccaaea32e7..fcf8efa278 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -212,13 +212,11 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
@@ -307,9 +305,8 @@ elif [[ "${step}" = "gempak" ]]; then
 
     export CFP_MP="YES"
 
-    if [[ ${CDUMP} == "gfs" ]]; then
-        npe_node_gempak=${npe_node_gempak_gfs}
-    fi
+    var="npe_node_gempak_${RUN}"
+    declare "npe_node_gempak"=${!var}
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
diff --git a/env/HERCULES.env b/env/HERCULES.env
index 0824ba913a..3e30d896da 100755
--- a/env/HERCULES.env
+++ b/env/HERCULES.env
@@ -205,13 +205,12 @@ case ${step} in
  "fcst" | "efcs")
 
     export OMP_STACKSIZE=512M
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
@@ -306,18 +305,7 @@ case ${step} in
  ;;
  "gempak")
 
-    export CFP_MP="YES"
-
-    if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
-        npe_node_gempak=${npe_node_gempak_gfs}
-    fi
-
-    nth_max=$((npe_node_max / npe_node_gempak))
-
-    export NTHREADS_GEMPAK=${nth_gempak:-1}
-    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+    echo "WARNING: ${step} is not enabled on ${machine}!"
 
  ;;
  "fit2obs")
diff --git a/env/JET.env b/env/JET.env
index 5bd88dc93a..16159a1919 100755
--- a/env/JET.env
+++ b/env/JET.env
@@ -185,13 +185,11 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
diff --git a/env/ORION.env b/env/ORION.env
index f701e55aa2..ed6f109219 100755
--- a/env/ORION.env
+++ b/env/ORION.env
@@ -219,14 +219,11 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    export OMP_STACKSIZE=512M
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
@@ -312,18 +309,7 @@ elif [[ "${step}" = "awips" ]]; then
 
 elif [[ "${step}" = "gempak" ]]; then
 
-    export CFP_MP="YES"
-
-    if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
-        npe_node_gempak=${npe_node_gempak_gfs}
-    fi
-
-    nth_max=$((npe_node_max / npe_node_gempak))
-
-    export NTHREADS_GEMPAK=${nth_gempak:-1}
-    [[ ${NTHREADS_GEMPAK} -gt ${nth_max} ]] && export NTHREADS_GEMPAK=${nth_max}
-    export APRUN="${launcher} -n ${npe_gempak} ${mpmd_opt}"
+    echo "WARNING: ${step} is not enabled on ${machine}!"
 
 elif [[ "${step}" = "fit2obs" ]]; then
 
diff --git a/env/S4.env b/env/S4.env
index 9ba3a61b01..e2cc1beb91 100755
--- a/env/S4.env
+++ b/env/S4.env
@@ -172,13 +172,11 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index 0876e4127d..005ab50abc 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -38,7 +38,10 @@ elif [[ "${step}" = "prep_emissions" ]]; then
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"
-    if [[ "${step}" = "waveprep" ]] && [[ "${CDUMP}" = "gfs" ]]; then export NTASKS=${NTASKS_gfs} ; fi
+    if [[ "${step}" = "waveprep" ]]; then
+        ntask_var="NTASKS_${RUN}"
+        export NTASKS=${!ntasks_var}
+    fi
     export wavempexec="${launcher} -np"
     export wave_mpmd=${mpmd_opt}
 
@@ -175,13 +178,11 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    if [[ "${CDUMP}" =~ "gfs" ]]; then
-        nprocs="npe_${step}_gfs"
-        ppn="npe_node_${step}_gfs" || ppn="npe_node_${step}"
-    else
-        nprocs="npe_${step}"
-        ppn="npe_node_${step}"
-    fi
+    ppn="npe_node_${step}_${RUN//enkf/}"
+    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN//enkf/}"
+    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
     # With ESMF threading, the model wants to use the full node
@@ -284,10 +285,8 @@ elif [[ "${step}" = "awips" ]]; then
 
 elif [[ "${step}" = "gempak" ]]; then
 
-    if [[ ${CDUMP} == "gfs" ]]; then
-        npe_gempak=${npe_gempak_gfs}
-        npe_node_gempak=${npe_node_gempak_gfs}
-    fi
+    var="npe_node_gempak_${RUN}"
+    declare "npe_node_gempak"=${!var}
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal
index 1a54b686d5..1eba5024a6 100644
--- a/parm/config/gfs/config.anal
+++ b/parm/config/gfs/config.anal
@@ -19,12 +19,10 @@ if [[ "${CDUMP}" = "gfs" ]] ; then
   export DIAG_TARBALL="YES"
 fi
 
-if [[ "${CDUMP}" == "gfs" ]] ; then
+if [[ "${RUN}" =~ "gfs" ]] ; then
   export npe_gsi=${npe_anal_gfs}
-  export nth_anal=${nth_anal_gfs}
-elif [[ "${CDUMP}" == "gdas" ]]; then
+elif [[ "${RUN}" =~ "gdas" ]]; then
   export npe_gsi=${npe_anal_gdas}
-  export nth_anal=${nth_anal_gdas}
 fi
 
 # Set parameters specific to L127
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 143e180758..7e74bd030a 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -598,6 +598,8 @@ case ${step} in
         ntasks_quilt=${ntasks_quilt_gfs}
         nthreads_fv3=${nthreads_fv3_gfs}
         nthreads_ufs=${nthreads_ufs_gfs}
+        # Will not be set if we are skipping the mediator
+        nthreads_mediator=${nthreads_mediator_gfs:-}
       elif [[ "${_CDUMP}" =~ "gdas" ]]; then
         export layout_x=${layout_x_gdas}
         export layout_y=${layout_y_gdas}
@@ -607,6 +609,7 @@ case ${step} in
         ntasks_quilt=${ntasks_quilt_gdas}
         nthreads_fv3=${nthreads_fv3_gdas}
         nthreads_ufs=${nthreads_ufs_gdas}
+        nthreads_mediator=${nthreads_mediator_gdas:-}
       fi
 
       # Determine if using ESMF-managed threading or traditional threading
diff --git a/parm/config/gfs/config.ufs b/parm/config/gfs/config.ufs
index bc3599bc75..b87435cfef 100644
--- a/parm/config/gfs/config.ufs
+++ b/parm/config/gfs/config.ufs
@@ -88,13 +88,13 @@ fi
 case "${fv3_res}" in
     "C48")
         export DELTIM=1200
-        export layout_x=1
-        export layout_y=1
+        export layout_x_gdas=1
+        export layout_y_gdas=1
         export layout_x_gfs=1
         export layout_y_gfs=1
-        export nthreads_fv3=1
+        export nthreads_fv3_gdas=1
         export nthreads_fv3_gfs=1
-        export nthreads_ufs=1
+        export nthreads_ufs_gdas=1
         export nthreads_ufs_gfs=1
         export xr_cnvcld=".false."  # Do not pass conv. clouds to Xu-Randall cloud fraction
         export cdmbgwd="0.071,2.1,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
@@ -105,8 +105,8 @@ case "${fv3_res}" in
         export tau=10.0
         export rf_cutoff=100.0
         export fv_sg_adj=3600
-        export WRITE_GROUP=1
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
+        export WRITE_GROUP_GDAS=1
+        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=1
         export WRITE_GROUP_GFS=1
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
         ;;
@@ -124,19 +124,19 @@ case "${fv3_res}" in
           export npy_nest=241
           export NEST_DLON=0.25
           export NEST_DLAT=0.25
-          export WRITE_GROUP=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=2
+          export WRITE_GROUP_GDAS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=2
           export WRITE_GROUP_GFS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=2
         else
           export DELTIM=600
-          export layout_x=2
-          export layout_y=2
+          export layout_x_gdas=2
+          export layout_y_gdas=2
           export layout_x_gfs=2
           export layout_y_gfs=2
-          export nthreads_fv3=1
+          export nthreads_fv3_gdas=1
           export nthreads_fv3_gfs=1
-          export nthreads_ufs=1
+          export nthreads_ufs_gdas=1
           export nthreads_ufs_gfs=1
           export xr_cnvcld=.false. # Do not pass conv. clouds to Xu-Randall cloud fraction
           export cdmbgwd="0.14,1.8,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
@@ -147,8 +147,8 @@ case "${fv3_res}" in
           export tau=8.0
           export rf_cutoff=100.0
           export fv_sg_adj=1800
-          export WRITE_GROUP=1
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=1
+          export WRITE_GROUP_GDAS=1
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=1
           export WRITE_GROUP_GFS=1
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=1
         fi
@@ -167,19 +167,19 @@ case "${fv3_res}" in
           export npy_nest=481
           export NEST_DLON=0.125
           export NEST_DLAT=0.125
-          export WRITE_GROUP=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=15
+          export WRITE_GROUP_GDAS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=15
           export WRITE_GROUP_GFS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=15
         else
           export DELTIM=600
-          export layout_x=4
-          export layout_y=6
+          export layout_x_gdas=4
+          export layout_y_gdas=6
           export layout_x_gfs=4
           export layout_y_gfs=6
-          export nthreads_fv3=1
+          export nthreads_fv3_gdas=1
           export nthreads_fv3_gfs=2
-          export nthreads_ufs=1
+          export nthreads_ufs_gdas=1
           export nthreads_ufs_gfs=2
           export cdmbgwd="0.23,1.5,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
           export cdmbgwd_gsl="10.0,3.5,1.0,1.0"   # settings for GSL drag suite
@@ -189,8 +189,8 @@ case "${fv3_res}" in
           export tau=6.0
           export rf_cutoff=100.0
           export fv_sg_adj=1800
-          export WRITE_GROUP=1
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GDAS=1
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10
           export WRITE_GROUP_GFS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=5
         fi
@@ -198,8 +198,8 @@ case "${fv3_res}" in
     "C384")
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=150
-          export layout_x=8
-          export layout_y=8
+          export layout_x_gdas=8
+          export layout_y_gdas=8
           export layout_x_gfs=8
           export layout_y_gfs=8
           export layout_x_nest=34
@@ -211,19 +211,19 @@ case "${fv3_res}" in
           export npy_nest=961
           export NEST_DLON=0.0625
           export NEST_DLAT=0.0625
-          export WRITE_GROUP=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=20
+          export WRITE_GROUP_GDAS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=20
           export WRITE_GROUP_GFS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20
         else
           export DELTIM=300
-          export layout_x=8
-          export layout_y=8
+          export layout_x_gdas=8
+          export layout_y_gdas=8
           export layout_x_gfs=8
           export layout_y_gfs=8
-          export nthreads_fv3=2
+          export nthreads_fv3_gdas=2
           export nthreads_fv3_gfs=2
-          export nthreads_ufs=2
+          export nthreads_ufs_gdas=2
           export nthreads_ufs_gfs=2
           export cdmbgwd="1.1,0.72,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
           export cdmbgwd_gsl="5.0,5.0,1.0,1.0"   # settings for GSL drag suite
@@ -233,8 +233,8 @@ case "${fv3_res}" in
           export tau=4.0
           export rf_cutoff=100.0
           export fv_sg_adj=900
-          export WRITE_GROUP=4
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GDAS=4
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10
           export WRITE_GROUP_GFS=4
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10
         fi
@@ -242,13 +242,14 @@ case "${fv3_res}" in
     "C768")
         if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
           export DELTIM=75
-          export layout_x=16
-          export layout_y=10
+          export layout_x_gdas=16
+          export layout_y_gdas=10
           export layout_x_gfs=16
           export layout_y_gfs=10
           export layout_x_nest=48
           export layout_y_nest=45
-          export nthreads_fv3=2
+          export nthreads_fv3_nest=2
+          export nthreads_fv3_gdas=2
           export nthreads_fv3_gfs=2
           export nest_refine=4
           export nest_ioffset=24
@@ -257,19 +258,19 @@ case "${fv3_res}" in
           export npy_nest=1921
           export NEST_DLON=0.0325
           export NEST_DLAT=0.0325
-          export WRITE_GROUP=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=90
+          export WRITE_GROUP_GDAS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=90
           export WRITE_GROUP_GFS=2
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=90
         else
           export DELTIM=150
-          export layout_x=8
-          export layout_y=12
+          export layout_x_gdas=8
+          export layout_y_gdas=12
           export layout_x_gfs=12
           export layout_y_gfs=16
-          export nthreads_fv3=4
+          export nthreads_fv3_gdas=4
           export nthreads_fv3_gfs=4
-          export nthreads_ufs=4
+          export nthreads_ufs_gdas=4
           export nthreads_ufs_gfs=4
           export cdmbgwd="4.0,0.15,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
           export cdmbgwd_gsl="2.5,7.5,1.0,1.0"   # settings for GSL drag suite
@@ -279,21 +280,21 @@ case "${fv3_res}" in
           export tau=3.0
           export rf_cutoff=100.0
           export fv_sg_adj=450
-          export WRITE_GROUP=2
-          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10
+          export WRITE_GROUP_GDAS=2
+          export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10
           export WRITE_GROUP_GFS=4
           export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20 #Note this should be 10 for WCOSS2
         fi
         ;;
     "C1152")
         export DELTIM=150
-        export layout_x=8
-        export layout_y=16
+        export layout_x_gdas=8
+        export layout_y_gdas=16
         export layout_x_gfs=8
         export layout_y_gfs=16
-        export nthreads_fv3=4
+        export nthreads_fv3_gdas=4
         export nthreads_fv3_gfs=4
-        export nthreads_ufs=4
+        export nthreads_ufs_gdas=4
         export nthreads_ufs_gfs=4
         export cdmbgwd="4.0,0.10,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="1.67,8.8,1.0,1.0"   # settings for GSL drag suite
@@ -303,20 +304,20 @@ case "${fv3_res}" in
         export tau=2.5
         export rf_cutoff=100.0
         export fv_sg_adj=450
-        export WRITE_GROUP=4
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10  # TODO: refine these numbers when a case is available
+        export WRITE_GROUP_GDAS=4
+        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10  # TODO: refine these numbers when a case is available
         export WRITE_GROUP_GFS=4
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=20  # TODO: refine these numbers when a case is available
         ;;
     "C3072")
         export DELTIM=90
         export layout_x=16
-        export layout_y=32
-        export layout_x_gfs=16
+        export layout_y_gdas=32
+        export layout_x_gfs_gdas=16
         export layout_y_gfs=32
-        export nthreads_fv3=4
+        export nthreads_fv3_gdas=4
         export nthreads_fv3_gfs=4
-        export nthreads_ufs=4
+        export nthreads_ufs_gdas=4
         export nthreads_ufs_gfs=4
         export cdmbgwd="4.0,0.05,1.0,1.0"  # mountain blocking, ogwd, cgwd, cgwd src scaling
         export cdmbgwd_gsl="0.625,14.1,1.0,1.0"   # settings for GSL drag suite
@@ -326,8 +327,8 @@ case "${fv3_res}" in
         export tau=0.5
         export rf_cutoff=100.0
         export fv_sg_adj=300
-        export WRITE_GROUP=4
-        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE=10  # TODO: refine these numbers when a case is available
+        export WRITE_GROUP_GDAS=4
+        export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS=10  # TODO: refine these numbers when a case is available
         export WRITE_GROUP_GFS=4
         export WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS=10  # TODO: refine these numbers when a case is available
         ;;
@@ -337,22 +338,22 @@ case "${fv3_res}" in
         ;;
 esac
 
-(( WRTTASK_PER_GROUP_PER_THREAD = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE * 6 ))
+(( WRTTASK_PER_GROUP_PER_THREAD_GDAS = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GDAS * 6 ))
 (( WRTTASK_PER_GROUP_PER_THREAD_GFS = WRTTASK_PER_GROUP_PER_THREAD_PER_TILE_GFS * 6 ))
-export WRTTASK_PER_GROUP_PER_THREAD
+export WRTTASK_PER_GROUP_PER_THREAD_GDAS
 export WRTTASK_PER_GROUP_PER_THREAD_GFS
 
-(( ntasks_fv3 = layout_x * layout_y * 6 ))
+(( ntasks_fv3_gdas = layout_x_gdas * layout_y_gdas * 6 ))
 (( ntasks_fv3_gfs = layout_x_gfs * layout_y_gfs * 6 ))
 if [[ "${DO_NEST:-NO}" == "YES" ]] ; then
   (( ntasks_fv3_gfs += layout_x_nest * layout_y_nest ))
 fi
-export ntasks_fv3
+export ntasks_fv3_gdas
 export ntasks_fv3_gfs
 
-(( ntasks_quilt = WRITE_GROUP * WRTTASK_PER_GROUP_PER_THREAD ))
+(( ntasks_quilt_gdas = WRITE_GROUP_GDAS * WRTTASK_PER_GROUP_PER_THREAD_GDAS ))
 (( ntasks_quilt_gfs = WRITE_GROUP_GFS * WRTTASK_PER_GROUP_PER_THREAD_GFS ))
-export ntasks_quilt
+export ntasks_quilt_gdas
 export ntasks_quilt_gfs
 
 # Determine whether to use parallel NetCDF based on resolution
@@ -389,7 +390,8 @@ model_list="atm"
 # Mediator specific settings
 if [[ "${skip_mediator}" == "false" ]]; then
   export cpl=".true."
-  export nthreads_mediator=${nthreads_fv3}  # Use same threads as FV3
+  export nthreads_mediator_gfs=${nthreads_fv3_gfs}  # Use same threads as FV3
+  export nthreads_mediator_gdas=${nthreads_fv3_gdas}
   export CCPP_SUITE="FV3_GFS_v17_coupled_p8_ugwpv1"  # TODO: Does this include FV3_GFS_v17_p8?  Can this be used instead of FV3_GFS_v17_p8?
 fi
 

From ffec31659fa0ea8cbd704721e15a8eea2575b403 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 11 Jun 2024 11:38:05 -0500
Subject: [PATCH 13/46] Strip enkf from RUN when reading config.anal

---
 parm/config/gfs/config.anal | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal
index 1eba5024a6..3679807889 100644
--- a/parm/config/gfs/config.anal
+++ b/parm/config/gfs/config.anal
@@ -19,11 +19,8 @@ if [[ "${CDUMP}" = "gfs" ]] ; then
   export DIAG_TARBALL="YES"
 fi
 
-if [[ "${RUN}" =~ "gfs" ]] ; then
-  export npe_gsi=${npe_anal_gfs}
-elif [[ "${RUN}" =~ "gdas" ]]; then
-  export npe_gsi=${npe_anal_gdas}
-fi
+npe_var="npe_anal_${RUN//enkf}"
+export npe_gsi="${!npe_var}"
 
 # Set parameters specific to L127
 if [[ ${LEVS} = "128" ]]; then

From b80814b387028081cfec862f03c54bb18c07f219 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 11 Jun 2024 11:41:39 -0500
Subject: [PATCH 14/46] Add double quotes

---
 env/HERA.env   | 2 +-
 env/WCOSS2.env | 6 +++---
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/env/HERA.env b/env/HERA.env
index fcf8efa278..674aa6e447 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -306,7 +306,7 @@ elif [[ "${step}" = "gempak" ]]; then
     export CFP_MP="YES"
 
     var="npe_node_gempak_${RUN}"
-    declare "npe_node_gempak"=${!var}
+    declare "npe_node_gempak"="${!var}"
 
     nth_max=$((npe_node_max / npe_node_gempak))
 
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index 005ab50abc..4eaf760302 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -39,8 +39,8 @@ elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}
 
     export USE_CFP="YES"
     if [[ "${step}" = "waveprep" ]]; then
-        ntask_var="NTASKS_${RUN}"
-        export NTASKS=${!ntasks_var}
+        ntasks_var="NTASKS_${RUN}"
+        export NTASKS="${!ntasks_var}"
     fi
     export wavempexec="${launcher} -np"
     export wave_mpmd=${mpmd_opt}
@@ -286,7 +286,7 @@ elif [[ "${step}" = "awips" ]]; then
 elif [[ "${step}" = "gempak" ]]; then
 
     var="npe_node_gempak_${RUN}"
-    declare "npe_node_gempak"=${!var}
+    declare "npe_node_gempak"="${!var}"
 
     nth_max=$((npe_node_max / npe_node_gempak))
 

From 63b97dca1d6aeb42906a206e74dcaef9d85aa97c Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 11 Jun 2024 12:10:57 -0500
Subject: [PATCH 15/46] Fix fcst/efcs RUNs in variable declarations

---
 env/AWSPW.env    | 8 ++++----
 env/GAEA.env     | 8 ++++----
 env/HERA.env     | 8 ++++----
 env/HERCULES.env | 8 ++++----
 env/JET.env      | 8 ++++----
 env/ORION.env    | 8 ++++----
 env/S4.env       | 8 ++++----
 env/WCOSS2.env   | 8 ++++----
 8 files changed, 32 insertions(+), 32 deletions(-)

diff --git a/env/AWSPW.env b/env/AWSPW.env
index 00eea066b9..e9bcd46e2d 100755
--- a/env/AWSPW.env
+++ b/env/AWSPW.env
@@ -26,10 +26,10 @@ ulimit -a
 
 if [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/GAEA.env b/env/GAEA.env
index 18980601be..d7d44d7534 100755
--- a/env/GAEA.env
+++ b/env/GAEA.env
@@ -19,10 +19,10 @@ ulimit -a
 
 if [[ "${step}" = "fcst" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/HERA.env b/env/HERA.env
index 674aa6e447..045ea122dc 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -212,10 +212,10 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/HERCULES.env b/env/HERCULES.env
index 3e30d896da..f14129d11d 100755
--- a/env/HERCULES.env
+++ b/env/HERCULES.env
@@ -206,10 +206,10 @@ case ${step} in
 
     export OMP_STACKSIZE=512M
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/JET.env b/env/JET.env
index 16159a1919..1da4e107d2 100755
--- a/env/JET.env
+++ b/env/JET.env
@@ -185,10 +185,10 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/ORION.env b/env/ORION.env
index ed6f109219..ea38ffdf0e 100755
--- a/env/ORION.env
+++ b/env/ORION.env
@@ -219,10 +219,10 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/S4.env b/env/S4.env
index e2cc1beb91..430442fe42 100755
--- a/env/S4.env
+++ b/env/S4.env
@@ -172,10 +172,10 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index 4eaf760302..4388765521 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -178,10 +178,10 @@ elif [[ "${step}" = "eupd" ]]; then
 
 elif [[ "${step}" = "fcst" ]] || [[ "${step}" = "efcs" ]]; then
 
-    ppn="npe_node_${step}_${RUN//enkf/}"
-    [[ -z "${!ppn}" ]] && ppn="npe_node_${step}"
-    nprocs="npe_${step}_${RUN//enkf/}"
-    [[ -z "${!nprocs}" ]] && nprocs="npe_${step}"
+    ppn="npe_node_${step}_${RUN}"
+    [[ -z "${!ppn+0}" ]] && ppn="npe_node_${step}"
+    nprocs="npe_${step}_${RUN}"
+    [[ -z ${!nprocs+0} ]] && nprocs="npe_${step}"
 
     (( nnodes = (${!nprocs}+${!ppn}-1)/${!ppn} ))
     (( ntasks = nnodes*${!ppn} ))

From 552ddda7d21c6f861630604ea4c6f990bf99d327 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 11 Jun 2024 19:01:49 +0000
Subject: [PATCH 16/46] Fix C768 analysis case in config.resources

---
 parm/config/gefs/config.resources |  2 +-
 parm/config/gfs/config.resources  | 35 +++++++++++++++++--------------
 2 files changed, 20 insertions(+), 17 deletions(-)

diff --git a/parm/config/gefs/config.resources b/parm/config/gefs/config.resources
index 04d55ae082..94dc5adad9 100644
--- a/parm/config/gefs/config.resources
+++ b/parm/config/gefs/config.resources
@@ -202,7 +202,7 @@ case ${step} in
         declare -x "wtime_${step}_gfs"="06:00:00"
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
         ;;
     esac
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 7e74bd030a..938b0b28be 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -274,7 +274,7 @@ case ${step} in
         layout_y=1
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
     esac
 
@@ -308,7 +308,7 @@ case ${step} in
         layout_y=1
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
     esac
 
@@ -341,7 +341,7 @@ case ${step} in
         layout_y=1
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
     esac
 
@@ -389,7 +389,7 @@ case ${step} in
       "050")  npes=16;;
       "500")  npes=16;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${OCNRES}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
         exit 4
     esac
 
@@ -416,7 +416,7 @@ case ${step} in
         memory_ocnanalrun="24GB"
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${OCNRES}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
         exit 4
     esac
 
@@ -444,7 +444,7 @@ case ${step} in
         memory_ocnanalecen="24GB"
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${OCNRES}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
         exit 4
     esac
 
@@ -472,7 +472,7 @@ case ${step} in
         memory_ocnanalletkf="24GB"
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${OCNRES}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
         exit 4
     esac
 
@@ -501,7 +501,7 @@ case ${step} in
         memory_ocnanalchkpt="32GB"
         npes=8;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${OCNRES}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${OCNRES}"
         exit 4
     esac
     export npe_ocnanalchkpt=${npes}
@@ -526,10 +526,12 @@ case ${step} in
   "anal")
     export wtime_anal_gdas="01:20:00"
     export wtime_anal_gfs="01:00:00"
-    export npe_anal_gdas=780
-    export nth_anal=5
-    export npe_anal_gfs=825
     case ${CASE} in
+      "C768")
+        export npe_anal_gdas=780
+        export npe_anal_gfs=825
+        export nth_anal=5
+        ;;
       "C384")
         export npe_anal_gdas=160
         export npe_anal_gfs=160
@@ -538,9 +540,10 @@ case ${step} in
       "C192" | "C96" | "C48")
         export npe_anal_gdas=84
         export npe_anal_gfs=84
+        export nth_anal=5
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
         ;;
     esac
@@ -729,7 +732,7 @@ case ${step} in
         declare -x "wtime_${step}_gfs"="06:00:00"
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
         ;;
     esac
@@ -760,7 +763,7 @@ case ${step} in
         export memory_upp="96GB"
       ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
       ;;
     esac
@@ -958,7 +961,7 @@ case ${step} in
       "C384")                 export npe_eobs=100;;
       "C192" | "C96" | "C48") export npe_eobs=40;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
         ;;
     esac
@@ -1003,7 +1006,7 @@ case ${step} in
         export nth_eupd=2
         ;;
       *)
-        echo "FATAL ERROR: Resources not defined for job ${job} at resolution ${CASE}"
+        echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
         exit 4
         ;;
     esac

From 27cdab4fa1657b73e2a4a3a07ff63ddb96fa45e5 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Wed, 12 Jun 2024 12:15:47 +0000
Subject: [PATCH 17/46] Corrected machine-specific resource config names

---
 parm/config/gfs/config.resources                            | 6 ++++++
 .../gfs/{config.resources.gaea => config.resources.GAEA}    | 0
 .../gfs/{config.resources.hera => config.resources.HERA}    | 0
 ...{config.resources.hercules => config.resources.HERCULES} | 0
 .../gfs/{config.resources.jet => config.resources.JET}      | 0
 .../config/gfs/{config.resources.s4 => config.resources.S4} | 0
 .../{config.resources.wcoss2 => config.resources.WCOSS2}    | 0
 7 files changed, 6 insertions(+)
 rename parm/config/gfs/{config.resources.gaea => config.resources.GAEA} (100%)
 rename parm/config/gfs/{config.resources.hera => config.resources.HERA} (100%)
 rename parm/config/gfs/{config.resources.hercules => config.resources.HERCULES} (100%)
 rename parm/config/gfs/{config.resources.jet => config.resources.JET} (100%)
 rename parm/config/gfs/{config.resources.s4 => config.resources.S4} (100%)
 rename parm/config/gfs/{config.resources.wcoss2 => config.resources.WCOSS2} (100%)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 938b0b28be..0219431286 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -3,6 +3,8 @@
 ########## config.resources ##########
 # Set resource information for job tasks
 # e.g. walltime, node, cores per node, memory etc.
+# Note: machine-specific resources should be placed into the appropriate config file:
+#       config.resources.${machine}
 
 if (( $# != 1 )); then
 
@@ -1239,4 +1241,8 @@ case ${step} in
 
 esac
 
+if [[ -f "${EXPDIR}/config.resources.${machine}" ]]; then
+   source "${EXPDIR}/config.resources.${machine}"
+fi
+
 echo "END: config.resources"
diff --git a/parm/config/gfs/config.resources.gaea b/parm/config/gfs/config.resources.GAEA
similarity index 100%
rename from parm/config/gfs/config.resources.gaea
rename to parm/config/gfs/config.resources.GAEA
diff --git a/parm/config/gfs/config.resources.hera b/parm/config/gfs/config.resources.HERA
similarity index 100%
rename from parm/config/gfs/config.resources.hera
rename to parm/config/gfs/config.resources.HERA
diff --git a/parm/config/gfs/config.resources.hercules b/parm/config/gfs/config.resources.HERCULES
similarity index 100%
rename from parm/config/gfs/config.resources.hercules
rename to parm/config/gfs/config.resources.HERCULES
diff --git a/parm/config/gfs/config.resources.jet b/parm/config/gfs/config.resources.JET
similarity index 100%
rename from parm/config/gfs/config.resources.jet
rename to parm/config/gfs/config.resources.JET
diff --git a/parm/config/gfs/config.resources.s4 b/parm/config/gfs/config.resources.S4
similarity index 100%
rename from parm/config/gfs/config.resources.s4
rename to parm/config/gfs/config.resources.S4
diff --git a/parm/config/gfs/config.resources.wcoss2 b/parm/config/gfs/config.resources.WCOSS2
similarity index 100%
rename from parm/config/gfs/config.resources.wcoss2
rename to parm/config/gfs/config.resources.WCOSS2

From 12b880bcdeb1c9b1850d05b76c33f35f8d66fa44 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Wed, 12 Jun 2024 12:16:54 +0000
Subject: [PATCH 18/46] Add memory requirement for WCOSS2 prep job

---
 parm/config/gfs/config.resources.WCOSS2 | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index 089fdeb24f..5927b82aed 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -5,6 +5,7 @@
 case ${step} in
   "prep")
     export is_exclusive=True
+    export memory_prep="480GB"
     ;;
 
   "anal")

From 41134d5205f426c76b03d41215f573b253044a32 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Wed, 12 Jun 2024 13:26:20 +0000
Subject: [PATCH 19/46] Adjust epos and analcalc high res time limits/memory
 upward

---
 parm/config/gfs/config.resources | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 0219431286..a36afddcf0 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -557,6 +557,7 @@ case ${step} in
 
   "analcalc")
     export wtime_analcalc="00:10:00"
+    [[ ${CASE} == "C384" || ${CASE} == "C768" ]] && export wtime_analcalc="00:15:00"
     export npe_analcalc=127
     export ntasks="${npe_analcalc}"
     export nth_analcalc=1
@@ -565,6 +566,9 @@ case ${step} in
     export nth_echgres_gfs=12
     export is_exclusive=True
     export memory_analcalc="48GB"
+    if [[ "${CASE}" == "C384" || "${CASE}" == "C768" ]]; then
+       memory_analcalc="80GB"
+    fi
     ;;
 
   "analdiag")
@@ -1040,6 +1044,7 @@ case ${step} in
 
   "epos")
     export wtime_epos="00:15:00"
+    [[ ${CASE} == "C768" ]] && export wtime_epos="00:25:00"
     export npe_epos=80
     export nth_epos=1
     export npe_node_epos=$(( npe_node_max / nth_epos ))

From 7271bfee960e417bc3eb4327d7d876fef4d7ee9d Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 12:03:18 +0000
Subject: [PATCH 20/46] Increase memory for UPP @C384 on WCOSS2.

---
 parm/config/gfs/config.resources.WCOSS2 | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index 5927b82aed..0e827f4ad3 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -13,7 +13,7 @@ case ${step} in
     ;;
 
   "upp")
-    if [[ "${CASE}" == "C768" ]]; then
+    if [[ "${CASE}" == "C768" || "${CASE}" == "C384" ]]; then
         export memory_upp="480GB"
     fi
     ;;

From 4e28b2c19ccd45ff2c32fd3a965b41d3ea67ab18 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 12:10:13 +0000
Subject: [PATCH 21/46] Make wtime for waveprep RUN-agnostic

---
 parm/config/gfs/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index a36afddcf0..893f6ec644 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -115,7 +115,7 @@ case ${step} in
     ;;
 
   "waveprep")
-    export wtime_waveprep_gdas="00:10:00"
+    export wtime_waveprep="00:10:00"
     export npe_waveprep_gdas=5
     export npe_waveprep_gfs=65
     export nth_waveprep_gdas=1

From 916693486908640224010ae1a412d0dc8a72db62 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 13:56:04 +0000
Subject: [PATCH 22/46] Declare runtime job resource variables based on RUN

---
 parm/config/gfs/config.analcalc  |  4 --
 parm/config/gfs/config.resources | 74 ++++++++++++++++++++++++++++++++
 2 files changed, 74 insertions(+), 4 deletions(-)

diff --git a/parm/config/gfs/config.analcalc b/parm/config/gfs/config.analcalc
index 9405114ecc..c02aafc2c3 100644
--- a/parm/config/gfs/config.analcalc
+++ b/parm/config/gfs/config.analcalc
@@ -8,8 +8,4 @@ echo "BEGIN: config.analcalc"
 # Get task specific resources
 . $EXPDIR/config.resources analcalc
 
-if [[ "$CDUMP" == "gfs" ]]; then
-   export nth_echgres=$nth_echgres_gfs
-fi
-
 echo "END: config.analcalc"
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 893f6ec644..9848d5ef80 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -126,6 +126,15 @@ case ${step} in
     export NTASKS_gfs=${npe_waveprep_gfs}
     export memory_waveprep_gdas="100GB"
     export memory_waveprep_gfs="150GB"
+
+    var_npe_node="npe_node_waveprep_${RUN}"; var_nth="nth_waveprep_${RUN}"
+    var_npe="npe_waveprep_${RUN}"; var_NTASKS="ntasks_${RUN}"
+    # RUN is set to a single value at setup time, so these won't be found
+    # TODO rework setup_xml.py to initialize RUN to the applicable option
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_waveprep"="${!var_npe_node}" nth_waveprep="${!var_nth}" \
+            "npe_waveprep"="${!var_npe}" NTASKS="${!var_NTASKS}"
+    fi
     ;;
 
   "wavepostsbs")
@@ -235,6 +244,13 @@ case ${step} in
     export npe_node_atmanlvar_gfs=$(( npe_node_max / nth_atmanlvar_gfs ))
     export memory_atmanlvar="96GB"
     export is_exclusive=True
+
+    var_npe_node="npe_node_atmanlvar_${RUN}"; var_nth="nth_atmanlvar_${RUN}"
+    var_npe="npe_atmanlvar_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_atmanlvar"="${!var_npe_node}" nth_atmanlvar="${!var_nth}" \
+            "npe_atmanlvar"="${!var_npe}"
+    fi
     ;;
 
   "atmanlfv3inc")
@@ -250,6 +266,13 @@ case ${step} in
     export npe_node_atmanlfv3inc_gfs=$(( npe_node_max / nth_atmanlfv3inc_gfs ))
     export memory_atmanlfv3inc="96GB"
     export is_exclusive=True
+
+    var_npe_node="npe_node_atmanlfv3inc_${RUN}"; var_nth="nth_atmanlfv3inc_${RUN}"
+    var_npe="npe_atmanlfv3inc_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" nth_atmanlfv3inc="${!var_nth}" \
+            "npe_atmanlfv3inc"="${!var_npe}"
+    fi
     ;;
 
   "atmanlfinal")
@@ -358,6 +381,13 @@ case ${step} in
     export npe_node_aeroanlrun_gdas=$(( npe_node_max / nth_aeroanlrun_gdas ))
     export npe_node_aeroanlrun_gfs=$(( npe_node_max / nth_aeroanlrun_gfs ))
     export is_exclusive=True
+
+    var_npe_node="npe_node_aeroanlrun_${RUN}"; var_nth="nth_aeroanlrun_${RUN}"
+    var_npe="npe_aeroanlrun_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_aeroanlrun"="${!var_npe_node}" nth_aeroanlrun="${!var_nth}" \
+            "npe_aeroanlrun"="${!var_npe}"
+    fi
     ;;
 
   "aeroanlfinal")
@@ -553,6 +583,11 @@ case ${step} in
     export nth_cycle=${nth_anal}
     export npe_node_cycle=$(( npe_node_max / nth_cycle ))
     export is_exclusive=True
+
+    var_npe="npe_anal_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_anal"="${!var_npe}"
+    fi
     ;;
 
   "analcalc")
@@ -569,6 +604,11 @@ case ${step} in
     if [[ "${CASE}" == "C384" || "${CASE}" == "C768" ]]; then
        memory_analcalc="80GB"
     fi
+
+    var_nth="nth_echgres_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x nth_echgres="${!var_nth}"
+    fi
     ;;
 
   "analdiag")
@@ -743,6 +783,13 @@ case ${step} in
         ;;
     esac
 
+    var_npe_node="npe_node_${step}_${RUN}"; var_nth="nth_${step}_${RUN}"
+    var_npe="npe_${step}_${RUN}"
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_${step}"="${!var_npe_node}" nth_${step}="${!var_nth}" \
+            "npe_${step}"="${!var_npe}"
+    fi
+
     unset _CDUMP _CDUMP_LIST
     unset NTASKS_TOT
     ;;
@@ -934,6 +981,15 @@ case ${step} in
     export npe_node_atmensanlletkf_enkfgfs=$(( npe_node_max / nth_atmensanlletkf_enkfgfs ))
     export memory_atmensanlletkf="96GB"
     export is_exclusive=True
+
+    var_npe_node="npe_node_atmensanlletkf_${RUN}"; var_nth="nth_atmensanlletkf_${RUN}"
+    var_npe="npe_atmensanlletkf_${RUN}"
+    # RUN is set to a single value at setup time, so these won't be found
+    # TODO rework setup_xml.py to initialize RUN to the applicable option
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
+                 "npe_atmensanlletkf"="${!var_npe}"
+    fi
     ;;
 
   "atmensanlfv3inc")
@@ -949,6 +1005,15 @@ case ${step} in
     export npe_node_atmensanlfv3inc_enkfgfs=$(( npe_node_max / nth_atmensanlfv3inc_enkfgfs ))
     export memory_atmensanlfv3inc="96GB"
     export is_exclusive=True
+
+    var_npe_node="npe_node_atmensanlletkf_${RUN}"; var_nth="nth_atmensanlletkf_${RUN}"
+    var_npe="npe_atmensanlletkf_${RUN}"
+    # RUN is set to a single value at setup time, so these won't be found
+    # TODO rework setup_xml.py to initialize RUN to the applicable option
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
+                 "npe_atmensanlletkf"="${!var_npe}"
+    fi
     ;;
 
   "atmensanlfinal")
@@ -1090,6 +1155,15 @@ case ${step} in
     export nth_gempak=1
     export memory_gempak_gdas="4GB"
     export memory_gempak_gfs="2GB"
+
+    var_npe_node="npe_node_gempak_${RUN}"; var_nth="nth_gempak_${RUN}"
+    var_npe="npe_gempak_${RUN}"
+    # RUN is set to a single value at setup time, so these won't be found
+    # TODO rework setup_xml.py to initialize RUN to the applicable option
+    if [[ -z "${!var_npe_node+0}" ]]; then
+      declare -x "npe_node_gempak"="${!var_npe_node}" nth_gempak="${!var_nth}" \
+                 "npe_gempak"="${!var_npe}"
+    fi
     ;;
 
   "mos_stn_prep")

From 10a75b9d5776f2aa22aa9ff5c185ebabd4c38862 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 13:56:32 +0000
Subject: [PATCH 23/46] Clean up env files

---
 env/AWSPW.env     |  5 -----
 env/CONTAINER.env |  5 -----
 env/GAEA.env      |  2 --
 env/HERA.env      |  8 --------
 env/HERCULES.env  |  3 ---
 env/JET.env       |  5 -----
 env/ORION.env     |  5 -----
 env/S4.env        |  5 -----
 env/WCOSS2.env    | 12 ------------
 9 files changed, 50 deletions(-)

diff --git a/env/AWSPW.env b/env/AWSPW.env
index e9bcd46e2d..a4f598d3d7 100755
--- a/env/AWSPW.env
+++ b/env/AWSPW.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
diff --git a/env/CONTAINER.env b/env/CONTAINER.env
index 77768b485b..c40543794b 100755
--- a/env/CONTAINER.env
+++ b/env/CONTAINER.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
diff --git a/env/GAEA.env b/env/GAEA.env
index d7d44d7534..5509a29a3f 100755
--- a/env/GAEA.env
+++ b/env/GAEA.env
@@ -3,8 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "fcst atmos_products"
     exit 1
 
 fi
diff --git a/env/HERA.env b/env/HERA.env
index 045ea122dc..863e4e5da6 100755
--- a/env/HERA.env
+++ b/env/HERA.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
@@ -305,9 +300,6 @@ elif [[ "${step}" = "gempak" ]]; then
 
     export CFP_MP="YES"
 
-    var="npe_node_gempak_${RUN}"
-    declare "npe_node_gempak"="${!var}"
-
     nth_max=$((npe_node_max / npe_node_gempak))
 
     export NTHREADS_GEMPAK=${nth_gempak:-1}
diff --git a/env/HERCULES.env b/env/HERCULES.env
index f14129d11d..abca411023 100755
--- a/env/HERCULES.env
+++ b/env/HERCULES.env
@@ -3,9 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "fcst post"
-    echo "Note: Hercules is only set up to run in forecast-only mode"
     exit 1
 
 fi
diff --git a/env/JET.env b/env/JET.env
index 1da4e107d2..1928e4182c 100755
--- a/env/JET.env
+++ b/env/JET.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
diff --git a/env/ORION.env b/env/ORION.env
index ea38ffdf0e..6831e9270b 100755
--- a/env/ORION.env
+++ b/env/ORION.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
diff --git a/env/S4.env b/env/S4.env
index 430442fe42..e8ff2414fd 100755
--- a/env/S4.env
+++ b/env/S4.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
diff --git a/env/WCOSS2.env b/env/WCOSS2.env
index 4388765521..3d0beab5c9 100755
--- a/env/WCOSS2.env
+++ b/env/WCOSS2.env
@@ -3,11 +3,6 @@
 if [[ $# -ne 1 ]]; then
 
     echo "Must specify an input argument to set runtime environment variables!"
-    echo "argument can be any one of the following:"
-    echo "atmanlvar atmanlfv3inc atmensanlletkf atmensanlfv3inc aeroanlrun snowanl"
-    echo "anal sfcanl fcst post metp"
-    echo "eobs eupd ecen esfc efcs epos"
-    echo "postsnd awips gempak"
     exit 1
 
 fi
@@ -38,10 +33,6 @@ elif [[ "${step}" = "prep_emissions" ]]; then
 elif [[ "${step}" = "waveinit" ]] || [[ "${step}" = "waveprep" ]] || [[ "${step}" = "wavepostsbs" ]] || [[ "${step}" = "wavepostbndpnt" ]] || [[ "${step}" = "wavepostbndpntbll"  ]] || [[ "${step}" = "wavepostpnt" ]]; then
 
     export USE_CFP="YES"
-    if [[ "${step}" = "waveprep" ]]; then
-        ntasks_var="NTASKS_${RUN}"
-        export NTASKS="${!ntasks_var}"
-    fi
     export wavempexec="${launcher} -np"
     export wave_mpmd=${mpmd_opt}
 
@@ -285,9 +276,6 @@ elif [[ "${step}" = "awips" ]]; then
 
 elif [[ "${step}" = "gempak" ]]; then
 
-    var="npe_node_gempak_${RUN}"
-    declare "npe_node_gempak"="${!var}"
-
     nth_max=$((npe_node_max / npe_node_gempak))
 
     export NTHREADS_GEMPAK=${nth_gempak:-1}

From d9e2f32846c6f4dcd9b4022928ff3f3205cb3640 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 13:58:29 +0000
Subject: [PATCH 24/46] Add {} around EXPDIR

---
 parm/config/gfs/config.analcalc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.analcalc b/parm/config/gfs/config.analcalc
index c02aafc2c3..d9501503f0 100644
--- a/parm/config/gfs/config.analcalc
+++ b/parm/config/gfs/config.analcalc
@@ -6,6 +6,6 @@
 echo "BEGIN: config.analcalc"
 
 # Get task specific resources
-. $EXPDIR/config.resources analcalc
+. ${EXPDIR}/config.resources analcalc
 
 echo "END: config.analcalc"

From 824c16f0ed4dabb75978f5a624d6b854e2be1783 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 14:01:25 +0000
Subject: [PATCH 25/46] Add double quotes to nth_

---
 parm/config/gfs/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 9848d5ef80..8a8ac4591e 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -786,7 +786,7 @@ case ${step} in
     var_npe_node="npe_node_${step}_${RUN}"; var_nth="nth_${step}_${RUN}"
     var_npe="npe_${step}_${RUN}"
     if [[ -z "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_${step}"="${!var_npe_node}" nth_${step}="${!var_nth}" \
+      declare -x "npe_node_${step}"="${!var_npe_node}" "nth_${step}"="${!var_nth}" \
             "npe_${step}"="${!var_npe}"
     fi
 

From b21f7363fccccb2482a98838bdd35d16350b8a0f Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 13:59:58 -0500
Subject: [PATCH 26/46] Fix resource tests

---
 parm/config/gfs/config.resources | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 8a8ac4591e..dd860200bd 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -131,7 +131,7 @@ case ${step} in
     var_npe="npe_waveprep_${RUN}"; var_NTASKS="ntasks_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_waveprep"="${!var_npe_node}" nth_waveprep="${!var_nth}" \
             "npe_waveprep"="${!var_npe}" NTASKS="${!var_NTASKS}"
     fi
@@ -247,7 +247,7 @@ case ${step} in
 
     var_npe_node="npe_node_atmanlvar_${RUN}"; var_nth="nth_atmanlvar_${RUN}"
     var_npe="npe_atmanlvar_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlvar"="${!var_npe_node}" nth_atmanlvar="${!var_nth}" \
             "npe_atmanlvar"="${!var_npe}"
     fi
@@ -269,7 +269,7 @@ case ${step} in
 
     var_npe_node="npe_node_atmanlfv3inc_${RUN}"; var_nth="nth_atmanlfv3inc_${RUN}"
     var_npe="npe_atmanlfv3inc_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" nth_atmanlfv3inc="${!var_nth}" \
             "npe_atmanlfv3inc"="${!var_npe}"
     fi
@@ -384,7 +384,7 @@ case ${step} in
 
     var_npe_node="npe_node_aeroanlrun_${RUN}"; var_nth="nth_aeroanlrun_${RUN}"
     var_npe="npe_aeroanlrun_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_aeroanlrun"="${!var_npe_node}" nth_aeroanlrun="${!var_nth}" \
             "npe_aeroanlrun"="${!var_npe}"
     fi
@@ -585,7 +585,7 @@ case ${step} in
     export is_exclusive=True
 
     var_npe="npe_anal_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe+0}" ]]; then
       declare -x "npe_anal"="${!var_npe}"
     fi
     ;;
@@ -606,7 +606,7 @@ case ${step} in
     fi
 
     var_nth="nth_echgres_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_nth+0}" ]]; then
       declare -x nth_echgres="${!var_nth}"
     fi
     ;;
@@ -785,7 +785,7 @@ case ${step} in
 
     var_npe_node="npe_node_${step}_${RUN}"; var_nth="nth_${step}_${RUN}"
     var_npe="npe_${step}_${RUN}"
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_${step}"="${!var_npe_node}" "nth_${step}"="${!var_nth}" \
             "npe_${step}"="${!var_npe}"
     fi
@@ -986,7 +986,7 @@ case ${step} in
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
@@ -1010,7 +1010,7 @@ case ${step} in
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
@@ -1160,7 +1160,7 @@ case ${step} in
     var_npe="npe_gempak_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ -z "${!var_npe_node+0}" ]]; then
+    if [[ ! -z "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_gempak"="${!var_npe_node}" nth_gempak="${!var_nth}" \
                  "npe_gempak"="${!var_npe}"
     fi

From 24946a6004beb02b75bc62d4fec79a52b68243a6 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 13 Jun 2024 19:17:35 +0000
Subject: [PATCH 27/46] Use -n instead of ! -z

---
 parm/config/gfs/config.resources | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index dd860200bd..f7f4dd4f4f 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -131,7 +131,7 @@ case ${step} in
     var_npe="npe_waveprep_${RUN}"; var_NTASKS="ntasks_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_waveprep"="${!var_npe_node}" nth_waveprep="${!var_nth}" \
             "npe_waveprep"="${!var_npe}" NTASKS="${!var_NTASKS}"
     fi
@@ -247,7 +247,7 @@ case ${step} in
 
     var_npe_node="npe_node_atmanlvar_${RUN}"; var_nth="nth_atmanlvar_${RUN}"
     var_npe="npe_atmanlvar_${RUN}"
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlvar"="${!var_npe_node}" nth_atmanlvar="${!var_nth}" \
             "npe_atmanlvar"="${!var_npe}"
     fi
@@ -269,7 +269,7 @@ case ${step} in
 
     var_npe_node="npe_node_atmanlfv3inc_${RUN}"; var_nth="nth_atmanlfv3inc_${RUN}"
     var_npe="npe_atmanlfv3inc_${RUN}"
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" nth_atmanlfv3inc="${!var_nth}" \
             "npe_atmanlfv3inc"="${!var_npe}"
     fi
@@ -384,7 +384,7 @@ case ${step} in
 
     var_npe_node="npe_node_aeroanlrun_${RUN}"; var_nth="nth_aeroanlrun_${RUN}"
     var_npe="npe_aeroanlrun_${RUN}"
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_aeroanlrun"="${!var_npe_node}" nth_aeroanlrun="${!var_nth}" \
             "npe_aeroanlrun"="${!var_npe}"
     fi
@@ -585,7 +585,7 @@ case ${step} in
     export is_exclusive=True
 
     var_npe="npe_anal_${RUN}"
-    if [[ ! -z "${!var_npe+0}" ]]; then
+    if [[ -n "${!var_npe+0}" ]]; then
       declare -x "npe_anal"="${!var_npe}"
     fi
     ;;
@@ -606,7 +606,7 @@ case ${step} in
     fi
 
     var_nth="nth_echgres_${RUN}"
-    if [[ ! -z "${!var_nth+0}" ]]; then
+    if [[ -n "${!var_nth+0}" ]]; then
       declare -x nth_echgres="${!var_nth}"
     fi
     ;;
@@ -785,7 +785,7 @@ case ${step} in
 
     var_npe_node="npe_node_${step}_${RUN}"; var_nth="nth_${step}_${RUN}"
     var_npe="npe_${step}_${RUN}"
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_${step}"="${!var_npe_node}" "nth_${step}"="${!var_nth}" \
             "npe_${step}"="${!var_npe}"
     fi
@@ -986,7 +986,7 @@ case ${step} in
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
@@ -1010,7 +1010,7 @@ case ${step} in
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
@@ -1160,7 +1160,7 @@ case ${step} in
     var_npe="npe_gempak_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
-    if [[ ! -z "${!var_npe_node+0}" ]]; then
+    if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_gempak"="${!var_npe_node}" nth_gempak="${!var_nth}" \
                  "npe_gempak"="${!var_npe}"
     fi

From 803add5a0c1da1d33017f068e1f8fcbed196c36b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 14 Jun 2024 07:17:18 -0500
Subject: [PATCH 28/46] Declare one  variable per line

---
 parm/config/gfs/config.resources | 27 ++++++++++++++++++---------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index f7f4dd4f4f..f7f9b40416 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -127,8 +127,10 @@ case ${step} in
     export memory_waveprep_gdas="100GB"
     export memory_waveprep_gfs="150GB"
 
-    var_npe_node="npe_node_waveprep_${RUN}"; var_nth="nth_waveprep_${RUN}"
-    var_npe="npe_waveprep_${RUN}"; var_NTASKS="ntasks_${RUN}"
+    var_npe_node="npe_node_waveprep_${RUN}"
+    var_nth="nth_waveprep_${RUN}"
+    var_npe="npe_waveprep_${RUN}"
+    var_NTASKS="ntasks_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
@@ -245,7 +247,8 @@ case ${step} in
     export memory_atmanlvar="96GB"
     export is_exclusive=True
 
-    var_npe_node="npe_node_atmanlvar_${RUN}"; var_nth="nth_atmanlvar_${RUN}"
+    var_npe_node="npe_node_atmanlvar_${RUN}"
+    var_nth="nth_atmanlvar_${RUN}"
     var_npe="npe_atmanlvar_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlvar"="${!var_npe_node}" nth_atmanlvar="${!var_nth}" \
@@ -267,7 +270,8 @@ case ${step} in
     export memory_atmanlfv3inc="96GB"
     export is_exclusive=True
 
-    var_npe_node="npe_node_atmanlfv3inc_${RUN}"; var_nth="nth_atmanlfv3inc_${RUN}"
+    var_npe_node="npe_node_atmanlfv3inc_${RUN}"
+    var_nth="nth_atmanlfv3inc_${RUN}"
     var_npe="npe_atmanlfv3inc_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" nth_atmanlfv3inc="${!var_nth}" \
@@ -382,7 +386,8 @@ case ${step} in
     export npe_node_aeroanlrun_gfs=$(( npe_node_max / nth_aeroanlrun_gfs ))
     export is_exclusive=True
 
-    var_npe_node="npe_node_aeroanlrun_${RUN}"; var_nth="nth_aeroanlrun_${RUN}"
+    var_npe_node="npe_node_aeroanlrun_${RUN}"
+    var_nth="nth_aeroanlrun_${RUN}"
     var_npe="npe_aeroanlrun_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_aeroanlrun"="${!var_npe_node}" nth_aeroanlrun="${!var_nth}" \
@@ -783,7 +788,8 @@ case ${step} in
         ;;
     esac
 
-    var_npe_node="npe_node_${step}_${RUN}"; var_nth="nth_${step}_${RUN}"
+    var_npe_node="npe_node_${step}_${RUN}"
+    var_nth="nth_${step}_${RUN}"
     var_npe="npe_${step}_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_${step}"="${!var_npe_node}" "nth_${step}"="${!var_nth}" \
@@ -982,7 +988,8 @@ case ${step} in
     export memory_atmensanlletkf="96GB"
     export is_exclusive=True
 
-    var_npe_node="npe_node_atmensanlletkf_${RUN}"; var_nth="nth_atmensanlletkf_${RUN}"
+    var_npe_node="npe_node_atmensanlletkf_${RUN}"
+    var_nth="nth_atmensanlletkf_${RUN}"
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
@@ -1006,7 +1013,8 @@ case ${step} in
     export memory_atmensanlfv3inc="96GB"
     export is_exclusive=True
 
-    var_npe_node="npe_node_atmensanlletkf_${RUN}"; var_nth="nth_atmensanlletkf_${RUN}"
+    var_npe_node="npe_node_atmensanlletkf_${RUN}"
+    var_nth="nth_atmensanlletkf_${RUN}"
     var_npe="npe_atmensanlletkf_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
@@ -1156,7 +1164,8 @@ case ${step} in
     export memory_gempak_gdas="4GB"
     export memory_gempak_gfs="2GB"
 
-    var_npe_node="npe_node_gempak_${RUN}"; var_nth="nth_gempak_${RUN}"
+    var_npe_node="npe_node_gempak_${RUN}"
+    var_nth="nth_gempak_${RUN}"
     var_npe="npe_gempak_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option

From dae27f0e8a11843dfa000b9ed3d1f5282a993d2d Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 14 Jun 2024 07:28:26 -0500
Subject: [PATCH 29/46] Use the environment bash for new config files

---
 parm/config/gfs/config.resources.GAEA     | 2 +-
 parm/config/gfs/config.resources.HERA     | 2 +-
 parm/config/gfs/config.resources.HERCULES | 2 +-
 parm/config/gfs/config.resources.JET      | 2 +-
 parm/config/gfs/config.resources.S4       | 2 +-
 parm/config/gfs/config.resources.WCOSS2   | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/parm/config/gfs/config.resources.GAEA b/parm/config/gfs/config.resources.GAEA
index 7d4cdcf241..64990b299f 100644
--- a/parm/config/gfs/config.resources.GAEA
+++ b/parm/config/gfs/config.resources.GAEA
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # Gaea-specific job resources
 
diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA
index c844f15503..db65166722 100644
--- a/parm/config/gfs/config.resources.HERA
+++ b/parm/config/gfs/config.resources.HERA
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # Hera-specific job resources
 
diff --git a/parm/config/gfs/config.resources.HERCULES b/parm/config/gfs/config.resources.HERCULES
index 401f5db3b0..7d2ca646e6 100644
--- a/parm/config/gfs/config.resources.HERCULES
+++ b/parm/config/gfs/config.resources.HERCULES
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # Hercules-specific job resources
 
diff --git a/parm/config/gfs/config.resources.JET b/parm/config/gfs/config.resources.JET
index c392d105ec..6dc11440f9 100644
--- a/parm/config/gfs/config.resources.JET
+++ b/parm/config/gfs/config.resources.JET
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # Jet-specific job resources
 
diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4
index a119bf627b..9c6f83a3f7 100644
--- a/parm/config/gfs/config.resources.S4
+++ b/parm/config/gfs/config.resources.S4
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # S4-specific job resources
 
diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index 0e827f4ad3..5f069e6f2e 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -1,4 +1,4 @@
-#!/usr/bin/bash
+#! /usr/bin/env bash
 
 # WCOSS2-specific job resources
 

From 32048242eb4f7094766e0a1f059bf7978f18b958 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 14 Jun 2024 07:40:32 -0500
Subject: [PATCH 30/46] Declare one variable per line; unset dynamic variables

---
 parm/config/gfs/config.resources | 43 +++++++++++++++++++++-----------
 1 file changed, 29 insertions(+), 14 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index f7f9b40416..f54deff9f8 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -134,8 +134,10 @@ case ${step} in
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_waveprep"="${!var_npe_node}" nth_waveprep="${!var_nth}" \
-            "npe_waveprep"="${!var_npe}" NTASKS="${!var_NTASKS}"
+      declare -x "npe_node_waveprep"="${!var_npe_node}" \
+                 "nth_waveprep"="${!var_nth}" \
+                 "npe_waveprep"="${!var_npe}" \
+                 "NTASKS"="${!var_NTASKS}"
     fi
     ;;
 
@@ -251,8 +253,9 @@ case ${step} in
     var_nth="nth_atmanlvar_${RUN}"
     var_npe="npe_atmanlvar_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_atmanlvar"="${!var_npe_node}" nth_atmanlvar="${!var_nth}" \
-            "npe_atmanlvar"="${!var_npe}"
+      declare -x "npe_node_atmanlvar"="${!var_npe_node}" \
+                 "nth_atmanlvar"="${!var_nth}" \
+                 "npe_atmanlvar"="${!var_npe}"
     fi
     ;;
 
@@ -274,8 +277,9 @@ case ${step} in
     var_nth="nth_atmanlfv3inc_${RUN}"
     var_npe="npe_atmanlfv3inc_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" nth_atmanlfv3inc="${!var_nth}" \
-            "npe_atmanlfv3inc"="${!var_npe}"
+      declare -x "npe_node_atmanlfv3inc"="${!var_npe_node}" \
+                 "nth_atmanlfv3inc"="${!var_nth}" \
+                 "npe_atmanlfv3inc"="${!var_npe}"
     fi
     ;;
 
@@ -390,8 +394,9 @@ case ${step} in
     var_nth="nth_aeroanlrun_${RUN}"
     var_npe="npe_aeroanlrun_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_aeroanlrun"="${!var_npe_node}" nth_aeroanlrun="${!var_nth}" \
-            "npe_aeroanlrun"="${!var_npe}"
+      declare -x "npe_node_aeroanlrun"="${!var_npe_node}" \
+                 "nth_aeroanlrun"="${!var_nth}" \
+                 "npe_aeroanlrun"="${!var_npe}"
     fi
     ;;
 
@@ -612,7 +617,7 @@ case ${step} in
 
     var_nth="nth_echgres_${RUN}"
     if [[ -n "${!var_nth+0}" ]]; then
-      declare -x nth_echgres="${!var_nth}"
+      declare -x "nth_echgres"="${!var_nth}"
     fi
     ;;
 
@@ -792,8 +797,9 @@ case ${step} in
     var_nth="nth_${step}_${RUN}"
     var_npe="npe_${step}_${RUN}"
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_${step}"="${!var_npe_node}" "nth_${step}"="${!var_nth}" \
-            "npe_${step}"="${!var_npe}"
+      declare -x "npe_node_${step}"="${!var_npe_node}" \
+                 "nth_${step}"="${!var_nth}" \
+                 "npe_${step}"="${!var_npe}"
     fi
 
     unset _CDUMP _CDUMP_LIST
@@ -994,7 +1000,8 @@ case ${step} in
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
+      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" \
+                 "nth_atmensanlletkf"="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
     ;;
@@ -1019,7 +1026,8 @@ case ${step} in
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" nth_atmensanlletkf="${!var_nth}" \
+      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" \
+                 "nth_atmensanlletkf"="${!var_nth}" \
                  "npe_atmensanlletkf"="${!var_npe}"
     fi
     ;;
@@ -1170,7 +1178,8 @@ case ${step} in
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_gempak"="${!var_npe_node}" nth_gempak="${!var_nth}" \
+      declare -x "npe_node_gempak"="${!var_npe_node}" \
+                 "nth_gempak"="${!var_nth}" \
                  "npe_gempak"="${!var_npe}"
     fi
     ;;
@@ -1329,6 +1338,12 @@ case ${step} in
 
 esac
 
+# Unset dynamic variable names
+unset var_NTASKS \
+      var_npe \
+      var_npe_node \
+      var_nth
+
 if [[ -f "${EXPDIR}/config.resources.${machine}" ]]; then
    source "${EXPDIR}/config.resources.${machine}"
 fi

From 7c4e0958bbc8df7feef08bef13ada3bc20a39f98 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Fri, 14 Jun 2024 07:41:59 -0500
Subject: [PATCH 31/46] Add comment on sourcing machine resources file

---
 parm/config/gfs/config.resources | 1 +
 1 file changed, 1 insertion(+)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index f54deff9f8..4619c38c3b 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -1344,6 +1344,7 @@ unset var_NTASKS \
       var_npe_node \
       var_nth
 
+# Get machine-specific resources, overriding/extending the above assignments
 if [[ -f "${EXPDIR}/config.resources.${machine}" ]]; then
    source "${EXPDIR}/config.resources.${machine}"
 fi

From 0286dd0f9c32c16b3dc710735f260e78fd69bbaf Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 17 Jun 2024 12:21:20 -0500
Subject: [PATCH 32/46] Rename mem/node variable, move to config.resources.

---
 parm/config/gfs/config.resources     | 57 +++++++++++++++++++++++-----
 parm/config/gfs/config.resources.JET | 14 +------
 2 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 1be71dd8c7..9e6657854a 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -35,16 +35,44 @@ step=$1
 echo "BEGIN: config.resources"
 
 case ${machine} in
-  "WCOSS2")   npe_node_max=128;;
-  "HERA")     npe_node_max=40;;
-  "GAEA")     npe_node_max=128;;
-  "ORION")    npe_node_max=40;;
-  "HERCULES") npe_node_max=80;;
+  "WCOSS2")
+              npe_node_max=128
+              mem_node_max="512GB"
+    ;;
+  "HERA")
+              npe_node_max=40
+              mem_node_max="96GB"
+    ;;
+  "GAEA")
+              npe_node_max=128
+              mem_node_max="251GB"
+    ;;
+  "ORION")
+              npe_node_max=40
+              mem_node_max="192GB"
+    ;;
+  "HERCULES")
+              npe_node_max=80
+              mem_node_max="512GB"
+    ;;
   "JET")
     case ${PARTITION_BATCH} in
-      "xjet")          npe_node_max=24;;
-      "vjet" | "sjet") npe_node_max=16;;
-      "kjet")          npe_node_max=40;;
+      "xjet")
+              npe_node_max=24
+              mem_node_max="61GB"
+        ;;
+      "vjet")
+              npe_node_max=16
+              mem_node_max="61GB"
+        ;;
+      "sjet")
+              npe_node_max=16
+              mem_node_max="29GB"
+        ;;
+      "kjet")
+              npe_node_max=40
+              mem_node_max="93GB"
+        ;;
       *)
         echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
         exit 3
@@ -52,8 +80,13 @@ case ${machine} in
     ;;
   "S4")
     case ${PARTITION_BATCH} in
-      "s4")  npe_node_max=32;;
-      "ivy") npe_node_max=20;;
+      "s4")   npe_node_max=32
+              mem_node_max="168GB"
+        ;;
+      "ivy")
+              npe_node_max=20
+              mem_node_max="128GB"
+        ;;
       *)
         echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"
         exit 3
@@ -62,9 +95,13 @@ case ${machine} in
   "AWSPW")
     export PARTITION_BATCH="compute"
     npe_node_max=40
+    # TODO Supply a max mem/node value for AWS
+    mem_node_max=""
     ;;
   "CONTAINER")
     npe_node_max=1
+    # TODO Supply a max mem/node value for a container
+    mem_node_max=""
     ;;
   *)
     echo "FATAL ERROR: Unknown machine encountered by ${BASH_SOURCE[0]}"
diff --git a/parm/config/gfs/config.resources.JET b/parm/config/gfs/config.resources.JET
index 6dc11440f9..0f6113605e 100644
--- a/parm/config/gfs/config.resources.JET
+++ b/parm/config/gfs/config.resources.JET
@@ -2,16 +2,6 @@
 
 # Jet-specific job resources
 
-case ${PARTITION_BATCH} in
-  "xjet") node_mem="61GB";;
-  "vjet") node_mem="61GB";;
-  "sjet") node_mem="29GB";;
-  "kjet") node_mem="93GB";;
-  *)
-    echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for Jet"
-    exit 3
-esac
-
 case ${step} in
   "anal")
     if [[ "${CASE}" == "C384" ]]; then
@@ -50,11 +40,11 @@ case ${step} in
     ;;
 
   "upp")
-    export memory_upp="${node_mem}"
+    export memory_upp="${mem_node_max}"
     ;;
 
   "esfc")
-    export memory_esfc="${node_mem}"
+    export memory_esfc="${mem_node_max}"
     ;;
 
   *)

From 238ae98b5566b48b870ff85cc9da050725907d3b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 17 Jun 2024 12:25:22 -0500
Subject: [PATCH 33/46] Ignore unused shellcheck warnings

---
 parm/config/gfs/config.resources | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 9e6657854a..d931a8279a 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -37,40 +37,49 @@ echo "BEGIN: config.resources"
 case ${machine} in
   "WCOSS2")
               npe_node_max=128
+              # shellcheck disable=SC2034
               mem_node_max="512GB"
     ;;
   "HERA")
               npe_node_max=40
+              # shellcheck disable=SC2034
               mem_node_max="96GB"
     ;;
   "GAEA")
               npe_node_max=128
+              # shellcheck disable=SC2034
               mem_node_max="251GB"
     ;;
   "ORION")
               npe_node_max=40
+              # shellcheck disable=SC2034
               mem_node_max="192GB"
     ;;
   "HERCULES")
               npe_node_max=80
+              # shellcheck disable=SC2034
               mem_node_max="512GB"
     ;;
   "JET")
     case ${PARTITION_BATCH} in
       "xjet")
               npe_node_max=24
+              # shellcheck disable=SC2034
               mem_node_max="61GB"
         ;;
       "vjet")
               npe_node_max=16
+              # shellcheck disable=SC2034
               mem_node_max="61GB"
         ;;
       "sjet")
               npe_node_max=16
+              # shellcheck disable=SC2034
               mem_node_max="29GB"
         ;;
       "kjet")
               npe_node_max=40
+              # shellcheck disable=SC2034
               mem_node_max="93GB"
         ;;
       *)
@@ -81,10 +90,12 @@ case ${machine} in
   "S4")
     case ${PARTITION_BATCH} in
       "s4")   npe_node_max=32
+              # shellcheck disable=SC2034
               mem_node_max="168GB"
         ;;
       "ivy")
               npe_node_max=20
+              # shellcheck disable=SC2034
               mem_node_max="128GB"
         ;;
       *)
@@ -96,11 +107,13 @@ case ${machine} in
     export PARTITION_BATCH="compute"
     npe_node_max=40
     # TODO Supply a max mem/node value for AWS
+    # shellcheck disable=SC2034
     mem_node_max=""
     ;;
   "CONTAINER")
     npe_node_max=1
     # TODO Supply a max mem/node value for a container
+    # shellcheck disable=SC2034
     mem_node_max=""
     ;;
   *)

From e5680e8de8a6628bc59610568ca2e117dfad81f7 Mon Sep 17 00:00:00 2001
From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
Date: Tue, 18 Jun 2024 11:43:36 +0000
Subject: [PATCH 34/46] Apply suggestions from Walter's code review

Co-authored-by: Walter Kolczynski - NOAA <Walter.Kolczynski@noaa.gov>
---
 parm/config/gfs/config.anal      | 2 +-
 parm/config/gfs/config.resources | 3 +--
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/parm/config/gfs/config.anal b/parm/config/gfs/config.anal
index 3679807889..2c55d85ff4 100644
--- a/parm/config/gfs/config.anal
+++ b/parm/config/gfs/config.anal
@@ -19,7 +19,7 @@ if [[ "${CDUMP}" = "gfs" ]] ; then
   export DIAG_TARBALL="YES"
 fi
 
-npe_var="npe_anal_${RUN//enkf}"
+npe_var="npe_anal_${RUN/enkf}"
 export npe_gsi="${!npe_var}"
 
 # Set parameters specific to L127
diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index d931a8279a..4de325acbb 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -659,8 +659,7 @@ case ${step} in
     ;;
 
   "analcalc")
-    export wtime_analcalc="00:10:00"
-    [[ ${CASE} == "C384" || ${CASE} == "C768" ]] && export wtime_analcalc="00:15:00"
+    export wtime_analcalc="00:15:00"
     export npe_analcalc=127
     export ntasks="${npe_analcalc}"
     export nth_analcalc=1

From ac0778d27c1026fd81dda9c6a7150541312eebcb Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Tue, 18 Jun 2024 13:17:26 +0000
Subject: [PATCH 35/46] Increase memory requirement on S4 for ediag

---
 parm/config/gfs/config.resources.S4 | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4
index 9c6f83a3f7..4c4b1e171c 100644
--- a/parm/config/gfs/config.resources.S4
+++ b/parm/config/gfs/config.resources.S4
@@ -49,6 +49,10 @@ case ${step} in
     export npe_node_eupd=$(( npe_node_max / nth_eupd ))
     ;;
 
+  "ediag")
+     export memory_ediag="${mem_node_max}"
+  ;;
+
   *)
     ;;
 esac

From c5659e33b6552976de0c3bf8c92797ef713b8cb8 Mon Sep 17 00:00:00 2001
From: Innocent Souopgui <Innocent.Souopgui@noaa.gov>
Date: Wed, 19 Jun 2024 13:33:21 +0000
Subject: [PATCH 36/46] Adjusted available memory on kjet to what is accepted
 by the job scheduler, 93GB is rejected

---
 parm/config/gfs/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index d931a8279a..e160f38334 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -80,7 +80,7 @@ case ${machine} in
       "kjet")
               npe_node_max=40
               # shellcheck disable=SC2034
-              mem_node_max="93GB"
+              mem_node_max="88GB"
         ;;
       *)
         echo "FATAL ERROR: Unknown partition ${PARTITION_BATCH} specified for ${machine}"

From 1caf612e52c40f77c19cba7117ad714c37190da8 Mon Sep 17 00:00:00 2001
From: Innocent Souopgui <innocent.souopgui@noaa.gov>
Date: Thu, 20 Jun 2024 15:50:02 +0000
Subject: [PATCH 37/46] Prevent OpenMP over-allocation of resources on S4

---
 env/S4.env | 1 +
 1 file changed, 1 insertion(+)

diff --git a/env/S4.env b/env/S4.env
index 11ecbdbb24..190c7295f4 100755
--- a/env/S4.env
+++ b/env/S4.env
@@ -191,6 +191,7 @@ elif [[ "${step}" = "upp" ]]; then
 
     export NTHREADS_UPP=${nth_upp:-1}
     [[ ${NTHREADS_UPP} -gt ${nth_max} ]] && export NTHREADS_UPP=${nth_max}
+    export OMP_NUM_THREADS="${NTHREADS_UPP}"
     export APRUN_UPP="${launcher} -n ${npe_upp}"
 
 elif [[ "${step}" = "atmos_products" ]]; then

From 7cc2753a703cfaee67897bf2223453552e9f30ff Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 20 Jun 2024 17:12:41 +0000
Subject: [PATCH 38/46] Add array length check for older bash versions.

---
 ush/forecast_postdet.sh | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 2cc34eaacd..006062a5a6 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -268,20 +268,23 @@ FV3_out() {
     fi
   fi
 
-  # Get list of FV3 restart files
-  local file_list fv3_file
-  file_list=$(FV3_restarts)
+  ### Check that there are restart files to copy
+  if [[ ${#restart_dates} -gt 0 ]]; then
+    # Get list of FV3 restart files
+    local file_list fv3_file
+    file_list=$(FV3_restarts)
 
-  # Copy restarts for the dates collected above to COM
-  for restart_date in "${restart_dates[@]}"; do
-    echo "Copying FV3 restarts for 'RUN=${RUN}' at ${restart_date}"
-    for fv3_file in ${file_list}; do
-      ${NCP} "${DATArestart}/FV3_RESTART/${restart_date}.${fv3_file}" \
-             "${COMOUT_ATMOS_RESTART}/${restart_date}.${fv3_file}"
+    # Copy restarts for the dates collected above to COM
+    for restart_date in "${restart_dates[@]}"; do
+      echo "Copying FV3 restarts for 'RUN=${RUN}' at ${restart_date}"
+      for fv3_file in ${file_list}; do
+        ${NCP} "${DATArestart}/FV3_RESTART/${restart_date}.${fv3_file}" \
+               "${COMOUT_ATMOS_RESTART}/${restart_date}.${fv3_file}"
+      done
     done
-  done
 
-  echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied"
+    echo "SUB ${FUNCNAME[0]}: Output data for FV3 copied"
+  fi
 }
 
 # Disable variable not used warnings

From 895d018773877cea4561e46d6db0d4f90bdd9bb3 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 20 Jun 2024 17:53:51 +0000
Subject: [PATCH 39/46] Recalculate analysis npe_node when redefining threads
 for machines

---
 parm/config/gfs/config.resources.HERA   | 1 +
 parm/config/gfs/config.resources.JET    | 1 +
 parm/config/gfs/config.resources.S4     | 1 +
 parm/config/gfs/config.resources.WCOSS2 | 1 +
 4 files changed, 4 insertions(+)

diff --git a/parm/config/gfs/config.resources.HERA b/parm/config/gfs/config.resources.HERA
index db65166722..cfd614961d 100644
--- a/parm/config/gfs/config.resources.HERA
+++ b/parm/config/gfs/config.resources.HERA
@@ -8,6 +8,7 @@ case ${step} in
       export npe_anal_gdas=270
       export npe_anal_gfs=270
       export nth_anal=8
+      export npe_node_anal=$(( npe_node_max / nth_anal ))
     fi
     ;;
 
diff --git a/parm/config/gfs/config.resources.JET b/parm/config/gfs/config.resources.JET
index 0f6113605e..de2ec6547a 100644
--- a/parm/config/gfs/config.resources.JET
+++ b/parm/config/gfs/config.resources.JET
@@ -8,6 +8,7 @@ case ${step} in
       export npe_anal_gdas=270
       export npe_anal_gfs=270
       export nth_anal=8
+      export npe_node_anal=$(( npe_node_max / nth_anal ))
     fi
     ;;
 
diff --git a/parm/config/gfs/config.resources.S4 b/parm/config/gfs/config.resources.S4
index 4c4b1e171c..3f6654f8d6 100644
--- a/parm/config/gfs/config.resources.S4
+++ b/parm/config/gfs/config.resources.S4
@@ -32,6 +32,7 @@ case ${step} in
       *)
         ;;
     esac
+    export npe_node_anal=$(( npe_node_max / nth_anal ))
     ;;
 
   "eobs")
diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index 5f069e6f2e..fdce6fefc1 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -10,6 +10,7 @@ case ${step} in
 
   "anal")
     export nth_anal=8
+    export npe_node_anal=$(( npe_node_max / nth_anal ))
     ;;
 
   "upp")

From b901c47a1028f1cc5116d33519d0d406a8740d9b Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Thu, 20 Jun 2024 12:56:53 -0500
Subject: [PATCH 40/46] Correct rocoto version in hercules setup module.

---
 modulefiles/module_gwsetup.hercules.lua | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/modulefiles/module_gwsetup.hercules.lua b/modulefiles/module_gwsetup.hercules.lua
index 795b295b30..9d845fb71d 100644
--- a/modulefiles/module_gwsetup.hercules.lua
+++ b/modulefiles/module_gwsetup.hercules.lua
@@ -3,7 +3,7 @@ Load environment to run GFS workflow ci scripts on Hercules
 ]])
 
 load(pathJoin("contrib","0.1"))
-load(pathJoin("rocoto","1.3.5"))
+load(pathJoin("rocoto","1.3.7"))
 
 prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/hercules/spack-stack-1.6.0/envs/gsi-addon-env/install/modulefiles/Core")
 

From fdd0850d9e8716142ab474978ebce8fcbb8dbefd Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Thu, 20 Jun 2024 18:13:54 +0000
Subject: [PATCH 41/46] Only run analyses @ 8 threads at C768 on WCOSS2.

---
 parm/config/gfs/config.resources.WCOSS2 | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index fdce6fefc1..375be37855 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -9,8 +9,13 @@ case ${step} in
     ;;
 
   "anal")
-    export nth_anal=8
-    export npe_node_anal=$(( npe_node_max / nth_anal ))
+    if [[ "${CASE}" == "C768" ]]; then
+        export nth_anal=8
+        # Make npe a multiple of 16
+        export npe_anal_gdas=784
+        export npe_anal_gfs=832
+        export npe_node_anal=$(( npe_node_max / nth_anal ))
+    fi
     ;;
 
   "upp")

From 859da0a2906792fc8c6adede53f31f30c600bf1b Mon Sep 17 00:00:00 2001
From: Innocent Souopgui <innocent.souopgui@noaa.gov>
Date: Fri, 21 Jun 2024 11:54:39 +0000
Subject: [PATCH 42/46] fix typo preventing a S4 to check array size

---
 ush/forecast_postdet.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ush/forecast_postdet.sh b/ush/forecast_postdet.sh
index 006062a5a6..993331d70b 100755
--- a/ush/forecast_postdet.sh
+++ b/ush/forecast_postdet.sh
@@ -269,7 +269,7 @@ FV3_out() {
   fi
 
   ### Check that there are restart files to copy
-  if [[ ${#restart_dates} -gt 0 ]]; then
+  if [[ ${#restart_dates[@]} -gt 0 ]]; then
     # Get list of FV3 restart files
     local file_list fv3_file
     file_list=$(FV3_restarts)

From f34bdd6ac94aae8c3b107536ce8f82d3ecbb8470 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Fri, 21 Jun 2024 12:55:01 +0000
Subject: [PATCH 43/46] High res wallclock/memory adjustments

---
 parm/config/gfs/config.resources        | 13 +++++--------
 parm/config/gfs/config.resources.WCOSS2 |  6 ------
 2 files changed, 5 insertions(+), 14 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 30bfe65dea..e7cfed0135 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -121,6 +121,7 @@ case ${machine} in
     exit 2
     ;;
 esac
+
 export npe_node_max
 
 case ${step} in
@@ -669,7 +670,7 @@ case ${step} in
     export is_exclusive=True
     export memory_analcalc="48GB"
     if [[ "${CASE}" == "C384" || "${CASE}" == "C768" ]]; then
-       memory_analcalc="80GB"
+       export memory_analcalc="${mem_node_max}"
     fi
 
     var_nth="nth_echgres_${RUN}"
@@ -841,7 +842,7 @@ case ${step} in
         ;;
       "C768" | "C1152")
         # Not valid resolutions for ensembles
-        declare -x "wtime_${step}_gdas"="00:30:00"
+        declare -x "wtime_${step}_gdas"="00:40:00"
         declare -x "wtime_${step}_gfs"="06:00:00"
         ;;
       *)
@@ -876,13 +877,9 @@ case ${step} in
       "C48" | "C96")
         export npe_upp=${CASE:1}
       ;;
-      "C192" | "C384")
-        export npe_upp=120
-        export memory_upp="96GB"
-      ;;
-      "C768")
+      "C192" | "C384" | "C768" )
         export npe_upp=120
-        export memory_upp="96GB"
+        export memory_upp="${mem_node_max}"
       ;;
       *)
         echo "FATAL ERROR: Resources not defined for job ${step} at resolution ${CASE}"
diff --git a/parm/config/gfs/config.resources.WCOSS2 b/parm/config/gfs/config.resources.WCOSS2
index 375be37855..7e4314a0e9 100644
--- a/parm/config/gfs/config.resources.WCOSS2
+++ b/parm/config/gfs/config.resources.WCOSS2
@@ -18,12 +18,6 @@ case ${step} in
     fi
     ;;
 
-  "upp")
-    if [[ "${CASE}" == "C768" || "${CASE}" == "C384" ]]; then
-        export memory_upp="480GB"
-    fi
-    ;;
-
   "fit2obs")
     export npe_node_fit2obs=3
     ;;

From 4b21dbcb942c178c96af399240a4a00d9dbd8468 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Fri, 21 Jun 2024 14:36:14 +0000
Subject: [PATCH 44/46] Correct max mem requestable on WCOSS2

---
 parm/config/gfs/config.resources | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index e7cfed0135..abce463eb2 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -38,7 +38,7 @@ case ${machine} in
   "WCOSS2")
               npe_node_max=128
               # shellcheck disable=SC2034
-              mem_node_max="512GB"
+              mem_node_max="500GB"
     ;;
   "HERA")
               npe_node_max=40

From 184d2359e1960dba2c9836a48b083620a9fbab08 Mon Sep 17 00:00:00 2001
From: DavidHuber <david.huber@noaa.gov>
Date: Sun, 23 Jun 2024 11:05:52 +0000
Subject: [PATCH 45/46] Fix atmensanlfv3inc resource declarations

---
 parm/config/gfs/config.resources | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 24f5fad769..48b6c31e87 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -1074,15 +1074,15 @@ case ${step} in
     export memory_atmensanlfv3inc="96GB"
     export is_exclusive=True
 
-    var_npe_node="npe_node_atmensanlletkf_${RUN}"
-    var_nth="nth_atmensanlletkf_${RUN}"
-    var_npe="npe_atmensanlletkf_${RUN}"
+    var_npe_node="npe_node_atmensanlfv3inc_${RUN}"
+    var_nth="nth_atmensanlfv3inc_${RUN}"
+    var_npe="npe_atmensanlfv3inc_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
-      declare -x "npe_node_atmensanlletkf"="${!var_npe_node}" \
-                 "nth_atmensanlletkf"="${!var_nth}" \
-                 "npe_atmensanlletkf"="${!var_npe}"
+      declare -x "npe_node_atmensanlfv3inc"="${!var_npe_node}" \
+                 "nth_atmensanlfv3inc"="${!var_nth}" \
+                 "npe_atmensanlfv3inc"="${!var_npe}"
     fi
     ;;
 

From 99171434513986dceea9af2426da5e4d9a48f891 Mon Sep 17 00:00:00 2001
From: David Huber <david.huber@noaa.gov>
Date: Mon, 24 Jun 2024 11:59:52 +0000
Subject: [PATCH 46/46] Remove reference to non-existent RUN-specific gempak
 variables

---
 parm/config/gfs/config.resources | 2 --
 1 file changed, 2 deletions(-)

diff --git a/parm/config/gfs/config.resources b/parm/config/gfs/config.resources
index 48b6c31e87..2e910d4eb4 100644
--- a/parm/config/gfs/config.resources
+++ b/parm/config/gfs/config.resources
@@ -1227,13 +1227,11 @@ case ${step} in
     export memory_gempak_gfs="2GB"
 
     var_npe_node="npe_node_gempak_${RUN}"
-    var_nth="nth_gempak_${RUN}"
     var_npe="npe_gempak_${RUN}"
     # RUN is set to a single value at setup time, so these won't be found
     # TODO rework setup_xml.py to initialize RUN to the applicable option
     if [[ -n "${!var_npe_node+0}" ]]; then
       declare -x "npe_node_gempak"="${!var_npe_node}" \
-                 "nth_gempak"="${!var_nth}" \
                  "npe_gempak"="${!var_npe}"
     fi
     ;;