From 73621e9b20b20f85e8b21e4c1e09ed1d80ecd46f Mon Sep 17 00:00:00 2001
From: David Huber <69919478+DavidHuber-NOAA@users.noreply.github.com>
Date: Sun, 3 Dec 2023 11:19:19 -0500
Subject: [PATCH] Upgrade to spack-stack/1.5.1 and Intel 2022 on non-production
 machines (#2084)

Update module files to spack-stack/1.5.1 and compilers to Intel 2022+ on all non-production machines.
---
 jobs/rocoto/anal.sh                      | 44 +++---------------------
 jobs/rocoto/efcs.sh                      | 11 ++++--
 jobs/rocoto/eobs.sh                      | 44 +++---------------------
 jobs/rocoto/eupd.sh                      | 44 +++---------------------
 jobs/rocoto/fcst.sh                      | 14 +++++---
 jobs/rocoto/post.sh                      | 39 ++++++++++-----------
 jobs/rocoto/prep.sh                      |  1 -
 modulefiles/module_base.hera.lua         | 41 ++++++++++++----------
 modulefiles/module_base.jet.lua          | 31 ++++++++++-------
 modulefiles/module_base.orion.lua        | 42 +++++++++++-----------
 modulefiles/module_base.s4.lua           | 36 +++++++++++--------
 modulefiles/module_gwci.hera.lua         | 12 +++----
 modulefiles/module_gwci.orion.lua        | 21 ++++-------
 modulefiles/module_gwsetup.hera.lua      | 15 +++++---
 modulefiles/module_gwsetup.jet.lua       | 22 +++++-------
 modulefiles/module_gwsetup.orion.lua     | 17 +++++----
 modulefiles/module_gwsetup.s4.lua        | 12 ++++++-
 modulefiles/module_gwsetup.wcoss2.lua    |  2 +-
 parm/config/gefs/config.base.emc.dyn     |  4 +--
 parm/config/gfs/config.base.emc.dyn      |  4 +--
 scripts/exgfs_atmos_awips_20km_1p0deg.sh |  2 +-
 scripts/exgfs_atmos_grib_awips.sh        |  2 +-
 scripts/exglobal_diag.sh                 |  2 +-
 sorc/checkout.sh                         | 12 +++----
 sorc/link_workflow.sh                    | 11 ++++--
 ush/getncdimlen                          |  2 +-
 ush/gsi_utils.py                         |  2 +-
 versions/build.hera.ver                  | 36 ++-----------------
 versions/build.jet.ver                   | 36 ++-----------------
 versions/build.orion.ver                 | 36 ++-----------------
 versions/build.s4.ver                    | 35 ++-----------------
 versions/build.spack.ver                 | 38 ++++++++++++++++++++
 versions/run.hera.ver                    | 29 ++++------------
 versions/run.jet.ver                     | 31 +++--------------
 versions/run.orion.ver                   | 28 ++-------------
 versions/run.s4.ver                      | 27 ++-------------
 versions/run.spack.ver                   | 27 +++++++++++++++
 37 files changed, 301 insertions(+), 511 deletions(-)
 create mode 100644 versions/build.spack.ver
 create mode 100644 versions/run.spack.ver

diff --git a/jobs/rocoto/anal.sh b/jobs/rocoto/anal.sh
index cb07749b9e..00f03e3832 100755
--- a/jobs/rocoto/anal.sh
+++ b/jobs/rocoto/anal.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="anal"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS
+"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh
index fa9dc0c269..c5667cb970 100755
--- a/jobs/rocoto/efcs.sh
+++ b/jobs/rocoto/efcs.sh
@@ -4,8 +4,13 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-. ${HOMEgfs}/ush/load_ufswm_modules.sh
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+   . ${HOMEgfs}/ush/load_ufswm_modules.sh
+else
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+fi
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 
@@ -14,7 +19,7 @@ export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_FCST
+"${HOMEgfs}/jobs/JGDAS_ENKF_FCST"
 status=$?
 
 exit ${status}
diff --git a/jobs/rocoto/eobs.sh b/jobs/rocoto/eobs.sh
index 4f813a8d84..1139c3e623 100755
--- a/jobs/rocoto/eobs.sh
+++ b/jobs/rocoto/eobs.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="eobs"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS
+"${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/eupd.sh b/jobs/rocoto/eupd.sh
index 9ddde92d36..9134ca5f14 100755
--- a/jobs/rocoto/eupd.sh
+++ b/jobs/rocoto/eupd.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="eupd"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE
+"${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh
index a5be293f9e..9138b4eb43 100755
--- a/jobs/rocoto/fcst.sh
+++ b/jobs/rocoto/fcst.sh
@@ -3,18 +3,22 @@
 source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
-# Source UFS Weather Model workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-. ${HOMEgfs}/ush/load_ufswm_modules.sh
+# Source FV3GFS workflow modules
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+   . ${HOMEgfs}/ush/load_ufswm_modules.sh
+else
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+fi
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 
 export job="fcst"
 export jobid="${job}.$$"
 
-###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGLOBAL_FORECAST
+"${HOMEgfs}/jobs/JGLOBAL_FORECAST"
 status=$?
 
 exit ${status}
diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh
index 5dc66926c9..92abef8767 100755
--- a/jobs/rocoto/post.sh
+++ b/jobs/rocoto/post.sh
@@ -7,29 +7,26 @@ source "${HOMEgfs}/ush/preamble.sh"
 ## FHRLST : forecast hourlist to be post-process (e.g. anl, f000, f000_f001_f002, ...)
 ###############################################################
 
-# Source FV3GFS workflow modules
-# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-# status=$?
-# [[ ${status} -ne 0 ]] && exit ${status}
-# Temporarily load modules from UPP
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
 source "${HOMEgfs}/ush/detect_machine.sh"
-source "${HOMEgfs}/ush/module-setup.sh"
-module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
-module load "${MACHINE_ID}"
-module load prod_util
 if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    module load cray-pals
-    module load cfp
-    module load libjpeg
-    module load grib_util
+   # Temporarily load modules from UPP
+   source "${HOMEgfs}/ush/module-setup.sh"
+   module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
+   module load "${MACHINE_ID}"
+   module load prod_util
+   module load cray-pals
+   module load cfp
+   module load libjpeg
+   module load grib_util
+   module load wgrib2
+   export WGRIB2=wgrib2
+   # End hack
 else
-    # shellcheck disable=SC2154
-    export UTILROOT="${prod_util_ROOT}"
-    module load grib-util
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+   status=$?
+   [[ ${status} -ne 0 ]] && exit ${status}
 fi
-module load wgrib2
-export WGRIB2=wgrib2
-# End hack
 
 export job="post"
 export jobid="${job}.$$"
@@ -39,9 +36,9 @@ fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g')
 #---------------------------------------------------------------
 for fhr in ${fhrlst}; do
     export post_times=${fhr}
-    ${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST"
     status=$?
-    [[ ${status} -ne 0 ]] && exit ${status}
+    [[ ${status} -ne 0 ]] && exit "${status}"
 done
 
 exit 0
diff --git a/jobs/rocoto/prep.sh b/jobs/rocoto/prep.sh
index f90be861d9..dfb541abb6 100755
--- a/jobs/rocoto/prep.sh
+++ b/jobs/rocoto/prep.sh
@@ -94,7 +94,6 @@ if [[ ${MAKE_PREPBUFR} = "YES" ]]; then
     fi
 
     export job="j${CDUMP}_prep_${cyc}"
-    export DATAROOT="${RUNDIR}/${CDATE}/${CDUMP}/prepbufr"
     export COMIN=${COM_OBS}
     export COMOUT=${COM_OBS}
     RUN="gdas" YMD=${PDY} HH=${cyc} generate_com -rx COMINgdas:COM_ATMOS_HISTORY_TMPL
diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua
index 7464ac7bec..cf77df520a 100644
--- a/modulefiles/module_base.hera.lua
+++ b/modulefiles/module_base.hera.lua
@@ -2,46 +2,49 @@ help([[
 Load environment to run GFS on Hera
 ]])
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
+spack_env=(os.getenv("spack_env") or "None")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
 
-load(pathJoin("hpc", os.getenv("hpc_ver")))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
 
 load(pathJoin("hpss", os.getenv("hpss_ver")))
 load(pathJoin("gempak", os.getenv("gempak_ver")))
 load(pathJoin("ncl", os.getenv("ncl_ver")))
 load(pathJoin("jasper", os.getenv("jasper_ver")))
-load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("libpng", os.getenv("libpng_ver")))
 load(pathJoin("cdo", os.getenv("cdo_ver")))
 load(pathJoin("R", os.getenv("R_ver")))
 
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
-load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver")))
+load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver")))
 
 load(pathJoin("nco", os.getenv("nco_ver")))
 load(pathJoin("prod_util", os.getenv("prod_util_ver")))
-load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("grib-util", os.getenv("grib_util_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
 load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver")))
+load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver")))
+load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver")))
+
+load(pathJoin("met", os.getenv("met_ver")))
+load(pathJoin("metplus", os.getenv("metplus_ver")))
+
 setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-intel", "18.0.5.274"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
-load(pathJoin("met", "9.1"))
-load(pathJoin("metplus", "3.1"))
-
 whatis("Description: GFS run environment")
diff --git a/modulefiles/module_base.jet.lua b/modulefiles/module_base.jet.lua
index c8cb9a9abf..7a0faaaeb6 100644
--- a/modulefiles/module_base.jet.lua
+++ b/modulefiles/module_base.jet.lua
@@ -2,13 +2,15 @@ help([[
 Load environment to run GFS on Jet
 ]])
 
-prepend_path("MODULEPATH", "/lfs4/HFIP/hfv3gfs/role.epic/hpc-stack/libs/intel-18.0.5.274/modulefiles/stack")
+spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
+spack_env=(os.getenv("spack_env") or "None")
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
 
-load(pathJoin("hpc", os.getenv("hpc_ver")))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
 
-load("hpss")
+load(pathJoin("hpss", os.getenv("hpss_ver")))
 load(pathJoin("gempak", os.getenv("gempak_ver")))
 load(pathJoin("ncl", os.getenv("ncl_ver")))
 load(pathJoin("jasper", os.getenv("jasper_ver")))
@@ -17,24 +19,29 @@ load(pathJoin("cdo", os.getenv("cdo_ver")))
 load(pathJoin("R", os.getenv("R_ver")))
 
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
-load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver")))
+load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver")))
 
 load(pathJoin("nco", os.getenv("nco_ver")))
 load(pathJoin("prod_util", os.getenv("prod_util_ver")))
-load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("grib-util", os.getenv("grib_util_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
 load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver")))
+load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver")))
+load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver")))
 
-prepend_path("MODULEPATH", "/contrib/anaconda/modulefiles")
-load(pathJoin("anaconda", os.getenv("anaconda_ver")))
+setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
---prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/lfs4/HFIP/hfv3gfs/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 whatis("Description: GFS run environment")
diff --git a/modulefiles/module_base.orion.lua b/modulefiles/module_base.orion.lua
index bb00594cce..3464accc89 100644
--- a/modulefiles/module_base.orion.lua
+++ b/modulefiles/module_base.orion.lua
@@ -2,45 +2,47 @@ help([[
 Load environment to run GFS on Orion
 ]])
 
-prepend_path("MODULEPATH", "/apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/stack")
+spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
+spack_env=(os.getenv("spack_env") or "None")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
 
-load(pathJoin("hpc", os.getenv("hpc_ver")))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
 
 load(pathJoin("gempak", os.getenv("gempak_ver")))
 load(pathJoin("ncl", os.getenv("ncl_ver")))
 load(pathJoin("jasper", os.getenv("jasper_ver")))
-load(pathJoin("zlib", os.getenv("zlib_ver")))
-load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("libpng", os.getenv("libpng_ver")))
 load(pathJoin("cdo", os.getenv("cdo_ver")))
 
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
-load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver")))
+load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver")))
 
 load(pathJoin("nco", os.getenv("nco_ver")))
 load(pathJoin("prod_util", os.getenv("prod_util_ver")))
-load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("grib-util", os.getenv("grib_util_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
 load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver")))
+load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver")))
+load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver")))
+
+load(pathJoin("met", os.getenv("met_ver")))
+load(pathJoin("metplus", os.getenv("metplus_ver")))
+
 setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
---prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/work/noaa/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-intel", "2018.4"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
-load(pathJoin("met", "9.1"))
-load(pathJoin("metplus", "3.1"))
-
 whatis("Description: GFS run environment")
diff --git a/modulefiles/module_base.s4.lua b/modulefiles/module_base.s4.lua
index 21b574e763..51eeee0f10 100644
--- a/modulefiles/module_base.s4.lua
+++ b/modulefiles/module_base.s4.lua
@@ -2,37 +2,43 @@ help([[
 Load environment to run GFS on S4
 ]])
 
-load("license_intel")
-prepend_path("MODULEPATH", "/data/prod/hpc-stack/modulefiles/stack")
+spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
+spack_env=(os.getenv("spack_env") or "None")
+prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
 
-load(pathJoin("hpc", os.getenv("hpc_ver")))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
 
-load(pathJoin("miniconda", os.getenv("miniconda_ver")))
 load(pathJoin("ncl", os.getenv("ncl_ver")))
-load(pathJoin("cdo", os.getenv("cdo_ver")))
 load(pathJoin("jasper", os.getenv("jasper_ver")))
-load(pathJoin("zlib", os.getenv("zlib_ver")))
-load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("libpng", os.getenv("libpng_ver")))
+load(pathJoin("cdo", os.getenv("cdo_ver")))
 
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
-load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver")))
+load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver")))
 
 load(pathJoin("nco", os.getenv("nco_ver")))
 load(pathJoin("prod_util", os.getenv("prod_util_ver")))
-load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("grib-util", os.getenv("grib_util_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
 load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver")))
+load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver")))
+load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver")))
+
 setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
---prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
-prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg/modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/data/prod/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
 whatis("Description: GFS run environment")
diff --git a/modulefiles/module_gwci.hera.lua b/modulefiles/module_gwci.hera.lua
index f4b62a5fd2..1aecddf549 100644
--- a/modulefiles/module_gwci.hera.lua
+++ b/modulefiles/module_gwci.hera.lua
@@ -2,14 +2,14 @@ help([[
 Load environment to run GFS workflow setup scripts on Hera
 ]])
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
-load(pathJoin("hpc", "1.1.0"))
-load(pathJoin("hpc-intel", "18.0.5.274"))
-load(pathJoin("hpc-impi", "2018.0.4"))
+load(pathJoin("stack-intel", os.getenv("2021.5.0")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1")))
 
-load(pathJoin("netcdf","4.7.4"))
-load(pathJoin("nccmp","1.8.7.0"))
+load(pathJoin("netcdf-c", os.getenv("4.9.2")))
+load(pathJoin("netcdf-fortran", os.getenv("4.6.0")))
+load(pathJoin("nccmp","1.9.0.1"))
 load(pathJoin("wgrib2", "2.0.8"))
 
 whatis("Description: GFS run setup CI environment")
diff --git a/modulefiles/module_gwci.orion.lua b/modulefiles/module_gwci.orion.lua
index 971ba01c65..18851ba7d4 100644
--- a/modulefiles/module_gwci.orion.lua
+++ b/modulefiles/module_gwci.orion.lua
@@ -2,21 +2,14 @@ help([[
 Load environment to run GFS workflow ci scripts on Orion
 ]])
 
-prepend_path("MODULEPATH", "/apps/contrib/NCEP/hpc-stack/libs/hpc-stack/modulefiles/stack")
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
-load(pathJoin("hpc", "1.1.0"))
-load(pathJoin("hpc-intel", "2018.4"))
-load(pathJoin("hpc-impi", "2018.4"))
-load(pathJoin("netcdf","4.7.4"))
-load(pathJoin("netcdf","4.7.4-parallel"))
-load(pathJoin("nccmp"," 1.8.7.0"))
-load(pathJoin("contrib","0.1"))
-load(pathJoin("wgrib2","3.0.2"))
+load(pathJoin("stack-intel", os.getenv("2022.0.2")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("2021.5.1")))
 
-prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-intel", "2018.4"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
+load(pathJoin("netcdf-c", os.getenv("4.9.2")))
+load(pathJoin("netcdf-fortran", os.getenv("4.6.0")))
+load(pathJoin("nccmp","1.9.0.1"))
+load(pathJoin("wgrib2", "2.0.8"))
 
 whatis("Description: GFS run ci top-level sripts environment")
diff --git a/modulefiles/module_gwsetup.hera.lua b/modulefiles/module_gwsetup.hera.lua
index a07b32b6a6..c86cac7b02 100644
--- a/modulefiles/module_gwsetup.hera.lua
+++ b/modulefiles/module_gwsetup.hera.lua
@@ -4,10 +4,15 @@ Load environment to run GFS workflow setup scripts on Hera
 
 load(pathJoin("rocoto"))
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+local python_ver=os.getenv("python_ver") or "3.10.8"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
 
 whatis("Description: GFS run setup environment")
diff --git a/modulefiles/module_gwsetup.jet.lua b/modulefiles/module_gwsetup.jet.lua
index 10fc39ce8a..d08389c711 100644
--- a/modulefiles/module_gwsetup.jet.lua
+++ b/modulefiles/module_gwsetup.jet.lua
@@ -4,19 +4,15 @@ Load environment to run GFS workflow setup scripts on Jet
 
 load(pathJoin("rocoto", "1.3.3"))
 
-if (mode() == "unload") then
-    -- `execute` delays commands until last, but we need conda deactivated
-    --   before unloading miniconda. `print` (bizarrely) still executes the
-    --   command, but does it immediately. The semicolon is necessary 
-    --   because otherwise other commands get tacked onto the same line.
-    print("conda deactivate;")
-end
+prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/mnt/lfs4/HFIP/hfv3gfs/role.epic/miniconda3/modulefiles")
-load(pathJoin("miniconda3", "4.12.0"))
-if (mode() == "load") then
-    execute{cmd="conda activate ufswm", modeA={"load"}}
-end
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+local python_ver=os.getenv("python_ver") or "3.10.8"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
 
 whatis("Description: GFS run setup environment")
diff --git a/modulefiles/module_gwsetup.orion.lua b/modulefiles/module_gwsetup.orion.lua
index 477ceb6251..93a59c8e50 100644
--- a/modulefiles/module_gwsetup.orion.lua
+++ b/modulefiles/module_gwsetup.orion.lua
@@ -7,10 +7,15 @@ load(pathJoin("contrib","0.1"))
 load(pathJoin("rocoto","1.3.3"))
 load(pathJoin("git","2.28.0"))
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/work2/noaa/global/wkolczyn/save/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
+prepend_path("MODULEPATH", "/work/noaa/epic/role-epic/spack-stack/orion/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
 
-whatis("Description: GFS run ci top-level sripts environment")
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2022.0.2"
+local python_ver=os.getenv("python_ver") or "3.10.8"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
+
+whatis("Description: GFS run setup environment")
diff --git a/modulefiles/module_gwsetup.s4.lua b/modulefiles/module_gwsetup.s4.lua
index 8324a148d8..291c654bb3 100644
--- a/modulefiles/module_gwsetup.s4.lua
+++ b/modulefiles/module_gwsetup.s4.lua
@@ -2,8 +2,18 @@ help([[
 Load environment to run GFS workflow setup scripts on S4
 ]])
 
-load(pathJoin("miniconda", "3.8-s4"))
 load(pathJoin("rocoto","1.3.5"))
 load(pathJoin("git","2.30.0"))
 
+prepend_path("MODULEPATH", "/data/prod/jedi/spack-stack/spack-stack-1.5.1/envs/gsi-addon/install/modulefiles/Core")
+
+local stack_intel_ver=os.getenv("stack_intel_ver") or "2021.5.0"
+local python_ver=os.getenv("python_ver") or "3.10.8"
+
+load(pathJoin("stack-intel", stack_intel_ver))
+load(pathJoin("python", python_ver))
+load("py-jinja2")
+load("py-pyyaml")
+load("py-numpy")
+
 whatis("Description: GFS run setup environment")
diff --git a/modulefiles/module_gwsetup.wcoss2.lua b/modulefiles/module_gwsetup.wcoss2.lua
index 31f644090b..d4e64548b0 100644
--- a/modulefiles/module_gwsetup.wcoss2.lua
+++ b/modulefiles/module_gwsetup.wcoss2.lua
@@ -7,4 +7,4 @@ load(pathJoin("git","2.29.0"))
 prepend_path("MODULEPATH", "/apps/ops/test/nco/modulefiles/core")
 load(pathJoin("rocoto","1.3.5"))
 
-whatis("Description: GFS run ci top-level sripts environment")
+whatis("Description: GFS run setup environment")
diff --git a/parm/config/gefs/config.base.emc.dyn b/parm/config/gefs/config.base.emc.dyn
index 16775cfc6c..238eb16991 100644
--- a/parm/config/gefs/config.base.emc.dyn
+++ b/parm/config/gefs/config.base.emc.dyn
@@ -82,7 +82,7 @@ export VERBOSE="YES"
 export KEEPDATA="NO"
 export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
 export CHGRP_CMD="@CHGRP_CMD@"
-export NCDUMP="${NETCDF:-}/bin/ncdump"
+export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump"
 export NCLEN="${HOMEgfs}/ush/getncdimlen"
 
 # Machine environment, jobs, and other utility scripts
@@ -125,7 +125,7 @@ export SENDECF=${SENDECF:-"NO"}
 export SENDSDM=${SENDSDM:-"NO"}
 export SENDDBN_NTC=${SENDDBN_NTC:-"NO"}
 export SENDDBN=${SENDDBN:-"NO"}
-export DBNROOT=${DBNROOT:-${UTILROOT}/fakedbn}
+export DBNROOT=${DBNROOT:-${UTILROOT:-}/fakedbn}
 
 # APP settings
 export APP=@APP@
diff --git a/parm/config/gfs/config.base.emc.dyn b/parm/config/gfs/config.base.emc.dyn
index 35495ed9df..22b72fe873 100644
--- a/parm/config/gfs/config.base.emc.dyn
+++ b/parm/config/gfs/config.base.emc.dyn
@@ -92,7 +92,7 @@ export VERBOSE="YES"
 export KEEPDATA="NO"
 export CHGRP_RSTPROD="@CHGRP_RSTPROD@"
 export CHGRP_CMD="@CHGRP_CMD@"
-export NCDUMP="${NETCDF:-}/bin/ncdump"
+export NCDUMP="${NETCDF:-${netcdf_c_ROOT:-}}/bin/ncdump"
 export NCLEN="${HOMEgfs}/ush/getncdimlen"
 
 # Machine environment, jobs, and other utility scripts
@@ -375,7 +375,7 @@ export netcdf_diag=".true."
 export binary_diag=".false."
 
 # Verification options
-export DO_METP="YES"         # Run METPLUS jobs - set METPLUS settings in config.metp
+export DO_METP="NO"          # Run METPLUS jobs - set METPLUS settings in config.metp; not supported with spack-stack
 export DO_FIT2OBS="YES"      # Run fit to observations package
 
 # Archiving options
diff --git a/scripts/exgfs_atmos_awips_20km_1p0deg.sh b/scripts/exgfs_atmos_awips_20km_1p0deg.sh
index caf7f9cc58..7546f3cabe 100755
--- a/scripts/exgfs_atmos_awips_20km_1p0deg.sh
+++ b/scripts/exgfs_atmos_awips_20km_1p0deg.sh
@@ -174,7 +174,7 @@ for GRID in conus ak prico pac 003; do
    export pgm; prep_step
    startmsg
 
-   if [[ ${GRID} = "003" && $(( fcsthrs % 6 )) == 0 ]]; then
+   if [[ ${GRID} = "003" && $(( 10#${fcsthrs} % 6 )) == 0 ]]; then
       export FORT11="awps_file_f${fcsthrs}_${GRID}"
       export FORT31="awps_file_fi${fcsthrs}_${GRID}"
       export FORT51="grib2.awpgfs${fcsthrs}.${GRID}"
diff --git a/scripts/exgfs_atmos_grib_awips.sh b/scripts/exgfs_atmos_grib_awips.sh
index efdd6beb81..037b4ce191 100755
--- a/scripts/exgfs_atmos_grib_awips.sh
+++ b/scripts/exgfs_atmos_grib_awips.sh
@@ -122,7 +122,7 @@ if [[ "${SENDDBN}" == 'YES' || "${SENDAWIP}" == 'YES' ]] ; then
     "${DBNROOT}/bin/dbn_alert" "${DBNALERT_TYPE}" "${NET}" "${job}" \
 			       "${COM_ATMOS_WMO}/xtrn.awpgfs${fcsthrs}.${GRID}.${job_name}"
 else
-    echo "File ${output_grb}.${job_name} not posted to db_net."
+    echo "File xtrn.awpgfs${fcsthrs}.${GRID}.${job_name} not posted to db_net."
 fi
 
 if [[ -e "${pgmout}" ]] ; then
diff --git a/scripts/exglobal_diag.sh b/scripts/exglobal_diag.sh
index 487b30c712..1f641c4fd7 100755
--- a/scripts/exglobal_diag.sh
+++ b/scripts/exglobal_diag.sh
@@ -35,7 +35,7 @@ export NMV=${NMV:-"/bin/mv"}
 export NLN=${NLN:-"/bin/ln -sf"}
 export CHGRP_CMD=${CHGRP_CMD:-"chgrp ${group_name:-rstprod}"}
 export NCLEN=${NCLEN:-$HOMEgfs/ush/getncdimlen}
-export CATEXEC=${CATEXEC:-$ncdiag_ROOT/bin/ncdiag_cat_serial.x}
+export CATEXEC=${CATEXEC:-${ncdiag_ROOT:-${gsi_ncdiag_ROOT}}/bin/ncdiag_cat_serial.x}
 COMPRESS=${COMPRESS:-gzip}
 UNCOMPRESS=${UNCOMPRESS:-gunzip}
 APRUNCFP=${APRUNCFP:-""}
diff --git a/sorc/checkout.sh b/sorc/checkout.sh
index 2b54502772..fc3842d968 100755
--- a/sorc/checkout.sh
+++ b/sorc/checkout.sh
@@ -155,21 +155,21 @@ checkout "ufs_model.fd"    "https://github.com/ufs-community/ufs-weather-model"
 
 # Run all other checkouts simultaneously with just 1 core each to handle submodules.
 checkout "wxflow"          "https://github.com/NOAA-EMC/wxflow"                 "528f5ab" &
-checkout "gfs_utils.fd"    "https://github.com/NOAA-EMC/gfs-utils"              "a283262" &
-checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "72a0471" &
+checkout "gfs_utils.fd"    "https://github.com/NOAA-EMC/gfs-utils"              "427d467" &
+checkout "ufs_utils.fd"    "https://github.com/ufs-community/UFS_UTILS.git"     "892b693" &
 checkout "verif-global.fd" "https://github.com/NOAA-EMC/EMC_verif-global.git"   "c267780" &
 
 if [[ ${checkout_gsi} == "YES" ]]; then
-  checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git" "ca19008" "1" "NO" &
+  checkout "gsi_enkf.fd" "https://github.com/NOAA-EMC/GSI.git"                  "c94bc72" "1" "NO" &
 fi
 
 if [[ ${checkout_gdas} == "YES" ]]; then
-  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git" "7659c10" &
+  checkout "gdas.cd" "https://github.com/NOAA-EMC/GDASApp.git"                  "7659c10" &
 fi
 
 if [[ ${checkout_gsi} == "YES" || ${checkout_gdas} == "YES" ]]; then
-  checkout "gsi_utils.fd"    "https://github.com/NOAA-EMC/GSI-Utils.git"   "322cc7b" &
-  checkout "gsi_monitor.fd"  "https://github.com/NOAA-EMC/GSI-Monitor.git" "45783e3" &
+  checkout "gsi_utils.fd"    "https://github.com/NOAA-EMC/GSI-Utils.git"        "f371890" &
+  checkout "gsi_monitor.fd"  "https://github.com/NOAA-EMC/GSI-Monitor.git"      "ae256c0" &
 fi
 
 # Go through each PID and verify no errors were reported.
diff --git a/sorc/link_workflow.sh b/sorc/link_workflow.sh
index 6c7e9a73f1..cdb9256479 100755
--- a/sorc/link_workflow.sh
+++ b/sorc/link_workflow.sh
@@ -49,8 +49,15 @@ machine=$(echo "${MACHINE_ID}" | cut -d. -f1)
 #------------------------------
 #--Set up build.ver and run.ver
 #------------------------------
-cp "${HOMEgfs}/versions/build.${machine}.ver" "${HOMEgfs}/versions/build.ver"
-cp "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/run.ver"
+if [[ "${machine}" == "wcoss2" ]]; then
+   cp "${HOMEgfs}/versions/build.${machine}.ver" "${HOMEgfs}/versions/build.ver"
+   cp "${HOMEgfs}/versions/run.${machine}.ver" "${HOMEgfs}/versions/run.ver"
+else
+   cp "${HOMEgfs}/versions/build.spack.ver" "${HOMEgfs}/versions/build.ver"
+   cp "${HOMEgfs}/versions/run.spack.ver" "${HOMEgfs}/versions/run.ver"
+   cat "${HOMEgfs}/versions/build.${machine}.ver" >> "${HOMEgfs}/versions/build.ver"
+   cat "${HOMEgfs}/versions/run.${machine}.ver" >> "${HOMEgfs}/versions/run.ver"
+fi
 
 #------------------------------
 #--model fix fields
diff --git a/ush/getncdimlen b/ush/getncdimlen
index 5d230f6cc3..fcf231947b 100755
--- a/ush/getncdimlen
+++ b/ush/getncdimlen
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # getncdimlen
 # cory.r.martin@noaa.gov
 # 2019-10-17
diff --git a/ush/gsi_utils.py b/ush/gsi_utils.py
index 94a2ae1348..8088cb7233 100644
--- a/ush/gsi_utils.py
+++ b/ush/gsi_utils.py
@@ -78,7 +78,7 @@ def get_ncdims(ncfile):
     try:
         import netCDF4 as nc
     except ImportError as err:
-        raise ImportError(f"Unable to import netCDF4 module\n{err}")
+        raise ImportError(f"Unable to import netCDF4 module")
     ncf = nc.Dataset(ncfile)
     ncdims = {}
     for d in ncf.dimensions.keys():
diff --git a/versions/build.hera.ver b/versions/build.hera.ver
index ae31afc75f..cba2bb5a76 100644
--- a/versions/build.hera.ver
+++ b/versions/build.hera.ver
@@ -1,34 +1,2 @@
-export hpc_ver=1.1.0
-export hpc_intel_ver=18.0.5.274
-export hpc_impi_ver=2018.0.4
-
-export cmake_ver=3.20.1
-
-export gempak_ver=7.4.2
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export zlib_ver=1.2.11
-export esmf_ver=8.3.0b09
-export pio_ver=2.5.2
-export fms_ver=2022.04
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export bacio_ver=2.4.1
-export w3nco_ver=2.4.1
-export nemsio_ver=2.5.2
-export sigio_ver=2.3.2
-export w3emc_ver=2.9.2
-export bufr_ver=11.7.0
-export g2_ver=3.4.5
-export sp_ver=2.3.3
-export ip_ver=3.3.3
-export wrf_io_ver=1.2.0
-export ncio_ver=1.1.2
-export ncdiag_ver=1.0.0
-export g2tmpl_ver=1.10.2
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export upp_ver=10.0.8
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
diff --git a/versions/build.jet.ver b/versions/build.jet.ver
index 71e330ad60..cba2bb5a76 100644
--- a/versions/build.jet.ver
+++ b/versions/build.jet.ver
@@ -1,34 +1,2 @@
-export hpc_ver=1.2.0
-export hpc_intel_ver=18.0.5.274
-export hpc_impi_ver=2018.4.274
-
-export cmake_ver=3.20.1
-
-export gempak_ver=7.4.2
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export zlib_ver=1.2.11
-export esmf_ver=8.3.0b09
-export pio_ver=2.5.2
-export fms_ver=2022.04
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export bacio_ver=2.4.1
-export w3nco_ver=2.4.1
-export nemsio_ver=2.5.2
-export sigio_ver=2.3.2
-export w3emc_ver=2.9.2
-export bufr_ver=11.7.0
-export g2_ver=3.4.5
-export sp_ver=2.3.3
-export ip_ver=3.3.3
-export wrf_io_ver=1.2.0
-export ncio_ver=1.1.2
-export ncdiag_ver=1.0.0
-export g2tmpl_ver=1.10.2
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export upp_ver=10.0.8
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
diff --git a/versions/build.orion.ver b/versions/build.orion.ver
index 0084e9e506..cba2bb5a76 100644
--- a/versions/build.orion.ver
+++ b/versions/build.orion.ver
@@ -1,34 +1,2 @@
-export hpc_ver=1.1.0
-export hpc_intel_ver=2018.4
-export hpc_impi_ver=2018.4
-
-export cmake_ver=3.22.1
-
-export gempak_ver=7.5.1
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export zlib_ver=1.2.11
-export esmf_ver=8.3.0b09
-export pio_ver=2.5.2
-export fms_ver=2022.04
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export bacio_ver=2.4.1
-export w3nco_ver=2.4.1
-export nemsio_ver=2.5.2
-export sigio_ver=2.3.2
-export w3emc_ver=2.9.2
-export bufr_ver=11.7.0
-export g2_ver=3.4.5
-export sp_ver=2.3.3
-export ip_ver=3.3.3
-export wrf_io_ver=1.2.0
-export ncio_ver=1.1.2
-export ncdiag_ver=1.0.0
-export g2tmpl_ver=1.10.0
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export upp_ver=10.0.8
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
diff --git a/versions/build.s4.ver b/versions/build.s4.ver
index 7712311425..a0aed0ee24 100644
--- a/versions/build.s4.ver
+++ b/versions/build.s4.ver
@@ -1,33 +1,2 @@
-export hpc_ver=1.2.0
-export hpc_intel_ver=18.0.4
-export hpc_impi_ver=18.0.4
-
-export cmake_ver=3.20.5
-
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export zlib_ver=1.2.11
-export esmf_ver=8.3.0b09
-export pio_ver=2.5.2
-export fms_ver=2022.04
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export bacio_ver=2.4.1
-export w3nco_ver=2.4.1
-export nemsio_ver=2.5.2
-export sigio_ver=2.3.2
-export w3emc_ver=2.9.2
-export bufr_ver=11.7.0
-export g2_ver=3.4.5
-export sp_ver=2.3.3
-export ip_ver=3.3.3
-export wrf_io_ver=1.2.0
-export ncio_ver=1.1.2
-export ncdiag_ver=1.0.0
-export g2tmpl_ver=1.10.2
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export upp_ver=10.0.8
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.0
diff --git a/versions/build.spack.ver b/versions/build.spack.ver
new file mode 100644
index 0000000000..679da91116
--- /dev/null
+++ b/versions/build.spack.ver
@@ -0,0 +1,38 @@
+export spack_stack_ver=1.5.1
+export spack_env=gsi-addon
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
+export python_ver=3.10.8
+
+export cmake_ver=3.23.1
+
+export gempak_ver=7.4.2
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export zlib_ver=1.2.13
+export esmf_ver=8.5.0
+export fms_ver=2023.02.01
+export cdo_ver=2.0.5
+export nco_ver=5.0.6
+export ncl_ver=6.6.2
+
+export hdf5_ver=1.14.0
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.0
+
+export bacio_ver=2.4.1
+export nemsio_ver=2.5.4
+export sigio_ver=2.3.2
+export w3emc_ver=2.10.0
+export bufr_ver=11.7.0
+export g2_ver=3.4.5
+export sp_ver=2.3.3
+export ip_ver=4.3.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.10.2
+export crtm_ver=2.4.0
+export wgrib2_ver=2.0.8
+export grib_util_ver=1.3.0
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=5.4.1
+export py_jinja2_ver=3.1.2
diff --git a/versions/run.hera.ver b/versions/run.hera.ver
index 3ebd49907b..d82e9fba62 100644
--- a/versions/run.hera.ver
+++ b/versions/run.hera.ver
@@ -1,28 +1,11 @@
-export hpc_ver=1.1.0
-export hpc_intel_ver=18.0.5.274
-export hpc_impi_ver=2018.0.4
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
 
 export hpss_ver=hpss
-export gempak_ver=7.4.2
 export ncl_ver=6.6.2
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export cdo_ver=1.9.5
 export R_ver=3.5.0
+export gempak_ver=7.4.2
 
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export nco_ver=4.9.1
-export prod_util_ver=1.2.2
-export grib_util_ver=1.2.2
-export g2tmpl_ver=1.10.2
-export ncdiag_ver=1.0.0
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export obsproc_run_ver=1.1.2
-export prepobs_run_ver=1.0.1
-
-export ens_tracker_ver=feature-GFSv17_com_reorg
-export fit2obs_ver=1.0.0
+#For metplus jobs, not currently working with spack-stack
+#export met_ver=9.1.3
+#export metplus_ver=3.1.1
diff --git a/versions/run.jet.ver b/versions/run.jet.ver
index b397d5588b..ab350da770 100644
--- a/versions/run.jet.ver
+++ b/versions/run.jet.ver
@@ -1,30 +1,7 @@
-export hpc_ver=1.2.0
-export hpc_intel_ver=18.0.5.274
-export hpc_impi_ver=2018.4.274
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.1
 
-export hpss_ver=hpss
-export gempak_ver=7.4.2
+export hpss_ver=
 export ncl_ver=6.6.2
-export jasper_ver=2.0.25
-export libpng_ver=1.6.35
-export cdo_ver=1.9.5
 export R_ver=4.0.2
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export nco_ver=4.9.1
-export prod_util_ver=1.2.2
-export grib_util_ver=1.2.2
-export g2tmpl_ver=1.10.2
-export ncdiag_ver=1.0.0
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export anaconda_ver=5.3.1
-
-export obsproc_run_ver=1.1.2
-export prepobs_run_ver=1.0.1
-
-export ens_tracker_ver=feature-GFSv17_com_reorg
-export fit2obs_ver=1.0.0
+export gempak_ver=7.4.2
diff --git a/versions/run.orion.ver b/versions/run.orion.ver
index 1b1400384e..ca3258ba8e 100644
--- a/versions/run.orion.ver
+++ b/versions/run.orion.ver
@@ -1,27 +1,5 @@
-export hpc_ver=1.1.0
-export hpc_intel_ver=2018.4
-export hpc_impi_ver=2018.4
+export stack_intel_ver=2022.0.2
+export stack_impi_ver=2021.5.1
 
-export gempak_ver=7.5.1
 export ncl_ver=6.6.2
-export jasper_ver=2.0.25
-export zlib_ver=1.2.11
-export libpng_ver=1.6.35
-export cdo_ver=1.9.5
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export nco_ver=4.8.1
-export prod_util_ver=1.2.2
-export grib_util_ver=1.2.2
-export g2tmpl_ver=1.10.0
-export ncdiag_ver=1.0.0
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export obsproc_run_ver=1.1.2
-export prepobs_run_ver=1.0.1
-
-export ens_tracker_ver=feature-GFSv17_com_reorg
-export fit2obs_ver=1.0.0
+export gempak_ver=7.5.1
diff --git a/versions/run.s4.ver b/versions/run.s4.ver
index b37c219cd4..72206b21d6 100644
--- a/versions/run.s4.ver
+++ b/versions/run.s4.ver
@@ -1,27 +1,4 @@
-export hpc_ver=1.1.0
-export hpc_intel_ver=18.0.4
-export hpc_impi_ver=18.0.4
+export stack_intel_ver=2021.5.0
+export stack_impi_ver=2021.5.0
 
-export miniconda_ver=3.8-s4
 export ncl_ver=6.4.0-precompiled
-export cdo_ver=1.9.8
-export jasper_ver=2.0.25
-export zlib_ver=1.2.11
-export libpng_ver=1.6.35
-
-export hdf5_ver=1.10.6
-export netcdf_ver=4.7.4
-
-export nco_ver=4.9.3
-export prod_util_ver=1.2.2
-export grib_util_ver=1.2.2
-export g2tmpl_ver=1.10.2
-export ncdiag_ver=1.0.0
-export crtm_ver=2.4.0
-export wgrib2_ver=2.0.8
-
-export obsproc_run_ver=1.1.2
-export prepobs_run_ver=1.0.1
-
-export ens_tracker_ver=feature-GFSv17_com_reorg
-export fit2obs_ver=1.0.0
diff --git a/versions/run.spack.ver b/versions/run.spack.ver
new file mode 100644
index 0000000000..7045f2ed01
--- /dev/null
+++ b/versions/run.spack.ver
@@ -0,0 +1,27 @@
+export spack_stack_ver=1.5.1
+export spack_env=gsi-addon
+export python_ver=3.10.8
+
+export jasper_ver=2.0.32
+export libpng_ver=1.6.37
+export cdo_ver=2.0.5
+export nco_ver=5.0.6
+
+export hdf5_ver=1.14.0
+export netcdf_c_ver=4.9.2
+export netcdf_fortran_ver=4.6.0
+
+export bufr_ver=11.7.0
+export gsi_ncdiag_ver=1.1.2
+export g2tmpl_ver=1.10.2
+export crtm_ver=2.4.0
+export wgrib2_ver=2.0.8
+export py_netcdf4_ver=1.5.8
+export py_pyyaml_ver=5.4.1
+export py_jinja2_ver=3.1.2
+
+export obsproc_run_ver=1.1.2
+export prepobs_run_ver=1.0.1
+
+export ens_tracker_ver=feature-GFSv17_com_reorg
+export fit2obs_ver=1.0.0