Upgrade to spack-stack/1.5.1 and Intel 2022 on non-production machines (

#2084) Update module files to spack-stack/1.5.1 and compilers to Intel 2022+ on all non-production machines.
NOAA-EMC · Dec 3, 2023 · 73621e9 · 73621e9
1 parent 6d548b2
commit 73621e9
Show file tree

Hide file tree

Showing 37 changed files with 301 additions and 511 deletions.
diff --git a/jobs/rocoto/anal.sh b/jobs/rocoto/anal.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="anal"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS
+"${HOMEgfs}/jobs/JGLOBAL_ATMOS_ANALYSIS"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/efcs.sh b/jobs/rocoto/efcs.sh
@@ -4,8 +4,13 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-. ${HOMEgfs}/ush/load_ufswm_modules.sh
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+   . ${HOMEgfs}/ush/load_ufswm_modules.sh
+else
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+fi
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 
@@ -14,7 +19,7 @@ export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_FCST
+"${HOMEgfs}/jobs/JGDAS_ENKF_FCST"
 status=$?
 
 exit ${status}
diff --git a/jobs/rocoto/eobs.sh b/jobs/rocoto/eobs.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="eobs"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS
+"${HOMEgfs}/jobs/JGDAS_ENKF_SELECT_OBS"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/eupd.sh b/jobs/rocoto/eupd.sh
@@ -4,51 +4,17 @@ source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
 # Source FV3GFS workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-#status=$?
-#[[ ${status} -ne 0 ]] && exit ${status}
-
-# TODO: clean this up
-#Source appropriate modulefiles based on machine
-
-source "${HOMEgfs}/ush/detect_machine.sh"
-if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    # Source FV3GFS workflow modules
-    . "${HOMEgfs}"/ush/load_fv3gfs_modules.sh
-    status=$?
-    [[ ${status} -ne 0 ]] && exit "${status}"
-else
-    # Append compiler (only on machines that have multiple compilers)
-    COMPILER=${COMPILER:-"intel"}
-    if [[ "${MACHINE_ID}" = "hera" ]] || [[ "${MACHINE_ID}" = "cheyenne" ]]; then
-        MACHINE_ID=${MACHINE_ID}.${COMPILER}
-    fi
-
-    # Source machine specific GSI-EnKF modules
-    set +x
-    source "${HOMEgfs}/ush/module-setup.sh"
-    module use "${HOMEgfs}/sorc/gsi_enkf.fd/modulefiles"
-    module load gsi_"${MACHINE_ID}"
-
-    if [[ "${MACHINE_ID}" = "orion" ]]; then
-       module load miniconda3/4.12.0
-       set +u
-       conda activate regional_workflow_cmaq
-       set_strict
-    fi
-
-    module list
-    unset MACHINE_ID
-    set_trace
-fi
+. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+status=$?
+[[ ${status} -ne 0 ]] && exit ${status}
 
 export job="eupd"
 export jobid="${job}.$$"
 
 ###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE
+"${HOMEgfs}/jobs/JGDAS_ENKF_UPDATE"
 status=$?
 
 
-exit ${status}
+exit "${status}"
diff --git a/jobs/rocoto/fcst.sh b/jobs/rocoto/fcst.sh
@@ -3,18 +3,22 @@
 source "${HOMEgfs}/ush/preamble.sh"
 
 ###############################################################
-# Source UFS Weather Model workflow modules
-#. ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-. ${HOMEgfs}/ush/load_ufswm_modules.sh
+# Source FV3GFS workflow modules
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
+source "${HOMEgfs}/ush/detect_machine.sh"
+if [[ "${MACHINE_ID}" == "wcoss2" ]]; then
+   . ${HOMEgfs}/ush/load_ufswm_modules.sh
+else
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+fi
 status=$?
 [[ ${status} -ne 0 ]] && exit ${status}
 
 export job="fcst"
 export jobid="${job}.$$"
 
-###############################################################
 # Execute the JJOB
-${HOMEgfs}/jobs/JGLOBAL_FORECAST
+"${HOMEgfs}/jobs/JGLOBAL_FORECAST"
 status=$?
 
 exit ${status}
diff --git a/jobs/rocoto/post.sh b/jobs/rocoto/post.sh
@@ -7,29 +7,26 @@ source "${HOMEgfs}/ush/preamble.sh"
 ## FHRLST : forecast hourlist to be post-process (e.g. anl, f000, f000_f001_f002, ...)
 ###############################################################
 
-# Source FV3GFS workflow modules
-# . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
-# status=$?
-# [[ ${status} -ne 0 ]] && exit ${status}
-# Temporarily load modules from UPP
+# TODO clean this up once ncdiag/1.1.2 is installed on WCOSS2
 source "${HOMEgfs}/ush/detect_machine.sh"
-source "${HOMEgfs}/ush/module-setup.sh"
-module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
-module load "${MACHINE_ID}"
-module load prod_util
 if [[ "${MACHINE_ID}" = "wcoss2" ]]; then
-    module load cray-pals
-    module load cfp
-    module load libjpeg
-    module load grib_util
+   # Temporarily load modules from UPP
+   source "${HOMEgfs}/ush/module-setup.sh"
+   module use "${HOMEgfs}/sorc/ufs_model.fd/FV3/upp/modulefiles"
+   module load "${MACHINE_ID}"
+   module load prod_util
+   module load cray-pals
+   module load cfp
+   module load libjpeg
+   module load grib_util
+   module load wgrib2
+   export WGRIB2=wgrib2
+   # End hack
 else
-    # shellcheck disable=SC2154
-    export UTILROOT="${prod_util_ROOT}"
-    module load grib-util
+   . ${HOMEgfs}/ush/load_fv3gfs_modules.sh
+   status=$?
+   [[ ${status} -ne 0 ]] && exit ${status}
 fi
-module load wgrib2
-export WGRIB2=wgrib2
-# End hack
 
 export job="post"
 export jobid="${job}.$$"
@@ -39,9 +36,9 @@ fhrlst=$(echo ${FHRLST} | sed -e 's/_/ /g; s/f/ /g; s/,/ /g')
 #---------------------------------------------------------------
 for fhr in ${fhrlst}; do
     export post_times=${fhr}
-    ${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST
+    "${HOMEgfs}/jobs/JGLOBAL_ATMOS_POST"
     status=$?
-    [[ ${status} -ne 0 ]] && exit ${status}
+    [[ ${status} -ne 0 ]] && exit "${status}"
 done
 
 exit 0
diff --git a/jobs/rocoto/prep.sh b/jobs/rocoto/prep.sh
@@ -94,7 +94,6 @@ if [[ ${MAKE_PREPBUFR} = "YES" ]]; then
     fi
 
     export job="j${CDUMP}_prep_${cyc}"
-    export DATAROOT="${RUNDIR}/${CDATE}/${CDUMP}/prepbufr"
     export COMIN=${COM_OBS}
     export COMOUT=${COM_OBS}
     RUN="gdas" YMD=${PDY} HH=${cyc} generate_com -rx COMINgdas:COM_ATMOS_HISTORY_TMPL

diff --git a/modulefiles/module_base.hera.lua b/modulefiles/module_base.hera.lua
@@ -2,46 +2,49 @@ help([[
 Load environment to run GFS on Hera
 ]])
 
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/nwprod/hpc-stack/libs/hpc-stack/modulefiles/stack")
+spack_stack_ver=(os.getenv("spack_stack_ver") or "None")
+spack_env=(os.getenv("spack_env") or "None")
+prepend_path("MODULEPATH", "/scratch1/NCEPDEV/nems/role.epic/spack-stack/spack-stack-" .. spack_stack_ver .. "/envs/" .. spack_env .. "/install/modulefiles/Core")
 
-load(pathJoin("hpc", os.getenv("hpc_ver")))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver")))
-load(pathJoin("hpc-impi", os.getenv("hpc_impi_ver")))
+load(pathJoin("stack-intel", os.getenv("stack_intel_ver")))
+load(pathJoin("stack-intel-oneapi-mpi", os.getenv("stack_impi_ver")))
+load(pathJoin("python", os.getenv("python_ver")))
 
 load(pathJoin("hpss", os.getenv("hpss_ver")))
 load(pathJoin("gempak", os.getenv("gempak_ver")))
 load(pathJoin("ncl", os.getenv("ncl_ver")))
 load(pathJoin("jasper", os.getenv("jasper_ver")))
-load(pathJoin("png", os.getenv("libpng_ver")))
+load(pathJoin("libpng", os.getenv("libpng_ver")))
 load(pathJoin("cdo", os.getenv("cdo_ver")))
 load(pathJoin("R", os.getenv("R_ver")))
 
 load(pathJoin("hdf5", os.getenv("hdf5_ver")))
-load(pathJoin("netcdf", os.getenv("netcdf_ver")))
+load(pathJoin("netcdf-c", os.getenv("netcdf_c_ver")))
+load(pathJoin("netcdf-fortran", os.getenv("netcdf_fortran_ver")))
 
 load(pathJoin("nco", os.getenv("nco_ver")))
 load(pathJoin("prod_util", os.getenv("prod_util_ver")))
-load(pathJoin("grib_util", os.getenv("grib_util_ver")))
+load(pathJoin("grib-util", os.getenv("grib_util_ver")))
 load(pathJoin("g2tmpl", os.getenv("g2tmpl_ver")))
-load(pathJoin("ncdiag", os.getenv("ncdiag_ver")))
+load(pathJoin("gsi-ncdiag", os.getenv("gsi_ncdiag_ver")))
 load(pathJoin("crtm", os.getenv("crtm_ver")))
+load(pathJoin("bufr", os.getenv("bufr_ver")))
 load(pathJoin("wgrib2", os.getenv("wgrib2_ver")))
+load(pathJoin("py-netcdf4", os.getenv("py_netcdf4_ver")))
+load(pathJoin("py-pyyaml", os.getenv("py_pyyaml_ver")))
+load(pathJoin("py-jinja2", os.getenv("py_jinja2_ver")))
+
+load(pathJoin("met", os.getenv("met_ver")))
+load(pathJoin("metplus", os.getenv("metplus_ver")))
+
 setenv("WGRIB2","wgrib2")
+setenv("UTILROOT",(os.getenv("prod_util_ROOT") or "None"))
 
---prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. os.getenv("prepobs_run_ver"), "modulefiles"))
+--prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/v" .. (os.getenv("prepobs_run_ver") or "None"), "modulefiles"))
 prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/prepobs/feature-GFSv17_com_reorg_log_update/modulefiles"))
 load(pathJoin("prepobs", os.getenv("prepobs_run_ver")))
 
-prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. os.getenv("fit2obs_ver"), "modulefiles"))
+prepend_path("MODULEPATH", pathJoin("/scratch1/NCEPDEV/global/glopara/git/Fit2Obs/v" .. (os.getenv("fit2obs_ver") or "None"), "modulefiles"))
 load(pathJoin("fit2obs", os.getenv("fit2obs_ver")))
 
--- Temporary until official hpc-stack is updated
-prepend_path("MODULEPATH", "/scratch2/NCEPDEV/ensemble/save/Walter.Kolczynski/hpc-stack/modulefiles/stack")
-load(pathJoin("hpc", "1.2.0"))
-load(pathJoin("hpc-intel", "18.0.5.274"))
-load(pathJoin("hpc-miniconda3", "4.6.14"))
-load(pathJoin("gfs_workflow", "1.0.0"))
-load(pathJoin("met", "9.1"))
-load(pathJoin("metplus", "3.1"))
-
 whatis("Description: GFS run environment")