[develop] update Gaea modulefile (ufs-community#836)

Updated build_gaea_intel.lua modulefile to use the new stack location on C3/C4 built following an upgrade with intel-classic-2022.0.2 compiler and cray-mpich/7.7.20. --------- Co-authored-by: Natalie Perlin <[email protected]> Co-authored-by: michael.lueken <[email protected]>
EdwardSnyder-NOAA · Aug 14, 2023 · 1031a28 · 1031a28
1 parent 6b57195
commit 1031a28
Show file tree

Hide file tree

Showing 11 changed files with 70 additions and 47 deletions.
diff --git a/.cicd/Jenkinsfile b/.cicd/Jenkinsfile
@@ -10,12 +10,10 @@ pipeline {
     parameters {
         // Allow job runner to filter based on platform
         // Use the line below to enable all PW clusters
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion', 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'], description: 'Specify the platform(s) to use')
         // Use the line below to enable the PW AWS cluster
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
-        // Use the line below to re-enable Gaea
-        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
-        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'hera', 'jet', 'orion'], description: 'Specify the platform(s) to use')
+        // choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion', 'pclusternoaav2use1'], description: 'Specify the platform(s) to use')
+        choice(name: 'SRW_PLATFORM_FILTER', choices: ['all', 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion'], description: 'Specify the platform(s) to use')
         // Allow job runner to filter based on compiler
         choice(name: 'SRW_COMPILER_FILTER', choices: ['all', 'gnu', 'intel'], description: 'Specify the compiler(s) to use to build')
         // Uncomment the following line to re-enable comprehensive tests
@@ -78,8 +76,7 @@ pipeline {
                 axes {
                     axis {
                         name 'SRW_PLATFORM'
-                        // values 'cheyenne', 'gaea', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
-                        values 'cheyenne', 'hera', 'jet', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                        values 'cheyenne', 'gaea', 'hera', 'jet-epic', 'orion' //, 'pclusternoaav2use1', 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                     }
 
                     axis {
@@ -93,8 +90,7 @@ pipeline {
                     exclude {
                             axis {
                             name 'SRW_PLATFORM'
-                            // values 'gaea', 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
-                            values 'jet', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
+                            values 'gaea', 'jet-epic', 'orion' //, 'pclusternoaav2use1' , 'azclusternoaav2eus1', 'gclusternoaav2usc1'
                         }
 
                         axis {

diff --git a/.cicd/scripts/srw_build.sh b/.cicd/scripts/srw_build.sh
@@ -24,6 +24,10 @@ else
     platform="${SRW_PLATFORM}"
 fi
 
+if [[ "${SRW_PLATFORM}" = jet-epic ]]; then
+    platform='jet'
+fi
+
 # Build and install
 cd ${workspace}/tests
 set +e

diff --git a/.cicd/scripts/srw_ftest.sh b/.cicd/scripts/srw_ftest.sh
@@ -42,6 +42,10 @@ else
     platform="${SRW_PLATFORM}"
 fi
 
+if [[ "${SRW_PLATFORM}" = jet-epic ]]; then
+    platform='jet'
+fi
+
 # Test directories
 we2e_experiment_base_dir="${workspace}/expt_dirs"
 we2e_test_dir="${workspace}/tests/WE2E"
@@ -64,7 +68,7 @@ sed "s|^workflow:|workflow:\n  EXPT_BASEDIR: ${workspace}/expt_dirs|1" -i ush/co
 sed "s|^workflow:|workflow:\n  EXEC_SUBDIR: ${workspace}/install_${SRW_COMPILER}/exec|1" -i ush/config.yaml
 
 # DATA_LOCATION differs on each platform ... find it.
-export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${SRW_PLATFORM,,}.yaml | awk '{printf "%s", $2}')
+export DATA_LOCATION=$(grep TEST_EXTRN_MDL_SOURCE_BASEDIR ${workspace}/ush/machine/${platform,,}.yaml | awk '{printf "%s", $2}')
 echo "DATA_LOCATION=${DATA_LOCATION}"
 
 # Configure a default test ...
@@ -129,18 +133,13 @@ rm -f ${results_file}
 status=0
 
 # Limit to machines that are fully ready
-deny_machines=( gaea )
-if [[ ${deny_machines[@]} =~ ${platform,,} ]] ; then
-    echo "# Deny ${platform} - incomplete configuration." | tee -a ${results_file}
-else
-    echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file}
-    for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
-                echo -n "./$task.sh ... "
-                ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))"
-                # stop at the first sign of trouble ...
-                [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file}
-    done
-fi
+echo "# Try ${platform} with the first few simple SRW tasks ..." | tee -a ${results_file}
+for task in ${TASKS[@]:0:${TASK_DEPTH}} ; do
+            echo -n "./$task.sh ... "
+            ./$task.sh > $task-log.txt 2>&1 && echo "COMPLETE" || echo "FAIL rc=$(( status+=$? ))"
+            # stop at the first sign of trouble ...
+            [[ 0 != ${status} ]] && echo "$task: FAIL" >> ${results_file} && break || echo "$task: COMPLETE" >> ${results_file}
+done
 
 # Set exit code to number of failures
 set +e

diff --git a/.cicd/scripts/srw_test.sh b/.cicd/scripts/srw_test.sh
@@ -25,6 +25,10 @@ else
     platform="${SRW_PLATFORM}"
 fi
 
+if [[ "${SRW_PLATFORM}" = jet-epic ]]; then
+    platform='jet'
+fi
+
 # Test directories
 we2e_experiment_base_dir="${workspace}/expt_dirs"
 we2e_test_dir="${workspace}/tests/WE2E"

diff --git a/etc/lmod-setup.csh b/etc/lmod-setup.csh
@@ -38,11 +38,7 @@ else if ( "$L_MACHINE" == singularity ) then
    module purge
 
 else if ( "$L_MACHINE" == gaea ) then
-   set ENV="/lustre/f2/dev/role.epic/contrib/apps/lmod/lmod/init/csh"
-   source $ENV
-
-   setenv LMOD_SYSTEM_DEFAULT_MODULES "modules/3.2.11.4"
-   module --initial_load --no_redirect restore
+   source /lustre/f2/dev/role.epic/contrib/Lmod_init.csh
 
 else if ( "$L_MACHINE" == odin ) then
    module unload modules

diff --git a/modulefiles/build_gaea_intel.lua b/modulefiles/build_gaea_intel.lua
@@ -1,21 +1,27 @@
 help([[
 This module loads libraries for building the UFS SRW App on
-the NOAA RDHPC machine Gaea using Intel-2022.1.2
+the NOAA RDHPC machine Gaea using Intel-2022.0.2
 ]])
 
 whatis([===[Loads libraries needed for building the UFS SRW App on Gaea ]===])
 
 load(pathJoin("cmake", os.getenv("cmake_ver") or "3.20.1"))
 
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-2021.3.0_noarch/modulefiles/stack")
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/hpc-stack/intel-classic-2022.0.2/modulefiles/stack")
 load(pathJoin("hpc", os.getenv("hpc_ver") or "1.2.0"))
-load(pathJoin("intel", os.getenv("intel_ver") or "2021.3.0"))
-load(pathJoin("hpc-intel", os.getenv("hpc_intel_ver") or "2021.3.0"))
-load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.11"))
-load(pathJoin("gcc", os.getenv("gcc_ver") or "8.3.0"))
-load(pathJoin("libpng", os.getenv("libpng_ver") or "1.6.37"))
+load(pathJoin("hpc-intel-classic", os.getenv("hpc_intel_classic_ver") or "2022.0.2"))
+load(pathJoin("hpc-cray-mpich", os.getenv("hpc_cray_mpich_ver") or "7.7.20"))
 
 load("srw_common")
+-- Need at runtime
+load("alps")
+
+local MKLROOT="/opt/intel/oneapi/mkl/2022.0.2/"
+prepend_path("LD_LIBRARY_PATH",pathJoin(MKLROOT,"lib/intel64"))
+pushenv("MKLROOT", MKLROOT)
+
+pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601")
+pushenv("CRAYPE_LINK_TYPE","dynamic")
 
 setenv("CC","cc")
 setenv("FC","ftn")

diff --git a/modulefiles/srw_common.lua b/modulefiles/srw_common.lua
@@ -27,6 +27,6 @@ load("sigio/2.3.2")
 load("w3nco/2.4.1")
 load("wrf_io/1.2.0")
 
-load("ncdiag/1.0.0")
+load("ncdiag/1.1.1")
 load("ncio/1.1.2")
 load("wgrib2/2.0.8")
diff --git a/modulefiles/tasks/gaea/plot_allvars.local.lua b/modulefiles/tasks/gaea/plot_allvars.local.lua
@@ -1,4 +1,4 @@
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles")
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles")
 load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
 
 setenv("SRW_ENV", "regional_workflow")
diff --git a/modulefiles/tasks/gaea/python_srw.lua b/modulefiles/tasks/gaea/python_srw.lua
@@ -1,4 +1,4 @@
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles")
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles")
 load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
 
 setenv("SRW_ENV", "workflow_tools")
diff --git a/modulefiles/wflow_gaea.lua b/modulefiles/wflow_gaea.lua
@@ -5,13 +5,17 @@ the NOAA RDHPC machine Gaea
 
 whatis([===[Loads libraries needed for running the UFS SRW App on gaea ]===])
 
+unload("python")
 load("set_pythonpath")
-prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/modulefiles")
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/miniconda3/modulefiles")
 load(pathJoin("miniconda3", os.getenv("miniconda3_ver") or "4.12.0"))
+prepend_path("MODULEPATH","/lustre/f2/dev/role.epic/contrib/rocoto/modulefiles")
 load("rocoto")
 load("alps")
 
-setenv("PROJ_LIB", "/lustre/f2/dev/role.epic/contrib/miniconda3/4.12.0/envs/regional_workflow/share/proj")
+pushenv("MKLROOT", "/opt/intel/oneapi/mkl/2022.0.2/")
+pushenv("GSI_BINARY_SOURCE_DIR", "/lustre/f2/dev/role.epic/contrib/GSI_data/fix/20230601")
+setenv("PMI_NO_PREINITIALIZE","1")
 
 if mode() == "load" then
    LmodMsgRaw([===[Please do the following to activate conda:

diff --git a/ush/machine/gaea.yaml b/ush/machine/gaea.yaml
@@ -6,17 +6,18 @@ platform:
   MRMS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/mrms/proc
   NDAS_OBS_DIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/obs_data/ndas/proc
   DOMAIN_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
-  QUEUE_DEFAULT: normal
-  QUEUE_FCST: normal
-  QUEUE_HPSS: normal
+  QUEUE_DEFAULT: windfall
+  QUEUE_FCST: windfall
+  QUEUE_HPSS: windfall
   REMOVE_MEMORY: True
-  RUN_CMD_FCST: srun --export=ALL --mpi=pmi2 -n ${PE_MEMBER01}
-  RUN_CMD_POST: srun --export=ALL --mpi=pmi2 -n $nprocs
-  RUN_CMD_PRDGEN: srun --export=ALL --mpi=pmi2 -n $nprocs
+  PARTITION_HPSS: eslogin
+  RUN_CMD_FCST: srun --export=ALL  -n ${PE_MEMBER01}
+  RUN_CMD_POST: srun --export=ALL  -n $nprocs
+  RUN_CMD_PRDGEN: srun --export=ALL  -n $nprocs
   RUN_CMD_SERIAL: time
   RUN_CMD_UTILS: srun --export=ALL --mpi=pmi2 -n $nprocs
-  SCHED_NATIVE_CMD: -M c3 --export=NONE
-  SCHED_NATIVE_CMD_HPSS: -M es --export=NONE
+  SCHED_NATIVE_CMD: --clusters=c4 --export=NONE
+  SCHED_NATIVE_CMD_HPSS: --clusters=es --export=NONE
   PRE_TASK_CMDS: '{ ulimit -s unlimited; ulimit -a; }'
   TEST_EXTRN_MDL_SOURCE_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data
   TEST_PREGEN_BASEDIR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/FV3LAM_pregen
@@ -30,8 +31,21 @@ platform:
   FIXorg: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_orog
   FIXsfc: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/fix/fix_sfc_climo
   FIXshp: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/NaturalEarth
+  EXTRN_MDL_DATA_STORES: aws
 data:
   ics_lbcs:
     FV3GFS:
       nemsio: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/nemsio/${yyyymmdd}${hh}
       grib2: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/FV3GFS/grib2/${yyyymmdd}${hh} 
+    RAP: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/RAP/${yyyymmdd}${hh}
+    HRRR: /lustre/f2/dev/role.epic/contrib/UFS_SRW_data/develop/input_model_data/HRRR/${yyyymmdd}${hh}
+rocoto:
+  tasks:
+    metatask_run_ensemble:
+      task_run_fcst_mem#mem#:
+        cores: '{{ task_run_fcst.PE_MEMBER01 // 1 }}'
+        native: '--cpus-per-task {{ task_run_fcst.OMP_NUM_THREADS_RUN_FCST|int }} --exclusive {{ platform.SCHED_NATIVE_CMD }}'
+        nodes:
+        nnodes:
+        nodesize:
+        ppn: