Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use HPC-stack libraries on WCOSS2 #975

Merged
merged 10 commits into from
Aug 21, 2024
Merged

Use HPC-stack libraries on WCOSS2 #975

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
15438a9
Update wcoss2 build module.
GeorgeGayno-NOAA Aug 8, 2024
3f53ae5
Add some comments to wcoss2 build module.
GeorgeGayno-NOAA Aug 8, 2024
1aace12
Boost memory for one of the chgres_cube regression tests.
GeorgeGayno-NOAA Aug 8, 2024
976a68b
Use stack version of nccmp in weight_gen regression test
GeorgeGayno-NOAA Aug 9, 2024
d65ff7a
Use stack version of nccmp utility in ocnice_prep regression
GeorgeGayno-NOAA Aug 9, 2024
9d85bd2
Use stack version of nccmp utility in grid_gen regression test
GeorgeGayno-NOAA Aug 9, 2024
6a986b0
Use stack version of nccmp utility in global_cycle regression
GeorgeGayno-NOAA Aug 9, 2024
f72ac50
Use stack version of nccmp utility in cpld_gridgen regression
GeorgeGayno-NOAA Aug 9, 2024
be7c21c
Use stack version of nccmp utility in chgres regression tests
GeorgeGayno-NOAA Aug 9, 2024
6f42420
Switch to ip v4.0.0 as v4.1.0 caused the snow2mdl regression
GeorgeGayno-NOAA Aug 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 13 additions & 11 deletions modulefiles/build.wcoss2.intel.lua
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@ load(pathJoin("craype", craype_ver))
intel_ver=os.getenv("intel_ver") or "19.1.3.304"
load(pathJoin("intel", intel_ver))

cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.7"
cray_mpich_ver=os.getenv("cray_mpich_ver") or "8.1.12"
load(pathJoin("cray-mpich", cray_mpich_ver))

libjpeg_ver=os.getenv("libjpeg_ver") or "9c"
Expand All @@ -26,11 +26,11 @@ load(pathJoin("zlib", zlib_ver))
libpng_ver=os.getenv("libpng_ver") or "1.6.37"
load(pathJoin("libpng", libpng_ver))

hdf5_ver=os.getenv("hdf5_ver") or "1.10.6"
load(pathJoin("hdf5", hdf5_ver))
hdf5_ver=os.getenv("hdf5_ver") or "1.14.0"
load(pathJoin("hdf5-C", hdf5_ver))

netcdf_ver=os.getenv("netcdf_ver") or "4.7.4"
load(pathJoin("netcdf", netcdf_ver))
netcdf_ver=os.getenv("netcdf_ver") or "4.9.2"
load(pathJoin("netcdf-C", netcdf_ver))

bacio_ver=os.getenv("bacio_ver") or "2.4.1"
load(pathJoin("bacio", bacio_ver))
Expand All @@ -42,13 +42,13 @@ load(pathJoin("bacio", bacio_ver))
w3emc_ver=os.getenv("w3emc_ver") or "2.9.2"
load(pathJoin("w3emc", w3emc_ver))

nemsio_ver=os.getenv("nemsio_ver") or "2.5.2"
nemsio_ver=os.getenv("nemsio_ver") or "2.5.4"
load(pathJoin("nemsio", nemsio_ver))

sigio_ver=os.getenv("sigio_ver") or "2.3.2"
load(pathJoin("sigio", sigio_ver))

sp_ver=os.getenv("sp_ver") or "2.3.3"
sp_ver=os.getenv("sp_ver") or "2.4.0"
load(pathJoin("sp", sp_ver))

ip_ver=os.getenv("ip_ver") or "4.0.0"
Expand All @@ -57,20 +57,22 @@ load(pathJoin("ip", ip_ver))
g2_ver=os.getenv("g2_ver") or "3.4.5"
load(pathJoin("g2", g2_ver))

-- for mpiexec command
-- Needed for mpiexec command.
cray_pals_ver=os.getenv("cray_pals_ver") or "1.2.2"
load(pathJoin("cray-pals", cray_pals_ver))

-- Needed at runtime for nco utilities.
udunits_ver=os.getenv("udunits_ver") or "2.2.28"
load(pathJoin("udunits", udunits_ver))

-- Needed at runtime for nco utilities.
gsl_ver=os.getenv("gsl_ver") or "2.7"
load(pathJoin("gsl", gsl_ver))

nco_ver=os.getenv("nco_ver") or "4.9.7"
nco_ver=os.getenv("nco_ver") or "5.0.6"
load(pathJoin("nco", nco_ver))

esmf_ver=os.getenv("esmf_ver") or "8.4.1"
load(pathJoin("esmf", esmf_ver))
esmf_ver=os.getenv("esmf_ver") or "8.6.0"
load(pathJoin("esmf-C", esmf_ver))

whatis("Description: UFS_UTILS build environment")
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/13km.conus.nam.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,8 @@ export TRACERS_INPUT=""

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/13km.conus.rap.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_RAPX

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/13km.na.gfs.ncei.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,8 @@ export CDATE=2019080100

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/25km.conus.gfs.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,8 @@ export CDATE=2019110400

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/25km.conus.gfs.pbgrib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@ export CDATE=2021062718

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/3km.conus.hrrr.gfssdf.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_HRRRX

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
2 changes: 2 additions & 0 deletions reg_tests/chgres_cube/3km.conus.hrrr.newsfc.grib2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,8 @@ export GEOGRID_FILE_INPUT=${HOMEufs}/fix/am/geo_em.d01.nc_HRRRX

export OMP_NUM_THREADS_CH=${OMP_NUM_THREADS:-1}

NCCMP=${NCCMP:-$(which nccmp)}

#-----------------------------------------------------------------------------
# Invoke chgres program.
#-----------------------------------------------------------------------------
Expand Down
7 changes: 4 additions & 3 deletions reg_tests/chgres_cube/driver.wcoss2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ compiler=${compiler:-"intel"}
source ../../sorc/machine-setup.sh > /dev/null 2>&1
module use ../../modulefiles
module load build.$target.$compiler
# load nccmp
module load netcdf
module load nccmp
GeorgeGayno-NOAA marked this conversation as resolved.
Show resolved Hide resolved
module list

export OUTDIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}"
Expand Down Expand Up @@ -60,8 +63,6 @@ rm -f $LOG_FILE* $SUM_FILE

export OMP_STACKSIZE=1024M

export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp
#export NCCMP=${NCCMP:-nccmp}
rm -fr $OUTDIR

this_dir=$PWD
Expand Down Expand Up @@ -163,7 +164,7 @@ TEST10=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltim
LOG_FILE=consistency.log11
export APRUN="mpiexec -n 12 -ppn 12 --cpu-bind core"
TEST11=$(qsub -V -o $LOG_FILE -e $LOG_FILE -q $QUEUE -A $PROJECT_CODE -l walltime=00:05:00 \
-N c96.fv3.netcdf2wam -l select=1:ncpus=12:ompthreads=1:mem=75GB $PWD/c96.fv3.netcdf2wam.sh)
-N c96.fv3.netcdf2wam -l select=1:ncpus=12:ompthreads=1:mem=80GB $PWD/c96.fv3.netcdf2wam.sh)

#-----------------------------------------------------------------------------
# Initialize CONUS 25-KM USING GFS PGRIB2+BGRIB2 files.
Expand Down
6 changes: 5 additions & 1 deletion reg_tests/cpld_gridgen/rt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ if [[ $target = wcoss2 ]]; then
export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core"
QUEUE=${QUEUE:-dev}
SBATCH_COMMAND="./cpld_gridgen.sh"
NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp
NCCMP=nccmp
elif [[ $target = hera ]]; then
STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER}
export MOM6_FIXDIR=/scratch1/NCEPDEV/global/glopara/fix/mom6/20220805
Expand Down Expand Up @@ -217,6 +217,10 @@ fi

module use $PATHTR/modulefiles
module load build.$target.$compiler
if [[ $target = wcoss2 ]]; then
module load netcdf
module load nccmp
fi
module list

if [[ $CREATE_BASELINE = true ]]; then
Expand Down
4 changes: 2 additions & 2 deletions reg_tests/global_cycle/driver.wcoss2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@ compiler=${compiler:-"intel"}
source ../../sorc/machine-setup.sh > /dev/null 2>&1
module use ../../modulefiles
module load build.$target.$compiler
module load netcdf
module load nccmp
module list

WORK_DIR="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}"
Expand Down Expand Up @@ -56,8 +58,6 @@ export NWPROD=$PWD/../..

reg_dir=$PWD

export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp

LOG_FILE=consistency.log
rm -f ${LOG_FILE}*

Expand Down
4 changes: 2 additions & 2 deletions reg_tests/grid_gen/driver.wcoss2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ compiler=${compiler:-"intel"}
source ../../sorc/machine-setup.sh > /dev/null 2>&1
module use ../../modulefiles
module load build.$target.$compiler
module load netcdf
module load nccmp
module list

set -x
Expand All @@ -44,8 +46,6 @@ if [ "$UPDATE_BASELINE" = "TRUE" ]; then
source ../get_hash.sh
fi

export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp

LOG_FILE=consistency.log
rm -f ${LOG_FILE}
SUM_FILE=summary.log
Expand Down
6 changes: 5 additions & 1 deletion reg_tests/ocnice_prep/rt.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,8 +130,8 @@ if [[ $target = wcoss2 ]]; then
ACCOUNT=${ACCOUNT:-GFS-DEV}
export APRUN="mpiexec -n 1 -ppn 1 --cpu-bind core"
QUEUE=${QUEUE:-dev}
NCCMP=nccmp
SBATCH_COMMAND="./ocnice_prep.sh"
NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp
elif [[ $target = hera ]]; then
STMP=${STMP:-/scratch1/NCEPDEV/stmp4/$USER}
BASELINE_ROOT=/scratch1/NCEPDEV/nems/role.ufsutils/ufs_utils/reg_tests/ocnice_prep/baseline_data
Expand Down Expand Up @@ -221,6 +221,10 @@ fi

module use $PATHTR/modulefiles
module load build.$target.$compiler
if [[ $target = wcoss2 ]]; then
module load netcdf
module load nccmp/1.8.9.0
fi
module list

if [[ $CREATE_BASELINE = true ]]; then
Expand Down
4 changes: 2 additions & 2 deletions reg_tests/weight_gen/driver.wcoss2.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,8 @@ compiler=${compiler:-"intel"}
source ../../sorc/machine-setup.sh > /dev/null 2>&1
module use ../../modulefiles
module load build.$target.$compiler
module load netcdf
module load nccmp
module list

export DATA="${WORK_DIR:-/lfs/h2/emc/stmp/$LOGNAME}"
Expand All @@ -54,8 +56,6 @@ fi
export HOMEreg=/lfs/h2/emc/nems/noscrub/emc.nems/UFS_UTILS/reg_tests/weight_gen
export HOMEufs=$PBS_O_WORKDIR/../..

export NCCMP=/lfs/h2/emc/global/noscrub/George.Gayno/util/nccmp/nccmp-1.8.5.0/src/nccmp

./weight_gen.sh

exit 0
Loading