From 1db9f2a1a1140fa70f2210c882bc0740dd940ce3 Mon Sep 17 00:00:00 2001 From: Florian Deconinck Date: Tue, 23 Apr 2024 10:17:45 -0400 Subject: [PATCH] SMTStack: 2024.04.00 --- sw_stack/discover/sles15/HISTORY.md | 8 +- .../SMTStack/2024.04.00-no-venv.lua | 44 ++++++ .../modulefiles/SMTStack/2024.04.00.lua | 49 ++++++ .../discover/sles15/src/2024.04.00/basics.sh | 37 +++++ .../sles15/src/2024.04.00/build_0_on-node.sh | 147 ++++++++++++++++++ .../sles15/src/2024.04.00/build_1_on-login.sh | 12 ++ .../sles15/src/2024.04.00/build_2_check.sh | 11 ++ .../src/2024.04.00/build_3_inode-clean.sh | 5 + .../sles15/src/2024.04.00/download.sh | 67 ++++++++ .../sles15/src/2024.04.00/osu-bench.sh | 86 ++++++++++ .../sles15/src/2024.04.00/verify_baselibs.sh | 10 ++ 11 files changed, 475 insertions(+), 1 deletion(-) create mode 100644 sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00-no-venv.lua create mode 100644 sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00.lua create mode 100755 sw_stack/discover/sles15/src/2024.04.00/basics.sh create mode 100755 sw_stack/discover/sles15/src/2024.04.00/build_0_on-node.sh create mode 100755 sw_stack/discover/sles15/src/2024.04.00/build_1_on-login.sh create mode 100755 sw_stack/discover/sles15/src/2024.04.00/build_2_check.sh create mode 100644 sw_stack/discover/sles15/src/2024.04.00/build_3_inode-clean.sh create mode 100755 sw_stack/discover/sles15/src/2024.04.00/download.sh create mode 100644 sw_stack/discover/sles15/src/2024.04.00/osu-bench.sh create mode 100644 sw_stack/discover/sles15/src/2024.04.00/verify_baselibs.sh diff --git a/sw_stack/discover/sles15/HISTORY.md b/sw_stack/discover/sles15/HISTORY.md index 27aca0d..3c08411 100644 --- a/sw_stack/discover/sles15/HISTORY.md +++ b/sw_stack/discover/sles15/HISTORY.md @@ -6,7 +6,13 @@ All versions of the software for a given version are saved in `basics.sh`. `build` directory is the throwaway directory where everything is downloaded then built. `install` is saves all library and executable once build is done. -Last edit: _March 22th 2024_ +Last edit: _April 22th 2024_ + +## v2024.04.00 + +### Changes + +- NDSL `2024.03.01` -> `2024.04.00` ## v2024.03.00 diff --git a/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00-no-venv.lua b/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00-no-venv.lua new file mode 100644 index 0000000..8e85696 --- /dev/null +++ b/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00-no-venv.lua @@ -0,0 +1,44 @@ +load("comp/gcc/12.3.0") +load("lib/mkl/2023.2.0") +load("nvidia/nvhpc-byo-compiler/23.9") + +local install_dir = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.04.00/install" + +-- Fix: GT4Py expects CUDA_HOME to be set -- +setenv("CUDA_HOME", os.getenv("NVHPC_ROOT")) + +-- UCX -- +local ucx_pkgdir = pathJoin(install_dir, "ucx") +prepend_path("LD_LIBRARY_PATH",pathJoin(ucx_pkgdir,"lib")) + +-- OMPI -- +local ompi_pkgdir = pathJoin(install_dir, "ompi") + +setenv("M_MPI_ROOT",ompi_pkgdir) +setenv("OPENMPI",ompi_pkgdir) +setenv("MPI_HOME",ompi_pkgdir) + +prepend_path("PATH",pathJoin(ompi_pkgdir,"bin")) +prepend_path("LD_LIBRARY_PATH",pathJoin(ompi_pkgdir,"lib")) +prepend_path("INCLUDE",pathJoin(ompi_pkgdir,"include")) +prepend_path("MANPATH",pathJoin(ompi_pkgdir,"share/man")) + +setenv("OMPI_MCA_orte_tmpdir_base","/tmp") +setenv("TMPDIR","/tmp") +setenv("OMP_STACKSIZE","1G") +setenv("OMPI_MCA_mca_base_component_show_load_errors","0") +setenv("PMIX_MCA_mca_base_component_show_load_errors","0") + +-- BOOST HEADERS (as expected by gt4py) -- +local boost_pkgdir = pathJoin(install_dir, "boost") +setenv("BOOST_ROOT", boost_pkgdir) + +-- Python 3 -- +local py_pkgdir = pathJoin(install_dir, "python3") +prepend_path("PATH",pathJoin(py_pkgdir,"bin")) +prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib")) +prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64")) + +-- Baselibs at a BASEDIR -- +local baselibs_pkgdir = pathJoin(install_dir, "baselibs-7.17.1/install/x86_64-pc-linux-gnu/") +setenv("BASEDIR", baselibs_pkgdir) diff --git a/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00.lua b/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00.lua new file mode 100644 index 0000000..b38c99c --- /dev/null +++ b/sw_stack/discover/sles15/modulefiles/SMTStack/2024.04.00.lua @@ -0,0 +1,49 @@ +load("comp/gcc/12.3.0") +load("lib/mkl/2023.2.0") +load("nvidia/nvhpc-byo-compiler/23.9") + +local install_dir = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.04.00/install" + +-- Fix: GT4Py expects CUDA_HOME to be set -- +setenv("CUDA_HOME", os.getenv("NVHPC_ROOT")) + +-- UCX -- +local ucx_pkgdir = pathJoin(install_dir, "ucx") +prepend_path("LD_LIBRARY_PATH",pathJoin(ucx_pkgdir,"lib")) + +-- OMPI -- +local ompi_pkgdir = pathJoin(install_dir, "ompi") + +setenv("M_MPI_ROOT",ompi_pkgdir) +setenv("OPENMPI",ompi_pkgdir) +setenv("MPI_HOME",ompi_pkgdir) + +prepend_path("PATH",pathJoin(ompi_pkgdir,"bin")) +prepend_path("LD_LIBRARY_PATH",pathJoin(ompi_pkgdir,"lib")) +prepend_path("INCLUDE",pathJoin(ompi_pkgdir,"include")) +prepend_path("MANPATH",pathJoin(ompi_pkgdir,"share/man")) + +setenv("OMPI_MCA_orte_tmpdir_base","/tmp") +setenv("TMPDIR","/tmp") +setenv("OMP_STACKSIZE","1G") +setenv("OMPI_MCA_mca_base_component_show_load_errors","0") +setenv("PMIX_MCA_mca_base_component_show_load_errors","0") + +-- BOOST HEADERS (as expected by gt4py) -- +local boost_pkgdir = pathJoin(install_dir, "boost") +setenv("BOOST_ROOT", boost_pkgdir) + +-- Python 3 -- +local py_pkgdir = pathJoin(install_dir, "python3") +prepend_path("PATH",pathJoin(py_pkgdir,"bin")) +prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib")) +prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64")) + +-- Load venv -- +local py_pkgdir = pathJoin(install_dir, "venv") +prepend_path("PATH",pathJoin(py_pkgdir,"bin")) +setenv("VIRTUAL_ENV", py_pkgdir) + +-- Baselibs at a BASEDIR -- +local baselibs_pkgdir = pathJoin(install_dir, "baselibs-7.17.1/install/x86_64-pc-linux-gnu/") +setenv("BASEDIR", baselibs_pkgdir) diff --git a/sw_stack/discover/sles15/src/2024.04.00/basics.sh b/sw_stack/discover/sles15/src/2024.04.00/basics.sh new file mode 100755 index 0000000..8869c3a --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/basics.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +export DSLSW_VERSION="2024.04.00" +echo "DSL Software Stack v${DSLSW_VERSION}" + +# Version +export DSLSW_GDRCOPY_VER=2.3 +export DSLSW_OMPI_MAJOR_VER=4.1 +export DSLSW_OMPI_VER=${DSLSW_OMPI_MAJOR_VER}.6 +export DSLSW_UCX_VER=1.15.0 +export DSLSW_CUDA_VER=12.2 +export DSLSW_OSUMICRO_VER=7.3 +export DSLSW_LAPACK_VER=3.11.0 +export DSLSW_PY_VER=3.11.7 +export DSLSW_BASELIBS_VER=7.17.1 +export DSLSW_SERIALBOX_VER=2.6.2-unreleased +export DSLSW_SERIALBOX_SHA=88ac4e4dfc824953d068fe63c8e7b3dd9560a914 +export DSLSW_GNU_VER=12.2.0 +export DSLSW_NDSL_VER=2024.04.00 +export DSLSW_BOOST_VER=1.76.0 +export DSLSW_BOOST_VER_STR=1_76_0 + +# Base directory & versioning +export DSLSW_BASE=$PWD/build +mkdir -p $DSLSW_BASE +export DSLSW_INSTALL_DIR=$PWD/install +mkdir -p $DSLSW_INSTALL_DIR + +# Modules +module use -a /discover/nobackup/projects/geosongpu/sw_sles15/live/modulefiles/ +module load SMTStack/${DSLSW_VERSION} +CUDA_DIR=/usr/local/other/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/ + +# Enforce proper compilers +export FC=gfortran +export CC=gcc +export CXX=g++ diff --git a/sw_stack/discover/sles15/src/2024.04.00/build_0_on-node.sh b/sw_stack/discover/sles15/src/2024.04.00/build_0_on-node.sh new file mode 100755 index 0000000..1bac314 --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/build_0_on-node.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +# Source the shared basics +source ./basics.sh + +echo " === GDR Copy (requires kernel running on the box) === " +#cd $DSLSW_BASE/gdrcopy-$DSLSW_GDRCOPY_VER +#make prefix=$DSLSW_INSTALL_DIR/gdrcopy CUDA=$CUDA_DIR all install +#exit 0 + +echo " === UCX === " +cd $DSLSW_BASE/ucx-$DSLSW_UCX_VER +./configure --prefix=$DSLSW_INSTALL_DIR/ucx \ + --enable-optimizations \ + --disable-logging \ + --disable-debug \ + --disable-assertions \ + --disable-params-check \ + --without-xpmem \ + --without-java \ + --without-go \ + --with-cuda=$CUDA_DIR \ + --with-gdrcopy=/usr/src/gdrdrv-$DSLSW_GDRCOPY_VER/ + +make -j 32 install +#exit 0 + +echo " === OpenMPI === " + +# NSL lib (-lnsl) was not symlink from libnsl.so.1 which lead to issues (--disable-getpwuid is an attempt to squash that, which seems unsucessful). Potentially, removing the LSF scheduler build would work. + +# libxml2 has a /zero on it's init (https://gitlab.gnome.org/GNOME/libxml2/-/blob/7846b0a677f8d3ce72486125fa281e92ac9970e8/xpath.c#L505) which seems to trigger a sigfpe. Relying on the internal but potentially wobly XML parser of OMPI + +cd $DSLSW_BASE/openmpi-${DSLSW_OMPI_VER} +./configure --prefix=$DSLSW_INSTALL_DIR/ompi \ + --disable-libxml2 \ + --disable-wrapper-rpath \ + --disable-wrapper-runpath \ + --with-pmix \ + --with-cuda=$CUDA_DIR \ + --with-cuda-libdir=$CUDA_DIR/lib64/stubs \ + --with-ucx=$DSLSW_INSTALL_DIR/ucx \ + --with-slurm \ + --enable-mpi1-compatibility + +make -j32 all +make install +export PATH=$DSLSW_INSTALL_DIR/ompi/bin:$PATH +export LD_LIBRARY_PATH=$DSLSW_INSTALL_DIR/ompi/lib:$DSLSW_INSTALL_DIR/ucx/lib:$LD_LIBRARY_PATH + +echo " === OSU === " + +cd $DSLSW_BASE/osu-micro-benchmarks-$DSLSW_OSUMICRO_VER +./configure \ + CC=mpicc \ + CXX=mpicxx \ + --prefix=$DSLSW_INSTALL_DIR/osu \ + --enable-cuda \ + --with-cuda-include=$CUDA_DIR/include \ + --with-cuda=$CUDA_DIR \ + --with-cuda-libpath=$CUDA_DIR/lib64/stubs/ + +make -j32 +make install + +echo " === Python === " +cd $DSLSW_BASE/Python-$DSLSW_PY_VER +./configure --prefix=$DSLSW_INSTALL_DIR/python3 --enable-shared --enable-optimizations + +make -j32 +make install + +echo " === Serialbox === " +cd $DSLSW_BASE/serialbox-$DSLSW_SERIALBOX_VER +mkdir build +cd build +cmake -DCMAKE_INSTALL_PREFIX=$DSLSW_INSTALL_DIR/serialbox \ + -DSERIALBOX_ENABLE_FORTRAN=ON \ + -DSERIALBOX_EXAMPLES=OFF \ + .. +make -j32 install + + +echo " === Baselibs === " +cd $DSLSW_BASE/baselibs-$DSLSW_BASELIBS_VER +make ESMF_COMM=openmpi \ + BUILD=ESSENTIALS \ + ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \ + prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \ + install + +if [ -z ${BUILD_GCC_OFFLOAD+x} ]; then + echo "Skip building offloaded GCC. Define BUILD_GCC_OFFLOAD to build." +else + echo " === GNU gcc/gfortran/g++ with OpenACC and OpenMP Offload on NVIDIA GPUs === " + module rm comp/gcc/12.3.0 + module rm nvidia/nvhpc-nompi/23.9 + unset CC + unset CXX + unset FC + + # Build assembler and linking tools + cd $DSLSW_BASE/gnu/nvptx-tools + ./configure \ + --with-cuda-driver-include=$CUDA_DIR/include \ + --with-cuda-driver-lib=$CUDA_DIR/lib64 \ + --prefix=$DSLSW_INSTALL_DIR/gnu + make || exit 1 + make install || exit 1 + cd .. + + # Set up the GCC source tree + cd $DSLSW_BASE/gnu/gcc + ln -s ../nvptx-newlib/newlib newlib + cd .. + export target=$(gcc/config.guess) + + # Build nvptx GCC + mkdir build-nvptx-gcc + cd build-nvptx-gcc + ../gcc/configure \ + --target=nvptx-none --with-build-time-tools=$DSLSW_INSTALL_DIR/gnu/nvptx-none/bin \ + --enable-as-accelerator-for=$target \ + --disable-sjlj-exceptions \ + --enable-newlib-io-long-long \ + --enable-languages="c,c++,fortran,lto" \ + --prefix=$DSLSW_INSTALL_DIR/gnu + make -j`nproc` || exit 1 + make install || exit 1 + cd .. + + # Build host GCC + mkdir build-host-gcc + cd build-host-gcc + ../gcc/configure \ + --enable-offload-targets=nvptx-none \ + --with-cuda-driver-include=$CUDA_DIR/include \ + --with-cuda-driver-lib=$CUDA_DIR/lib64 \ + --disable-bootstrap \ + --disable-multilib \ + --enable-languages="c,c++,fortran,lto" \ + --prefix=$DSLSW_INSTALL_DIR/gnu + make -j`nproc` || exit 1 + make install || exit 1 + cd .. +fi + diff --git a/sw_stack/discover/sles15/src/2024.04.00/build_1_on-login.sh b/sw_stack/discover/sles15/src/2024.04.00/build_1_on-login.sh new file mode 100755 index 0000000..5be5232 --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/build_1_on-login.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +# Source the shared basics +source ./basics.sh + +echo " === Make NDSL venv === " +cd $DSLSW_INSTALL_DIR +python3 -m venv venv +source ./venv/bin/activate +pip install --upgrade setuptools pip +pip install -e $DSLSW_INSTALL_DIR/ndsl +pip install mpi4py cffi cupy-cuda12x diff --git a/sw_stack/discover/sles15/src/2024.04.00/build_2_check.sh b/sw_stack/discover/sles15/src/2024.04.00/build_2_check.sh new file mode 100755 index 0000000..1740e49 --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/build_2_check.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +source ./basics.sh + +echo $DSLSW_INSTALL_DIR +echo `which $FC` +echo `which $CC` + +echo $LD_LIBRARY_PATH + +rm -rf build \ No newline at end of file diff --git a/sw_stack/discover/sles15/src/2024.04.00/build_3_inode-clean.sh b/sw_stack/discover/sles15/src/2024.04.00/build_3_inode-clean.sh new file mode 100644 index 0000000..da814f9 --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/build_3_inode-clean.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +source ./basics.sh + +rm -rf $DSLSW_BASE \ No newline at end of file diff --git a/sw_stack/discover/sles15/src/2024.04.00/download.sh b/sw_stack/discover/sles15/src/2024.04.00/download.sh new file mode 100755 index 0000000..6e0b2f6 --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/download.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +# Source the share basics +source ./basics.sh + +cd $DSLSW_BASE + +# GDR Copy should be present in /usr/src/gdrdrv-* +#wget -c https://github.com/NVIDIA/gdrcopy/archive/refs/tags/v$DSLSW_GDRCOPY_VER.tar.gz +#tar zxpvf v$DSLSW_GDRCOPY_VER.tar.gz +#rm v$DSLSW_GDRCOPY_VER.tar.gz + +wget https://github.com/openucx/ucx/releases/download/v${DSLSW_UCX_VER}/ucx-${DSLSW_UCX_VER}.tar.gz +tar xfp ucx-$DSLSW_UCX_VER.tar.gz +rm ucx-$DSLSW_UCX_VER.tar.gz + +wget https://download.open-mpi.org/release/open-mpi/v$DSLSW_OMPI_MAJOR_VER/openmpi-${DSLSW_OMPI_VER}.tar.gz +tar xfzp openmpi-$DSLSW_OMPI_VER.tar.gz +rm openmpi-$DSLSW_OMPI_VER.tar.gz + +wget https://mvapich.cse.ohio-state.edu/download/mvapich/osu-micro-benchmarks-$DSLSW_OSUMICRO_VER.tar.gz +tar xfp osu-micro-benchmarks-$DSLSW_OSUMICRO_VER.tar.gz +rm osu-micro-benchmarks-$DSLSW_OSUMICRO_VER.tar.gz + +wget https://www.python.org/ftp/python/$DSLSW_PY_VER/Python-$DSLSW_PY_VER.tgz +tar zxpvf Python-$DSLSW_PY_VER.tgz +rm Python-$DSLSW_PY_VER.tgz + +git clone https://github.com/GridTools/serialbox.git serialbox-$DSLSW_SERIALBOX_VER +cd serialbox-$DSLSW_SERIALBOX_VER +git checkout $DSLSW_SERIALBOX_SHA +cd $DSLSW_BASE + +git clone --recurse-submodules -b v$DSLSW_BASELIBS_VER https://github.com/GEOS-ESM/ESMA-Baselibs.git ./baselibs-$DSLSW_BASELIBS_VER +cd ./baselibs-$DSLSW_BASELIBS_VER +make download +echo "=>Baselibs >> Removing HDF4 from the ESSENTIALS" +sed -i 's/ESSENTIAL_DIRS = jpeg zlib szlib hdf4 hdf5/ESSENTIAL_DIRS = jpeg zlib szlib hdf5/g' GNUmakefile +sed -i 's/\/zlib \/szlib \/jpeg \/hdf5 \/hdf \/netcdf,\\/\/ \/zlib \/szlib \/jpeg \/hdf5 \/netcdf,\\/g' GNUmakefile +cd $DSLSW_BASE + +if [ -z ${BUILD_GCC_OFFLOAD+x} ]; then + echo "Skip building offloaded GCC. Define BUILD_GCC_OFFLOAD to build." +else + mkdir gnu + cd gnu + git clone https://github.com/SourceryTools/nvptx-tools + git clone git://sourceware.org/git/newlib-cygwin.git nvptx-newlib + git clone --branch releases/gcc-${DSLSW_GNU_VER} git://gcc.gnu.org/git/gcc.git gcc + cd gcc + contrib/download_prerequisites +fi + +# Stream include out of boost source +cd $DSLSW_INSTALL_DIR +wget https://boostorg.jfrog.io/artifactory/main/release/$DSLSW_BOOST_VER/source/boost_$DSLSW_BOOST_VER_STR.tar.gz +tar zxpvf boost_$DSLSW_BOOST_VER_STR.tar.gz +rm boost_$DSLSW_BOOST_VER_STR.tar.gz +mkdir -p boost/include +mv boost_$DSLSW_BOOST_VER_STR/boost boost/include +rm -r boost_$DSLSW_BOOST_VER_STR +cd $DSLSW_BASE + +# Git clone `ndsl`, with the minimuum amount of history +cd $DSLSW_INSTALL_DIR +git clone --recurse-submodules --shallow-submodules -b $DSLSW_NDSL_VER --single-branch --depth 1 https://github.com/NOAA-GFDL/NDSL.git ndsl +cd $DSLSW_BASE diff --git a/sw_stack/discover/sles15/src/2024.04.00/osu-bench.sh b/sw_stack/discover/sles15/src/2024.04.00/osu-bench.sh new file mode 100644 index 0000000..4658a1b --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/osu-bench.sh @@ -0,0 +1,86 @@ +#!/bin/bash + +source ./basics.sh + +# need to be ran on a salloc with 2 process on one node each +if [[ $(hostname -s) != warpa* ]]; then + echo "salloc on a 2 warpa nodes!" + exit 0 +fi + +which 'mpirun' +which 'osu_bw' + +echo "Host to Host" +mpirun -x UCX_IB_GPU_DIRECT_RDMA=0 \ + -mca btl '^openib' \ + -mca pml ucx \ + -mca opal_warn_on_missing_libcuda 1 \ + -mca btl_smcuda_cuda_ipc_verbose 1 \ + -mca btl_openib_want_cuda_gdr 1 \ + -mca orte_tmpdir_base $TSE_TMPDIR \ + -mca orte_base_help_aggregate 0 \ + -x UCX_TLS=rc,cuda,cuda_copy,gdr_copy \ + -x UCX_RNDV_THRESH=8192 \ + osu_bw -x 100 -i 100 -d cuda H H + +echo "Device: RDMA + GDRCOPY" +mpirun -x UCX_IB_GPU_DIRECT_RDMA=1 \ + -x UCX_RNDV_SCHEME=get_zcopy \ + -x UCX_MEMTYPE_CACHE=n \ + -x UCX_RNDV_THRESH=8192 \ + -x UCX_TLS=rc,cuda,cuda_copy,gdr_copy \ + -mca pml ucx \ + -mca opal_warn_on_missing_libcuda 1 \ + -mca btl_smcuda_cuda_ipc_verbose 1 \ + -mca btl_openib_want_cuda_gdr 1 \ + -mca orte_tmpdir_base $TSE_TMPDIR \ + -mca orte_base_help_aggregate 0 \ + -mca btl '^openib' \ + osu_bw -x 100 -i 100 -d cuda D D + +echo "Device: RDMA only" +mpirun -x UCX_IB_GPU_DIRECT_RDMA=1 \ + -x UCX_RNDV_SCHEME=get_zcopy \ + -x UCX_MEMTYPE_CACHE=n \ + -x UCX_RNDV_THRESH=8192 \ + -x UCX_TLS=rc,cuda,cuda_copy \ + -mca pml ucx \ + -mca opal_warn_on_missing_libcuda 1 \ + -mca btl_smcuda_cuda_ipc_verbose 1 \ + -mca btl_openib_want_cuda_gdr 1 \ + -mca orte_tmpdir_base $TSE_TMPDIR \ + -mca orte_base_help_aggregate 0 \ + -mca btl '^openib' \ + osu_bw -x 100 -i 100 -d cuda D D + +echo "Device: GDR_COPY only" +mpirun -x UCX_IB_GPU_DIRECT_RDMA=0 \ + -x UCX_RNDV_SCHEME=get_zcopy \ + -x UCX_MEMTYPE_CACHE=n \ + -x UCX_RNDV_THRESH=8192 \ + -x UCX_TLS=rc,cuda,cuda_copy,gdr_copy \ + -mca pml ucx \ + -mca opal_warn_on_missing_libcuda 1 \ + -mca btl_smcuda_cuda_ipc_verbose 1 \ + -mca btl_openib_want_cuda_gdr 1 \ + -mca orte_tmpdir_base $TSE_TMPDIR \ + -mca orte_base_help_aggregate 0 \ + -mca btl '^openib' \ + osu_bw -x 10i -i 10 -d cuda D D + + +echo "Device: default" +mpirun -x UCX_IB_GPU_DIRECT_RDMA=0 \ + -x UCX_RNDV_SCHEME=get_zcopy \ + -x UCX_MEMTYPE_CACHE=n \ + -x UCX_RNDV_THRESH=8192 \ + -x UCX_TLS=rc,cuda,cuda_copy \ + -mca pml ucx \ + -mca opal_warn_on_missing_libcuda 1 \ + -mca btl_smcuda_cuda_ipc_verbose 1 \ + -mca btl_openib_want_cuda_gdr 1 \ + -mca orte_tmpdir_base $TSE_TMPDIR \ + -mca orte_base_help_aggregate 0 \ + -mca btl '^openib' \ + osu_bw -x 10 -i 10 -d cuda D D diff --git a/sw_stack/discover/sles15/src/2024.04.00/verify_baselibs.sh b/sw_stack/discover/sles15/src/2024.04.00/verify_baselibs.sh new file mode 100644 index 0000000..4763f7a --- /dev/null +++ b/sw_stack/discover/sles15/src/2024.04.00/verify_baselibs.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +source ./basics.sh + +cd $DSLSW_BASE/baselibs-$DSLSW_BASELIBS_VER +make ESMF_COMM=openmpi \ + BUILD=ESSENTIALS \ + ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \ + prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \ + verify