Skip to content

Commit

Permalink
Merge commit '8dfc4228cfdddb2398ecd6bb02eeea592b48c203'
Browse files Browse the repository at this point in the history
  • Loading branch information
FlorianDeconinck committed Mar 21, 2024
2 parents 98ea2cf + 8dfc422 commit 07ee7b0
Show file tree
Hide file tree
Showing 11 changed files with 218 additions and 150 deletions.
39 changes: 19 additions & 20 deletions sw_stack/discover/sles15/HISTORY.md
Original file line number Diff line number Diff line change
Expand Up @@ -6,20 +6,34 @@ All versions of the software for a given version are saved in `basics.sh`.
`build` directory is the throwaway directory where everything is downloaded then built.
`install` is saves all library and executable once build is done.

Last edit: _December 29th 2023_
Last edit: _March 18th 2024_

## v1.0.0
## v2024.03.00

### OpenMPI
### Options

`BUILD_GCC_OFFLOAD`: builds an offload ready GCC 12.2

### Stack

We build OpenMPI throught the UCX layer with cuda-enabled and GRDCopy and GPUDirect on.

- GDRCOPY: Must be installed on the compiling machine as a kernel module.
- GCC: 12.3.0 [^1] (via `comp/gcc/12.3.0` on discover)
- CUDA (via `nvhpc`): 12.2 [^2] (via `nvidia/nvhpc-nompi/23.9` on discover)
- MKL: 2023.2.0 (via `lib/mkl/2023.2.0` on discover)
- UCX: 1.15.0
- OpenMPI: 4.1.6 [^3]
- OSU-MICROBENCHMARK: 7.3
- Boost headers: 1.76.0
- BASELIBS: 7.17.1 [^4]
- Python: 3.11.7
- NDSL: 2024.03.01
- Serialbox: 2.6.2-unreleased [^5]

When defining `BUILD_GCC_OFFLOAD`:

- GCC with offload: 12.2.0

Test of the stack can be done via the `osu-microbenchmark` with latency & bandwith saved in `osu-bench.sh`.

Expand All @@ -28,20 +42,5 @@ _Note:_
- [^1]: `gcc-13.2.0` fails during GEOS with an internal compiler error
- [^2]: `nvhpc` ships with a prebuilt `openmpi` which can cause issues. Make sure to load the `nompi` module.
- [^3]: `openmpi-5.0.0` fails at GEOS runtime on a call to `libxml2` that does a divide by zero (triggering a sigfpe). We revert to `4.1.6`.

### Baselibs

- LAPACK/BLAS: 3.11.0
- BASELIBS: 7.14.1

### Python

- Python: 3.8.10 [^4]

### Serialbox

- Latest stable is 2.6.1. Development is over.

_Note:_

- [^4]: `3.10.12` leads to failure in DaCe parsing.
- [^4]: As per GEOS 11.5.2 @env
- [^5]: Latest stable is 2.6.1. Development is crawlingly slow but some fixes for GCC 12+ are in `main`
1 change: 0 additions & 1 deletion sw_stack/discover/sles15/TODO.md
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# TODO

- Test the build of baselibs
- Build GCC with OpenACC & OpenMP offloading
Original file line number Diff line number Diff line change
@@ -1,7 +1,8 @@
load("comp/gcc/12.3.0")
load("nvidia/nvhpc-nompi/23.9")
load("lib/mkl/2023.2.0")
load("nvidia/nvhpc-byo-compiler/23.9")

local install_dir = "/discover/nobackup/projects/geosongpu/sw_sles15/v1.0.0/install/"
local install_dir = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.03.00/install"

-- UCX --
local ucx_pkgdir = pathJoin(install_dir, "ucx")
Expand All @@ -25,7 +26,12 @@ setenv("OMP_STACKSIZE","1G")
setenv("OMPI_MCA_mca_base_component_show_load_errors","0")
setenv("PMIX_MCA_mca_base_component_show_load_errors","0")

-- BOOST HEADERS (as expected by gt4py) --
local boost_pkgdir = pathJoin(install_dir, "boost")
setenv("BOOST_ROOT", boost_pkgdir)

-- Python 3 --
local py_pkgdir = pathJoin(install_dir, "python3")
prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib"))
prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64"))
42 changes: 42 additions & 0 deletions sw_stack/discover/sles15/modulefiles/SMTStack/2024.03.00.lua
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
load("comp/gcc/12.3.0")
load("lib/mkl/2023.2.0")
load("nvidia/nvhpc-byo-compiler/23.9")

local install_dir = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.03.00/install"

-- UCX --
local ucx_pkgdir = pathJoin(install_dir, "ucx")
prepend_path("LD_LIBRARY_PATH",pathJoin(ucx_pkgdir,"lib"))

-- OMPI --
local ompi_pkgdir = pathJoin(install_dir, "ompi")

setenv("M_MPI_ROOT",ompi_pkgdir)
setenv("OPENMPI",ompi_pkgdir)
setenv("MPI_HOME",ompi_pkgdir)

prepend_path("PATH",pathJoin(ompi_pkgdir,"bin"))
prepend_path("LD_LIBRARY_PATH",pathJoin(ompi_pkgdir,"lib"))
prepend_path("INCLUDE",pathJoin(ompi_pkgdir,"include"))
prepend_path("MANPATH",pathJoin(ompi_pkgdir,"share/man"))

setenv("OMPI_MCA_orte_tmpdir_base","/tmp")
setenv("TMPDIR","/tmp")
setenv("OMP_STACKSIZE","1G")
setenv("OMPI_MCA_mca_base_component_show_load_errors","0")
setenv("PMIX_MCA_mca_base_component_show_load_errors","0")

-- BOOST HEADERS (as expected by gt4py) --
local boost_pkgdir = pathJoin(install_dir, "boost")
setenv("BOOST_ROOT", boost_pkgdir)

-- Python 3 --
local py_pkgdir = pathJoin(install_dir, "python3")
prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib"))
prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64"))

-- Load venv --
local py_pkgdir = pathJoin(install_dir, "venv")
prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
setenv("VIRTUAL_ENV", py_pkgdir)
41 changes: 0 additions & 41 deletions sw_stack/discover/sles15/src/1.0.0/basics.sh

This file was deleted.

37 changes: 37 additions & 0 deletions sw_stack/discover/sles15/src/2024.03.00/basics.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
#!/bin/bash

export DSLSW_VERSION="2024.03.00"
echo "DSL Software Stack v${DSLSW_VERSION}"

# Version
export DSLSW_GDRCOPY_VER=2.3
export DSLSW_OMPI_MAJOR_VER=4.1
export DSLSW_OMPI_VER=${DSLSW_OMPI_MAJOR_VER}.6
export DSLSW_UCX_VER=1.15.0
export DSLSW_CUDA_VER=12.2
export DSLSW_OSUMICRO_VER=7.3
export DSLSW_LAPACK_VER=3.11.0
export DSLSW_PY_VER=3.11.7
export DSLSW_BASELIBS_VER=7.17.1
export DSLSW_SERIALBOX_VER=2.6.2-unreleased
export DSLSW_SERIALBOX_SHA=88ac4e4dfc824953d068fe63c8e7b3dd9560a914
export DSLSW_GNU_VER=12.2.0
export DSLSW_NDSL_VER=2024.03.01
export DSLSW_BOOST_VER=1.76.0
export DSLSW_BOOST_VER_STR=1_76_0

# Base directory & versioning
export DSLSW_BASE=$PWD/build
mkdir -p $DSLSW_BASE
export DSLSW_INSTALL_DIR=$PWD/install
mkdir -p $DSLSW_INSTALL_DIR

# Modules
module use -a /discover/nobackup/projects/geosongpu/sw_sles15/live/modulefiles/
module load SMTStack/${DSLSW_VERSION}
CUDA_DIR=/usr/local/other/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/

# Enforce proper compilers
export FC=gfortran
export CC=gcc
export CXX=g++
12 changes: 12 additions & 0 deletions sw_stack/discover/sles15/src/2024.03.00/build_0_on-login.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
#!/bin/bash

# Source the shared basics
source ./basics.sh

echo " === Make NDSL venv === "
cd $DSLSW_INSTALL_DIR
./python3/bin/python3 -m venv venv
source ./venv/bin/activate
pip install --upgrade setuptools pip
pip install -e $DSLSW_INSTALL_DIR/ndsl
pip install mpi4py cffi
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash

# Source the shared basics
source ./basics.v1.0.0.sh
source ./basics.sh

echo " === GDR Copy (requires kernel running on the box) === "
#cd $DSLSW_BASE/gdrcopy-$DSLSW_GDRCOPY_VER
Expand Down Expand Up @@ -63,13 +63,6 @@ cd $DSLSW_BASE/osu-micro-benchmarks-$DSLSW_OSUMICRO_VER
make -j32
make install

echo " === Lapack === "
cd $DSLSW_BASE/lapack-$DSLSW_LAPACK_VER
mkdir build
cd build
cmake .. -DCMAKE_INSTALL_PREFIX=$DSLSW_INSTALL_DIR/lapack
make -j32 install

echo " === Python === "
cd $DSLSW_BASE/Python-$DSLSW_PY_VER
./configure --prefix=$DSLSW_INSTALL_DIR/python3 --enable-shared --enable-optimizations
Expand All @@ -82,67 +75,73 @@ cd $DSLSW_BASE/serialbox-$DSLSW_SERIALBOX_VER
mkdir build
cd build
cmake -DCMAKE_INSTALL_PREFIX=$DSLSW_INSTALL_DIR/serialbox \
-DSERIALBOX_ENABLE_FORTRAN=ON \
-DSERIALBOX_EXAMPLES=OFF \
..
-DSERIALBOX_ENABLE_FORTRAN=ON \
-DSERIALBOX_EXAMPLES=OFF \
..
make -j32 install


echo " === Baselibs === "
cd $DSLSW_BASE/baselibs-$DSLSW_BASELIBS_VER
make ESMF_COMM=openmpi \
BUILD=ESSENTIALS \
ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \
--prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \
install

echo " === GNU gcc/gfortran/g++ with OpenACC and OpenMP Offload on NVIDIA GPUs === "
module rm comp/gcc/12.3.0
module rm nvidia/nvhpc-nompi/23.9
unset CC
unset CXX
unset FC

# Build assembler and linking tools
cd $DSLSW_BASE/gnu/nvptx-tools
./configure \
--with-cuda-driver-include=$CUDA_DIR/include \
--with-cuda-driver-lib=$CUDA_DIR/lib64 \
--prefix=$DSLSW_INSTALL_DIR/gnu
make || exit 1
make install || exit 1
cd ..

# Set up the GCC source tree
cd $DSLSW_BASE/gnu/gcc
ln -s ../nvptx-newlib/newlib newlib
cd ..
export target=$(gcc/config.guess)

# Build nvptx GCC
mkdir build-nvptx-gcc
cd build-nvptx-gcc
../gcc/configure \
--target=nvptx-none --with-build-time-tools=$DSLSW_INSTALL_DIR/gnu/nvptx-none/bin \
--enable-as-accelerator-for=$target \
--disable-sjlj-exceptions \
--enable-newlib-io-long-long \
--enable-languages="c,c++,fortran,lto" \
--prefix=$DSLSW_INSTALL_DIR/gnu
make -j`nproc` || exit 1
make install || exit 1
cd ..

# Build host GCC
mkdir build-host-gcc
cd build-host-gcc
../gcc/configure \
--enable-offload-targets=nvptx-none \
--with-cuda-driver-include=$CUDA_DIR/include \
--with-cuda-driver-lib=$CUDA_DIR/lib64 \
--disable-bootstrap \
--disable-multilib \
--enable-languages="c,c++,fortran,lto" \
--prefix=$DSLSW_INSTALL_DIR/gnu
make -j`nproc` || exit 1
make install || exit 1
cd ..
BUILD=ESSENTIALS \
ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \
prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \
install

if [ -z ${BUILD_GCC_OFFLOAD+x} ]; then
echo "Skip building offloaded GCC. Define BUILD_GCC_OFFLOAD to build."
else
echo " === GNU gcc/gfortran/g++ with OpenACC and OpenMP Offload on NVIDIA GPUs === "
module rm comp/gcc/12.3.0
module rm nvidia/nvhpc-nompi/23.9
unset CC
unset CXX
unset FC

# Build assembler and linking tools
cd $DSLSW_BASE/gnu/nvptx-tools
./configure \
--with-cuda-driver-include=$CUDA_DIR/include \
--with-cuda-driver-lib=$CUDA_DIR/lib64 \
--prefix=$DSLSW_INSTALL_DIR/gnu
make || exit 1
make install || exit 1
cd ..

# Set up the GCC source tree
cd $DSLSW_BASE/gnu/gcc
ln -s ../nvptx-newlib/newlib newlib
cd ..
export target=$(gcc/config.guess)

# Build nvptx GCC
mkdir build-nvptx-gcc
cd build-nvptx-gcc
../gcc/configure \
--target=nvptx-none --with-build-time-tools=$DSLSW_INSTALL_DIR/gnu/nvptx-none/bin \
--enable-as-accelerator-for=$target \
--disable-sjlj-exceptions \
--enable-newlib-io-long-long \
--enable-languages="c,c++,fortran,lto" \
--prefix=$DSLSW_INSTALL_DIR/gnu
make -j`nproc` || exit 1
make install || exit 1
cd ..

# Build host GCC
mkdir build-host-gcc
cd build-host-gcc
../gcc/configure \
--enable-offload-targets=nvptx-none \
--with-cuda-driver-include=$CUDA_DIR/include \
--with-cuda-driver-lib=$CUDA_DIR/lib64 \
--disable-bootstrap \
--disable-multilib \
--enable-languages="c,c++,fortran,lto" \
--prefix=$DSLSW_INSTALL_DIR/gnu
make -j`nproc` || exit 1
make install || exit 1
cd ..
fi

Original file line number Diff line number Diff line change
@@ -1,9 +1,11 @@
#!/bin/bash

source ./basics.v1.0.0.sh
source ./basics.sh

echo $DSLSW_INSTALL_DIR
echo `which $FC`
echo `which $CC`

echo $LD_LIBRARY_PATH

rm -rf build
Loading

0 comments on commit 07ee7b0

Please sign in to comment.