Merge commit '8dfc4228cfdddb2398ecd6bb02eeea592b48c203'

sw_stack/discover/sles15/HISTORY.md

@@ -6,20 +6,34 @@ All versions of the software for a given version are saved in `basics.sh`.
 `build` directory is the throwaway directory where everything is downloaded then built.
 `install` is saves all library and executable once build is done.
 
-Last edit: _December 29th 2023_
+Last edit: _March 18th 2024_
 
-## v1.0.0
+## v2024.03.00
 
-### OpenMPI
+### Options
+
+`BUILD_GCC_OFFLOAD`: builds an offload ready GCC 12.2
+
+### Stack
 
 We build OpenMPI throught the UCX layer with cuda-enabled and GRDCopy and GPUDirect on.
 
 - GDRCOPY: Must be installed on the compiling machine as a kernel module.
 - GCC: 12.3.0 [^1] (via `comp/gcc/12.3.0` on discover)
 - CUDA (via `nvhpc`): 12.2 [^2] (via `nvidia/nvhpc-nompi/23.9` on discover)
+- MKL: 2023.2.0 (via `lib/mkl/2023.2.0` on discover)
 - UCX: 1.15.0
 - OpenMPI: 4.1.6 [^3]
 - OSU-MICROBENCHMARK: 7.3
+- Boost headers: 1.76.0
+- BASELIBS: 7.17.1 [^4]
+- Python: 3.11.7
+- NDSL: 2024.03.01
+- Serialbox: 2.6.2-unreleased [^5]
+
+When defining `BUILD_GCC_OFFLOAD`:
+
+- GCC with offload: 12.2.0
 
 Test of the stack can be done via the `osu-microbenchmark` with latency & bandwith saved in `osu-bench.sh`.
 
@@ -28,20 +42,5 @@ _Note:_
 - [^1]: `gcc-13.2.0` fails during GEOS with an internal compiler error
 - [^2]: `nvhpc` ships with a prebuilt `openmpi` which can cause issues. Make sure to load the `nompi` module.
 - [^3]: `openmpi-5.0.0` fails at GEOS runtime on a call to `libxml2` that does a divide by zero (triggering a sigfpe). We revert to `4.1.6`.
-
-### Baselibs
-
-- LAPACK/BLAS: 3.11.0
-- BASELIBS: 7.14.1
-
-### Python
-
-- Python: 3.8.10 [^4]
-
-### Serialbox
-
-- Latest stable is 2.6.1. Development is over.
-
-_Note:_
-
-- [^4]: `3.10.12` leads to failure in DaCe parsing.
+- [^4]: As per GEOS 11.5.2 @env
+- [^5]: Latest stable is 2.6.1. Development is crawlingly slow but some fixes for GCC 12+ are in `main`

sw_stack/discover/sles15/TODO.md

@@ -1,4 +1,3 @@
 # TODO
 
 - Test the build of baselibs
-- Build GCC with OpenACC & OpenMP offloading

sw_stack/discover/sles15/modulefiles/DSLwork/1.0.0.lua → sw_stack/discover/sles15/modulefiles/SMTStack/2024.03.00-no-ndsl.lua

@@ -1,7 +1,8 @@
 load("comp/gcc/12.3.0")
-load("nvidia/nvhpc-nompi/23.9")
+load("lib/mkl/2023.2.0")
+load("nvidia/nvhpc-byo-compiler/23.9")
 
-local install_dir  = "/discover/nobackup/projects/geosongpu/sw_sles15/v1.0.0/install/"
+local install_dir  = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.03.00/install"
 
 -- UCX --
 local ucx_pkgdir = pathJoin(install_dir, "ucx")
@@ -25,7 +26,12 @@ setenv("OMP_STACKSIZE","1G")
 setenv("OMPI_MCA_mca_base_component_show_load_errors","0")
 setenv("PMIX_MCA_mca_base_component_show_load_errors","0")
 
+-- BOOST HEADERS (as expected by gt4py) --
+local boost_pkgdir = pathJoin(install_dir, "boost")
+setenv("BOOST_ROOT", boost_pkgdir)
+
 -- Python 3 --
 local py_pkgdir = pathJoin(install_dir, "python3")
 prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
 prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib"))
+prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64"))

sw_stack/discover/sles15/modulefiles/SMTStack/2024.03.00.lua

@@ -0,0 +1,42 @@
+load("comp/gcc/12.3.0")
+load("lib/mkl/2023.2.0")
+load("nvidia/nvhpc-byo-compiler/23.9")
+
+local install_dir  = "/discover/nobackup/projects/geosongpu/sw_sles15/live/src/2024.03.00/install"
+
+-- UCX --
+local ucx_pkgdir = pathJoin(install_dir, "ucx")
+prepend_path("LD_LIBRARY_PATH",pathJoin(ucx_pkgdir,"lib"))
+
+-- OMPI --
+local ompi_pkgdir = pathJoin(install_dir, "ompi")
+
+setenv("M_MPI_ROOT",ompi_pkgdir)
+setenv("OPENMPI",ompi_pkgdir)
+setenv("MPI_HOME",ompi_pkgdir)
+
+prepend_path("PATH",pathJoin(ompi_pkgdir,"bin"))
+prepend_path("LD_LIBRARY_PATH",pathJoin(ompi_pkgdir,"lib"))
+prepend_path("INCLUDE",pathJoin(ompi_pkgdir,"include"))
+prepend_path("MANPATH",pathJoin(ompi_pkgdir,"share/man"))
+
+setenv("OMPI_MCA_orte_tmpdir_base","/tmp")
+setenv("TMPDIR","/tmp")
+setenv("OMP_STACKSIZE","1G")
+setenv("OMPI_MCA_mca_base_component_show_load_errors","0")
+setenv("PMIX_MCA_mca_base_component_show_load_errors","0")
+
+-- BOOST HEADERS (as expected by gt4py) --
+local boost_pkgdir = pathJoin(install_dir, "boost")
+setenv("BOOST_ROOT", boost_pkgdir)
+
+-- Python 3 --
+local py_pkgdir = pathJoin(install_dir, "python3")
+prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
+prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib"))
+prepend_path("LD_LIBRARY_PATH",pathJoin(py_pkgdir,"lib64"))
+
+-- Load venv --
+local py_pkgdir = pathJoin(install_dir, "venv")
+prepend_path("PATH",pathJoin(py_pkgdir,"bin"))
+setenv("VIRTUAL_ENV", py_pkgdir)

sw_stack/discover/sles15/src/2024.03.00/basics.sh

@@ -0,0 +1,37 @@
+#!/bin/bash
+
+export DSLSW_VERSION="2024.03.00"
+echo "DSL Software Stack v${DSLSW_VERSION}"
+
+# Version
+export DSLSW_GDRCOPY_VER=2.3
+export DSLSW_OMPI_MAJOR_VER=4.1
+export DSLSW_OMPI_VER=${DSLSW_OMPI_MAJOR_VER}.6
+export DSLSW_UCX_VER=1.15.0
+export DSLSW_CUDA_VER=12.2
+export DSLSW_OSUMICRO_VER=7.3
+export DSLSW_LAPACK_VER=3.11.0
+export DSLSW_PY_VER=3.11.7
+export DSLSW_BASELIBS_VER=7.17.1
+export DSLSW_SERIALBOX_VER=2.6.2-unreleased
+export DSLSW_SERIALBOX_SHA=88ac4e4dfc824953d068fe63c8e7b3dd9560a914
+export DSLSW_GNU_VER=12.2.0
+export DSLSW_NDSL_VER=2024.03.01
+export DSLSW_BOOST_VER=1.76.0
+export DSLSW_BOOST_VER_STR=1_76_0 
+
+# Base directory & versioning
+export DSLSW_BASE=$PWD/build
+mkdir -p $DSLSW_BASE
+export DSLSW_INSTALL_DIR=$PWD/install
+mkdir -p $DSLSW_INSTALL_DIR
+
+# Modules
+module use -a /discover/nobackup/projects/geosongpu/sw_sles15/live/modulefiles/
+module load SMTStack/${DSLSW_VERSION}
+CUDA_DIR=/usr/local/other/nvidia/hpc_sdk/Linux_x86_64/23.9/cuda/
+
+# Enforce proper compilers
+export FC=gfortran
+export CC=gcc
+export CXX=g++

sw_stack/discover/sles15/src/2024.03.00/build_0_on-login.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Source the shared basics
+source ./basics.sh
+
+echo " === Make NDSL venv === "
+cd $DSLSW_INSTALL_DIR
+./python3/bin/python3 -m venv venv
+source ./venv/bin/activate
+pip install --upgrade setuptools pip
+pip install -e $DSLSW_INSTALL_DIR/ndsl
+pip install mpi4py cffi

sw_stack/discover/sles15/src/1.0.0/build.sh → sw_stack/discover/sles15/src/2024.03.00/build_1_on-node.sh

@@ -1,7 +1,7 @@
 #!/bin/bash
 
 # Source the shared basics
-source ./basics.v1.0.0.sh
+source ./basics.sh
 
 echo " === GDR Copy (requires kernel running on the box) === "
 #cd $DSLSW_BASE/gdrcopy-$DSLSW_GDRCOPY_VER
@@ -63,13 +63,6 @@ cd $DSLSW_BASE/osu-micro-benchmarks-$DSLSW_OSUMICRO_VER
 make -j32
 make install
 
-echo " === Lapack === "
-cd $DSLSW_BASE/lapack-$DSLSW_LAPACK_VER
-mkdir build
-cd build
-cmake .. -DCMAKE_INSTALL_PREFIX=$DSLSW_INSTALL_DIR/lapack
-make -j32 install
-
 echo " === Python === "
 cd $DSLSW_BASE/Python-$DSLSW_PY_VER
 ./configure --prefix=$DSLSW_INSTALL_DIR/python3 --enable-shared --enable-optimizations
@@ -82,67 +75,73 @@ cd $DSLSW_BASE/serialbox-$DSLSW_SERIALBOX_VER
 mkdir build
 cd build
 cmake -DCMAKE_INSTALL_PREFIX=$DSLSW_INSTALL_DIR/serialbox \
-      -DSERIALBOX_ENABLE_FORTRAN=ON \
-      -DSERIALBOX_EXAMPLES=OFF \
-      ..
+    -DSERIALBOX_ENABLE_FORTRAN=ON \
+    -DSERIALBOX_EXAMPLES=OFF \
+    ..
 make -j32 install
 
 
 echo " === Baselibs === "
+cd $DSLSW_BASE/baselibs-$DSLSW_BASELIBS_VER
 make ESMF_COMM=openmpi \
-      BUILD=ESSENTIALS \
-      ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \
-      --prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \
-      install
-
-echo " === GNU gcc/gfortran/g++ with OpenACC and OpenMP Offload on NVIDIA GPUs === "
-module rm comp/gcc/12.3.0
-module rm nvidia/nvhpc-nompi/23.9
-unset CC
-unset CXX
-unset FC
-
-# Build assembler and linking tools
-cd $DSLSW_BASE/gnu/nvptx-tools
-./configure \
-    --with-cuda-driver-include=$CUDA_DIR/include \
-    --with-cuda-driver-lib=$CUDA_DIR/lib64 \
-    --prefix=$DSLSW_INSTALL_DIR/gnu
-make || exit 1
-make install || exit 1
-cd ..
-
-# Set up the GCC source tree
-cd $DSLSW_BASE/gnu/gcc
-ln -s ../nvptx-newlib/newlib newlib
-cd ..
-export target=$(gcc/config.guess)
-
-# Build nvptx GCC
-mkdir build-nvptx-gcc
-cd build-nvptx-gcc
-../gcc/configure \
-    --target=nvptx-none --with-build-time-tools=$DSLSW_INSTALL_DIR/gnu/nvptx-none/bin \
-    --enable-as-accelerator-for=$target \
-    --disable-sjlj-exceptions \
-    --enable-newlib-io-long-long \
-    --enable-languages="c,c++,fortran,lto" \
-    --prefix=$DSLSW_INSTALL_DIR/gnu
-make -j`nproc` || exit 1
-make install || exit 1
-cd ..
-
-# Build host GCC
-mkdir build-host-gcc
-cd  build-host-gcc
-../gcc/configure \
-    --enable-offload-targets=nvptx-none \
-    --with-cuda-driver-include=$CUDA_DIR/include \
-    --with-cuda-driver-lib=$CUDA_DIR/lib64 \
-    --disable-bootstrap \
-    --disable-multilib \
-    --enable-languages="c,c++,fortran,lto" \
-    --prefix=$DSLSW_INSTALL_DIR/gnu
-make -j`nproc` || exit 1
-make install || exit 1
-cd ..
+    BUILD=ESSENTIALS \
+    ALLOW_ARGUMENT_MISMATCH=-fallow-argument-mismatch \
+    prefix=$DSLSW_INSTALL_DIR/baselibs-$DSLSW_BASELIBS_VER/install/x86_64-pc-linux-gnu/Linux \
+    install
+
+if [ -z ${BUILD_GCC_OFFLOAD+x} ]; then
+    echo "Skip building offloaded GCC. Define BUILD_GCC_OFFLOAD to build."
+else
+    echo " === GNU gcc/gfortran/g++ with OpenACC and OpenMP Offload on NVIDIA GPUs === "
+    module rm comp/gcc/12.3.0
+    module rm nvidia/nvhpc-nompi/23.9
+    unset CC
+    unset CXX
+    unset FC
+
+    # Build assembler and linking tools
+    cd $DSLSW_BASE/gnu/nvptx-tools
+    ./configure \
+        --with-cuda-driver-include=$CUDA_DIR/include \
+        --with-cuda-driver-lib=$CUDA_DIR/lib64 \
+        --prefix=$DSLSW_INSTALL_DIR/gnu
+    make || exit 1
+    make install || exit 1
+    cd ..
+
+    # Set up the GCC source tree
+    cd $DSLSW_BASE/gnu/gcc
+    ln -s ../nvptx-newlib/newlib newlib
+    cd ..
+    export target=$(gcc/config.guess)
+
+    # Build nvptx GCC
+    mkdir build-nvptx-gcc
+    cd build-nvptx-gcc
+    ../gcc/configure \
+        --target=nvptx-none --with-build-time-tools=$DSLSW_INSTALL_DIR/gnu/nvptx-none/bin \
+        --enable-as-accelerator-for=$target \
+        --disable-sjlj-exceptions \
+        --enable-newlib-io-long-long \
+        --enable-languages="c,c++,fortran,lto" \
+        --prefix=$DSLSW_INSTALL_DIR/gnu
+    make -j`nproc` || exit 1
+    make install || exit 1
+    cd ..
+
+    # Build host GCC
+    mkdir build-host-gcc
+    cd  build-host-gcc
+    ../gcc/configure \
+        --enable-offload-targets=nvptx-none \
+        --with-cuda-driver-include=$CUDA_DIR/include \
+        --with-cuda-driver-lib=$CUDA_DIR/lib64 \
+        --disable-bootstrap \
+        --disable-multilib \
+        --enable-languages="c,c++,fortran,lto" \
+        --prefix=$DSLSW_INSTALL_DIR/gnu
+    make -j`nproc` || exit 1
+    make install || exit 1
+    cd ..
+fi
+

sw_stack/discover/sles15/src/1.0.0/check.sh → sw_stack/discover/sles15/src/2024.03.00/check.sh

@@ -1,9 +1,11 @@
 #!/bin/bash
 
-source ./basics.v1.0.0.sh
+source ./basics.sh
 
 echo $DSLSW_INSTALL_DIR
 echo `which $FC`
 echo `which $CC`
 
 echo $LD_LIBRARY_PATH
+
+rm -rf build