Merge pull request #862 from ndellingwood/master

Release 3.3.0
kokkos · Dec 19, 2020 · f64b1c5 · f64b1c5
2 parents 42defc5 + 439a8d7
commit f64b1c5
Show file tree

Hide file tree

Showing 267 changed files with 9,249 additions and 4,242 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,34 @@
 # Change Log
 
+## [3.3.00](https://github.com/kokkos/kokkos-kernels/tree/3.3.00) (2020-12-16)
+[Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.2.01...3.3.00)
+
+**Implemented enhancements:**
+- Add permanent RCM reordering interface, and a basic serial implementation [\#854](https://github.com/kokkos/kokkos/pull/#854)
+- Half\_t explicit conversions [\#849](https://github.com/kokkos/kokkos/pull/#849)
+- Add batched gemm performance tests [\#838](https://github.com/kokkos/kokkos/pull/#838)
+- Add HIP support to src and perf\_test [\#828](https://github.com/kokkos/kokkos/pull/#828)
+- Factor out coarsening [\#827](https://github.com/kokkos/kokkos/pull/#827)
+- Allow enabling/disabling components at configuration time [\#823](https://github.com/kokkos/kokkos/pull/#823)
+- HIP: CMake work on tests and ETI  [\#820](https://github.com/kokkos/kokkos/pull/#820)
+- HIP: KokkosBatched - hip specialization [\#812](https://github.com/kokkos/kokkos/pull/#812)
+- Distance-2 maximal independent set [\#801](https://github.com/kokkos/kokkos/pull/#801)
+- Use batched TRTRI & TRMM for Supernode-sptrsv setup [\#797](https://github.com/kokkos/kokkos/pull/#797)
+- Initial support for half precision [\#794](https://github.com/kokkos/kokkos/pull/#794)
+
+**Fixed bugs:**
+- Fix issue with HIP and Kokkos\_ArithTraits [\#844](https://github.com/kokkos/kokkos/pull/#844)
+- HIP: fixing round of issues on AMD [\#840](https://github.com/kokkos/kokkos/pull/#840)
+- Throw an exception if BLAS GESV is not enabled [\#837](https://github.com/kokkos/kokkos/pull/#837)
+- Fixes -Werror for gcc with c++20 [\#836](https://github.com/kokkos/kokkos/pull/#836)
+- Add fallback condition to use spmv\_native when cuSPARSE does not work [\#834](https://github.com/kokkos/kokkos/pull/#834)
+- Fix install testing refactor for inline builds [\#811](https://github.com/kokkos/kokkos/pull/#811)
+- HIP: fix ArithTraits to support HIP backend [\#809](https://github.com/kokkos/kokkos/pull/#809)
+- cuSPARSE 11: fix spgemm and spmv\_struct\_tunning compilation error [\#804](https://github.com/kokkos/kokkos/pull/#804)
+
+**Incompatibilities:**
+- Remove pre-3.0 deprecated code [\#825](https://github.com/kokkos/kokkos/pull/#825)
+
 ## [3.2.01](https://github.com/kokkos/kokkos-kernels/tree/3.2.01) (2020-11-17)
 [Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.2.00...3.2.01)
 
@@ -28,8 +57,8 @@
 - Nightly test failure: spgemm unit tests failing on White \(Power8\) [\#780](https://github.com/kokkos/kokkos-kernels/issues/780)
 - supernodal does not build with UVM enabled [\#633](https://github.com/kokkos/kokkos-kernels/issues/633)
 
-## [3.1.1](https://github.com/kokkos/kokkos-kernels/tree/3.1.1) (2020-05-04)
-[Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.1.00...3.1.1)
+## [3.1.01](https://github.com/kokkos/kokkos-kernels/tree/3.1.01) (2020-05-04)
+[Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.1.00...3.1.01)
 
 ** Fixed bugs:** 
 

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -24,8 +24,8 @@ IF(NOT KOKKOSKERNELS_HAS_TRILINOS)
     PROJECT(KokkosKernels CXX)
   ENDIF()
   SET(KokkosKernels_VERSION_MAJOR 3)
-  SET(KokkosKernels_VERSION_MINOR 2)
-  SET(KokkosKernels_VERSION_PATCH 1)
+  SET(KokkosKernels_VERSION_MINOR 3)
+  SET(KokkosKernels_VERSION_PATCH 0)
 ENDIF()
 
 IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0")
@@ -83,7 +83,8 @@ IF (KokkosKernels_INSTALL_TESTING)
 ELSE()
   # Regular build, not install testing
   # Do all the regular option processing
-  IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
+  IF (NOT KOKKOSKERNELS_HAS_TRILINOS AND NOT KOKKOSKERNELS_HAS_PARENT)
+   # This is a standalone build
    FIND_PACKAGE(Kokkos REQUIRED)
    MESSAGE(STATUS "Found Kokkos at ${Kokkos_DIR}")
    KOKKOS_CHECK(OPTIONS CUDA_UVM RETURN_VALUE KOKKOS_ENABLE_CUDA_UVM)
@@ -139,6 +140,15 @@ ELSE()
     BOOL
     "Whether to restrict testing to ETI types. Default: ON"
     )
+
+  KOKKOSKERNELS_ADD_OPTION(
+    ENABLED_COMPONENTS
+    "ALL"
+    STRING
+    "A list of components to enable in testing and building"
+    VALID_ENTRIES BATCHED BLAS GRAPH SPARSE ALL
+  )
+
   # ==================================================================
   # Enable Device Types for ETI (exec- + mem-space)
   # ==================================================================
@@ -159,7 +169,6 @@ ELSE()
   # Enable Layout Types for ETI
   # ==================================================================
   INCLUDE(cmake/kokkoskernels_eti_layouts.cmake)
-
   # ==================================================================
   # Enable Third Party Libraries
   # ==================================================================

diff --git a/cm_generate_makefile.bash b/cm_generate_makefile.bash
@@ -21,8 +21,18 @@ get_kokkos_device_list() {
   for DEVICE_ in $PARSE_DEVICES_LST
   do 
      UC_DEVICE=$(echo $DEVICE_ | tr "[:lower:]" "[:upper:]")
+     if [ "${UC_DEVICE}" == "CUDA" ]; then
+       WITH_CUDA_BACKEND=ON
+     fi
+     if [ "${UC_DEVICE}" == "HIP" ]; then
+       WITH_HIP_BACKEND=ON
+     fi
      KOKKOS_DEVICE_CMD="-DKokkos_ENABLE_${UC_DEVICE}=ON ${KOKKOS_DEVICE_CMD}"
   done
+  if [ "${WITH_CUDA_BACKEND}" == "ON" ] && [ "${WITH_HIP_BACKEND}" == "ON" ]; then
+     echo "Invalid configuration - Cuda and Hip cannot be simultaneously enabled"
+     exit
+  fi
 }
 
 get_kokkos_arch_list() {
@@ -59,6 +69,24 @@ get_kokkos_cuda_option_list() {
   done
 }
 
+get_kokkos_hip_option_list() {
+  echo parsing KOKKOS_HIP_OPTIONS=$KOKKOS_HIP_OPTIONS
+  KOKKOS_HIP_OPTION_CMD=
+  PARSE_HIP_LST=$(echo $KOKKOS_HIP_OPTIONS | tr "," "\n")
+  for HIP_ in $PARSE_HIP_LST
+  do
+     HIP_OPT_NAME=
+     if  [ "${HIP_}" == "rdc" ]; then
+        HIP_OPT_NAME=HIP_RELOCATABLE_DEVICE_CODE
+     else
+        echo "${HIP_} is not a valid hip option..."
+     fi
+     if [ "${HIP_OPT_NAME}" != "" ]; then
+        KOKKOS_HIP_OPTION_CMD="-DKokkos_ENABLE_${HIP_OPT_NAME}=ON ${KOKKOS_HIP_OPTION_CMD}"
+     fi
+  done
+}
+
 get_kokkos_option_list() {
   echo parsing KOKKOS_OPTIONS=$KOKKOS_OPTIONS
   KOKKOS_OPTION_CMD=
@@ -196,15 +224,21 @@ display_help_text() {
       echo "--prefix=/Install/Path:                       Path to install the KokkosKernels library."
       echo ""
       echo "--with-cuda[=/Path/To/Cuda]:                  Enable Cuda and set path to Cuda Toolkit."
+      echo "--with-hip[=/Path/To/Hip]:                    Enable Hip and set path to ROCM Toolkit."
       echo "--with-openmp:                                Enable OpenMP backend."
       echo "--with-pthread:                               Enable Pthreads backend."
       echo "--with-serial:                                Enable Serial backend."
       echo "--with-devices:                               Explicitly add a set of backends."
       echo ""
       echo "--arch=[OPT]:  Set target architectures. Options are:"
-      echo "               [AMD]"
+      echo "               [AMD: CPU]"
       echo "                 AMDAVX          = AMD CPU"
-      echo "                 EPYC            = AMD EPYC Zen-Core CPU"
+      echo "                 ZEN             = AMD Zen-Core CPU"
+      echo "                 ZEN2            = AMD Zen2-Core CPU"
+      echo "               [AMD: GPU]"
+      echo "                 VEGA900         = AMD GPU MI25 GFX900"
+      echo "                 VEGA906         = AMD GPU MI50/MI60 GFX906"
+      echo "                 VEGA908         = AMD GPU"
       echo "               [ARM]"
       echo "                 ARMV80          = ARMv8.0 Compatible CPU"
       echo "                 ARMV81          = ARMv8.1 Compatible CPU"
@@ -264,6 +298,8 @@ display_help_text() {
       echo "                                "
       echo "--with-cuda-options=[OPT]:    Additional options to CUDA:"
       echo "                                force_uvm, use_ldg, enable_lambda, rdc"
+      echo "--with-hip-options=[OPT]:     Additional options to HIP:"
+      echo "                                rdc"
       echo "--with-scalars=[SCALARS]:     Set scalars to be instantiated."
       echo "                                Options: float, double, complex_float, complex_double"
       echo "--with-ordinals=[ORDINALS]:   Set ordinals to be instantiated."
@@ -307,6 +343,10 @@ KOKKOS_MAKEINSTALL_J=4
 
 KERNELS_DEFAULT_ETI_OPTION=""
 
+# For tracking if Cuda and Hip devices are enabled simultaneously
+WITH_CUDA_BACKEND=OFF
+WITH_HIP_BACKEND=OFF
+
 while [[ $# > 0 ]]
 do
   key="$1"
@@ -340,6 +380,19 @@ do
       update_kokkos_devices Cuda
       CUDA_PATH="${key#*=}"
       ;;
+    --with-hip)
+      update_kokkos_devices Hip
+      HIP_PATH_HIPCC=$(command -v hipcc)
+      HIP_PATH=${HIP_PATH_HIPCC%/bin/hipcc}
+      ;;
+    # Catch this before '--with-hip*'
+    --with-hip-options*)
+      KOKKOS_HIP_OPTIONS="${key#*=}"
+      ;;
+    --with-hip*)
+      update_kokkos_devices Hip
+      HIP_PATH="${key#*=}"
+      ;;
     --with-openmp)
       update_kokkos_devices OpenMP
       ;;
@@ -606,6 +659,7 @@ get_kokkos_device_list
 get_kokkos_option_list
 get_kokkos_arch_list
 get_kokkos_cuda_option_list
+get_kokkos_hip_option_list
 
 get_kernels_scalar_list
 get_kernels_ordinals_list
@@ -655,9 +709,9 @@ cd ${KOKKOS_INSTALL_PATH}
 
 # Configure kokkos
 echo ""
-echo cmake $COMPILER_CMD  -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
+echo cmake $COMPILER_CMD  -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} ${KOKKOS_HIP_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
 echo ""
-cmake $COMPILER_CMD  -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS//\"}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
+cmake $COMPILER_CMD  -DCMAKE_CXX_FLAGS="${KOKKOS_CXXFLAGS//\"}" -DCMAKE_EXE_LINKER_FLAGS="${KOKKOS_LDFLAGS//\"}" -DCMAKE_INSTALL_PREFIX=${KOKKOS_INSTALL_PATH} ${KOKKOS_DEVICE_CMD} ${KOKKOS_ARCH_CMD} -DKokkos_ENABLE_TESTS=${KOKKOS_DO_TESTS} -DKokkos_ENABLE_EXAMPLES=${KOKKOS_DO_EXAMPLES} ${KOKKOS_OPTION_CMD} ${KOKKOS_CUDA_OPTION_CMD} ${KOKKOS_HIP_OPTION_CMD} -DCMAKE_VERBOSE_MAKEFILE=ON -DCMAKE_CXX_EXTENSIONS=OFF ${STANDARD_CMD} ${KOKKOS_BUILDTYPE_CMD} ${KOKKOS_BC_CMD} ${KOKKOS_HWLOC_CMD} ${KOKKOS_HWLOC_PATH_CMD} ${KOKKOS_MEMKIND_CMD} ${KOKKOS_MEMKIND_PATH_CMD} ${KOKKOS_PATH}
 
 # Install kokkos library
 make install -j $KOKKOS_MAKEINSTALL_J

diff --git a/cmake/KokkosKernelsConfig.cmake.in b/cmake/KokkosKernelsConfig.cmake.in
@@ -11,6 +11,7 @@ find_dependency(Kokkos HINTS @Kokkos_DIR@)
 
 SET(Kokkos_ENABLE_OPENMP  @Kokkos_ENABLE_OPENMP@)
 SET(Kokkos_ENABLE_CUDA    @Kokkos_ENABLE_CUDA@)
+SET(Kokkos_ENABLE_HIP     @Kokkos_ENABLE_HIP@)
 SET(Kokkos_ENABLE_PTHREAD @Kokkos_ENABLE_PTHREAD@) 
 SET(Kokkos_ENABLE_SERIAL  @Kokkos_ENABLE_SERIAL@)
 

diff --git a/cmake/KokkosKernels_config.h.in b/cmake/KokkosKernels_config.h.in
@@ -34,6 +34,9 @@
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_CUDA
 #cmakedefine KOKKOSKERNELS_INST_MEMSPACE_CUDASPACE
 #cmakedefine KOKKOSKERNELS_INST_MEMSPACE_CUDAUVMSPACE
+/* Whether to build kernels for execution space Kokkos::Experimental::HIP */
+#cmakedefine KOKKOSKERNELS_INST_EXECSPACE_HIP
+#cmakedefine KOKKOSKERNELS_INST_MEMSPACE_HIPSPACE
 /* Whether to build kernels for execution space Kokkos::OpenMP */
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_OPENMP
 /* Whether to build kernels for execution space Kokkos::Threads */