Merge pull request #941 from ndellingwood/rc-3.4.0

Release 3.4.0

.jenkins/nightly.groovy

@@ -0,0 +1,41 @@
+pipeline {
+    agent none
+
+    stages {
+        stage('HIP-ROCm-3.10-C++14') {
+            agent {
+                dockerfile {
+                    filename 'Dockerfile.hip'
+                    dir 'scripts/docker'
+                    additionalBuildArgs '--build-arg BASE=rocm/dev-ubuntu-20.04:3.10'
+                    label 'rocm-docker && vega'
+                    args '-v /tmp/ccache.kokkos:/tmp/ccache --device=/dev/kfd --device=/dev/dri --security-opt seccomp=unconfined --group-add video --env HIP_VISIBLE_DEVICES=$HIP_VISIBLE_DEVICES'
+                }
+            }
+            steps {
+                sh '''rm -rf kokkos &&
+                      git clone -b develop https://github.com/kokkos/kokkos.git && cd kokkos && \
+                      mkdir build && cd build && \
+                      cmake \
+                        -DCMAKE_CXX_COMPILER=hipcc \
+                        -DKokkos_ENABLE_HIP=ON \
+                        -DKokkos_ARCH_VEGA906=ON \
+                        .. && \
+                       make -j8 && make install && \
+                       cd ../.. && rm -rf kokkos'''
+                sh '''rm -rf build && mkdir -p build && cd build && \
+                      cmake \
+                        -DCMAKE_BUILD_TYPE=RelWithDebInfo \
+                        -DCMAKE_CXX_COMPILER=hipcc \
+                        -DKokkosKernels_ENABLE_TESTS=ON \
+                        -DKokkosKernels_ENABLE_EXAMPLES=ON \
+                        -DKokkos_ENABLE_HIP=ON \
+                        -DKokkosKernels_INST_DOUBLE=ON \
+                        -DKokkosKernels_INST_ORDINAL_INT=ON \
+                        -DKokkosKernels_INST_OFFSET_INT=ON \
+                      .. && \
+                      make -j8 && ctest --verbose'''
+            }
+        }
+    }
+}

BUILD.md

@@ -125,6 +125,12 @@ endif()
 * CUSPARSE_LIBRARY_DIRS: STRING
   * Optional override for the library directories that comprise TPL CUSPARSE.
   * Default: None. Default common library locations will be searched
+* ARMPL_LIBRARIES: STRING
+  * Optional override for the libraries that comprise TPL ARMPL.
+  * Default: None. Default common library names will be searched
+* ARMPL_LIBRARY_DIRS: STRING
+  * Optional override for the library directories that comprise TPL ARMPL.
+  * Default: None. Default common library locations will be searched
 * KokkosKernels_BLAS_ROOT: PATH
   * Location of BLAS install root.
   * Default: None or the value of the environment variable BLAS_ROOT if set
@@ -161,6 +167,9 @@ endif()
 * KokkosKernels_ENABLE_TPL_MKL: BOOL
   * Whether to enable MKL
   * Default: OFF
+* KokkosKernels_ENABLE_TPL_ARMPL: BOOL
+  * Whether to enable ARMPL
+  * Default: OFF
 * KokkosKernels_ETI_ONLY: BOOL
   * Whether to restrict availability of kernels to ETI types only. Turning this on guarantees that kernels are never built inside of object files which simply call KokkosKernels functions.
   * Default: OFF

CHANGELOG.md

@@ -1,5 +1,40 @@
 # Change Log
 
+## [3.4.00](https://github.com/kokkos/kokkos-kernels/tree/3.4.00) (2021-04-25)
+[Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.3.01...3.4.00)
+
+**Features:**
+- SYCL: adding ETI and CMake logic for SYCL backend [\#924](https://github.com/kokkos/kokkos-kernels/pull/924)
+
+**Implemented enhancements Algorithms and Archs:**
+- Two-stage GS: add damping factors [\#921](https://github.com/kokkos/kokkos-kernels/pull/921)
+- Supernodal SpTRSV, improve symbolic performance [\#899](https://github.com/kokkos/kokkos-kernels/pull/899)
+- Add MKL SpMV wrapper [\#895](https://github.com/kokkos/kokkos-kernels/pull/895)
+- Serial code path for spmv [\#893](https://github.com/kokkos/kokkos-kernels/pull/893)
+
+**Implemented enhancements BuildSystem:**
+- Cmake: Update ArmPL support [\#901](https://github.com/kokkos/kokkos-kernels/pull/901)
+- Cmake: Add ARMPL TPL support [\#880](https://github.com/kokkos/kokkos-kernels/pull/880)
+- IntelClang guarding __assume_aligned with !defined(__clang__) [\#878](https://github.com/kokkos/kokkos-kernels/pull/878)
+
+**Implemented enhancements Other:**
+- Add static_assert/throw in batched eigendecomp [\#931](https://github.com/kokkos/kokkos-kernels/pull/931)
+- Workaround using new/delete in kernel code [\#925](https://github.com/kokkos/kokkos-kernels/pull/925)
+- Blas perf_test updates [\#892](https://github.com/kokkos/kokkos-kernels/pull/892)
+
+**Fixed bugs:**
+- Fix ctor CrsMat mirror with CrsGraph mirror [\#918](https://github.com/kokkos/kokkos-kernels/pull/918)
+- Fix nrm1, removed cublas nrminf, improved blas tests [\#915](https://github.com/kokkos/kokkos-kernels/pull/915)
+- Fix and testing coverage mainly in graph coarsening [\#910](https://github.com/kokkos/kokkos-kernels/pull/910)
+- Fix KokkosSparse for nightly test failure [\#898](https://github.com/kokkos/kokkos-kernels/pull/898)
+- Fix view types across ternary operator [\#894](https://github.com/kokkos/kokkos-kernels/pull/894)
+- Make work_view_t typedef consistent [\#885](https://github.com/kokkos/kokkos-kernels/pull/885)
+- Fix supernodal SpTRSV build with serial+openmp+cuda [\#884](https://github.com/kokkos/kokkos-kernels/pull/884)
+- Construct SpGEMM C with correct ncols [\#883](https://github.com/kokkos/kokkos-kernels/pull/883)
+- Matrix Converter: fixing issue with deallocation after Kokkos::fininalize [\#882](https://github.com/kokkos/kokkos-kernels/pull/882)
+- Fix >1024 team size error in sort_crs_* [\#872](https://github.com/kokkos/kokkos-kernels/pull/872)
+- Fixing seg fault with empty matrix in kspiluk [\#871](https://github.com/kokkos/kokkos-kernels/pull/871)
+
 ## [3.3.01](https://github.com/kokkos/kokkos-kernels/tree/3.3.01) (2021-01-18)
 [Full Changelog](https://github.com/kokkos/kokkos-kernels/compare/3.3.00...3.3.01)
 

CMakeLists.txt

@@ -24,8 +24,8 @@ IF(NOT KOKKOSKERNELS_HAS_TRILINOS)
     PROJECT(KokkosKernels CXX)
   ENDIF()
   SET(KokkosKernels_VERSION_MAJOR 3)
-  SET(KokkosKernels_VERSION_MINOR 3)
-  SET(KokkosKernels_VERSION_PATCH 1)
+  SET(KokkosKernels_VERSION_MINOR 4)
+  SET(KokkosKernels_VERSION_PATCH 0)
 ENDIF()
 
 IF(${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.12.0")
@@ -196,6 +196,10 @@ ELSE()
   MESSAGE("")
   # Skip building Kokkos Kernels if we are doing an installation test
   ADD_SUBDIRECTORY(src)
+  IF(KokkosKernels_ENABLE_INSTALL_TEST)
+    ADD_SUBDIRECTORY(install_test)
+    MESSAGE("The install test has been enabled, you will need to peform: make install before running the tests otherwise install_test will fail")
+  ENDIF()
   KOKKOSKERNELS_ADD_TEST_DIRECTORIES(test_common)
   KOKKOSKERNELS_ADD_TEST_DIRECTORIES(perf_test)
   KOKKOSKERNELS_ADD_TEST_DIRECTORIES(unit_test)

CheckHostBlasReturnComplex.cmake

@@ -5,7 +5,12 @@ FUNCTION(CHECK_HOST_BLAS_RETURN_COMPLEX VARNAME)
   IF (KOKKOSKERNELS_HAS_TRILINOS)
     SET(CMAKE_REQUIRED_LIBRARIES ${TPL_BLAS_LIBRARIES})
   ELSE()
-    SET(CMAKE_REQUIRED_LIBRARIES ${BLAS_LIBRARIES})
+    # For TPLs, just pull out the required libraries from the target properies.
+    IF (KOKKOSKERNELS_ENABLE_TPL_ARMPL)
+      GET_TARGET_PROPERTY(CMAKE_REQUIRED_LIBRARIES KokkosKernels::ARMPL INTERFACE_LINK_LIBRARIES)
+    ELSE()
+      SET(CMAKE_REQUIRED_LIBRARIES ${BLAS_LIBRARIES})
+    ENDIF()
   ENDIF()
 
   SET(SOURCE

cm_generate_makefile.bash

@@ -225,6 +225,8 @@ display_help_text() {
       echo ""
       echo "--with-cuda[=/Path/To/Cuda]:                  Enable Cuda and set path to Cuda Toolkit."
       echo "--with-hip[=/Path/To/Hip]:                    Enable Hip and set path to ROCM Toolkit."
+      echo "--with-openmptarget:                          Enable OpenMPTarget backend."
+      echo "--with-sycl:                                  Enable Sycl backend."
       echo "--with-openmp:                                Enable OpenMP backend."
       echo "--with-pthread:                               Enable Pthreads backend."
       echo "--with-serial:                                Enable Serial backend."
@@ -313,7 +315,7 @@ display_help_text() {
       echo "--with-tpls=[TPLS]:           Set tpls to be instantiated (Proper support requies that appropriate compiler and device must be enabled)."
       echo "                              This may require providing paths and the library name if using custom installs not on a default path"
       echo "                              that CMake searches"
-      echo "                                Options: blas, mkl, cublas, cusparse, magma"
+      echo "                                Options: blas, mkl, cublas, cusparse, magma, armpl"
       echo "--user-blas-path=[PATH]:      Set path to location of user-specified BLAS library."
       echo "--user-blas-lib=[LIB]:        Library name of desired BLAS install."
       echo "                                Example: For the typical \"libblas.a\" provide \"blas\""
@@ -396,6 +398,12 @@ do
     --with-openmp)
       update_kokkos_devices OpenMP
       ;;
+    --with-openmptarget)
+      update_kokkos_devices OpenMPTarget
+      ;;
+    --with-sycl)
+      update_kokkos_devices Sycl
+      ;;
     --with-pthread)
       update_kokkos_devices Pthread
       ;;
@@ -569,7 +577,7 @@ done
 if [ "$KOKKOS_CXX_STANDARD" == "" ]; then
     STANDARD_CMD=
 else
-    STANDARD_CMD=-DKokkos_CXX_STANDARD=${KOKKOS_CXX_STANDARD}
+    STANDARD_CMD=-DCMAKE_CXX_STANDARD=${KOKKOS_CXX_STANDARD}
 fi
 
 if [ "$COMPILER" == "" ]; then

cmake/Dependencies.cmake

@@ -1,5 +1,5 @@
 TRIBITS_PACKAGE_DEFINE_DEPENDENCIES(
   LIB_REQUIRED_PACKAGES KokkosCore KokkosContainers KokkosAlgorithms 
-  LIB_OPTIONAL_TPLS quadmath MKL BLAS LAPACK CUSPARSE MAGMA SUPERLU CHOLMOD LAPACKE CBLAS
+  LIB_OPTIONAL_TPLS quadmath MKL BLAS LAPACK CUSPARSE MAGMA SUPERLU CHOLMOD LAPACKE CBLAS ARMPL
   TEST_OPTIONAL_TPLS yaml-cpp
 )

cmake/KokkosKernelsConfig.cmake.in

@@ -12,6 +12,7 @@ find_dependency(Kokkos HINTS @Kokkos_DIR@)
 SET(Kokkos_ENABLE_OPENMP  @Kokkos_ENABLE_OPENMP@)
 SET(Kokkos_ENABLE_CUDA    @Kokkos_ENABLE_CUDA@)
 SET(Kokkos_ENABLE_HIP     @Kokkos_ENABLE_HIP@)
+SET(Kokkos_ENABLE_SYCL    @Kokkos_ENABLE_SYCL@)
 SET(Kokkos_ENABLE_PTHREAD @Kokkos_ENABLE_PTHREAD@) 
 SET(Kokkos_ENABLE_SERIAL  @Kokkos_ENABLE_SERIAL@)
 

cmake/KokkosKernels_config.h.in

@@ -37,6 +37,10 @@
 /* Whether to build kernels for execution space Kokkos::Experimental::HIP */
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_HIP
 #cmakedefine KOKKOSKERNELS_INST_MEMSPACE_HIPSPACE
+/* Whether to build kernels for execution space Kokkos::Experimental::SYCL */
+#cmakedefine KOKKOSKERNELS_INST_EXECSPACE_SYCL
+#cmakedefine KOKKOSKERNELS_INST_MEMSPACE_SYCLSPACE
+#cmakedefine KOKKOSKERNELS_INST_MEMSPACE_SYCLSHAREDSPACE
 /* Whether to build kernels for execution space Kokkos::OpenMP */
 #cmakedefine KOKKOSKERNELS_INST_EXECSPACE_OPENMP
 /* Whether to build kernels for execution space Kokkos::Threads */
@@ -102,11 +106,14 @@
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_LAPACKE
 /* METIS */
 #cmakedefine KOKKOSKERNELS_ENABLE_TPL_METIS
+/* ARMPL */
+#cmakedefine KOKKOSKERNELS_ENABLE_TPL_ARMPL
 
 #cmakedefine KOKKOSKERNELS_ENABLE_SUPERNODAL_SPTRSV
 
-/* if MKL, BLAS is also defined */
-#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) 
+/* if MKL or ARMPL, BLAS is also defined */
+#if defined(KOKKOSKERNELS_ENABLE_TPL_MKL) ||\
+    defined(KOKKOSKERNELS_ENABLE_TPL_ARMPL)
 #if !defined(KOKKOSKERNELS_ENABLE_TPL_BLAS)
 #define KOKKOSKERNELS_ENABLE_TPL_BLAS
 #endif

cmake/Modules/FindTPLARMPL.cmake

@@ -0,0 +1,47 @@
+# Both the armpl_mp and armpl libraries define the same public symbol names.
+# In order to link against the openmp  armpl symbols, instruct cmake to link against armpl_mp.
+# In order to link against the default armpl symbols, instruct cmake to link against armpl.
+IF(KOKKOSKERNELS_INST_EXECSPACE_OPENMP)
+  SET(ARMPL_LIB armpl_mp)
+ELSE()
+  SET(ARMPL_LIB armpl)
+ENDIF()
+
+IF (ARMPL_LIBRARY_DIRS AND ARMPL_LIBRARIES)
+  KOKKOSKERNELS_FIND_IMPORTED(ARMPL INTERFACE LIBRARIES ${ARMPL_LIBRARIES} LIBRARY_PATHS ${ARMPL_LIBRARY_DIRS})
+ELSEIF (ARMPL_LIBRARIES)
+  KOKKOSKERNELS_FIND_IMPORTED(ARMPL INTERFACE LIBRARIES ${ARMPL_LIBRARIES})
+ELSEIF (ARMPL_LIBRARY_DIRS)
+  KOKKOSKERNELS_FIND_IMPORTED(ARMPL INTERFACE LIBRARIES amath ${ARMPL_LIB} LIBRARY_PATHS ${ARMPL_LIBRARY_DIRS})
+ELSEIF (DEFINED ENV{ARMPL_DIR})
+  SET(ARMPL_ROOT $ENV{ARMPL_DIR})
+  KOKKOSKERNELS_FIND_IMPORTED(ARMPL INTERFACE
+    LIBRARIES
+      amath
+      ${ARMPL_LIB}
+    LIBRARY_PATHS
+      ${ARMPL_ROOT}/lib
+    HEADERS
+      armpl.h
+    HEADER_PATHS
+      ${ARMPL_ROOT}/include
+  )
+ELSE()
+  FIND_PACKAGE(ARMPL REQUIRED)
+  KOKKOSKERNELS_CREATE_IMPORTED_TPL(ARMPL INTERFACE LINK_LIBRARIES ${ARMPL_LIBRARIES})
+ENDIF()
+
+TRY_COMPILE(KOKKOSKERNELS_TRY_COMPILE_ARMPL
+  ${KOKKOSKERNELS_TOP_BUILD_DIR}/tpl_tests
+  ${KOKKOSKERNELS_TOP_SOURCE_DIR}/cmake/compile_tests/armpl.cpp
+  LINK_LIBRARIES -l${ARMPL_LIB} -lgfortran -lamath -lm
+  OUTPUT_VARIABLE KOKKOSKERNELS_TRY_COMPILE_ARMPL_OUT)
+IF(NOT KOKKOSKERNELS_TRY_COMPILE_ARMPL)
+  MESSAGE(FATAL_ERROR "KOKKOSKERNELS_TRY_COMPILE_ARMPL_OUT=${KOKKOSKERNELS_TRY_COMPILE_ARMPL_OUT}")
+ELSE()
+  # KokkosKernels::ARMPL is an alias to the ARMPL target.
+  # Let's add in the libgfortran and libm dependencies for users here.
+  GET_TARGET_PROPERTY(ARMPL_INTERFACE_LINK_LIBRARIES KokkosKernels::ARMPL INTERFACE_LINK_LIBRARIES)
+  SET(ARMPL_INTERFACE_LINK_LIBRARIES "${ARMPL_INTERFACE_LINK_LIBRARIES};-lgfortran;-lm")
+  SET_TARGET_PROPERTIES(ARMPL PROPERTIES INTERFACE_LINK_LIBRARIES "${ARMPL_INTERFACE_LINK_LIBRARIES}")
+ENDIF()

cmake/compile_tests/armpl.cpp

@@ -0,0 +1,5 @@
+#include <armpl.h>
+
+int main(void) {
+	return 0;
+}

cmake/kokkos_backends.cmake

@@ -11,6 +11,7 @@ ENDMACRO(CHECK_KOKKOS_BACKEND)
 
 CHECK_KOKKOS_BACKEND(CUDA)
 CHECK_KOKKOS_BACKEND(HIP)
+CHECK_KOKKOS_BACKEND(SYCL)
 CHECK_KOKKOS_BACKEND(OPENMP)
 CHECK_KOKKOS_BACKEND(PTHREAD)
 CHECK_KOKKOS_BACKEND(SERIAL)

cmake/kokkoskernels_eti_devices.cmake

@@ -5,12 +5,14 @@
 SET(EXEC_SPACES
   EXECSPACE_CUDA
   EXECSPACE_HIP
+  EXECSPACE_SYCL
   EXECSPACE_OPENMP
   EXECSPACE_PTHREAD
   EXECSPACE_SERIAL
 )
 SET(EXECSPACE_CUDA_CPP_TYPE Kokkos::Cuda)
 SET(EXECSPACE_HIP_CPP_TYPE Kokkos::Experimental::HIP)
+SET(EXECSPACE_SYCL_CPP_TYPE Kokkos::Experimental::SYCL)
 SET(EXECSPACE_OPENMP_CPP_TYPE Kokkos::OpenMP)
 SET(EXECSPACE_PTHREAD_CPP_TYPE Kokkos::Threads)
 SET(EXECSPACE_SERIAL_CPP_TYPE Kokkos::Serial)
@@ -19,14 +21,18 @@ SET(MEM_SPACES
   MEMSPACE_CUDASPACE
   MEMSPACE_CUDAUVMSPACE
   MEMSPACE_HIPSPACE
+  MEMSPACE_SYCLSPACE
+  MEMSPACE_SYCLSHAREDSPACE
   MEMSPACE_HOSTSPACE
   MEMSPACE_HBWSPACE
 )
-SET(MEMSPACE_CUDASPACE_CPP_TYPE    Kokkos::CudaSpace)
-SET(MEMSPACE_CUDAUVMSPACE_CPP_TYPE Kokkos::CudaUVMSpace)
-SET(MEMSPACE_HIPSPACE_CPP_TYPE     Kokkos::Experimental::HIPSpace)
-SET(MEMSPACE_HOSTSPACE_CPP_TYPE    Kokkos::HostSpace)
-SET(MEMSPACE_HBWSPACE_CPP_TYPE     Kokkos::HBWSpace)
+SET(MEMSPACE_CUDASPACE_CPP_TYPE       Kokkos::CudaSpace)
+SET(MEMSPACE_CUDAUVMSPACE_CPP_TYPE    Kokkos::CudaUVMSpace)
+SET(MEMSPACE_HIPSPACE_CPP_TYPE        Kokkos::Experimental::HIPSpace)
+SET(MEMSPACE_SYCLSPACE_CPP_TYPE       Kokkos::Experimental::SYCLDeviceUSMSpace)
+SET(MEMSPACE_SYCLSHAREDSPACE_CPP_TYPE Kokkos::Experimental::SYCLSharedUSMSpace)
+SET(MEMSPACE_HOSTSPACE_CPP_TYPE       Kokkos::HostSpace)
+SET(MEMSPACE_HBWSPACE_CPP_TYPE        Kokkos::HBWSpace)
 
 IF(KOKKOS_ENABLE_CUDA)
  KOKKOSKERNELS_ADD_OPTION(
@@ -85,6 +91,33 @@ IF(KOKKOS_ENABLE_HIP)
 
 ENDIF()
 
+IF(KOKKOS_ENABLE_SYCL)
+ KOKKOSKERNELS_ADD_OPTION(
+   INST_EXECSPACE_SYCL
+   ${KOKKOSKERNELS_INST_EXECSPACE_SYCL_DEFAULT}
+   BOOL
+   "Whether to pre instantiate kernels for the execution space Kokkos::Experimental::SYCL. Disabling this when Kokkos_ENABLE_SYCL is enabled may increase build times. Default: ON if Kokkos is SYCL-enabled, OFF otherwise."
+   )
+ KOKKOSKERNELS_ADD_OPTION(
+   INST_MEMSPACE_SYCLSPACE
+   ${KOKKOSKERNELS_INST_EXECSPACE_SYCL_DEFAULT}
+   BOOL
+   "Whether to pre instantiate kernels for the memory space Kokkos::Experimental::SYCLSpace.  Disabling this when Kokkos_ENABLE_SYCL is enabled may increase build times. Default: ON if Kokkos is SYCL-enabled, OFF otherwise."
+   )
+
+  IF(KOKKOSKERNELS_INST_EXECSPACE_SYCL AND KOKKOSKERNELS_INST_MEMSPACE_SYCLSPACE)
+    LIST(APPEND DEVICE_LIST "<SYCL,SYCLDeviceUSMSpace>")
+  ENDIF()
+  IF(KOKKOSKERNELS_INST_EXECSPACE_SYCL AND KOKKOSKERNELS_INST_MEMSPACE_SYCLSHAREDSPACE)
+    LIST(APPEND DEVICE_LIST "<SYCL,SYCLSharedUSMSpace>")
+  ENDIF()
+
+  IF( Trilinos_ENABLE_COMPLEX_DOUBLE AND ((NOT DEFINED CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS) OR (NOT CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS)) )
+    MESSAGE( WARNING "The CMake option CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS is either undefined or OFF.  Please set CMAKE_CXX_USE_RESPONSE_FILE_FOR_OBJECTS:BOOL=ON when building with SYCL and complex double enabled.")
+  ENDIF()
+
+ENDIF()
+
 KOKKOSKERNELS_ADD_OPTION(
  INST_MEMSPACE_HOSTSPACE
  ${KOKKOSKERNELS_ADD_DEFAULT_ETI}
@@ -138,6 +171,7 @@ KOKKOSKERNELS_ADD_OPTION(
 
 SET(EXECSPACE_CUDA_VALID_MEM_SPACES      CUDASPACE CUDAUVMSPACE)
 SET(EXECSPACE_HIP_VALID_MEM_SPACES       HIPSPACE)
+SET(EXECSPACE_SYCL_VALID_MEM_SPACES      SYCLSPACE SYCLSHAREDSPACE)
 SET(EXECSPACE_SERIAL_VALID_MEM_SPACES    HBWSPACE HOSTSPACE)
 SET(EXECSPACE_OPENMP_VALID_MEM_SPACES    HBWSPACE HOSTSPACE)
 SET(EXECSPACE_PTHREAD_VALID_MEM_SPACES   HBWSPACE HOSTSPACE)

cmake/kokkoskernels_features.cmake

@@ -24,7 +24,7 @@ KOKKOSKERNELS_FEATURE_DEPENDS_ON_TPLS(
 # Fortran Complex BLAS
 # ==================================================================
 
-IF (KOKKOSKERNELS_ENABLE_TPL_BLAS OR KOKKOSKERNELS_ENABLE_TPL_MKL)
+IF (KOKKOSKERNELS_ENABLE_TPL_BLAS OR KOKKOSKERNELS_ENABLE_TPL_MKL OR KOKKOSKERNELS_ENABLE_TPL_ARMPL)
   INCLUDE(CheckHostBlasReturnComplex.cmake)
   CHECK_HOST_BLAS_RETURN_COMPLEX(KOKKOSKERNELS_TPL_BLAS_RETURN_COMPLEX)
 ENDIF()

cmake/kokkoskernels_tpls.cmake

@@ -420,11 +420,12 @@ KOKKOSKERNELS_ADD_TPL_OPTION(MKL  OFF "Whether to enable MKL")
 KOKKOSKERNELS_ADD_TPL_OPTION(MAGMA    OFF "Whether to enable MAGMA")
 KOKKOSKERNELS_ADD_TPL_OPTION(CBLAS OFF "Whether to enable CBLAS")
 KOKKOSKERNELS_ADD_TPL_OPTION(LAPACKE OFF "Whether to enable LAPACKE")
+KOKKOSKERNELS_ADD_TPL_OPTION(ARMPL   OFF  "Whether to enable ARMPL")
 
 # Set F77_BLAS_MANGLE macro based on Fortran-C interface (unless already set
 # by Trilinos or user)
 IF ("${F77_BLAS_MANGLE}" STREQUAL "")
-  IF (KOKKOSKERNELS_ENABLE_TPL_BLAS OR KOKKOSKERNELS_ENABLE_TPL_MKL OR KOKKOSKERNELS_ENABLE_TPL_MAGMA)
+  IF (KOKKOSKERNELS_ENABLE_TPL_BLAS OR KOKKOSKERNELS_ENABLE_TPL_MKL OR KOKKOSKERNELS_ENABLE_TPL_MAGMA OR KOKKOSKERNELS_ENABLE_TPL_ARMPL)
     ENABLE_LANGUAGE(C)
     ENABLE_LANGUAGE(Fortran)
     INCLUDE(FortranCInterface)
@@ -481,6 +482,7 @@ IF (NOT KOKKOSKERNELS_HAS_TRILINOS)
   KOKKOSKERNELS_IMPORT_TPL(CHOLMOD)
   KOKKOSKERNELS_IMPORT_TPL(SUPERLU)
   KOKKOSKERNELS_IMPORT_TPL(METIS)
+  KOKKOSKERNELS_IMPORT_TPL(ARMPL)
 ENDIF()
 
 #Convert list to newlines (which CMake doesn't always like in cache variables)