Tests: Adopt do_regtest.py for CMake builds

hfp · Aug 21, 2024 · 44379fe · 44379fe
1 parent 6eaca95
commit 44379fe
Show file tree

Hide file tree

Showing 8 changed files with 20 additions and 75 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -92,10 +92,6 @@ endforeach()
 # =================================================================================================
 # OPTIONS
 
-# option(CP2K_BUILD_DBCSR "Build dbcsr before building cp2k" OFF)
-option(CP2K_ENABLE_REGTESTS
-       "Enable installation of the binaries for running regtests afterwards"
-       OFF)
 option(CMAKE_POSITION_INDEPENDENT_CODE "Enable position independent code" ON)
 option(CP2K_ENABLE_CONSISTENCY_CHECKS
        "Check that the list of compiled files and files contained in src match"
@@ -377,21 +373,9 @@ if(CP2K_ENABLE_CONSISTENCY_CHECKS)
   return()
 endif()
 
-# for the time being I change the bin directory to point to
-# ../exe/build-cmake-{cuda,hip,cpu} when we want to run the regtests afterwards.
-# This solution does not require modifying the regtests scripts at all and cmake
-# does not mind this change when we want to install cp2k with a command like
-# `make install`
-if(CP2K_ENABLE_REGTESTS)
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY
-      "${cp2k_SOURCE_DIR}/exe/${__cp2k_cmake_name}"
-      CACHE PATH "Single output directory for building all executables.")
-else()
-  set(CMAKE_RUNTIME_OUTPUT_DIRECTORY
-      ${cp2k_BINARY_DIR}/bin
-      CACHE PATH "Single output directory for building all executables.")
-
-endif()
+set(CMAKE_RUNTIME_OUTPUT_DIRECTORY
+    ${cp2k_BINARY_DIR}/bin
+    CACHE PATH "Single output directory for building all executables.")
 
 # Python
 #
@@ -1034,10 +1018,10 @@ if(NOT CP2K_USE_LIBTORCH)
   message("   - libtorch")
 endif()
 
-if(CP2K_ENABLE_REGTESTS OR cp2k_TESTS)
+if(cp2k_TESTS)
   message("\n\n" "To run the regtests you need to run the following commands\n"
           "\n\n cd ..\n" " export CP2K_DATA_DIR=${CMAKE_SOURCE_DIR}/data/\n"
-          " ./tests/do_regtest.py ${__cp2k_cmake_name} ${__cp2k_ext}\n\n")
+          " ./tests/do_regtest.py ${cp2k_BINARY_DIR}/bin ${__cp2k_ext}\n\n")
 endif()
 
 # files needed for cmake

diff --git a/Makefile b/Makefile
@@ -253,11 +253,11 @@ $(EXEDIR)/cp2k_shell.$(ONEVERSION): $(EXEDIR)/cp2k.$(ONEVERSION)
 	cd $(EXEDIR); ln -sf cp2k.$(ONEVERSION) cp2k_shell.$(ONEVERSION)
 
 test:
-	@$(CP2KHOME)/tests/do_regtest.py $(ARCH) $(ONEVERSION) $(TESTOPTS)
+	@$(CP2KHOME)/tests/do_regtest.py $(EXEDIR) $(ONEVERSION) $(TESTOPTS)
 
 testbg:
 	@echo "testing: $(ONEVERSION) : full log in $(TSTDIR)/regtest.log "
-	@$(CP2KHOME)/tests/do_regtest.py $(ARCH) $(ONEVERSION) $(TESTOPTS) > $(TSTDIR)/regtest.log 2>&1
+	@$(CP2KHOME)/tests/do_regtest.py $(EXEDIR) $(ONEVERSION) $(TESTOPTS) > $(TSTDIR)/regtest.log 2>&1
 	@grep -e "Summary:" -e "Status:" $(TSTDIR)/regtest.log
 
 endif

diff --git a/docs/getting-started/CMake.md b/docs/getting-started/CMake.md
@@ -99,14 +99,6 @@ with `-DCP2K_USE_<LIBRARY>=ON`. `<LIBRARY>` is the name of the optional dependen
 Please refer to the `CMakeLists.txt` file for an up-to-date list of the dependencies enabled by each
 option.
 
-### Regression Testing
-
-By default, [CMake] creates the executables in the `bin/` subdirectory of the build directory. This
-is incompatible with CP2K's `do_regtest.py` script to run regression tests.
-
-`-DCP2K_ENABLE_REGTESTS=ON` instructs [CMake] to create binaries in the usual location (`exe` folder
-in the root of the source tree), so that `do_regtest.py` works as expected.
-
 ### GPUs
 
 CP2K is GPU-accelerated. In order to enable GPU acceleration with \[CUDA\] or \[HIP\],
@@ -125,7 +117,6 @@ Build CP2K with CUDA acceleration for Nvidia A100 GPUs, with multiple optional d
 cd <CP2K_REPOSITORY> && make build/
 cmake -S . -B build \
     -GNinja \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_USE_LIBXC=ON \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_SPGLIB=ON \

diff --git a/tests/do_regtest.py b/tests/do_regtest.py
@@ -42,10 +42,6 @@
 ]
 
 
-def cp2k_stem() -> str:
-    return os.getenv("CP2K_STEM", "cp2k")
-
-
 # ======================================================================================
 async def main() -> None:
     parser = argparse.ArgumentParser(description="Runs CP2K regression test suite.")
@@ -71,17 +67,15 @@ async def main() -> None:
     parser.add_argument("--workbasedir", type=Path)
     parser.add_argument("--skip_unittests", action="store_true")
     parser.add_argument("--skip_regtests", action="store_true")
-    parser.add_argument("arch")
+    parser.add_argument("binary_dir", type=Path)
     parser.add_argument("version")
     cfg = Config(parser.parse_args())
 
     print("*************************** Testing started ****************************")
     start_time = time.perf_counter()
 
     # Query CP2K binary for feature flags.
-    version_bytes, _ = await (
-        await cfg.launch_exe(cp2k_stem(), "--version")
-    ).communicate()
+    version_bytes, _ = await (await cfg.launch_exe("cp2k", "--version")).communicate()
     version_output = version_bytes.decode("utf8", errors="replace")
     flags_line = re.search(r" cp2kflags:(.*)\n", version_output)
     if not flags_line:
@@ -103,7 +97,7 @@ async def main() -> None:
     print(f"Keepalive:      {cfg.keepalive}")
     print(f"Flag slow:      {cfg.flag_slow}")
     print(f"Debug:          {cfg.debug}")
-    print(f"ARCH:           {cfg.arch}")
+    print(f"Binary dir:     {cfg.binary_dir}")
     print(f"VERSION:        {cfg.version}")
     print(f"Flags:          " + ",".join(flags))
 
@@ -262,7 +256,7 @@ def __init__(self, args: argparse.Namespace):
         self.valgrind = args.valgrind
         self.keepalive = args.keepalive
         self.flag_slow = args.flagslow
-        self.arch = args.arch
+        self.binary_dir = args.binary_dir.resolve()
         self.version = args.version
         self.debug = args.debug
         self.max_errors = args.maxerrors
@@ -271,12 +265,8 @@ def __init__(self, args: argparse.Namespace):
         self.skip_unittests = args.skip_unittests
         self.skip_regtests = args.skip_regtests
         datestamp = datetime.now().strftime("%Y-%m-%d_%H-%M-%S")
-        leaf_dir = f"TEST-{args.arch}-{args.version}-{datestamp}"
-        self.work_base_dir = (
-            args.workbasedir / leaf_dir
-            if args.workbasedir
-            else self.cp2k_root / "regtesting" / args.arch / args.version / leaf_dir
-        )
+        leaf_dir = f"TEST-{args.version}-{datestamp}"
+        self.work_base_dir = (args.workbasedir or args.binary_dir).resolve() / leaf_dir
         self.error_summary = self.work_base_dir / "error_summary"
 
         # Parse suppression files.
@@ -316,9 +306,7 @@ def launch_exe(
         if exe_path.is_absolute():
             cmd = [str(exe_path)]
         else:
-            cmd = [
-                str(self.cp2k_root / "exe" / self.arch / f"{exe_stem}.{self.version}")
-            ]
+            cmd = [str(self.binary_dir / f"{exe_stem}.{self.version}")]
         if self.valgrind:
             cmd = ["valgrind", "--error-exitcode=42", "--exit-on-first-error=yes"] + cmd
         if self.use_mpi:
@@ -454,9 +442,7 @@ async def stop(self) -> None:
 
     async def start(self) -> None:
         assert self._child is None
-        self._child = await self.cfg.launch_exe(
-            cp2k_stem(), "--shell", cwd=self.workdir
-        )
+        self._child = await self.cfg.launch_exe("cp2k", "--shell", cwd=self.workdir)
         await self.ready()
         await self.sendline("HARSH")  # With harsh mode any error leads to an abort.
         await self.ready()
@@ -601,7 +587,7 @@ async def run_regtests_classic(batch: Batch, cfg: Config) -> List[TestResult]:
     for test in batch.regtests:
         start_time = time.perf_counter()
         start_dirsize = dirsize(batch.workdir)
-        child = await cfg.launch_exe(cp2k_stem(), test.inp_fn, cwd=batch.workdir)
+        child = await cfg.launch_exe("cp2k", test.inp_fn, cwd=batch.workdir)
         output, returncode, timed_out = await wait_for_child_process(child, cfg.timeout)
         test.out_path.write_bytes(output)
         duration = time.perf_counter() - start_time

diff --git a/tools/docker/scripts/build_cp2k_cmake.sh b/tools/docker/scripts/build_cp2k_cmake.sh
@@ -43,7 +43,6 @@ if [[ "${PROFILE}" == "spack" ]] && [[ "${VERSION}" == "psmp" ]]; then
     -DCP2K_USE_LIBTORCH=OFF \
     -DCP2K_USE_MPI=ON \
     -DCP2K_USE_MPI_F08=ON \
-    -DCP2K_ENABLE_REGTESTS=ON \
     .. |& tee ./cmake.log
   CMAKE_EXIT_CODE=$?
 
@@ -52,7 +51,6 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "ssmp" ]]; then
     -GNinja \
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -73,7 +71,6 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "sdbg" ]]; then
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
     -DCP2K_DEBUG_MODE=ON \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -95,7 +92,6 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "psmp" ]]; then
     -GNinja \
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -107,7 +103,6 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "psmp" ]]; then
     -DCP2K_USE_LIBXSMM=ON \
     -DCP2K_USE_SUPERLU=ON \
     -DCP2K_USE_PLUMED=ON \
-    -DCP2K_USE_PEXSI=ON \
     -DCP2K_USE_SPLA=ON \
     -DCP2K_USE_METIS=ON \
     -DCP2K_USE_ELPA=OFF \
@@ -126,7 +121,6 @@ elif [[ "${PROFILE}" == "toolchain" ]] && [[ "${VERSION}" == "pdbg" ]]; then
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
     -DCP2K_DEBUG_MODE=ON \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -147,7 +141,6 @@ elif [[ "${PROFILE}" == "ubuntu" ]] && [[ "${VERSION}" == "ssmp" ]]; then
     -GNinja \
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -169,7 +162,6 @@ elif [[ "${PROFILE}" == "ubuntu_i386" ]] && [[ "${VERSION}" == "ssmp" ]]; then
     -GNinja \
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=ON \
     -DCP2K_USE_LIBXC=ON \
@@ -190,7 +182,6 @@ elif [[ "${PROFILE}" == "minimal" ]] && [[ "${VERSION}" == "ssmp" ]]; then
     -GNinja \
     -DCMAKE_INSTALL_PREFIX=/opt/cp2k \
     -Werror=dev \
-    -DCP2K_ENABLE_REGTESTS=ON \
     -DCP2K_BLAS_VENDOR=OpenBLAS \
     -DCP2K_USE_LIBINT2=OFF \
     -DCP2K_USE_LIBXC=OFF \

diff --git a/tools/docker/scripts/test_regtest_cmake.sh b/tools/docker/scripts/test_regtest_cmake.sh
@@ -48,7 +48,7 @@ fi
 echo -e "\n========== Running Regtests =========="
 set -x
 # shellcheck disable=SC2086
-./tests/do_regtest.py local ${VERSION} ${TESTOPTS}
+./tests/do_regtest.py ./build/bin/ ${VERSION} ${TESTOPTS}
 
 exit 0 # Prevent CI from overwriting do_regtest's summary message.
 

diff --git a/tools/fedora/cp2k.spec b/tools/fedora/cp2k.spec
@@ -164,9 +164,6 @@ cmake_common_args=(
   "-DCP2K_DEBUG_MODE:BOOL=OFF"
   "-DCP2K_BLAS_VENDOR:STRING=FlexiBLAS"
   "-DCP2K_USE_STATIC_BLAS:BOOL=OFF"
-  # Unit tests are included in REGTESTS
-  # Note: Enabling this will write build files in the source folder :/
-  "-DCP2K_ENABLE_REGTESTS:BOOL=ON"
   # Dependencies equivalent with Default
   "-DCP2K_USE_FFTW3:BOOL=ON"
   "-DCP2K_USE_COSMA:BOOL=OFF"  # Not packaged
@@ -231,7 +228,6 @@ for mpi in '' mpich %{?with_openmpi:openmpi} ; do
     module load mpi/${mpi}-%{_arch}
     bindir=${MPI_BIN}
     libdir=${MPI_LIB}
-    export CP2K_STEM=%{buildroot}${MPI_BIN}/cp2k
     # Note, final position arguments are also here
     test_mpi_args=(
       "--mpiranks 2"
@@ -241,7 +237,6 @@ for mpi in '' mpich %{?with_openmpi:openmpi} ; do
   else
     bindir=%{_bindir}
     libdir=%{_libdir}
-    export CP2K_STEM=%{buildroot}/usr/bin/cp2k
     test_mpi_args=(
       "local"
       "ssmp"
@@ -250,7 +245,7 @@ for mpi in '' mpich %{?with_openmpi:openmpi} ; do
   # Run packaged do_regtest.sh with appropriate buildroot runpaths
   env PATH=%{buildroot}${bindir}:${PATH} \
     LD_LIBRARY_PATH=%{buildroot}${libdir} \
-    tests/do_regtest.py ${test_common_args[@]} ${test_mpi_args[@]}
+    tests/do_regtest.py %{buildroot}${bindir} ${test_mpi_args[@]}
   [ -n "$mpi" ] && module unload mpi/${mpi}-%{_arch}
 done
 

diff --git a/tools/fedora/tests/do_regtest.sh b/tools/fedora/tests/do_regtest.sh
@@ -17,12 +17,10 @@ rlJournalStart
     if [[ "${CP2K_VARIANT}" != "serial" ]]; then
       rlRun "module avail" 0 "Show available modules"
       rlRun "module load mpi/${CP2K_VARIANT}" 0 "Load MPI module: ${CP2K_VARIANT}"
-      rlRun "export CP2K_STEM=$MPI_BIN/cp2k" 0 "Export CP2K_STEM"
       rlRun "args=\"\$args --mpiranks 2\"" 0 "Set MPI arguments"
-      rlRun "args=\"\$args local_${CP2K_VARIANT} psmp\"" 0 "Set run specific arguments"
+      rlRun "args=\"\$args $MPI_BIN psmp\"" 0 "Set run specific arguments"
     else
-      rlRun "export CP2K_STEM=/usr/bin/cp2k" 0 "Export CP2K_STEM"
-      rlRun "args=\"\$args local ssmp\"" 0 "Set run specific arguments"
+      rlRun "args=\"\$args /usr/bin ssmp\"" 0 "Set run specific arguments"
     fi
     rlRun "./do_regtest.py $args" 0 "Run regression tests"
   rlPhaseEnd