Merge pull request #3029 from easybuilders/4.8.x

release EasyBuild v4.8.2

RELEASE_NOTES

@@ -6,6 +6,31 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en
 The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks.
 
 
+v4.8.2 (29 October 2023)
+------------------------
+
+update/bugfix release
+
+- minor enhancements and updates, including:
+  - allow use of `test_cmd` without `runtest` for `ConfigureMake` (#2837)
+  - enhance `CMakeMake` easyblock to run `ctest` command if `runtest` is `True` (#2838)
+  - enhance `Conda` easyblock: add support for using custom `conda` command (like `mamba`), and clean up after installation (#2996)
+  - update WRF easyblock to correctly determine `WRF*` subdirectory for WRF versions >= 4.5.1 (#2997)
+  - update numpy easyblock to be able to use FlexiBLAS with MKL as dependency (#2999)
+  - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003, #3005)
+  - don't blindly overwrite `-Dccflags` + honour `preconfigopts` in Perl easyblock (#3010)
+  - use more test programs in sanity check step of OpenMPI easyblock (#3016)
+  - fix `CMAKE_PREFIX_PATH` in imkl easyblock (#3021)
+  - update sanity check in wxPython easyblock for version 4.2.0 (#3023)
+- various bug fixes, including:
+  - fix creation of symlink for libraries in TBB easyblock (#2927)
+  - fix `--sanity-check-only` and `--module-only` for UCX plugins (#3007)
+  - replace umlaut in docstring of PerlBundle easyblock (to fix Non-ASCII character error when using Python 2.7) (#3008)
+  - enhance TensorFlow easyblock to avoid use of `-mcpu=native` for XNNPACK component when building on aarch64 (#3011)
+  - only use `-DCMAKE_SKIP_RPATH=ON` for CMake < 3.5.0 (#3012)
+  - fix the blas/lapack name passed to meson when building recent scipy versions (>= 1.9.0) on top of Intel MKL (#3024)
+
+
 v4.8.1 (11 September 2023)
 --------------------------
 

easybuild/easyblocks/__init__.py

@@ -43,7 +43,7 @@
 # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like
 #   UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0'
 # This causes problems further up the dependency chain...
-VERSION = LooseVersion('4.8.1')
+VERSION = LooseVersion('4.8.2')
 UNKNOWN = 'UNKNOWN'
 
 

easybuild/easyblocks/generic/cmakemake.py

@@ -39,7 +39,7 @@
 from distutils.version import LooseVersion
 
 from easybuild.easyblocks.generic.configuremake import ConfigureMake
-from easybuild.framework.easyconfig import CUSTOM
+from easybuild.framework.easyconfig import BUILD, CUSTOM
 from easybuild.tools.build_log import EasyBuildError, print_warning
 from easybuild.tools.config import build_option
 from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which
@@ -105,6 +105,7 @@ def extra_options(extra_vars=None):
             'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM],
             'generator': [None, "Build file generator to use. None to use CMakes default", CUSTOM],
             'install_target_subdir': [None, "Subdirectory to use as installation target", CUSTOM],
+            'runtest': [None, "Make target to test build or True to use CTest", BUILD],
             'srcdir': [None, "Source directory location to provide to cmake command", CUSTOM],
             'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM],
         })
@@ -114,6 +115,7 @@ def __init__(self, *args, **kwargs):
         """Constructor for CMakeMake easyblock"""
         super(CMakeMake, self).__init__(*args, **kwargs)
         self._lib_ext = None
+        self._cmake_version = None
         self.separate_build_dir = None
 
     @property
@@ -131,6 +133,14 @@ def lib_ext(self):
     def lib_ext(self, value):
         self._lib_ext = value
 
+    @property
+    def cmake_version(self):
+        """Return the used CMake version, caching the value for reuse"""
+        if self._cmake_version is None:
+            self._cmake_version = det_cmake_version()
+            self.log.debug('Determined CMake version: %s', self._cmake_version)
+        return self._cmake_version
+
     @property
     def build_type(self):
         """Build type set in the EasyConfig with default determined by toolchainopts"""
@@ -241,10 +251,8 @@ def configure_step(self, srcdir=None, builddir=None):
             if fc:
                 setvar('FC', fc)
 
-        cmake_version = det_cmake_version()
-
         # Flags are read from environment variables already since at least CMake 2.8.0
-        if LooseVersion(cmake_version) < LooseVersion('2.8.0') or cache_exists:
+        if LooseVersion(self.cmake_version) < LooseVersion('2.8.0') or cache_exists:
             env_to_options.update({
                 'CFLAGS': 'CMAKE_C_FLAGS',
                 'CXXFLAGS': 'CMAKE_CXX_FLAGS',
@@ -259,9 +267,10 @@ def configure_step(self, srcdir=None, builddir=None):
                     self.log.info("Using absolute path to compiler command: %s", value)
                 options[option] = value
 
-        if build_option('rpath'):
+        if build_option('rpath') and LooseVersion(self.cmake_version) < LooseVersion('3.5.0'):
             # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install...
-            # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416
+            # this is only required for CMake < 3.5.0, since newer version are more careful w.r.t. RPATH,
+            # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5
             options['CMAKE_SKIP_RPATH'] = 'ON'
 
         # show what CMake is doing by default
@@ -310,4 +319,12 @@ def test_step(self):
         """CMake specific test setup"""
         # When using ctest for tests (default) then show verbose output if a test fails
         setvar('CTEST_OUTPUT_ON_FAILURE', 'True')
+        # Handle `runtest = True` if `test_cmd` has not been set
+        if self.cfg.get('runtest') is True and not self.cfg.get('test_cmd'):
+            test_cmd = 'ctest'
+            if LooseVersion(self.cmake_version) >= '3.17.0':
+                test_cmd += ' --no-tests=error'
+            self.log.debug("`runtest = True` found, using '%s' as test_cmd", test_cmd)
+            self.cfg['test_cmd'] = test_cmd
+
         super(CMakeMake, self).test_step()

easybuild/easyblocks/generic/conda.py

@@ -34,6 +34,8 @@
 from easybuild.easyblocks.generic.binary import Binary
 from easybuild.framework.easyconfig import CUSTOM
 from easybuild.tools.run import run_cmd
+from easybuild.tools.modules import get_software_root
+from easybuild.tools.build_log import EasyBuildError
 
 
 class Conda(Binary):
@@ -57,11 +59,21 @@ def extract_step(self):
             super(Conda, self).extract_step()
 
     def install_step(self):
-        """Install software using 'conda env create' or 'conda create' & 'conda install'."""
+        """Install software using 'conda env create' or 'conda create' & 'conda install'
+        (or the 'mamba', etc., equivalent)."""
+        if (get_software_root('anaconda2') or get_software_root('miniconda2') or
+                get_software_root('anaconda3') or get_software_root('miniconda3')):
+            conda_cmd = 'conda'
+        elif get_software_root('mamba'):
+            conda_cmd = 'mamba'
+        elif get_software_root('micromamba'):
+            conda_cmd = 'micromamba'
+        else:
+            raise EasyBuildError("No conda/mamba/micromamba available.")
 
         # initialize conda environment
         # setuptools is just a choice, but *something* needs to be there
-        cmd = "conda config --add create_default_packages setuptools"
+        cmd = "%s config --add create_default_packages setuptools" % conda_cmd
         run_cmd(cmd, log_all=True, simple=True)
 
         if self.cfg['environment_file'] or self.cfg['remote_environment']:
@@ -72,7 +84,8 @@ def install_step(self):
                 env_spec = self.cfg['remote_environment']
 
             # use --force to ignore existing installation directory
-            cmd = "%s conda env create --force %s -p %s" % (self.cfg['preinstallopts'], env_spec, self.installdir)
+            cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd,
+                                                         env_spec, self.installdir)
             run_cmd(cmd, log_all=True, simple=True)
 
         else:
@@ -85,9 +98,14 @@ def install_step(self):
 
                 self.log.info("Installed conda requirements")
 
-            cmd = "%s conda create --force -y -p %s %s" % (self.cfg['preinstallopts'], self.installdir, install_args)
+            cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd,
+                                                        self.installdir, install_args)
             run_cmd(cmd, log_all=True, simple=True)
 
+        # clean up
+        cmd = "%s clean -ya" % conda_cmd
+        run_cmd(cmd, log_all=True, simple=True)
+
     def make_module_extra(self):
         """Add the install directory to the PATH."""
         txt = super(Conda, self).make_module_extra()

easybuild/easyblocks/generic/configuremake.py

@@ -48,6 +48,7 @@
 from easybuild.tools.config import source_paths, build_option
 from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file
 from easybuild.tools.filetools import read_file, remove_file
+from easybuild.tools.py2vs3 import string_type
 from easybuild.tools.run import run_cmd
 
 # string that indicates that a configure script was generated by Autoconf
@@ -192,7 +193,8 @@ def extra_options(extra_vars=None):
             'install_cmd': [DEFAULT_INSTALL_CMD, "Install command to use", CUSTOM],
             'prefix_opt': [None, "Prefix command line option for configure script ('--prefix=' if None)", CUSTOM],
             'tar_config_opts': [False, "Override tar settings as determined by configure.", CUSTOM],
-            'test_cmd': [DEFAULT_TEST_CMD, "Test command to use ('runtest' value is appended)", CUSTOM],
+            'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD,
+                         CUSTOM],
         })
         return extra_vars
 
@@ -337,7 +339,7 @@ def build_step(self, verbose=False, path=None):
 
         targets = self.cfg.get('build_cmd_targets') or DEFAULT_BUILD_TARGET
         # ensure strings are converted to list
-        targets = [targets] if isinstance(targets, str) else targets
+        targets = [targets] if isinstance(targets, string_type) else targets
 
         for target in targets:
             cmd = ' '.join([
@@ -360,13 +362,13 @@ def test_step(self):
         """
 
         test_cmd = self.cfg.get('test_cmd') or DEFAULT_TEST_CMD
-        if self.cfg['runtest'] or test_cmd != DEFAULT_TEST_CMD:
-            cmd = ' '.join([
-                self.cfg['pretestopts'],
-                test_cmd,
-                self.cfg['runtest'],
-                self.cfg['testopts'],
-            ])
+        runtest = self.cfg['runtest']
+        if runtest or test_cmd != DEFAULT_TEST_CMD:
+            # Make run_test a string (empty if it is e.g. a boolean)
+            if not isinstance(runtest, string_type):
+                runtest = ''
+            # Compose command filtering out empty values
+            cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x])
             (out, _) = run_cmd(cmd, log_all=True, simple=False)
 
             return out

easybuild/easyblocks/generic/perlbundle.py

@@ -25,7 +25,7 @@
 """
 EasyBuild support for installing a bundle of Perl modules, implemented as a generic easyblock
 
-@author: Mikael Öhman (Chalmers University of Technology)
+@author: Mikael Oehman (Chalmers University of Technology)
 """
 import os
 

easybuild/easyblocks/i/imkl.py

@@ -546,11 +546,13 @@ def make_module_req_guess(self):
                     os.path.join(self.mkl_basedir, 'include'),
                     os.path.join(self.mkl_basedir, 'include', 'fftw'),
                 ]
+                cmake_prefix_path = [self.mkl_basedir]
                 guesses.update({
                     'PATH': [],
                     'LD_LIBRARY_PATH': library_path,
                     'LIBRARY_PATH': library_path,
                     'CPATH': cpath,
+                    'CMAKE_PREFIX_PATH': cmake_prefix_path,
                     'PKG_CONFIG_PATH': pkg_config_path,
                 })
                 if self.cfg['flexiblas']:

easybuild/easyblocks/n/numpy.py

@@ -79,7 +79,11 @@ def configure_step(self):
             "search_static_first=True",
         ])
 
-        if get_software_root("imkl"):
+        # If both FlexiBLAS and MKL are found, we assume that FlexiBLAS has a dependency on MKL.
+        # In this case we want to link to FlexiBLAS and not directly to MKL.
+        imkl_direct = get_software_root("imkl") and not get_software_root("FlexiBLAS")
+
+        if imkl_direct:
 
             if self.toolchain.comp_family() == toolchain.GCC:
                 # see https://software.intel.com/en-us/articles/numpyscipy-with-intel-mkl,
@@ -112,7 +116,7 @@ def configure_step(self):
         lapack = None
         fft = None
 
-        if get_software_root("imkl"):
+        if imkl_direct:
             # with IMKL, no spaces and use '-Wl:'
             # redefine 'Wl,' to 'Wl:' so that the patch file can do its job
             def get_libs_for_mkl(varname):

easybuild/easyblocks/o/openmpi.py

@@ -211,8 +211,18 @@ def sanity_check_step(self):
         mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version)
         # Limit number of ranks to 8 to avoid it failing due to hyperthreading
         ranks = min(8, self.cfg['parallel'])
-        for src, compiler in (('hello_c.c', 'mpicc'), ('hello_mpifh.f', 'mpifort'), ('hello_usempi.f90', 'mpif90')):
-            src_path = os.path.join(self.cfg['start_dir'], 'examples', src)
+        for srcdir, src, compiler in (
+            ('examples', 'hello_c.c', 'mpicc'),
+            ('examples', 'hello_mpifh.f', 'mpifort'),
+            ('examples', 'hello_usempi.f90', 'mpif90'),
+            ('examples', 'ring_c.c', 'mpicc'),
+            ('examples', 'ring_mpifh.f', 'mpifort'),
+            ('examples', 'ring_usempi.f90', 'mpif90'),
+            ('test/simple', 'thread_init.c', 'mpicc'),
+            ('test/simple', 'intercomm1.c', 'mpicc'),
+            ('test/simple', 'mpi_barrier.c', 'mpicc'),
+        ):
+            src_path = os.path.join(self.cfg['start_dir'], srcdir, src)
             if os.path.exists(src_path):
                 test_exe = os.path.join(self.builddir, 'mpi_test_' + os.path.splitext(src)[0])
                 self.log.info("Adding minimal MPI test program to sanity checks: %s", test_exe)

easybuild/easyblocks/p/perl.py

@@ -75,7 +75,7 @@ def configure_step(self):
         configopts = [
             self.cfg['configopts'],
             '-Dcc="{0}"'.format(os.getenv('CC')),
-            '-Dccflags="{0}"'.format(os.getenv('CFLAGS')),
+            '-Dccflags="{0}"'.format(os.getenv('CFLAGS')) if '-Dccflags' not in self.cfg['configopts'] else '',
             '-Dinc_version_list=none',
             '-Dprefix=%(installdir)s',
             # guarantee that scripts are installed in /bin in the installation directory (and not in a guessed path)
@@ -116,7 +116,7 @@ def configure_step(self):
         if os.getenv('COLUMNS', None) == '0':
             unset_env_vars(['COLUMNS'])
 
-        cmd = './Configure -de %s' % configopts
+        cmd = '%s ./Configure -de %s' % (self.cfg['preconfigopts'], configopts)
         run_cmd(cmd, log_all=True, simple=True)
 
     def test_step(self):

easybuild/easyblocks/p/pytorch.py

@@ -318,7 +318,7 @@ def get_count_for_pattern(regex, text):
             return 0
 
         # Create clear summary report
-        failure_report = ""
+        failure_report = []
         failure_cnt = 0
         error_cnt = 0
         failed_test_suites = []
@@ -337,9 +337,9 @@ def get_count_for_pattern(regex, text):
             # E.g. 'failures=3, errors=10, skipped=190, expected failures=6'
             failure_summary = m.group('failure_summary')
             total, test_suite = m.group('test_cnt', 'failed_test_suite_name')
-            failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format(
+            failure_report.append("{test_suite} ({total} total tests, {failure_summary})".format(
                     test_suite=test_suite, total=total, failure_summary=failure_summary
-                )
+                ))
             failure_cnt += get_count_for_pattern(r"(?<!expected )failures=([0-9]+)", failure_summary)
             error_cnt += get_count_for_pattern(r"errors=([0-9]+)", failure_summary)
             failed_test_suites.append(test_suite)
@@ -362,27 +362,55 @@ def get_count_for_pattern(regex, text):
             # E.g. '2 failed, 128 passed, 2 skipped, 2 warnings'
             failure_summary = m.group('failure_summary')
             test_suite = m.group('failed_test_suite_name')
-            failure_report += "{test_suite} ({failure_summary})\n".format(
+            failure_report.append("{test_suite} ({failure_summary})".format(
                     test_suite=test_suite, failure_summary=failure_summary
-                )
+                ))
             failure_cnt += get_count_for_pattern(r"([0-9]+) failed", failure_summary)
             error_cnt += get_count_for_pattern(r"([0-9]+) error", failure_summary)
             failed_test_suites.append(test_suite)
 
-        # Make the names unique and sorted
-        failed_test_suites = sorted(set(failed_test_suites))
+        # Grep for patterns like:
+        # AssertionError: 2 unit test(s) failed:
+        #         DistributedDataParallelTest.test_find_unused_parameters_kwarg_debug_detail
+        #         DistributedDataParallelTest.test_find_unused_parameters_kwarg_grad_is_view_debug_detail
+        #
+        # FINISHED PRINTING LOG FILE of distributed/test_c10d_nccl (<snip>)
+        #
+        # distributed/test_c10d_nccl failed!
+
+        regex = (
+            r"^AssertionError: (?P<failure_summary>[0-9]+ unit test\(s\) failed):\n"
+            r"(\s+.*\n)+"
+            r"(((?!failed!).)*\n){0,5}"
+            r"(?P<failed_test_suite_name>.*) failed!$"
+        )
+
+        for m in re.finditer(regex, tests_out, re.M):
+            # E.g. '2 unit test(s) failed'
+            failure_summary = m.group('failure_summary')
+            test_suite = m.group('failed_test_suite_name')
+            failure_report.append("{test_suite} ({failure_summary})".format(
+                    test_suite=test_suite, failure_summary=failure_summary
+                ))
+            failure_cnt += get_count_for_pattern(r"([0-9]+) unit test\(s\) failed", failure_summary)
+            failed_test_suites.append(test_suite)
+
+        # Make the names unique
+        failed_test_suites = set(failed_test_suites)
         # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors)
-        # Also unique and sorted to be able to compare the lists below
-        all_failed_test_suites = sorted(set(
+        # Also unique to be able to compare the lists below
+        all_failed_test_suites = set(
             re.findall(r"^(?P<test_name>.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M)
-        ))
+        )
         # If we missed any test suites prepend a list of all failed test suites
         if failed_test_suites != all_failed_test_suites:
-            failure_report_save = failure_report
-            failure_report = 'Failed tests (suites/files):\n'
-            failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites)
-            if failure_report_save:
-                failure_report += '\n' + failure_report_save
+            failure_report = ['Failed tests (suites/files):'] + failure_report
+            # Test suites where we didn't match a specific regexp and hence likely didn't count the failures
+            failure_report.extend('+ %s' % t for t in sorted(all_failed_test_suites - failed_test_suites))
+            # Test suites not included in the catch-all regexp but counted. Should be empty.
+            failure_report.extend('? %s' % t for t in sorted(failed_test_suites - all_failed_test_suites))
+
+        failure_report = '\n'.join(failure_report)
 
         # Calculate total number of unsuccesful and total tests
         failed_test_cnt = failure_cnt + error_cnt