diff --git a/RELEASE_NOTES b/RELEASE_NOTES index 8dd90e0374..027e4a47e3 100644 --- a/RELEASE_NOTES +++ b/RELEASE_NOTES @@ -6,6 +6,31 @@ These release notes can also be consulted at http://easybuild.readthedocs.org/en The latest version of easybuild-easyblocks provides 251 software-specific easyblocks and 42 generic easyblocks. +v4.8.2 (29 October 2023) +------------------------ + +update/bugfix release + +- minor enhancements and updates, including: + - allow use of `test_cmd` without `runtest` for `ConfigureMake` (#2837) + - enhance `CMakeMake` easyblock to run `ctest` command if `runtest` is `True` (#2838) + - enhance `Conda` easyblock: add support for using custom `conda` command (like `mamba`), and clean up after installation (#2996) + - update WRF easyblock to correctly determine `WRF*` subdirectory for WRF versions >= 4.5.1 (#2997) + - update numpy easyblock to be able to use FlexiBLAS with MKL as dependency (#2999) + - add error regexp for PyTorch tests run in subprocess and enhance error message (#3003, #3005) + - don't blindly overwrite `-Dccflags` + honour `preconfigopts` in Perl easyblock (#3010) + - use more test programs in sanity check step of OpenMPI easyblock (#3016) + - fix `CMAKE_PREFIX_PATH` in imkl easyblock (#3021) + - update sanity check in wxPython easyblock for version 4.2.0 (#3023) +- various bug fixes, including: + - fix creation of symlink for libraries in TBB easyblock (#2927) + - fix `--sanity-check-only` and `--module-only` for UCX plugins (#3007) + - replace umlaut in docstring of PerlBundle easyblock (to fix Non-ASCII character error when using Python 2.7) (#3008) + - enhance TensorFlow easyblock to avoid use of `-mcpu=native` for XNNPACK component when building on aarch64 (#3011) + - only use `-DCMAKE_SKIP_RPATH=ON` for CMake < 3.5.0 (#3012) + - fix the blas/lapack name passed to meson when building recent scipy versions (>= 1.9.0) on top of Intel MKL (#3024) + + v4.8.1 (11 September 2023) -------------------------- diff --git a/easybuild/easyblocks/__init__.py b/easybuild/easyblocks/__init__.py index d8bbee0eb0..7d1118ed84 100644 --- a/easybuild/easyblocks/__init__.py +++ b/easybuild/easyblocks/__init__.py @@ -43,7 +43,7 @@ # recent setuptools versions will *TRANSFORM* something like 'X.Y.Zdev' into 'X.Y.Z.dev0', with a warning like # UserWarning: Normalizing '2.4.0dev' to '2.4.0.dev0' # This causes problems further up the dependency chain... -VERSION = LooseVersion('4.8.1') +VERSION = LooseVersion('4.8.2') UNKNOWN = 'UNKNOWN' diff --git a/easybuild/easyblocks/generic/cmakemake.py b/easybuild/easyblocks/generic/cmakemake.py index 48cdae593a..0401f872e4 100644 --- a/easybuild/easyblocks/generic/cmakemake.py +++ b/easybuild/easyblocks/generic/cmakemake.py @@ -39,7 +39,7 @@ from distutils.version import LooseVersion from easybuild.easyblocks.generic.configuremake import ConfigureMake -from easybuild.framework.easyconfig import CUSTOM +from easybuild.framework.easyconfig import BUILD, CUSTOM from easybuild.tools.build_log import EasyBuildError, print_warning from easybuild.tools.config import build_option from easybuild.tools.filetools import change_dir, create_unused_dir, mkdir, which @@ -105,6 +105,7 @@ def extra_options(extra_vars=None): 'configure_cmd': [DEFAULT_CONFIGURE_CMD, "Configure command to use", CUSTOM], 'generator': [None, "Build file generator to use. None to use CMakes default", CUSTOM], 'install_target_subdir': [None, "Subdirectory to use as installation target", CUSTOM], + 'runtest': [None, "Make target to test build or True to use CTest", BUILD], 'srcdir': [None, "Source directory location to provide to cmake command", CUSTOM], 'separate_build_dir': [True, "Perform build in a separate directory", CUSTOM], }) @@ -114,6 +115,7 @@ def __init__(self, *args, **kwargs): """Constructor for CMakeMake easyblock""" super(CMakeMake, self).__init__(*args, **kwargs) self._lib_ext = None + self._cmake_version = None self.separate_build_dir = None @property @@ -131,6 +133,14 @@ def lib_ext(self): def lib_ext(self, value): self._lib_ext = value + @property + def cmake_version(self): + """Return the used CMake version, caching the value for reuse""" + if self._cmake_version is None: + self._cmake_version = det_cmake_version() + self.log.debug('Determined CMake version: %s', self._cmake_version) + return self._cmake_version + @property def build_type(self): """Build type set in the EasyConfig with default determined by toolchainopts""" @@ -241,10 +251,8 @@ def configure_step(self, srcdir=None, builddir=None): if fc: setvar('FC', fc) - cmake_version = det_cmake_version() - # Flags are read from environment variables already since at least CMake 2.8.0 - if LooseVersion(cmake_version) < LooseVersion('2.8.0') or cache_exists: + if LooseVersion(self.cmake_version) < LooseVersion('2.8.0') or cache_exists: env_to_options.update({ 'CFLAGS': 'CMAKE_C_FLAGS', 'CXXFLAGS': 'CMAKE_CXX_FLAGS', @@ -259,9 +267,10 @@ def configure_step(self, srcdir=None, builddir=None): self.log.info("Using absolute path to compiler command: %s", value) options[option] = value - if build_option('rpath'): + if build_option('rpath') and LooseVersion(self.cmake_version) < LooseVersion('3.5.0'): # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install... - # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416 + # this is only required for CMake < 3.5.0, since newer version are more careful w.r.t. RPATH, + # see https://github.com/Kitware/CMake/commit/3ec9226779776811240bde88a3f173c29aa935b5 options['CMAKE_SKIP_RPATH'] = 'ON' # show what CMake is doing by default @@ -310,4 +319,12 @@ def test_step(self): """CMake specific test setup""" # When using ctest for tests (default) then show verbose output if a test fails setvar('CTEST_OUTPUT_ON_FAILURE', 'True') + # Handle `runtest = True` if `test_cmd` has not been set + if self.cfg.get('runtest') is True and not self.cfg.get('test_cmd'): + test_cmd = 'ctest' + if LooseVersion(self.cmake_version) >= '3.17.0': + test_cmd += ' --no-tests=error' + self.log.debug("`runtest = True` found, using '%s' as test_cmd", test_cmd) + self.cfg['test_cmd'] = test_cmd + super(CMakeMake, self).test_step() diff --git a/easybuild/easyblocks/generic/conda.py b/easybuild/easyblocks/generic/conda.py index b3e0fed5b0..03e78ab42e 100644 --- a/easybuild/easyblocks/generic/conda.py +++ b/easybuild/easyblocks/generic/conda.py @@ -34,6 +34,8 @@ from easybuild.easyblocks.generic.binary import Binary from easybuild.framework.easyconfig import CUSTOM from easybuild.tools.run import run_cmd +from easybuild.tools.modules import get_software_root +from easybuild.tools.build_log import EasyBuildError class Conda(Binary): @@ -57,11 +59,21 @@ def extract_step(self): super(Conda, self).extract_step() def install_step(self): - """Install software using 'conda env create' or 'conda create' & 'conda install'.""" + """Install software using 'conda env create' or 'conda create' & 'conda install' + (or the 'mamba', etc., equivalent).""" + if (get_software_root('anaconda2') or get_software_root('miniconda2') or + get_software_root('anaconda3') or get_software_root('miniconda3')): + conda_cmd = 'conda' + elif get_software_root('mamba'): + conda_cmd = 'mamba' + elif get_software_root('micromamba'): + conda_cmd = 'micromamba' + else: + raise EasyBuildError("No conda/mamba/micromamba available.") # initialize conda environment # setuptools is just a choice, but *something* needs to be there - cmd = "conda config --add create_default_packages setuptools" + cmd = "%s config --add create_default_packages setuptools" % conda_cmd run_cmd(cmd, log_all=True, simple=True) if self.cfg['environment_file'] or self.cfg['remote_environment']: @@ -72,7 +84,8 @@ def install_step(self): env_spec = self.cfg['remote_environment'] # use --force to ignore existing installation directory - cmd = "%s conda env create --force %s -p %s" % (self.cfg['preinstallopts'], env_spec, self.installdir) + cmd = "%s %s env create --force %s -p %s" % (self.cfg['preinstallopts'], conda_cmd, + env_spec, self.installdir) run_cmd(cmd, log_all=True, simple=True) else: @@ -85,9 +98,14 @@ def install_step(self): self.log.info("Installed conda requirements") - cmd = "%s conda create --force -y -p %s %s" % (self.cfg['preinstallopts'], self.installdir, install_args) + cmd = "%s %s create --force -y -p %s %s" % (self.cfg['preinstallopts'], conda_cmd, + self.installdir, install_args) run_cmd(cmd, log_all=True, simple=True) + # clean up + cmd = "%s clean -ya" % conda_cmd + run_cmd(cmd, log_all=True, simple=True) + def make_module_extra(self): """Add the install directory to the PATH.""" txt = super(Conda, self).make_module_extra() diff --git a/easybuild/easyblocks/generic/configuremake.py b/easybuild/easyblocks/generic/configuremake.py index a8527e1088..3b88de8f6c 100644 --- a/easybuild/easyblocks/generic/configuremake.py +++ b/easybuild/easyblocks/generic/configuremake.py @@ -48,6 +48,7 @@ from easybuild.tools.config import source_paths, build_option from easybuild.tools.filetools import CHECKSUM_TYPE_SHA256, adjust_permissions, compute_checksum, download_file from easybuild.tools.filetools import read_file, remove_file +from easybuild.tools.py2vs3 import string_type from easybuild.tools.run import run_cmd # string that indicates that a configure script was generated by Autoconf @@ -192,7 +193,8 @@ def extra_options(extra_vars=None): 'install_cmd': [DEFAULT_INSTALL_CMD, "Install command to use", CUSTOM], 'prefix_opt': [None, "Prefix command line option for configure script ('--prefix=' if None)", CUSTOM], 'tar_config_opts': [False, "Override tar settings as determined by configure.", CUSTOM], - 'test_cmd': [DEFAULT_TEST_CMD, "Test command to use ('runtest' value is appended)", CUSTOM], + 'test_cmd': [None, "Test command to use ('runtest' value is appended, default: '%s')" % DEFAULT_TEST_CMD, + CUSTOM], }) return extra_vars @@ -337,7 +339,7 @@ def build_step(self, verbose=False, path=None): targets = self.cfg.get('build_cmd_targets') or DEFAULT_BUILD_TARGET # ensure strings are converted to list - targets = [targets] if isinstance(targets, str) else targets + targets = [targets] if isinstance(targets, string_type) else targets for target in targets: cmd = ' '.join([ @@ -360,13 +362,13 @@ def test_step(self): """ test_cmd = self.cfg.get('test_cmd') or DEFAULT_TEST_CMD - if self.cfg['runtest'] or test_cmd != DEFAULT_TEST_CMD: - cmd = ' '.join([ - self.cfg['pretestopts'], - test_cmd, - self.cfg['runtest'], - self.cfg['testopts'], - ]) + runtest = self.cfg['runtest'] + if runtest or test_cmd != DEFAULT_TEST_CMD: + # Make run_test a string (empty if it is e.g. a boolean) + if not isinstance(runtest, string_type): + runtest = '' + # Compose command filtering out empty values + cmd = ' '.join([x for x in (self.cfg['pretestopts'], test_cmd, runtest, self.cfg['testopts']) if x]) (out, _) = run_cmd(cmd, log_all=True, simple=False) return out diff --git a/easybuild/easyblocks/generic/perlbundle.py b/easybuild/easyblocks/generic/perlbundle.py index fcb5ce48f8..43cd4545c1 100644 --- a/easybuild/easyblocks/generic/perlbundle.py +++ b/easybuild/easyblocks/generic/perlbundle.py @@ -25,7 +25,7 @@ """ EasyBuild support for installing a bundle of Perl modules, implemented as a generic easyblock -@author: Mikael Öhman (Chalmers University of Technology) +@author: Mikael Oehman (Chalmers University of Technology) """ import os diff --git a/easybuild/easyblocks/i/imkl.py b/easybuild/easyblocks/i/imkl.py index 978b42137c..a342a4686c 100644 --- a/easybuild/easyblocks/i/imkl.py +++ b/easybuild/easyblocks/i/imkl.py @@ -546,11 +546,13 @@ def make_module_req_guess(self): os.path.join(self.mkl_basedir, 'include'), os.path.join(self.mkl_basedir, 'include', 'fftw'), ] + cmake_prefix_path = [self.mkl_basedir] guesses.update({ 'PATH': [], 'LD_LIBRARY_PATH': library_path, 'LIBRARY_PATH': library_path, 'CPATH': cpath, + 'CMAKE_PREFIX_PATH': cmake_prefix_path, 'PKG_CONFIG_PATH': pkg_config_path, }) if self.cfg['flexiblas']: diff --git a/easybuild/easyblocks/n/numpy.py b/easybuild/easyblocks/n/numpy.py index f72e2671d4..f42b914e55 100644 --- a/easybuild/easyblocks/n/numpy.py +++ b/easybuild/easyblocks/n/numpy.py @@ -79,7 +79,11 @@ def configure_step(self): "search_static_first=True", ]) - if get_software_root("imkl"): + # If both FlexiBLAS and MKL are found, we assume that FlexiBLAS has a dependency on MKL. + # In this case we want to link to FlexiBLAS and not directly to MKL. + imkl_direct = get_software_root("imkl") and not get_software_root("FlexiBLAS") + + if imkl_direct: if self.toolchain.comp_family() == toolchain.GCC: # see https://software.intel.com/en-us/articles/numpyscipy-with-intel-mkl, @@ -112,7 +116,7 @@ def configure_step(self): lapack = None fft = None - if get_software_root("imkl"): + if imkl_direct: # with IMKL, no spaces and use '-Wl:' # redefine 'Wl,' to 'Wl:' so that the patch file can do its job def get_libs_for_mkl(varname): diff --git a/easybuild/easyblocks/o/openmpi.py b/easybuild/easyblocks/o/openmpi.py index cf7c73151f..e76dd131bf 100644 --- a/easybuild/easyblocks/o/openmpi.py +++ b/easybuild/easyblocks/o/openmpi.py @@ -211,8 +211,18 @@ def sanity_check_step(self): mpi_cmd_tmpl, params = get_mpi_cmd_template(toolchain.OPENMPI, dict(), mpi_version=self.version) # Limit number of ranks to 8 to avoid it failing due to hyperthreading ranks = min(8, self.cfg['parallel']) - for src, compiler in (('hello_c.c', 'mpicc'), ('hello_mpifh.f', 'mpifort'), ('hello_usempi.f90', 'mpif90')): - src_path = os.path.join(self.cfg['start_dir'], 'examples', src) + for srcdir, src, compiler in ( + ('examples', 'hello_c.c', 'mpicc'), + ('examples', 'hello_mpifh.f', 'mpifort'), + ('examples', 'hello_usempi.f90', 'mpif90'), + ('examples', 'ring_c.c', 'mpicc'), + ('examples', 'ring_mpifh.f', 'mpifort'), + ('examples', 'ring_usempi.f90', 'mpif90'), + ('test/simple', 'thread_init.c', 'mpicc'), + ('test/simple', 'intercomm1.c', 'mpicc'), + ('test/simple', 'mpi_barrier.c', 'mpicc'), + ): + src_path = os.path.join(self.cfg['start_dir'], srcdir, src) if os.path.exists(src_path): test_exe = os.path.join(self.builddir, 'mpi_test_' + os.path.splitext(src)[0]) self.log.info("Adding minimal MPI test program to sanity checks: %s", test_exe) diff --git a/easybuild/easyblocks/p/perl.py b/easybuild/easyblocks/p/perl.py index d96505ce7d..11542473cf 100644 --- a/easybuild/easyblocks/p/perl.py +++ b/easybuild/easyblocks/p/perl.py @@ -75,7 +75,7 @@ def configure_step(self): configopts = [ self.cfg['configopts'], '-Dcc="{0}"'.format(os.getenv('CC')), - '-Dccflags="{0}"'.format(os.getenv('CFLAGS')), + '-Dccflags="{0}"'.format(os.getenv('CFLAGS')) if '-Dccflags' not in self.cfg['configopts'] else '', '-Dinc_version_list=none', '-Dprefix=%(installdir)s', # guarantee that scripts are installed in /bin in the installation directory (and not in a guessed path) @@ -116,7 +116,7 @@ def configure_step(self): if os.getenv('COLUMNS', None) == '0': unset_env_vars(['COLUMNS']) - cmd = './Configure -de %s' % configopts + cmd = '%s ./Configure -de %s' % (self.cfg['preconfigopts'], configopts) run_cmd(cmd, log_all=True, simple=True) def test_step(self): diff --git a/easybuild/easyblocks/p/pytorch.py b/easybuild/easyblocks/p/pytorch.py index 86da42a51a..58a5e9bdb8 100644 --- a/easybuild/easyblocks/p/pytorch.py +++ b/easybuild/easyblocks/p/pytorch.py @@ -318,7 +318,7 @@ def get_count_for_pattern(regex, text): return 0 # Create clear summary report - failure_report = "" + failure_report = [] failure_cnt = 0 error_cnt = 0 failed_test_suites = [] @@ -337,9 +337,9 @@ def get_count_for_pattern(regex, text): # E.g. 'failures=3, errors=10, skipped=190, expected failures=6' failure_summary = m.group('failure_summary') total, test_suite = m.group('test_cnt', 'failed_test_suite_name') - failure_report += "{test_suite} ({total} total tests, {failure_summary})\n".format( + failure_report.append("{test_suite} ({total} total tests, {failure_summary})".format( test_suite=test_suite, total=total, failure_summary=failure_summary - ) + )) failure_cnt += get_count_for_pattern(r"(?) + # + # distributed/test_c10d_nccl failed! + + regex = ( + r"^AssertionError: (?P[0-9]+ unit test\(s\) failed):\n" + r"(\s+.*\n)+" + r"(((?!failed!).)*\n){0,5}" + r"(?P.*) failed!$" + ) + + for m in re.finditer(regex, tests_out, re.M): + # E.g. '2 unit test(s) failed' + failure_summary = m.group('failure_summary') + test_suite = m.group('failed_test_suite_name') + failure_report.append("{test_suite} ({failure_summary})".format( + test_suite=test_suite, failure_summary=failure_summary + )) + failure_cnt += get_count_for_pattern(r"([0-9]+) unit test\(s\) failed", failure_summary) + failed_test_suites.append(test_suite) + + # Make the names unique + failed_test_suites = set(failed_test_suites) # Gather all failed tests suites in case we missed any (e.g. when it exited due to syntax errors) - # Also unique and sorted to be able to compare the lists below - all_failed_test_suites = sorted(set( + # Also unique to be able to compare the lists below + all_failed_test_suites = set( re.findall(r"^(?P.*) failed!(?: Received signal: \w+)?\s*$", tests_out, re.M) - )) + ) # If we missed any test suites prepend a list of all failed test suites if failed_test_suites != all_failed_test_suites: - failure_report_save = failure_report - failure_report = 'Failed tests (suites/files):\n' - failure_report += '\n'.join('* %s' % t for t in all_failed_test_suites) - if failure_report_save: - failure_report += '\n' + failure_report_save + failure_report = ['Failed tests (suites/files):'] + failure_report + # Test suites where we didn't match a specific regexp and hence likely didn't count the failures + failure_report.extend('+ %s' % t for t in sorted(all_failed_test_suites - failed_test_suites)) + # Test suites not included in the catch-all regexp but counted. Should be empty. + failure_report.extend('? %s' % t for t in sorted(failed_test_suites - all_failed_test_suites)) + + failure_report = '\n'.join(failure_report) # Calculate total number of unsuccesful and total tests failed_test_cnt = failure_cnt + error_cnt diff --git a/easybuild/easyblocks/s/scipy.py b/easybuild/easyblocks/s/scipy.py index fa82d0a866..094a0259b5 100644 --- a/easybuild/easyblocks/s/scipy.py +++ b/easybuild/easyblocks/s/scipy.py @@ -124,7 +124,7 @@ def configure_step(self): if lapack_lib == toolchain.FLEXIBLAS: blas_lapack = 'flexiblas' elif lapack_lib == toolchain.INTELMKL: - blas_lapack = 'mkl' + blas_lapack = 'mkl-dynamic-lp64-seq' elif lapack_lib == toolchain.OPENBLAS: blas_lapack = 'openblas' else: diff --git a/easybuild/easyblocks/t/tbb.py b/easybuild/easyblocks/t/tbb.py index 1ce9ee6750..e3f34e1ab8 100644 --- a/easybuild/easyblocks/t/tbb.py +++ b/easybuild/easyblocks/t/tbb.py @@ -43,7 +43,7 @@ from easybuild.easyblocks.generic.intelbase import INSTALL_MODE_NAME_2015, INSTALL_MODE_2015 from easybuild.easyblocks.generic.intelbase import IntelBase, ACTIVATION_NAME_2012, LICENSE_FILE_NAME_2012 from easybuild.framework.easyconfig import CUSTOM -from easybuild.tools.filetools import move_file, symlink +from easybuild.tools.filetools import find_glob_pattern, move_file, symlink from easybuild.tools.build_log import EasyBuildError from easybuild.tools.modules import get_software_version from easybuild.tools.systemtools import POWER, get_cpu_architecture, get_gcc_version, get_platform_name @@ -173,39 +173,37 @@ def install_step(self): IntelBase.install_step(self, silent_cfg_names_map=silent_cfg_names_map, silent_cfg_extras=silent_cfg_extras) # determine libdir - os.chdir(self.installdir) - libpath = os.path.join('tbb', 'libs', 'intel64') + libpath = os.path.join(self.installdir, 'tbb', 'libs', 'intel64') if LooseVersion(self.version) < LooseVersion('4.1.0'): - libglob = 'tbb/lib/intel64/cc*libc*_kernel*' + libglob = os.path.join(libpath, 'cc*libc*_kernel*') libs = sorted(glob.glob(libglob), key=LooseVersion) if libs: # take the last one, should be ordered by cc version # we're only interested in the last bit - libdir = os.path.basename(libs[-1]) + libpath = libs[-1] else: raise EasyBuildError("No libs found using %s in %s", libglob, self.installdir) else: - libdir = get_tbb_gccprefix(libpath) + libpath = os.path.join(libpath, get_tbb_gccprefix(libpath)) - libpath = os.path.join(libpath, libdir) # applications go looking into tbb/lib so we move what's in there to tbb/libs shutil.move(install_tbb_lib_path, os.path.join(self.installdir, 'tbb', 'libs')) # And add a symlink of the library folder to tbb/lib - symlink(os.path.join(self.installdir, libpath), install_tbb_lib_path) + symlink(libpath, install_tbb_lib_path) else: # no custom install step when building from source (building is done in install directory) - cand_lib_paths = glob.glob(os.path.join(self.installdir, 'build', '*_release')) - if len(cand_lib_paths) == 1: - libpath = os.path.join('build', os.path.basename(cand_lib_paths[0])) - else: - raise EasyBuildError("Failed to isolate location of libraries: %s", cand_lib_paths) + libpath = find_glob_pattern(os.path.join(self.installdir, 'build', '*_release')) - self.log.debug("libpath: %s" % libpath) - # applications usually look into /lib, so we move the folder there and symlink the original one to it + real_libpath = os.path.realpath(libpath) + self.log.debug("libpath: %s, resolved: %s" % (libpath, real_libpath)) + libpath = real_libpath + # applications usually look into /lib, so we move the folder there # This is also important so that /lib and /lib64 are actually on the same level root_lib_path = os.path.join(self.installdir, 'lib') move_file(libpath, root_lib_path) - symlink(os.path.relpath(root_lib_path, os.path.join(libpath)), libpath, use_abspath_source=False) + # Create a relative symlink at the original location as-if we didn't move it. + # Note that the path must be relative from the folder where the symlink will be! + symlink(os.path.relpath(root_lib_path, os.path.dirname(libpath)), libpath, use_abspath_source=False) # Install CMake config files if possible if self._has_cmake() and LooseVersion(self.version) >= LooseVersion('2020.0'): diff --git a/easybuild/easyblocks/t/tensorflow.py b/easybuild/easyblocks/t/tensorflow.py index de9b97296a..926a1920cf 100644 --- a/easybuild/easyblocks/t/tensorflow.py +++ b/easybuild/easyblocks/t/tensorflow.py @@ -48,7 +48,7 @@ from easybuild.tools.filetools import is_readable, read_file, which, write_file, remove_file from easybuild.tools.modules import get_software_root, get_software_version, get_software_libdir from easybuild.tools.run import run_cmd -from easybuild.tools.systemtools import X86_64, get_cpu_architecture, get_os_name, get_os_version +from easybuild.tools.systemtools import AARCH64, X86_64, get_cpu_architecture, get_os_name, get_os_version CPU_DEVICE = 'cpu' @@ -687,6 +687,19 @@ def configure_step(self): cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts'] run_cmd(cmd, log_all=True, simple=True) + # when building on Arm 64-bit we can't just use --copt=-mcpu=native (or likewise for any -mcpu=...), + # because it breaks the build of XNNPACK; + # see also https://github.com/easybuilders/easybuild-easyconfigs/issues/18899 + if get_cpu_architecture() == AARCH64: + tf_conf_bazelrc = os.path.join(self.start_dir, '.tf_configure.bazelrc') + regex_subs = [ + # use --per_file_copt instead of --copt to selectively use -mcpu=native (not for XNNPACK), + # the leading '-' ensures that -mcpu=native is *not* used when building XNNPACK; + # see https://github.com/google/XNNPACK/issues/5566 + https://bazel.build/docs/user-manual#per-file-copt + ('--copt=-mcpu=', '--per_file_copt=-.*XNNPACK/.*@-mcpu='), + ] + apply_regex_substitutions(tf_conf_bazelrc, regex_subs) + def patch_crosstool_files(self): """Patches the CROSSTOOL files to include EasyBuild provided compiler paths""" inc_paths, lib_paths = [], [] diff --git a/easybuild/easyblocks/u/ucx_plugins.py b/easybuild/easyblocks/u/ucx_plugins.py index 10e5a97e13..a00512c48c 100644 --- a/easybuild/easyblocks/u/ucx_plugins.py +++ b/easybuild/easyblocks/u/ucx_plugins.py @@ -29,6 +29,7 @@ @author: Mikael Öhman (Chalmers University of Techonology) """ from collections import defaultdict +from itertools import chain import os from easybuild.easyblocks.generic.configuremake import ConfigureMake @@ -43,16 +44,37 @@ class EB_UCX_Plugins(ConfigureMake): """Support for building additional plugins for a existing UCX module""" def __init__(self, *args, **kwargs): - """Custom initialization for UCX: set correct module name.""" + """Custom initialization for UCX-Plugins.""" super(EB_UCX_Plugins, self).__init__(*args, **kwargs) - self.plugins = {} + self._plugins = None self.makefile_dirs = [] + @property + def plugins(self): + """Property to determine list of plugins based on loaded dependencies, or return cached list of plugins.""" + if self._plugins is None: + plugins = defaultdict(list) + dep_names = self.cfg.dependency_names() + + if 'CUDAcore' in dep_names or 'CUDA' in dep_names: + for key in ('ucm', 'uct', 'ucx_perftest'): + plugins[key].append('cuda') + + if 'GDRCopy' in dep_names: + plugins['uct_cuda'].append('gdrcopy') + + if 'ROCm' in dep_names: + for key in ('ucm', 'uct', 'ucx_perftest'): + plugins[key].append('rocm') + + self._plugins = dict(plugins) + self.log.info("Creating plugins for %s", ", ".join(sorted(set(chain(*plugins.values()))))) + return self._plugins + def configure_step(self): """Customize configuration for building requested plugins.""" # make sure that required dependencies are loaded - ucxroot = get_software_root('UCX') - if not ucxroot: + if not get_software_root('UCX'): raise EasyBuildError("UCX is a required dependency") self.cfg['configure_cmd'] = 'contrib/configure-release' @@ -62,30 +84,21 @@ def configure_step(self): # omit the lib subdirectory since we are just installing plugins configopts += '--libdir=%(installdir)s ' - plugins = defaultdict(list) cudaroot = get_software_root('CUDAcore') or get_software_root('CUDA') if cudaroot: configopts += '--with-cuda=%s ' % cudaroot - for key in ('ucm', 'uct', 'ucx_perftest'): - plugins[key].append('cuda') gdrcopyroot = get_software_root('GDRCopy') if gdrcopyroot: - plugins['uct_cuda'].append('gdrcopy') configopts += '--with-gdrcopy=%s ' % gdrcopyroot self.makefile_dirs.extend(os.path.join(x, 'cuda') for x in ('uct', 'ucm', 'tools/perf')) - # To be supported in the future: rocmroot = get_software_root('ROCm') if rocmroot: - for key in ('ucm', 'uct', 'ucx_perftest'): - plugins[key].append('rocm') configopts += '--with-rocm=%s ' % rocmroot self.makefile_dirs.extend(os.path.join(x, 'rocm') for x in ('uct', 'ucm', 'tools/perf')) - self.plugins = dict(plugins) - self.cfg.update('configopts', configopts) super(EB_UCX_Plugins, self).configure_step() diff --git a/easybuild/easyblocks/w/wrf.py b/easybuild/easyblocks/w/wrf.py index b232400724..ddd6387fd8 100644 --- a/easybuild/easyblocks/w/wrf.py +++ b/easybuild/easyblocks/w/wrf.py @@ -56,6 +56,8 @@ def det_wrf_subdir(wrf_version): if LooseVersion(wrf_version) < LooseVersion('4.0'): wrf_subdir = 'WRFV%s' % wrf_version.split('.')[0] + elif LooseVersion(wrf_version) >= LooseVersion('4.5.1'): + wrf_subdir = 'WRFV%s' % wrf_version else: wrf_subdir = 'WRF-%s' % wrf_version diff --git a/easybuild/easyblocks/w/wxpython.py b/easybuild/easyblocks/w/wxpython.py index 35b22b1867..939e2718a1 100644 --- a/easybuild/easyblocks/w/wxpython.py +++ b/easybuild/easyblocks/w/wxpython.py @@ -127,6 +127,9 @@ def sanity_check_step(self): files.extend([os.path.join('bin', 'wxrc')]) dirs.extend(['include', 'share']) py_bins.extend(['alacarte', 'alamode', 'wrap']) + elif LooseVersion(self.version) >= LooseVersion("4.2"): + majver = '3.2' # this is 3.2 in ver 4.2.x + py_bins.extend(['slices', 'slicesshell']) elif LooseVersion(self.version) >= LooseVersion("4.1"): majver = '3.1' # this is 3.1 in ver 4.1.x py_bins.extend(['slices', 'slicesshell'])