From 250303a500b1036c2e42723c8d76c176efda329b Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Mon, 31 Jul 2023 16:15:50 +1000 Subject: [PATCH] run ldd to obtain the required binaries when submodels is initialised --- .gitignore | 3 ++- payu/envmod.py | 21 +++++++++------------ payu/experiment.py | 7 +++++-- payu/fsops.py | 40 ++++++++++++++++++++++++++++++++++++++++ payu/models/fms.py | 3 ++- test/test_payu.py | 19 +++++++++++++++++++ 6 files changed, 77 insertions(+), 16 deletions(-) diff --git a/.gitignore b/.gitignore index 015dcb77..47c37851 100644 --- a/.gitignore +++ b/.gitignore @@ -8,4 +8,5 @@ MANIFEST /docs/_build .coverage .ipynb_checkpoints -.vscode \ No newline at end of file +.vscode +/test/tmp/ \ No newline at end of file diff --git a/payu/envmod.py b/payu/envmod.py index e9ab1b54..a74bedea 100644 --- a/payu/envmod.py +++ b/payu/envmod.py @@ -92,24 +92,21 @@ def module(command, *args): exec(envs) -def lib_update(bin_path, lib_name): +def lib_update(required_libs, lib_name): # Local import to avoid reversion interference # TODO: Bad design, fixme! # NOTE: We may be able to move this now that reversion is going away from payu import fsops - # TODO: Use objdump instead of ldd - cmd = 'ldd {0}'.format(bin_path) - ldd_output = subprocess.check_output(shlex.split(cmd)).decode('ascii') - slibs = ldd_output.split('\n') - - for lib_entry in slibs: - if lib_name in lib_entry and 'spack' not in lib_entry: - # Only load enviroment modules available - assuming spack built libs includes 'spack' in full path of library - lib_path = lib_entry.split()[2] - + for lib_filename, lib_path in required_libs.items(): + if lib_filename.startswith(lib_name) and lib_path.startswith('/apps/'): + # Load nci's /apps/ version of module if required # pylint: disable=unbalanced-tuple-unpacking mod_name, mod_version = fsops.splitpath(lib_path)[2:4] module('unload', mod_name) - module('load', os.path.join(mod_name, mod_version)) \ No newline at end of file + module('load', os.path.join(mod_name, mod_version)) + return '{0}/{1}'.format(mod_name, mod_version) + + # If there are no libraries, return an empty string + return '' \ No newline at end of file diff --git a/payu/experiment.py b/payu/experiment.py index 306acc14..b8363389 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -25,7 +25,7 @@ # Local from payu import envmod -from payu.fsops import mkdir_p, make_symlink, read_config, movetree +from payu.fsops import mkdir_p, make_symlink, read_config, movetree, required_libs from payu.schedulers.pbs import get_job_info, pbs_env_init, get_job_id from payu.models import index as model_index import payu.profilers @@ -138,6 +138,9 @@ def init_models(self): submodels = self.config.get('submodels', []) + for sm in submodels: + sm['required_libs'] = required_libs(sm['exe']) + solo_model = self.config.get('model') if not solo_model: sys.exit('payu: error: Unknown model configuration.') @@ -515,7 +518,7 @@ def run(self, *user_flags): # TODO: Check for MPI library mismatch across multiple binaries if mpi_module is None: envmod.lib_update( - model.exec_path_local, + model.config.get('required_libs'), 'libmpi.so' ) diff --git a/payu/fsops.py b/payu/fsops.py index b1b991dc..b8f0cc82 100644 --- a/payu/fsops.py +++ b/payu/fsops.py @@ -12,6 +12,8 @@ import os import shutil import sys +import shlex +import subprocess # Extensions import yaml @@ -171,3 +173,41 @@ def is_conda(): """Return True if python interpreter is in a conda environment""" return os.path.exists(os.path.join(sys.prefix, 'conda-meta')) + + +def required_libs(bin_path): + """ + Parses ldd output. + This function should only be called once per binary + i.e. Use a singleton pattern in the caller object. + PARAMETERS: + string bin_path: full path to the binary + RETURN: + dict: {filename-of-lib: fullpath-of-file} + """ + + # NOTE: ldd + + # Example 1: + # libmpi.so.40 => /apps/openmpi/4.0.2/lib/libmpi.so.40 (0x00007fa493665000) + # libmpi_usempif08_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_usempif08_Intel.so.40 (0x00007fa492a7c000) + # libmpi_usempi_ignore_tkr_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_usempi_ignore_tkr_Intel.so.40 (0x00007fa492863000) + # libmpi_mpifh_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_mpifh_Intel.so.40 (0x00007fa4925cb000) + + # Example 2: + # libmpi_usempif08.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempif08.so.40 (0x00007f12671c0000) + # libmpi_usempi_ignore_tkr.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempi_ignore_tkr.so.40 (0x00007f1266fb4000) + # libmpi_mpifh.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_mpifh.so.40 (0x00007f1266d44000) + # libmpi.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi.so.40 (0x00007f12667c3000) + + needed_libs = {} + cmd = 'ldd {0}'.format(bin_path) + ldd_out = subprocess.check_output(shlex.split(cmd)).decode('ascii') + + # awk '$2 ~ /=>/{print $1" "$3}' + for line in ldd_out.split("\n"): + word_list = line.split() + if len(word_list) >= 3 and word_list[1] == '=>': + needed_libs[word_list[0]] = word_list[2] + print("Needed libs: ", needed_libs) + return needed_libs \ No newline at end of file diff --git a/payu/models/fms.py b/payu/models/fms.py index e510bce0..f2f3e743 100644 --- a/payu/models/fms.py +++ b/payu/models/fms.py @@ -19,6 +19,7 @@ from payu.models.model import Model from payu import envmod +from payu.fsops import required_libs # There is a limit on the number of command line arguments in a forked # MPI process. This applies only to mppnccombine-fast. The limit is higher @@ -109,7 +110,7 @@ def fms_collate(model): # and mppnccombine-fast uses an explicit -o flag to specify # the output collate_flags = " ".join([collate_flags, '-o']) - envmod.lib_update(mppnc_path, 'libmpi.so') + envmod.lib_update(required_libs(mppnc_path), 'libmpi.so') # Import list of collated files to ignore collate_ignore = collate_config.get('ignore') diff --git a/test/test_payu.py b/test/test_payu.py index 78952752..92134dc9 100644 --- a/test/test_payu.py +++ b/test/test_payu.py @@ -10,6 +10,7 @@ import payu import payu.fsops import payu.laboratory +import payu.envmod from .common import testdir, tmpdir, ctrldir, labdir, workdir from .common import make_exe, make_inputs, make_restarts, make_all_files @@ -214,3 +215,21 @@ def test_lab_new(): sys.stdout = StringIO() lab = payu.laboratory.Laboratory('model') sys.stdout = sys.__stdout__ + + +def test_lib_update_lib_if_required(): + required_libs_dict = { + 'libmpi.so.40': '/apps/openmpi/4.0.2/lib/libmpi.so.40', + 'libmpi_usempif08_Intel.so.40': '/apps/openmpi/4.0.2/lib/libmpi_usempif08_Intel.so.40' + } + result = payu.envmod.lib_update(required_libs_dict, 'libmpi.so') + assert(result == 'openmpi/4.0.2') + + +def test_lib_update_if_nci_module_not_required(): + required_libs_dict = { + 'libmpi.so.40': '/$HOME/spack-microarchitectures.git/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi.so.40', + 'libmpi_usempif08.so.40': '/$HOME/exe/spack-microarchitectures.git/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempif08.so.40', + } + result = payu.envmod.lib_update(required_libs_dict, 'libmpi.so') + assert(result == '') \ No newline at end of file