run ldd to obtain the required binaries when submodels is initialised

.gitignore

@@ -8,4 +8,5 @@ MANIFEST
 /docs/_build
 .coverage
 .ipynb_checkpoints
-.vscode
+.vscode
+/test/tmp/

payu/envmod.py

@@ -92,24 +92,21 @@ def module(command, *args):
     exec(envs)
 
 
-def lib_update(bin_path, lib_name):
+def lib_update(required_libs, lib_name):
     # Local import to avoid reversion interference
     # TODO: Bad design, fixme!
     # NOTE: We may be able to move this now that reversion is going away
     from payu import fsops
 
-    # TODO: Use objdump instead of ldd
-    cmd = 'ldd {0}'.format(bin_path)
-    ldd_output = subprocess.check_output(shlex.split(cmd)).decode('ascii')
-    slibs = ldd_output.split('\n')
-
-    for lib_entry in slibs:
-        if lib_name in lib_entry and 'spack' not in lib_entry:
-            # Only load enviroment modules available - assuming spack built libs includes 'spack' in full path of library
-            lib_path = lib_entry.split()[2]
-
+    for lib_filename, lib_path in required_libs.items():
+        if lib_filename.startswith(lib_name) and lib_path.startswith('/apps/'): 
+            # Load nci's /apps/ version of module if required 
             # pylint: disable=unbalanced-tuple-unpacking
             mod_name, mod_version = fsops.splitpath(lib_path)[2:4]
 
             module('unload', mod_name)
-            module('load', os.path.join(mod_name, mod_version))
+            module('load', os.path.join(mod_name, mod_version))
+            return '{0}/{1}'.format(mod_name, mod_version)
+
+    # If there are no libraries, return an empty string
+    return ''

payu/experiment.py

@@ -25,7 +25,7 @@
 
 # Local
 from payu import envmod
-from payu.fsops import mkdir_p, make_symlink, read_config, movetree
+from payu.fsops import mkdir_p, make_symlink, read_config, movetree, required_libs
 from payu.schedulers.pbs import get_job_info, pbs_env_init, get_job_id
 from payu.models import index as model_index
 import payu.profilers
@@ -138,6 +138,9 @@ def init_models(self):
 
         submodels = self.config.get('submodels', [])
 
+        for sm in submodels:
+            sm['required_libs'] = required_libs(sm['exe'])
+
         solo_model = self.config.get('model')
         if not solo_model:
             sys.exit('payu: error: Unknown model configuration.')
@@ -515,7 +518,7 @@ def run(self, *user_flags):
             # TODO: Check for MPI library mismatch across multiple binaries
             if mpi_module is None:
                 envmod.lib_update(
-                    model.exec_path_local,
+                    model.config.get('required_libs'),
                     'libmpi.so'
                 )
 

payu/fsops.py

@@ -12,6 +12,8 @@
 import os
 import shutil
 import sys
+import shlex
+import subprocess
 
 # Extensions
 import yaml
@@ -171,3 +173,41 @@ def is_conda():
     """Return True if python interpreter is in a conda environment"""
 
     return os.path.exists(os.path.join(sys.prefix, 'conda-meta'))
+
+
+def required_libs(bin_path):
+    """
+    Parses ldd output.
+    This function should only be called once per binary
+    i.e. Use a singleton pattern in the caller object.
+    PARAMETERS:
+        string bin_path: full path to the binary
+    RETURN:
+        dict: {filename-of-lib: fullpath-of-file}
+    """
+
+    # NOTE: ldd <binary>
+
+    # Example 1:
+    # libmpi.so.40 => /apps/openmpi/4.0.2/lib/libmpi.so.40 (0x00007fa493665000)
+    # libmpi_usempif08_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_usempif08_Intel.so.40 (0x00007fa492a7c000)
+    # libmpi_usempi_ignore_tkr_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_usempi_ignore_tkr_Intel.so.40 (0x00007fa492863000)
+    # libmpi_mpifh_Intel.so.40 => /apps/openmpi/4.0.2/lib/libmpi_mpifh_Intel.so.40 (0x00007fa4925cb000)
+
+    # Example 2:
+    # libmpi_usempif08.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempif08.so.40 (0x00007f12671c0000)
+    # libmpi_usempi_ignore_tkr.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempi_ignore_tkr.so.40 (0x00007f1266fb4000)
+    # libmpi_mpifh.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_mpifh.so.40 (0x00007f1266d44000)
+    # libmpi.so.40 => $SPACKDIR/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi.so.40 (0x00007f12667c3000)
+
+    needed_libs = {}
+    cmd = 'ldd {0}'.format(bin_path)
+    ldd_out = subprocess.check_output(shlex.split(cmd)).decode('ascii')
+
+    # awk '$2 ~ /=>/{print $1" "$3}'
+    for line in ldd_out.split("\n"):
+        word_list = line.split()
+        if len(word_list) >= 3 and word_list[1] == '=>':
+            needed_libs[word_list[0]] = word_list[2]
+    print("Needed libs: ", needed_libs)
+    return needed_libs

payu/models/fms.py

@@ -19,6 +19,7 @@
 
 from payu.models.model import Model
 from payu import envmod
+from payu.fsops import required_libs
 
 # There is a limit on the number of command line arguments in a forked
 # MPI process. This applies only to mppnccombine-fast. The limit is higher
@@ -109,7 +110,7 @@ def fms_collate(model):
         # and mppnccombine-fast uses an explicit -o flag to specify
         # the output
         collate_flags = " ".join([collate_flags, '-o'])
-        envmod.lib_update(mppnc_path, 'libmpi.so')
+        envmod.lib_update(required_libs(mppnc_path), 'libmpi.so')
 
     # Import list of collated files to ignore
     collate_ignore = collate_config.get('ignore')

test/test_payu.py

@@ -10,6 +10,7 @@
 import payu
 import payu.fsops
 import payu.laboratory
+import payu.envmod
 
 from .common import testdir, tmpdir, ctrldir, labdir, workdir
 from .common import make_exe, make_inputs, make_restarts, make_all_files
@@ -214,3 +215,21 @@ def test_lab_new():
     sys.stdout = StringIO()
     lab = payu.laboratory.Laboratory('model')
     sys.stdout = sys.__stdout__
+
+
+def test_lib_update_lib_if_required():
+    required_libs_dict = {
+        'libmpi.so.40': '/apps/openmpi/4.0.2/lib/libmpi.so.40',
+        'libmpi_usempif08_Intel.so.40': '/apps/openmpi/4.0.2/lib/libmpi_usempif08_Intel.so.40'
+    }
+    result = payu.envmod.lib_update(required_libs_dict, 'libmpi.so')
+    assert(result == 'openmpi/4.0.2')
+
+
+def test_lib_update_if_nci_module_not_required():
+    required_libs_dict = {
+         'libmpi.so.40': '/$HOME/spack-microarchitectures.git/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi.so.40', 
+         'libmpi_usempif08.so.40': '/$HOME/exe/spack-microarchitectures.git/opt/spack/linux-rocky8-cascadelake/intel-2019.5.281/openmpi-4.1.5-ooyg5wc7sa3tvmcpazqqb44pzip3wbyo/lib/libmpi_usempif08.so.40',
+    }
+    result = payu.envmod.lib_update(required_libs_dict, 'libmpi.so')
+    assert(result == '')