Merge pull request #356 from jo-basevi/354-solo-model-req-libs

Store required libs on model object

payu/experiment.py

@@ -138,17 +138,9 @@ def init_models(self):
 
         submodels = self.config.get('submodels', [])
 
-        # Inject information about required dynamically loaded libraries into submodel configuration
-        for sm in submodels:
-            sm['required_libs'] = required_libs(sm['exe'])
-
-        solo_model = self.config.get('model')
-        if not solo_model:
-            sys.exit('payu: error: Unknown model configuration.')
-
         submodel_config = dict((f, self.config[f]) for f in model_fields
                                if f in self.config)
-        submodel_config['name'] = solo_model
+        submodel_config['name'] = self.model_name
 
         submodels.append(submodel_config)
 
@@ -521,9 +513,9 @@ def run(self, *user_flags):
 
             # Update MPI library module (if not explicitly set)
             # TODO: Check for MPI library mismatch across multiple binaries
-            if mpi_module is None:
+            if mpi_module is None and model.required_libs is not None:
                 envmod.lib_update(
-                    model.config.get('required_libs'),
+                    model.required_libs,
                     'libmpi.so'
                 )
 

payu/fsops.py

@@ -202,5 +202,9 @@ def required_libs(bin_path):
         dict: {filename-of-lib: fullpath-of-file}
     """
     cmd = 'ldd {0}'.format(bin_path)
-    ldd_out = subprocess.check_output(shlex.split(cmd)).decode('ascii')
+    try: 
+        ldd_out = subprocess.check_output(shlex.split(cmd)).decode('ascii')
+    except:
+        print("payu: error running ldd command on exe path: ", bin_path)
+        return {}
     return parse_ldd_output(ldd_out)

payu/models/model.py

@@ -11,7 +11,7 @@
 import subprocess as sp
 
 from payu import envmod
-from payu.fsops import mkdir_p
+from payu.fsops import mkdir_p, required_libs
 
 
 class Model(object):
@@ -52,6 +52,8 @@ def __init__(self, expt, model_name, model_config):
         self.build_exec_path = None
         self.build_path = None
 
+        self.required_libs = None
+
         # Control flags
         self.copy_restarts = False
         self.copy_inputs = False
@@ -291,6 +293,9 @@ def setup(self):
                     self.exec_path_local,
                     self.exec_path
                 )
+
+            # Populate information about required dynamically loaded libraries
+            self.required_libs = required_libs(self.exec_path)
 
         timestep = self.config.get('timestep')
         if timestep:

payu/schedulers/pbs.py

@@ -108,6 +108,8 @@ def submit(self, pbs_script, pbs_config, pbs_vars=None, python_exe=None):
         short_path = pbs_config.get('shortpath', None)
         if short_path is not None:
             extra_search_paths.append(short_path)
+        module_use_paths = pbs_config.get('modules', {}).get('use', [])
+        extra_search_paths.extend(module_use_paths)
 
         storages.update(find_mounts(extra_search_paths, mounts))
         storages.update(find_mounts(get_manifest_paths(), mounts))

test/test_pbs.py

@@ -167,6 +167,9 @@ def test_run():
         config['laboratory'] = '/f/data/c000/blah'
         config['shortpath'] = '/f/data/y00'
 
+        config['modules'] = {}
+        config['modules']['use'] = ['/f/data/mm01', '/f/data/mm02/test/modules']
+
         cmd = sched.submit(payu_cmd, config, pbs_vars, python_exe)
 
         print(cmd)
@@ -210,8 +213,8 @@ def test_run():
             assert(resources_found[resource] == str(config[resource]))
 
         assert(resources_found['storage'] ==
-               ('fdata/a000+fdata/c000+fdata/m000+fdata/x00+' +
-                'fdata/xyz999+fdata/y00+test/x00'))
+               ('fdata/a000+fdata/c000+fdata/m000+fdata/mm01+fdata/mm02+' +
+                'fdata/x00+fdata/xyz999+fdata/y00+test/x00'))
 
         # Check other auto-added resources are present
         for resource in other_resources: