Merge pull request #353 from jo-basevi/347-setting-module-use

Support setting 'module use' directories in config.yaml

.gitignore

@@ -9,4 +9,5 @@ MANIFEST
 .coverage
 .ipynb_checkpoints
 .vscode
+.idea
 /test/tmp/

docs/source/config.rst

@@ -427,17 +427,21 @@ Miscellaneous
    is generally only used for testing purposes, such as bit reproducibility.
 
 ``modules``
-   Specify a list of environment modules to load at the start of the PBS job,
-   for example::
+   Specify lists of environment modules and/or directories
+   to load/use at the start of the PBS job, for example::
 
       modules:
-         - netcdf-c-4.9.0
-         - parallel-netcdf-1.12.3
-         - xerces-c-3.2.3
+         use:
+            - /path/to/module/directory
+         load:
+            - netcdf-c-4.9.0
+            - parallel-netcdf-1.12.3
+            - xerces-c-3.2.3
 
    This is seldom needed, because payu is good at automatically determining
    the environment modules required by model executables. If the modules
-   require `module use` inorder to be found, do this prior to `payu run`,
+   require `module use` in order to be found, this command can also be run
+   prior to `payu run` instead of listing the directory under the `use` option,
    e.g.::
 
       module use /path/to/module/directory

payu/experiment.py

@@ -258,7 +258,7 @@ def load_modules(self):
             envmod.module('load', mod)
 
         # User-defined modules
-        user_modules = self.config.get('modules', [])
+        user_modules = self.config.get('modules', {}).get('load', [])
         for mod in user_modules:
             envmod.module('load', mod)
 
@@ -457,6 +457,10 @@ def run(self, *user_flags):
 
         # XXX: This was previously done in reversion
         envmod.setup()
+
+        # Add any user-defined module dir(s) to MODULEPATH
+        for module_dir in self.config.get('modules', {}).get('use', []):
+            envmod.module('use', module_dir) 
 
         self.load_modules()
 

payu/fsops.py

@@ -96,6 +96,12 @@ def read_config(config_fname=None):
 
     config['collate'] = collate_config
 
+    # Transform legacy modules config options
+    modules_config = config.pop('modules', {})
+    if type(modules_config) is list:
+        modules_config = {'load': modules_config}
+    config['modules'] = modules_config
+
     # Local "control" path. Must be set here so it can be
     # scanned for storage points
     config["control_path"] = config.get('control',

test/resources/config_legacy_modules.yaml

@@ -0,0 +1,3 @@
+modules:
+  - module_1
+  - module_2

test/resources/config_modules.yaml

@@ -0,0 +1,7 @@
+modules: 
+  use:
+    - path/to/module/dir/1
+    - path/to/module/dir/2
+  load:
+    - module_1
+    - module_2

test/test_payu.py

@@ -125,7 +125,7 @@ def test_movetree():
 
 
 def test_read_config():
-    config_path = os.path.join('test', 'config_mom5.yaml')
+    config_path = os.path.join('test', 'resources', 'config_mom5.yaml')
     config = payu.fsops.read_config(config_path)
 
     # Test control_path is not set in read_config
@@ -148,11 +148,34 @@ def test_read_config():
 
     assert(config.pop('collate') == {})
     assert(config.pop('control_path') == os.getcwd())
+    assert(config.pop('modules') == {})
     assert(config == {})
 
     os.remove(config_tmp)
 
 
+def test_read_config_modules_legacy_option():
+    # Test transform legacy modules option
+    config_path = os.path.join('test', 'resources', 'config_legacy_modules.yaml')
+
+    config = payu.fsops.read_config(config_path)
+    modules_config = config.get('modules', {})
+
+    assert(modules_config.get('load', []) == ['module_1', 'module_2'])
+    assert(modules_config.get('use', []) == [])
+
+
+def test_read_config_modules_option():
+    # Test modules with load/use options is unchanged
+    config_path = os.path.join('test', 'resources', 'config_modules.yaml')
+
+    config = payu.fsops.read_config(config_path)
+    modules_config = config.get('modules', {})
+
+    assert(modules_config.get('load', []) == ['module_1', 'module_2'])
+    assert(modules_config.get('use', []) == ['path/to/module/dir/1', 'path/to/module/dir/2'])
+
+
 def test_make_symlink():
     tmp_path = 'tmp_file'
     tmp_sym = 'tmp_sym'
@@ -222,8 +245,8 @@ def test_parse_ldd_output():
     with open(ldd_output_path, 'r') as f:
         ldd_output = f.read()
     required_libs = payu.fsops.parse_ldd_output(ldd_output)
-    assert(len(required_libs), 4) 
-    assert(required_libs['libmpi.so.40'], '/apps/openmpi/4.0.2/lib/libmpi.so.40')
+    assert(len(required_libs) == 4)
+    assert(required_libs['libmpi.so.40'] == '/apps/openmpi/4.0.2/lib/libmpi.so.40')
 
 
 def test_lib_update_lib_if_required():