Merge pull request #447 from ACCESS-NRI/446-add-cmd-line-arg-to-disab…

…le-metadata

Add cmd-line `--metadata-off` flag to `payu setup` and `payu sweep` to  disable metadata generation and commits

payu/experiment.py

@@ -49,7 +49,7 @@
 
 class Experiment(object):
 
-    def __init__(self, lab, reproduce=False, force=False):
+    def __init__(self, lab, reproduce=False, force=False, metadata_off=False):
         self.lab = lab
 
         if not force:
@@ -61,7 +61,7 @@ def __init__(self, lab, reproduce=False, force=False):
         self.start_time = datetime.datetime.now()
 
         # Initialise experiment metadata - uuid and experiment name
-        self.metadata = Metadata(Path(lab.archive_path))
+        self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off)
         self.metadata.setup()
 
         # TODO: replace with dict, check versions via key-value pairs

payu/metadata.py

@@ -58,13 +58,18 @@ class Metadata:
         config_path : Optional[Path]
             Configuration Path. The default is config.yaml in the current
             working directory. This is also set in fsop.read_config
+        disabled : bool, default False
+            Flag to disable metadata and UUID generation and commits. The
+            legacy name (control directory name) for experiments names
+            in archive will be used instead.
     """
 
     def __init__(self,
                  laboratory_archive_path: Path,
                  config_path: Optional[Path] = None,
                  branch: Optional[str] = None,
-                 control_path: Optional[Path] = None) -> None:
+                 control_path: Optional[Path] = None,
+                 disabled: Optional[bool] = False) -> None:
         self.config = read_config(config_path)
         self.metadata_config = self.config.get('metadata', {})
 
@@ -74,8 +79,12 @@ def __init__(self,
         self.filepath = self.control_path / METADATA_FILENAME
         self.lab_archive_path = laboratory_archive_path
 
-        # Config flag to disable creating metadata files and UUIDs
-        self.enabled = self.metadata_config.get('enable', True)
+        # Check if metadata has been disabled in call, env flag under PBS,
+        # or in config.yaml
+        self.enabled = (
+            not disabled and
+            self.metadata_config.get('enable', True)
+        )
 
         if self.enabled:
             self.repo = GitRepository(self.control_path, catch_error=True)
@@ -161,7 +170,7 @@ def set_experiment_name(self,
             # Metadata/UUID generation is disabled, so leave UUID out of
             # experiment name
             self.experiment_name = legacy_name
-            print("Metadata is disabled in config.yaml.",
+            print("Metadata and UUID generation is disabled.",
                   f"Experiment name used for archival: {self.experiment_name}")
             return
 

payu/subcommands/args.py

@@ -275,3 +275,15 @@
         'help': 'Display metadata of branches in remote directory'
     }
 }
+
+
+# Disable metadata + UUID generation
+metadata_off = {
+    'flags': ['--metadata-off', '-M'],
+    'parameters': {
+        'dest': 'metadata_off',
+        'action': 'store_true',
+        'default': False,
+        'help': 'Disable experiment metadata and UUID generation and commits'
+    }
+}

payu/subcommands/setup_cmd.py

@@ -13,15 +13,17 @@
     args.laboratory,
     args.force_archive,
     args.reproduce,
-    args.force
+    args.force,
+    args.metadata_off
 ]
 
 
 def runcmd(model_type, config_path, lab_path, force_archive,
-           reproduce=False, force=False):
+           reproduce=False, force=False, metadata_off=False):
 
     lab = Laboratory(model_type, config_path, lab_path)
-    expt = Experiment(lab, reproduce=reproduce, force=force)
+    expt = Experiment(lab, reproduce=reproduce, force=force,
+                      metadata_off=metadata_off)
 
     expt.setup(force_archive=force_archive)
 

payu/subcommands/sweep_cmd.py

@@ -7,13 +7,14 @@
 title = 'sweep'
 parameters = {'description': 'Delete any temporary files from prior runs'}
 
-arguments = [args.model, args.config, args.hard_sweep, args.laboratory]
+arguments = [args.model, args.config, args.hard_sweep, args.laboratory,
+             args.metadata_off]
 
 
-def runcmd(model_type, config_path, hard_sweep, lab_path):
+def runcmd(model_type, config_path, hard_sweep, lab_path, metadata_off):
 
     lab = Laboratory(model_type, config_path, lab_path)
-    expt = Experiment(lab)
+    expt = Experiment(lab, metadata_off=metadata_off)
 
     expt.sweep(hard_sweep)
 

test/common.py

@@ -106,7 +106,8 @@ def sweep_work(hard_sweep=False):
         payu_sweep(model_type=None,
                    config_path=None,
                    hard_sweep=hard_sweep,
-                   lab_path=str(labdir))
+                   lab_path=str(labdir),
+                   metadata_off=False)
 
 
 def payu_setup(model_type=None,
@@ -115,7 +116,8 @@ def payu_setup(model_type=None,
                force_archive=None,
                reproduce=None,
                sweep=True,
-               force=False):
+               force=False,
+               metadata_off=False):
     """
     Wrapper around original setup command to provide default arguments
     and run in ctrldir
@@ -125,13 +127,15 @@ def payu_setup(model_type=None,
             payu_sweep(model_type=None,
                        config_path=None,
                        hard_sweep=False,
-                       lab_path=str(labdir))
+                       lab_path=str(labdir),
+                       metadata_off=False)
         payu_setup_orignal(model_type,
                            config_path,
                            lab_path,
                            force_archive,
                            reproduce,
-                           force)
+                           force,
+                           metadata_off=False)
 
 
 def write_config(config, path=config_path):

test/test_cli.py

@@ -62,6 +62,7 @@ def test_parse_setup():
     assert args.pop('force_archive') is False
     assert args.pop('reproduce') is False
     assert args.pop('force') is False
+    assert args.pop('metadata_off') is False
 
     assert len(args) == 0
 
@@ -72,7 +73,8 @@ def test_parse_setup():
                             '--laboratory path/to/lab '
                             '--archive '
                             '--force '
-                            '--reproduce'.format(cmd=cmd))
+                            '--reproduce '
+                            '--metadata-off'.format(cmd=cmd))
 
     args = vars(parser.parse_args(arguments[1:]))
 
@@ -86,6 +88,7 @@ def test_parse_setup():
     assert args.pop('force_archive') is True
     assert args.pop('reproduce') is True
     assert args.pop('force') is True
+    assert args.pop('metadata_off') is True
 
     assert len(args) == 0
 
@@ -95,7 +98,8 @@ def test_parse_setup():
                             '-c path/to/config.yaml '
                             '-l path/to/lab '
                             '-f '
-                            '-r'.format(cmd=cmd))
+                            '-r '
+                            '-M'.format(cmd=cmd))
 
     args = vars(parser.parse_args(arguments[1:]))
 
@@ -109,6 +113,7 @@ def test_parse_setup():
     assert args.pop('force_archive') is False
     assert args.pop('reproduce') is True
     assert args.pop('force') is True
+    assert args.pop('metadata_off') is True
 
     assert len(args) == 0
 
@@ -209,6 +214,7 @@ def test_parse_sweep():
     assert args.pop('config_path') is None
     assert args.pop('lab_path') is None
     assert args.pop('hard_sweep') is False
+    assert args.pop('metadata_off') is False
 
     assert len(args) == 0
 
@@ -217,7 +223,8 @@ def test_parse_sweep():
                             '--model mom '
                             '--config path/to/config.yaml '
                             '--laboratory path/to/lab '
-                            '--hard'.format(cmd=cmd))
+                            '--hard '
+                            '--metadata-off'.format(cmd=cmd))
 
     args = vars(parser.parse_args(arguments[1:]))
 
@@ -229,14 +236,16 @@ def test_parse_sweep():
     assert args.pop('config_path') == 'path/to/config.yaml'
     assert args.pop('lab_path') == 'path/to/lab'
     assert args.pop('hard_sweep') is True
+    assert args.pop('metadata_off') is True
 
     assert len(args) == 0
 
     # Test short options
     arguments = shlex.split('payu {cmd} '
                             '-m mom '
                             '-c path/to/config.yaml '
-                            '-l path/to/lab '.format(cmd=cmd))
+                            '-l path/to/lab '
+                            '-M'.format(cmd=cmd))
 
     args = vars(parser.parse_args(arguments[1:]))
 
@@ -248,6 +257,7 @@ def test_parse_sweep():
     assert args.pop('config_path') == 'path/to/config.yaml'
     assert args.pop('lab_path') == 'path/to/lab'
     assert args.pop('hard_sweep') is False
+    assert args.pop('metadata_off') is True
 
     assert len(args) == 0
 

test/test_metadata.py

@@ -1,4 +1,5 @@
 import copy
+import os
 import shutil
 from datetime import datetime
 
@@ -354,13 +355,25 @@ def test_metadata_enable_false():
     }
     write_config(test_config)
 
-    with patch('payu.metadata.GitRepository.get_branch_name') as mock_branch:
-        mock_branch.return_value = "mock-branch"
+    with cd(ctrldir):
+        metadata = Metadata(archive_dir)
+        metadata.setup()
+        metadata.write_metadata()
 
-        with cd(ctrldir):
-            metadata = Metadata(archive_dir)
-            metadata.setup()
-            metadata.write_metadata()
+    # Test UUID kept out of experiment name and metadata file is not written
+    assert metadata.experiment_name == "ctrl"
+    assert not (ctrldir / "metadata.yaml").exists()
+
+
+def test_metadata_disable():
+    # Set metadata to True in config file
+    write_config(config)
+
+    with cd(ctrldir):
+        # Pass disabled flag to Metadata initialisation call
+        metadata = Metadata(archive_dir, disabled=True)
+        metadata.setup()
+        metadata.write_metadata()
 
     # Test UUID kept out of experiment name and metadata file is not written
     assert metadata.experiment_name == "ctrl"