From c2d10c3e5382e9f1ac0afcda5620dcd87105558b Mon Sep 17 00:00:00 2001 From: Jo Basevi Date: Thu, 22 Aug 2024 13:57:17 +1000 Subject: [PATCH] Add cmd-line flag to setup + sweep to disable metadata generation and commits --- payu/experiment.py | 4 ++-- payu/metadata.py | 17 +++++++++++++---- payu/subcommands/args.py | 12 ++++++++++++ payu/subcommands/setup_cmd.py | 8 +++++--- payu/subcommands/sweep_cmd.py | 7 ++++--- test/common.py | 12 ++++++++---- test/test_cli.py | 18 ++++++++++++++---- test/test_metadata.py | 25 +++++++++++++++++++------ 8 files changed, 77 insertions(+), 26 deletions(-) diff --git a/payu/experiment.py b/payu/experiment.py index 53d23c8b..a2bc89ef 100644 --- a/payu/experiment.py +++ b/payu/experiment.py @@ -49,7 +49,7 @@ class Experiment(object): - def __init__(self, lab, reproduce=False, force=False): + def __init__(self, lab, reproduce=False, force=False, metadata_off=False): self.lab = lab if not force: @@ -61,7 +61,7 @@ def __init__(self, lab, reproduce=False, force=False): self.start_time = datetime.datetime.now() # Initialise experiment metadata - uuid and experiment name - self.metadata = Metadata(Path(lab.archive_path)) + self.metadata = Metadata(Path(lab.archive_path), disabled=metadata_off) self.metadata.setup() # TODO: replace with dict, check versions via key-value pairs diff --git a/payu/metadata.py b/payu/metadata.py index 025ffc44..54d12b95 100644 --- a/payu/metadata.py +++ b/payu/metadata.py @@ -58,13 +58,18 @@ class Metadata: config_path : Optional[Path] Configuration Path. The default is config.yaml in the current working directory. This is also set in fsop.read_config + disabled : bool, default False + Flag to disable metadata and UUID generation and commits. The + legacy name (control directory name) for experiments names + in archive will be used instead. """ def __init__(self, laboratory_archive_path: Path, config_path: Optional[Path] = None, branch: Optional[str] = None, - control_path: Optional[Path] = None) -> None: + control_path: Optional[Path] = None, + disabled: Optional[bool] = False) -> None: self.config = read_config(config_path) self.metadata_config = self.config.get('metadata', {}) @@ -74,8 +79,12 @@ def __init__(self, self.filepath = self.control_path / METADATA_FILENAME self.lab_archive_path = laboratory_archive_path - # Config flag to disable creating metadata files and UUIDs - self.enabled = self.metadata_config.get('enable', True) + # Check if metadata has been disabled in call, env flag under PBS, + # or in config.yaml + self.enabled = ( + not disabled and + self.metadata_config.get('enable', True) + ) if self.enabled: self.repo = GitRepository(self.control_path, catch_error=True) @@ -161,7 +170,7 @@ def set_experiment_name(self, # Metadata/UUID generation is disabled, so leave UUID out of # experiment name self.experiment_name = legacy_name - print("Metadata is disabled in config.yaml.", + print("Metadata and UUID generation is disabled.", f"Experiment name used for archival: {self.experiment_name}") return diff --git a/payu/subcommands/args.py b/payu/subcommands/args.py index cf3da7b3..bb2ec3d6 100644 --- a/payu/subcommands/args.py +++ b/payu/subcommands/args.py @@ -275,3 +275,15 @@ 'help': 'Display metadata of branches in remote directory' } } + + +# Disable metadata + UUID generation +metadata_off = { + 'flags': ['--metadata-off', '-M'], + 'parameters': { + 'dest': 'metadata_off', + 'action': 'store_true', + 'default': False, + 'help': 'Disable experiment metadata and UUID generation and commits' + } +} \ No newline at end of file diff --git a/payu/subcommands/setup_cmd.py b/payu/subcommands/setup_cmd.py index fa0823bc..c08492cc 100644 --- a/payu/subcommands/setup_cmd.py +++ b/payu/subcommands/setup_cmd.py @@ -13,15 +13,17 @@ args.laboratory, args.force_archive, args.reproduce, - args.force + args.force, + args.metadata_off ] def runcmd(model_type, config_path, lab_path, force_archive, - reproduce=False, force=False): + reproduce=False, force=False, metadata_off=False): lab = Laboratory(model_type, config_path, lab_path) - expt = Experiment(lab, reproduce=reproduce, force=force) + expt = Experiment(lab, reproduce=reproduce, force=force, + metadata_off=metadata_off) expt.setup(force_archive=force_archive) diff --git a/payu/subcommands/sweep_cmd.py b/payu/subcommands/sweep_cmd.py index e5c27ec1..abbadad9 100644 --- a/payu/subcommands/sweep_cmd.py +++ b/payu/subcommands/sweep_cmd.py @@ -7,13 +7,14 @@ title = 'sweep' parameters = {'description': 'Delete any temporary files from prior runs'} -arguments = [args.model, args.config, args.hard_sweep, args.laboratory] +arguments = [args.model, args.config, args.hard_sweep, args.laboratory, + args.metadata_off] -def runcmd(model_type, config_path, hard_sweep, lab_path): +def runcmd(model_type, config_path, hard_sweep, lab_path, metadata_off): lab = Laboratory(model_type, config_path, lab_path) - expt = Experiment(lab) + expt = Experiment(lab, metadata_off=metadata_off) expt.sweep(hard_sweep) diff --git a/test/common.py b/test/common.py index cbb6b9c9..c0754a0d 100644 --- a/test/common.py +++ b/test/common.py @@ -107,7 +107,8 @@ def sweep_work(hard_sweep=False): payu_sweep(model_type=None, config_path=None, hard_sweep=hard_sweep, - lab_path=str(labdir)) + lab_path=str(labdir), + metadata_off=False) def payu_setup(model_type=None, @@ -116,7 +117,8 @@ def payu_setup(model_type=None, force_archive=None, reproduce=None, sweep=True, - force=False): + force=False, + metadata_off=False): """ Wrapper around original setup command to provide default arguments and run in ctrldir @@ -126,13 +128,15 @@ def payu_setup(model_type=None, payu_sweep(model_type=None, config_path=None, hard_sweep=False, - lab_path=str(labdir)) + lab_path=str(labdir), + metadata_off=False) payu_setup_orignal(model_type, config_path, lab_path, force_archive, reproduce, - force) + force, + metadata_off=False) def write_config(config, path=config_path): diff --git a/test/test_cli.py b/test/test_cli.py index b2b8933c..7cf6da40 100644 --- a/test/test_cli.py +++ b/test/test_cli.py @@ -62,6 +62,7 @@ def test_parse_setup(): assert args.pop('force_archive') is False assert args.pop('reproduce') is False assert args.pop('force') is False + assert args.pop('metadata_off') is False assert len(args) == 0 @@ -72,7 +73,8 @@ def test_parse_setup(): '--laboratory path/to/lab ' '--archive ' '--force ' - '--reproduce'.format(cmd=cmd)) + '--reproduce ' + '--metadata-off'.format(cmd=cmd)) args = vars(parser.parse_args(arguments[1:])) @@ -86,6 +88,7 @@ def test_parse_setup(): assert args.pop('force_archive') is True assert args.pop('reproduce') is True assert args.pop('force') is True + assert args.pop('metadata_off') is True assert len(args) == 0 @@ -95,7 +98,8 @@ def test_parse_setup(): '-c path/to/config.yaml ' '-l path/to/lab ' '-f ' - '-r'.format(cmd=cmd)) + '-r ' + '-M'.format(cmd=cmd)) args = vars(parser.parse_args(arguments[1:])) @@ -109,6 +113,7 @@ def test_parse_setup(): assert args.pop('force_archive') is False assert args.pop('reproduce') is True assert args.pop('force') is True + assert args.pop('metadata_off') is True assert len(args) == 0 @@ -209,6 +214,7 @@ def test_parse_sweep(): assert args.pop('config_path') is None assert args.pop('lab_path') is None assert args.pop('hard_sweep') is False + assert args.pop('metadata_off') is False assert len(args) == 0 @@ -217,7 +223,8 @@ def test_parse_sweep(): '--model mom ' '--config path/to/config.yaml ' '--laboratory path/to/lab ' - '--hard'.format(cmd=cmd)) + '--hard ' + '--metadata-off'.format(cmd=cmd)) args = vars(parser.parse_args(arguments[1:])) @@ -229,6 +236,7 @@ def test_parse_sweep(): assert args.pop('config_path') == 'path/to/config.yaml' assert args.pop('lab_path') == 'path/to/lab' assert args.pop('hard_sweep') is True + assert args.pop('metadata_off') is True assert len(args) == 0 @@ -236,7 +244,8 @@ def test_parse_sweep(): arguments = shlex.split('payu {cmd} ' '-m mom ' '-c path/to/config.yaml ' - '-l path/to/lab '.format(cmd=cmd)) + '-l path/to/lab ' + '-M'.format(cmd=cmd)) args = vars(parser.parse_args(arguments[1:])) @@ -248,6 +257,7 @@ def test_parse_sweep(): assert args.pop('config_path') == 'path/to/config.yaml' assert args.pop('lab_path') == 'path/to/lab' assert args.pop('hard_sweep') is False + assert args.pop('metadata_off') is True assert len(args) == 0 diff --git a/test/test_metadata.py b/test/test_metadata.py index 29d28661..a8443539 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -1,4 +1,5 @@ import copy +import os import shutil from datetime import datetime @@ -354,13 +355,25 @@ def test_metadata_enable_false(): } write_config(test_config) - with patch('payu.metadata.GitRepository.get_branch_name') as mock_branch: - mock_branch.return_value = "mock-branch" + with cd(ctrldir): + metadata = Metadata(archive_dir) + metadata.setup() + metadata.write_metadata() - with cd(ctrldir): - metadata = Metadata(archive_dir) - metadata.setup() - metadata.write_metadata() + # Test UUID kept out of experiment name and metadata file is not written + assert metadata.experiment_name == "ctrl" + assert not (ctrldir / "metadata.yaml").exists() + + +def test_metadata_disable(): + # Set metadata to True in config file + write_config(config) + + with cd(ctrldir): + # Pass disabled flag to Metadata initialisation call + metadata = Metadata(archive_dir, disabled=True) + metadata.setup() + metadata.write_metadata() # Test UUID kept out of experiment name and metadata file is not written assert metadata.experiment_name == "ctrl"