Skip to content

Commit

Permalink
Add parent experiment UUID as argument to payu clone
Browse files Browse the repository at this point in the history
- Minor edit to documentation
  • Loading branch information
Jo Basevi committed Dec 21, 2023
1 parent 6e59c50 commit 088e1df
Show file tree
Hide file tree
Showing 6 changed files with 54 additions and 28 deletions.
13 changes: 5 additions & 8 deletions docs/source/usage.rst
Original file line number Diff line number Diff line change
Expand Up @@ -390,15 +390,12 @@ the experiment name would be:
``perturb``.

* ``my_expt-416af8c6`` - if the control directory was not a git repository or
experiments was run from the ``main`` or ``master`` branch.
experiments was run from the ``main`` or ``master`` git branch.

* ``my_expt`` - if running an older experiment that has a pre-existing
archive.

* ``set_expt_name`` - if the ``experiment`` value is configured as ``set_expt_name``
(see :ref:`config`). Note: to use branches in the same control directory while setting
the experiment name explicitly would require each configured ``experiment`` value to
be unique for each branch. This is possible but not recommended practice.
To preserve backwards compatibility, if there's a pre-existing archive under
the *control directory* name, this will remain the experiment name (e.g.
``my_expt`` in the above example). Similarly, if the ``experiment`` value is
configured (see :ref:`config`), this will be used for the experiment name.

Switching between related experiments
-------------------------------------
Expand Down
20 changes: 14 additions & 6 deletions payu/branch.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@
import os
import warnings
from pathlib import Path
import re
from typing import Optional

from ruamel.yaml import YAML, CommentedMap
Expand Down Expand Up @@ -108,7 +107,8 @@ def checkout_branch(branch_name: str,
config_path: Optional[Path] = None,
control_path: Optional[Path] = None,
model_type: Optional[str] = None,
lab_path: Optional[Path] = None) -> None:
lab_path: Optional[Path] = None,
parent_experiment: Optional[str] = None) -> None:
"""Checkout branch, setup metadata and add symlinks
Parameters
Expand All @@ -134,6 +134,8 @@ def checkout_branch(branch_name: str,
Type of model - used for creating a Laboratory
lab_path: Optional[Path]
Path to laboratory directory
parent_experiment: Optional[str]
Parent experiment UUID to add to generated metadata
"""
if control_path is None:
control_path = get_control_path(config_path)
Expand Down Expand Up @@ -164,7 +166,8 @@ def checkout_branch(branch_name: str,

# Create/update and commit metadata file
metadata.write_metadata(set_template_values=True,
restart_path=prior_restart_path)
restart_path=prior_restart_path,
parent_experiment=parent_experiment)

# Add restart option to config
if prior_restart_path:
Expand Down Expand Up @@ -203,7 +206,8 @@ def clone(repository: str,
model_type: Optional[str] = None,
config_path: Optional[Path] = None,
lab_path: Optional[Path] = None,
restart_path: Optional[Path] = None) -> None:
restart_path: Optional[Path] = None,
parent_experiment: Optional[str] = None) -> None:
"""Clone an experiment control repository.
Parameters:
Expand All @@ -229,6 +233,8 @@ def clone(repository: str,
Path to laboratory directory
restart_path: Optional[Path]
Absolute restart path to start experiment from
parent_experiment: Optional[str]
Parent experiment UUID to add to generated metadata
Returns: None
"""
Expand All @@ -253,7 +259,8 @@ def clone(repository: str,
config_path=config_path,
control_path=control_path,
model_type=model_type,
lab_path=lab_path)
lab_path=lab_path,
parent_experiment=parent_experiment)
else:
# Checkout branch
if branch is None:
Expand All @@ -266,7 +273,8 @@ def clone(repository: str,
control_path=control_path,
model_type=model_type,
lab_path=lab_path,
is_new_experiment=True)
is_new_experiment=True,
parent_experiment=parent_experiment)
finally:
# Change back to original working directory
os.chdir(owd)
Expand Down
25 changes: 15 additions & 10 deletions payu/metadata.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
:license: Apache License, Version 2.0, see LICENSE for details.
"""

import re
import requests
import shutil
import uuid
Expand Down Expand Up @@ -218,7 +217,8 @@ def set_new_uuid(self, is_new_experiment: bool = False) -> None:

def write_metadata(self,
restart_path: Optional[Union[Path, str]] = None,
set_template_values: bool = False) -> None:
set_template_values: bool = False,
parent_experiment: Optional[str] = None) -> None:
"""Create/update metadata file, commit any changes and
copy metadata file to the experiment archive.
Expand All @@ -228,6 +228,8 @@ def write_metadata(self,
set_template_values: bool, default False
Read schema and set metadata template values for new
experiments
parent_experiment: Optional[str]
Parent experiment UUID to add to generated metadata
Return: None
Expand All @@ -241,24 +243,27 @@ def write_metadata(self,
# Update metadata if UUID has changed
restart_path = Path(restart_path) if restart_path else None
self.update_file(restart_path=restart_path,
set_template_values=set_template_values)
set_template_values=set_template_values,
parent_experiment=parent_experiment)
self.commit_file()

self.copy_to_archive()

def update_file(self,
restart_path: Optional[Path] = None,
set_template_values: bool = False) -> None:
set_template_values: bool = False,
parent_experiment: Optional[str] = None) -> None:
"""Write any updates to metadata file"""
metadata = self.read_file()

# Add UUID field
metadata[UUID_FIELD] = self.uuid

# Update parent UUID field
parent_uuid = self.get_parent_experiment(restart_path)
if parent_uuid and parent_uuid != self.uuid:
metadata[PARENT_UUID_FIELD] = parent_uuid
if parent_experiment is None:
parent_experiment = self.get_parent_experiment(restart_path)
if parent_experiment and parent_experiment != self.uuid:
metadata[PARENT_UUID_FIELD] = parent_experiment

# Add extra fields if new branch-uuid experiment
# so to not over-write fields if it's a pre-existing legacy experiment
Expand Down Expand Up @@ -290,10 +295,10 @@ def update_file(self,

def get_model_name(self) -> str:
"""Get model name from config file"""
# Use model name unless specific model is specified in metadata config
default_model_name = self.config.get('model')
# Use capitilised model name unless a specific model name is defined
default_model_name = self.config.get('model').upper()
model_name = self.metadata_config.get('model', default_model_name)
return model_name.upper()
return model_name

def get_parent_experiment(self, prior_restart_path: Path) -> None:
"""Searches UUID in the metadata in the parent directory that
Expand Down
11 changes: 11 additions & 0 deletions payu/subcommands/args.py
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,17 @@
}
}

# Clone parent experiment
parent_experiment = {
'flags': ('--parent-experiment', '-p'),
'parameters': {
'action': 'store',
'dest': 'parent_experiment',
'default': None,
'help': 'The parent experiment UUID to add to generated metadata'
}
}

# Branch name
branch_name = {
'flags': [],
Expand Down
9 changes: 6 additions & 3 deletions payu/subcommands/clone_cmd.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,15 +17,17 @@
arguments = [args.model, args.config, args.laboratory,
args.keep_uuid, args.clone_branch,
args.repository, args.local_directory,
args.new_branch_name, args.restart_path]
args.new_branch_name, args.restart_path,
args.parent_experiment]


def transform_strings_to_path(path_str=None):
return Path(path_str) if path_str is not None else None


def runcmd(model_type, config_path, lab_path, keep_uuid,
branch, repository, local_directory, new_branch_name, restart_path):
branch, repository, local_directory, new_branch_name, restart_path,
parent_experiment):
"""Execute the command."""
config_path = transform_strings_to_path(config_path)
restart_path = transform_strings_to_path(restart_path)
Expand All @@ -40,7 +42,8 @@ def runcmd(model_type, config_path, lab_path, keep_uuid,
config_path=config_path,
lab_path=lab_path,
new_branch_name=new_branch_name,
restart_path=restart_path)
restart_path=restart_path,
parent_experiment=parent_experiment)


runscript = runcmd
4 changes: 3 additions & 1 deletion test/test_branch.py
Original file line number Diff line number Diff line change
Expand Up @@ -515,7 +515,8 @@ def test_clone(mock_uuid):
# Run clone
with cd(tmpdir):
clone(str(cloned_repo_path), Path("clonedRepo2"),
lab_path=labdir, new_branch_name="Branch2", branch="Branch1")
lab_path=labdir, new_branch_name="Branch2", branch="Branch1",
parent_experiment=uuid1)

# Check new commit added and expected metadata
cloned_repo2 = git.Repo(tmpdir / "clonedRepo2")
Expand All @@ -524,6 +525,7 @@ def test_clone(mock_uuid):
expected_current_branch="Branch2",
expected_uuid=uuid2,
expected_experiment="clonedRepo2-Branch2-fd7b4804",
expected_parent_uuid=uuid1,
metadata_file=metadata_file)

# Check branched from Branch1
Expand Down

0 comments on commit 088e1df

Please sign in to comment.