diff --git a/payu/sync.py b/payu/sync.py index ac05dd86..336d3078 100644 --- a/payu/sync.py +++ b/payu/sync.py @@ -101,6 +101,21 @@ def add_restarts_to_sync(self): for path in protected_paths]) self.source_paths.extend([SourcePath(path) for path in paths]) + def add_extra_source_paths(self): + """Add additional paths to sync to remote archive""" + paths = self.config.get('extra_paths', []) + if isinstance(paths, str): + paths = [paths] + + for path in paths: + # First check if path exists + if os.path.exists(path): + # Add extra path to protected paths - so they can't be deleted + self.source_paths.append(SourcePath(path=path, protected=True)) + else: + print(f"payu: error: Path {path} not found. " + "Failed to sync path to remote archive") + def set_destination_path(self): "set or create destination path to sync archive to" # Remote archive user @@ -159,8 +174,11 @@ def set_base_rsync_cmd(self): def set_excludes_flags(self): """Add lists of patterns of filepaths to exclude from sync commands""" # Get any excludes - excludes_list = self.config.get('excludes', []) - excludes = ' '.join(['--exclude ' + value for value in excludes_list]) + exclude = self.config.get('exclude', []) + if isinstance(exclude, str): + exclude = [exclude] + + excludes = ' '.join(['--exclude ' + pattern for pattern in exclude]) if "--exclude *.nc.*" not in excludes: # TODO: Useful enough to keep?? # Uncollated files are always excluded @@ -183,6 +201,7 @@ def build_cmd(self, source_path): def run_cmd(self, source_path): """Given an source path, build and run rsync command""" cmd = self.build_cmd(source_path) + print(cmd) try: subprocess.check_call(cmd, shell=True) except subprocess.CalledProcessError as e: @@ -211,11 +230,7 @@ def run(self): is_log_file=True)) # Add any additional paths to protected paths - # TODO: Could change this to configured list of control paths - paths = self.config.get('paths', []) - for path in paths: - if os.path.exists(path): - self.source_paths.append(path=path, protected=True) + self.add_extra_source_paths() # Set rsync command components self.set_destination_path() diff --git a/test/test_sync.py b/test/test_sync.py index ebaaf702..59b1b8cc 100644 --- a/test/test_sync.py +++ b/test/test_sync.py @@ -192,7 +192,7 @@ def test_rsync_components(): "url": "test.domain", "user": "test-usr", "path": "remote/path", - "excludes": ["iceh.????-??-??.nc", "*-DEPRECATED"] + "exclude": ["iceh.????-??-??.nc", "*-DEPRECATED"] }} sync = setup_sync(additional_config=additional_config)