add single or list options to extra paths to sync and exclude

payu/sync.py

@@ -101,6 +101,21 @@ def add_restarts_to_sync(self):
                                   for path in protected_paths])
         self.source_paths.extend([SourcePath(path) for path in paths])
 
+    def add_extra_source_paths(self):
+        """Add additional paths to sync to remote archive"""
+        paths = self.config.get('extra_paths', [])
+        if isinstance(paths, str):
+            paths = [paths]
+
+        for path in paths:
+            # First check if path exists
+            if os.path.exists(path):
+                # Add extra path to protected paths - so they can't be deleted
+                self.source_paths.append(SourcePath(path=path, protected=True))
+            else:
+                print(f"payu: error: Path {path} not found. "
+                      "Failed to sync path to remote archive")
+
     def set_destination_path(self):
         "set or create destination path to sync archive to"
         # Remote archive user
@@ -159,8 +174,11 @@ def set_base_rsync_cmd(self):
     def set_excludes_flags(self):
         """Add lists of patterns of filepaths to exclude from sync commands"""
         # Get any excludes
-        excludes_list = self.config.get('excludes', [])
-        excludes = ' '.join(['--exclude ' + value for value in excludes_list])
+        exclude = self.config.get('exclude', [])
+        if isinstance(exclude, str):
+            exclude = [exclude]
+
+        excludes = ' '.join(['--exclude ' + pattern for pattern in exclude])
         if "--exclude *.nc.*" not in excludes:
             # TODO: Useful enough to keep??
             # Uncollated files are always excluded
@@ -183,6 +201,7 @@ def build_cmd(self, source_path):
     def run_cmd(self, source_path):
         """Given an source path, build and run rsync command"""
         cmd = self.build_cmd(source_path)
+        print(cmd)
         try:
             subprocess.check_call(cmd, shell=True)
         except subprocess.CalledProcessError as e:
@@ -211,11 +230,7 @@ def run(self):
                                                     is_log_file=True))
 
         # Add any additional paths to protected paths
-        # TODO: Could change this to configured list of control paths
-        paths = self.config.get('paths', [])
-        for path in paths:
-            if os.path.exists(path):
-                self.source_paths.append(path=path, protected=True)
+        self.add_extra_source_paths()
 
         # Set rsync command components
         self.set_destination_path()

test/test_sync.py

@@ -192,7 +192,7 @@ def test_rsync_components():
             "url": "test.domain",
             "user": "test-usr",
             "path": "remote/path",
-            "excludes": ["iceh.????-??-??.nc", "*-DEPRECATED"]
+            "exclude": ["iceh.????-??-??.nc", "*-DEPRECATED"]
         }}
     sync = setup_sync(additional_config=additional_config)