Walk file tree on directory publish

modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy

@@ -19,11 +19,14 @@ package nextflow.processor
 import java.nio.file.FileAlreadyExistsException
 import java.nio.file.FileSystem
 import java.nio.file.FileSystems
+import java.nio.file.FileVisitResult
 import java.nio.file.Files
 import java.nio.file.LinkOption
 import java.nio.file.NoSuchFileException
 import java.nio.file.Path
 import java.nio.file.PathMatcher
+import java.nio.file.SimpleFileVisitor
+import java.nio.file.attribute.BasicFileAttributes
 import java.time.temporal.ChronoUnit
 import java.util.concurrent.ExecutorService
 
@@ -337,10 +340,10 @@ class PublishDir {
         }
 
         if( inProcess ) {
-            safeProcessFile(source, destination)
+            safeProcessPath(source, destination)
         }
         else {
-            threadPool.submit({ safeProcessFile(source, destination) } as Runnable)
+            threadPool.submit({ safeProcessPath(source, destination) } as Runnable)
         }
 
     }
@@ -363,9 +366,23 @@ class PublishDir {
         throw new IllegalArgumentException("Not a valid publish target path: `$target` [${target?.class?.name}]")
     }
 
-    protected void safeProcessFile(Path source, Path target) {
+    protected void safeProcessPath(Path source, Path target) {
         try {
-            retryableProcessFile(source, target)
+            // publish each file in the directory tree
+            if( Files.isDirectory(source) ) {
+                Files.walkFileTree(source, new SimpleFileVisitor<Path>() {
+                    FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) {
+                        final targetFile = target.resolve(source.relativize(sourceFile).toString())
+                        retryableProcessFile(sourceFile, targetFile)
+                        FileVisitResult.CONTINUE
+                    }
+                })
+            }
+
+            // otherwise publish file directly
+            else {
+                retryableProcessFile(source, target)
+            }
         }
         catch( Throwable e ) {
             final msg =  "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details"
@@ -395,7 +412,7 @@ class PublishDir {
             .build()
         Failsafe
             .with( retryPolicy )
-            .get({it-> processFile(source, target)})
+            .get { it-> processFile(source, target) }
     }
 
     protected void processFile( Path source, Path destination ) {

plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy

@@ -63,7 +63,7 @@ class PublishDirS3Test extends Specification {
         when:
         spy.apply1(source, true)
         then:
-        1 * spy.safeProcessFile(source, _) >> { sourceFile, s3File ->
+        1 * spy.safeProcessPath(source, _) >> { sourceFile, s3File ->
             assert s3File instanceof S3Path
             assert (s3File as S3Path).getTagsList().find{ it.getKey()=='FOO'}.value == 'this'
             assert (s3File as S3Path).getTagsList().find{ it.getKey()=='BAR'}.value == 'that'