From a835b600b166bd267c73451cc2d4a9478550ebf0 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Wed, 10 May 2023 09:23:38 -0500 Subject: [PATCH 1/6] Walk file tree on directory publish Signed-off-by: Ben Sherman --- .../nextflow/processor/PublishDir.groovy | 29 ++++++++++++++++--- .../processor/PublishDirS3Test.groovy | 2 +- 2 files changed, 26 insertions(+), 5 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index b1353db3fe..8aeb11613f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -19,11 +19,14 @@ package nextflow.processor import java.nio.file.FileAlreadyExistsException import java.nio.file.FileSystem import java.nio.file.FileSystems +import java.nio.file.FileVisitResult import java.nio.file.Files import java.nio.file.LinkOption import java.nio.file.NoSuchFileException import java.nio.file.Path import java.nio.file.PathMatcher +import java.nio.file.SimpleFileVisitor +import java.nio.file.attribute.BasicFileAttributes import java.util.concurrent.ExecutorService import groovy.transform.CompileDynamic @@ -326,10 +329,10 @@ class PublishDir { } if( inProcess ) { - safeProcessFile(source, destination) + safeProcessPath(source, destination) } else { - threadPool.submit({ safeProcessFile(source, destination) } as Runnable) + threadPool.submit({ safeProcessPath(source, destination) } as Runnable) } } @@ -354,9 +357,9 @@ class PublishDir { } @CompileStatic - protected void safeProcessFile(Path source, Path target) { + protected void safeProcessPath(Path source, Path target) { try { - processFile(source, target) + processPath(source, target) } catch( Throwable e ) { log.warn "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details", e @@ -367,6 +370,24 @@ class PublishDir { } } + @CompileStatic + protected void processPath( Path source, Path target ) { + + // publish each file in the directory tree + if( Files.isDirectory(source) ) + Files.walkFileTree(source, new SimpleFileVisitor() { + FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { + final targetFile = target.resolve(source.relativize(sourceFile)) + processFile(sourceFile, targetFile) + FileVisitResult.CONTINUE + } + }) + + // otherwise publish file directly + else + processFile(source, target) + } + @CompileStatic protected void processFile( Path source, Path destination ) { diff --git a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy index 460de36581..e60909da96 100644 --- a/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy +++ b/plugins/nf-amazon/src/test/nextflow/processor/PublishDirS3Test.groovy @@ -63,7 +63,7 @@ class PublishDirS3Test extends Specification { when: spy.apply1(source, true) then: - 1 * spy.safeProcessFile(source, _) >> { sourceFile, s3File -> + 1 * spy.safeProcessPath(source, _) >> { sourceFile, s3File -> assert s3File instanceof S3Path assert (s3File as S3Path).getTagsList().find{ it.getKey()=='FOO'}.value == 'this' assert (s3File as S3Path).getTagsList().find{ it.getKey()=='BAR'}.value == 'that' From b7b69674fb28c33b0e68f5e8f233dc288fcc3b23 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Thu, 8 Feb 2024 16:40:11 -0600 Subject: [PATCH 2/6] Fix failing tests Signed-off-by: Ben Sherman --- .../src/main/groovy/nextflow/processor/PublishDir.groovy | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index 02d33fefdd..f2f151ab83 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -366,19 +366,19 @@ class PublishDir { } catch( Throwable e ) { log.warn "Failed to publish file: ${source.toUriString()}; to: ${target.toUriString()} [${mode.toString().toLowerCase()}] -- See log file for details", e - if( NF.strictMode || failOnError){ + if( NF.strictMode || failOnError ) { session?.abort(e) } } } - protected void processPath( Path source, Path target ) { + protected void processPath(Path source, Path target) { // publish each file in the directory tree if( Files.isDirectory(source) ) Files.walkFileTree(source, new SimpleFileVisitor() { FileVisitResult visitFile(Path sourceFile, BasicFileAttributes attrs) { - final targetFile = target.resolve(source.relativize(sourceFile)) + final targetFile = target.resolve(source.relativize(sourceFile).toString()) processFile(sourceFile, targetFile) FileVisitResult.CONTINUE } From 807e5d4d256f234b9fc0b2fc1979deacb6af3b40 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Fri, 9 Feb 2024 09:26:22 -0600 Subject: [PATCH 3/6] Overwrite published files only if they are stale Signed-off-by: Ben Sherman --- docs/process.md | 5 ++- .../nextflow/processor/PublishDir.groovy | 7 ++-- .../nextflow/processor/TaskProcessor.groovy | 5 --- .../nextflow/processor/PublishDirTest.groovy | 42 +++++++++++++++++++ .../main/nextflow/extension/FilesEx.groovy | 15 +++++++ .../main/nextflow/file/ETagAwareFile.groovy | 29 +++++++++++++ .../main/nextflow/cloud/aws/nio/S3Path.java | 11 ++++- .../cloud/azure/nio/AzFileAttributes.groovy | 8 ++++ .../nextflow/cloud/azure/nio/AzPath.groovy | 8 +++- 9 files changed, 119 insertions(+), 11 deletions(-) create mode 100644 modules/nf-commons/src/main/nextflow/file/ETagAwareFile.groovy diff --git a/docs/process.md b/docs/process.md index da84bd9fcc..a2b3d6ae19 100644 --- a/docs/process.md +++ b/docs/process.md @@ -2230,7 +2230,10 @@ Available options: - `'symlink'`: Creates an absolute symbolic link in the publish directory for each output file (default). `overwrite` -: When `true` any existing file in the specified folder will be overridden (default: `true` during normal pipeline execution and `false` when pipeline execution is `resumed`). +: :::{versionchanged} 24.02.0-edge + Prior to this version, the default behavior was `false` on a resumed run and `true` otherwise. + ::: +: Determines whether to overwrite a published file if it already exists. By default, existing files are overwritten only if they are stale, i.e. checksum does not match the new file. `path` : Specifies the directory where files need to be published. **Note**: the syntax `publishDir '/some/dir'` is a shortcut for `publishDir path: '/some/dir'`. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy index f2f151ab83..039230ca60 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/PublishDir.groovy @@ -71,7 +71,7 @@ class PublishDir { Path path /** - * Whenever overwrite existing files + * Whether to overwrite existing files */ Boolean overwrite @@ -411,8 +411,9 @@ class PublishDir { // see https://github.com/nextflow-io/nextflow/issues/2177 if( checkSourcePathConflicts(destination)) return - - if( overwrite ) { + + // overwrite only if explicitly enabled or destination is stale + if( overwrite || (overwrite == null && source.getChecksum() != destination.getChecksum()) ) { FileHelper.deletePath(destination) processFileImpl(source, destination) } diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 134d6b28b8..7cf1d88a09 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -1324,7 +1324,6 @@ class TaskProcessor { * Publish output files to a specified target folder * * @param task The task whose outputs need to be published - * @param overwrite When {@code true} any existing file will be overwritten, otherwise the publishing is ignored */ @CompileStatic protected void publishOutputs( TaskRun task ) { @@ -1340,10 +1339,6 @@ class TaskProcessor { private void publishOutputs0( TaskRun task, PublishDir publish ) { - if( publish.overwrite == null ) { - publish.overwrite = !task.cached - } - HashSet files = [] def outputs = task.getOutputsByType(FileOutParam) for( Map.Entry entry : outputs ) { diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy index 613fead059..7a2f8f554f 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy @@ -224,6 +224,48 @@ class PublishDirTest extends Specification { } + def 'should overwrite published files only if they are stale' () { + + given: + def session = new Session() + def folder = Files.createTempDirectory('nxf') + def sourceDir = folder.resolve('work-dir') + def targetDir = folder.resolve('pub-dir') + sourceDir.mkdir() + sourceDir.resolve('file1.txt').text = 'aaa' + sourceDir.resolve('file2.bam').text = 'bbb' + targetDir.mkdir() + targetDir.resolve('file1.txt').text = 'aaa' + targetDir.resolve('file2.bam').text = 'bbb (old)' + + def task = new TaskRun(workDir: sourceDir, config: new TaskConfig(), name: 'foo') + + when: + def outputs = [ + sourceDir.resolve('file1.txt'), + sourceDir.resolve('file2.bam') + ] as Set + def publisher = new PublishDir(path: targetDir, mode: 'copy') + and: + def timestamp1 = targetDir.resolve('file1.txt').lastModified() + def timestamp2 = targetDir.resolve('file2.bam').lastModified() + and: + publisher.apply( outputs, task ) + and: + session.@publishPoolManager.shutdown(false) + + then: + timestamp1 == targetDir.resolve('file1.txt').lastModified() + timestamp2 != targetDir.resolve('file2.bam').lastModified() + + targetDir.resolve('file1.txt').text == 'aaa' + targetDir.resolve('file2.bam').text == 'bbb' + + cleanup: + folder?.deleteDir() + + } + def 'should apply saveAs closure' () { given: diff --git a/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy b/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy index 389dab5107..86c70f596b 100644 --- a/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy +++ b/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy @@ -35,14 +35,17 @@ import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFilePermission import java.nio.file.attribute.PosixFilePermissions +import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.transform.stc.ClosureParams import groovy.transform.stc.FromString import groovy.util.logging.Slf4j +import nextflow.file.ETagAwareFile import nextflow.file.FileHelper import nextflow.file.FileSystemPathFactory import nextflow.io.ByteBufferBackedInputStream +import nextflow.util.CacheHelper import nextflow.util.CharsetHelper import nextflow.util.CheckHelper @@ -1599,4 +1602,16 @@ class FilesEx { static String getScheme(Path path) { path.getFileSystem().provider().getScheme() } + + static String getChecksum(Path path) { + if( Files.isDirectory(path) ) + return null + + // use etag if available + if( path instanceof ETagAwareFile ) + return path.getETag() + + // otherwise compute checksum manually + CacheHelper.hasher(Hashing.md5(), path, CacheHelper.HashMode.DEEP).hash().toString() + } } diff --git a/modules/nf-commons/src/main/nextflow/file/ETagAwareFile.groovy b/modules/nf-commons/src/main/nextflow/file/ETagAwareFile.groovy new file mode 100644 index 0000000000..f1c40073b3 --- /dev/null +++ b/modules/nf-commons/src/main/nextflow/file/ETagAwareFile.groovy @@ -0,0 +1,29 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package nextflow.file + +/** + * Defines the interface for files that have an ETag + * + * @author Ben Sherman + */ +interface ETagAwareFile { + + String getETag() + +} diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Path.java b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Path.java index 2a5e193b8c..a86a884869 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Path.java +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/nio/S3Path.java @@ -42,13 +42,14 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; +import nextflow.file.ETagAwareFile; import nextflow.file.TagAwareFile; import static com.google.common.collect.Iterables.concat; import static com.google.common.collect.Iterables.filter; import static com.google.common.collect.Iterables.transform; import static java.lang.String.format; -public class S3Path implements Path, TagAwareFile { +public class S3Path implements Path, ETagAwareFile, TagAwareFile { public static final String PATH_SEPARATOR = "/"; /** @@ -566,6 +567,14 @@ public String getStorageClass() { return storageClass; } + @Override + public String getETag() { + return fileSystem + .getClient() + .getObjectMetadata(getBucket(), getKey()) + .getETag(); + } + // ~ helpers methods private static Function strip(final String ... strs) { diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributes.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributes.groovy index a9960aff56..047a4e0ce3 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributes.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzFileAttributes.groovy @@ -46,6 +46,8 @@ class AzFileAttributes implements BasicFileAttributes { private String objectId + private String etag + static AzFileAttributes root() { new AzFileAttributes(size: 0, objectId: '/', directory: true) } @@ -60,6 +62,7 @@ class AzFileAttributes implements BasicFileAttributes { updateTime = time(props.getLastModified()) directory = client.blobName.endsWith('/') size = props.getBlobSize() + etag = props.getETag() // Support for Azure Data Lake Storage Gen2 with hierarchical namespace enabled final meta = props.getMetadata() @@ -75,6 +78,7 @@ class AzFileAttributes implements BasicFileAttributes { creationTime = time(item.properties.getCreationTime()) updateTime = time(item.properties.getLastModified()) size = item.properties.getContentLength() + etag = item.properties.getETag() } } @@ -150,6 +154,10 @@ class AzFileAttributes implements BasicFileAttributes { return objectId } + String getETag() { + return etag + } + @Override boolean equals( Object obj ) { if( this.class != obj?.class ) return false diff --git a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPath.groovy b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPath.groovy index 2f654b4ad8..596ab38260 100644 --- a/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPath.groovy +++ b/plugins/nf-azure/src/main/nextflow/cloud/azure/nio/AzPath.groovy @@ -29,6 +29,7 @@ import com.azure.storage.blob.models.BlobItem import groovy.transform.CompileStatic import groovy.transform.EqualsAndHashCode import groovy.transform.PackageScope +import nextflow.file.ETagAwareFile /** * Implements Azure path object @@ -37,7 +38,7 @@ import groovy.transform.PackageScope */ @CompileStatic @EqualsAndHashCode(includes = 'fs,path,directory', includeFields = true) -class AzPath implements Path { +class AzPath implements Path, ETagAwareFile { private AzFileSystem fs @@ -333,4 +334,9 @@ class AzPath implements Path { return result } + @Override + String getETag() { + return attributes.getETag() + } + } From dc3207b94e88c9046ac516992fa6754c6fb0a69d Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Fri, 9 Feb 2024 09:30:38 -0600 Subject: [PATCH 4/6] Update docs Signed-off-by: Ben Sherman --- docs/process.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/process.md b/docs/process.md index a2b3d6ae19..68e069a872 100644 --- a/docs/process.md +++ b/docs/process.md @@ -2231,7 +2231,7 @@ Available options: `overwrite` : :::{versionchanged} 24.02.0-edge - Prior to this version, the default behavior was `false` on a resumed run and `true` otherwise. + Prior to this version, the default behavior was `false` if the task was cached on a resumed run and `true` otherwise. ::: : Determines whether to overwrite a published file if it already exists. By default, existing files are overwritten only if they are stale, i.e. checksum does not match the new file. From 787e053f6ceb0418073119e00665d99d9af5ef0e Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Mon, 12 Feb 2024 12:57:34 -0600 Subject: [PATCH 5/6] Add wrapper file system for Google cloud storage with checksum support Signed-off-by: Ben Sherman --- .../nextflow/cloud/aws/AmazonPlugin.groovy | 2 +- .../cloud/google/GoogleCloudPlugin.groovy | 8 + .../batch/GoogleBatchScriptLauncher.groovy | 8 +- .../batch/GoogleBatchTaskHandler.groovy | 2 +- .../google/{util => file}/GsBashLib.groovy | 2 +- .../{util => file}/GsPathFactory.groovy | 16 +- .../{util => file}/GsPathSerializer.groovy | 20 +-- .../GoogleLifeSciencesFileCopyStrategy.groovy | 2 +- .../cloud/google/nio/GsFileSystem.groovy | 63 +++++++ .../google/nio/GsFileSystemProvider.groovy | 148 +++++++++++++++ .../nextflow/cloud/google/nio/GsPath.groovy | 168 ++++++++++++++++++ .../src/resources/META-INF/extensions.idx | 4 +- .../java.nio.file.spi.FileSystemProvider | 17 ++ .../cloud/google/GoogleSpecification.groovy | 14 +- .../{util => file}/GsBashLibTest.groovy | 2 +- .../{util => file}/GsPathFactoryTest.groovy | 2 +- .../GsPathSerializerTest.groovy | 15 +- .../GoogleLifeSciencesExecutorTest.groovy | 2 +- .../nextflow/extension/EscapeTest2.groovy | 4 +- .../nextflow/extension/FilesExTest2.groovy | 9 +- .../nextflow/file/FileHelperGsTest.groovy | 39 ++-- .../src/test/nextflow/file/GsPathTest.groovy | 10 +- 22 files changed, 479 insertions(+), 78 deletions(-) rename plugins/nf-google/src/main/nextflow/cloud/google/{util => file}/GsBashLib.groovy (99%) rename plugins/nf-google/src/main/nextflow/cloud/google/{util => file}/GsPathFactory.groovy (88%) rename plugins/nf-google/src/main/nextflow/cloud/google/{util => file}/GsPathSerializer.groovy (70%) create mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy create mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy create mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy create mode 100644 plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider rename plugins/nf-google/src/test/nextflow/cloud/google/{util => file}/GsBashLibTest.groovy (99%) rename plugins/nf-google/src/test/nextflow/cloud/google/{util => file}/GsPathFactoryTest.groovy (99%) rename plugins/nf-google/src/test/nextflow/cloud/google/{util => file}/GsPathSerializerTest.groovy (80%) diff --git a/plugins/nf-amazon/src/main/nextflow/cloud/aws/AmazonPlugin.groovy b/plugins/nf-amazon/src/main/nextflow/cloud/aws/AmazonPlugin.groovy index e0d173b3c6..c3cebf54b0 100644 --- a/plugins/nf-amazon/src/main/nextflow/cloud/aws/AmazonPlugin.groovy +++ b/plugins/nf-amazon/src/main/nextflow/cloud/aws/AmazonPlugin.groovy @@ -15,8 +15,8 @@ */ package nextflow.cloud.aws -import nextflow.cloud.aws.nio.S3FileSystemProvider import groovy.transform.CompileStatic +import nextflow.cloud.aws.nio.S3FileSystemProvider import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import org.pf4j.PluginWrapper diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy index 9338a26965..cb936a5247 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy @@ -16,6 +16,8 @@ package nextflow.cloud.google import groovy.transform.CompileStatic +import nextflow.cloud.google.nio.GsFileSystemProvider +import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import org.pf4j.PluginWrapper /** @@ -30,4 +32,10 @@ class GoogleCloudPlugin extends BasePlugin { super(wrapper) } + @Override + void start() { + super.start() + FileHelper.getOrInstallProvider(GsFileSystemProvider) + } + } diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index c9ed889685..7b5650dafc 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -22,9 +22,9 @@ import java.nio.file.Paths import com.google.cloud.batch.v1.GCS import com.google.cloud.batch.v1.Volume -import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.cloud.google.nio.GsPath import nextflow.executor.BashWrapperBuilder import nextflow.extension.FilesEx import nextflow.processor.TaskBean @@ -43,7 +43,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc private static final String MOUNT_ROOT = '/mnt/disks' - private CloudStoragePath remoteWorkDir + private GsPath remoteWorkDir private Path remoteBinDir private Set buckets = new HashSet<>() private PathTrie pathTrie = new PathTrie() @@ -54,7 +54,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir) { super(bean) // keep track the google storage work dir - this.remoteWorkDir = (CloudStoragePath) bean.workDir + this.remoteWorkDir = (GsPath) bean.workDir this.remoteBinDir = toContainerMount(remoteBinDir) // map bean work and target dirs to container mount @@ -97,7 +97,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc } protected Path toContainerMount(Path path, boolean parent=false) { - if( path instanceof CloudStoragePath ) { + if( path instanceof GsPath ) { buckets.add(path.bucket()) pathTrie.add( (parent ? "/${path.bucket()}${path.parent}" : "/${path.bucket()}${path}").toString() ) final containerMount = "$MOUNT_ROOT/${path.bucket()}${path}" diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy index b6057cb67e..31bcb156cc 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy @@ -90,7 +90,7 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { this.client = executor.getClient() this.jobId = "nf-${task.hashLog.replace('/','')}-${System.currentTimeMillis()}" this.executor = executor - // those files are access via NF runtime, keep based on CloudStoragePath + // those files are access via NF runtime, keep based on GsPath this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE) this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE) this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy similarity index 99% rename from plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy index fca97ba598..8ad682de09 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import groovy.transform.CompileStatic import groovy.transform.Memoized diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy similarity index 88% rename from plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy index 1924432e4a..8b4bd79a02 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy @@ -14,19 +14,19 @@ * limitations under the License. */ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import java.nio.file.Path import com.google.api.gax.retrying.RetrySettings import com.google.cloud.storage.StorageOptions import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration -import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem -import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import nextflow.Global import nextflow.Session import nextflow.cloud.google.GoogleOpts +import nextflow.cloud.google.nio.GsFileSystem +import nextflow.cloud.google.nio.GsPath import nextflow.cloud.google.lifesciences.GoogleLifeSciencesFileCopyStrategy import nextflow.file.FileSystemPathFactory /** @@ -98,13 +98,13 @@ class GsPathFactory extends FileSystemPathFactory { final str = uri.substring(5) final p = str.indexOf('/') return p == -1 - ? CloudStorageFileSystem.forBucket(str, storageConfig, storageOptions).getPath('') - : CloudStorageFileSystem.forBucket(str.substring(0,p), storageConfig, storageOptions).getPath(str.substring(p)) + ? GsFileSystem.forBucket(str, storageConfig, storageOptions).getPath('') + : GsFileSystem.forBucket(str.substring(0,p), storageConfig, storageOptions).getPath(str.substring(p)) } @Override protected String toUriString(Path path) { - if( path instanceof CloudStoragePath ) { + if( path instanceof GsPath ) { return "gs://${path.bucket()}$path".toString() } return null @@ -112,7 +112,7 @@ class GsPathFactory extends FileSystemPathFactory { @Override protected String getBashLib(Path path) { - if( path instanceof CloudStoragePath ) { + if( path instanceof GsPath ) { return GsBashLib.fromSession( Global.session as Session ) } return null @@ -120,7 +120,7 @@ class GsPathFactory extends FileSystemPathFactory { @Override protected String getUploadCmd(String source, Path target) { - if( target instanceof CloudStoragePath ) { + if( target instanceof GsPath ) { GoogleLifeSciencesFileCopyStrategy.uploadCmd(source,target) } return null diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy similarity index 70% rename from plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy index e042061495..7b4b2f2fbc 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy @@ -14,48 +14,48 @@ * limitations under the License. */ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.Serializer import com.esotericsoftware.kryo.io.Input import com.esotericsoftware.kryo.io.Output import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem -import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import groovy.util.logging.Slf4j +import nextflow.cloud.google.nio.GsPath import nextflow.util.SerializerRegistrant import org.pf4j.Extension /** - * Serializer for a {@link CloudStoragePath} + * Serializer for a {@link GsPath} * * @author Paolo Di Tommaso */ @Slf4j @Extension @CompileStatic -class GsPathSerializer extends Serializer implements SerializerRegistrant { +class GsPathSerializer extends Serializer implements SerializerRegistrant { @Override - void write(Kryo kryo, Output output, CloudStoragePath target) { + void write(Kryo kryo, Output output, GsPath target) { def path = target.toString() if( !path.startsWith('/') ) // <-- it looks a bug in the google nio library, in some case the path returned is not absolute path = '/' + path path = target.bucket() + path - log.trace "Google CloudStoragePath serialisation > path=$path" + log.trace "Google GsPath serialisation > path=$path" output.writeString(path) } @Override - CloudStoragePath read(Kryo kryo, Input input, Class type) { + GsPath read(Kryo kryo, Input input, Class type) { final path = input.readString() - log.trace "Google CloudStoragePath de-serialization > path=$path" + log.trace "Google GsPath de-serialization > path=$path" def uri = CloudStorageFileSystem.URI_SCHEME + '://' + path - (CloudStoragePath) GsPathFactory.parse(uri) + (GsPath) GsPathFactory.parse(uri) } @Override void register(Map serializers) { - serializers.put(CloudStoragePath, GsPathSerializer) + serializers.put(GsPath, GsPathSerializer) } } diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy index c013011a93..2644ff50cd 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy @@ -23,7 +23,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cloud.google.util.GsBashLib +import nextflow.cloud.google.file.GsBashLib import nextflow.executor.SimpleFileCopyStrategy import nextflow.processor.TaskBean import nextflow.processor.TaskRun diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy new file mode 100644 index 0000000000..10963cbaef --- /dev/null +++ b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy @@ -0,0 +1,63 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.google.nio + +import java.nio.file.FileSystem + +import com.google.cloud.storage.StorageOptions +import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +import groovy.transform.CompileStatic +import groovy.transform.Memoized +/** + * + * @author Ben Sherman + */ +@CompileStatic +class GsFileSystem extends FileSystem { + + @Delegate + CloudStorageFileSystem target + + private GsFileSystemProvider provider + + GsFileSystem(CloudStorageFileSystem target) { + this.target = target + this.provider = new GsFileSystemProvider(target.provider()) + } + + @Override + GsPath getPath(String path, String... more) { + return new GsPath(this, target.getPath(path, more)) + } + + @Override + GsFileSystemProvider provider() { + return provider + } + + static GsFileSystem forBucket(String bucket) { + final target = CloudStorageFileSystem.forBucket(bucket) + return new GsFileSystem(target) + } + + static GsFileSystem forBucket(String bucket, CloudStorageConfiguration config, StorageOptions storageOptions) { + final target = CloudStorageFileSystem.forBucket(bucket, config, storageOptions) + return new GsFileSystem(target) + } + +} diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy new file mode 100644 index 0000000000..d33c061575 --- /dev/null +++ b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy @@ -0,0 +1,148 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.google.nio + +import java.nio.file.AccessMode +import java.nio.file.CopyOption +import java.nio.file.DirectoryStream +import java.nio.file.LinkOption +import java.nio.file.OpenOption +import java.nio.file.Path +import java.nio.file.attribute.BasicFileAttributes +import java.nio.file.attribute.FileAttribute +import java.nio.file.attribute.FileAttributeView +import java.nio.file.spi.FileSystemProvider + +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystemProvider +import com.google.cloud.storage.contrib.nio.CloudStoragePath +import groovy.transform.CompileStatic +/** + * + * @author Ben Sherman + */ +@CompileStatic +class GsFileSystemProvider extends FileSystemProvider { + + @Delegate + CloudStorageFileSystemProvider delegate + + GsFileSystemProvider() { + this(new CloudStorageFileSystemProvider()) + } + + GsFileSystemProvider(CloudStorageFileSystemProvider delegate) { + this.delegate = delegate + } + + @Override + void checkAccess(Path path, AccessMode... modes) { + delegate.checkAccess(unwrap0(path), modes) + } + + @Override + void copy(Path source, Path target, CopyOption... options) { + delegate.copy(unwrap0(source), unwrap0(target), options) + } + + @Override + void createDirectory(Path dir, FileAttribute... attrs) { + delegate.createDirectory(unwrap0(dir), attrs) + } + + @Override + void delete(Path path) { + delegate.delete(unwrap0(path)) + } + + @Override + boolean deleteIfExists(Path path) { + delegate.deleteIfExists(unwrap0(path)) + } + + @Override + V getFileAttributeView(Path path, Class type, LinkOption... options) { + delegate.getFileAttributeView(unwrap0(path), type, options) + } + + @Override + boolean isHidden(Path path) { + delegate.isHidden(unwrap0(path)) + } + + @Override + boolean isSameFile(Path path, Path path2) { + delegate.isSameFile(unwrap0(path), unwrap0(path2)) + } + + @Override + void move(Path source, Path target, CopyOption... options) { + delegate.move(unwrap0(source), unwrap0(target), options) + } + + @Override + DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) { + final directoryStream = delegate.newDirectoryStream(unwrap0(dir), filter) + final iterator = new GsPathIterator(directoryStream.iterator()) + return new DirectoryStream() { + @Override + Iterator iterator() { iterator } + + @Override + void close() {} + } + } + + @Override + GsFileSystem newFileSystem(URI uri, Map env) { + new GsFileSystem(delegate.newFileSystem(uri, env)) + } + + @Override + InputStream newInputStream(Path path, OpenOption... options) { + delegate.newInputStream(unwrap0(path), options) + } + + @Override + OutputStream newOutputStream(Path path, OpenOption... options) { + delegate.newOutputStream(unwrap0(path), options) + } + + @Override + A readAttributes(Path path, Class type, LinkOption... options) { + delegate.readAttributes(unwrap0(path), type, options) + } + + private static Path unwrap0(Path path) { + path instanceof GsPath ? path.target : path + } + + private static class GsPathIterator implements Iterator { + @Delegate + private Iterator delegate + + GsPathIterator(Iterator delegate) { + this.delegate = delegate + } + + @Override + Path next() { + final path = delegate.next() + path instanceof CloudStoragePath ? new GsPath(path) : path + } + } + +} diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy new file mode 100644 index 0000000000..dd75777f28 --- /dev/null +++ b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy @@ -0,0 +1,168 @@ +/* + * Copyright 2013-2023, Seqera Labs + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package nextflow.cloud.google.nio + +import java.nio.file.LinkOption +import java.nio.file.Path + +import com.google.cloud.storage.Blob +import com.google.cloud.storage.StorageOptions +import com.google.cloud.storage.contrib.nio.CloudStoragePath +import groovy.transform.CompileStatic +import groovy.util.logging.Slf4j +import nextflow.Global +import nextflow.Session +import nextflow.cloud.google.GoogleOpts +import nextflow.file.ETagAwareFile +/** + * Implements an ETag-aware wrapper around the NIO filesystem + * provided by the Google Cloud SDK. + * + * @author Ben Sherman + */ +@Slf4j +@CompileStatic +class GsPath implements Path, ETagAwareFile { + + @Delegate + CloudStoragePath target + + private GsFileSystem fileSystem + + private Blob metadata + + GsPath(CloudStoragePath target) { + this(new GsFileSystem(target.fileSystem), target) + } + + GsPath(GsFileSystem fileSystem, CloudStoragePath target) { + this.target = target + this.fileSystem = fileSystem + } + + @Override + int compareTo(Path other) { + return target.compareTo(unwrap0(other)) + } + + @Override + boolean equals(Object other) { + return target.equals(other instanceof GsPath ? other.target : other) + } + + @Override + Path getFileName() { + return wrap0(target.getFileName()) + } + + @Override + Path getName(int index) { + return wrap0(target.getName(index)) + } + + @Override + Path getParent() { + return wrap0(target.getParent()) + } + + @Override + Path getRoot() { + return wrap0(target.getRoot()) + } + + @Override + GsFileSystem getFileSystem() { + return fileSystem + } + + @Override + int hashCode() { + return target.hashCode() + } + + @Override + Path normalize() { + return wrap0(target.normalize()) + } + + @Override + Path relativize(Path other) { + return wrap0(target.relativize(unwrap0(other))) + } + + @Override + Path resolve(String other) { + return wrap0(target.resolve(other)) + } + + @Override + Path resolve(Path other) { + return wrap0(target.resolve(unwrap0(other))) + } + + @Override + Path resolveSibling(String other) { + return wrap0(target.resolveSibling(other)) + } + + @Override + Path resolveSibling(Path other) { + return wrap0(target.resolveSibling(unwrap0(other))) + } + + @Override + Path subpath(int beginIndex, int endIndex) { + return wrap0(target.subpath(beginIndex, endIndex)) + } + + @Override + Path toAbsolutePath() { + return wrap0(target.toAbsolutePath()) + } + + @Override + Path toRealPath(LinkOption... options) { + return wrap0(target.toRealPath(options)) + } + + @Override + String toString() { + return target.toString() + } + + @Override + String getETag() { + if( metadata == null ) { + final googleOpts = GoogleOpts.create(Global.session as Session) + final client = StorageOptions.newBuilder() + .setProjectId(googleOpts.projectId) + .build() + .getService() + metadata = client.get(target.bucket(), target.toString()) + } + return metadata?.getMd5ToHexString() + } + + private static Path unwrap0(Path path) { + path instanceof GsPath ? path.target : path + } + + private Path wrap0(Path path) { + path instanceof CloudStoragePath ? new GsPath(fileSystem, path) : path + } + +} diff --git a/plugins/nf-google/src/resources/META-INF/extensions.idx b/plugins/nf-google/src/resources/META-INF/extensions.idx index c4f30a7034..b921b3bcf4 100644 --- a/plugins/nf-google/src/resources/META-INF/extensions.idx +++ b/plugins/nf-google/src/resources/META-INF/extensions.idx @@ -16,5 +16,5 @@ nextflow.cloud.google.batch.GoogleBatchExecutor nextflow.cloud.google.lifesciences.GoogleLifeSciencesExecutor -nextflow.cloud.google.util.GsPathSerializer -nextflow.cloud.google.util.GsPathFactory +nextflow.cloud.google.file.GsPathSerializer +nextflow.cloud.google.file.GsPathFactory diff --git a/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider new file mode 100644 index 0000000000..58226f86b3 --- /dev/null +++ b/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider @@ -0,0 +1,17 @@ +# +# Copyright 2013-2023, Seqera Labs +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +nextflow.cloud.google.nio.GsFileSystemProvider diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy index d6630b3cee..276e9df47b 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy @@ -16,16 +16,16 @@ package nextflow.cloud.google -import java.nio.file.Paths - -import com.google.cloud.storage.contrib.nio.CloudStoragePath -import spock.lang.Specification - import java.nio.file.FileSystem import java.nio.file.Path +import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributes import java.nio.file.spi.FileSystemProvider +import nextflow.cloud.google.nio.GsPath +import nextflow.file.FileHelper +import spock.lang.Specification + /** * * @author Paolo Di Tommaso @@ -71,9 +71,9 @@ abstract class GoogleSpecification extends Specification { } - static CloudStoragePath gsPath(String path) { + static GsPath gsPath(String path) { assert path.startsWith('gs://') - (CloudStoragePath) Paths.get( new URI(null,null,path,null,null)) + FileHelper.asPath(path) } } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy similarity index 99% rename from plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy index 18cd512dae..32272413d3 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy @@ -1,4 +1,4 @@ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import nextflow.Session diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy similarity index 99% rename from plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy index 8dd6bde8f4..74b95ae0f5 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import com.google.cloud.storage.StorageOptions import nextflow.Global diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy similarity index 80% rename from plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy index 0d309a85de..5d42b346f1 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy @@ -14,14 +14,14 @@ * limitations under the License. */ -package nextflow.cloud.google.util +package nextflow.cloud.google.file import java.nio.file.Path -import java.nio.file.Paths -import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session +import nextflow.cloud.google.nio.GsPath +import nextflow.file.FileHelper import nextflow.util.KryoHelper import spock.lang.Specification @@ -36,15 +36,14 @@ class GsPathSerializerTest extends Specification { Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] } - + when: - def uri = URI.create("gs://my-seq/data/ggal/sample.fq") - def path = Paths.get(uri) + def path = FileHelper.asPath("gs://my-seq/data/ggal/sample.fq") def buffer = KryoHelper.serialize(path) def copy = (Path)KryoHelper.deserialize(buffer) then: - copy instanceof CloudStoragePath - copy.toUri() == uri + copy instanceof GsPath + copy.toUri() == URI.create("gs://my-seq/data/ggal/sample.fq") copy.toUriString() == "gs://my-seq/data/ggal/sample.fq" } } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy index b391600ce2..0b4bb53389 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy @@ -115,7 +115,7 @@ class GoogleLifeSciencesExecutorTest extends GoogleSpecification { err.message.startsWith('Project Id `another-project` declared in the nextflow config file') } - def 'should abort operation when the workdir is not a CloudStoragePath'() { + def 'should abort operation when the workdir is not a GsPath'() { given: def session = Stub(Session) session.workDir = Stub(Path) diff --git a/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy b/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy index eddab69141..cc35c1563b 100644 --- a/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy +++ b/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy @@ -18,7 +18,7 @@ package nextflow.extension import java.nio.file.Path -import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +import nextflow.cloud.google.nio.GsFileSystem import nextflow.util.Escape import spock.lang.Specification @@ -30,7 +30,7 @@ class EscapeTest2 extends Specification { Path asPath(String bucket, String path) { - CloudStorageFileSystem.forBucket(bucket).getPath(path) + GsFileSystem.forBucket(bucket).getPath(path) } diff --git a/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy b/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy index a7b1419aef..5c969c738c 100644 --- a/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy +++ b/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy @@ -16,14 +16,13 @@ package nextflow.extension -import spock.lang.Specification -import spock.lang.Unroll - import java.nio.file.Path -import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session +import nextflow.cloud.google.nio.GsPath +import spock.lang.Specification +import spock.lang.Unroll /** * @@ -41,7 +40,7 @@ class FilesExTest2 extends Specification { when: def path = PATH as Path then: - path instanceof CloudStoragePath + path instanceof GsPath println FilesEx.toUriString(path) FilesEx.toUriString(path) == PATH diff --git a/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy b/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy index be16a7e16d..5c891a7f7a 100644 --- a/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy +++ b/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy @@ -21,13 +21,12 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +import nextflow.Global +import nextflow.Session import nextflow.SysEnv +import nextflow.cloud.google.nio.GsFileSystem import spock.lang.Ignore import spock.lang.Specification - -import nextflow.Global -import nextflow.Session import spock.lang.Unroll /** @@ -48,23 +47,23 @@ class FileHelperGsTest extends Specification { Paths.get('file.txt') and: FileHelper.asPath('gs://foo') == - CloudStorageFileSystem.forBucket('foo').getPath('') + GsFileSystem.forBucket('foo').getPath('') and: FileHelper.asPath('gs://foo/this/and/that.txt') == - CloudStorageFileSystem.forBucket('foo').getPath('/this/and/that.txt') + GsFileSystem.forBucket('foo').getPath('/this/and/that.txt') and: FileHelper.asPath('gs://foo/b a r.txt') == - CloudStorageFileSystem.forBucket('foo').getPath('/b a r.txt') + GsFileSystem.forBucket('foo').getPath('/b a r.txt') and: FileHelper.asPath('gs://f o o/bar.txt') == - CloudStorageFileSystem.forBucket('f o o').getPath('/bar.txt') + GsFileSystem.forBucket('f o o').getPath('/bar.txt') and: FileHelper.asPath('gs://f_o_o/bar.txt') == - CloudStorageFileSystem.forBucket('f_o_o').getPath('/bar.txt') + GsFileSystem.forBucket('f_o_o').getPath('/bar.txt') } @@ -89,8 +88,8 @@ class FileHelperGsTest extends Specification { @Ignore def 'should throw FileAlreadyExistsException'() { given: - def foo = CloudStorageFileSystem.forBucket('nf-bucket').getPath('foo.txt') - def bar = CloudStorageFileSystem.forBucket('nf-bucket').getPath('bar.txt') + def foo = GsFileSystem.forBucket('nf-bucket').getPath('foo.txt') + def bar = GsFileSystem.forBucket('nf-bucket').getPath('bar.txt') and: if( !Files.exists(foo) ) Files.createFile(foo) if( !Files.exists(bar) ) Files.createFile(bar) @@ -108,7 +107,7 @@ class FileHelperGsTest extends Specification { SysEnv.push(NXF_FILE_ROOT: 'gs://host.com/work') expect: - FileHelper.toCanonicalPath(VALUE) == (EXPECTED ? FileHelper.asPath(EXPECTED) : null) + FileHelper.toCanonicalPath(VALUE) == EXPECTED cleanup: SysEnv.pop() @@ -117,15 +116,15 @@ class FileHelperGsTest extends Specification { where: VALUE | EXPECTED null | null - 'file.txt' | 'gs://host.com/work/file.txt' - Path.of('file.txt') | 'gs://host.com/work/file.txt' + 'file.txt' | FileHelper.asPath('gs://host.com/work/file.txt') + Path.of('file.txt') | FileHelper.asPath('gs://host.com/work/file.txt') and: - './file.txt' | 'gs://host.com/work/file.txt' - '.' | 'gs://host.com/work' - './' | 'gs://host.com/work' - '../file.txt' | 'gs://host.com/file.txt' + './file.txt' | FileHelper.asPath('gs://host.com/work/file.txt') + '.' | FileHelper.asPath('gs://host.com/work') + './' | FileHelper.asPath('gs://host.com/work') + '../file.txt' | FileHelper.asPath('gs://host.com/file.txt') and: - '/file.txt' | '/file.txt' - Path.of('/file.txt') | '/file.txt' + '/file.txt' | Path.of('/file.txt') + Path.of('/file.txt') | Path.of('/file.txt') } } diff --git a/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy b/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy index 695abe5cb1..5338e5d174 100644 --- a/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy +++ b/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy @@ -17,9 +17,9 @@ package nextflow.file -import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session +import nextflow.cloud.google.nio.GsPath import spock.lang.Specification /** @@ -41,10 +41,10 @@ class GsPathTest extends Specification { def path4 = FileHelper.asPath('gs://bar/some/foo.txt') expect: - path1 instanceof CloudStoragePath - path2 instanceof CloudStoragePath - path3 instanceof CloudStoragePath - path4 instanceof CloudStoragePath + path1 instanceof GsPath + path2 instanceof GsPath + path3 instanceof GsPath + path4 instanceof GsPath and: path1 == path2 From 9edc7968a3697148dea173a9e60921792ec24ac5 Mon Sep 17 00:00:00 2001 From: Ben Sherman Date: Sun, 19 May 2024 01:21:28 -0500 Subject: [PATCH 6/6] Merge upstream changes, move google nio filesystem to separate branch Signed-off-by: Ben Sherman --- docs/process.md | 5 +- .../nextflow/processor/TaskProcessor.groovy | 5 + .../nextflow/processor/PublishDirTest.groovy | 2 +- .../main/nextflow/extension/FilesEx.groovy | 15 -- .../src/main/nextflow/util/HashBuilder.java | 7 + .../cloud/google/GoogleCloudPlugin.groovy | 8 - .../batch/GoogleBatchScriptLauncher.groovy | 13 +- .../batch/GoogleBatchTaskHandler.groovy | 4 - .../GoogleLifeSciencesFileCopyStrategy.groovy | 2 +- .../cloud/google/nio/GsFileSystem.groovy | 63 ------- .../google/nio/GsFileSystemProvider.groovy | 148 --------------- .../nextflow/cloud/google/nio/GsPath.groovy | 168 ------------------ .../google/{file => util}/GsBashLib.groovy | 2 +- .../{file => util}/GsPathFactory.groovy | 16 +- .../{file => util}/GsPathSerializer.groovy | 20 +-- .../src/resources/META-INF/extensions.idx | 4 +- .../java.nio.file.spi.FileSystemProvider | 17 -- .../cloud/google/GoogleSpecification.groovy | 14 +- .../GoogleLifeSciencesExecutorTest.groovy | 2 +- .../{file => util}/GsBashLibTest.groovy | 2 +- .../{file => util}/GsPathFactoryTest.groovy | 2 +- .../GsPathSerializerTest.groovy | 15 +- .../nextflow/extension/EscapeTest2.groovy | 4 +- .../nextflow/extension/FilesExTest2.groovy | 9 +- .../nextflow/file/FileHelperGsTest.groovy | 39 ++-- .../src/test/nextflow/file/GsPathTest.groovy | 10 +- 26 files changed, 89 insertions(+), 507 deletions(-) delete mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy delete mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy delete mode 100644 plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy rename plugins/nf-google/src/main/nextflow/cloud/google/{file => util}/GsBashLib.groovy (99%) rename plugins/nf-google/src/main/nextflow/cloud/google/{file => util}/GsPathFactory.groovy (88%) rename plugins/nf-google/src/main/nextflow/cloud/google/{file => util}/GsPathSerializer.groovy (70%) delete mode 100644 plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider rename plugins/nf-google/src/test/nextflow/cloud/google/{file => util}/GsBashLibTest.groovy (99%) rename plugins/nf-google/src/test/nextflow/cloud/google/{file => util}/GsPathFactoryTest.groovy (99%) rename plugins/nf-google/src/test/nextflow/cloud/google/{file => util}/GsPathSerializerTest.groovy (80%) diff --git a/docs/process.md b/docs/process.md index 1a65a44168..40be0a6135 100644 --- a/docs/process.md +++ b/docs/process.md @@ -2318,10 +2318,7 @@ Available options: - `'symlink'`: Creates an absolute symbolic link in the publish directory for each output file (default). `overwrite` -: :::{versionchanged} 24.02.0-edge - Prior to this version, the default behavior was `false` if the task was cached on a resumed run and `true` otherwise. - ::: -: Determines whether to overwrite a published file if it already exists. By default, existing files are overwritten only if they are stale, i.e. checksum does not match the new file. +: When `true` any existing file in the specified folder will be overridden (default: `true` during normal pipeline execution and `false` when pipeline execution is `resumed`). `path` : Specifies the directory where files need to be published. **Note**: the syntax `publishDir '/some/dir'` is a shortcut for `publishDir path: '/some/dir'`. diff --git a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy index 4e685eaf66..19f4f1888f 100644 --- a/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy +++ b/modules/nextflow/src/main/groovy/nextflow/processor/TaskProcessor.groovy @@ -1372,6 +1372,7 @@ class TaskProcessor { * Publish output files to a specified target folder * * @param task The task whose outputs need to be published + * @param overwrite When {@code true} any existing file will be overwritten, otherwise the publishing is ignored */ @CompileStatic protected void publishOutputs( TaskRun task ) { @@ -1387,6 +1388,10 @@ class TaskProcessor { private void publishOutputs0( TaskRun task, PublishDir publish ) { + if( publish.overwrite == null ) { + publish.overwrite = !task.cached + } + HashSet files = [] def outputs = task.getOutputsByType(FileOutParam) for( Map.Entry entry : outputs ) { diff --git a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy index efa7f500ac..60c484e402 100644 --- a/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy +++ b/modules/nextflow/src/test/groovy/nextflow/processor/PublishDirTest.groovy @@ -239,7 +239,7 @@ class PublishDirTest extends Specification { sourceDir.resolve('file1.txt'), sourceDir.resolve('file2.bam') ] as Set - def publisher = new PublishDir(path: targetDir, mode: 'copy') + def publisher = new PublishDir(path: targetDir, mode: 'copy', overwrite: 'deep') and: def timestamp1 = targetDir.resolve('file1.txt').lastModified() def timestamp2 = targetDir.resolve('file2.bam').lastModified() diff --git a/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy b/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy index 0377535829..255d52bdd3 100644 --- a/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy +++ b/modules/nf-commons/src/main/nextflow/extension/FilesEx.groovy @@ -35,17 +35,14 @@ import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFilePermission import java.nio.file.attribute.PosixFilePermissions -import com.google.common.hash.Hashing import groovy.transform.CompileStatic import groovy.transform.PackageScope import groovy.transform.stc.ClosureParams import groovy.transform.stc.FromString import groovy.util.logging.Slf4j -import nextflow.file.ETagAwareFile import nextflow.file.FileHelper import nextflow.file.FileSystemPathFactory import nextflow.io.ByteBufferBackedInputStream -import nextflow.util.CacheHelper import nextflow.util.CharsetHelper import nextflow.util.CheckHelper @@ -1602,16 +1599,4 @@ class FilesEx { static String getScheme(Path path) { path.getFileSystem().provider().getScheme() } - - static String getChecksum(Path path) { - if( Files.isDirectory(path) ) - return null - - // use etag if available - if( path instanceof ETagAwareFile ) - return path.getETag() - - // otherwise compute checksum manually - CacheHelper.hasher(Hashing.md5(), path, CacheHelper.HashMode.DEEP).hash().toString() - } } diff --git a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java index 46c3fedf84..6d14c64142 100644 --- a/modules/nf-commons/src/main/nextflow/util/HashBuilder.java +++ b/modules/nf-commons/src/main/nextflow/util/HashBuilder.java @@ -18,6 +18,7 @@ import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; @@ -44,6 +45,7 @@ import nextflow.ISession; import nextflow.extension.Bolts; import nextflow.extension.FilesEx; +import nextflow.file.ETagAwareFile; import nextflow.file.FileHolder; import nextflow.io.SerializableMarker; import org.slf4j.Logger; @@ -413,6 +415,11 @@ static private Hasher hashFileMetadata( Hasher hasher, Path file, BasicFileAttri */ static private Hasher hashFileContent( Hasher hasher, Path path ) { + // use etag if available + if( path instanceof ETagAwareFile ) + hasher.putBytes(((ETagAwareFile)path).getETag().getBytes(StandardCharsets.UTF_8)); + + // otherwise compute checksum manually OutputStream output = Funnels.asOutputStream(hasher); try { Files.copy(path, output); diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy index d8b4aa4a40..836d9a6b7b 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/GoogleCloudPlugin.groovy @@ -16,8 +16,6 @@ package nextflow.cloud.google import groovy.transform.CompileStatic -import nextflow.cloud.google.nio.GsFileSystemProvider -import nextflow.file.FileHelper import nextflow.plugin.BasePlugin import org.pf4j.PluginWrapper /** @@ -32,10 +30,4 @@ class GoogleCloudPlugin extends BasePlugin { super(wrapper) } - @Override - void start() { - super.start() - FileHelper.getOrInstallProvider(GsFileSystemProvider) - } - } diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy index e08a838153..c54b92f615 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy @@ -22,13 +22,10 @@ import java.nio.file.Paths import com.google.cloud.batch.v1.GCS import com.google.cloud.batch.v1.Volume +import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -<<<<<<< HEAD -import nextflow.cloud.google.nio.GsPath -======= import nextflow.cloud.google.batch.client.BatchConfig ->>>>>>> master import nextflow.executor.BashWrapperBuilder import nextflow.extension.FilesEx import nextflow.processor.TaskBean @@ -47,12 +44,8 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc private static final String MOUNT_ROOT = '/mnt/disks' -<<<<<<< HEAD - private GsPath remoteWorkDir -======= private BatchConfig config private CloudStoragePath remoteWorkDir ->>>>>>> master private Path remoteBinDir private Set buckets = new HashSet<>() private PathTrie pathTrie = new PathTrie() @@ -63,7 +56,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir) { super(bean) // keep track the google storage work dir - this.remoteWorkDir = (GsPath) bean.workDir + this.remoteWorkDir = (CloudStoragePath) bean.workDir this.remoteBinDir = toContainerMount(remoteBinDir) // map bean work and target dirs to container mount @@ -112,7 +105,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc } protected Path toContainerMount(Path path, boolean parent=false) { - if( path instanceof GsPath ) { + if( path instanceof CloudStoragePath ) { buckets.add(path.bucket()) pathTrie.add( (parent ? "/${path.bucket()}${path.parent}" : "/${path.bucket()}${path}").toString() ) final containerMount = containerMountPath(path) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy index aa0505cd3a..f292b335e7 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy @@ -98,12 +98,8 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask { super(task) this.client = executor.getClient() this.executor = executor -<<<<<<< HEAD - // those files are access via NF runtime, keep based on GsPath -======= this.jobId = customJobName(task) ?: "nf-${task.hashLog.replace('/','')}-${System.currentTimeMillis()}" // those files are access via NF runtime, keep based on CloudStoragePath ->>>>>>> master this.outputFile = task.workDir.resolve(TaskRun.CMD_OUTFILE) this.errorFile = task.workDir.resolve(TaskRun.CMD_ERRFILE) this.exitFile = task.workDir.resolve(TaskRun.CMD_EXIT) diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy index 2644ff50cd..c013011a93 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/lifesciences/GoogleLifeSciencesFileCopyStrategy.groovy @@ -23,7 +23,7 @@ import java.nio.file.Path import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cloud.google.file.GsBashLib +import nextflow.cloud.google.util.GsBashLib import nextflow.executor.SimpleFileCopyStrategy import nextflow.processor.TaskBean import nextflow.processor.TaskRun diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy deleted file mode 100644 index 10963cbaef..0000000000 --- a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystem.groovy +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.cloud.google.nio - -import java.nio.file.FileSystem - -import com.google.cloud.storage.StorageOptions -import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration -import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem -import groovy.transform.CompileStatic -import groovy.transform.Memoized -/** - * - * @author Ben Sherman - */ -@CompileStatic -class GsFileSystem extends FileSystem { - - @Delegate - CloudStorageFileSystem target - - private GsFileSystemProvider provider - - GsFileSystem(CloudStorageFileSystem target) { - this.target = target - this.provider = new GsFileSystemProvider(target.provider()) - } - - @Override - GsPath getPath(String path, String... more) { - return new GsPath(this, target.getPath(path, more)) - } - - @Override - GsFileSystemProvider provider() { - return provider - } - - static GsFileSystem forBucket(String bucket) { - final target = CloudStorageFileSystem.forBucket(bucket) - return new GsFileSystem(target) - } - - static GsFileSystem forBucket(String bucket, CloudStorageConfiguration config, StorageOptions storageOptions) { - final target = CloudStorageFileSystem.forBucket(bucket, config, storageOptions) - return new GsFileSystem(target) - } - -} diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy deleted file mode 100644 index d33c061575..0000000000 --- a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsFileSystemProvider.groovy +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.cloud.google.nio - -import java.nio.file.AccessMode -import java.nio.file.CopyOption -import java.nio.file.DirectoryStream -import java.nio.file.LinkOption -import java.nio.file.OpenOption -import java.nio.file.Path -import java.nio.file.attribute.BasicFileAttributes -import java.nio.file.attribute.FileAttribute -import java.nio.file.attribute.FileAttributeView -import java.nio.file.spi.FileSystemProvider - -import com.google.cloud.storage.contrib.nio.CloudStorageFileSystemProvider -import com.google.cloud.storage.contrib.nio.CloudStoragePath -import groovy.transform.CompileStatic -/** - * - * @author Ben Sherman - */ -@CompileStatic -class GsFileSystemProvider extends FileSystemProvider { - - @Delegate - CloudStorageFileSystemProvider delegate - - GsFileSystemProvider() { - this(new CloudStorageFileSystemProvider()) - } - - GsFileSystemProvider(CloudStorageFileSystemProvider delegate) { - this.delegate = delegate - } - - @Override - void checkAccess(Path path, AccessMode... modes) { - delegate.checkAccess(unwrap0(path), modes) - } - - @Override - void copy(Path source, Path target, CopyOption... options) { - delegate.copy(unwrap0(source), unwrap0(target), options) - } - - @Override - void createDirectory(Path dir, FileAttribute... attrs) { - delegate.createDirectory(unwrap0(dir), attrs) - } - - @Override - void delete(Path path) { - delegate.delete(unwrap0(path)) - } - - @Override - boolean deleteIfExists(Path path) { - delegate.deleteIfExists(unwrap0(path)) - } - - @Override - V getFileAttributeView(Path path, Class type, LinkOption... options) { - delegate.getFileAttributeView(unwrap0(path), type, options) - } - - @Override - boolean isHidden(Path path) { - delegate.isHidden(unwrap0(path)) - } - - @Override - boolean isSameFile(Path path, Path path2) { - delegate.isSameFile(unwrap0(path), unwrap0(path2)) - } - - @Override - void move(Path source, Path target, CopyOption... options) { - delegate.move(unwrap0(source), unwrap0(target), options) - } - - @Override - DirectoryStream newDirectoryStream(Path dir, DirectoryStream.Filter filter) { - final directoryStream = delegate.newDirectoryStream(unwrap0(dir), filter) - final iterator = new GsPathIterator(directoryStream.iterator()) - return new DirectoryStream() { - @Override - Iterator iterator() { iterator } - - @Override - void close() {} - } - } - - @Override - GsFileSystem newFileSystem(URI uri, Map env) { - new GsFileSystem(delegate.newFileSystem(uri, env)) - } - - @Override - InputStream newInputStream(Path path, OpenOption... options) { - delegate.newInputStream(unwrap0(path), options) - } - - @Override - OutputStream newOutputStream(Path path, OpenOption... options) { - delegate.newOutputStream(unwrap0(path), options) - } - - @Override - A readAttributes(Path path, Class type, LinkOption... options) { - delegate.readAttributes(unwrap0(path), type, options) - } - - private static Path unwrap0(Path path) { - path instanceof GsPath ? path.target : path - } - - private static class GsPathIterator implements Iterator { - @Delegate - private Iterator delegate - - GsPathIterator(Iterator delegate) { - this.delegate = delegate - } - - @Override - Path next() { - final path = delegate.next() - path instanceof CloudStoragePath ? new GsPath(path) : path - } - } - -} diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy deleted file mode 100644 index dd75777f28..0000000000 --- a/plugins/nf-google/src/main/nextflow/cloud/google/nio/GsPath.groovy +++ /dev/null @@ -1,168 +0,0 @@ -/* - * Copyright 2013-2023, Seqera Labs - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package nextflow.cloud.google.nio - -import java.nio.file.LinkOption -import java.nio.file.Path - -import com.google.cloud.storage.Blob -import com.google.cloud.storage.StorageOptions -import com.google.cloud.storage.contrib.nio.CloudStoragePath -import groovy.transform.CompileStatic -import groovy.util.logging.Slf4j -import nextflow.Global -import nextflow.Session -import nextflow.cloud.google.GoogleOpts -import nextflow.file.ETagAwareFile -/** - * Implements an ETag-aware wrapper around the NIO filesystem - * provided by the Google Cloud SDK. - * - * @author Ben Sherman - */ -@Slf4j -@CompileStatic -class GsPath implements Path, ETagAwareFile { - - @Delegate - CloudStoragePath target - - private GsFileSystem fileSystem - - private Blob metadata - - GsPath(CloudStoragePath target) { - this(new GsFileSystem(target.fileSystem), target) - } - - GsPath(GsFileSystem fileSystem, CloudStoragePath target) { - this.target = target - this.fileSystem = fileSystem - } - - @Override - int compareTo(Path other) { - return target.compareTo(unwrap0(other)) - } - - @Override - boolean equals(Object other) { - return target.equals(other instanceof GsPath ? other.target : other) - } - - @Override - Path getFileName() { - return wrap0(target.getFileName()) - } - - @Override - Path getName(int index) { - return wrap0(target.getName(index)) - } - - @Override - Path getParent() { - return wrap0(target.getParent()) - } - - @Override - Path getRoot() { - return wrap0(target.getRoot()) - } - - @Override - GsFileSystem getFileSystem() { - return fileSystem - } - - @Override - int hashCode() { - return target.hashCode() - } - - @Override - Path normalize() { - return wrap0(target.normalize()) - } - - @Override - Path relativize(Path other) { - return wrap0(target.relativize(unwrap0(other))) - } - - @Override - Path resolve(String other) { - return wrap0(target.resolve(other)) - } - - @Override - Path resolve(Path other) { - return wrap0(target.resolve(unwrap0(other))) - } - - @Override - Path resolveSibling(String other) { - return wrap0(target.resolveSibling(other)) - } - - @Override - Path resolveSibling(Path other) { - return wrap0(target.resolveSibling(unwrap0(other))) - } - - @Override - Path subpath(int beginIndex, int endIndex) { - return wrap0(target.subpath(beginIndex, endIndex)) - } - - @Override - Path toAbsolutePath() { - return wrap0(target.toAbsolutePath()) - } - - @Override - Path toRealPath(LinkOption... options) { - return wrap0(target.toRealPath(options)) - } - - @Override - String toString() { - return target.toString() - } - - @Override - String getETag() { - if( metadata == null ) { - final googleOpts = GoogleOpts.create(Global.session as Session) - final client = StorageOptions.newBuilder() - .setProjectId(googleOpts.projectId) - .build() - .getService() - metadata = client.get(target.bucket(), target.toString()) - } - return metadata?.getMd5ToHexString() - } - - private static Path unwrap0(Path path) { - path instanceof GsPath ? path.target : path - } - - private Path wrap0(Path path) { - path instanceof CloudStoragePath ? new GsPath(fileSystem, path) : path - } - -} diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy similarity index 99% rename from plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy index 476e6a19db..6eb6d0c24d 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsBashLib.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsBashLib.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import groovy.transform.CompileStatic import groovy.transform.Memoized diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy similarity index 88% rename from plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy index 8b4bd79a02..1924432e4a 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathFactory.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathFactory.groovy @@ -14,19 +14,19 @@ * limitations under the License. */ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import java.nio.file.Path import com.google.api.gax.retrying.RetrySettings import com.google.cloud.storage.StorageOptions import com.google.cloud.storage.contrib.nio.CloudStorageConfiguration +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import nextflow.Global import nextflow.Session import nextflow.cloud.google.GoogleOpts -import nextflow.cloud.google.nio.GsFileSystem -import nextflow.cloud.google.nio.GsPath import nextflow.cloud.google.lifesciences.GoogleLifeSciencesFileCopyStrategy import nextflow.file.FileSystemPathFactory /** @@ -98,13 +98,13 @@ class GsPathFactory extends FileSystemPathFactory { final str = uri.substring(5) final p = str.indexOf('/') return p == -1 - ? GsFileSystem.forBucket(str, storageConfig, storageOptions).getPath('') - : GsFileSystem.forBucket(str.substring(0,p), storageConfig, storageOptions).getPath(str.substring(p)) + ? CloudStorageFileSystem.forBucket(str, storageConfig, storageOptions).getPath('') + : CloudStorageFileSystem.forBucket(str.substring(0,p), storageConfig, storageOptions).getPath(str.substring(p)) } @Override protected String toUriString(Path path) { - if( path instanceof GsPath ) { + if( path instanceof CloudStoragePath ) { return "gs://${path.bucket()}$path".toString() } return null @@ -112,7 +112,7 @@ class GsPathFactory extends FileSystemPathFactory { @Override protected String getBashLib(Path path) { - if( path instanceof GsPath ) { + if( path instanceof CloudStoragePath ) { return GsBashLib.fromSession( Global.session as Session ) } return null @@ -120,7 +120,7 @@ class GsPathFactory extends FileSystemPathFactory { @Override protected String getUploadCmd(String source, Path target) { - if( target instanceof GsPath ) { + if( target instanceof CloudStoragePath ) { GoogleLifeSciencesFileCopyStrategy.uploadCmd(source,target) } return null diff --git a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy similarity index 70% rename from plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy rename to plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy index 5b22e38fab..c2903d4ebc 100644 --- a/plugins/nf-google/src/main/nextflow/cloud/google/file/GsPathSerializer.groovy +++ b/plugins/nf-google/src/main/nextflow/cloud/google/util/GsPathSerializer.groovy @@ -14,48 +14,48 @@ * limitations under the License. */ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import com.esotericsoftware.kryo.Kryo import com.esotericsoftware.kryo.Serializer import com.esotericsoftware.kryo.io.Input import com.esotericsoftware.kryo.io.Output import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem +import com.google.cloud.storage.contrib.nio.CloudStoragePath import groovy.transform.CompileStatic import groovy.util.logging.Slf4j -import nextflow.cloud.google.nio.GsPath import nextflow.util.SerializerRegistrant import org.pf4j.Extension /** - * Serializer for a {@link GsPath} + * Serializer for a {@link CloudStoragePath} * * @author Paolo Di Tommaso */ @Slf4j @Extension @CompileStatic -class GsPathSerializer extends Serializer implements SerializerRegistrant { +class GsPathSerializer extends Serializer implements SerializerRegistrant { @Override - void write(Kryo kryo, Output output, GsPath target) { + void write(Kryo kryo, Output output, CloudStoragePath target) { def path = target.toString() if( !path.startsWith('/') ) // <-- it looks a bug in the google nio library, in some case the path returned is not absolute path = '/' + path path = target.bucket() + path - log.trace "Google GsPath serialisation > path=$path" + log.trace "Google CloudStoragePath serialisation > path=$path" output.writeString(path) } @Override - GsPath read(Kryo kryo, Input input, Class type) { + CloudStoragePath read(Kryo kryo, Input input, Class type) { final path = input.readString() - log.trace "Google GsPath de-serialization > path=$path" + log.trace "Google CloudStoragePath de-serialization > path=$path" def uri = CloudStorageFileSystem.URI_SCHEME + '://' + path - (GsPath) GsPathFactory.parse(uri) + (CloudStoragePath) GsPathFactory.parse(uri) } @Override void register(Map serializers) { - serializers.put(GsPath, GsPathSerializer) + serializers.put(CloudStoragePath, GsPathSerializer) } } diff --git a/plugins/nf-google/src/resources/META-INF/extensions.idx b/plugins/nf-google/src/resources/META-INF/extensions.idx index 74fc8becf6..354b2d7688 100644 --- a/plugins/nf-google/src/resources/META-INF/extensions.idx +++ b/plugins/nf-google/src/resources/META-INF/extensions.idx @@ -16,5 +16,5 @@ nextflow.cloud.google.batch.GoogleBatchExecutor nextflow.cloud.google.lifesciences.GoogleLifeSciencesExecutor -nextflow.cloud.google.file.GsPathSerializer -nextflow.cloud.google.file.GsPathFactory +nextflow.cloud.google.util.GsPathSerializer +nextflow.cloud.google.util.GsPathFactory diff --git a/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider b/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider deleted file mode 100644 index 58226f86b3..0000000000 --- a/plugins/nf-google/src/resources/META-INF/services/java.nio.file.spi.FileSystemProvider +++ /dev/null @@ -1,17 +0,0 @@ -# -# Copyright 2013-2023, Seqera Labs -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -nextflow.cloud.google.nio.GsFileSystemProvider diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy index fd6149143f..8e530db943 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/GoogleSpecification.groovy @@ -16,16 +16,16 @@ package nextflow.cloud.google +import java.nio.file.Paths + +import com.google.cloud.storage.contrib.nio.CloudStoragePath +import spock.lang.Specification + import java.nio.file.FileSystem import java.nio.file.Path -import java.nio.file.Paths import java.nio.file.attribute.BasicFileAttributes import java.nio.file.spi.FileSystemProvider -import nextflow.cloud.google.nio.GsPath -import nextflow.file.FileHelper -import spock.lang.Specification - /** * * @author Paolo Di Tommaso @@ -71,9 +71,9 @@ abstract class GoogleSpecification extends Specification { } - static GsPath gsPath(String path) { + static CloudStoragePath gsPath(String path) { assert path.startsWith('gs://') - FileHelper.asPath(path) + (CloudStoragePath) Paths.get( new URI(null,null,path,null,null)) } } diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy index 0d4e597f57..bb57f201a3 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/lifesciences/GoogleLifeSciencesExecutorTest.groovy @@ -115,7 +115,7 @@ class GoogleLifeSciencesExecutorTest extends GoogleSpecification { err.message.startsWith('Project Id `another-project` declared in the nextflow config file') } - def 'should abort operation when the workdir is not a GsPath'() { + def 'should abort operation when the workdir is not a CloudStoragePath'() { given: def session = Stub(Session) session.workDir = Stub(Path) diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy similarity index 99% rename from plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy index 32272413d3..18cd512dae 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsBashLibTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsBashLibTest.groovy @@ -1,4 +1,4 @@ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import nextflow.Session diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy similarity index 99% rename from plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy index 3643764636..61b9349112 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathFactoryTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathFactoryTest.groovy @@ -14,7 +14,7 @@ * limitations under the License. */ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import com.google.cloud.storage.StorageOptions import nextflow.Global diff --git a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy similarity index 80% rename from plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy rename to plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy index ac66ea1562..6180a38344 100644 --- a/plugins/nf-google/src/test/nextflow/cloud/google/file/GsPathSerializerTest.groovy +++ b/plugins/nf-google/src/test/nextflow/cloud/google/util/GsPathSerializerTest.groovy @@ -14,14 +14,14 @@ * limitations under the License. */ -package nextflow.cloud.google.file +package nextflow.cloud.google.util import java.nio.file.Path +import java.nio.file.Paths +import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session -import nextflow.cloud.google.nio.GsPath -import nextflow.file.FileHelper import nextflow.util.KryoHelper import spock.lang.Specification @@ -36,14 +36,15 @@ class GsPathSerializerTest extends Specification { Global.session = Mock(Session) { getConfig() >> [google:[project:'foo', region:'x']] } - + when: - def path = FileHelper.asPath("gs://my-seq/data/ggal/sample.fq") + def uri = URI.create("gs://my-seq/data/ggal/sample.fq") + def path = Paths.get(uri) def buffer = KryoHelper.serialize(path) def copy = (Path)KryoHelper.deserialize(buffer) then: - copy instanceof GsPath - copy.toUri() == URI.create("gs://my-seq/data/ggal/sample.fq") + copy instanceof CloudStoragePath + copy.toUri() == uri copy.toUriString() == "gs://my-seq/data/ggal/sample.fq" } } diff --git a/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy b/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy index 6b04150717..a116ffe8e5 100644 --- a/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy +++ b/plugins/nf-google/src/test/nextflow/extension/EscapeTest2.groovy @@ -18,7 +18,7 @@ package nextflow.extension import java.nio.file.Path -import nextflow.cloud.google.nio.GsFileSystem +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem import nextflow.util.Escape import spock.lang.Specification @@ -30,7 +30,7 @@ class EscapeTest2 extends Specification { Path asPath(String bucket, String path) { - GsFileSystem.forBucket(bucket).getPath(path) + CloudStorageFileSystem.forBucket(bucket).getPath(path) } diff --git a/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy b/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy index 8efedf673f..cd9b70c2d3 100644 --- a/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy +++ b/plugins/nf-google/src/test/nextflow/extension/FilesExTest2.groovy @@ -16,13 +16,14 @@ package nextflow.extension +import spock.lang.Specification +import spock.lang.Unroll + import java.nio.file.Path +import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session -import nextflow.cloud.google.nio.GsPath -import spock.lang.Specification -import spock.lang.Unroll /** * @@ -40,7 +41,7 @@ class FilesExTest2 extends Specification { when: def path = PATH as Path then: - path instanceof GsPath + path instanceof CloudStoragePath println FilesEx.toUriString(path) FilesEx.toUriString(path) == PATH diff --git a/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy b/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy index af0216ffb6..ee4d7181d3 100644 --- a/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy +++ b/plugins/nf-google/src/test/nextflow/file/FileHelperGsTest.groovy @@ -21,12 +21,13 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths -import nextflow.Global -import nextflow.Session +import com.google.cloud.storage.contrib.nio.CloudStorageFileSystem import nextflow.SysEnv -import nextflow.cloud.google.nio.GsFileSystem import spock.lang.Ignore import spock.lang.Specification + +import nextflow.Global +import nextflow.Session import spock.lang.Unroll /** @@ -47,23 +48,23 @@ class FileHelperGsTest extends Specification { Paths.get('file.txt') and: FileHelper.asPath('gs://foo') == - GsFileSystem.forBucket('foo').getPath('') + CloudStorageFileSystem.forBucket('foo').getPath('') and: FileHelper.asPath('gs://foo/this/and/that.txt') == - GsFileSystem.forBucket('foo').getPath('/this/and/that.txt') + CloudStorageFileSystem.forBucket('foo').getPath('/this/and/that.txt') and: FileHelper.asPath('gs://foo/b a r.txt') == - GsFileSystem.forBucket('foo').getPath('/b a r.txt') + CloudStorageFileSystem.forBucket('foo').getPath('/b a r.txt') and: FileHelper.asPath('gs://f o o/bar.txt') == - GsFileSystem.forBucket('f o o').getPath('/bar.txt') + CloudStorageFileSystem.forBucket('f o o').getPath('/bar.txt') and: FileHelper.asPath('gs://f_o_o/bar.txt') == - GsFileSystem.forBucket('f_o_o').getPath('/bar.txt') + CloudStorageFileSystem.forBucket('f_o_o').getPath('/bar.txt') } @@ -88,8 +89,8 @@ class FileHelperGsTest extends Specification { @Ignore def 'should throw FileAlreadyExistsException'() { given: - def foo = GsFileSystem.forBucket('nf-bucket').getPath('foo.txt') - def bar = GsFileSystem.forBucket('nf-bucket').getPath('bar.txt') + def foo = CloudStorageFileSystem.forBucket('nf-bucket').getPath('foo.txt') + def bar = CloudStorageFileSystem.forBucket('nf-bucket').getPath('bar.txt') and: if( !Files.exists(foo) ) Files.createFile(foo) if( !Files.exists(bar) ) Files.createFile(bar) @@ -107,7 +108,7 @@ class FileHelperGsTest extends Specification { SysEnv.push(NXF_FILE_ROOT: 'gs://host.com/work') expect: - FileHelper.toCanonicalPath(VALUE) == EXPECTED + FileHelper.toCanonicalPath(VALUE) == (EXPECTED ? FileHelper.asPath(EXPECTED) : null) cleanup: SysEnv.pop() @@ -116,15 +117,15 @@ class FileHelperGsTest extends Specification { where: VALUE | EXPECTED null | null - 'file.txt' | FileHelper.asPath('gs://host.com/work/file.txt') - Path.of('file.txt') | FileHelper.asPath('gs://host.com/work/file.txt') + 'file.txt' | 'gs://host.com/work/file.txt' + Path.of('file.txt') | 'gs://host.com/work/file.txt' and: - './file.txt' | FileHelper.asPath('gs://host.com/work/file.txt') - '.' | FileHelper.asPath('gs://host.com/work') - './' | FileHelper.asPath('gs://host.com/work') - '../file.txt' | FileHelper.asPath('gs://host.com/file.txt') + './file.txt' | 'gs://host.com/work/file.txt' + '.' | 'gs://host.com/work' + './' | 'gs://host.com/work' + '../file.txt' | 'gs://host.com/file.txt' and: - '/file.txt' | Path.of('/file.txt') - Path.of('/file.txt') | Path.of('/file.txt') + '/file.txt' | '/file.txt' + Path.of('/file.txt') | '/file.txt' } } diff --git a/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy b/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy index 72082e8685..3a3cc770e7 100644 --- a/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy +++ b/plugins/nf-google/src/test/nextflow/file/GsPathTest.groovy @@ -17,9 +17,9 @@ package nextflow.file +import com.google.cloud.storage.contrib.nio.CloudStoragePath import nextflow.Global import nextflow.Session -import nextflow.cloud.google.nio.GsPath import spock.lang.Specification /** @@ -41,10 +41,10 @@ class GsPathTest extends Specification { def path4 = FileHelper.asPath('gs://bar/some/foo.txt') expect: - path1 instanceof GsPath - path2 instanceof GsPath - path3 instanceof GsPath - path4 instanceof GsPath + path1 instanceof CloudStoragePath + path2 instanceof CloudStoragePath + path3 instanceof CloudStoragePath + path4 instanceof CloudStoragePath and: path1 == path2