Allow GCS backend to upload more than 32 parts

This recursively combines up to 32 sets of 32 parts, allowing 1024 part multipart uploads. Fixes #330.
gaul · Mar 7, 2021 · 37e843a · 37e843a
1 parent 29413b1
commit 37e843a
Showing 1 changed file with 42 additions and 9 deletions.
diff --git a/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java b/src/main/java/org/gaul/s3proxy/S3ProxyHandler.java
@@ -2197,20 +2197,20 @@ private void handleCompleteMultipartUpload(HttpServletRequest request,
             HttpServletResponse response, InputStream is,
             final BlobStore blobStore, String containerName, String blobName,
             String uploadId) throws IOException, S3Exception {
-        final MultipartUpload mpu;
+        BlobMetadata metadata;
+        PutOptions options;
         if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(
                 blobStore))) {
-            Blob stubBlob = blobStore.getBlob(containerName, uploadId);
+            metadata = blobStore.getBlob(containerName, uploadId).getMetadata();
             BlobAccess access = blobStore.getBlobAccess(containerName,
                     uploadId);
-            mpu = MultipartUpload.create(containerName,
-                    blobName, uploadId, stubBlob.getMetadata(),
-                    new PutOptions().setBlobAccess(access));
+            options = new PutOptions().setBlobAccess(access);
         } else {
-            mpu = MultipartUpload.create(containerName,
-                    blobName, uploadId, new MutableBlobMetadataImpl(),
-                    new PutOptions());
+            metadata = new MutableBlobMetadataImpl();
+            options = new PutOptions();
         }
+        final MultipartUpload mpu = MultipartUpload.create(containerName,
+                blobName, uploadId, metadata, options);
 
         // List parts to get part sizes and to map multiple Azure parts
         // into single parts.
@@ -2219,7 +2219,6 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
         for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
             builder.put(part.partNumber(), part);
         }
-        ImmutableMap<Integer, MultipartPart> partsByListing = builder.build();
 
         final List<MultipartPart> parts = new ArrayList<>();
         String blobStoreType = getBlobStoreType(blobStore);
@@ -2228,6 +2227,28 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
             for (MultipartPart part : blobStore.listMultipartUpload(mpu)) {
                 parts.add(part);
             }
+        } else if (blobStoreType.equals("google-cloud-storage")) {
+            // GCS only supports 32 parts but we can support up to 1024 by
+            // recursively combining objects.
+            for (int partNumber = 1;; ++partNumber) {
+                MultipartUpload mpu2 = MultipartUpload.create(
+                        containerName,
+                        String.format("%s_%08d", mpu.id(), partNumber),
+                        String.format("%s_%08d", mpu.id(), partNumber),
+                        metadata, options);
+                List<MultipartPart> subParts = blobStore.listMultipartUpload(
+                        mpu2);
+                if (subParts.isEmpty()) {
+                    break;
+                }
+                long partSize = 0;
+                for (MultipartPart part : subParts) {
+                    partSize += part.partSize();
+                }
+                String eTag = blobStore.completeMultipartUpload(mpu2, subParts);
+                parts.add(MultipartPart.create(
+                        partNumber, partSize, eTag, /*lastModified=*/ null));
+            }
         } else {
             CompleteMultipartUploadRequest cmu;
             try {
@@ -2244,6 +2265,9 @@ blobName, uploadId, new MutableBlobMetadataImpl(),
                     requestParts.put(part.partNumber, part.eTag);
                 }
             }
+
+            ImmutableMap<Integer, MultipartPart> partsByListing =
+                    builder.build();
             for (Iterator<Map.Entry<Integer, String>> it =
                     requestParts.entrySet().iterator(); it.hasNext();) {
                 Map.Entry<Integer, String> entry = it.next();
@@ -2722,6 +2746,15 @@ private void handleUploadPart(HttpServletRequest request,
                             "ArgumentValue", partNumberString));
         }
 
+        // GCS only supports 32 parts so partition MPU into 32-part chunks.
+        String blobStoreType = getBlobStoreType(blobStore);
+        if (blobStoreType.equals("google-cloud-storage")) {
+            // fix up 1-based part numbers
+            uploadId = String.format(
+                    "%s_%08d", uploadId, ((partNumber - 1) / 32) + 1);
+            partNumber = ((partNumber - 1) % 32) + 1;
+        }
+
         // TODO: how to reconstruct original mpu?
         BlobMetadata blobMetadata;
         if (Quirks.MULTIPART_REQUIRES_STUB.contains(getBlobStoreType(