set part size for s3:CopyObject minimum 128Mb, look details https://r…

…epost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data
Altinity · Mar 24, 2024 · 5e41c8b · 5e41c8b
1 parent 30579e6
commit 5e41c8b
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 2 deletions.
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -1,3 +1,7 @@
+# v2.4.35
+IMPROVEMENTS
+- set part size for `s3:CopyObject` minimum 128Mb, look details https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data
+
 # v2.4.34
 BUG FIXES
 - fixed wrong behavior for CLICKHOUSE_SKIP_TABLES_ENGINES for engine=EngineName without parameters

diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go
@@ -492,8 +492,9 @@ func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, d
 	if srcSize%s.Config.MaxPartsCount > 0 {
 		partSize++
 	}
-	if partSize < 5*1024*1024 {
-		partSize = 5 * 1024 * 1024
+	// 128Mb part size recommendation from https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data
+	if partSize < 128*1024*1024 {
+		partSize = 128 * 1024 * 1024
 	}
 
 	// Calculate the number of parts