From 5e41c8be05849a4ce9e171fe2e11f55f9ece309b Mon Sep 17 00:00:00 2001 From: Slach Date: Sun, 24 Mar 2024 20:09:30 +0400 Subject: [PATCH] set part size for s3:CopyObject minimum 128Mb, look details https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data --- ChangeLog.md | 4 ++++ pkg/storage/s3.go | 5 +++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ChangeLog.md b/ChangeLog.md index f491aaa4..ef194368 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,7 @@ +# v2.4.35 +IMPROVEMENTS +- set part size for `s3:CopyObject` minimum 128Mb, look details https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data + # v2.4.34 BUG FIXES - fixed wrong behavior for CLICKHOUSE_SKIP_TABLES_ENGINES for engine=EngineName without parameters diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go index eebba9d9..bb62bae4 100644 --- a/pkg/storage/s3.go +++ b/pkg/storage/s3.go @@ -492,8 +492,9 @@ func (s *S3) CopyObject(ctx context.Context, srcSize int64, srcBucket, srcKey, d if srcSize%s.Config.MaxPartsCount > 0 { partSize++ } - if partSize < 5*1024*1024 { - partSize = 5 * 1024 * 1024 + // 128Mb part size recommendation from https://repost.aws/questions/QUtW2_XaALTK63wv9XLSywiQ/s3-sync-command-is-slow-to-start-on-some-data + if partSize < 128*1024*1024 { + partSize = 128 * 1024 * 1024 } // Calculate the number of parts