fix possible wrong merging after ATTACH PART for Collapsing and Repla…

…cing engines without version, look ClickHouse/ClickHouse#71009 for details

Signed-off-by: Slach <[email protected]>

ChangeLog.md

@@ -1,3 +1,10 @@
+# v2.6.4
+
+BUG FIXES
+
+- fix possible wrong merging after ATTACH PART for Collapsing and Replacing engines without version,
+  look https://github.com/ClickHouse/ClickHouse/issues/71009 for details
+
 # v2.6.3
 IMPROVEMENTS
 - implement new format for *.state2 files boltdb key value (please, check memory RSS usage)

pkg/clickhouse/clickhouse.go

@@ -7,6 +7,7 @@ import (
 	"database/sql"
 	"errors"
 	"fmt"
+	"github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper"
 	"os"
 	"path"
 	"path/filepath"
@@ -770,6 +771,8 @@ func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Tab
 		return nil
 	}
 	for disk := range table.Parts {
+		// https://github.com/ClickHouse/ClickHouse/issues/71009
+		filesystemhelper.SortPartsByMinBlock(table.Parts[disk])
 		for _, part := range table.Parts[disk] {
 			if !strings.HasSuffix(part.Name, ".proj") {
 				query := fmt.Sprintf("ALTER TABLE `%s`.`%s` ATTACH PART '%s'", table.Database, table.Table, part.Name)

pkg/filesystemhelper/filesystemhelper.go

@@ -7,6 +7,8 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -297,9 +299,27 @@ func MoveShadowToBackup(shadowPath, backupPartsPath string, partitionsBackupMap
 			return os.Link(filePath, dstFilePath)
 		}
 	})
+	// https://github.com/ClickHouse/ClickHouse/issues/71009
+	SortPartsByMinBlock(parts)
 	return parts, size, err
 }
 
+// SortPartsByMinBlock need to avoid wrong restore for Replacing, Collapsing, https://github.com/ClickHouse/ClickHouse/issues/71009
+func SortPartsByMinBlock(parts []metadata.Part) {
+	sort.Slice(parts, func(i, j int) bool {
+		namePartsI := strings.Split(parts[i].Name, "_")
+		namePartsJ := strings.Split(parts[j].Name, "_")
+		// partitions different
+		if namePartsI[0] != namePartsJ[0] {
+			return namePartsI[0] < namePartsJ[0]
+		}
+		// partition same, min block
+		minBlockI, _ := strconv.Atoi(namePartsI[1])
+		minBlockJ, _ := strconv.Atoi(namePartsJ[1])
+		return minBlockI < minBlockJ
+	})
+}
+
 func addRequiredPartIfNotExists(parts []metadata.Part, relativePath string, tableDiffFromRemote metadata.TableMetadata, disk clickhouse.Disk) ([]metadata.Part, bool, bool) {
 	isRequiredPartFound := false
 	exists := false

test/integration/run.sh

@@ -99,7 +99,6 @@ set +e
 go test -parallel "${RUN_PARALLEL}" -race -timeout "${TEST_TIMEOUT:-60m}" -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v "${CUR_DIR}/integration_test.go"
 TEST_FAILED=$?
 set -e
-docker buildx prune -f --filter=until=1h  --keep-storage=1G
 
 if [[ "0" == "${TEST_FAILED}" ]]; then
   go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out"
@@ -121,3 +120,5 @@ if [[ "1" == "${CLEAN_AFTER:-0}" || "0" == "${TEST_FAILED}" ]]; then
     fi
   done
 fi
+
+docker buildx prune -f --filter=until=1h  --keep-storage=1G