diff --git a/ChangeLog.md b/ChangeLog.md index 45fdc132..c5832a46 100644 --- a/ChangeLog.md +++ b/ChangeLog.md @@ -1,3 +1,10 @@ +# v2.6.4 + +BUG FIXES + +- fix possible wrong merging after ATTACH PART for Collapsing and Replacing engines without version, + look https://github.com/ClickHouse/ClickHouse/issues/71009 for details + # v2.6.3 IMPROVEMENTS - implement new format for *.state2 files boltdb key value (please, check memory RSS usage) diff --git a/pkg/clickhouse/clickhouse.go b/pkg/clickhouse/clickhouse.go index ff820757..63f12eb3 100644 --- a/pkg/clickhouse/clickhouse.go +++ b/pkg/clickhouse/clickhouse.go @@ -7,6 +7,7 @@ import ( "database/sql" "errors" "fmt" + "github.com/Altinity/clickhouse-backup/v2/pkg/filesystemhelper" "os" "path" "path/filepath" @@ -770,6 +771,8 @@ func (ch *ClickHouse) AttachDataParts(table metadata.TableMetadata, dstTable Tab return nil } for disk := range table.Parts { + // https://github.com/ClickHouse/ClickHouse/issues/71009 + filesystemhelper.SortPartsByMinBlock(table.Parts[disk]) for _, part := range table.Parts[disk] { if !strings.HasSuffix(part.Name, ".proj") { query := fmt.Sprintf("ALTER TABLE `%s`.`%s` ATTACH PART '%s'", table.Database, table.Table, part.Name) diff --git a/pkg/filesystemhelper/filesystemhelper.go b/pkg/filesystemhelper/filesystemhelper.go index 2284e0fd..43b8a0e1 100644 --- a/pkg/filesystemhelper/filesystemhelper.go +++ b/pkg/filesystemhelper/filesystemhelper.go @@ -7,6 +7,8 @@ import ( "os" "path" "path/filepath" + "sort" + "strconv" "strings" "sync" "syscall" @@ -297,9 +299,27 @@ func MoveShadowToBackup(shadowPath, backupPartsPath string, partitionsBackupMap return os.Link(filePath, dstFilePath) } }) + // https://github.com/ClickHouse/ClickHouse/issues/71009 + SortPartsByMinBlock(parts) return parts, size, err } +// SortPartsByMinBlock need to avoid wrong restore for Replacing, Collapsing, https://github.com/ClickHouse/ClickHouse/issues/71009 +func SortPartsByMinBlock(parts []metadata.Part) { + sort.Slice(parts, func(i, j int) bool { + namePartsI := strings.Split(parts[i].Name, "_") + namePartsJ := strings.Split(parts[j].Name, "_") + // partitions different + if namePartsI[0] != namePartsJ[0] { + return namePartsI[0] < namePartsJ[0] + } + // partition same, min block + minBlockI, _ := strconv.Atoi(namePartsI[1]) + minBlockJ, _ := strconv.Atoi(namePartsJ[1]) + return minBlockI < minBlockJ + }) +} + func addRequiredPartIfNotExists(parts []metadata.Part, relativePath string, tableDiffFromRemote metadata.TableMetadata, disk clickhouse.Disk) ([]metadata.Part, bool, bool) { isRequiredPartFound := false exists := false diff --git a/test/integration/run.sh b/test/integration/run.sh index c341a7ae..82adeaee 100755 --- a/test/integration/run.sh +++ b/test/integration/run.sh @@ -99,7 +99,6 @@ set +e go test -parallel "${RUN_PARALLEL}" -race -timeout "${TEST_TIMEOUT:-60m}" -failfast -tags=integration -run "${RUN_TESTS:-.+}" -v "${CUR_DIR}/integration_test.go" TEST_FAILED=$? set -e -docker buildx prune -f --filter=until=1h --keep-storage=1G if [[ "0" == "${TEST_FAILED}" ]]; then go tool covdata textfmt -i "${CUR_DIR}/_coverage_/" -o "${CUR_DIR}/_coverage_/coverage.out" @@ -121,3 +120,5 @@ if [[ "1" == "${CLEAN_AFTER:-0}" || "0" == "${TEST_FAILED}" ]]; then fi done fi + +docker buildx prune -f --filter=until=1h --keep-storage=1G