From 0cdb888d4093cd6faef9ea05cb64c2510ea0c414 Mon Sep 17 00:00:00 2001 From: Stefan Majer Date: Tue, 22 Aug 2023 16:05:54 +0200 Subject: [PATCH] Support Postgres Upgrades (#48) --- README.md | 30 +- api/v1/initializer.pb.go | 34 +- cmd/internal/database/contract.go | 14 + cmd/internal/database/etcd/etcd.go | 7 +- cmd/internal/database/postgres/postgres.go | 4 +- cmd/internal/database/postgres/upgrade.go | 318 +++++++++ cmd/internal/database/rethinkdb/rethinkdb.go | 7 +- cmd/internal/initializer/initializer.go | 9 +- docs/sequence.drawio | 1 - docs/sequence.drawio.svg | 657 +++++++++++++++++++ docs/sequence.png | 3 - go.mod | 1 + go.sum | 2 + proto/Makefile | 2 +- proto/buf.gen.yaml | 2 +- proto/v1/initializer.proto | 1 + 16 files changed, 1056 insertions(+), 36 deletions(-) create mode 100644 cmd/internal/database/postgres/upgrade.go delete mode 100644 docs/sequence.drawio create mode 100644 docs/sequence.drawio.svg delete mode 100644 docs/sequence.png diff --git a/README.md b/README.md index 3e4880f..4783438 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,30 @@ Probably, it does not make sense to use this project with large databases. Howev ## Supported Databases -| Database | Image | Status | -| --------- | --------- | ------ | -| postgres | 12-alpine | alpha | -| rethinkdb | >= 2.4.0 | alpha | -| ETCD | >= 3.5 | alpha | +| Database | Image | Status | Upgrade Support | +| --------- | ------------ | :----: | :-------------: | +| postgres | >= 12-alpine | beta | ✅ | +| rethinkdb | >= 2.4.0 | beta | ❌ | +| ETCD | >= 3.5 | alpha | ❌ | + +## Database Upgrades + +### Postgres + +Postgres requires special treatment if a major version upgrade is planned. `pg_upgrade` needs to be called with the old and new binaries, also the old data directory and a already initialized data directory which was initialized with the new binary, e.g. `initdb `. + +To make this process as smooth as possible, backup-restore-sidecar will detect if the version of the database files and the version of the postgres binary. If the binary is newer than the database files it will start the upgrade process. Strict validation to ensure all prerequisites are met is done before actually starting the upgrade process. + +To achieve this, `backup-restore-sidecar` saves the postgres binaries in the database directory in the form of `pg-bin-v12` for postgres 12. If later the database version is upgraded, the previous postgres binaries are present for doing the actual upgrade. ## Supported Compression Methods With `--compression-method` you can define how generated backups are compressed before stored at the storage provider. Available compression methods are: -| compression-method | suffix | comments | -| ------------------ | -------- | -------- | -| tar | .tar | no compression, best suited for already compressed content | -| targz | .tar.gz | tar and gzip, most commonly used, best compression ratio, average performance | +| compression-method | suffix | comments | +|--------------------|----------|----------------------------------------------------------------------------------------------| +| tar | .tar | no compression, best suited for already compressed content | +| targz | .tar.gz | tar and gzip, most commonly used, best compression ratio, average performance | | tarlz4 | .tar.lz4 | tar and lz4, very fast compression/decompression speed compared to gz, slightly bigger files | ## Supported Storage Providers @@ -32,7 +42,7 @@ With `--compression-method` you can define how generated backups are compressed ## How it works -![Sequence Diagram](docs/sequence.png) +![Sequence Diagram](docs/sequence.drawio.svg) ## Limitations diff --git a/api/v1/initializer.pb.go b/api/v1/initializer.pb.go index 41e1fbc..3adafa6 100644 --- a/api/v1/initializer.pb.go +++ b/api/v1/initializer.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.28.1 +// protoc-gen-go v1.31.0 // protoc (unknown) // source: v1/initializer.proto @@ -26,6 +26,7 @@ const ( StatusResponse_CHECKING StatusResponse_InitializerStatus = 0 StatusResponse_RESTORING StatusResponse_InitializerStatus = 1 StatusResponse_DONE StatusResponse_InitializerStatus = 2 + StatusResponse_UPGRADING StatusResponse_InitializerStatus = 3 ) // Enum value maps for StatusResponse_InitializerStatus. @@ -34,11 +35,13 @@ var ( 0: "CHECKING", 1: "RESTORING", 2: "DONE", + 3: "UPGRADING", } StatusResponse_InitializerStatus_value = map[string]int32{ "CHECKING": 0, "RESTORING": 1, "DONE": 2, + "UPGRADING": 3, } ) @@ -167,28 +170,29 @@ var File_v1_initializer_proto protoreflect.FileDescriptor var file_v1_initializer_proto_rawDesc = []byte{ 0x0a, 0x14, 0x76, 0x31, 0x2f, 0x69, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x02, 0x76, 0x31, 0x22, 0x07, 0x0a, 0x05, 0x45, 0x6d, - 0x70, 0x74, 0x79, 0x22, 0xa4, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, + 0x70, 0x74, 0x79, 0x22, 0xb3, 0x01, 0x0a, 0x0e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0e, 0x32, 0x24, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2e, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x18, - 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x3a, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x6d, 0x65, 0x73, 0x73, 0x61, 0x67, 0x65, 0x22, 0x49, 0x0a, 0x11, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x0c, 0x0a, 0x08, 0x43, 0x48, 0x45, 0x43, 0x4b, 0x49, 0x4e, 0x47, 0x10, 0x00, 0x12, 0x0d, 0x0a, 0x09, 0x52, 0x45, 0x53, 0x54, 0x4f, 0x52, 0x49, 0x4e, 0x47, 0x10, 0x01, - 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x32, 0x3d, 0x0a, 0x12, 0x49, 0x6e, - 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, - 0x12, 0x27, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x09, 0x2e, 0x76, 0x31, 0x2e, - 0x45, 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, - 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x6c, 0x0a, 0x06, 0x63, 0x6f, 0x6d, - 0x2e, 0x76, 0x31, 0x42, 0x10, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, - 0x50, 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, - 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, - 0x64, 0x72, 0x6f, 0x70, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, - 0x31, 0xa2, 0x02, 0x03, 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, - 0x31, 0xe2, 0x02, 0x0e, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, - 0x74, 0x61, 0xea, 0x02, 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x12, 0x08, 0x0a, 0x04, 0x44, 0x4f, 0x4e, 0x45, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x55, 0x50, + 0x47, 0x52, 0x41, 0x44, 0x49, 0x4e, 0x47, 0x10, 0x03, 0x32, 0x3d, 0x0a, 0x12, 0x49, 0x6e, 0x69, + 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x53, 0x65, 0x72, 0x76, 0x69, 0x63, 0x65, 0x12, + 0x27, 0x0a, 0x06, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x09, 0x2e, 0x76, 0x31, 0x2e, 0x45, + 0x6d, 0x70, 0x74, 0x79, 0x1a, 0x12, 0x2e, 0x76, 0x31, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x6c, 0x0a, 0x06, 0x63, 0x6f, 0x6d, 0x2e, + 0x76, 0x31, 0x42, 0x10, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x65, 0x72, 0x50, + 0x72, 0x6f, 0x74, 0x6f, 0x50, 0x01, 0x5a, 0x28, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, + 0x6f, 0x6d, 0x2f, 0x6d, 0x65, 0x74, 0x61, 0x6c, 0x2d, 0x73, 0x74, 0x61, 0x63, 0x6b, 0x2f, 0x64, + 0x72, 0x6f, 0x70, 0x74, 0x61, 0x69, 0x6c, 0x65, 0x72, 0x2f, 0x61, 0x70, 0x69, 0x2f, 0x76, 0x31, + 0xa2, 0x02, 0x03, 0x56, 0x58, 0x58, 0xaa, 0x02, 0x02, 0x56, 0x31, 0xca, 0x02, 0x02, 0x56, 0x31, + 0xe2, 0x02, 0x0e, 0x56, 0x31, 0x5c, 0x47, 0x50, 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, + 0x61, 0xea, 0x02, 0x02, 0x56, 0x31, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/cmd/internal/database/contract.go b/cmd/internal/database/contract.go index a420030..71426d8 100644 --- a/cmd/internal/database/contract.go +++ b/cmd/internal/database/contract.go @@ -1,12 +1,26 @@ package database type DatabaseInitializer interface { + // Check indicates whether a restore of the database is required or not. Check() (bool, error) + + // Recover performs a restore of the database. Recover() error + + // Upgrade performs an upgrade of the database in case a newer version of the database is detected. + // + // The function aborts the update without returning an error as long as the old data stays unmodified and only logs the error. + // This behavior is intended to reduce unnecessary downtime caused by misconfigurations. + // + // Once the upgrade was made, any error condition will require to recover the database from backup. + Upgrade() error } type DatabaseProber interface { + // Probe figures out if the database is running and available for taking backups. Probe() error + + // Backup creates a backup of the database. Backup() error } diff --git a/cmd/internal/database/etcd/etcd.go b/cmd/internal/database/etcd/etcd.go index 86a579e..491cac1 100644 --- a/cmd/internal/database/etcd/etcd.go +++ b/cmd/internal/database/etcd/etcd.go @@ -127,7 +127,7 @@ func (db *Etcd) Recover() error { return nil } -// Probe indicates whether the database is running +// Probe figures out if the database is running and available for taking backups. func (db *Etcd) Probe() error { out, err := db.etcdctl(true, "get", "foo") if err != nil { @@ -136,6 +136,11 @@ func (db *Etcd) Probe() error { return nil } +// Upgrade performs an upgrade of the database in case a newer version of the database is detected. +func (db *Etcd) Upgrade() error { + return nil +} + func (db *Etcd) etcdctl(withConnectionArgs bool, args ...string) (string, error) { var ( etcdctlEnvs []string diff --git a/cmd/internal/database/postgres/postgres.go b/cmd/internal/database/postgres/postgres.go index 343379f..7f94889 100644 --- a/cmd/internal/database/postgres/postgres.go +++ b/cmd/internal/database/postgres/postgres.go @@ -137,12 +137,12 @@ func (db *Postgres) Recover() error { db.log.Debugw("restored postgres pg_wal backup", "output", out) - db.log.Infow("successfully restored postgres database") + db.log.Info("successfully restored postgres database") return nil } -// Probe indicates whether the database is running +// Probe figures out if the database is running and available for taking backups. func (db *Postgres) Probe() error { conn, err := net.DialTimeout("tcp", net.JoinHostPort(db.host, strconv.Itoa(db.port)), connectionTimeout) if err != nil { diff --git a/cmd/internal/database/postgres/upgrade.go b/cmd/internal/database/postgres/upgrade.go new file mode 100644 index 0000000..d567112 --- /dev/null +++ b/cmd/internal/database/postgres/upgrade.go @@ -0,0 +1,318 @@ +package postgres + +import ( + "errors" + "fmt" + "io/fs" + "os" + "os/exec" + "os/user" + "path" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/Masterminds/semver/v3" +) + +const ( + postgresHBAConf = "pg_hba.conf" + postgresqlConf = "postgresql.conf" + postgresConfigCmd = "pg_config" + postgresUpgradeCmd = "pg_upgrade" + postgresInitDBCmd = "initdb" + postgresVersionFile = "PG_VERSION" + postgresBinBackupPrefix = "pg-bin-v" +) + +var ( + requiredCommands = []string{postgresUpgradeCmd, postgresConfigCmd, postgresInitDBCmd} +) + +// Upgrade performs an upgrade of the database in case a newer version of the database is detected. +func (db *Postgres) Upgrade() error { + start := time.Now() + + err := db.copyPostgresBinaries() + if err != nil { + return err + } + + // First check if there are data already present + pgVersionFile := path.Join(db.datadir, postgresVersionFile) + if _, err := os.Stat(pgVersionFile); errors.Is(err, fs.ErrNotExist) { + db.log.Infof("%q is not present, no upgrade required", pgVersionFile) + return nil + } + + // Check if required commands are present + for _, command := range requiredCommands { + if ok := db.isCommandPresent(command); !ok { + db.log.Errorf("%q is not present, skipping upgrade", command) + return nil + } + } + + // Then check the version of the existing database + pgVersion, err := db.getDatabaseVersion(pgVersionFile) + if err != nil { + db.log.Errorw("unable get database version, skipping upgrade", "error", err) + return nil + } + + // Now check the version of the actual postgres binaries + binaryVersionMajor, err := db.getBinaryVersion(postgresConfigCmd) + if err != nil { + db.log.Errorw("unable to get binary version, skipping upgrade", "error", err) + return nil + } + + if pgVersion == binaryVersionMajor { + db.log.Infow("no version difference, no upgrade required", "database-version", pgVersion, "binary-version", binaryVersionMajor) + return nil + } + if pgVersion > binaryVersionMajor { + db.log.Errorw("database is newer than postgres binary, aborting", "database-version", pgVersion, "binary-version", binaryVersionMajor) + return fmt.Errorf("database is newer than postgres binary") + } + + oldPostgresBinDir := path.Join(db.datadir, fmt.Sprintf("%s%d", postgresBinBackupPrefix, pgVersion)) + + // Check if old pg_config are present and match pgVersion + oldPostgresConfigCmd := path.Join(oldPostgresBinDir, postgresConfigCmd) + if ok := db.isCommandPresent(oldPostgresConfigCmd); !ok { + db.log.Infof("%q is not present, skipping upgrade", oldPostgresConfigCmd) + return nil + } + + // We need to upgrade, therefore old binaries are required + oldBinaryVersionMajor, err := db.getBinaryVersion(oldPostgresConfigCmd) + if err != nil { + db.log.Errorw("unable to get old binary version, skipping upgrade", "error", err) + return nil + } + + if oldBinaryVersionMajor != pgVersion { + db.log.Errorw("database version and old binary version do not match, skipping upgrade", "old database", pgVersion, "old binary", oldBinaryVersionMajor) + return nil + } + + // OK we need to upgrade the database in place, maybe taking a backup before is recommended + db.log.Infow("start upgrading from", "old database", pgVersion, "old binary", oldBinaryVersionMajor, "new binary", binaryVersionMajor) + + // run the pg_upgrade command as postgres user + pgUser, err := user.Lookup("postgres") + if err != nil { + return err + } + uid, err := strconv.Atoi(pgUser.Uid) + if err != nil { + return err + } + + // remove /data/postgres-new if present + newDataDirTemp := path.Join("/data", "postgres-new") + err = os.RemoveAll(newDataDirTemp) + if err != nil { + db.log.Errorw("unable to remove new datadir, skipping upgrade", "error", err) + return nil + } + + // initdb -D /data/postgres-new + cmd := exec.Command(postgresInitDBCmd, "-D", newDataDirTemp) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uint32(uid)}, + } + err = cmd.Run() + if err != nil { + db.log.Errorw("unable to run initdb on new new datadir, skipping upgrade", "error", err) + return nil + } + + db.log.Infow("new database directory initialized") + + // restore old pg_hba.conf and postgresql.conf + for _, config := range []string{postgresHBAConf, postgresqlConf} { + db.log.Infow("restore old configuration into new datadir", "config", config) + + cfg, err := os.ReadFile(path.Join(db.datadir, config)) + if err != nil { + return err + } + + err = os.WriteFile(path.Join(newDataDirTemp, config), cfg, 0600) + if err != nil { + return err + } + } + + err = db.restoreOldPostgresBinaries(db.datadir, newDataDirTemp) + if err != nil { + return err + } + + newPostgresBinDir, err := db.getBinDir(postgresConfigCmd) + if err != nil { + return fmt.Errorf("unable to detect bin dir of actual postgres %w", err) + } + + pgUpgradeArgs := []string{ + "--old-datadir", db.datadir, + "--new-datadir", newDataDirTemp, + "--old-bindir", oldPostgresBinDir, + "--new-bindir", newPostgresBinDir, + "--link", + } + cmd = exec.Command(postgresUpgradeCmd, pgUpgradeArgs...) //nolint:gosec + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.SysProcAttr = &syscall.SysProcAttr{ + Credential: &syscall.Credential{Uid: uint32(uid)}, + } + cmd.Dir = pgUser.HomeDir + + db.log.Infow("running pg_upgrade with", "args", pgUpgradeArgs) + err = cmd.Run() + if err != nil { + db.log.Errorw("unable to run pg_upgrade on new new datadir, abort upgrade", "error", err) + return fmt.Errorf("unable to run pg_upgrade %w", err) + } + + db.log.Infow("pg_upgrade done") + + // rm -rf /data/postgres + err = os.RemoveAll(db.datadir) + if err != nil { + return fmt.Errorf("unable to remove old data dir: %w", err) + } + + err = os.Rename(newDataDirTemp, db.datadir) + if err != nil { + return fmt.Errorf("unable to rename upgraded datadir to destination, a full restore is required: %w", err) + } + + db.log.Infow("pg_upgrade done and new data in place", "took", time.Since(start)) + + return nil +} + +// Helpers + +func (db *Postgres) getBinaryVersion(pgConfigCmd string) (int, error) { + // pg_config --version + // PostgreSQL 12.16 + cmd := exec.Command(pgConfigCmd, "--version") + out, err := cmd.CombinedOutput() + if err != nil { + return 0, fmt.Errorf("unable to detect postgres binary version: %w", err) + } + + _, binaryVersionString, found := strings.Cut(string(out), "PostgreSQL ") + if !found { + return 0, fmt.Errorf("unable to detect postgres binary version in pg_config output %q", binaryVersionString) + } + + v, err := semver.NewVersion(strings.TrimSpace(binaryVersionString)) + if err != nil { + return 0, fmt.Errorf("unable to parse postgres binary version in %q: %w", binaryVersionString, err) + } + + return int(v.Major()), nil +} + +func (db *Postgres) getDatabaseVersion(pgVersionFile string) (int, error) { + // cat PG_VERSION + // 12 + pgVersionBytes, err := os.ReadFile(pgVersionFile) + if err != nil { + return 0, fmt.Errorf("unable to read %q: %w", pgVersionFile, err) + } + + pgVersion, err := strconv.Atoi(strings.TrimSpace(string(pgVersionBytes))) + if err != nil { + return 0, fmt.Errorf("unable to convert content of %q (content: %q) to integer: %w", pgVersionFile, string(pgVersionBytes), err) + } + + return pgVersion, nil +} + +func (db *Postgres) isCommandPresent(command string) bool { + p, err := exec.LookPath(command) + if err != nil { + return false + } + + if _, err := os.Stat(p); errors.Is(err, fs.ErrNotExist) { + return false + } + + return true +} + +func (db *Postgres) getBinDir(pgConfigCmd string) (string, error) { + cmd := exec.Command(pgConfigCmd, "--bindir") + out, err := cmd.CombinedOutput() + if err != nil { + return "", err + } + + return strings.TrimSpace(string(out)), nil +} + +func (db *Postgres) copyPostgresBinaries() error { + binDir, err := db.getBinDir(postgresConfigCmd) + if err != nil { + return err + } + + version, err := db.getBinaryVersion(postgresConfigCmd) + if err != nil { + return err + } + + pgBinDir := path.Join(db.datadir, fmt.Sprintf("%s%d", postgresBinBackupPrefix, version)) + + err = os.RemoveAll(pgBinDir) + if err != nil { + return fmt.Errorf("unable to remove old pg bin dir: %w", err) + } + + db.log.Infow("copying postgres binaries for later upgrades", "from", binDir, "to", pgBinDir) + copy := exec.Command("cp", "-av", binDir, pgBinDir) + copy.Stdout = os.Stdout + copy.Stderr = os.Stderr + err = copy.Run() + if err != nil { + return fmt.Errorf("unable to copy pg bin dir: %w", err) + } + + return nil +} + +func (db *Postgres) restoreOldPostgresBinaries(src, dst string) error { + return filepath.WalkDir(src, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + if !strings.HasPrefix(d.Name(), postgresBinBackupPrefix) { + return nil + } + + db.log.Infow("copying postgres binaries from old datadir to new datadir", "from", path, "to", dst) + + copy := exec.Command("cp", "-av", path, dst) + copy.Stdout = os.Stdout + copy.Stderr = os.Stderr + err = copy.Run() + if err != nil { + return fmt.Errorf("unable to copy pg bin dir: %w", err) + } + + return nil + }) +} diff --git a/cmd/internal/database/rethinkdb/rethinkdb.go b/cmd/internal/database/rethinkdb/rethinkdb.go index d9f2c23..d3348ae 100644 --- a/cmd/internal/database/rethinkdb/rethinkdb.go +++ b/cmd/internal/database/rethinkdb/rethinkdb.go @@ -175,7 +175,7 @@ func (db *RethinkDB) Recover() error { return nil } -// Probe indicates whether the database is running +// Probe figures out if the database is running and available for taking backups. func (db *RethinkDB) Probe() error { conn, err := net.DialTimeout("tcp", db.url, connectionTimeout) if err != nil { @@ -184,3 +184,8 @@ func (db *RethinkDB) Probe() error { defer conn.Close() return nil } + +// Upgrade performs an upgrade of the database in case a newer version of the database is detected. +func (db *RethinkDB) Upgrade() error { + return nil +} diff --git a/cmd/internal/initializer/initializer.go b/cmd/internal/initializer/initializer.go index e4333d2..5037b53 100644 --- a/cmd/internal/initializer/initializer.go +++ b/cmd/internal/initializer/initializer.go @@ -88,7 +88,14 @@ func (i *Initializer) Start(stop <-chan struct{}) { err = i.initialize() if err != nil { - i.log.Fatal(fmt.Errorf("error initializing database, shutting down:%w", err)) + i.log.Fatalw("error initializing database, shutting down", "error", err) + } + + i.currentStatus.Status = v1.StatusResponse_UPGRADING + i.currentStatus.Message = "start upgrading database" + err = i.db.Upgrade() + if err != nil { + i.log.Fatalw("upgrade database failed", "error", err) } i.log.Info("initializer done") diff --git a/docs/sequence.drawio b/docs/sequence.drawio deleted file mode 100644 index 7a8b920..0000000 --- a/docs/sequence.drawio +++ /dev/null @@ -1 +0,0 @@ -7V1tk6M2Ev41rrr7MC6ExNvHnZndbHKXq6mbu2TvUwob2VYGIw7wvOTXR8ICIwlssIXtnczUVq0RshDS091Pt1ryBN6tX3/IwnT1M41wPLGt6HUC7ye2DZBtT/g/K3rblvgB2hYsMxKJSruCR/IHFoWWKN2QCOdSxYLSuCCpXDinSYLnhVQWZhl9kastaCw/NQ2XWCt4nIexXvoriYqVKAWWtbvxFZPlSjzad8SNWTh/WmZ0k4jnTWy4KP+2t9dh1Zaon6/CiL40iuDnCbzLKC22n9avdzjmY1sN2/Z7Xzru1v3OcFL0+cLzt5//9Y/119XrT48P6dpf///XL8WNmKvnMN7g6jXKzhZv1QCVr4h5I9YE3r6sSIEf03DO774wSLCyVbGO2RVgH0VzOCvwa2c/Qf32DFWYrnGRvbEqrzIw3uTLl93kuEiUrRrz4nmiMBSAWNYt7waFfRDj0j5G/mo5++3xt09O8gDv/vt090PwI7iBnjZIjwyz8zBjhXc0KUKS4Ewbt/yFrOMwYVe3cTjD8W2Nlzsa06ysVCGmGjhtlFrGsnPgbF8eOaSPXA1GaeTgaCMHtJHLi7DY5FxfUDY06pg1kMTfmzAx/RSTZcLKZrQo6JrdwEn0ics9K6MpZrduozBflQAF29tCyfj9Rn7BZvBLuCYxH7ZfcBaFSSiKRUPA5u1GTG2InuJ4Rl8+7wpuywJ2o+o0L3olxTdWdGNNLeSLgv9xKZoiD4nr+1chVuXFW+PiAWeETQLORBnvTqP3VvnXipu9GOZv0QdLrNMuEMohw3FYkGdZX7ZhRbT2QAnrS93UjQckWDq2KzdRhNkSF+JbO8SxSQ7fGtVSXiHvfg5ASH3OFEEFw9tGOxqwwNSxrfrP1vo9Dfxg9ye/RU432Rxrb7F7SlWRLhY5LiaqbNVzcLy4ATiKNj9JKUELSqMIoa9pJeS0aCUQGNBK3iychWCOZnPbW6CZdWPvUeVMNWV8oubrSFfmqzDlHzfr+J9kgWNS6vW0IaSxKG4I7qHhnde2o7qO4zDNyax8Kp+fDM83Wc5k79843+oiXko3BX/SXc2Iyqr1xPKmarJhle2uyVx87qEQ8yKjTzUZAv1VZChU9ZzBpXx/E/iBgUwHbB0/NVdr4sc1YdU0/FRsZJ+INWZY6CxW7NxOnHsFMjQrVnRJkzBugkaW0KETmdASmWPOYadQ9Z5U5CgUT2cqoGVKndGICgi0aV0QRuxti5k+nBeC7m/SUxnLLKbzp4Oz13Ou+k/N1sTuGYBKMPvyA85qgkqVn0gQHBkNtudMg+B4O1spDm9Qs4NJyCUMvKOhtLJZOc6euQ/CBMSN2UTfztiFu+Sf/lZTbobDsv9/HxnFx7DrhI3Mt5IdB9XllixbW7LMC3Zsubx6a16pfLm/bGwBdUDfNwWos1J/cg1RoJBVy4goBVMfWBAhn6Hdd5lPrOjZQBIB/0jR2v8UqDrihvh9INNxAKz9nVSqI6n6AVfAkT3pG0d5pfFEvKI3DRnfJMzupxnO84YZssJsvuJA0eQ9LO0Wq17QDHNHOywYXvOTne0Lma5e4jlA8hzPlufWMyN6wNovFU5gwqo5stqAqpNkSNZ8WXigu1/W1OrBEFnzZFlTtYcRUdOAo3vKPyaEd2pPPO+du4Bn8BwGe38OksHR2/sL7G75Pd756xFf+cs5f3DonCrOn9PP9xvFm9dt7S4a9BKSv2ow6Bo1AXDkOGJvTeD4YyBHjxd8aAJ/6JweGQaCaJTInr7Kl2b0mekDzrpJEmYEcwL+TEJOONdpwd8kIjpRuLwzPXCpqlM893NvIZB7XWMhJwPCSsitnn5qWAnIGoOpjF6ceyhXdpQVDhD4A9iv7am99OVedjD6kcJO2gQiWxMLQZG5MOSVuVSd0FJg3jotZ0ILfNgkCisHUIskFTRtqKkYL/hTc9YUSZb/4ffub9wLa1Hz9NfrafOgc7qCbF9GHmT2zh5VMKb4+scTgOXLAlwbEsPqq1IrptUXsk9RX3UAolJftndJ9dUKWt2wz1d4/sT1F1lIAbKW6HmCccRrFpTTALyLrOluwcXtvgE4Q4jkuNVNtTpzIpxhgKT4mGL1fM/Imk8gM0rY05gOlxllbcn39/ZLrV/1q2cwWnGYFeU+Yj6JbvtTmm9d0kaEucPGM14QHbbxGS0YrGgiDPEiJukvlZFln79OzpqJAmCLNbUNuBvtsX49meC2Cu8/bH2PjzDkNQQfXPfYJJSKIJnHjp5auc8gvcsIxH6hGjkgaSSu1P4GOsuuV/3YW+VcW57IPY5OnL0QHzez/H4ihUFKZjpShdsQu3Acv/U5fZfT7P297CBTxhaw9S0FTG8kMQ2jSWP52hCQRwieibxtaxrYwaSZtu0AMBmatl3ntfDI1kRKbGFEeGIksaVFRq4uxwsGcGo3k6ddGaSON/Xtk/m/G3hDmv0ucr5sPelLzwG5kDU47zYKA+LYKTxj2w6o5Hsh79RNEJ1C4AHA5FYDff1gZwr93SYH5U1GNg7Q1cB8v4u+XNFeKqiY+patVK2b0EZZpka6n7hbpybJ78z14uGoluiVWKzidiSt5/QjIL9/q5Q7FCxASfWCGlhAmwdhYhmz/Q10vJCEFIRNzh9lknCVEXxt3KsXt7kY/7ctRZUCM0FLqJCWnkFLYzpZj/J9oMU8Whw0CloONGsOO3p+rh6m+D5w09hrAGx1twFyjDplezMogmuCK5BzeG3Ub7mxBaDu/oZGhGSL4as3v5RrfoyLnIrH78FZYji2K+doi2vgVrENA86SmZT8k/WrHCWz1fVxQ9G4ms4pz+m5Pt7x7XM5XLq/9X3o6ONsu30Z0257RvZ2qKb9QLMj00Kgx5342pFtJTwTgudMsP9otvtMkjlNcpIXuHwpvsqkwor5U4XiUZaenLq6QOJYKdJWg1RYrkkUxV0urOx0nimCoKwWQLfFLQzOutysJy3X1nG7mP/OzWPnrJ7bViFodi9yP8uFBDPrabl8t/XbZwsV6psz7rt2Mn7kRJw/JwLJ8DjvuRztiPnYlHFAlI5NiUAt8e+2iKaRM3taX6ElBC6OPhiVuQ5Md9gtHcsrVYFB5+uUrc6VQJy81Vlu1/G8KfJO5r9qs67VL5d48EFhdqB2f3+/lPpVv3pu8FDOIThb9ibS43NpRmdjnwNg9PAPxzF0+EefyMWZvEeobHh3VcLfX160hqaVCJ3JSayO3Gvm1sdk6wCmtDyZpnQI3rFTUSl9MLV8V1b7rjtY7fc/5nGwOTB2oCP0FX0Ix/FfNHyL5/Q/zdF3LS3ToWoMsQk6wWadQX/rAZgifCp3o5b5P9xt54yVRhyJsb7Z7v3p9UYykBze9v1gsKgdL1Iny4+cC+S6zhS4Q1Ny2gCvNOue3RzoPnvj6KJyC1XLVlHD2Wwjn1M0nIarZ33VzozhnDLXs49DDgzkQJHXEzbDt01ZrR3u3jalHg0k1T9gAFB7CtcZFLceh9ikSv7z1alqo1nPnVJzOEd54IFeZ8xR9kAwRo7ygWavLUe59VcEdKbSyOp8oJGG9+tYCJq0R8XM/liBoSUedrn7qYjtvO1+jwN+/hM= \ No newline at end of file diff --git a/docs/sequence.drawio.svg b/docs/sequence.drawio.svg new file mode 100644 index 0000000..6c39927 --- /dev/null +++ b/docs/sequence.drawio.svg @@ -0,0 +1,657 @@ + + + + + + + + + + + Sidecar Container + + + + + + + + +
+
+
+ Sidecar start cmd +
+
+
+ + + Sidecar start cmd + + + + + + + + +
+
+
+ find latest backup +
+
+
+ + + find latest backup + + + + + + + + +
+
+
+ start server +
+ (status endpoint) +
+
+
+ + + start server... + + + + + + + + +
+
+
+ uncompress backup archive +
+ and restore database +
+
+
+ + + uncompress backup archive... + + + + + + + + + +
+
+
+ Init Container +
+
+
+ + + Init Container + + + + + + + + + +
+
+
+ Sidecar wait cmd +
+
+
+ + + Sidecar wait cmd + + + + + + + + + +
+
+
+ provide binaries via empty dir +
+
+
+ + + provide binaries via empty dir + + + + + + + + +
+
+
+ Contains sidecar +
+ binary +
+
+
+ + + Contains sidecar... + + + + + + + + +
+
+
+ checks if database +
+ needs to be restored +
+
+
+ + + checks if database... + + + + + + + +
+
+
+ possible restore +
+
+
+ + + possible restore + + + + + + + + +
+
+
+ Backup Provider +
+
+
+ + + Backup Provider + + + + + + + + + +
+
+
+ backup version +
+
+
+ + + backup version + + + + + + + + +
+
+
+ download backup version +
+
+
+ + + download backup version + + + + + + + + +
+
+
+ backup archive +
+
+
+ + + backup archive + + + + + + + + + + Database Container + + + + + + + +
+
+
+ Sidecar injected +
+ via entrypoint +
+
+
+ + + Sidecar injected... + + + + + + + + +
+
+
+ status done +
+
+
+ + + status done + + + + + + + + +
+
+
+ probe +
+
+
+ + + probe + + + + + + + + +
+
+
+ client port open +
+
+
+ + + client port open + + + + + + + + +
+
+
+ take backups periodically +
+
+
+ + + take backups periodically + + + + + + + + +
+
+
+ compress to +
+ backup archive +
+
+
+ + + compress to... + + + + + + + + +
+
+
+ upload backup +
+
+
+ + + upload backup + + + + + + + + + +
+
+
+ initializer status +
+
+
+ + + initializer status + + + + + + + + +
+
+
+ initializer status +
+
+
+ + + initializer status + + + + + + + + +
+
+
+ initializer status +
+
+
+ + + initializer status + + + + + + + + +
+
+
+ status checking +
+
+
+ + + status checking + + + + + + + + +
+
+
+ initializer status +
+
+
+ + + initializer status + + + + + + +
+
+
+ [ no data or data inconsistent ] +
+
+
+ + + [ no data or data inconsistent ] + + + + + + + + +
+
+
+ status restoring +
+
+
+ + + status restoring + + + + + + + + +
+
+
+ Database +
+
+
+ + + Database + + + + + + +
+
+
+ Database Pod +
+
+
+ + + Database Pod + + + + + + + + +
+
+
+ possible upgrade +
+
+
+ + + possible upgrade + + + + + + + + +
+
+
+ upgrade database +
+
+
+ + + upgrade database + + + + + + + + +
+
+
+ [ database image updated  ] +
+
+
+ + + [ database image updated  ] + + + + + + + + +
+
+
+ start +
+
+
+ + + start + + + + + + + + + Text is not SVG - cannot display + + + + diff --git a/docs/sequence.png b/docs/sequence.png deleted file mode 100644 index 61b177a..0000000 --- a/docs/sequence.png +++ /dev/null @@ -1,3 +0,0 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:e12e2a393beee105556fac72b054762e32ee90b36ab9fe63f69d4ed30eca9d0a -size 91035 diff --git a/go.mod b/go.mod index 2a6c70e..e20447d 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( cloud.google.com/go/storage v1.32.0 + github.com/Masterminds/semver/v3 v3.2.1 github.com/aws/aws-sdk-go v1.44.324 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/metal-stack/v v1.0.3 diff --git a/go.sum b/go.sum index 4de89f9..b179e57 100644 --- a/go.sum +++ b/go.sum @@ -48,6 +48,8 @@ cloud.google.com/go/storage v1.32.0/go.mod h1:Hhh/dogNRGca7IWv1RC2YqEn0c0G77ctA/ dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= +github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0= +github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= diff --git a/proto/Makefile b/proto/Makefile index fce7ea7..8c71e32 100644 --- a/proto/Makefile +++ b/proto/Makefile @@ -1,5 +1,5 @@ MAKEFLAGS += --no-print-directory -BUF_VERSION := 1.15.0 +BUF_VERSION := 1.26.1 _buf: docker run --rm \ diff --git a/proto/buf.gen.yaml b/proto/buf.gen.yaml index 6377dc2..4c47b49 100644 --- a/proto/buf.gen.yaml +++ b/proto/buf.gen.yaml @@ -5,7 +5,7 @@ managed: default: github.com/metal-stack/droptailer/api plugins: # generate go structs for protocol buffer definition - - plugin: buf.build/protocolbuffers/go:v1.28.1 + - plugin: buf.build/protocolbuffers/go:v1.31.0 out: ../api opt: paths=source_relative - plugin: buf.build/grpc/go:v1.3.0 diff --git a/proto/v1/initializer.proto b/proto/v1/initializer.proto index 899f9e6..1f1be67 100644 --- a/proto/v1/initializer.proto +++ b/proto/v1/initializer.proto @@ -15,6 +15,7 @@ message StatusResponse { CHECKING = 0; RESTORING = 1; DONE = 2; + UPGRADING = 3; } InitializerStatus status = 1; string message = 2;