From d276b85f61679c907db87bedb0b266233ef46692 Mon Sep 17 00:00:00 2001 From: Gerrit91 Date: Mon, 29 Jan 2024 16:08:59 +0100 Subject: [PATCH 1/4] Allow upgrading Postgres with timescaleDB extension. --- cmd/internal/database/postgres/postgres.go | 2 + cmd/internal/database/postgres/upgrade.go | 33 ++++++++++ cmd/internal/utils/cmd.go | 30 ++++++++- .../postgres_timescaledb_upgrade_test.go | 65 +++++++++++++++++++ 4 files changed, 128 insertions(+), 2 deletions(-) create mode 100644 integration/postgres_timescaledb_upgrade_test.go diff --git a/cmd/internal/database/postgres/postgres.go b/cmd/internal/database/postgres/postgres.go index 2fd6a2d..0fd4f47 100644 --- a/cmd/internal/database/postgres/postgres.go +++ b/cmd/internal/database/postgres/postgres.go @@ -11,6 +11,8 @@ import ( "github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils" "github.com/metal-stack/backup-restore-sidecar/pkg/constants" "go.uber.org/zap" + + _ "github.com/lib/pq" ) const ( diff --git a/cmd/internal/database/postgres/upgrade.go b/cmd/internal/database/postgres/upgrade.go index d1c5dc2..cba8582 100644 --- a/cmd/internal/database/postgres/upgrade.go +++ b/cmd/internal/database/postgres/upgrade.go @@ -171,6 +171,23 @@ func (db *Postgres) Upgrade(ctx context.Context) error { "--new-bindir", newPostgresBinDir, "--link", } + + runsTimescaleDB, err := db.runningTimescaleDB(ctx, postgresConfigCmd) + if err != nil { + return err + } + + if runsTimescaleDB { + db.log.Infow("running timescaledb, applying custom options for upgrade command") + + // timescaledb libraries in this container are only compatible with the current postgres version + // do not load them anymore with the old postgresql server + pgUpgradeArgs = append(pgUpgradeArgs, + "--old-options", "-c shared_preload_libraries=''", + "--new-options", "-c timescaledb.restoring=on -c shared_preload_libraries=timescaledb", + ) + } + cmd = exec.CommandContext(ctx, postgresUpgradeCmd, pgUpgradeArgs...) //nolint:gosec cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr @@ -255,6 +272,22 @@ func (db *Postgres) getBinDir(ctx context.Context, pgConfigCmd string) (string, return strings.TrimSpace(string(out)), nil } +func (db *Postgres) runningTimescaleDB(ctx context.Context, pgConfigCmd string) (bool, error) { + cmd := exec.CommandContext(ctx, pgConfigCmd, "--pkglibdir") + out, err := cmd.CombinedOutput() + if err != nil { + return false, err + } + + libDir := strings.TrimSpace(string(out)) + + if _, err := os.Stat(path.Join(libDir, "timescaledb.so")); err == nil { + return true, nil + } + + return false, nil +} + // copyPostgresBinaries is needed to save old postgres binaries for a later major upgrade func (db *Postgres) copyPostgresBinaries(ctx context.Context, override bool) error { binDir, err := db.getBinDir(ctx, postgresConfigCmd) diff --git a/cmd/internal/utils/cmd.go b/cmd/internal/utils/cmd.go index 37a1ed5..fd482ce 100644 --- a/cmd/internal/utils/cmd.go +++ b/cmd/internal/utils/cmd.go @@ -6,6 +6,7 @@ import ( "os" "os/exec" "strings" + "time" "go.uber.org/zap" ) @@ -52,7 +53,7 @@ func (c *CmdExecutor) ExecWithStreamingOutput(ctx context.Context, command strin parts := strings.Fields(command) - cmd := exec.CommandContext(ctx, parts[0], parts[1:]...) // nolint:gosec + cmd := exec.Command(parts[0], parts[1:]...) // nolint:gosec c.log.Debugw("running command", "command", cmd.Path, "args", cmd.Args) @@ -61,5 +62,30 @@ func (c *CmdExecutor) ExecWithStreamingOutput(ctx context.Context, command strin cmd.Stdout = os.Stdout cmd.Stderr = os.Stdout - return cmd.Run() + err := cmd.Start() + if err != nil { + return err + } + + go func() { + <-ctx.Done() + + go func() { + time.Sleep(10 * time.Second) + + c.log.Infow("force killing post-exec command now") + if err := cmd.Process.Signal(os.Kill); err != nil { + panic(err) + } + }() + + c.log.Infow("sending sigint to post-exec command process") + + err := cmd.Process.Signal(os.Interrupt) + if err != nil { + c.log.Errorw("unable to send interrupt to post-exec command", "error", err) + } + }() + + return cmd.Wait() } diff --git a/integration/postgres_timescaledb_upgrade_test.go b/integration/postgres_timescaledb_upgrade_test.go new file mode 100644 index 0000000..06b740b --- /dev/null +++ b/integration/postgres_timescaledb_upgrade_test.go @@ -0,0 +1,65 @@ +//go:build integration + +package integration_test + +import ( + "testing" + + "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + _ "github.com/lib/pq" +) + +func Test_Postgres_TimescaleDB_Upgrade(t *testing.T) { + backingResources := examples.PostgresBackingResources(namespaceName(t)) + + modified := false + + for _, r := range backingResources { + cm, ok := r.(*corev1.ConfigMap) + if !ok { + continue + } + + if cm.Name != "backup-restore-sidecar-config-postgres" { + continue + } + + cm.Data = map[string]string{ + "config.yaml": `--- +bind-addr: 0.0.0.0 +db: postgres +db-data-directory: /data/postgres/ +backup-provider: local +backup-cron-schedule: "*/1 * * * *" +object-prefix: postgres-test +compression-method: tar +post-exec-cmds: +- docker-entrypoint.sh postgres -c shared_preload_libraries=timescaledb +`} + + modified = true + break + } + + require.True(t, modified) + + upgradeFlow(t, &upgradeFlowSpec{ + flowSpec: flowSpec{ + databaseType: examples.Postgres, + sts: examples.PostgresSts, + backingResources: func(namespace string) []client.Object { + return backingResources + }, + addTestData: addPostgresTestData, + verifyTestData: verifyPostgresTestData, + }, + databaseImages: []string{ + "timescale/timescaledb:2.11.2-pg12", + "timescale/timescaledb:2.11.2-pg15", + }, + }) +} From 5b6b0649df5c1bb5910f513cf847ce09dee243fd Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 6 Feb 2024 10:27:45 +0100 Subject: [PATCH 2/4] Progress. --- README.md | 16 +++---- cmd/internal/database/postgres/postgres.go | 27 +++++++++++- cmd/internal/database/postgres/upgrade.go | 16 +++++-- .../postgres_timescaledb_upgrade_test.go | 42 ++++++++++++++++++- 4 files changed, 87 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 06a19b3..cdc6007 100644 --- a/README.md +++ b/README.md @@ -9,13 +9,15 @@ Probably, it does not make sense to use this project with large databases. Howev ## Supported Databases | Database | Image | Status | Upgrade Support | -|-------------|--------------|:------:|:---------------:| -| postgres | >= 12-alpine | beta | ✅ | -| rethinkdb | >= 2.4.0 | beta | ❌ | -| ETCD | >= 3.5 | alpha | ❌ | -| meilisearch | >= 1.2.0 | alpha | ✅ | -| redis | >= 6.0 | alpha | ❌ | -| keydb | >= 6.0 | alpha | ❌ | +| ----------- | ------------ | :----: | :-------------: | +| postgres | >= 12-alpine | beta | ✅ | +| rethinkdb | >= 2.4.0 | beta | ❌ | +| ETCD | >= 3.5 | alpha | ❌ | +| meilisearch | >= 1.2.0 | alpha | ✅ | +| redis | >= 6.0 | alpha | ❌ | +| keydb | >= 6.0 | alpha | ❌ | + +Postgres also supports updates when using the TimescaleDB extension. Please consider the integration test for supported upgrade paths. ## Database Upgrades diff --git a/cmd/internal/database/postgres/postgres.go b/cmd/internal/database/postgres/postgres.go index 0fd4f47..3aba5f0 100644 --- a/cmd/internal/database/postgres/postgres.go +++ b/cmd/internal/database/postgres/postgres.go @@ -155,7 +155,7 @@ func (db *Postgres) Recover(ctx context.Context) error { func (db *Postgres) Probe(ctx context.Context) error { // TODO is postgres db OK ? connString := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=postgres sslmode=disable", db.host, db.port, db.user, db.password) - var err error + dbc, err := sql.Open("postgres", connString) if err != nil { return fmt.Errorf("unable to open postgres connection %w", err) @@ -166,5 +166,30 @@ func (db *Postgres) Probe(ctx context.Context) error { if err != nil { return fmt.Errorf("unable to ping postgres connection %w", err) } + + runsTimescaleDB, err := db.runningTimescaleDB(ctx, postgresConfigCmd) + if err == nil && runsTimescaleDB { + db.log.Infow("detected running timescaledb, running post-start hook to update timescaledb extension if necessary") + + _, err = dbc.ExecContext(ctx, "ALTER EXTENSION timescaledb UPDATE;") + if err != nil { + return fmt.Errorf("unable to alter extension: %w", err) + } + + // we also need to upgrade the extension in the template1 database because there it is also installed + + connString := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=template1 sslmode=disable", db.host, db.port, db.user, db.password) + dbcTemplate1, err := sql.Open("postgres", connString) + if err != nil { + return fmt.Errorf("unable to open postgres connection %w", err) + } + defer dbcTemplate1.Close() + + _, err = dbcTemplate1.ExecContext(ctx, "ALTER EXTENSION timescaledb UPDATE;") + if err != nil { + return fmt.Errorf("unable to alter extension for template database: %w", err) + } + } + return nil } diff --git a/cmd/internal/database/postgres/upgrade.go b/cmd/internal/database/postgres/upgrade.go index cba8582..4fe2012 100644 --- a/cmd/internal/database/postgres/upgrade.go +++ b/cmd/internal/database/postgres/upgrade.go @@ -178,6 +178,7 @@ func (db *Postgres) Upgrade(ctx context.Context) error { } if runsTimescaleDB { + // see https://github.com/timescale/timescaledb/issues/1844 and https://github.com/timescale/timescaledb/issues/4503#issuecomment-1860883843 db.log.Infow("running timescaledb, applying custom options for upgrade command") // timescaledb libraries in this container are only compatible with the current postgres version @@ -273,14 +274,11 @@ func (db *Postgres) getBinDir(ctx context.Context, pgConfigCmd string) (string, } func (db *Postgres) runningTimescaleDB(ctx context.Context, pgConfigCmd string) (bool, error) { - cmd := exec.CommandContext(ctx, pgConfigCmd, "--pkglibdir") - out, err := cmd.CombinedOutput() + libDir, err := db.getLibDir(ctx, pgConfigCmd) if err != nil { return false, err } - libDir := strings.TrimSpace(string(out)) - if _, err := os.Stat(path.Join(libDir, "timescaledb.so")); err == nil { return true, nil } @@ -288,6 +286,16 @@ func (db *Postgres) runningTimescaleDB(ctx context.Context, pgConfigCmd string) return false, nil } +func (db *Postgres) getLibDir(ctx context.Context, pgConfigCmd string) (string, error) { + cmd := exec.CommandContext(ctx, pgConfigCmd, "--pkglibdir") + out, err := cmd.CombinedOutput() + if err != nil { + return "", fmt.Errorf("unable to figure out lib dir: %w", err) + } + + return strings.TrimSpace(string(out)), nil +} + // copyPostgresBinaries is needed to save old postgres binaries for a later major upgrade func (db *Postgres) copyPostgresBinaries(ctx context.Context, override bool) error { binDir, err := db.getBinDir(ctx, postgresConfigCmd) diff --git a/integration/postgres_timescaledb_upgrade_test.go b/integration/postgres_timescaledb_upgrade_test.go index 06b740b..a4e997e 100644 --- a/integration/postgres_timescaledb_upgrade_test.go +++ b/integration/postgres_timescaledb_upgrade_test.go @@ -3,6 +3,7 @@ package integration_test import ( + "context" "testing" "github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples" @@ -38,7 +39,7 @@ backup-cron-schedule: "*/1 * * * *" object-prefix: postgres-test compression-method: tar post-exec-cmds: -- docker-entrypoint.sh postgres -c shared_preload_libraries=timescaledb +- docker-entrypoint.sh postgres -c shared_preload_libraries=timescaledb `} modified = true @@ -54,12 +55,49 @@ post-exec-cmds: backingResources: func(namespace string) []client.Object { return backingResources }, - addTestData: addPostgresTestData, + addTestData: addTimescaleDbTestData, verifyTestData: verifyPostgresTestData, }, databaseImages: []string{ "timescale/timescaledb:2.11.2-pg12", "timescale/timescaledb:2.11.2-pg15", + "timescale/timescaledb:2.12.2-pg15", + "timescale/timescaledb:2.13.1-pg15", + "timescale/timescaledb:2.13.1-pg16", }, }) } + +func addTimescaleDbTestData(t *testing.T, ctx context.Context) { + db := newPostgresSession(t, ctx) + defer db.Close() + + var ( + createStmt = ` + + CREATE EXTENSION IF NOT EXISTS timescaledb; + + CREATE TABLE IF NOT EXISTS backuprestore ( + timestamp timestamp NOT NULL, + data text NOT NULL, + PRIMARY KEY(timestamp, data) + ); + SELECT create_hypertable('backuprestore', 'timestamp', chunk_time_interval => INTERVAL '1 days', if_not_exists => TRUE); + + ALTER TABLE backuprestore SET ( + timescaledb.compress, + timescaledb.compress_segmentby = 'data', + timescaledb.compress_orderby = 'timestamp' + ); + SELECT add_compression_policy('backuprestore', INTERVAL '1 days'); + + ` + insertStmt = `INSERT INTO backuprestore("timestamp", "data") VALUES ('2024-01-01 12:00:00.000', 'I am precious');` + ) + + _, err := db.Exec(createStmt) + require.NoError(t, err) + + _, err = db.Exec(insertStmt) + require.NoError(t, err) +} From 3cd686e128038210acb603916076bee5b42052b9 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 6 Feb 2024 13:07:40 +0100 Subject: [PATCH 3/4] Review. --- cmd/internal/database/postgres/postgres.go | 70 ++++++++++++++++--- .../postgres_timescaledb_upgrade_test.go | 3 +- 2 files changed, 64 insertions(+), 9 deletions(-) diff --git a/cmd/internal/database/postgres/postgres.go b/cmd/internal/database/postgres/postgres.go index 3aba5f0..cc7daf4 100644 --- a/cmd/internal/database/postgres/postgres.go +++ b/cmd/internal/database/postgres/postgres.go @@ -171,23 +171,77 @@ func (db *Postgres) Probe(ctx context.Context) error { if err == nil && runsTimescaleDB { db.log.Infow("detected running timescaledb, running post-start hook to update timescaledb extension if necessary") - _, err = dbc.ExecContext(ctx, "ALTER EXTENSION timescaledb UPDATE;") + err = db.updateTimescaleDB(ctx, dbc) if err != nil { - return fmt.Errorf("unable to alter extension: %w", err) + return fmt.Errorf("unable to update timescaledb: %w", err) } + } + + return nil +} + +func (db *Postgres) updateTimescaleDB(ctx context.Context, dbc *sql.DB) error { + var ( + databaseNames []string + ) - // we also need to upgrade the extension in the template1 database because there it is also installed + databaseNameRows, err := dbc.QueryContext(ctx, "SELECT datname,datallowconn FROM pg_database") + if err != nil { + return fmt.Errorf("unable to get database names: %w", err) + } + defer databaseNameRows.Close() + + for databaseNameRows.Next() { + var name string + var allowed bool + if err := databaseNameRows.Scan(&name, &allowed); err != nil { + return err + } + + if allowed { + databaseNames = append(databaseNames, name) + } + } + if err := databaseNameRows.Err(); err != nil { + return err + } - connString := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=template1 sslmode=disable", db.host, db.port, db.user, db.password) - dbcTemplate1, err := sql.Open("postgres", connString) + for _, dbName := range databaseNames { + connString := fmt.Sprintf("host=%s port=%d user=%s password=%s dbname=%s sslmode=disable", db.host, db.port, db.user, db.password, dbName) + dbc2, err := sql.Open("postgres", connString) if err != nil { return fmt.Errorf("unable to open postgres connection %w", err) } - defer dbcTemplate1.Close() + defer dbc2.Close() - _, err = dbcTemplate1.ExecContext(ctx, "ALTER EXTENSION timescaledb UPDATE;") + rows, err := dbc2.QueryContext(ctx, "SELECT extname FROM pg_extension") if err != nil { - return fmt.Errorf("unable to alter extension for template database: %w", err) + return fmt.Errorf("unable to get extensions: %w", err) + } + defer rows.Close() + + for rows.Next() { + var extName string + if err := rows.Scan(&extName); err != nil { + return err + } + + if extName != "timescaledb" { + continue + } + + db.log.Infow("updating timescaledb extension", "db-name", dbName) + + _, err = dbc2.ExecContext(ctx, "ALTER EXTENSION timescaledb UPDATE") + if err != nil { + return fmt.Errorf("unable to update extension: %w", err) + } + + break + } + + if err := rows.Err(); err != nil { + return err } } diff --git a/integration/postgres_timescaledb_upgrade_test.go b/integration/postgres_timescaledb_upgrade_test.go index a4e997e..4976184 100644 --- a/integration/postgres_timescaledb_upgrade_test.go +++ b/integration/postgres_timescaledb_upgrade_test.go @@ -61,7 +61,8 @@ post-exec-cmds: databaseImages: []string{ "timescale/timescaledb:2.11.2-pg12", "timescale/timescaledb:2.11.2-pg15", - "timescale/timescaledb:2.12.2-pg15", + // it is allowed to skip a minor version + // "timescale/timescaledb:2.12.2-pg15", "timescale/timescaledb:2.13.1-pg15", "timescale/timescaledb:2.13.1-pg16", }, From 7b4ee55feb2a4709447eb2457cc97eebd427df03 Mon Sep 17 00:00:00 2001 From: Gerrit91 Date: Tue, 6 Feb 2024 13:26:29 +0100 Subject: [PATCH 4/4] Chown. --- cmd/internal/database/postgres/upgrade.go | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/cmd/internal/database/postgres/upgrade.go b/cmd/internal/database/postgres/upgrade.go index 4fe2012..7e34558 100644 --- a/cmd/internal/database/postgres/upgrade.go +++ b/cmd/internal/database/postgres/upgrade.go @@ -114,15 +114,24 @@ func (db *Postgres) Upgrade(ctx context.Context) error { if err != nil { return err } + gid, err := strconv.Atoi(pgUser.Gid) + if err != nil { + return err + } // remove /data/postgres-new if present - newDataDirTemp := path.Join("/data", "postgres-new") + newDataDirTemp := path.Join("/data", "postgres-new") // TODO: /data should not be hardcoded err = os.RemoveAll(newDataDirTemp) if err != nil { db.log.Errorw("unable to remove new datadir, skipping upgrade", "error", err) return nil } + err = os.Chown("/data", uid, gid) + if err != nil { + return err + } + // initdb -D /data/postgres-new cmd := exec.Command(postgresInitDBCmd, "-D", newDataDirTemp) cmd.Stdout = os.Stdout