First pass!!

.github/workflows/docker.yaml

@@ -57,3 +57,25 @@ jobs:
         context: .
         push: true
         tags: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ env.tag }}
+
+  integration:
+    name: Integration Test
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+
+    - name: Set up Go 1.21
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21.x'
+
+    - name: Create k8s Kind Cluster
+      uses: helm/[email protected]
+      with:
+        install_only: true
+
+    - name: Test
+      run: |
+        make test

Makefile

@@ -23,7 +23,7 @@ all:
 	strip bin/backup-restore-sidecar
 
 .PHONY: test-integration
-test-integration:
+test-integration: dockerimage
 	kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest
 	KUBECONFIG=$(KUBECONFIG) go test -v -p 1 -timeout 10m ./integration/...
 
@@ -71,6 +71,7 @@ kind-cluster-create: dockerimage
 	@if ! kind get clusters | grep backup-restore-sidecar > /dev/null; then \
 		kind create cluster \
 		--name backup-restore-sidecar \
+		--config kind.yaml \
 		--kubeconfig $(KUBECONFIG); fi
 
 .PHONY: cleanup

cmd/internal/database/rethinkdb/rethinkdb.go

@@ -3,7 +3,6 @@ package rethinkdb
 import (
 	"context"
 	"fmt"
-	"net"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -17,12 +16,14 @@ import (
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils"
 	"github.com/metal-stack/backup-restore-sidecar/pkg/constants"
 	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+
+	r "gopkg.in/rethinkdb/rethinkdb-go.v6"
 )
 
 const (
-	connectionTimeout              = 1 * time.Second
-	restoreDatabaseStartupTimeout  = 30 * time.Second
-	restoreDatabaseShutdownTimeout = 10 * time.Second
+	connectionTimeout             = 1 * time.Second
+	restoreDatabaseStartupTimeout = 30 * time.Second
 
 	rethinkDBCmd        = "rethinkdb"
 	rethinkDBDumpCmd    = "rethinkdb-dump"
@@ -112,44 +113,45 @@ func (db *RethinkDB) Recover() error {
 		return fmt.Errorf("restore file not present: %s", rethinkDBRestoreFilePath)
 	}
 
+	passwordRaw, err := os.ReadFile(db.passwordFile)
+	if err != nil {
+		return fmt.Errorf("unable to read rethinkdb password file at %s: %w", db.passwordFile, err)
+	}
+
 	// rethinkdb requires to be running when restoring a backup.
 	// however, if we let the real database container start, we cannot interrupt it anymore in case
 	// an issue occurs during the restoration. therefore, we spin up an own instance of rethinkdb
 	// inside the sidecar against which we can restore.
 
-	db.log.Infow("starting rethinkdb database within sidecar for restore")
-	//nolint
-	cmd := exec.Command(rethinkDBCmd, "--bind", "all", "--driver-port", "1", "--directory", db.datadir)
-	if err := cmd.Start(); err != nil {
-		return fmt.Errorf("unable to start database within sidecar for restore: %w", err)
-	}
-	defer func() {
-		_ = cmd.Process.Kill()
-	}()
+	var (
+		cmd  *exec.Cmd
+		g, _ = errgroup.WithContext(context.Background())
+	)
 
-	db.log.Infow("waiting for rethinkdb database to come up")
+	g.Go(func() error {
+		args := []string{"--bind", "all", "--driver-port", "1", "--directory", db.datadir, "--initial-password", strings.TrimSpace(string(passwordRaw))}
+		db.log.Infow("execute rethinkdb", "args", args)
 
-	restoreDB := New(db.ctx, db.log, db.datadir, "localhost:1", "")
+		cmd = exec.Command(rethinkDBCmd, args...) // nolint:gosec
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		err := cmd.Run()
+		if err != nil {
+			return fmt.Errorf("unable to run rethinkdb: %w", err)
+		}
+
+		db.log.Info("rethinkdb finished")
 
-	done := make(chan bool)
-	defer close(done)
+		return nil
+	})
 
+	db.log.Infow("waiting for rethinkdb database to come up")
 	probeCtx, probeCancel := context.WithTimeout(context.Background(), restoreDatabaseStartupTimeout)
 	defer probeCancel()
-
-	var err error
-	go func() {
-		err = probe.Start(probeCtx, restoreDB.log, restoreDB)
-		done <- true
-	}()
-	select {
-	case <-done:
-		if err != nil {
-			return fmt.Errorf("error while probing: %w", err)
-		}
-		db.log.Infow("rethinkdb in sidecar is now available, now triggering restore commands...")
-	case <-probeCtx.Done():
-		return errors.New("rethinkdb database did not come up in time")
+	restoreDB := New(db.ctx, db.log, db.datadir, "localhost:1", db.passwordFile)
+	err = probe.Start(probeCtx, restoreDB.log, restoreDB)
+	if err != nil {
+		return fmt.Errorf("rethinkdb did not come up: %w", err)
 	}
 
 	args := []string{}
@@ -166,32 +168,44 @@ func (db *RethinkDB) Recover() error {
 		return fmt.Errorf("error running restore command: %s %w", out, err)
 	}
 
+	db.log.Infow("successfully restored rethinkdb database", "output", out)
+
 	if err := cmd.Process.Signal(syscall.SIGTERM); err != nil {
 		return fmt.Errorf("failed to send sigterm signal to rethinkdb: %w", err)
 	}
 
-	wait := make(chan error)
-	go func() { wait <- cmd.Wait() }()
-	select {
-	case err := <-wait:
-		if err != nil {
-			return fmt.Errorf("rethinkdb did not shutdown cleanly: %w", err)
-		}
-		db.log.Infow("successfully restored rethinkdb database", "output", out)
-	case <-time.After(restoreDatabaseShutdownTimeout):
-		return fmt.Errorf("rethinkdb did not shutdown cleanly after %s", restoreDatabaseShutdownTimeout)
+	err = g.Wait()
+	if err != nil {
+		// sending a TERM signal will always result in a error response.
+		db.log.Errorw("importing dump terminated but reported an error which can be ignored", "error", err)
 	}
 
 	return nil
 }
 
 // Probe figures out if the database is running and available for taking backups.
 func (db *RethinkDB) Probe() error {
-	conn, err := net.DialTimeout("tcp", db.url, connectionTimeout)
+	passwordRaw, err := os.ReadFile(db.passwordFile)
+	if err != nil {
+		return fmt.Errorf("unable to read rethinkdb password file at %s: %w", db.passwordFile, err)
+	}
+
+	session, err := r.Connect(r.ConnectOpts{
+		Addresses: []string{db.url},
+		Username:  "admin",
+		Password:  strings.TrimSpace(string(passwordRaw)),
+		MaxIdle:   10,
+		MaxOpen:   20,
+	})
+	if err != nil {
+		return fmt.Errorf("cannot create rethinkdb client: %w", err)
+	}
+
+	_, err = r.DB("rethinkdb").Table("server_status").Run(session)
 	if err != nil {
-		return fmt.Errorf("connection error: %w", err)
+		return fmt.Errorf("error retrieving rethinkdb server status: %w", err)
 	}
-	defer conn.Close()
+
 	return nil
 }
 

deploy/rethinkdb-test.yaml

@@ -3,6 +3,7 @@
 data:
   config.yaml: |
     ---
+    bind-addr: 0.0.0.0
     db: rethinkdb
     db-data-directory: /data/rethinkdb/
     backup-provider: local
@@ -116,6 +117,7 @@ spec:
         - mountPath: /usr/local/bin/rethinkdb-restore
           name: bin-provision
           subPath: rethinkdb-restore
+      hostNetwork: true
       initContainers:
       - command:
         - cp

go.mod

@@ -18,13 +18,13 @@ require (
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	go.uber.org/zap v1.25.0
+	golang.org/x/sync v0.3.0
 	google.golang.org/api v0.137.0
 	google.golang.org/grpc v1.57.0
 	google.golang.org/protobuf v1.31.0
 	gopkg.in/rethinkdb/rethinkdb-go.v6 v6.2.2
 	k8s.io/api v0.28.1
 	k8s.io/apimachinery v0.28.1
-	k8s.io/cli-runtime v0.28.1
 	k8s.io/client-go v0.28.1
 	sigs.k8s.io/controller-runtime v0.16.0
 	sigs.k8s.io/yaml v1.3.0
@@ -41,10 +41,8 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect
 	github.com/emicklei/go-restful/v3 v3.10.2 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.6.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
-	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-logr/logr v1.2.4 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
@@ -53,16 +51,13 @@ require (
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/golang/snappy v0.0.4 // indirect
-	github.com/google/btree v1.1.2 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/s2a-go v0.1.5 // indirect
-	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/google/uuid v1.3.0 // indirect
 	github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
 	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
-	github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/imdario/mergo v0.3.16 // indirect
@@ -72,21 +67,17 @@ require (
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/klauspost/compress v1.16.7 // indirect
 	github.com/klauspost/pgzip v1.2.6 // indirect
-	github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de // indirect
 	github.com/magiconair/properties v1.8.7 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
-	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nwaples/rardecode v1.1.3 // indirect
 	github.com/opentracing/opentracing-go v1.1.0 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.9 // indirect
-	github.com/peterbourgon/diskv v2.0.1+incompatible // indirect
 	github.com/pierrec/lz4/v4 v4.1.18 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -102,14 +93,11 @@ require (
 	github.com/subosito/gotenv v1.6.0 // indirect
 	github.com/ulikunitz/xz v0.5.11 // indirect
 	github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
-	github.com/xlab/treeprint v1.2.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.starlark.net v0.0.0-20230525235612-a134d8f9ddca // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
 	golang.org/x/net v0.14.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
-	golang.org/x/sync v0.3.0 // indirect
 	golang.org/x/sys v0.11.0 // indirect
 	golang.org/x/term v0.11.0 // indirect
 	golang.org/x/text v0.12.0 // indirect
@@ -128,7 +116,5 @@ require (
 	k8s.io/kube-openapi v0.0.0-20230717233707-2695361300d9 // indirect
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/kustomize/api v0.13.5-0.20230601165947-6ce0bf390ce3 // indirect
-	sigs.k8s.io/kustomize/kyaml v0.14.3-0.20230601165947-6ce0bf390ce3 // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 )