add support for localfs provider

Makefile

@@ -72,6 +72,10 @@ start-meilisearch:
 start-redis:
 	$(MAKE)	start	DB=redis
 
+.PHONY: start-localfs
+start-localfs:
+	$(MAKE)	start	DB=localfs	
+
 .PHONY: start
 start: kind-cluster-create
 	kind --name backup-restore-sidecar load docker-image ghcr.io/metal-stack/backup-restore-sidecar:latest

README.md

@@ -16,6 +16,7 @@ Probably, it does not make sense to use this project with large databases. Howev
 | meilisearch | >= 1.2.0     | alpha  |       ✅        |
 | redis       | >= 6.0       | alpha  |       ❌        |
 | keydb       | >= 6.0       | alpha  |       ❌        |
+| localfs     |              | alpha  |       ❌        |
 
 Postgres also supports updates when using the TimescaleDB extension. Please consider the integration test for supported upgrade paths.
 

cmd/internal/database/localfs/localfs.go

@@ -0,0 +1,79 @@
+package localfs
+
+import (
+	"context"
+	"fmt"
+	"log/slog"
+	"os"
+
+	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/utils"
+	"github.com/metal-stack/backup-restore-sidecar/pkg/constants"
+)
+
+type LocalFS struct {
+	datadir string
+	log     *slog.Logger
+}
+
+func New(log *slog.Logger, datadir string) *LocalFS {
+	return &LocalFS{
+		datadir: datadir,
+		log:     log,
+	}
+}
+
+// Check if Datadir is empty
+func (l *LocalFS) Check(ctx context.Context) (bool, error) {
+	empty, err := utils.IsEmpty(l.datadir)
+	if err != nil {
+		return false, err
+	}
+	if empty {
+		l.log.Info("data directory is empty")
+		return true, err
+	}
+
+	return false, nil
+}
+
+// put Datadir into constants.BackupDir directory
+func (l *LocalFS) Backup(ctx context.Context) error {
+	if err := os.RemoveAll(constants.BackupDir); err != nil {
+		return fmt.Errorf("could not clean backup directory: %w", err)
+	}
+
+	if err := os.MkdirAll(constants.BackupDir, 0777); err != nil {
+		return fmt.Errorf("could not create backup directory: %w", err)
+	}
+
+	if err := utils.CopyFS(constants.BackupDir, os.DirFS(l.datadir)); err != nil {
+		return fmt.Errorf("could not copy contents: %w", err)
+	}
+
+	l.log.Debug("Sucessfully took backup of localfs")
+	return nil
+}
+
+// get data from constants.RestoreDir
+func (l *LocalFS) Recover(ctx context.Context) error {
+	if err := utils.RemoveContents(l.datadir); err != nil {
+		return fmt.Errorf("Could not cleanup Datadir: %w", err)
+	}
+
+	if err := utils.CopyFS(l.datadir, os.DirFS(constants.RestoreDir)); err != nil {
+		return fmt.Errorf("could not copy contents: %w", err)
+	}
+
+	l.log.Debug("Successfully restored localfs")
+	return nil
+}
+
+func (l *LocalFS) Probe(ctx context.Context) error {
+	//Nothing to do, not a real Database
+	return nil
+}
+
+func (_ *LocalFS) Upgrade(ctx context.Context) error {
+	// Nothing to do here
+	return nil
+}

cmd/internal/utils/files.go

@@ -2,6 +2,7 @@ package utils
 
 import (
 	"errors"
+	"fmt"
 	"io"
 	"io/fs"
 	"os"
@@ -80,3 +81,37 @@ func IsCommandPresent(command string) bool {
 
 	return true
 }
+
+// TODO: replace once go-1.23 is released
+func CopyFS(dir string, fsys fs.FS) error {
+	return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, error error) error {
+		targ := filepath.Join(dir, filepath.FromSlash(path))
+		if d.IsDir() {
+			if err := os.MkdirAll(targ, 0777); err != nil {
+				return err
+			}
+			return nil
+		}
+		r, err := fsys.Open(path)
+		if err != nil {
+			return err
+		}
+		defer r.Close()
+		info, err := r.Stat()
+		if err != nil {
+			return err
+		}
+		w, err := os.OpenFile(targ, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0666|info.Mode()&0777)
+		if err != nil {
+			return err
+		}
+		if _, err := io.Copy(w, r); err != nil {
+			w.Close()
+			return fmt.Errorf("copying %s: %w", path, err)
+		}
+		if err := w.Close(); err != nil {
+			return err
+		}
+		return nil
+	})
+}

cmd/main.go

@@ -18,6 +18,7 @@ import (
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/compress"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/etcd"
+	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/localfs"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/meilisearch"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/postgres"
 	"github.com/metal-stack/backup-restore-sidecar/cmd/internal/database/redis"
@@ -273,7 +274,7 @@ func init() {
 	rootCmd.AddCommand(startCmd, waitCmd, restoreCmd, createBackupCmd)
 
 	rootCmd.PersistentFlags().StringP(logLevelFlg, "", "info", "sets the application log level")
-	rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd|meilisearch|redis|keydb]")
+	rootCmd.PersistentFlags().StringP(databaseFlg, "", "", "the kind of the database [postgres|rethinkdb|etcd|meilisearch|redis|keydb|localfs]")
 	rootCmd.PersistentFlags().StringP(databaseDatadirFlg, "", "", "the directory where the database stores its data in")
 
 	err := viper.BindPFlags(rootCmd.PersistentFlags())
@@ -459,6 +460,11 @@ func initDatabase() error {
 		if err != nil {
 			return err
 		}
+	case "localfs":
+		db = localfs.New(
+			logger.WithGroup("localfs"),
+			datadir,
+		)
 	default:
 		return fmt.Errorf("unsupported database type: %s", dbString)
 	}

deploy/localfs-local.yaml

@@ -0,0 +1,124 @@
+# THESE EXAMPLES ARE GENERATED!
+# Use them as a template for your deployment, but do not commit manual changes to these files.
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  creationTimestamp: null
+  labels:
+    app: localfs
+  name: localfs
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: localfs
+  serviceName: localfs
+  template:
+    metadata:
+      creationTimestamp: null
+      labels:
+        app: localfs
+    spec:
+      containers:
+      - command:
+        - backup-restore-sidecar
+        - wait
+        image: alpine:3.19
+        name: localfs
+        resources: {}
+        volumeMounts:
+        - mountPath: /data
+          name: data
+        - mountPath: /usr/local/bin/backup-restore-sidecar
+          name: bin-provision
+          subPath: backup-restore-sidecar
+        - mountPath: /etc/backup-restore-sidecar
+          name: backup-restore-sidecar-config
+      - command:
+        - backup-restore-sidecar
+        - start
+        - --log-level=debug
+        image: alpine:3.19
+        name: backup-restore-sidecar
+        ports:
+        - containerPort: 8000
+          name: grpc
+        resources: {}
+        volumeMounts:
+        - mountPath: /backup
+          name: backup
+        - mountPath: /data
+          name: data
+        - mountPath: /etc/backup-restore-sidecar
+          name: backup-restore-sidecar-config
+        - mountPath: /usr/local/bin/backup-restore-sidecar
+          name: bin-provision
+          subPath: backup-restore-sidecar
+      initContainers:
+      - command:
+        - cp
+        - /backup-restore-sidecar
+        - /bin-provision
+        image: ghcr.io/metal-stack/backup-restore-sidecar:latest
+        imagePullPolicy: IfNotPresent
+        name: backup-restore-sidecar-provider
+        resources: {}
+        volumeMounts:
+        - mountPath: /bin-provision
+          name: bin-provision
+      volumes:
+      - name: data
+        persistentVolumeClaim:
+          claimName: data
+      - name: backup
+        persistentVolumeClaim:
+          claimName: backup
+      - configMap:
+          name: backup-restore-sidecar-config-localfs
+        name: backup-restore-sidecar-config
+      - emptyDir: {}
+        name: bin-provision
+  updateStrategy: {}
+  volumeClaimTemplates:
+  - metadata:
+      creationTimestamp: null
+      name: data
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Gi
+    status: {}
+  - metadata:
+      creationTimestamp: null
+      name: backup
+    spec:
+      accessModes:
+      - ReadWriteOnce
+      resources:
+        requests:
+          storage: 1Gi
+    status: {}
+status:
+  availableReplicas: 0
+  replicas: 0
+---
+apiVersion: v1
+data:
+  config.yaml: |
+    ---
+    bind-addr: 0.0.0.0
+    db: localfs
+    db-data-directory: /data/
+    backup-provider: local
+    backup-cron-schedule: "*/1 * * * *"
+    object-prefix: localfs-test
+    redis-addr: localhost:6379
+    post-exec-cmds:
+    - tail -f /etc/hosts
+kind: ConfigMap
+metadata:
+  creationTimestamp: null
+  name: backup-restore-sidecar-config-localfs

go.mod

@@ -97,6 +97,7 @@ require (
 	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/patternmatcher v0.6.0 // indirect
+	github.com/moby/spdystream v0.2.0 // indirect
 	github.com/moby/sys/sequential v0.5.0 // indirect
 	github.com/moby/sys/user v0.1.0 // indirect
 	github.com/moby/term v0.5.0 // indirect

go.sum

@@ -26,6 +26,8 @@ github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu
 github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
 github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M=
 github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
+github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/avast/retry-go/v4 v4.5.1 h1:AxIx0HGi4VZ3I02jr78j5lZ3M6x1E0Ivxa6b0pUUh7o=
 github.com/avast/retry-go/v4 v4.5.1/go.mod h1:/sipNsvNB3RRuT5iNcb6h73nw3IBmXJ/H3XrCQYSOpc=
 github.com/aws/aws-sdk-go v1.45.7 h1:k4QsvWZhm8409TYeRuTV1P6+j3lLKoe+giFA/j3VAps=
@@ -179,6 +181,7 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfF
 github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
 github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
 github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 h1:UH//fgunKIs4JdUbpDl1VZCDaL56wXCB/5+wF6uHfaI=
 github.com/grpc-ecosystem/go-grpc-middleware v1.4.0/go.mod h1:g5qyo/la0ALbONm6Vbp88Yd8NsDy6rZz+RcrMPxvld8=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
@@ -246,6 +249,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/patternmatcher v0.6.0 h1:GmP9lR19aU5GqSSFko+5pRqHi+Ohk1O69aFiKkVGiPk=
 github.com/moby/patternmatcher v0.6.0/go.mod h1:hDPoyOpDY7OrrMDLaYoY3hf52gNCR/YOUYxkhApJIxc=
+github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8=
+github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c=
 github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
 github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
 github.com/moby/sys/user v0.1.0 h1:WmZ93f5Ux6het5iituh9x2zAG7NFY9Aqi49jjE1PaQg=

integration/localfs_test.go

@@ -0,0 +1,36 @@
+//go:build integration
+
+package integration_test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/metal-stack/backup-restore-sidecar/pkg/generate/examples/examples"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Localfs_Restore(t *testing.T) {
+	restoreFlow(t, &flowSpec{
+		databaseType:     examples.Localfs,
+		sts:              examples.LocalfsSts,
+		backingResources: examples.LocalfsBackingResources,
+		addTestData:      addLocalfsTestData,
+		verifyTestData:   verifyLocalfsTestData,
+	})
+}
+
+func addLocalfsTestData(t *testing.T, ctx context.Context) {
+	namespace := namespaceName(t)
+	_, err := execCommand(ctx, "localfs-0", namespace, "backup-restore-sidecar", []string{"sh", "-c", "echo 'I am precious' > /data/test.txt"})
+	require.NoError(t, err)
+}
+
+func verifyLocalfsTestData(t *testing.T, ctx context.Context) {
+	namespace := namespaceName(t)
+	resp, err := execCommand(ctx, "localfs-0", namespace, "backup-restore-sidecar", []string{"cat", "/data/test.txt"})
+	require.NoError(t, err)
+
+	assert.Equal(t, "I am precious", resp)
+}