From 6f3bb47adb474c1f96096a585358b948fd8d5e19 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 16:01:28 +1000 Subject: [PATCH 01/12] Removed hard coded restore from DB, print DB conneciton info for dump/restore as it's not working in a real env --- api/ws/wsHelpers/sync-mongo-dump.go | 6 ++++++ api/ws/wsHelpers/sync-mongo-restore.go | 11 ++++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index f0c6399b..1adf1fd8 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -34,6 +34,12 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + passSeg := "" + if len(auth.Password) > 5 { + passSeg = auth.Password[0:5] + } + fmt.Printf("MongoDump connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + toolOptions = &options.ToolOptions{ SSL: &ssl, Connection: connection, diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 355a425f..43eb85bc 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -30,6 +30,12 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + passSeg := "" + if len(auth.Password) > 5 { + passSeg = auth.Password[0:5] + } + fmt.Printf("MongoRestore connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + toolOptions = &options.ToolOptions{ SSL: &ssl, Connection: connection, @@ -56,9 +62,8 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta nsOptions := &mongorestore.NSOptions{ NSInclude: []string{"*"}, - //NSInclude: []string{"pixlise-prodv4-15-jul-2024.ownership"}, - NSFrom: []string{"pixlise-prodv4-15-jul-2024"}, - NSTo: []string{restoreToDBName}, + NSFrom: []string{restoreFromDBName}, + NSTo: []string{restoreToDBName}, } //log.SetVerbosity(toolOptions.Verbosity) From b6764320b4d41aab21ce2a07156dc437483f0bb5 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 16:28:05 +1000 Subject: [PATCH 02/12] Bump build --- api/ws/handlers/system.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index 92eaaa70..49f9d5f6 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -15,13 +15,13 @@ import ( func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (*protos.BackupDBResp, error) { if len(hctx.Svcs.Config.DataBackupBucket) <= 0 { - err := "PIXLISE Backup bucket not configured!" + err := "PIXLISE Backup bucket not configured" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } if !hctx.Svcs.Config.BackupEnabled { - err := "PIXLISE Backup not enabled!" + err := "PIXLISE Backup not enabled" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } @@ -126,7 +126,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A func HandleRestoreDBReq(req *protos.RestoreDBReq, hctx wsHelpers.HandlerContext) (*protos.RestoreDBResp, error) { // Only allow restore if enabled and we're NOT prod if !hctx.Svcs.Config.RestoreEnabled { - err := "PIXLISE Restore not enabled!" + err := "PIXLISE Restore not enabled" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } From a398f0de1f9ea8289ab9ddcce793a4bc727ebe9d Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 16:38:24 +1000 Subject: [PATCH 03/12] Better logging of mongo dump/restore connectivity --- api/ws/handlers/system.go | 4 ++-- api/ws/wsHelpers/sync-mongo-dump.go | 5 +++-- api/ws/wsHelpers/sync-mongo-restore.go | 5 +++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index 49f9d5f6..7c1d6c18 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -37,7 +37,7 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) ( hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket) // Run MongoDump, save to a local archive file - dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) + dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) err = dump.Init() if err != nil { @@ -171,7 +171,7 @@ func runRestore(startTimestamp int64, svcs *services.APIServices, downloadRemote } if errDBRestore == nil { - restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName) + restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName) if errDBRestore == nil { result := restore.Restore() diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index 1adf1fd8..ccaa396f 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -10,13 +10,14 @@ import ( "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" ) var dataBackupLocalPath = "./backup" var dataBackupS3Path = "DB" -func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, dbName string) *mongodump.MongoDump { +func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump { var toolOptions *options.ToolOptions ssl := options.SSL{} @@ -38,7 +39,7 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails if len(auth.Password) > 5 { passSeg = auth.Password[0:5] } - fmt.Printf("MongoDump connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + logger.Infof("MongoDump connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) toolOptions = &options.ToolOptions{ SSL: &ssl, diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 43eb85bc..9affdcb0 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -9,10 +9,11 @@ import ( "github.com/mongodb/mongo-tools/mongorestore" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" ) -func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { +func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { var toolOptions *options.ToolOptions ssl := options.SSL{} @@ -34,7 +35,7 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta if len(auth.Password) > 5 { passSeg = auth.Password[0:5] } - fmt.Printf("MongoRestore connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + logger.Infof("MongoRestore connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) toolOptions = &options.ToolOptions{ SSL: &ssl, From c4d40b63e5319716aa8762e720e2843d0e1f1d34 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 17:00:04 +1000 Subject: [PATCH 04/12] Mongo dump/restore more verbose logging --- api/ws/wsHelpers/sync-mongo-dump.go | 3 ++- api/ws/wsHelpers/sync-mongo-restore.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index ccaa396f..efde9dea 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -6,6 +6,7 @@ import ( "path" "strings" + "github.com/mongodb/mongo-tools/common/log" "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" @@ -60,7 +61,7 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails } inputOptions := &mongodump.InputOptions{} - //log.SetVerbosity(toolOptions.Verbosity) + log.SetVerbosity(nil /*toolOptions.Verbosity*/) return &mongodump.MongoDump{ ToolOptions: toolOptions, diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 9affdcb0..48dd8926 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -5,6 +5,7 @@ import ( "path" "strings" + "github.com/mongodb/mongo-tools/common/log" "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongorestore" "github.com/pixlise/core/v4/api/services" @@ -67,7 +68,7 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta NSTo: []string{restoreToDBName}, } - //log.SetVerbosity(toolOptions.Verbosity) + log.SetVerbosity(nil /*toolOptions.Verbosity*/) return mongorestore.New(mongorestore.Options{ ToolOptions: toolOptions, From 0d88406ff980d7fb05a9f258efd9fc88e65c09af Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 17:10:31 +1000 Subject: [PATCH 05/12] Mongo dump/restore more verbose logging --- api/ws/wsHelpers/sync-mongo-dump.go | 2 ++ api/ws/wsHelpers/sync-mongo-restore.go | 11 +++++++++++ 2 files changed, 13 insertions(+) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index efde9dea..2882003d 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -62,6 +62,8 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails inputOptions := &mongodump.InputOptions{} log.SetVerbosity(nil /*toolOptions.Verbosity*/) + lw := LogWriter{logger: logger} + log.SetWriter(lw) return &mongodump.MongoDump{ ToolOptions: toolOptions, diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 48dd8926..29734679 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -69,6 +69,8 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta } log.SetVerbosity(nil /*toolOptions.Verbosity*/) + lw := LogWriter{logger: logger} + log.SetWriter(lw) return mongorestore.New(mongorestore.Options{ ToolOptions: toolOptions, @@ -79,6 +81,15 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta }) } +type LogWriter struct { + logger logger.ILogger +} + +func (w LogWriter) Write(p []byte) (n int, err error) { + w.logger.Infof(string(p)) + return len(p), nil +} + func DownloadArchive(svcs *services.APIServices) (string, error) { svcs.Log.Infof("Downloading PIXLISE DB Dump files...") From a6460aea9c8d39d6ee5d6d521eca1a00e90d76ba Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 19:34:55 +1000 Subject: [PATCH 06/12] Mongo dump/restore document DB connection fixes --- api/ws/wsHelpers/sync-mongo-dump.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index 2882003d..a1a29192 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -36,18 +36,28 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") + passSeg := "" if len(auth.Password) > 5 { passSeg = auth.Password[0:5] } logger.Infof("MongoDump connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + uri, err := options.NewURI(connectionURI) + if err != nil { + logger.Errorf("%v", err) + return nil + } + toolOptions = &options.ToolOptions{ SSL: &ssl, Connection: connection, Auth: &auth, Verbosity: &options.Verbosity{}, - URI: &options.URI{}, + URI: uri, /*&options.URI{ + ConnectionString: connectionURI, + },*/ } toolOptions.Namespace = &options.Namespace{DB: dbName} From 5407ff7cdc267f535c5d51a2148b56995ed95d01 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 19:47:42 +1000 Subject: [PATCH 07/12] Fix test output --- internal/cmd-line-tools/api-integration-test/testImage.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/cmd-line-tools/api-integration-test/testImage.go b/internal/cmd-line-tools/api-integration-test/testImage.go index fe44744f..c0634446 100644 --- a/internal/cmd-line-tools/api-integration-test/testImage.go +++ b/internal/cmd-line-tools/api-integration-test/testImage.go @@ -132,7 +132,7 @@ func doHTTPRequest(scheme string, method string, apiHost string, urlPath string, bodyReader = io.NopCloser(reqBody) } - fmt.Printf("Sending HTTP %v request: %v...", method, wsConnectUrl.String()) + fmt.Printf("Sending HTTP %v request: %v...\n", method, wsConnectUrl.String()) client := &http.Client{} req, err := http.NewRequest(method, wsConnectUrl.String(), bodyReader) From d9eb0a2ba3886a2c5ea0700e210813bc225c87f7 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 20:14:10 +1000 Subject: [PATCH 08/12] AWS documentdb connection fixes for mongodump/restore --- api/ws/wsHelpers/sync-mongo-dump.go | 31 +++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index a1a29192..a851cf83 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -21,10 +21,20 @@ var dataBackupS3Path = "DB" func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump { var toolOptions *options.ToolOptions - ssl := options.SSL{} + ssl := options.SSL{ + UseSSL: true, + SSLCAFile: "./global-bundle.pem", + SSLPEMKeyFile: "./global-bundle.pem", + //SSLAllowInvalidCert: true, + //SSLAllowInvalidHost: true, + // SSLFipsMode bool `long:"sslFIPSMode" description:"use FIPS mode of the installed openssl library"` + //TLSInsecure: true, + } + auth := options.Auth{ - Username: mongoDetails.User, - Password: mongoDetails.Password, + Username: mongoDetails.User, + Password: mongoDetails.Password, + Mechanism: "DEFAULT", } connection := &options.Connection{ @@ -50,14 +60,15 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails return nil } + retryWrites := false + toolOptions = &options.ToolOptions{ - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: uri, /*&options.URI{ - ConnectionString: connectionURI, - },*/ + RetryWrites: &retryWrites, + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: uri, } toolOptions.Namespace = &options.Namespace{DB: dbName} From 483865dfb09f6cc35833c8f2f85374433caf1cc9 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 20:21:07 +1000 Subject: [PATCH 09/12] AWS documentdb connection fixes for mongodump/restore --- api/ws/wsHelpers/sync-mongo-dump.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index a851cf83..c671d5b5 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -32,9 +32,9 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails } auth := options.Auth{ - Username: mongoDetails.User, - Password: mongoDetails.Password, - Mechanism: "DEFAULT", + Username: mongoDetails.User, + Password: mongoDetails.Password, + //Mechanism: "DEFAULT", } connection := &options.Connection{ From e955f453d22c8bb39c73e614432f5c41fe3161c1 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 20:32:33 +1000 Subject: [PATCH 10/12] Finally mongo dump works on dev/documentDB, setting same settings for restore --- api/ws/wsHelpers/sync-mongo-dump.go | 12 +----------- api/ws/wsHelpers/sync-mongo-restore.go | 27 ++++++++++++++------------ 2 files changed, 16 insertions(+), 23 deletions(-) diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index c671d5b5..89139cbf 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -25,21 +25,15 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails UseSSL: true, SSLCAFile: "./global-bundle.pem", SSLPEMKeyFile: "./global-bundle.pem", - //SSLAllowInvalidCert: true, - //SSLAllowInvalidHost: true, - // SSLFipsMode bool `long:"sslFIPSMode" description:"use FIPS mode of the installed openssl library"` - //TLSInsecure: true, } auth := options.Auth{ Username: mongoDetails.User, Password: mongoDetails.Password, - //Mechanism: "DEFAULT", } connection := &options.Connection{ Host: mongoDetails.Host, - //Port: db.DefaultTestPort, } // Trim excess @@ -48,11 +42,7 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") - passSeg := "" - if len(auth.Password) > 5 { - passSeg = auth.Password[0:5] - } - logger.Infof("MongoDump connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + logger.Infof("MongoDump connecting to: %v, user %v...", connection.Host, auth.Username) uri, err := options.NewURI(connectionURI) if err != nil { diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 29734679..3f983887 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -17,7 +17,12 @@ import ( func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { var toolOptions *options.ToolOptions - ssl := options.SSL{} + ssl := options.SSL{ + UseSSL: true, + SSLCAFile: "./global-bundle.pem", + SSLPEMKeyFile: "./global-bundle.pem", + } + auth := options.Auth{ Username: mongoDetails.User, Password: mongoDetails.Password, @@ -25,25 +30,23 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta connection := &options.Connection{ Host: mongoDetails.Host, - //Port: db.DefaultTestPort, } // Trim excess protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) - passSeg := "" - if len(auth.Password) > 5 { - passSeg = auth.Password[0:5] - } - logger.Infof("MongoRestore connecting to: %v, user %v, pass %v...", connection.Host, auth.Username, passSeg) + logger.Infof("MongoRestore connecting to: %v, user %v...", connection.Host, auth.Username) + + retryWrites := false toolOptions = &options.ToolOptions{ - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: &options.URI{}, + RetryWrites: &retryWrites, + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: &options.URI{}, } toolOptions.Namespace = &options.Namespace{DB: restoreToDBName} From 4b376c3f9ca727c699ba7d6401942030c6c59dac Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 20:44:37 +1000 Subject: [PATCH 11/12] Clear backup bucket progress --- api/ws/handlers/system.go | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index 7c1d6c18..76743e95 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -9,6 +9,8 @@ import ( "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/api/ws/wsHelpers" + "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" protos "github.com/pixlise/core/v4/generated-protos" ) @@ -50,6 +52,27 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) ( return &protos.BackupDBResp{}, nil } +func clearBucket(bucket string, fs fileaccess.FileAccess, logger logger.ILogger) error { + files, err := fs.ListObjects(bucket, "") + if err != nil { + return err + } + + logger.Infof("Clearing %v files from bucket: %v", len(files), bucket) + + for c, file := range files { + logger.Infof("Clearing file %v of %v...", c, len(files)) + + err = fs.DeleteObject(bucket, file) + if err != nil { + return err + } + } + + logger.Infof("Bucket cleared: %v", bucket) + return nil +} + func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.APIServices) { var wg sync.WaitGroup var errDBDump error @@ -57,6 +80,14 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A var errImageSync error var errQuantSync error + svcs.Log.Infof("Clearing PIXLISE backup bucket: %v", svcs.Config.DataBackupBucket) + + err := clearBucket(svcs.Config.DataBackupBucket, svcs.FS, svcs.Log) + if err != nil { + svcs.Log.Errorf("PIXLISE Backup bucket clear failed: %v", err) + return + } + wg.Add(1) go func() { defer wg.Done() @@ -96,7 +127,6 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A // Wait for all sync tasks wg.Wait() - var err error if errDBDump != nil { err = fmt.Errorf("PIXLISE Backup DB dump failed: %v", errDBDump) } @@ -114,6 +144,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A } if err != nil { + svcs.Log.Errorf("%v", err) return } From 1426e7325b154d3db2280069327abd8f98ddd45f Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Mon, 16 Sep 2024 20:54:21 +1000 Subject: [PATCH 12/12] Less verbose progress --- api/ws/handlers/system.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index 76743e95..75621c7a 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -61,7 +61,9 @@ func clearBucket(bucket string, fs fileaccess.FileAccess, logger logger.ILogger) logger.Infof("Clearing %v files from bucket: %v", len(files), bucket) for c, file := range files { - logger.Infof("Clearing file %v of %v...", c, len(files)) + if c%100 == 0 { + logger.Infof("Clearing file %v of %v...", c, len(files)) + } err = fs.DeleteObject(bucket, file) if err != nil {