diff --git a/api/ws/handlers/system.go b/api/ws/handlers/system.go index 92eaaa70..75621c7a 100644 --- a/api/ws/handlers/system.go +++ b/api/ws/handlers/system.go @@ -9,19 +9,21 @@ import ( "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/api/ws/wsHelpers" + "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" protos "github.com/pixlise/core/v4/generated-protos" ) func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (*protos.BackupDBResp, error) { if len(hctx.Svcs.Config.DataBackupBucket) <= 0 { - err := "PIXLISE Backup bucket not configured!" + err := "PIXLISE Backup bucket not configured" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } if !hctx.Svcs.Config.BackupEnabled { - err := "PIXLISE Backup not enabled!" + err := "PIXLISE Backup not enabled" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } @@ -37,7 +39,7 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) ( hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket) // Run MongoDump, save to a local archive file - dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) + dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName)) err = dump.Init() if err != nil { @@ -50,6 +52,29 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) ( return &protos.BackupDBResp{}, nil } +func clearBucket(bucket string, fs fileaccess.FileAccess, logger logger.ILogger) error { + files, err := fs.ListObjects(bucket, "") + if err != nil { + return err + } + + logger.Infof("Clearing %v files from bucket: %v", len(files), bucket) + + for c, file := range files { + if c%100 == 0 { + logger.Infof("Clearing file %v of %v...", c, len(files)) + } + + err = fs.DeleteObject(bucket, file) + if err != nil { + return err + } + } + + logger.Infof("Bucket cleared: %v", bucket) + return nil +} + func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.APIServices) { var wg sync.WaitGroup var errDBDump error @@ -57,6 +82,14 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A var errImageSync error var errQuantSync error + svcs.Log.Infof("Clearing PIXLISE backup bucket: %v", svcs.Config.DataBackupBucket) + + err := clearBucket(svcs.Config.DataBackupBucket, svcs.FS, svcs.Log) + if err != nil { + svcs.Log.Errorf("PIXLISE Backup bucket clear failed: %v", err) + return + } + wg.Add(1) go func() { defer wg.Done() @@ -96,7 +129,6 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A // Wait for all sync tasks wg.Wait() - var err error if errDBDump != nil { err = fmt.Errorf("PIXLISE Backup DB dump failed: %v", errDBDump) } @@ -114,6 +146,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A } if err != nil { + svcs.Log.Errorf("%v", err) return } @@ -126,7 +159,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A func HandleRestoreDBReq(req *protos.RestoreDBReq, hctx wsHelpers.HandlerContext) (*protos.RestoreDBResp, error) { // Only allow restore if enabled and we're NOT prod if !hctx.Svcs.Config.RestoreEnabled { - err := "PIXLISE Restore not enabled!" + err := "PIXLISE Restore not enabled" hctx.Svcs.Log.Errorf(err) return nil, errors.New(err) } @@ -171,7 +204,7 @@ func runRestore(startTimestamp int64, svcs *services.APIServices, downloadRemote } if errDBRestore == nil { - restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName) + restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName) if errDBRestore == nil { result := restore.Restore() diff --git a/api/ws/wsHelpers/sync-mongo-dump.go b/api/ws/wsHelpers/sync-mongo-dump.go index f0c6399b..89139cbf 100644 --- a/api/ws/wsHelpers/sync-mongo-dump.go +++ b/api/ws/wsHelpers/sync-mongo-dump.go @@ -6,20 +6,27 @@ import ( "path" "strings" + "github.com/mongodb/mongo-tools/common/log" "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongodump" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" ) var dataBackupLocalPath = "./backup" var dataBackupS3Path = "DB" -func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, dbName string) *mongodump.MongoDump { +func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump { var toolOptions *options.ToolOptions - ssl := options.SSL{} + ssl := options.SSL{ + UseSSL: true, + SSLCAFile: "./global-bundle.pem", + SSLPEMKeyFile: "./global-bundle.pem", + } + auth := options.Auth{ Username: mongoDetails.User, Password: mongoDetails.Password, @@ -27,19 +34,31 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails connection := &options.Connection{ Host: mongoDetails.Host, - //Port: db.DefaultTestPort, } // Trim excess protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "") + + logger.Infof("MongoDump connecting to: %v, user %v...", connection.Host, auth.Username) + + uri, err := options.NewURI(connectionURI) + if err != nil { + logger.Errorf("%v", err) + return nil + } + + retryWrites := false + toolOptions = &options.ToolOptions{ - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: &options.URI{}, + RetryWrites: &retryWrites, + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: uri, } toolOptions.Namespace = &options.Namespace{DB: dbName} @@ -53,7 +72,9 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails } inputOptions := &mongodump.InputOptions{} - //log.SetVerbosity(toolOptions.Verbosity) + log.SetVerbosity(nil /*toolOptions.Verbosity*/) + lw := LogWriter{logger: logger} + log.SetWriter(lw) return &mongodump.MongoDump{ ToolOptions: toolOptions, diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 355a425f..3f983887 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -5,17 +5,24 @@ import ( "path" "strings" + "github.com/mongodb/mongo-tools/common/log" "github.com/mongodb/mongo-tools/common/options" "github.com/mongodb/mongo-tools/mongorestore" "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/fileaccess" + "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" ) -func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { +func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { var toolOptions *options.ToolOptions - ssl := options.SSL{} + ssl := options.SSL{ + UseSSL: true, + SSLCAFile: "./global-bundle.pem", + SSLPEMKeyFile: "./global-bundle.pem", + } + auth := options.Auth{ Username: mongoDetails.User, Password: mongoDetails.Password, @@ -23,19 +30,23 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta connection := &options.Connection{ Host: mongoDetails.Host, - //Port: db.DefaultTestPort, } // Trim excess protocolPrefix := "mongodb://" connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix) + logger.Infof("MongoRestore connecting to: %v, user %v...", connection.Host, auth.Username) + + retryWrites := false + toolOptions = &options.ToolOptions{ - SSL: &ssl, - Connection: connection, - Auth: &auth, - Verbosity: &options.Verbosity{}, - URI: &options.URI{}, + RetryWrites: &retryWrites, + SSL: &ssl, + Connection: connection, + Auth: &auth, + Verbosity: &options.Verbosity{}, + URI: &options.URI{}, } toolOptions.Namespace = &options.Namespace{DB: restoreToDBName} @@ -56,12 +67,13 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta nsOptions := &mongorestore.NSOptions{ NSInclude: []string{"*"}, - //NSInclude: []string{"pixlise-prodv4-15-jul-2024.ownership"}, - NSFrom: []string{"pixlise-prodv4-15-jul-2024"}, - NSTo: []string{restoreToDBName}, + NSFrom: []string{restoreFromDBName}, + NSTo: []string{restoreToDBName}, } - //log.SetVerbosity(toolOptions.Verbosity) + log.SetVerbosity(nil /*toolOptions.Verbosity*/) + lw := LogWriter{logger: logger} + log.SetWriter(lw) return mongorestore.New(mongorestore.Options{ ToolOptions: toolOptions, @@ -72,6 +84,15 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta }) } +type LogWriter struct { + logger logger.ILogger +} + +func (w LogWriter) Write(p []byte) (n int, err error) { + w.logger.Infof(string(p)) + return len(p), nil +} + func DownloadArchive(svcs *services.APIServices) (string, error) { svcs.Log.Infof("Downloading PIXLISE DB Dump files...") diff --git a/internal/cmd-line-tools/api-integration-test/testImage.go b/internal/cmd-line-tools/api-integration-test/testImage.go index fe44744f..c0634446 100644 --- a/internal/cmd-line-tools/api-integration-test/testImage.go +++ b/internal/cmd-line-tools/api-integration-test/testImage.go @@ -132,7 +132,7 @@ func doHTTPRequest(scheme string, method string, apiHost string, urlPath string, bodyReader = io.NopCloser(reqBody) } - fmt.Printf("Sending HTTP %v request: %v...", method, wsConnectUrl.String()) + fmt.Printf("Sending HTTP %v request: %v...\n", method, wsConnectUrl.String()) client := &http.Client{} req, err := http.NewRequest(method, wsConnectUrl.String(), bodyReader)