Skip to content

Commit

Permalink
Merge pull request #319 from pixlise/development
Browse files Browse the repository at this point in the history 
Release 4.34.0
  • Loading branch information
@pnemere
pnemere authored Sep 16, 2024
2 parents 8baac19 + 91fdef9 commit 7bb92c5
Show file tree
Hide file tree
Showing 4 changed files with 103 additions and 28 deletions.
45 changes: 39 additions & 6 deletions api/ws/handlers/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,19 +9,21 @@ import (
"github.com/mongodb/mongo-tools/mongodump"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/api/ws/wsHelpers"
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/mongoDBConnection"
protos "github.com/pixlise/core/v4/generated-protos"
)

func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (*protos.BackupDBResp, error) {
if len(hctx.Svcs.Config.DataBackupBucket) <= 0 {
err := "PIXLISE Backup bucket not configured!"
err := "PIXLISE Backup bucket not configured"
hctx.Svcs.Log.Errorf(err)
return nil, errors.New(err)
}

if !hctx.Svcs.Config.BackupEnabled {
err := "PIXLISE Backup not enabled!"
err := "PIXLISE Backup not enabled"
hctx.Svcs.Log.Errorf(err)
return nil, errors.New(err)
}
Expand All @@ -37,7 +39,7 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (
hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket)

// Run MongoDump, save to a local archive file
dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))
dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))

err = dump.Init()
if err != nil {
Expand All @@ -50,13 +52,44 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (
return &protos.BackupDBResp{}, nil
}

func clearBucket(bucket string, fs fileaccess.FileAccess, logger logger.ILogger) error {
files, err := fs.ListObjects(bucket, "")
if err != nil {
return err
}

logger.Infof("Clearing %v files from bucket: %v", len(files), bucket)

for c, file := range files {
if c%100 == 0 {
logger.Infof("Clearing file %v of %v...", c, len(files))
}

err = fs.DeleteObject(bucket, file)
if err != nil {
return err
}
}

logger.Infof("Bucket cleared: %v", bucket)
return nil
}

func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.APIServices) {
var wg sync.WaitGroup
var errDBDump error
var errScanSync error
var errImageSync error
var errQuantSync error

svcs.Log.Infof("Clearing PIXLISE backup bucket: %v", svcs.Config.DataBackupBucket)

err := clearBucket(svcs.Config.DataBackupBucket, svcs.FS, svcs.Log)
if err != nil {
svcs.Log.Errorf("PIXLISE Backup bucket clear failed: %v", err)
return
}

wg.Add(1)
go func() {
defer wg.Done()
Expand Down Expand Up @@ -96,7 +129,6 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A
// Wait for all sync tasks
wg.Wait()

var err error
if errDBDump != nil {
err = fmt.Errorf("PIXLISE Backup DB dump failed: %v", errDBDump)
}
Expand All @@ -114,6 +146,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A
}

if err != nil {
svcs.Log.Errorf("%v", err)
return
}

Expand All @@ -126,7 +159,7 @@ func runBackup(dump *mongodump.MongoDump, startTimestamp int64, svcs *services.A
func HandleRestoreDBReq(req *protos.RestoreDBReq, hctx wsHelpers.HandlerContext) (*protos.RestoreDBResp, error) {
// Only allow restore if enabled and we're NOT prod
if !hctx.Svcs.Config.RestoreEnabled {
err := "PIXLISE Restore not enabled!"
err := "PIXLISE Restore not enabled"
hctx.Svcs.Log.Errorf(err)
return nil, errors.New(err)
}
Expand Down Expand Up @@ -171,7 +204,7 @@ func runRestore(startTimestamp int64, svcs *services.APIServices, downloadRemote
}

if errDBRestore == nil {
restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName)
restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName)

if errDBRestore == nil {
result := restore.Restore()
Expand Down
39 changes: 30 additions & 9 deletions api/ws/wsHelpers/sync-mongo-dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,40 +6,59 @@ import (
"path"
"strings"

"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/mongodump"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/mongoDBConnection"
)

var dataBackupLocalPath = "./backup"
var dataBackupS3Path = "DB"

func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, dbName string) *mongodump.MongoDump {
func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump {
var toolOptions *options.ToolOptions

ssl := options.SSL{}
ssl := options.SSL{
UseSSL: true,
SSLCAFile: "./global-bundle.pem",
SSLPEMKeyFile: "./global-bundle.pem",
}

auth := options.Auth{
Username: mongoDetails.User,
Password: mongoDetails.Password,
}

connection := &options.Connection{
Host: mongoDetails.Host,
//Port: db.DefaultTestPort,
}

// Trim excess
protocolPrefix := "mongodb://"
connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)

connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")

logger.Infof("MongoDump connecting to: %v, user %v...", connection.Host, auth.Username)

uri, err := options.NewURI(connectionURI)
if err != nil {
logger.Errorf("%v", err)
return nil
}

retryWrites := false

toolOptions = &options.ToolOptions{
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: &options.URI{},
RetryWrites: &retryWrites,
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: uri,
}

toolOptions.Namespace = &options.Namespace{DB: dbName}
Expand All @@ -53,7 +72,9 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails
}
inputOptions := &mongodump.InputOptions{}

//log.SetVerbosity(toolOptions.Verbosity)
log.SetVerbosity(nil /*toolOptions.Verbosity*/)
lw := LogWriter{logger: logger}
log.SetWriter(lw)

return &mongodump.MongoDump{
ToolOptions: toolOptions,
Expand Down
45 changes: 33 additions & 12 deletions api/ws/wsHelpers/sync-mongo-restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,48 @@ import (
"path"
"strings"

"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/mongorestore"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/mongoDBConnection"
)

func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) {
func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) {
var toolOptions *options.ToolOptions

ssl := options.SSL{}
ssl := options.SSL{
UseSSL: true,
SSLCAFile: "./global-bundle.pem",
SSLPEMKeyFile: "./global-bundle.pem",
}

auth := options.Auth{
Username: mongoDetails.User,
Password: mongoDetails.Password,
}

connection := &options.Connection{
Host: mongoDetails.Host,
//Port: db.DefaultTestPort,
}

// Trim excess
protocolPrefix := "mongodb://"
connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)

logger.Infof("MongoRestore connecting to: %v, user %v...", connection.Host, auth.Username)

retryWrites := false

toolOptions = &options.ToolOptions{
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: &options.URI{},
RetryWrites: &retryWrites,
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: &options.URI{},
}

toolOptions.Namespace = &options.Namespace{DB: restoreToDBName}
Expand All @@ -56,12 +67,13 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta

nsOptions := &mongorestore.NSOptions{
NSInclude: []string{"*"},
//NSInclude: []string{"pixlise-prodv4-15-jul-2024.ownership"},
NSFrom: []string{"pixlise-prodv4-15-jul-2024"},
NSTo: []string{restoreToDBName},
NSFrom: []string{restoreFromDBName},
NSTo: []string{restoreToDBName},
}

//log.SetVerbosity(toolOptions.Verbosity)
log.SetVerbosity(nil /*toolOptions.Verbosity*/)
lw := LogWriter{logger: logger}
log.SetWriter(lw)

return mongorestore.New(mongorestore.Options{
ToolOptions: toolOptions,
Expand All @@ -72,6 +84,15 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta
})
}

type LogWriter struct {
logger logger.ILogger
}

func (w LogWriter) Write(p []byte) (n int, err error) {
w.logger.Infof(string(p))
return len(p), nil
}

func DownloadArchive(svcs *services.APIServices) (string, error) {
svcs.Log.Infof("Downloading PIXLISE DB Dump files...")

Expand Down
2 changes: 1 addition & 1 deletion internal/cmd-line-tools/api-integration-test/testImage.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ func doHTTPRequest(scheme string, method string, apiHost string, urlPath string,
bodyReader = io.NopCloser(reqBody)
}

fmt.Printf("Sending HTTP %v request: %v...", method, wsConnectUrl.String())
fmt.Printf("Sending HTTP %v request: %v...\n", method, wsConnectUrl.String())

client := &http.Client{}
req, err := http.NewRequest(method, wsConnectUrl.String(), bodyReader)
Expand Down

0 comments on commit 7bb92c5

Please sign in to comment.