Skip to content

Commit

Permalink
Merge pull request #325 from pixlise/development
Browse files Browse the repository at this point in the history 
Release 4.35.0
  • Loading branch information
@pnemere
pnemere authored Sep 25, 2024
2 parents 7bb92c5 + ec93460 commit 9fca471
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 91 deletions.
4 changes: 3 additions & 1 deletion api/ws/handlers/expression-group.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -65,7 +65,9 @@ func HandleExpressionGroupGetReq(req *protos.ExpressionGroupGetReq, hctx wsHelpe
}

func validateExpressionGroup(egroup *protos.ExpressionGroup) error {
if err := wsHelpers.CheckStringField(&egroup.Name, "Name", 1, 100); err != nil {
// NOTE: name allows for 200 chars. Original was 100. Raised due to temporary RGB mixes generating names that are too long. We intend
// to fix RGB mixes in a different way in future and should lower this back at that point
if err := wsHelpers.CheckStringField(&egroup.Name, "Name", 1, 200); err != nil {
return err
}
if err := wsHelpers.CheckStringField(&egroup.Description, "Description", 0, wsHelpers.DescriptionFieldMaxLength); err != nil {
Expand Down
13 changes: 11 additions & 2 deletions api/ws/handlers/system.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"sync"

"github.com/mongodb/mongo-tools/mongodump"
"github.com/mongodb/mongo-tools/mongorestore"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/api/ws/wsHelpers"
"github.com/pixlise/core/v4/core/fileaccess"
Expand Down Expand Up @@ -39,7 +40,12 @@ func HandleBackupDBReq(req *protos.BackupDBReq, hctx wsHelpers.HandlerContext) (
hctx.Svcs.Log.Infof("PIXLISE Backup Requested, will be written to bucket: %v", hctx.Svcs.Config.DataBackupBucket)

// Run MongoDump, save to a local archive file
dump := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))
dump, err := wsHelpers.MakeMongoDumpInstance(hctx.Svcs.MongoDetails, hctx.Svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", hctx.Svcs.Config.EnvironmentName))

if err != nil {
hctx.Svcs.Log.Errorf("Failed to create dump instance: %v", err)
return nil, err
}

err = dump.Init()
if err != nil {
Expand Down Expand Up @@ -204,9 +210,11 @@ func runRestore(startTimestamp int64, svcs *services.APIServices, downloadRemote
}

if errDBRestore == nil {
restore, errDBRestore := wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName)
var restore *mongorestore.MongoRestore
restore, errDBRestore = wsHelpers.MakeMongoRestoreInstance(svcs.MongoDetails, svcs.Log, mongoDBConnection.GetDatabaseName("pixlise", svcs.Config.EnvironmentName), restoreFromDBName)

if errDBRestore == nil {
svcs.Log.Infof("Mongo Restore starting...")
result := restore.Restore()
if result.Err != nil {
errDBRestore = result.Err
Expand Down Expand Up @@ -267,6 +275,7 @@ func runRestore(startTimestamp int64, svcs *services.APIServices, downloadRemote
}

if err != nil {
svcs.Log.Errorf("%v", err)
return
}

Expand Down
54 changes: 5 additions & 49 deletions api/ws/wsHelpers/sync-mongo-dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,7 @@ import (
"fmt"
"os"
"path"
"strings"

"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/mongodump"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/core/fileaccess"
Expand All @@ -18,50 +15,13 @@ import (
var dataBackupLocalPath = "./backup"
var dataBackupS3Path = "DB"

func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) *mongodump.MongoDump {
var toolOptions *options.ToolOptions

ssl := options.SSL{
UseSSL: true,
SSLCAFile: "./global-bundle.pem",
SSLPEMKeyFile: "./global-bundle.pem",
}

auth := options.Auth{
Username: mongoDetails.User,
Password: mongoDetails.Password,
}

connection := &options.Connection{
Host: mongoDetails.Host,
}

// Trim excess
protocolPrefix := "mongodb://"
connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)

connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")

logger.Infof("MongoDump connecting to: %v, user %v...", connection.Host, auth.Username)

uri, err := options.NewURI(connectionURI)
func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbName string) (*mongodump.MongoDump, error) {
toolOptions, err := makeMongoToolOptions(mongoDetails, logger, dbName)
if err != nil {
logger.Errorf("%v", err)
return nil
return nil, err
}

retryWrites := false

toolOptions = &options.ToolOptions{
RetryWrites: &retryWrites,
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: uri,
}

toolOptions.Namespace = &options.Namespace{DB: dbName}
logger.Infof("MongoDump connecting to: %v, user %v, db-to-dump: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, dbName)

outputOptions := &mongodump.OutputOptions{
Out: dataBackupLocalPath,
Expand All @@ -72,15 +32,11 @@ func MakeMongoDumpInstance(mongoDetails mongoDBConnection.MongoConnectionDetails
}
inputOptions := &mongodump.InputOptions{}

log.SetVerbosity(nil /*toolOptions.Verbosity*/)
lw := LogWriter{logger: logger}
log.SetWriter(lw)

return &mongodump.MongoDump{
ToolOptions: toolOptions,
InputOptions: inputOptions,
OutputOptions: outputOptions,
}
}, nil
}

func UploadArchive(svcs *services.APIServices) error {
Expand Down
43 changes: 4 additions & 39 deletions api/ws/wsHelpers/sync-mongo-restore.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,6 @@ import (
"path"
"strings"

"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/mongodb/mongo-tools/mongorestore"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/core/fileaccess"
Expand All @@ -15,41 +13,12 @@ import (
)

func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) {
var toolOptions *options.ToolOptions

ssl := options.SSL{
UseSSL: true,
SSLCAFile: "./global-bundle.pem",
SSLPEMKeyFile: "./global-bundle.pem",
}

auth := options.Auth{
Username: mongoDetails.User,
Password: mongoDetails.Password,
}

connection := &options.Connection{
Host: mongoDetails.Host,
}

// Trim excess
protocolPrefix := "mongodb://"
connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)

logger.Infof("MongoRestore connecting to: %v, user %v...", connection.Host, auth.Username)

retryWrites := false

toolOptions = &options.ToolOptions{
RetryWrites: &retryWrites,
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: &options.URI{},
toolOptions, err := makeMongoToolOptions(mongoDetails, logger, restoreToDBName)
if err != nil {
return nil, err
}

toolOptions.Namespace = &options.Namespace{DB: restoreToDBName}
logger.Infof("MongoRestore connecting to: %v, user %v, restore-to-db: %v, restore-from-db: %v...", toolOptions.URI.ConnectionString, toolOptions.Auth.Username, restoreToDBName, restoreFromDBName)

outputOptions := &mongorestore.OutputOptions{
NumParallelCollections: 1,
Expand All @@ -71,10 +40,6 @@ func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDeta
NSTo: []string{restoreToDBName},
}

log.SetVerbosity(nil /*toolOptions.Verbosity*/)
lw := LogWriter{logger: logger}
log.SetWriter(lw)

return mongorestore.New(mongorestore.Options{
ToolOptions: toolOptions,
InputOptions: inputOptions,
Expand Down
60 changes: 60 additions & 0 deletions api/ws/wsHelpers/sync-mongo.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,12 @@ package wsHelpers
import (
"fmt"
"os"
"strings"

"github.com/mongodb/mongo-tools/common/log"
"github.com/mongodb/mongo-tools/common/options"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/mongoDBConnection"
)

func ResetLocalMongoBackupDir() error {
Expand All @@ -23,3 +29,57 @@ func ClearLocalMongoArchive() error {

return nil
}

func makeMongoToolOptions(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, dbNamespace string) (*options.ToolOptions, error) {
var toolOptions *options.ToolOptions

log.SetVerbosity(nil /*toolOptions.Verbosity*/)
lw := LogWriter{logger: logger}
log.SetWriter(lw)

ssl := options.SSL{}

isLocal := strings.Contains(mongoDetails.Host, "localhost") && len(mongoDetails.User) <= 0 && len(mongoDetails.Password) <= 0

if !isLocal {
ssl = options.SSL{
UseSSL: true,
SSLCAFile: "./global-bundle.pem",
SSLPEMKeyFile: "./global-bundle.pem",
}
}

auth := options.Auth{
Username: mongoDetails.User,
Password: mongoDetails.Password,
}

connection := &options.Connection{
Host: mongoDetails.Host,
}

// Trim excess
protocolPrefix := "mongodb://"
connection.Host = strings.TrimPrefix(connection.Host, protocolPrefix)

connectionURI := fmt.Sprintf("mongodb://%s/%s", connection.Host, "")

uri, err := options.NewURI(connectionURI)
if err != nil {
return nil, err
}

retryWrites := false

toolOptions = &options.ToolOptions{
RetryWrites: &retryWrites,
SSL: &ssl,
Connection: connection,
Auth: &auth,
Verbosity: &options.Verbosity{},
URI: uri,
Namespace: &options.Namespace{DB: dbNamespace},
}

return toolOptions, nil
}

0 comments on commit 9fca471

Please sign in to comment.