diff --git a/api/dataimport/import.go b/api/dataimport/import.go index e26ec1ac..53c63f33 100644 --- a/api/dataimport/import.go +++ b/api/dataimport/import.go @@ -81,7 +81,7 @@ func ImportDataset( // Firstly, we download from the archive archive := datasetArchive.NewDatasetArchiveDownloader(remoteFS, localFS, log, datasetBucket, manualUploadBucket) - localDownloadPath, localUnzippedPath, zipCount, lastZipName, err := archive.DownloadFromDatasetArchive(datasetID, workingDir) + localDownloadPath, localUnzippedPath, zipCount, _, err := archive.DownloadFromDatasetArchive(datasetID, workingDir) if err != nil { return workingDir, savedSummary, "", false, err } diff --git a/api/ws/wsHelpers/sync-mongo-restore.go b/api/ws/wsHelpers/sync-mongo-restore.go index 79034887..a3137ee3 100644 --- a/api/ws/wsHelpers/sync-mongo-restore.go +++ b/api/ws/wsHelpers/sync-mongo-restore.go @@ -10,6 +10,7 @@ import ( "github.com/pixlise/core/v4/core/fileaccess" "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/mongoDBConnection" + "github.com/pixlise/core/v4/core/utils" ) func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) { @@ -69,10 +70,16 @@ func DownloadArchive(svcs *services.APIServices) (string, error) { svcs.Log.Infof("Found %v remote DB Dump files...", len(remoteDBFiles)) localFS := fileaccess.FSAccess{} - dbName := "" + for _, dbFile := range remoteDBFiles { - svcs.Log.Infof(" Downloading: %v...", dbFile) + // Report free space remaining + freeBytes, err := utils.GetDiskAvailableBytes() + if err != nil { + svcs.Log.Errorf(" Failed to get free disk bytes: %v", err) + } + + svcs.Log.Infof(" Downloading: %v... free space: %v bytes", dbFile, freeBytes) dbFileBytes, err := svcs.FS.ReadObject(svcs.Config.DataBackupBucket, dbFile) if err != nil { diff --git a/core/utils/diskfree_nonwindows.go b/core/utils/diskfree_nonwindows.go new file mode 100644 index 00000000..7a16aabe --- /dev/null +++ b/core/utils/diskfree_nonwindows.go @@ -0,0 +1,22 @@ +//go:build !windows + +package utils + +import ( + "os" + + "golang.org/x/sys/unix" +) + +func GetDiskAvailableBytes() (uint64, error) { + var stat unix.Statfs_t + wd, err := os.Getwd() + if err != nil { + return 0, err + } + + unix.Statfs(wd, &stat) + + // Available blocks * size per block = available space in bytes + return stat.Bavail * uint64(stat.Bsize), nil +} diff --git a/core/utils/diskfree_windows.go b/core/utils/diskfree_windows.go new file mode 100644 index 00000000..ce6f6f36 --- /dev/null +++ b/core/utils/diskfree_windows.go @@ -0,0 +1,16 @@ +package utils + +import "golang.org/x/sys/windows" + +func GetDiskAvailableBytes() (uint64, error) { + var freeBytesAvailable uint64 + var totalNumberOfBytes uint64 + var totalNumberOfFreeBytes uint64 + + err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr("C:"), + &freeBytesAvailable, &totalNumberOfBytes, &totalNumberOfFreeBytes) + if err != nil { + return 0, err + } + return freeBytesAvailable, nil +} diff --git a/go.mod b/go.mod index 7f4fbb47..733e7e8a 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( golang.org/x/net v0.25.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.22.0 // indirect + golang.org/x/sys v0.26.0 // indirect golang.org/x/term v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect golang.org/x/time v0.3.0 // indirect diff --git a/go.sum b/go.sum index 83e331c1..1f3ebbb8 100644 --- a/go.sum +++ b/go.sum @@ -241,6 +241,8 @@ golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI= golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= diff --git a/internal/cmd-line-tools/dataset-archive-optimiser/main.go b/internal/cmd-line-tools/dataset-archive-optimiser/main.go index 426ae4cd..7d81e75e 100644 --- a/internal/cmd-line-tools/dataset-archive-optimiser/main.go +++ b/internal/cmd-line-tools/dataset-archive-optimiser/main.go @@ -122,7 +122,10 @@ func main() { log.Fatalf("Failed to download archive for scan %v: %v", scan.Id, err) } - if zipCount == 0 { + if zipCount == 1 { + l.Infof("Only one zip was loaded, nothing to optimise...") + } + if zipCount <= 1 { // Stuff already logged... l.Infof("No archive zip files found for scan %v\n", scan.Id) continue }