Merge pull request #347 from pixlise/feature/archive-optimiser-tool

Show disk free space when restoring PIXLISE backup. Also ensure datas…

api/dataimport/import.go

@@ -81,7 +81,7 @@ func ImportDataset(
 
 	// Firstly, we download from the archive
 	archive := datasetArchive.NewDatasetArchiveDownloader(remoteFS, localFS, log, datasetBucket, manualUploadBucket)
-	localDownloadPath, localUnzippedPath, zipCount, lastZipName, err := archive.DownloadFromDatasetArchive(datasetID, workingDir)
+	localDownloadPath, localUnzippedPath, zipCount, _, err := archive.DownloadFromDatasetArchive(datasetID, workingDir)
 	if err != nil {
 		return workingDir, savedSummary, "", false, err
 	}

api/ws/wsHelpers/sync-mongo-restore.go

@@ -10,6 +10,7 @@ import (
 	"github.com/pixlise/core/v4/core/fileaccess"
 	"github.com/pixlise/core/v4/core/logger"
 	"github.com/pixlise/core/v4/core/mongoDBConnection"
+	"github.com/pixlise/core/v4/core/utils"
 )
 
 func MakeMongoRestoreInstance(mongoDetails mongoDBConnection.MongoConnectionDetails, logger logger.ILogger, restoreToDBName string, restoreFromDBName string) (*mongorestore.MongoRestore, error) {
@@ -69,10 +70,16 @@ func DownloadArchive(svcs *services.APIServices) (string, error) {
 	svcs.Log.Infof("Found %v remote DB Dump files...", len(remoteDBFiles))
 
 	localFS := fileaccess.FSAccess{}
-
 	dbName := ""
+
 	for _, dbFile := range remoteDBFiles {
-		svcs.Log.Infof(" Downloading: %v...", dbFile)
+		// Report free space remaining
+		freeBytes, err := utils.GetDiskAvailableBytes()
+		if err != nil {
+			svcs.Log.Errorf(" Failed to get free disk bytes: %v", err)
+		}
+
+		svcs.Log.Infof(" Downloading: %v... free space: %v bytes", dbFile, freeBytes)
 
 		dbFileBytes, err := svcs.FS.ReadObject(svcs.Config.DataBackupBucket, dbFile)
 		if err != nil {

core/utils/diskfree_nonwindows.go

@@ -0,0 +1,22 @@
+//go:build !windows
+
+package utils
+
+import (
+	"os"
+
+	"golang.org/x/sys/unix"
+)
+
+func GetDiskAvailableBytes() (uint64, error) {
+	var stat unix.Statfs_t
+	wd, err := os.Getwd()
+	if err != nil {
+		return 0, err
+	}
+
+	unix.Statfs(wd, &stat)
+
+	// Available blocks * size per block = available space in bytes
+	return stat.Bavail * uint64(stat.Bsize), nil
+}

core/utils/diskfree_windows.go

@@ -0,0 +1,16 @@
+package utils
+
+import "golang.org/x/sys/windows"
+
+func GetDiskAvailableBytes() (uint64, error) {
+	var freeBytesAvailable uint64
+	var totalNumberOfBytes uint64
+	var totalNumberOfFreeBytes uint64
+
+	err := windows.GetDiskFreeSpaceEx(windows.StringToUTF16Ptr("C:"),
+		&freeBytesAvailable, &totalNumberOfBytes, &totalNumberOfFreeBytes)
+	if err != nil {
+		return 0, err
+	}
+	return freeBytesAvailable, nil
+}

go.mod

@@ -69,7 +69,7 @@ require (
 	golang.org/x/net v0.25.0 // indirect
 	golang.org/x/oauth2 v0.8.0 // indirect
 	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.22.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
 	golang.org/x/term v0.22.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	golang.org/x/time v0.3.0 // indirect

go.sum

@@ -241,6 +241,8 @@ golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
 golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

internal/cmd-line-tools/dataset-archive-optimiser/main.go

@@ -122,7 +122,10 @@ func main() {
 				log.Fatalf("Failed to download archive for scan %v: %v", scan.Id, err)
 			}
 
-			if zipCount == 0 {
+			if zipCount == 1 {
+				l.Infof("Only one zip was loaded, nothing to optimise...")
+			}
+			if zipCount <= 1 {
 				// Stuff already logged... l.Infof("No archive zip files found for scan %v\n", scan.Id)
 				continue
 			}