Added extra logging to importer, brought back check for single dir in…

… for EM unzipped data

api/dataimport/internal/converterSelector/selector.go

@@ -50,6 +50,7 @@ func SelectDataConverter(localFS fileaccess.FileAccess, remoteFS fileaccess.File
 		}
 	*/
 	// Check if it's a PIXL FM style dataset
+	log.Infof("Checking path \"%v\" for PIXL FM structure...", importPath)
 	pathType, err := pixlfm.DetectPIXLFMStructure(importPath)
 	if len(pathType) > 0 && err == nil {
 		// We know it's a PIXL FM type dataset... it'll later be determined which one
@@ -75,10 +76,14 @@ func SelectDataConverter(localFS fileaccess.FileAccess, remoteFS fileaccess.File
 	var detectorFile dataimportModel.DetectorChoice
 	err = localFS.ReadJSON(detPath, "", &detectorFile, false)
 	if err == nil {
+		log.Infof("Loaded detector.json...")
+
 		// We found it, work out based on what's in there
 		if strings.HasSuffix(detectorFile.Detector, "-breadboard") {
+			log.Infof("Assuming breadboard dataset...")
 			return jplbreadboard.MSATestData{}, nil
 		} else if detectorFile.Detector == "pixl-em" {
+			log.Infof("Assuming PIXL EM dataset...")
 			return pixlem.PIXLEM{}, nil
 		}
 	} else {

api/dataimport/internal/converters/pixlem/import.go

@@ -18,6 +18,10 @@
 package pixlem
 
 import (
+	"errors"
+	"os"
+	"path/filepath"
+
 	"github.com/pixlise/core/v4/api/dataimport/internal/converters/pixlfm"
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
 	"github.com/pixlise/core/v4/core/logger"
@@ -32,11 +36,31 @@ type PIXLEM struct {
 }
 
 func (p PIXLEM) Import(importPath string, pseudoIntensityRangesPath string, datasetIDExpected string, log logger.ILogger) (*dataConvertModels.OutputData, string, error) {
+	// Find the subdir
+	subdir := ""
+
+	c, _ := os.ReadDir(importPath)
+	for _, entry := range c {
+		if entry.IsDir() {
+			// If it's not the first one, we can't do this
+			if len(subdir) > 0 {
+				return nil, "", errors.New("Found multiple subdirs, expected one in: " + importPath)
+			}
+			subdir = entry.Name()
+		}
+	}
+
+	if len(subdir) <= 0 {
+		return nil, "", errors.New("Failed to find PIXL data subdir in: " + importPath)
+	}
+
+	// Form the actual path to the files
+	subImportPath := filepath.Join(importPath, subdir)
 	fmImporter := pixlfm.PIXLFM{}
 
 	// Override importers group and detector
 	fmImporter.SetOverrides(protos.ScanInstrument_PIXL_EM, "PIXL-EM-E2E")
 
 	// Now we can import it like normal
-	return fmImporter.Import(importPath, pseudoIntensityRangesPath, datasetIDExpected, log)
+	return fmImporter.Import(subImportPath, pseudoIntensityRangesPath, datasetIDExpected, log)
 }

api/dataimport/internal/converters/pixlfm/import.go

@@ -85,11 +85,13 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 	rgbuImgDir := fileStructure{}
 	discoImgDir := fileStructure{}
 
+	log.Infof("Checking path \"%v\" for FM dataset type", importPath)
 	pathType, err := DetectPIXLFMStructure(importPath)
 	if err != nil {
 		return nil, "", err
 	}
 
+	log.Infof("Found path \"%v\" is of type %v", importPath, pathType)
 	if pathType == "DataDrive" {
 		// This is the official way we receive PIXL FM data from Mars
 		// We expect these directories to exist...

api/endpoints/Scan.go

@@ -51,6 +51,19 @@ func PutScanData(params apiRouter.ApiHandlerGenericParams) error {
 	s3PathStart := path.Join(filepaths.DatasetUploadRoot, scanId)
 
 	// NOTE: We overwrite any previous attempts without worry!
+	existing, err := params.Svcs.FS.ListObjects(destBucket, s3PathStart+"/")
+	if err == nil && len(existing) > 0 {
+		// Delete all that exists
+		msg := fmt.Sprintf("PutScan for \"%v\": Deleting existing file...\n", scanId)
+		for _, existingItem := range existing {
+			msg += existingItem + "\n"
+			if err := params.Svcs.FS.DeleteObject(destBucket, existingItem); err != nil {
+				return fmt.Errorf("Failed to delete: \"%v\", error: %v", existing, err)
+			}
+		}
+
+		params.Svcs.Log.Infof(msg)
+	}
 
 	// Read in body
 	zippedData, err := io.ReadAll(params.Request.Body)