Merge pull request #245 from pixlise/feature/beam-loc-version

Feature/beam loc version

api/dataimport/for-trigger_test.go

@@ -362,7 +362,7 @@ func Example_importForTrigger_OCS_DatasetEdit() {
 	// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
 }
 
-func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) {
+func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32, beamLocLBLFileName string, beamVersion uint32) {
 	// Ensure these log msgs appeared...
 	requiredLogs := []string{
 		"Downloading archived zip files...",
@@ -380,6 +380,12 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
 		"No auto-share destination found, so only importing user will be able to access this dataset.",
 	}
 
+	if len(beamLocLBLFileName) > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", beamLocLBLFileName, beamVersion))
+	} else if beamVersion > 0 {
+		requiredLogs = append(requiredLogs, fmt.Sprintf("Saving as beam geometry tool version: %v", beamVersion))
+	}
+
 	for _, msg := range requiredLogs {
 		fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg))
 	}
@@ -412,7 +418,7 @@ func Example_importForTrigger_Manual_JPL() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234", 3)
+	printManualOKLogOutput(log, db, "test1234", 3, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -445,7 +451,7 @@ func Example_importForTrigger_Manual_SBU() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -478,7 +484,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "test1234sbu", 4)
+	printManualOKLogOutput(log, db, "test1234sbu", 4, "", 0)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -529,7 +535,41 @@ func Example_importForTrigger_Manual_EM() {
 
 	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
 
-	printManualOKLogOutput(log, db, "048300551", 3)
+	printManualOKLogOutput(log, db, "048300551", 3, "PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL", 2)
+
+	// Output:
+	// Errors: <nil>, changes: unknown, isUpdate: false
+	// Logged "Downloading archived zip files...": true
+	// Logged "Downloaded 0 zip files, unzipped 0 files": true
+	// Logged "No zip files found in archive, dataset may have been manually uploaded. Trying to download...": true
+	// Logged "Dataset 048300551 downloaded 3 files from manual upload area": true
+	// Logged "Downloading pseudo-intensity ranges...": true
+	// Logged "Downloading user customisation files...": true
+	// Logged "Reading 1261 files from spectrum directory...": false
+	// Logged "Reading spectrum [1135/1260] 90%": false
+	// Logged "PMC 1261 has 4 MSA/spectrum entries": false
+	// Logged "WARNING: No main context image determined": false
+	// Logged "Diffraction db saved successfully": true
+	// Logged "Warning: No import.json found, defaults will be used": false
+	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "WARNING: Beam location LBL file (PE__0125_0678031418_000RXL_N004171204830055100910__J01.LBL) could not be read. Beam Version version is assumed to be: 2": true
+	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
+}
+
+// Import a breadboard dataset from manual uploaded zip file
+func Example_importForTrigger_Manual_EM_WithBeamV2() {
+	remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("ManualEM_Beamv2_OK", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId")
+
+	trigger := `{
+	"datasetID": "048300551",
+	"jobID": "dataimport-unittest048300551"
+}`
+
+	result, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS)
+
+	fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)
+
+	printManualOKLogOutput(log, db, "048300551", 3, "", 2)
 
 	// Output:
 	// Errors: <nil>, changes: unknown, isUpdate: false
@@ -546,6 +586,7 @@ func Example_importForTrigger_Manual_EM() {
 	// Logged "Diffraction db saved successfully": true
 	// Logged "Warning: No import.json found, defaults will be used": false
 	// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
+	// Logged "Saving as beam geometry tool version: 2": true
 	// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
 }
 

api/dataimport/internal/converters/pixlfm/import.go

@@ -23,6 +23,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"unicode/utf8"
 
 	"github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels"
 	"github.com/pixlise/core/v4/api/dataimport/internal/importerutils"
@@ -118,6 +119,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 
 	// Allocate everything needed (empty, if we find & load stuff, great, but we still need the data struct for the last step)
 	beamLookup := dataConvertModels.BeamLocationByPMC{}
+	beamToolVersion := 0
 	hkData := dataConvertModels.HousekeepingData{}
 	locSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
 	bulkMaxSpectraLookup := dataConvertModels.DetectorSampleByPMC{}
@@ -214,10 +216,38 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 			for file, beamCsvMeta := range latestVersionFoundPaths {
 				if beamCsvMeta.ProdType == "RXL" {
 					// If files don't conform, don't read...
-					beamLookup, err = importerutils.ReadBeamLocationsFile(filepath.Join(pathToSubdir, file), true, 1, log)
+					beamFilePath := filepath.Join(pathToSubdir, file)
+					beamLookup, err = importerutils.ReadBeamLocationsFile(beamFilePath, true, 1, log)
 					if err != nil {
 						return nil, "", err
 					} else {
+						// Import worked, find out what version of the beam geometry tool generated this by looking in the LBL file
+						lblPath := beamFilePath[0:len(beamFilePath)-3] + "LBL"
+						lblFileBytes, err := os.ReadFile(lblPath)
+						if err != nil {
+							//return nil, "", fmt.Errorf("Failed to read LBL file: %v. Beam geometry version could not be determined.", lblPath)
+							beamToolVersion = 2 // If we don't have an LBL file but it's a "recent" copy of the data, it has to be using beam geometry v2...
+							log.Errorf("WARNING: Beam location LBL file (%v) could not be read. Beam Version version is assumed to be: %v", filepath.Base(lblPath), beamToolVersion)
+						} else {
+							// Read each line and find the beam geometry version
+							lblLines := strings.Split(string(lblFileBytes), "\n")
+							searchStr := "Beam Geometry Tool Version "
+
+							for _, line := range lblLines {
+								if pos := strings.Index(line, searchStr); pos > -1 {
+									verPos := pos + len(searchStr)
+
+									log.Infof("Beam Geometry Tool version text found: %v", line[verPos:])
+
+									verChar := line[verPos : verPos+1]
+									r, _ := utf8.DecodeRuneInString(verChar)
+									beamToolVersion = int(r) - '0'
+									log.Infof("Saving as beam geometry tool version: %v", beamToolVersion)
+									break
+								}
+							}
+						}
+
 						// Found it, why keep looping?
 						break
 					}
@@ -358,7 +388,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 		datasetIDExpected,
 		p.overrideInstrument,
 		p.overrideDetector,
-		1, // TODO: Retrieve beam version and set it here!
+		uint32(beamToolVersion),
 		log,
 	)
 

api/dataimport/internal/output/output.go

@@ -335,11 +335,7 @@ func (s *PIXLISEDataSaver) Save(
 	// as we switched to storing them in DB (to allow import of other images with a corresponding set of beam locations)
 	// Redundant, but this is how it evolved...
 	for idx, imgItem := range exp.AlignedContextImages {
-		beamVer := data.BeamVersion
-		if beamVer < 1 {
-			beamVer = 1
-		}
-		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, beamVer, idx, &exp, db, jobLog)
+		err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, data.BeamVersion, idx, &exp, db, jobLog)
 		if err != nil {
 			return fmt.Errorf("Failed to import beam locations for image %v into DB. Error: %v", imgItem.Image, err)
 		}

api/dataimport/test-data/ManualEM_Beamv2_OK/config-bucket/DatasetConfig/StandardPseudoIntensities.csv

@@ -0,0 +1,23 @@
+Name, StartChannel, EndChannel
+Na, 122, 142
+Mg, 148, 170
+Al, 176, 200
+Si, 207, 233
+P, 241, 268
+S, 277, 306
+Cl, 315, 346
+K, 400, 435
+Ca, 446, 483
+Ti, 547, 588
+Ce, 588, 630
+Cr, 658, 703
+Mn, 718, 765
+Fe, 780, 829
+Ni, 913, 966
+Ge, 1211, 1271
+As, 1293, 1355
+Zn, 1057, 1113
+Sr, 1742, 1814
+Y, 1840, 1914
+Zr, 1938, 2014
+Ba, 542, 582

api/dataimport/test-data/ManualEM_Beamv2_OK/dataset-bucket/.gitignore

@@ -0,0 +1,2 @@
+Scans/
+Images/

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/creator.json

@@ -0,0 +1,5 @@
+{
+    "name": "peternemere",
+    "user_id": "peter123",
+    "email": "[email protected]"
+}

api/dataimport/test-data/ManualEM_Beamv2_OK/manual-bucket/UploadedDatasets/048300551/detector.json

@@ -0,0 +1,3 @@
+{
+    "detector": "pixl-em"
+}

api/ws/handlers/image-beam-location.go

@@ -21,7 +21,10 @@ func HandleImageBeamLocationsReq(req *protos.ImageBeamLocationsReq, hctx wsHelpe
 	// If we have generateForScanId set, we don't want to have an image name!
 	if len(req.GenerateForScanId) > 0 {
 		if len(req.ImageName) > 0 {
-			return nil, errorwithstatus.MakeBadRequestError(errors.New("Expected empty image name for request with GenerateForScanId set"))
+			return nil, errorwithstatus.MakeBadRequestError(errors.New("Expected empty imageName for request with GenerateForScanId set"))
+		}
+		if len(req.ScanBeamVersions) > 0 {
+			return nil, errorwithstatus.MakeBadRequestError(errors.New("Expected empty scanBeamVersions for request with GenerateForScanId set"))
 		}
 
 		// Check user has access to this scan
@@ -68,17 +71,70 @@ func HandleImageBeamLocationsReq(req *protos.ImageBeamLocationsReq, hctx wsHelpe
 			return nil, result.Err()
 		}
 
-		err := result.Decode(&locs)
+		var dbLocs *protos.ImageLocations
+		err := result.Decode(&dbLocs)
 		if err != nil {
 			return nil, err
 		}
 
-		if len(locs.LocationPerScan) <= 0 {
+		if len(dbLocs.LocationPerScan) <= 0 {
 			return nil, fmt.Errorf("No beams defined for image: %v", req.ImageName)
 		}
 
-		for _, scanLocs := range locs.LocationPerScan {
-			_, err := wsHelpers.CheckObjectAccess(false, scanLocs.ScanId, protos.ObjectType_OT_SCAN, hctx)
+		// Build list of unique scans so we don't run the object access check
+		dbScanIds := map[string]bool{}
+		for _, scanLocs := range dbLocs.LocationPerScan {
+			dbScanIds[scanLocs.ScanId] = true
+		}
+		scanBeamVersionsToReturn := req.ScanBeamVersions
+		if scanBeamVersionsToReturn == nil {
+			scanBeamVersionsToReturn = map[string]uint32{}
+		}
+
+		// If they didn't specify versions to return, return the latest version for each scan represented
+		if len(scanBeamVersionsToReturn) <= 0 {
+			for _, scanLocs := range dbLocs.LocationPerScan {
+				// Add to map if it doesn't exist yet
+				if ver, ok := scanBeamVersionsToReturn[scanLocs.ScanId]; !ok {
+					scanBeamVersionsToReturn[scanLocs.ScanId] = scanLocs.BeamVersion
+				} else {
+					// Check if this beam version is larger
+					if scanLocs.BeamVersion > ver {
+						scanBeamVersionsToReturn[scanLocs.ScanId] = scanLocs.BeamVersion
+					}
+				}
+			}
+		}
+
+		// Run through what we're planning to return and make sure user has access while building the result list
+		locs = &protos.ImageLocations{
+			ImageName:       dbLocs.ImageName,
+			LocationPerScan: []*protos.ImageLocationsForScan{},
+		}
+
+		// Return the specified scan/beam versions
+		for scan, ver := range scanBeamVersionsToReturn {
+			if !dbScanIds[scan] {
+				return nil, fmt.Errorf("No beams defined for image: %v and scan: %v", req.ImageName, scan)
+			}
+
+			// This is a valid scan choice, now make sure the version requested exists
+			verFound := false
+			for _, scanLocs := range dbLocs.LocationPerScan {
+				if scanLocs.ScanId == scan && scanLocs.BeamVersion == ver {
+					verFound = true
+					locs.LocationPerScan = append(locs.LocationPerScan, scanLocs)
+					break
+				}
+			}
+
+			if !verFound {
+				return nil, fmt.Errorf("No beams defined for image: %v and scan: %v with version: %v", req.ImageName, scan, ver)
+			}
+		}
+
+		for scanId, _ := range scanBeamVersionsToReturn {
+			_, err := wsHelpers.CheckObjectAccess(false, scanId, protos.ObjectType_OT_SCAN, hctx)
 			if err != nil {
 				return nil, err
 			}
@@ -141,3 +197,46 @@ func generateIJs(imageName string, scanId string, instrument protos.ScanInstrume
 
 	return &locs, nil
 }
+
+func HandleImageBeamLocationVersionsReq(req *protos.ImageBeamLocationVersionsReq, hctx wsHelpers.HandlerContext) (*protos.ImageBeamLocationVersionsResp, error) {
+	ctx := context.TODO()
+
+	if err := wsHelpers.CheckStringField(&req.ImageName, "ImageName", 1, 255); err != nil {
+		return nil, err
+	}
+
+	coll := hctx.Svcs.MongoDB.Collection(dbCollections.ImageBeamLocationsName)
+	vers := map[string]*protos.ImageBeamLocationVersionsResp_AvailableVersions{}
+
+	// Read the image and check that the user has access to all scans associated with it
+	result := coll.FindOne(ctx, bson.M{"_id": req.ImageName})
+	if result.Err() != nil {
+		if result.Err() == mongo.ErrNoDocuments {
+			// If there are no beam locations, don't return an error, just return a message with no items in it
+			return &protos.ImageBeamLocationVersionsResp{
+				BeamVersionPerScan: vers,
+			}, nil
+		}
+		return nil, result.Err()
+	}
+
+	var locs *protos.ImageLocations
+	err := result.Decode(&locs)
+	if err != nil {
+		return nil, err
+	}
+
+	for _, locPerScan := range locs.LocationPerScan {
+		var availVersions *protos.ImageBeamLocationVersionsResp_AvailableVersions = vers[locPerScan.ScanId]
+		if availVersions == nil {
+			availVersions = &protos.ImageBeamLocationVersionsResp_AvailableVersions{Versions: []uint32{}}
+			vers[locPerScan.ScanId] = availVersions
+		}
+
+		availVersions.Versions = append(availVersions.Versions, locPerScan.BeamVersion)
+	}
+
+	return &protos.ImageBeamLocationVersionsResp{
+		BeamVersionPerScan: vers,
+	}, nil
+}

api/ws/handlers/job.go

@@ -0,0 +1,11 @@
+package wsHandler
+
+import (
+	"errors"
+	protos "github.com/pixlise/core/v4/generated-protos"
+	"github.com/pixlise/core/v4/api/ws/wsHelpers"
+)
+
+func HandleJobListReq(req *protos.JobListReq, hctx wsHelpers.HandlerContext) (*protos.JobListResp, error) {
+    return nil, errors.New("HandleJobListReq not implemented yet")
+}

api/ws/handlers/piquant.go

@@ -103,7 +103,7 @@ func HandlePiquantWriteCurrentVersionReq(req *protos.PiquantWriteCurrentVersionR
 		return nil, err
 	}
 
-	if result.MatchedCount != 1 {
+	if result.MatchedCount != 1 && result.UpsertedCount != 1 {
 		hctx.Svcs.Log.Errorf("PiquantWriteCurrentVersionReq UpdateByID result had unexpected counts %+v", result)
 	}