Merge pull request #344 from pixlise/feature/em-import

Fixed PIXL EM importer support in lambda function. Shoud be able to r…

api/dataimport/dataimportHelpers/beamLocation.go

@@ -18,8 +18,10 @@
 package dataImportHelpers
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
+	"os"
 	"slices"
 	"strconv"
 	"strings"
@@ -29,14 +31,35 @@ import (
 )
 
 // ReadBeamLocationsFile - Reads beam location CSV. Old style (expectMultipleIJ=false) or new multi-image IJ coord CSVs.
-func ReadBeamLocationsFile(path string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) {
-	rowsToSkip := 0
-	if !expectMultipleIJ {
-		// If we're loading the old style test data, that had an extra header that we skip
-		rowsToSkip = 1
+func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) {
+	// Find the first row that has the start of data we're interested in!
+	file, err := os.Open(beamPath)
+	if err != nil {
+		return nil, err
+	}
+	defer file.Close()
+
+	scanner := bufio.NewScanner(file)
+
+	lineNo := 0
+	for scanner.Scan() {
+		line := scanner.Text()
+		lineNo++
+
+		if strings.HasPrefix(line, "PMC,") {
+			break
+		}
+
+		if lineNo > 4 {
+			return nil, fmt.Errorf("Failed to find header row of beam location data")
+		}
 	}
+
 	// read CSV
-	rows, err := ReadCSV(path, rowsToSkip, ',', jobLog)
+	if lineNo > 0 {
+		lineNo--
+	}
+	rows, err := ReadCSV(beamPath, lineNo, ',', jobLog)
 	if err != nil {
 		return nil, err
 	}

api/dataimport/dataimportHelpers/readcsv.go

@@ -16,6 +16,7 @@ func ReadCSV(filePath string, headerIdx int, sep rune, jobLog logger.ILogger) ([
 		return nil, err
 	}
 
+	seekPos := int64(0)
 	if headerIdx > 0 {
 		n := 0
 		for n < headerIdx {
@@ -24,7 +25,8 @@ func ReadCSV(filePath string, headerIdx int, sep rune, jobLog logger.ILogger) ([
 			if err != nil {
 				return nil, err
 			}
-			_, err = csvFile.Seek(int64(len(row1)), io.SeekStart)
+			seekPos += int64(len(row1))
+			_, err = csvFile.Seek(seekPos, io.SeekStart)
 			if err != nil {
 				return nil, err
 			}

api/dataimport/internal/converters/jplbreadboard/contextImages.go

@@ -18,10 +18,7 @@
 package jplbreadboard
 
 import (
-	"path/filepath"
-	"strconv"
-	"strings"
-
+	"github.com/pixlise/core/v4/api/dataimport/internal/importerutils"
 	"github.com/pixlise/core/v4/core/fileaccess"
 	"github.com/pixlise/core/v4/core/logger"
 )
@@ -35,30 +32,5 @@ func processContextImages(path string, jobLog logger.ILogger, fs fileaccess.File
 		return nil, err
 	}
 
-	return getContextImagesPerPMCFromListing(contextImgDirFiles, jobLog), nil
-}
-
-func getContextImagesPerPMCFromListing(paths []string, jobLog logger.ILogger) map[int32]string {
-	result := make(map[int32]string)
-
-	for _, pathitem := range paths {
-		_, file := filepath.Split(pathitem)
-		extension := filepath.Ext(file)
-		if extension == ".jpg" {
-			fileNameBits := strings.Split(file, "_")
-			if len(fileNameBits) != 3 {
-				jobLog.Infof("Ignored unexpected image file name \"%v\" when searching for context images.", pathitem)
-			} else {
-				pmcStr := fileNameBits[len(fileNameBits)-1]
-				pmcStr = pmcStr[0 : len(pmcStr)-len(extension)]
-				pmcI, err := strconv.Atoi(pmcStr)
-				if err != nil {
-					jobLog.Infof("Ignored unexpected image file name \"%v\", couldn't parse PMC.", pathitem)
-				} else {
-					result[int32(pmcI)] = file
-				}
-			}
-		}
-	}
-	return result
+	return importerutils.GetContextImagesPerPMCFromListing(contextImgDirFiles, jobLog), nil
 }

api/dataimport/internal/converters/jplbreadboard/import.go

@@ -93,7 +93,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,
 		}
 	}
 
-	minContextPMC := getMinimumContextPMC(contextImgsPerPMC)
+	minContextPMC := importerutils.GetMinimumContextPMC(contextImgsPerPMC)
 
 	var hkData dataConvertModels.HousekeepingData
 	var beamLookup = make(dataConvertModels.BeamLocationByPMC)
@@ -153,7 +153,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,
 	spectrafiles, _ := getSpectraFiles(allMSAFiles, verifyreadtype, jobLog)
 
 	jobLog.Infof("  Found %v usable spectrum files...", len(allMSAFiles))
-	spectraLookup, err := makeSpectraLookup(spectraPath, spectrafiles, params.SingleDetectorMSAs, params.GenPMCs, params.ReadTypeOverride, params.DetectorADuplicate, jobLog)
+	spectraLookup, err := MakeSpectraLookup(spectraPath, spectrafiles, params.SingleDetectorMSAs, params.GenPMCs, params.ReadTypeOverride, params.DetectorADuplicate, jobLog)
 	if err != nil {
 		return nil, "", err
 	}
@@ -236,19 +236,3 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,
 	data.SetPMCData(beamLookup, hkData, spectraLookup, contextImgsPerPMC, pseudoIntensityData, map[int32]string{})
 	return data, contextImageSrcDir, nil
 }
-
-// Check what the minimum PMC is we have a context image for
-func getMinimumContextPMC(contextImgsPerPMC map[int32]string) int32 {
-	minContextPMC := int32(0)
-
-	for contextPMC := range contextImgsPerPMC {
-		if minContextPMC == 0 || contextPMC < minContextPMC {
-			minContextPMC = contextPMC
-		}
-	}
-	if minContextPMC == 0 {
-		minContextPMC = 1
-	}
-
-	return minContextPMC
-}

api/dataimport/internal/converters/jplbreadboard/spectra.go

@@ -122,7 +122,7 @@ func getMSASeqNo(path string) (int64, error) {
 	return int64(seqNo), nil
 }
 
-func makeSpectraLookup(inputpath string, spectraFiles []string, singleDetectorMSAs bool, genPMCs bool, readTypeOverride string, detectorADuplicate bool, jobLog logger.ILogger) (dataConvertModels.DetectorSampleByPMC, error) {
+func MakeSpectraLookup(inputpath string, spectraFiles []string, singleDetectorMSAs bool, genPMCs bool, readTypeOverride string, detectorADuplicate bool, jobLog logger.ILogger) (dataConvertModels.DetectorSampleByPMC, error) {
 	spectraLookup := make(dataConvertModels.DetectorSampleByPMC)
 
 	reportInterval := len(spectraFiles) / 10
@@ -168,7 +168,7 @@ func makeSpectraLookup(inputpath string, spectraFiles []string, singleDetectorMS
 				return spectraLookup, fmt.Errorf("Unexpected SOURCEFILE metadata already defined in %v", path)
 			}
 
-			s.Meta["SOURCEFILE"] = dataConvertModels.StringMetaValue(f)
+			s.Meta["SOURCEFILE"] = dataConvertModels.StringMetaValue(filepath.Base(f))
 
 			// Use the override if it's provided
 			rt := readTypeOverride