EM IMPORT: Exclude images that have no beam locations, fix SOL to be …

…current date

api/dataimport/dataimportHelpers/beamLocation.go

@@ -31,11 +31,11 @@ import (
 )
 
 // ReadBeamLocationsFile - Reads beam location CSV. Old style (expectMultipleIJ=false) or new multi-image IJ coord CSVs.
-func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) {
+func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, []int32, error) {
 	// Find the first row that has the start of data we're interested in!
 	file, err := os.Open(beamPath)
 	if err != nil {
-		return nil, err
+		return nil, []int32{}, err
 	}
 	defer file.Close()
 
@@ -51,7 +51,7 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
 		}
 
 		if lineNo > 4 {
-			return nil, fmt.Errorf("Failed to find header row of beam location data")
+			return nil, []int32{}, fmt.Errorf("Failed to find header row of beam location data")
 		}
 	}
 
@@ -61,16 +61,17 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
 	}
 	rows, err := ReadCSV(beamPath, lineNo, ',', jobLog)
 	if err != nil {
-		return nil, err
+		return nil, []int32{}, err
 	}
 
 	return parseBeamLocations(rows, expectMultipleIJ, mainImagePMC, ignoreColumns)
 }
 
-func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, error) {
+func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, []int32, error) {
+	pmcs := []int32{}
 	headerLookup, geom_corrIdx, err := parseBeamLocationHeaders(rows[0], expectMultipleIJ, mainImagePMC, ignoreColumns)
 	if err != nil {
-		return nil, err
+		return nil, pmcs, err
 	}
 
 	// Read in each row and store based on the header lookup we made
@@ -79,16 +80,21 @@ func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int
 	for line, row := range rows[1:] {
 		pmc, locData, err := parseBeamLocationRow(row, headerLookup, geom_corrIdx)
 		if err != nil {
-			return nil, fmt.Errorf("line [%v] - ERROR: %v", line, err)
+			return nil, pmcs, fmt.Errorf("line [%v] - ERROR: %v", line, err)
 		}
 		if _, ok := result[pmc]; ok {
-			return nil, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
+			return nil, pmcs, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
 		}
 
 		result[pmc] = locData
 	}
 
-	return result, nil
+	// Also save a list of PMCs who had ijs
+	for _, item := range headerLookup {
+		pmcs = append(pmcs, item.pmc)
+	}
+
+	return result, pmcs, nil
 }
 
 type pmcColIdxs struct {

api/dataimport/dataimportHelpers/beamLocation_test.go

@@ -127,53 +127,56 @@ func Example_parseBeamLocationRow() {
 }
 
 func Example_parseBeamLocations() {
-	data, err := parseBeamLocations([][]string{[]string{"PMC", "x", "y", "z", "image_i", "image_j"}, []string{"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
-	fmt.Printf("%v|%v\n", data, err)
+	data, pmcs, err := parseBeamLocations([][]string{{"PMC", "x", "y", "z", "image_i", "image_j"}, {"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
+	fmt.Printf("%v|%v|%v\n", data, pmcs, err)
 
-	data, err = parseBeamLocations([][]string{
-		[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
-		[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
-		[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
+	data, pmcs, err = parseBeamLocations([][]string{
+		{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
+		{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
+		{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
 	}, true, 333, []string{})
-	fmt.Printf("%v|%v\n", data, err)
+	fmt.Printf("%v|%v|%v\n", data, pmcs, err)
 
-	data, err = parseBeamLocations([][]string{
-		[]string{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
-		[]string{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
-		[]string{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
+	data, pmcs, err = parseBeamLocations([][]string{
+		{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
+		{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
+		{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
 	}, true, 333, []string{})
-	fmt.Printf("%v|%v\n", data, err)
+	fmt.Printf("%v|%v|%v\n", data, pmcs, err)
 
-	data, err = parseBeamLocations([][]string{
-		[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
-		[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
-		[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
+	data, pmcs, err = parseBeamLocations([][]string{
+		{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
+		{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
+		{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
 	}, true, 333, []string{})
-	fmt.Printf("%v|%v\n", data, err)
+	fmt.Printf("%v|%v|%v\n", data, pmcs, err)
 
 	// Output:
-	// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|<nil>
-	// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
-	// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
-	// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
+	// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|[222]|<nil>
+	// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
+	// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
+	// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
 }
 
 // This is kind of redunant, was already tested elsewhere, but this is an easy point to add a file and run the test to make sure it imports!
 func Example_ReadBeamLocationsFile() {
-	rxl, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
-	fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
+	rxl, pmcs, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
+	fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))
 
-	rxl, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
-	fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
+	rxl, pmcs, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
+	fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))
 
-	rxl, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
-	fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
+	rxl, pmcs, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
+	fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))
 
 	// Output:
 	// <nil>
+	// pmcs: [4 457 5 458]
 	// rxl: 3
 	// <nil>
+	// pmcs: [4 457 5 458]
 	// rxl: 3
 	// Unexpected count of i/j columns
+	// pmcs: []
 	// rxl: 0
 }

api/dataimport/internal/converters/jplbreadboard/import.go

@@ -100,7 +100,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,
 
 	if params.MsaBeamParams == "" && params.BeamFile != "" {
 		jobLog.Infof("  Reading Beam Locations: \"%v\", using minimum context image PMC detected: %v\n", params.BeamFile, minContextPMC)
-		beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
+		beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
 		if err != nil {
 			return nil, "", err
 		}

api/dataimport/internal/converters/pixlem/import.go

@@ -23,6 +23,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"time"
 
 	dataImportHelpers "github.com/pixlise/core/v4/api/dataimport/dataimportHelpers"
 	"github.com/pixlise/core/v4/api/dataimport/internal/converters/jplbreadboard"
@@ -214,14 +215,14 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin
 	minContextPMC := importerutils.GetMinimumContextPMC(contextImgsPerPMC)
 
 	// Read Beams
-	beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
+	beamLookup, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
 	if err != nil {
 		return nil, err
 	}
 
 	// Remove any images which don't have beam locations
 	for pmc, img := range contextImgsPerPMC {
-		if _, ok := beamLookup[pmc]; !ok {
+		if !utils.ItemInSlice(pmc, ijPMCs) {
 			logger.Infof("Excluding image due to not having beam locations: %v", img)
 			delete(contextImgsPerPMC, pmc)
 		}
@@ -240,7 +241,11 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin
 	site := "000"
 	drive := "0000"
 	product := "???"
-	sol := "D000"
+
+	// Use current date encoded as a test sol
+	// A=2017, 'A' is 65 ascii
+	sol := fmt.Sprintf("%v%v", string(65+time.Now().Year()-2017), time.Now().YearDay())
+
 	ftype := "??" // PE
 	producer := "J"
 	version := "01"

api/dataimport/internal/converters/pixlfm/import.go

@@ -218,7 +218,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
 				if beamCsvMeta.ProdType == "RXL" {
 					// If files don't conform, don't read...
 					beamFilePath := filepath.Join(pathToSubdir, file)
-					beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
+					beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
 					if err != nil {
 						return nil, "", err
 					} else {

api/dataimport/internal/converters/soff/import.go

@@ -148,7 +148,7 @@ func (s *SOFFImport) Import(importPath string, pseudoIntensityRangesPath string,
 	}
 
 	// Read each one
-	beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
+	beamLookup, _, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
 	if err != nil {
 		return nil, "", err
 	}

internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go

@@ -134,30 +134,16 @@ func main() {
 
 	// Find out what PMCs we have ij's for, and find the corresponding image file name to import for
 	// this way we can import into ImageBeamLocations using the file name, and insert an entry for v3
-	beamLocs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})
+	beamLocs, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})
 
 	if err != nil {
 		log.Fatalln(err)
 	}
 
-	for _, beam := range beamLocs {
-		// They should all be the same so only checking first one
-		// NOTE: Also ensure we don't have any images stored for PMCs that we don't have beam data for!
-		validPMCs := []int32{}
-		for imgPMC := range beam.IJ {
-			if _, ok := pmcImageLookup[imgPMC]; !ok {
-				log.Fatalf("Failed to find image for ij PMC: %v", imgPMC)
-			} else {
-				validPMCs = append(validPMCs, imgPMC)
-			}
-		}
-
-		for pmc := range pmcImageLookup {
-			if !utils.ItemInSlice(pmc, validPMCs) {
-				delete(pmcImageLookup, pmc)
-			}
+	for pmc := range pmcImageLookup {
+		if !utils.ItemInSlice(pmc, ijPMCs) {
+			delete(pmcImageLookup, pmc)
 		}
-		break
 	}
 
 	s3Path := fmt.Sprintf("Scans/%v/dataset.bin", scanId)