Skip to content

Commit

Permalink
Merge pull request #359 from pixlise/feature/em-import
Browse files Browse the repository at this point in the history 
EM IMPORT: Exclude images that have no beam locations, fix SOL to be …
  • Loading branch information
@pnemere
pnemere authored Nov 13, 2024
2 parents 74d0036 + 67fcbfb commit 7c66256
Show file tree
Hide file tree
Showing 7 changed files with 60 additions and 60 deletions.
24 changes: 15 additions & 9 deletions api/dataimport/dataimportHelpers/beamLocation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,11 @@ import (
)

// ReadBeamLocationsFile - Reads beam location CSV. Old style (expectMultipleIJ=false) or new multi-image IJ coord CSVs.
func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, error) {
func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string, jobLog logger.ILogger) (dataConvertModels.BeamLocationByPMC, []int32, error) {
// Find the first row that has the start of data we're interested in!
file, err := os.Open(beamPath)
if err != nil {
return nil, err
return nil, []int32{}, err
}
defer file.Close()

Expand All @@ -51,7 +51,7 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
}

if lineNo > 4 {
return nil, fmt.Errorf("Failed to find header row of beam location data")
return nil, []int32{}, fmt.Errorf("Failed to find header row of beam location data")
}
}

Expand All @@ -61,16 +61,17 @@ func ReadBeamLocationsFile(beamPath string, expectMultipleIJ bool, mainImagePMC
}
rows, err := ReadCSV(beamPath, lineNo, ',', jobLog)
if err != nil {
return nil, err
return nil, []int32{}, err
}

return parseBeamLocations(rows, expectMultipleIJ, mainImagePMC, ignoreColumns)
}

func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, error) {
func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int32, ignoreColumns []string) (dataConvertModels.BeamLocationByPMC, []int32, error) {
pmcs := []int32{}
headerLookup, geom_corrIdx, err := parseBeamLocationHeaders(rows[0], expectMultipleIJ, mainImagePMC, ignoreColumns)
if err != nil {
return nil, err
return nil, pmcs, err
}

// Read in each row and store based on the header lookup we made
Expand All @@ -79,16 +80,21 @@ func parseBeamLocations(rows [][]string, expectMultipleIJ bool, mainImagePMC int
for line, row := range rows[1:] {
pmc, locData, err := parseBeamLocationRow(row, headerLookup, geom_corrIdx)
if err != nil {
return nil, fmt.Errorf("line [%v] - ERROR: %v", line, err)
return nil, pmcs, fmt.Errorf("line [%v] - ERROR: %v", line, err)
}
if _, ok := result[pmc]; ok {
return nil, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
return nil, pmcs, fmt.Errorf("line [%v] - ERROR: duplicate PMC %v", line, pmc)
}

result[pmc] = locData
}

return result, nil
// Also save a list of PMCs who had ijs
for _, item := range headerLookup {
pmcs = append(pmcs, item.pmc)
}

return result, pmcs, nil
}

type pmcColIdxs struct {
Expand Down
57 changes: 30 additions & 27 deletions api/dataimport/dataimportHelpers/beamLocation_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,53 +127,56 @@ func Example_parseBeamLocationRow() {
}

func Example_parseBeamLocations() {
data, err := parseBeamLocations([][]string{[]string{"PMC", "x", "y", "z", "image_i", "image_j"}, []string{"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
fmt.Printf("%v|%v\n", data, err)
data, pmcs, err := parseBeamLocations([][]string{{"PMC", "x", "y", "z", "image_i", "image_j"}, {"33", "1.1", "1.2", "1.3", "55.1", "55.2"}}, false, 222, []string{})
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "geom_corr", "PMC_22_MCC_i", "PMC_22_MCC_j", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "1.03", "355.1", "355.2", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "0.99", "955.1", "955.2", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

data, err = parseBeamLocations([][]string{
[]string{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
[]string{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
[]string{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
data, pmcs, err = parseBeamLocations([][]string{
{"PMC", "x", "y", "z", "PMC_22_MCC_i", "PMC_22_MCC_j", "geom_corr", "PMC_62_MCC_i", "PMC_62_MCC_j"},
{"33", "31.1", "31.2", "31.3", "355.1", "355.2", "1.03", "3121.4", "3121.5"},
{"66", "91.1", "91.2", "91.3", "955.1", "955.2", "0.99", "9121.4", "9121.5"},
}, true, 333, []string{})
fmt.Printf("%v|%v\n", data, err)
fmt.Printf("%v|%v|%v\n", data, pmcs, err)

// Output:
// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|<nil>
// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|<nil>
// map[33:{1.1 1.2 1.3 0 map[222:{55.1 55.2}]}]|[222]|<nil>
// map[33:{31.1 31.2 31.3 0 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
// map[33:{31.1 31.2 31.3 1.03 map[22:{355.1 355.2} 62:{3121.4 3121.5}]} 66:{91.1 91.2 91.3 0.99 map[22:{955.1 955.2} 62:{9121.4 9121.5}]}]|[22 62]|<nil>
}

// This is kind of redunant, was already tested elsewhere, but this is an easy point to add a file and run the test to make sure it imports!
func Example_ReadBeamLocationsFile() {
rxl, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err := ReadBeamLocationsFile("./test-data/GeomCorrMoved.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

rxl, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err = ReadBeamLocationsFile("./test-data/GeomCorrAtExpected.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

rxl, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\nrxl: %v\n", err, len(rxl))
rxl, pmcs, err = ReadBeamLocationsFile("./test-data/MissingJ.CSV", true, 4, []string{}, &logger.StdOutLoggerForTest{})
fmt.Printf("%v\npmcs: %v\nrxl: %v\n", err, pmcs, len(rxl))

// Output:
// <nil>
// pmcs: [4 457 5 458]
// rxl: 3
// <nil>
// pmcs: [4 457 5 458]
// rxl: 3
// Unexpected count of i/j columns
// pmcs: []
// rxl: 0
}
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/jplbreadboard/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ func (m MSATestData) Import(importPath string, pseudoIntensityRangesPath string,

if params.MsaBeamParams == "" && params.BeamFile != "" {
jobLog.Infof(" Reading Beam Locations: \"%v\", using minimum context image PMC detected: %v\n", params.BeamFile, minContextPMC)
beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, params.BeamFile), false, minContextPMC, []string{}, jobLog)
if err != nil {
return nil, "", err
}
Expand Down
11 changes: 8 additions & 3 deletions api/dataimport/internal/converters/pixlem/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"path/filepath"
"strconv"
"strings"
"time"

dataImportHelpers "github.com/pixlise/core/v4/api/dataimport/dataimportHelpers"
"github.com/pixlise/core/v4/api/dataimport/internal/converters/jplbreadboard"
Expand Down Expand Up @@ -214,14 +215,14 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin
minContextPMC := importerutils.GetMinimumContextPMC(contextImgsPerPMC)

// Read Beams
beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
beamLookup, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(beamLocPath, true, minContextPMC, []string{"drift_x", "drift_y", "drift_z"}, logger)
if err != nil {
return nil, err
}

// Remove any images which don't have beam locations
for pmc, img := range contextImgsPerPMC {
if _, ok := beamLookup[pmc]; !ok {
if !utils.ItemInSlice(pmc, ijPMCs) {
logger.Infof("Excluding image due to not having beam locations: %v", img)
delete(contextImgsPerPMC, pmc)
}
Expand All @@ -240,7 +241,11 @@ func importEMData(creatorId string, rtt string, beamLocPath string, hkPath strin
site := "000"
drive := "0000"
product := "???"
sol := "D000"

// Use current date encoded as a test sol
// A=2017, 'A' is 65 ascii
sol := fmt.Sprintf("%v%v", string(65+time.Now().Year()-2017), time.Now().YearDay())

Check failure on line 247 in api/dataimport/internal/converters/pixlem/import.go

View workflow job for this annotation

GitHub Actions / prerelease / test 

conversion from int to string yields a string of one rune, not a string of digits (did you mean fmt.Sprint(x)?)

Check failure on line 247 in api/dataimport/internal/converters/pixlem/import.go

View workflow job for this annotation

GitHub Actions / release / test 

conversion from int to string yields a string of one rune, not a string of digits (did you mean fmt.Sprint(x)?)

ftype := "??" // PE
producer := "J"
version := "01"
Expand Down
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/pixlfm/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,7 +218,7 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data
if beamCsvMeta.ProdType == "RXL" {
// If files don't conform, don't read...
beamFilePath := filepath.Join(pathToSubdir, file)
beamLookup, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
beamLookup, _, err = dataImportHelpers.ReadBeamLocationsFile(beamFilePath, true, 1, []string{}, log)
if err != nil {
return nil, "", err
} else {
Expand Down
2 changes: 1 addition & 1 deletion api/dataimport/internal/converters/soff/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ func (s *SOFFImport) Import(importPath string, pseudoIntensityRangesPath string,
}

// Read each one
beamLookup, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
beamLookup, _, err := dataImportHelpers.ReadBeamLocationsFile(filepath.Join(importPath, importPathAndOffsets["Xray_beam_positions"].fileName), true, 1, []string{}, s.log)
if err != nil {
return nil, "", err
}
Expand Down
22 changes: 4 additions & 18 deletions internal/cmd-line-tools/beam-geom-v3-csv-importer/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,30 +134,16 @@ func main() {

// Find out what PMCs we have ij's for, and find the corresponding image file name to import for
// this way we can import into ImageBeamLocations using the file name, and insert an entry for v3
beamLocs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})
beamLocs, ijPMCs, err := dataImportHelpers.ReadBeamLocationsFile(fileName, true, 0, []string{"drift_x", "drift_y", "drift_z"}, &logger.StdOutLogger{})

if err != nil {
log.Fatalln(err)
}

for _, beam := range beamLocs {
// They should all be the same so only checking first one
// NOTE: Also ensure we don't have any images stored for PMCs that we don't have beam data for!
validPMCs := []int32{}
for imgPMC := range beam.IJ {
if _, ok := pmcImageLookup[imgPMC]; !ok {
log.Fatalf("Failed to find image for ij PMC: %v", imgPMC)
} else {
validPMCs = append(validPMCs, imgPMC)
}
}

for pmc := range pmcImageLookup {
if !utils.ItemInSlice(pmc, validPMCs) {
delete(pmcImageLookup, pmc)
}
for pmc := range pmcImageLookup {
if !utils.ItemInSlice(pmc, ijPMCs) {
delete(pmcImageLookup, pmc)
}
break
}

s3Path := fmt.Sprintf("Scans/%v/dataset.bin", scanId)
Expand Down

0 comments on commit 7c66256

Please sign in to comment.