Skip to content

Commit

Permalink
Merge pull request #236 from pixlise/development
Browse files Browse the repository at this point in the history
Releasing variogram and bug fixes
  • Loading branch information
pnemere authored May 24, 2024
2 parents c821d86 + 1f862a6 commit 5c1e502
Show file tree
Hide file tree
Showing 95 changed files with 1,418 additions and 1,110 deletions.
2 changes: 2 additions & 0 deletions api/dataimport/for-trigger_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -234,6 +234,7 @@ func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Datab
}
// Clear the time stamp so it doesn't change next time we run test
summary.TimestampUnixSec = 0
summary.CompleteTimeStampUnixSec = 0

b, err := protojson.Marshal(summary)
if err != nil {
Expand Down Expand Up @@ -390,6 +391,7 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
} else {
// Clear the time stamp so it doesn't change next time we run test
summary.TimestampUnixSec = 0
summary.CompleteTimeStampUnixSec = 0
}

b, err := protojson.Marshal(summary)
Expand Down
9 changes: 8 additions & 1 deletion api/dataimport/internal/output/output.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ import (
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/gdsfilename"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/scan"
"github.com/pixlise/core/v4/core/utils"
protos "github.com/pixlise/core/v4/generated-protos"
"go.mongodb.org/mongo-driver/bson"
Expand Down Expand Up @@ -361,7 +362,13 @@ func (s *PIXLISEDataSaver) Save(
return fmt.Errorf("Failed to get dataset file size for: %v", outFilePath)
}

summaryData := makeSummaryFileContent(&exp, data.DatasetID, data.Instrument, data.Meta, int(fi.Size()), creationUnixTimeSec, data.CreatorUserId)
var prevSavedScan *protos.ScanItem
if prevSavedScan, err = scan.ReadScanItem(data.DatasetID, db); err != nil {
// NOTE: this is largely paranoia, in case ReadScanItem changes and returns errors even on read
prevSavedScan = nil
}

summaryData := makeSummaryFileContent(&exp, prevSavedScan, data.DatasetID, data.Instrument, data.Meta /*int(fi.Size()),*/, creationUnixTimeSec, data.CreatorUserId)

jobLog.Infof("Writing summary to DB for %v...", summaryData.Id)

Expand Down
59 changes: 48 additions & 11 deletions api/dataimport/internal/output/summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ import (

func makeSummaryFileContent(
exp *protos.Experiment,
prevSavedScan *protos.ScanItem,
datasetID string,
sourceInstrument protos.ScanInstrument,
meta dataConvertModels.FileMetaData,
fileSize int,
//fileSize int,
creationUnixTimeSec int64,
creatorUserId string) *protos.ScanItem {
contextImgCount := len(exp.AlignedContextImages) + len(exp.UnalignedContextImages) + len(exp.MatchedAlignedContextImages)
Expand Down Expand Up @@ -93,16 +94,52 @@ func makeSummaryFileContent(
// what it's displaying

s := &protos.ScanItem{
Id: datasetID,
Title: meta.Title,
Description: "",
DataTypes: dataTypes,
Instrument: sourceInstrument,
InstrumentConfig: exp.DetectorConfig,
TimestampUnixSec: uint32(creationUnixTimeSec),
Meta: saveMeta,
ContentCounts: contentCounts,
CreatorUserId: creatorUserId,
Id: datasetID,
Title: meta.Title,
Description: "",
DataTypes: dataTypes,
Instrument: sourceInstrument,
InstrumentConfig: exp.DetectorConfig,
TimestampUnixSec: uint32(creationUnixTimeSec),
Meta: saveMeta,
ContentCounts: contentCounts,
CreatorUserId: creatorUserId,
PreviousImportTimesUnixSec: []uint32{},
}

// If we've got a previously stored ScanItem, we are updating it, so read its time stamp into the array of previous time stamps
isComplete := exp.NormalSpectra == exp.PseudoIntensities*2

if prevSavedScan != nil {
// Build the list of previous import times
// Preserve the previous list...
s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.PreviousImportTimesUnixSec...)

// Add new time at the end
s.PreviousImportTimesUnixSec = append(s.PreviousImportTimesUnixSec, prevSavedScan.TimestampUnixSec)
}

// Save a time stamp for completion
if isComplete {
// If previous scan was also complete, DON'T update the time stamp, just preserve it
if prevSavedScan != nil && prevSavedScan.CompleteTimeStampUnixSec > 0 {
s.CompleteTimeStampUnixSec = prevSavedScan.CompleteTimeStampUnixSec
} else {
// We must've just completed now, so save the time
s.CompleteTimeStampUnixSec = uint32(creationUnixTimeSec)
}
}

// Preserve user-editable fields
if prevSavedScan != nil {
s.Tags = prevSavedScan.Tags
s.Description = prevSavedScan.Description

if len(prevSavedScan.Description) > 0 {
// User has entered a description so perserve the title too
s.Title = prevSavedScan.Title
}
}

return s
}
39 changes: 32 additions & 7 deletions api/quantification/autoQuantification.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,12 @@ package quantification
import (
"context"
"fmt"
"strconv"

"github.com/pixlise/core/v4/api/dbCollections"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/api/specialUserIds"
"github.com/pixlise/core/v4/api/ws/wsHelpers"
protos "github.com/pixlise/core/v4/generated-protos"
"go.mongodb.org/mongo-driver/bson"
"go.mongodb.org/mongo-driver/mongo/options"
Expand All @@ -23,11 +25,14 @@ func RunAutoQuantifications(scanId string, svcs *services.APIServices) {
quantNames := []string{"AutoQuant-PDS", "AutoQuant-PIXL"}
quantModes := []string{quantModeCombinedAB, quantModeSeparateAB}
quantElements := [][]string{
// PDS
[]string{"Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "Br"},
[]string{"Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "GeO", "Br", "Rb2O", "SrO", "Y2O3", "ZrO2"},
// PDS: intended "Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "Br"
// But we must specify elements only! Expecting PIQUANT to determine the oxide states to write
[]string{"Na", "Mg", "Al", "Si", "P", "S", "Cl", "K", "Ca", "Ti", "Cr", "Mn", "Fe", "Ni", "Zn", "Br"},
// PIXL: intended "Na2O", "MgO", "Al2O3", "SiO2", "P2O5", "SO3", "Cl", "K2O", "CaO", "TiO2", "Cr2O3", "MnO", "FeO-T", "NiO", "ZnO", "GeO", "Br", "Rb2O", "SrO", "Y2O3", "ZrO2"
// But we must specify elements only! Expecting PIQUANT to determine the oxide states to write
[]string{"Na", "Mg", "Al", "Si", "P", "S", "Cl", "K", "Ca", "Ti", "Cr", "Mn", "Fe", "Ni", "Zn", "Ge", "Br", "Rb", "Sr", "Y", "Zr"},
}
detector := "PIXL/v7"
detector := "PIXL/PiquantConfigs/v7"

allNames := []string{}
for _, name := range quantNames {
Expand All @@ -52,18 +57,38 @@ func RunAutoQuantifications(scanId string, svcs *services.APIServices) {
return
}

exprPB, err := wsHelpers.ReadDatasetFile(scanId, svcs)
if err != nil {
svcs.Log.Errorf("AutoQuant failed to read scan %v to determine PMC list: %v", scanId, err)
return
}

pmcs := []int32{}
for _, loc := range exprPB.Locations {
pmc, err := strconv.Atoi(loc.Id)
if err != nil {
svcs.Log.Errorf("AutoQuant: Failed to read PMC %v from scan %v. Skipping...", loc.Id, scanId)
continue
}

if len(loc.PseudoIntensities) > 0 && len(loc.PseudoIntensities[0].ElementIntensities) > 0 {
// We have quantifiable data for this
pmcs = append(pmcs, int32(pmc))
}
}

// Start all the quants
for c, name := range quantNames {
for _, m := range quantModes {
params := &protos.QuantCreateParams{
Command: "map",
Name: makeAutoQuantName(name, m),
ScanId: scanId,
Pmcs: []int32{},
Pmcs: pmcs,
Elements: quantElements[c],
DetectorConfig: detector,
Parameters: "",
RunTimeSec: 0,
Parameters: "-Fe,1",
RunTimeSec: 300,
QuantMode: m,
RoiIDs: []string{},
IncludeDwells: false,
Expand Down
11 changes: 11 additions & 0 deletions api/ws/handlers/scan.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ import (
"github.com/pixlise/core/v4/api/dbCollections"
"github.com/pixlise/core/v4/api/filepaths"
"github.com/pixlise/core/v4/api/job"
"github.com/pixlise/core/v4/api/quantification"
"github.com/pixlise/core/v4/api/services"
"github.com/pixlise/core/v4/api/ws/wsHelpers"
"github.com/pixlise/core/v4/core/errorwithstatus"
Expand Down Expand Up @@ -708,3 +709,13 @@ func HandleScanAutoShareWriteReq(req *protos.ScanAutoShareWriteReq, hctx wsHelpe

return &protos.ScanAutoShareWriteResp{}, nil
}

func HandleScanTriggerAutoQuantReq(req *protos.ScanTriggerAutoQuantReq, hctx wsHelpers.HandlerContext) (*protos.ScanTriggerAutoQuantResp, error) {
if err := wsHelpers.CheckStringField(&req.ScanId, "ScanId", 1, wsHelpers.IdFieldMaxLength); err != nil {
return nil, err
}

quantification.RunAutoQuantifications(req.ScanId, hctx.Svcs)

return &protos.ScanTriggerAutoQuantResp{}, nil
}
2 changes: 1 addition & 1 deletion api/ws/handlers/selection-entry.go
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ func writeSelection(id string, idxs *protos.ScanEntryRange, db *mongo.Database,
}

// Modified and Upsert counts will be 0 if the selection hasn't changed, so we just check matched
if dbResult.MatchedCount != 1 {
if dbResult.MatchedCount != 1 && dbResult.UpsertedCount != 1 {
logger.Errorf("writeSelection (%v) UpdateByID result had unexpected counts %+v", id, dbResult)
}

Expand Down
4 changes: 2 additions & 2 deletions generated-protos/detector-config-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/detector-config.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/diffraction-data.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/diffraction-detected-peak-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/diffraction-manual-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/diffraction-status-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/diffraction.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/doi-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/doi.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/element-set-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/element-set.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/experiment.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/export-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/export.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/expression-group-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/expression-group.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

4 changes: 2 additions & 2 deletions generated-protos/expression-msgs.pb.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 5c1e502

Please sign in to comment.