diff --git a/api/coreg/import_test.go b/api/coreg/import_test.go index 3a18d29e..85fd0429 100644 --- a/api/coreg/import_test.go +++ b/api/coreg/import_test.go @@ -1,10 +1,10 @@ package coreg import ( - "encoding/json" "fmt" "log" + "github.com/pixlise/core/v4/core/utils" protos "github.com/pixlise/core/v4/generated-protos" "google.golang.org/protobuf/encoding/protojson" ) @@ -16,16 +16,7 @@ func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) { log.Fatalln(err) } else { // Proto isn't deterministic, so we read as JSON and write it back out again - var anyJson map[string]interface{} - err = json.Unmarshal(b, &anyJson) - if err != nil { - log.Fatalln(err) - } - b, err = json.MarshalIndent(anyJson, "", " ") - if err != nil { - log.Fatalln(err) - } - fmt.Printf("%v\n", string(b)) + fmt.Printf("%v\n", utils.MakeDeterministicJSON(b, false)) } } diff --git a/api/dataimport/for-trigger_test.go b/api/dataimport/for-trigger_test.go index 784933e2..81d49531 100644 --- a/api/dataimport/for-trigger_test.go +++ b/api/dataimport/for-trigger_test.go @@ -20,6 +20,7 @@ package dataimport import ( "context" "fmt" + "log" "os" "strings" @@ -28,6 +29,7 @@ import ( "github.com/pixlise/core/v4/core/fileaccess" "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/scan" + "github.com/pixlise/core/v4/core/utils" "github.com/pixlise/core/v4/core/wstestlib" protos "github.com/pixlise/core/v4/generated-protos" "go.mongodb.org/mongo-driver/mongo" @@ -205,7 +207,7 @@ func Example_importForTrigger_OCS_Archive_Exists() { // Log shows exists in archive: true } -func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database) { +func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Database) { // Ensure these log msgs appeared... requiredLogs := []string{ "Downloading archived zip files...", @@ -221,7 +223,7 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database } for _, msg := range requiredLogs { - fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg)) + fmt.Printf("Logged \"%v\": %v\n", msg, logger.LogContains(msg)) } // Dump contents of summary file, this verifies most things imported as expected @@ -234,8 +236,10 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database summary.TimestampUnixSec = 0 b, err := protojson.Marshal(summary) - s := strings.ReplaceAll(string(b), " ", "") - fmt.Printf("%v|%v\n", err, s) + if err != nil { + log.Fatalf("%v", err) + } + fmt.Printf("%v|%v\n", err, utils.MakeDeterministicJSON(b, true)) } // Import FM-style (simulate trigger by OCS pipeline), file goes to archive, then all files downloaded from archive and dataset created @@ -301,7 +305,7 @@ func Example_importForTrigger_OCS_Archive_OK() { // Logged "Diffraction db saved successfully": true // Logged "Applying custom title: Naltsos": true // Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true - // |{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"} } // Import FM-style (simulate trigger by dataset edit screen), should create dataset with custom name+image @@ -354,7 +358,7 @@ func Example_importForTrigger_OCS_DatasetEdit() { // Logged "Diffraction db saved successfully": true // Logged "Applying custom title: Naltsos": true // Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true - // |{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"} } func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) { @@ -423,7 +427,7 @@ func Example_importForTrigger_Manual_JPL() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": true // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"} + // |{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"} } // Import a breadboard dataset from manual uploaded zip file @@ -456,7 +460,7 @@ func Example_importForTrigger_Manual_SBU() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} + // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } // Import a breadboard dataset from manual uploaded zip file @@ -489,7 +493,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": true - // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} + // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } /* Didnt get this working when the above was changed. Problem is this still generates the user name: SBUImport, so the @@ -540,7 +544,7 @@ func Example_importForTrigger_Manual_EM() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} } /* NOT TESTED YET, because it's not done yet! diff --git a/api/dataimport/internal/output/summary.go b/api/dataimport/internal/output/summary.go index 1f155d1f..10721293 100644 --- a/api/dataimport/internal/output/summary.go +++ b/api/dataimport/internal/output/summary.go @@ -67,12 +67,12 @@ func makeSummaryFileContent( } saveMeta := map[string]string{ - "TargetID": exp.TargetId, - "SiteID": fmt.Sprintf("%v", meta.SiteID), - "DriveID": fmt.Sprintf("%v", meta.DriveID), + "TargetId": exp.TargetId, + "SiteId": fmt.Sprintf("%v", meta.SiteID), + "DriveId": fmt.Sprintf("%v", meta.DriveID), "Target": meta.Target, "Site": meta.Site, - "SOL": meta.SOL, + "Sol": meta.SOL, "RTT": meta.RTT, "SCLK": fmt.Sprintf("%v", meta.SCLK), } diff --git a/core/utils/jsonprint.go b/core/utils/jsonprint.go new file mode 100644 index 00000000..1e3cff47 --- /dev/null +++ b/core/utils/jsonprint.go @@ -0,0 +1,29 @@ +package utils + +import ( + "encoding/json" + "log" + "strings" +) + +func MakeDeterministicJSON(b []byte, flat bool) string { + var anyJson map[string]interface{} + err := json.Unmarshal(b, &anyJson) + if err != nil { + log.Fatalln(err) + } + indent := "" + if !flat { + indent = " " + } + b2, err := json.MarshalIndent(anyJson, "", indent) + if err != nil { + log.Fatalln(err) + } + + result := string(b2) + if flat { + result = strings.ReplaceAll(result, "\n", "") + } + return result +}