Skip to content

Commit

Permalink
Fixing example tests which rely on json output which was not determin…
Browse files Browse the repository at this point in the history 
…istic
  • Loading branch information
Peter Nemere committed Mar 27, 2024
1 parent 63abf40 commit 2ccd6ee
Show file tree
Hide file tree
Showing 3 changed files with 45 additions and 21 deletions.
13 changes: 2 additions & 11 deletions api/coreg/import_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package coreg

import (
"encoding/json"
"fmt"
"log"

"github.com/pixlise/core/v4/core/utils"
protos "github.com/pixlise/core/v4/generated-protos"
"google.golang.org/protobuf/encoding/protojson"
)
Expand All @@ -16,16 +16,7 @@ func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) {
log.Fatalln(err)
} else {
// Proto isn't deterministic, so we read as JSON and write it back out again
var anyJson map[string]interface{}
err = json.Unmarshal(b, &anyJson)
if err != nil {
log.Fatalln(err)
}
b, err = json.MarshalIndent(anyJson, "", " ")
if err != nil {
log.Fatalln(err)
}
fmt.Printf("%v\n", string(b))
fmt.Printf("%v\n", utils.MakeDeterministicJSON(b, false))
}
}

Expand Down
24 changes: 14 additions & 10 deletions api/dataimport/for-trigger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ package dataimport
import (
"context"
"fmt"
"log"
"os"
"strings"

Expand All @@ -28,6 +29,7 @@ import (
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/scan"
"github.com/pixlise/core/v4/core/utils"
"github.com/pixlise/core/v4/core/wstestlib"
protos "github.com/pixlise/core/v4/generated-protos"
"go.mongodb.org/mongo-driver/mongo"
Expand Down Expand Up @@ -205,7 +207,7 @@ func Example_importForTrigger_OCS_Archive_Exists() {
// Log shows exists in archive: true
}

func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database) {
func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Database) {
// Ensure these log msgs appeared...
requiredLogs := []string{
"Downloading archived zip files...",
Expand All @@ -221,7 +223,7 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database
}

for _, msg := range requiredLogs {
fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg))
fmt.Printf("Logged \"%v\": %v\n", msg, logger.LogContains(msg))
}

// Dump contents of summary file, this verifies most things imported as expected
Expand All @@ -234,8 +236,10 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database
summary.TimestampUnixSec = 0

b, err := protojson.Marshal(summary)
s := strings.ReplaceAll(string(b), " ", "")
fmt.Printf("%v|%v\n", err, s)
if err != nil {
log.Fatalf("%v", err)
}
fmt.Printf("%v|%v\n", err, utils.MakeDeterministicJSON(b, true))
}

// Import FM-style (simulate trigger by OCS pipeline), file goes to archive, then all files downloaded from archive and dataset created
Expand Down Expand Up @@ -301,7 +305,7 @@ func Example_importForTrigger_OCS_Archive_OK() {
// Logged "Diffraction db saved successfully": true
// Logged "Applying custom title: Naltsos": true
// Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true
// <nil>|{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
}

// Import FM-style (simulate trigger by dataset edit screen), should create dataset with custom name+image
Expand Down Expand Up @@ -354,7 +358,7 @@ func Example_importForTrigger_OCS_DatasetEdit() {
// Logged "Diffraction db saved successfully": true
// Logged "Applying custom title: Naltsos": true
// Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true
// <nil>|{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
}

func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) {
Expand Down Expand Up @@ -423,7 +427,7 @@ func Example_importForTrigger_Manual_JPL() {
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": true
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"}
// <nil>|{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"}
}

// Import a breadboard dataset from manual uploaded zip file
Expand Down Expand Up @@ -456,7 +460,7 @@ func Example_importForTrigger_Manual_SBU() {
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
}

// Import a breadboard dataset from manual uploaded zip file
Expand Down Expand Up @@ -489,7 +493,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() {
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": true
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
}

/* Didnt get this working when the above was changed. Problem is this still generates the user name: SBUImport, so the
Expand Down Expand Up @@ -540,7 +544,7 @@ func Example_importForTrigger_Manual_EM() {
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
}

/* NOT TESTED YET, because it's not done yet!
Expand Down
29 changes: 29 additions & 0 deletions core/utils/jsonprint.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,29 @@
package utils

import (
"encoding/json"
"log"
"strings"
)

func MakeDeterministicJSON(b []byte, flat bool) string {
var anyJson map[string]interface{}
err := json.Unmarshal(b, &anyJson)
if err != nil {
log.Fatalln(err)
}
indent := ""
if !flat {
indent = " "
}
b2, err := json.MarshalIndent(anyJson, "", indent)
if err != nil {
log.Fatalln(err)
}

result := string(b2)
if flat {
result = strings.ReplaceAll(result, "\n", "")
}
return result
}

0 comments on commit 2ccd6ee

Please sign in to comment.