Skip to content

Commit

Permalink
Merge pull request #203 from pixlise/development
Browse files Browse the repository at this point in the history 
v4 Patch
  • Loading branch information
@pnemere
pnemere authored Mar 27, 2024
2 parents a8df66e + ac82155 commit 70ae13c
Show file tree
Hide file tree
Showing 114 changed files with 589 additions and 352 deletions.
2 changes: 1 addition & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ build-linux:
echo "sha: ${GITHUB_SHA}"
GOOS=linux GOARCH=amd64 go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/pixlise-api-linux ./internal/api
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -o ./_out/bootstrap ./internal/lambdas/data-import
GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/bootstrap ./internal/lambdas/data-import
# GOOS=linux GOARCH=amd64 go build -v -o ./_out/importtest-linux ./internal/cmdline-tools/import-integration-test
# GOOS=linux GOARCH=amd64 go build -v -o ./_out/integrationtest-linux ./internal/cmdline-tools/api-integration-test

Expand Down
17 changes: 13 additions & 4 deletions api/coreg/import.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -257,10 +257,15 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp
return "", nil, nil, err
}

imgWidth := uint32(0)
imgHeight := uint32(0)

var img image.Image

if imageExt == ".IMG" {
w, h, d, err := imgFormat.ReadIMGFile(imgData)
imgWidth = uint32(w)
imgHeight = uint32(h)
if err != nil {
return "", nil, nil, fmt.Errorf("Failed to read IMG file: %v. Error: %v", imageUrl, err)
}
Expand All @@ -280,7 +285,7 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp

imgData = buf.Bytes()
} else {
img, _, err = image.Decode(bytes.NewReader(imgData))
imgWidth, imgHeight, err = utils.ReadImageDimensions(path.Base(imgPath), imgData)
if err != nil {
return "", nil, nil, fmt.Errorf("Failed to read image file: %v. Error: %v", imageUrl, err)
}
Expand All @@ -296,7 +301,9 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp
"", //baseRTT,
"",
nil,
img)
imgWidth,
imgHeight,
)

ctx := context.TODO()
coll := hctx.Svcs.MongoDB.Collection(dbCollections.ImagesName)
Expand Down Expand Up @@ -523,7 +530,7 @@ func importWarpedImage(warpedImageUrl string, rttWarpedTo string, baseImage stri
return err
}

img, _, err := image.Decode(bytes.NewReader(imgData))
imgWidth, imgHeight, err := utils.ReadImageDimensions(path.Base(warpedSrcPath), imgData)
if err != nil {
return err
}
Expand All @@ -545,7 +552,9 @@ func importWarpedImage(warpedImageUrl string, rttWarpedTo string, baseImage stri
rttWarpedTo,
"",
matchInfo,
img)
imgWidth,
imgHeight,
)

ctx := context.TODO()
coll := hctx.Svcs.MongoDB.Collection(dbCollections.ImagesName)
Expand Down
76 changes: 57 additions & 19 deletions api/coreg/import_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,31 +1,69 @@
package coreg

import "fmt"
import (
"fmt"
"log"

"github.com/pixlise/core/v4/core/utils"
protos "github.com/pixlise/core/v4/generated-protos"
"google.golang.org/protobuf/encoding/protojson"
)

func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) {
fmt.Printf("%v|%v\n", name, err)

if b, err := protojson.Marshal(xform); err != nil {
log.Fatalln(err)
} else {
// Proto isn't deterministic, so we read as JSON and write it back out again
fmt.Printf("%v\n", utils.MakeDeterministicJSON(b, false))
}
}

func Example_readWarpedImageTransform() {
xform, name, err := readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png"))

xform, name, err = readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png"))

xform, name, err = readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png"))

xform, name, err = readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png"))

xform, name, err = readWarpedImageTransform("warped-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png"))

xform, name, err = readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01.png")
fmt.Printf("%+v|%v|%v\n", xform, name, err)
printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01.png"))

// Output:
// xOffset:40 yOffset:519 xScale:4.478153 yScale:4.478153|coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png|<nil>
// xOffset:186 yOffset:216 xScale:1.1359178 yScale:1.1359178|coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png|<nil>
// xOffset:40 yOffset:519 xScale:4.478153 yScale:4.478153|coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png|<nil>
// xOffset:186 yOffset:216 xScale:1.1359178 yScale:1.1359178|coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png|<nil>
// <nil>||Warped image name does not have expected components
// <nil>||Failed to find GDS file name section in image name: SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01. Error: Failed to parse meta from file name
// coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png|<nil>
// {
// "xOffset": 40,
// "xScale": 4.478153,
// "yOffset": 519,
// "yScale": 4.478153
// }
// coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png|<nil>
// {
// "xOffset": 186,
// "xScale": 1.1359178,
// "yOffset": 216,
// "yScale": 1.1359178
// }
// coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png|<nil>
// {
// "xOffset": 40,
// "xScale": 4.478153,
// "yOffset": 519,
// "yScale": 4.478153
// }
// coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png|<nil>
// {
// "xOffset": 186,
// "xScale": 1.1359178,
// "yOffset": 216,
// "yScale": 1.1359178
// }
// |Warped image name does not have expected components
// {}
// |Failed to find GDS file name section in image name: SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01. Error: Failed to parse meta from file name
// {}
}
65 changes: 53 additions & 12 deletions api/dataimport/for-trigger_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,15 +20,16 @@ package dataimport
import (
"context"
"fmt"
"log"
"os"
"strings"
"testing"

"github.com/pixlise/core/v4/api/dbCollections"
"github.com/pixlise/core/v4/api/specialUserIds"
"github.com/pixlise/core/v4/core/fileaccess"
"github.com/pixlise/core/v4/core/logger"
"github.com/pixlise/core/v4/core/scan"
"github.com/pixlise/core/v4/core/utils"
"github.com/pixlise/core/v4/core/wstestlib"
protos "github.com/pixlise/core/v4/generated-protos"
"go.mongodb.org/mongo-driver/mongo"
Expand Down Expand Up @@ -206,7 +207,7 @@ func Example_importForTrigger_OCS_Archive_Exists() {
// Log shows exists in archive: true
}

func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database) {
func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Database) {
// Ensure these log msgs appeared...
requiredLogs := []string{
"Downloading archived zip files...",
Expand All @@ -222,7 +223,7 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database
}

for _, msg := range requiredLogs {
fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg))
fmt.Printf("Logged \"%v\": %v\n", msg, logger.LogContains(msg))
}

// Dump contents of summary file, this verifies most things imported as expected
Expand All @@ -235,8 +236,10 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database
summary.TimestampUnixSec = 0

b, err := protojson.Marshal(summary)
s := strings.ReplaceAll(string(b), " ", "")
fmt.Printf("%v|%v\n", err, s)
if err != nil {
log.Fatalf("%v", err)
}
fmt.Printf("%v|%v\n", err, utils.MakeDeterministicJSON(b, true))
}

// Import FM-style (simulate trigger by OCS pipeline), file goes to archive, then all files downloaded from archive and dataset created
Expand Down Expand Up @@ -302,7 +305,7 @@ func Example_importForTrigger_OCS_Archive_OK() {
// Logged "Diffraction db saved successfully": true
// Logged "Applying custom title: Naltsos": true
// Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true
// <nil>|{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
}

// Import FM-style (simulate trigger by dataset edit screen), should create dataset with custom name+image
Expand Down Expand Up @@ -355,7 +358,7 @@ func Example_importForTrigger_OCS_DatasetEdit() {
// Logged "Diffraction db saved successfully": true
// Logged "Applying custom title: Naltsos": true
// Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true
// <nil>|{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// <nil>|{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"}
}

func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) {
Expand All @@ -373,6 +376,7 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database,
"WARNING: No main context image determined",
"Diffraction db saved successfully",
"Warning: No import.json found, defaults will be used",
"No auto-share destination found, so only importing user will be able to access this dataset.",
}

for _, msg := range requiredLogs {
Expand Down Expand Up @@ -422,7 +426,8 @@ func Example_importForTrigger_Manual_JPL() {
// Logged "WARNING: No main context image determined": true
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": true
// <nil>|{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"}
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"}
}

// Import a breadboard dataset from manual uploaded zip file
Expand Down Expand Up @@ -454,13 +459,48 @@ func Example_importForTrigger_Manual_SBU() {
// Logged "WARNING: No main context image determined": true
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
}

// Import a breadboard dataset from manual uploaded zip file
func Test_ImportForTrigger_Manual_SBU_NoAutoShare(t *testing.T) {
func Example_ImportForTrigger_Manual_SBU_NoAutoShare() {
remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("Manual_OK2", specialUserIds.JPLImport, "JPLTestUserGroupId")

trigger := `{
"datasetID": "test1234sbu",
"jobID": "dataimport-unittest123sbu"
}`

result, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS)

fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate)

printManualOKLogOutput(log, db, "test1234sbu", 4)

// Output:
// Errors: <nil>, changes: unknown, isUpdate: false
// Logged "Downloading archived zip files...": true
// Logged "Downloaded 0 zip files, unzipped 0 files": true
// Logged "No zip files found in archive, dataset may have been manually uploaded. Trying to download...": true
// Logged "Dataset test1234sbu downloaded 4 files from manual upload area": true
// Logged "Downloading pseudo-intensity ranges...": true
// Logged "Downloading user customisation files...": true
// Logged "Reading 1261 files from spectrum directory...": true
// Logged "Reading spectrum [1135/1260] 90%": true
// Logged "PMC 1261 has 4 MSA/spectrum entries": true
// Logged "WARNING: No main context image determined": true
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": true
// <nil>|{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"}
}

/* Didnt get this working when the above was changed. Problem is this still generates the user name: SBUImport, so the
premise of the test fails because it doesn't end up with no user id at that point!
func Test_ImportForTrigger_Manual_SBU_NoAutoShare_FailForPipeline(t *testing.T) {
remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("Manual_OK2", "", "")
trigger := `{
"datasetID": "test1234sbu",
"jobID": "dataimport-unittest123sbu"
Expand All @@ -473,7 +513,7 @@ func Test_ImportForTrigger_Manual_SBU_NoAutoShare(t *testing.T) {
t.Errorf("ImportForTrigger didnt return expected error")
}
}

*/
// Import a breadboard dataset from manual uploaded zip file
func Example_importForTrigger_Manual_EM() {
remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("ManualEM_OK", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId")
Expand Down Expand Up @@ -503,7 +543,8 @@ func Example_importForTrigger_Manual_EM() {
// Logged "WARNING: No main context image determined": false
// Logged "Diffraction db saved successfully": true
// Logged "Warning: No import.json found, defaults will be used": false
// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
// Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false
// <nil>|{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"}
}

/* NOT TESTED YET, because it's not done yet!
Expand Down
Loading

0 comments on commit 70ae13c

Please sign in to comment.