From b2d5c7880f2c2b522cd1e6c64e64f279a16dbd9a Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 06:51:49 +1000 Subject: [PATCH 01/19] Fix intermittently failing test, don't know why but sometimes Sprintf(%+v) puts 2 spaces between variables, sometimes one. Now we format to JSON in the example test and this probably won't happen again --- api/coreg/import_test.go | 54 ++++++++++++++++++++++++++-------------- 1 file changed, 35 insertions(+), 19 deletions(-) diff --git a/api/coreg/import_test.go b/api/coreg/import_test.go index 3be52e05..7fe82dda 100644 --- a/api/coreg/import_test.go +++ b/api/coreg/import_test.go @@ -1,31 +1,47 @@ package coreg -import "fmt" +import ( + "fmt" + "log" + + protos "github.com/pixlise/core/v4/generated-protos" + "google.golang.org/protobuf/encoding/protojson" +) + +func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) { + fmt.Printf("%v|%v\n", name, err) + + if b, err := protojson.Marshal(xform); err != nil { + log.Fatalln(err) + } else { + fmt.Printf("%v\n", string(b)) + } +} func Example_readWarpedImageTransform() { - xform, name, err := readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png")) - xform, name, err = readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png")) - xform, name, err = readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png")) - xform, name, err = readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-zoom_1.1359177671479777-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png")) - xform, name, err = readWarpedImageTransform("warped-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-win_216_186_167_183-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png")) - xform, name, err = readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01.png") - fmt.Printf("%+v|%v|%v\n", xform, name, err) + printWarpXform(readWarpedImageTransform("warped-zoom_4.478153138946561-win_519_40_1232_1183-SN100D0-SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01.png")) // Output: - // xOffset:40 yOffset:519 xScale:4.478153 yScale:4.478153|coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png| - // xOffset:186 yOffset:216 xScale:1.1359178 yScale:1.1359178|coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png| - // xOffset:40 yOffset:519 xScale:4.478153 yScale:4.478153|coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png| - // xOffset:186 yOffset:216 xScale:1.1359178 yScale:1.1359178|coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png| - // ||Warped image name does not have expected components - // ||Failed to find GDS file name section in image name: SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01. Error: Failed to parse meta from file name + // coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png| + // {"xOffset":40, "yOffset":519, "xScale":4.478153, "yScale":4.478153} + // coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png| + // {"xOffset":186, "yOffset":216, "xScale":1.1359178, "yScale":1.1359178} + // coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png| + // {"xOffset":40, "yOffset":519, "xScale":4.478153, "yScale":4.478153} + // coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png| + // {"xOffset":186, "yOffset":216, "xScale":1.1359178, "yScale":1.1359178} + // |Warped image name does not have expected components + // {} + // |Failed to find GDS file name section in image name: SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01. Error: Failed to parse meta from file name + // {} } From 42661302a1d807962760464682edfe8adfae40aa Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 06:52:01 +1000 Subject: [PATCH 02/19] Print out lambda version on startup --- internal/lambdas/data-import/main.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/internal/lambdas/data-import/main.go b/internal/lambdas/data-import/main.go index 26cc31fa..fed9c2bb 100644 --- a/internal/lambdas/data-import/main.go +++ b/internal/lambdas/data-import/main.go @@ -25,6 +25,7 @@ import ( "github.com/aws/aws-lambda-go/lambda" "github.com/pixlise/core/v4/api/dataimport" + "github.com/pixlise/core/v4/api/services" "github.com/pixlise/core/v4/core/awsutil" "github.com/pixlise/core/v4/core/fileaccess" "github.com/pixlise/core/v4/core/logger" @@ -32,6 +33,8 @@ import ( ) func HandleRequest(ctx context.Context, event awsutil.Event) (string, error) { + fmt.Printf("Data Importer Lambda version: %v\n", services.ApiVersion) + configBucket := os.Getenv("CONFIG_BUCKET") datasetBucket := os.Getenv("DATASETS_BUCKET") manualBucket := os.Getenv("MANUAL_BUCKET") From 12c10300864dfdcec416c2bb9f8282a9ef314fda Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 09:17:34 +1000 Subject: [PATCH 03/19] Make test deterministic --- api/coreg/import_test.go | 39 +++++++++++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/api/coreg/import_test.go b/api/coreg/import_test.go index 7fe82dda..3a18d29e 100644 --- a/api/coreg/import_test.go +++ b/api/coreg/import_test.go @@ -1,6 +1,7 @@ package coreg import ( + "encoding/json" "fmt" "log" @@ -14,6 +15,16 @@ func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) { if b, err := protojson.Marshal(xform); err != nil { log.Fatalln(err) } else { + // Proto isn't deterministic, so we read as JSON and write it back out again + var anyJson map[string]interface{} + err = json.Unmarshal(b, &anyJson) + if err != nil { + log.Fatalln(err) + } + b, err = json.MarshalIndent(anyJson, "", " ") + if err != nil { + log.Fatalln(err) + } fmt.Printf("%v\n", string(b)) } } @@ -33,13 +44,33 @@ func Example_readWarpedImageTransform() { // Output: // coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01-A.png| - // {"xOffset":40, "yOffset":519, "xScale":4.478153, "yScale":4.478153} + // { + // "xOffset": 40, + // "xScale": 4.478153, + // "yOffset": 519, + // "yScale": 4.478153 + // } // coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01-A.png| - // {"xOffset":186, "yOffset":216, "xScale":1.1359178, "yScale":1.1359178} + // { + // "xOffset": 186, + // "xScale": 1.1359178, + // "yOffset": 216, + // "yScale": 1.1359178 + // } // coreg-40_519-SC3_0921_0748732957_027RAS_N0450000SRLC11373_0000LMJ01.png| - // {"xOffset":40, "yOffset":519, "xScale":4.478153, "yScale":4.478153} + // { + // "xOffset": 40, + // "xScale": 4.478153, + // "yOffset": 519, + // "yScale": 4.478153 + // } // coreg-186_216-PCB_0921_0748739251_000RAS_N045000032302746300020LUJ01.png| - // {"xOffset":186, "yOffset":216, "xScale":1.1359178, "yScale":1.1359178} + // { + // "xOffset": 186, + // "xScale": 1.1359178, + // "yOffset": 216, + // "yScale": 1.1359178 + // } // |Warped image name does not have expected components // {} // |Failed to find GDS file name section in image name: SC3_0921_0748732957_027RASS_N0450000SRLC11373_0000LMJ01. Error: Failed to parse meta from file name From 4fd782d4abbe10feeb34d117a6b44b7228eb48b6 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 10:28:16 +1000 Subject: [PATCH 04/19] Dont append random chars to uploaded dataset ids because theyre not found by lambda --- api/ws/handlers/scan.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/api/ws/handlers/scan.go b/api/ws/handlers/scan.go index c1c8e689..2a7140ed 100644 --- a/api/ws/handlers/scan.go +++ b/api/ws/handlers/scan.go @@ -310,7 +310,7 @@ func HandleScanUploadReq(req *protos.ScanUploadReq, hctx wsHelpers.HandlerContex destBucket := hctx.Svcs.Config.ManualUploadBucket fs := hctx.Svcs.FS logger := hctx.Svcs.Log - logger.Infof("Dataset create started for format: %v, id: %v", req.Id, req.Format) + logger.Infof("Dataset create started for format: %v, id: %v", req.Format, req.Id) // Validate the dataset ID - can't contain funny characters because it ends up as an S3 path // NOTE: we also turn space to _ here! Having spaces in the path broke quants because the @@ -325,7 +325,7 @@ func HandleScanUploadReq(req *protos.ScanUploadReq, hctx wsHelpers.HandlerContex s3PathStart := path.Join(filepaths.DatasetUploadRoot, datasetID) // Append a few random chars to make it more unique from this point on - datasetID += "_" + utils.RandStringBytesMaskImpr(6) + //datasetID += "_" + utils.RandStringBytesMaskImpr(6) // We don't need to check anything beyond does this file exist? /* From 3433999b68d1597e7b94f5c461e40e2a5afaf32a Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 11:36:15 +1000 Subject: [PATCH 05/19] If data importing user has no config, import so they are the only editor, but if no user (auto-import pipeline), still fail --- api/dataimport/internal/output/output.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 871b234c..948b3091 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -294,7 +294,17 @@ func (s *PIXLISEDataSaver) Save( if autoShareResult.Err() != nil { // We couldn't find someone to auto-share it with, if we don't have anyone to share with, just fail here if autoShareResult.Err() == mongo.ErrNoDocuments { - return fmt.Errorf("Cannot work out groups to auto-share imported dataset with") + // If the user has no auto-share destination configured, share with just the user - BUT if we're + // not dealing with a user here, we must be importing via the pipeline, in which case it should've + // been configured to share already... + if len(data.CreatorUserId) > 0 { + jobLog.Infof("No auto-share destination found, so only importing user will be able to access this dataset.") + autoShare.Id = data.CreatorUserId + autoShare.Viewers = &protos.UserGroupList{UserIds: []string{}, GroupIds: []string{}} + autoShare.Editors = &protos.UserGroupList{UserIds: []string{data.CreatorUserId}, GroupIds: []string{}} + } else { + return fmt.Errorf("Cannot work out groups to auto-share imported dataset with") + } } return autoShareResult.Err() } else { From 327b107280a8d029d5464610f947f4bcd1c1418a Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 12:18:37 +1000 Subject: [PATCH 06/19] Fixing test for autoshare issue --- api/dataimport/for-trigger_test.go | 33 +++++++++++++++++++----- api/dataimport/internal/output/output.go | 3 ++- 2 files changed, 28 insertions(+), 8 deletions(-) diff --git a/api/dataimport/for-trigger_test.go b/api/dataimport/for-trigger_test.go index eb99d82b..39ce1297 100644 --- a/api/dataimport/for-trigger_test.go +++ b/api/dataimport/for-trigger_test.go @@ -22,7 +22,6 @@ import ( "fmt" "os" "strings" - "testing" "github.com/pixlise/core/v4/api/dbCollections" "github.com/pixlise/core/v4/api/specialUserIds" @@ -373,6 +372,7 @@ func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, "WARNING: No main context image determined", "Diffraction db saved successfully", "Warning: No import.json found, defaults will be used", + "No auto-share destination found, so only importing user will be able to access this dataset.", } for _, msg := range requiredLogs { @@ -422,6 +422,7 @@ func Example_importForTrigger_Manual_JPL() { // Logged "WARNING: No main context image determined": true // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": true + // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false // |{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"} } @@ -454,11 +455,12 @@ func Example_importForTrigger_Manual_SBU() { // Logged "WARNING: No main context image determined": true // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false + // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } // Import a breadboard dataset from manual uploaded zip file -func Test_ImportForTrigger_Manual_SBU_NoAutoShare(t *testing.T) { +func Example_ImportForTrigger_Manual_SBU_NoAutoShare() { remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("Manual_OK2", specialUserIds.JPLImport, "JPLTestUserGroupId") trigger := `{ @@ -466,12 +468,28 @@ func Test_ImportForTrigger_Manual_SBU_NoAutoShare(t *testing.T) { "jobID": "dataimport-unittest123sbu" }` - _, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS) + result, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS) - // Make sure we got the error - if !strings.HasSuffix(err.Error(), "Cannot work out groups to auto-share imported dataset with") { - t.Errorf("ImportForTrigger didnt return expected error") - } + fmt.Printf("Errors: %v, changes: %v, isUpdate: %v\n", err, result.WhatChanged, result.IsUpdate) + + printManualOKLogOutput(log, db, "test1234sbu", 4) + + // Output: + // Errors: , changes: unknown, isUpdate: false + // Logged "Downloading archived zip files...": true + // Logged "Downloaded 0 zip files, unzipped 0 files": true + // Logged "No zip files found in archive, dataset may have been manually uploaded. Trying to download...": true + // Logged "Dataset test1234sbu downloaded 4 files from manual upload area": true + // Logged "Downloading pseudo-intensity ranges...": true + // Logged "Downloading user customisation files...": true + // Logged "Reading 1261 files from spectrum directory...": true + // Logged "Reading spectrum [1135/1260] 90%": true + // Logged "PMC 1261 has 4 MSA/spectrum entries": true + // Logged "WARNING: No main context image determined": true + // Logged "Diffraction db saved successfully": true + // Logged "Warning: No import.json found, defaults will be used": false + // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": true + // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } // Import a breadboard dataset from manual uploaded zip file @@ -503,6 +521,7 @@ func Example_importForTrigger_Manual_EM() { // Logged "WARNING: No main context image determined": false // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false + // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false // |{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} } diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 948b3091..6c18b290 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -305,8 +305,9 @@ func (s *PIXLISEDataSaver) Save( } else { return fmt.Errorf("Cannot work out groups to auto-share imported dataset with") } + } else { + return autoShareResult.Err() } - return autoShareResult.Err() } else { err := autoShareResult.Decode(autoShare) From 1a86e0463a25714a1e82c21d589a7048a63f9c5a Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 13:16:06 +1000 Subject: [PATCH 07/19] PIXL EM importer was assuming 1 subdir, not sure why, no longer --- api/dataimport/for-trigger_test.go | 18 +++++++++++++ .../internal/converters/pixlem/import.go | 26 +------------------ 2 files changed, 19 insertions(+), 25 deletions(-) diff --git a/api/dataimport/for-trigger_test.go b/api/dataimport/for-trigger_test.go index 39ce1297..784933e2 100644 --- a/api/dataimport/for-trigger_test.go +++ b/api/dataimport/for-trigger_test.go @@ -492,6 +492,24 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() { // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } +/* Didnt get this working when the above was changed. Problem is this still generates the user name: SBUImport, so the + premise of the test fails because it doesn't end up with no user id at that point! +func Test_ImportForTrigger_Manual_SBU_NoAutoShare_FailForPipeline(t *testing.T) { + remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("Manual_OK2", "", "") + + trigger := `{ + "datasetID": "test1234sbu", + "jobID": "dataimport-unittest123sbu" +}` + + _, err := ImportForTrigger([]byte(trigger), envName, configBucket, datasetBucket, manualBucket, db, log, remoteFS) + + // Make sure we got the error + if !strings.HasSuffix(err.Error(), "Cannot work out groups to auto-share imported dataset with") { + t.Errorf("ImportForTrigger didnt return expected error") + } +} +*/ // Import a breadboard dataset from manual uploaded zip file func Example_importForTrigger_Manual_EM() { remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("ManualEM_OK", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId") diff --git a/api/dataimport/internal/converters/pixlem/import.go b/api/dataimport/internal/converters/pixlem/import.go index 480be3bd..7fa6c785 100644 --- a/api/dataimport/internal/converters/pixlem/import.go +++ b/api/dataimport/internal/converters/pixlem/import.go @@ -18,10 +18,6 @@ package pixlem import ( - "errors" - "os" - "path/filepath" - "github.com/pixlise/core/v4/api/dataimport/internal/converters/pixlfm" "github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels" "github.com/pixlise/core/v4/core/logger" @@ -36,31 +32,11 @@ type PIXLEM struct { } func (p PIXLEM) Import(importPath string, pseudoIntensityRangesPath string, datasetIDExpected string, log logger.ILogger) (*dataConvertModels.OutputData, string, error) { - // Find the subdir - subdir := "" - - c, _ := os.ReadDir(importPath) - for _, entry := range c { - if entry.IsDir() { - // If it's not the first one, we can't do this - if len(subdir) > 0 { - return nil, "", errors.New("Found multiple subdirs, expected one in: " + importPath) - } - subdir = entry.Name() - } - } - - if len(subdir) <= 0 { - return nil, "", errors.New("Failed to find PIXL data subdir in: " + importPath) - } - - // Form the actual path to the files - subImportPath := filepath.Join(importPath, subdir) fmImporter := pixlfm.PIXLFM{} // Override importers group and detector fmImporter.SetOverrides(protos.ScanInstrument_PIXL_EM, "PIXL-EM-E2E") // Now we can import it like normal - return fmImporter.Import(subImportPath, pseudoIntensityRangesPath, datasetIDExpected, log) + return fmImporter.Import(importPath, pseudoIntensityRangesPath, datasetIDExpected, log) } From e0398db0c0f9714efe43e77204b8375210b4c7bb Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 14:44:20 +1000 Subject: [PATCH 08/19] Added extra logging to importer, brought back check for single dir in for EM unzipped data --- .../internal/converterSelector/selector.go | 5 ++++ .../internal/converters/pixlem/import.go | 26 ++++++++++++++++++- .../internal/converters/pixlfm/import.go | 2 ++ api/endpoints/Scan.go | 13 ++++++++++ 4 files changed, 45 insertions(+), 1 deletion(-) diff --git a/api/dataimport/internal/converterSelector/selector.go b/api/dataimport/internal/converterSelector/selector.go index d75a1c12..c15c6356 100644 --- a/api/dataimport/internal/converterSelector/selector.go +++ b/api/dataimport/internal/converterSelector/selector.go @@ -50,6 +50,7 @@ func SelectDataConverter(localFS fileaccess.FileAccess, remoteFS fileaccess.File } */ // Check if it's a PIXL FM style dataset + log.Infof("Checking path \"%v\" for PIXL FM structure...", importPath) pathType, err := pixlfm.DetectPIXLFMStructure(importPath) if len(pathType) > 0 && err == nil { // We know it's a PIXL FM type dataset... it'll later be determined which one @@ -75,10 +76,14 @@ func SelectDataConverter(localFS fileaccess.FileAccess, remoteFS fileaccess.File var detectorFile dataimportModel.DetectorChoice err = localFS.ReadJSON(detPath, "", &detectorFile, false) if err == nil { + log.Infof("Loaded detector.json...") + // We found it, work out based on what's in there if strings.HasSuffix(detectorFile.Detector, "-breadboard") { + log.Infof("Assuming breadboard dataset...") return jplbreadboard.MSATestData{}, nil } else if detectorFile.Detector == "pixl-em" { + log.Infof("Assuming PIXL EM dataset...") return pixlem.PIXLEM{}, nil } } else { diff --git a/api/dataimport/internal/converters/pixlem/import.go b/api/dataimport/internal/converters/pixlem/import.go index 7fa6c785..480be3bd 100644 --- a/api/dataimport/internal/converters/pixlem/import.go +++ b/api/dataimport/internal/converters/pixlem/import.go @@ -18,6 +18,10 @@ package pixlem import ( + "errors" + "os" + "path/filepath" + "github.com/pixlise/core/v4/api/dataimport/internal/converters/pixlfm" "github.com/pixlise/core/v4/api/dataimport/internal/dataConvertModels" "github.com/pixlise/core/v4/core/logger" @@ -32,11 +36,31 @@ type PIXLEM struct { } func (p PIXLEM) Import(importPath string, pseudoIntensityRangesPath string, datasetIDExpected string, log logger.ILogger) (*dataConvertModels.OutputData, string, error) { + // Find the subdir + subdir := "" + + c, _ := os.ReadDir(importPath) + for _, entry := range c { + if entry.IsDir() { + // If it's not the first one, we can't do this + if len(subdir) > 0 { + return nil, "", errors.New("Found multiple subdirs, expected one in: " + importPath) + } + subdir = entry.Name() + } + } + + if len(subdir) <= 0 { + return nil, "", errors.New("Failed to find PIXL data subdir in: " + importPath) + } + + // Form the actual path to the files + subImportPath := filepath.Join(importPath, subdir) fmImporter := pixlfm.PIXLFM{} // Override importers group and detector fmImporter.SetOverrides(protos.ScanInstrument_PIXL_EM, "PIXL-EM-E2E") // Now we can import it like normal - return fmImporter.Import(importPath, pseudoIntensityRangesPath, datasetIDExpected, log) + return fmImporter.Import(subImportPath, pseudoIntensityRangesPath, datasetIDExpected, log) } diff --git a/api/dataimport/internal/converters/pixlfm/import.go b/api/dataimport/internal/converters/pixlfm/import.go index 78f515aa..acb117f7 100644 --- a/api/dataimport/internal/converters/pixlfm/import.go +++ b/api/dataimport/internal/converters/pixlfm/import.go @@ -85,11 +85,13 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data rgbuImgDir := fileStructure{} discoImgDir := fileStructure{} + log.Infof("Checking path \"%v\" for FM dataset type", importPath) pathType, err := DetectPIXLFMStructure(importPath) if err != nil { return nil, "", err } + log.Infof("Found path \"%v\" is of type %v", importPath, pathType) if pathType == "DataDrive" { // This is the official way we receive PIXL FM data from Mars // We expect these directories to exist... diff --git a/api/endpoints/Scan.go b/api/endpoints/Scan.go index 84c49d05..34f1439a 100644 --- a/api/endpoints/Scan.go +++ b/api/endpoints/Scan.go @@ -51,6 +51,19 @@ func PutScanData(params apiRouter.ApiHandlerGenericParams) error { s3PathStart := path.Join(filepaths.DatasetUploadRoot, scanId) // NOTE: We overwrite any previous attempts without worry! + existing, err := params.Svcs.FS.ListObjects(destBucket, s3PathStart+"/") + if err == nil && len(existing) > 0 { + // Delete all that exists + msg := fmt.Sprintf("PutScan for \"%v\": Deleting existing file...\n", scanId) + for _, existingItem := range existing { + msg += existingItem + "\n" + if err := params.Svcs.FS.DeleteObject(destBucket, existingItem); err != nil { + return fmt.Errorf("Failed to delete: \"%v\", error: %v", existing, err) + } + } + + params.Svcs.Log.Infof(msg) + } // Read in body zippedData, err := io.ReadAll(params.Request.Body) From e3d7f29b03a78e9865a4c9e2edc19e9d94acc412 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 15:04:08 +1000 Subject: [PATCH 09/19] Fixing EM uploads where data zip is uploaded again, and 2 zips are found later in the process. Also print out path names of zips in the multiple subdir case --- .../internal/converters/pixlem/import.go | 3 ++- api/ws/handlers/scan.go | 17 ++++++++++------- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/api/dataimport/internal/converters/pixlem/import.go b/api/dataimport/internal/converters/pixlem/import.go index 480be3bd..17ddad4e 100644 --- a/api/dataimport/internal/converters/pixlem/import.go +++ b/api/dataimport/internal/converters/pixlem/import.go @@ -19,6 +19,7 @@ package pixlem import ( "errors" + "fmt" "os" "path/filepath" @@ -44,7 +45,7 @@ func (p PIXLEM) Import(importPath string, pseudoIntensityRangesPath string, data if entry.IsDir() { // If it's not the first one, we can't do this if len(subdir) > 0 { - return nil, "", errors.New("Found multiple subdirs, expected one in: " + importPath) + return nil, "", fmt.Errorf("Found multiple subdirs (\"%v\", \"%v\"), expected one in: \"%v\"", subdir, entry.Name(), importPath) } subdir = entry.Name() } diff --git a/api/ws/handlers/scan.go b/api/ws/handlers/scan.go index 2a7140ed..7845ba10 100644 --- a/api/ws/handlers/scan.go +++ b/api/ws/handlers/scan.go @@ -394,13 +394,16 @@ func HandleScanUploadReq(req *protos.ScanUploadReq, hctx wsHelpers.HandlerContex return nil, fmt.Errorf("Zip file missing RFS sub-directory") } - // Save the contents as a zip file in the uploads area - savePath := path.Join(s3PathStart, "data.zip") - err = fs.WriteObject(destBucket, savePath, zippedData) - if err != nil { - return nil, err - } - logger.Infof(" Wrote: s3://%v/%v", destBucket, savePath) + // NOTE: we just downloaded the zip from here, no point uploading it again! + /* + // Save the contents as a zip file in the uploads area + savePath := path.Join(s3PathStart, "data.zip") + err = fs.WriteObject(destBucket, savePath, zippedData) + if err != nil { + return nil, err + } + logger.Infof(" Wrote: s3://%v/%v", destBucket, savePath) + */ } else { // Expecting flat zip of MSA files count := 0 From af7fdf7680b4d19ef84c01455a66c60d35d24009 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 16:23:08 +1000 Subject: [PATCH 10/19] Fixing default image path saved for importer --- api/dataimport/internal/output/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 6c18b290..7b99b006 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -408,7 +408,7 @@ func insertDefaultImage(db *mongo.Database, scanId string, defaultImage string, coll := db.Collection(dbCollections.ScanDefaultImagesName) opt := options.InsertOne() - defaultImageResult, err := coll.InsertOne(context.TODO(), &protos.ScanImageDefaultDB{ScanId: scanId, DefaultImageFileName: defaultImage}, opt) + defaultImageResult, err := coll.InsertOne(context.TODO(), &protos.ScanImageDefaultDB{ScanId: scanId, DefaultImageFileName: path.Join(scanId, defaultImage)}, opt) if err != nil { if mongo.IsDuplicateKeyError(err) { // Don't overwrite, so we're OK with this From 7375ea597bc83fc45b63d913d94a6197c256ec47 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Tue, 26 Mar 2024 17:31:23 +1000 Subject: [PATCH 11/19] Fixing beam location save ID for data imports --- api/dataimport/internal/output/output.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 7b99b006..0d56d79d 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -334,7 +334,7 @@ func (s *PIXLISEDataSaver) Save( if beamVer < 1 { beamVer = 1 } - err := beamLocation.ImportBeamLocationToDB(imgItem.Image, data.Instrument, data.DatasetID, beamVer, idx, &exp, db, jobLog) + err := beamLocation.ImportBeamLocationToDB(path.Join(data.DatasetID, imgItem.Image), data.Instrument, data.DatasetID, beamVer, idx, &exp, db, jobLog) if err != nil { return fmt.Errorf("Failed to import beam locations for image %v into DB. Error: %v", imgItem.Image, err) } From 6ce736b31a26ca4f1b434c2c9796146651bd4ead Mon Sep 17 00:00:00 2001 From: Ryan Stonebraker Date: Tue, 26 Mar 2024 12:52:14 -0700 Subject: [PATCH 12/19] Bumps data formats for single axis and parallel coords --- data-formats | 2 +- generated-protos/detector-config-msgs.pb.go | 4 +- generated-protos/detector-config.pb.go | 4 +- generated-protos/diffraction-data.pb.go | 4 +- .../diffraction-detected-peak-msgs.pb.go | 4 +- .../diffraction-manual-msgs.pb.go | 4 +- .../diffraction-status-msgs.pb.go | 4 +- generated-protos/diffraction.pb.go | 4 +- generated-protos/doi-msgs.pb.go | 4 +- generated-protos/doi.pb.go | 4 +- generated-protos/element-set-msgs.pb.go | 4 +- generated-protos/element-set.pb.go | 4 +- generated-protos/experiment.pb.go | 4 +- generated-protos/export-msgs.pb.go | 4 +- generated-protos/expression-group-msgs.pb.go | 4 +- generated-protos/expression-group.pb.go | 4 +- generated-protos/expression-msgs.pb.go | 4 +- generated-protos/expressions.pb.go | 4 +- .../image-beam-location-msgs.pb.go | 4 +- generated-protos/image-beam-location.pb.go | 4 +- generated-protos/image-coreg-msgs.pb.go | 4 +- generated-protos/image-coreg.pb.go | 4 +- generated-protos/image-msgs.pb.go | 4 +- generated-protos/image.pb.go | 4 +- generated-protos/job-msgs.pb.go | 4 +- generated-protos/job.pb.go | 4 +- generated-protos/log-msgs.pb.go | 4 +- generated-protos/log.pb.go | 4 +- generated-protos/memoisation-msgs.pb.go | 4 +- generated-protos/memoisation.pb.go | 4 +- generated-protos/module-msgs.pb.go | 4 +- generated-protos/modules.pb.go | 4 +- generated-protos/notification-msgs.pb.go | 4 +- generated-protos/notification.pb.go | 4 +- generated-protos/ownership-access-msgs.pb.go | 4 +- generated-protos/ownership-access.pb.go | 4 +- generated-protos/permissions.pb.go | 4 +- generated-protos/piquant-config.pb.go | 4 +- generated-protos/piquant-msgs.pb.go | 4 +- .../pseudo-intensities-msgs.pb.go | 4 +- generated-protos/pseudo-intensities.pb.go | 4 +- generated-protos/quantification-create.pb.go | 4 +- .../quantification-management-msgs.pb.go | 4 +- generated-protos/quantification-meta.pb.go | 4 +- .../quantification-multi-msgs.pb.go | 4 +- generated-protos/quantification-multi.pb.go | 4 +- .../quantification-retrieval-msgs.pb.go | 4 +- .../quantification-upload-msgs.pb.go | 4 +- generated-protos/quantification.pb.go | 4 +- generated-protos/restmsgs.pb.go | 4 +- generated-protos/roi-msgs.pb.go | 4 +- generated-protos/roi.pb.go | 4 +- .../scan-beam-location-msgs.pb.go | 4 +- generated-protos/scan-beam-location.pb.go | 4 +- .../scan-entry-metadata-msgs.pb.go | 4 +- generated-protos/scan-entry-metadata.pb.go | 4 +- generated-protos/scan-entry-msgs.pb.go | 4 +- generated-protos/scan-entry.pb.go | 4 +- generated-protos/scan-msgs.pb.go | 4 +- generated-protos/scan.pb.go | 4 +- .../screen-configuration-msgs.pb.go | 4 +- generated-protos/screen-configuration.pb.go | 4 +- generated-protos/search-params.pb.go | 4 +- generated-protos/selection-entry-msgs.pb.go | 4 +- generated-protos/selection-pixel-msgs.pb.go | 4 +- generated-protos/spectrum-msgs.pb.go | 4 +- generated-protos/spectrum.pb.go | 4 +- generated-protos/tag-msgs.pb.go | 4 +- generated-protos/tags.pb.go | 4 +- generated-protos/test-msgs.pb.go | 4 +- generated-protos/user-db-items.pb.go | 4 +- generated-protos/user-group-admins-msgs.pb.go | 4 +- .../user-group-joining-msgs.pb.go | 4 +- .../user-group-management-msgs.pb.go | 4 +- .../user-group-membership-msgs.pb.go | 4 +- .../user-group-retrieval-msgs.pb.go | 4 +- generated-protos/user-group.pb.go | 4 +- generated-protos/user-management-msgs.pb.go | 4 +- generated-protos/user-msgs.pb.go | 4 +- .../user-notification-setting-msgs.pb.go | 4 +- .../user-notification-settings.pb.go | 4 +- generated-protos/user.pb.go | 4 +- generated-protos/version.pb.go | 4 +- generated-protos/viewstate-msgs.pb.go | 4 +- generated-protos/viewstate.pb.go | 4 +- generated-protos/websocket.pb.go | 4 +- generated-protos/widget-data-msgs.pb.go | 4 +- generated-protos/widget-data.pb.go | 182 +++++++++++------- 88 files changed, 290 insertions(+), 238 deletions(-) diff --git a/data-formats b/data-formats index 159a4b73..1f7f7a03 160000 --- a/data-formats +++ b/data-formats @@ -1 +1 @@ -Subproject commit 159a4b730c42f1953f17e311aae616154d3fe471 +Subproject commit 1f7f7a032dad793bcddd84dbde4b44dacc1b34f2 diff --git a/generated-protos/detector-config-msgs.pb.go b/generated-protos/detector-config-msgs.pb.go index e1d80b24..594ad16b 100644 --- a/generated-protos/detector-config-msgs.pb.go +++ b/generated-protos/detector-config-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: detector-config-msgs.proto package protos diff --git a/generated-protos/detector-config.pb.go b/generated-protos/detector-config.pb.go index 5d5b19c5..7dddbd9d 100644 --- a/generated-protos/detector-config.pb.go +++ b/generated-protos/detector-config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: detector-config.proto package protos diff --git a/generated-protos/diffraction-data.pb.go b/generated-protos/diffraction-data.pb.go index 58ce0764..a16350e0 100644 --- a/generated-protos/diffraction-data.pb.go +++ b/generated-protos/diffraction-data.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: diffraction-data.proto package protos diff --git a/generated-protos/diffraction-detected-peak-msgs.pb.go b/generated-protos/diffraction-detected-peak-msgs.pb.go index 250ee0d2..17a2234c 100644 --- a/generated-protos/diffraction-detected-peak-msgs.pb.go +++ b/generated-protos/diffraction-detected-peak-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: diffraction-detected-peak-msgs.proto package protos diff --git a/generated-protos/diffraction-manual-msgs.pb.go b/generated-protos/diffraction-manual-msgs.pb.go index f20cb474..6683e807 100644 --- a/generated-protos/diffraction-manual-msgs.pb.go +++ b/generated-protos/diffraction-manual-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: diffraction-manual-msgs.proto package protos diff --git a/generated-protos/diffraction-status-msgs.pb.go b/generated-protos/diffraction-status-msgs.pb.go index 966801f2..710eee91 100644 --- a/generated-protos/diffraction-status-msgs.pb.go +++ b/generated-protos/diffraction-status-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: diffraction-status-msgs.proto package protos diff --git a/generated-protos/diffraction.pb.go b/generated-protos/diffraction.pb.go index 281a37a9..77ca2271 100644 --- a/generated-protos/diffraction.pb.go +++ b/generated-protos/diffraction.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: data-formats/file-formats/diffraction.proto package protos diff --git a/generated-protos/doi-msgs.pb.go b/generated-protos/doi-msgs.pb.go index 13f76db2..8542205e 100644 --- a/generated-protos/doi-msgs.pb.go +++ b/generated-protos/doi-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: doi-msgs.proto package protos diff --git a/generated-protos/doi.pb.go b/generated-protos/doi.pb.go index 7d5bd02f..893a7e80 100644 --- a/generated-protos/doi.pb.go +++ b/generated-protos/doi.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: doi.proto package protos diff --git a/generated-protos/element-set-msgs.pb.go b/generated-protos/element-set-msgs.pb.go index 2d58e2db..9e3b01d3 100644 --- a/generated-protos/element-set-msgs.pb.go +++ b/generated-protos/element-set-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: element-set-msgs.proto package protos diff --git a/generated-protos/element-set.pb.go b/generated-protos/element-set.pb.go index 37d9bd0e..b96c3637 100644 --- a/generated-protos/element-set.pb.go +++ b/generated-protos/element-set.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: element-set.proto package protos diff --git a/generated-protos/experiment.pb.go b/generated-protos/experiment.pb.go index 6092e776..1bde8d45 100644 --- a/generated-protos/experiment.pb.go +++ b/generated-protos/experiment.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: data-formats/file-formats/experiment.proto package protos diff --git a/generated-protos/export-msgs.pb.go b/generated-protos/export-msgs.pb.go index 913bada3..2df4ffab 100644 --- a/generated-protos/export-msgs.pb.go +++ b/generated-protos/export-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: export-msgs.proto package protos diff --git a/generated-protos/expression-group-msgs.pb.go b/generated-protos/expression-group-msgs.pb.go index d7975d7e..f867bfa9 100644 --- a/generated-protos/expression-group-msgs.pb.go +++ b/generated-protos/expression-group-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: expression-group-msgs.proto package protos diff --git a/generated-protos/expression-group.pb.go b/generated-protos/expression-group.pb.go index 90f1716b..f71ded8f 100644 --- a/generated-protos/expression-group.pb.go +++ b/generated-protos/expression-group.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: expression-group.proto package protos diff --git a/generated-protos/expression-msgs.pb.go b/generated-protos/expression-msgs.pb.go index 35082e36..0be580e9 100644 --- a/generated-protos/expression-msgs.pb.go +++ b/generated-protos/expression-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: expression-msgs.proto package protos diff --git a/generated-protos/expressions.pb.go b/generated-protos/expressions.pb.go index a7fc9b8b..92b6c1b9 100644 --- a/generated-protos/expressions.pb.go +++ b/generated-protos/expressions.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: expressions.proto package protos diff --git a/generated-protos/image-beam-location-msgs.pb.go b/generated-protos/image-beam-location-msgs.pb.go index c48fccee..3cd5a92e 100644 --- a/generated-protos/image-beam-location-msgs.pb.go +++ b/generated-protos/image-beam-location-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image-beam-location-msgs.proto package protos diff --git a/generated-protos/image-beam-location.pb.go b/generated-protos/image-beam-location.pb.go index 6e8ae2ab..7a7b0344 100644 --- a/generated-protos/image-beam-location.pb.go +++ b/generated-protos/image-beam-location.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image-beam-location.proto package protos diff --git a/generated-protos/image-coreg-msgs.pb.go b/generated-protos/image-coreg-msgs.pb.go index a2cbb23d..d736058a 100644 --- a/generated-protos/image-coreg-msgs.pb.go +++ b/generated-protos/image-coreg-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image-coreg-msgs.proto package protos diff --git a/generated-protos/image-coreg.pb.go b/generated-protos/image-coreg.pb.go index 7f636355..edf35579 100644 --- a/generated-protos/image-coreg.pb.go +++ b/generated-protos/image-coreg.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image-coreg.proto package protos diff --git a/generated-protos/image-msgs.pb.go b/generated-protos/image-msgs.pb.go index 5784f9b5..2c5974fb 100644 --- a/generated-protos/image-msgs.pb.go +++ b/generated-protos/image-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image-msgs.proto package protos diff --git a/generated-protos/image.pb.go b/generated-protos/image.pb.go index a8316b61..df12738d 100644 --- a/generated-protos/image.pb.go +++ b/generated-protos/image.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: image.proto package protos diff --git a/generated-protos/job-msgs.pb.go b/generated-protos/job-msgs.pb.go index 63a874bd..344a7e94 100644 --- a/generated-protos/job-msgs.pb.go +++ b/generated-protos/job-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: job-msgs.proto package protos diff --git a/generated-protos/job.pb.go b/generated-protos/job.pb.go index 91edb26c..7686c5ff 100644 --- a/generated-protos/job.pb.go +++ b/generated-protos/job.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: job.proto package protos diff --git a/generated-protos/log-msgs.pb.go b/generated-protos/log-msgs.pb.go index 1b490026..b741e1d5 100644 --- a/generated-protos/log-msgs.pb.go +++ b/generated-protos/log-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: log-msgs.proto package protos diff --git a/generated-protos/log.pb.go b/generated-protos/log.pb.go index ac5666d3..82e2509f 100644 --- a/generated-protos/log.pb.go +++ b/generated-protos/log.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: log.proto package protos diff --git a/generated-protos/memoisation-msgs.pb.go b/generated-protos/memoisation-msgs.pb.go index 3b40cbad..b34b5fcb 100644 --- a/generated-protos/memoisation-msgs.pb.go +++ b/generated-protos/memoisation-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: memoisation-msgs.proto package protos diff --git a/generated-protos/memoisation.pb.go b/generated-protos/memoisation.pb.go index b01797d2..07349006 100644 --- a/generated-protos/memoisation.pb.go +++ b/generated-protos/memoisation.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: memoisation.proto package protos diff --git a/generated-protos/module-msgs.pb.go b/generated-protos/module-msgs.pb.go index bc17fe28..b6b752da 100644 --- a/generated-protos/module-msgs.pb.go +++ b/generated-protos/module-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: module-msgs.proto package protos diff --git a/generated-protos/modules.pb.go b/generated-protos/modules.pb.go index fd1124d4..21b3054a 100644 --- a/generated-protos/modules.pb.go +++ b/generated-protos/modules.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: modules.proto package protos diff --git a/generated-protos/notification-msgs.pb.go b/generated-protos/notification-msgs.pb.go index ef76689d..200f5230 100644 --- a/generated-protos/notification-msgs.pb.go +++ b/generated-protos/notification-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: notification-msgs.proto package protos diff --git a/generated-protos/notification.pb.go b/generated-protos/notification.pb.go index 48d4c45a..ec0f2890 100644 --- a/generated-protos/notification.pb.go +++ b/generated-protos/notification.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: notification.proto package protos diff --git a/generated-protos/ownership-access-msgs.pb.go b/generated-protos/ownership-access-msgs.pb.go index 36d54a65..679277e4 100644 --- a/generated-protos/ownership-access-msgs.pb.go +++ b/generated-protos/ownership-access-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: ownership-access-msgs.proto package protos diff --git a/generated-protos/ownership-access.pb.go b/generated-protos/ownership-access.pb.go index 730662a0..9fa2e2a4 100644 --- a/generated-protos/ownership-access.pb.go +++ b/generated-protos/ownership-access.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: ownership-access.proto package protos diff --git a/generated-protos/permissions.pb.go b/generated-protos/permissions.pb.go index 6ab98ee6..676e6fe1 100644 --- a/generated-protos/permissions.pb.go +++ b/generated-protos/permissions.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: permissions.proto package protos diff --git a/generated-protos/piquant-config.pb.go b/generated-protos/piquant-config.pb.go index 0c3e38cf..67854fde 100644 --- a/generated-protos/piquant-config.pb.go +++ b/generated-protos/piquant-config.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: piquant-config.proto package protos diff --git a/generated-protos/piquant-msgs.pb.go b/generated-protos/piquant-msgs.pb.go index 474254bd..91f97ad1 100644 --- a/generated-protos/piquant-msgs.pb.go +++ b/generated-protos/piquant-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: piquant-msgs.proto package protos diff --git a/generated-protos/pseudo-intensities-msgs.pb.go b/generated-protos/pseudo-intensities-msgs.pb.go index bf12ad6d..4fc6fc1a 100644 --- a/generated-protos/pseudo-intensities-msgs.pb.go +++ b/generated-protos/pseudo-intensities-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: pseudo-intensities-msgs.proto package protos diff --git a/generated-protos/pseudo-intensities.pb.go b/generated-protos/pseudo-intensities.pb.go index 549fa381..141c3b8f 100644 --- a/generated-protos/pseudo-intensities.pb.go +++ b/generated-protos/pseudo-intensities.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: pseudo-intensities.proto package protos diff --git a/generated-protos/quantification-create.pb.go b/generated-protos/quantification-create.pb.go index 7ae937f0..cb6b40e7 100644 --- a/generated-protos/quantification-create.pb.go +++ b/generated-protos/quantification-create.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-create.proto package protos diff --git a/generated-protos/quantification-management-msgs.pb.go b/generated-protos/quantification-management-msgs.pb.go index 9f77418f..dde9e2bc 100644 --- a/generated-protos/quantification-management-msgs.pb.go +++ b/generated-protos/quantification-management-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-management-msgs.proto package protos diff --git a/generated-protos/quantification-meta.pb.go b/generated-protos/quantification-meta.pb.go index 50f3cab7..5625f11e 100644 --- a/generated-protos/quantification-meta.pb.go +++ b/generated-protos/quantification-meta.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-meta.proto package protos diff --git a/generated-protos/quantification-multi-msgs.pb.go b/generated-protos/quantification-multi-msgs.pb.go index b40544b4..615831f1 100644 --- a/generated-protos/quantification-multi-msgs.pb.go +++ b/generated-protos/quantification-multi-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-multi-msgs.proto package protos diff --git a/generated-protos/quantification-multi.pb.go b/generated-protos/quantification-multi.pb.go index a7c08ac4..3dfc1203 100644 --- a/generated-protos/quantification-multi.pb.go +++ b/generated-protos/quantification-multi.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-multi.proto package protos diff --git a/generated-protos/quantification-retrieval-msgs.pb.go b/generated-protos/quantification-retrieval-msgs.pb.go index e6a5625a..f86b04aa 100644 --- a/generated-protos/quantification-retrieval-msgs.pb.go +++ b/generated-protos/quantification-retrieval-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-retrieval-msgs.proto package protos diff --git a/generated-protos/quantification-upload-msgs.pb.go b/generated-protos/quantification-upload-msgs.pb.go index 9408dce0..580d28b6 100644 --- a/generated-protos/quantification-upload-msgs.pb.go +++ b/generated-protos/quantification-upload-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification-upload-msgs.proto package protos diff --git a/generated-protos/quantification.pb.go b/generated-protos/quantification.pb.go index d5737b8f..333fc858 100644 --- a/generated-protos/quantification.pb.go +++ b/generated-protos/quantification.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: quantification.proto package protos diff --git a/generated-protos/restmsgs.pb.go b/generated-protos/restmsgs.pb.go index 077d2be9..3c69f5dc 100644 --- a/generated-protos/restmsgs.pb.go +++ b/generated-protos/restmsgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: restmsgs.proto package protos diff --git a/generated-protos/roi-msgs.pb.go b/generated-protos/roi-msgs.pb.go index 25cd024a..e3fd1680 100644 --- a/generated-protos/roi-msgs.pb.go +++ b/generated-protos/roi-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: roi-msgs.proto package protos diff --git a/generated-protos/roi.pb.go b/generated-protos/roi.pb.go index 50e24b29..c8e3a69b 100644 --- a/generated-protos/roi.pb.go +++ b/generated-protos/roi.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: roi.proto package protos diff --git a/generated-protos/scan-beam-location-msgs.pb.go b/generated-protos/scan-beam-location-msgs.pb.go index e0163691..482c2696 100644 --- a/generated-protos/scan-beam-location-msgs.pb.go +++ b/generated-protos/scan-beam-location-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-beam-location-msgs.proto package protos diff --git a/generated-protos/scan-beam-location.pb.go b/generated-protos/scan-beam-location.pb.go index 22773ca8..f6531300 100644 --- a/generated-protos/scan-beam-location.pb.go +++ b/generated-protos/scan-beam-location.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-beam-location.proto package protos diff --git a/generated-protos/scan-entry-metadata-msgs.pb.go b/generated-protos/scan-entry-metadata-msgs.pb.go index 5e748e1b..037c7947 100644 --- a/generated-protos/scan-entry-metadata-msgs.pb.go +++ b/generated-protos/scan-entry-metadata-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-entry-metadata-msgs.proto package protos diff --git a/generated-protos/scan-entry-metadata.pb.go b/generated-protos/scan-entry-metadata.pb.go index 05565a9e..65cd4ebb 100644 --- a/generated-protos/scan-entry-metadata.pb.go +++ b/generated-protos/scan-entry-metadata.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-entry-metadata.proto package protos diff --git a/generated-protos/scan-entry-msgs.pb.go b/generated-protos/scan-entry-msgs.pb.go index d4cda4ad..8c15ab1e 100644 --- a/generated-protos/scan-entry-msgs.pb.go +++ b/generated-protos/scan-entry-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-entry-msgs.proto package protos diff --git a/generated-protos/scan-entry.pb.go b/generated-protos/scan-entry.pb.go index 6f0c3d57..1a8f62cd 100644 --- a/generated-protos/scan-entry.pb.go +++ b/generated-protos/scan-entry.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-entry.proto package protos diff --git a/generated-protos/scan-msgs.pb.go b/generated-protos/scan-msgs.pb.go index 433c6bf7..686ef44f 100644 --- a/generated-protos/scan-msgs.pb.go +++ b/generated-protos/scan-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan-msgs.proto package protos diff --git a/generated-protos/scan.pb.go b/generated-protos/scan.pb.go index 73a2adce..aeac3fe0 100644 --- a/generated-protos/scan.pb.go +++ b/generated-protos/scan.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: scan.proto package protos diff --git a/generated-protos/screen-configuration-msgs.pb.go b/generated-protos/screen-configuration-msgs.pb.go index 0586c064..b58d8e1f 100644 --- a/generated-protos/screen-configuration-msgs.pb.go +++ b/generated-protos/screen-configuration-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: screen-configuration-msgs.proto package protos diff --git a/generated-protos/screen-configuration.pb.go b/generated-protos/screen-configuration.pb.go index d2ebb60d..0a7352c7 100644 --- a/generated-protos/screen-configuration.pb.go +++ b/generated-protos/screen-configuration.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: screen-configuration.proto package protos diff --git a/generated-protos/search-params.pb.go b/generated-protos/search-params.pb.go index 309b142f..c00313fa 100644 --- a/generated-protos/search-params.pb.go +++ b/generated-protos/search-params.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: search-params.proto package protos diff --git a/generated-protos/selection-entry-msgs.pb.go b/generated-protos/selection-entry-msgs.pb.go index 1c5c18f1..8a0e9cc8 100644 --- a/generated-protos/selection-entry-msgs.pb.go +++ b/generated-protos/selection-entry-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: selection-entry-msgs.proto package protos diff --git a/generated-protos/selection-pixel-msgs.pb.go b/generated-protos/selection-pixel-msgs.pb.go index 8bad0215..fda4be2d 100644 --- a/generated-protos/selection-pixel-msgs.pb.go +++ b/generated-protos/selection-pixel-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: selection-pixel-msgs.proto package protos diff --git a/generated-protos/spectrum-msgs.pb.go b/generated-protos/spectrum-msgs.pb.go index 72ef5fbe..faf94fac 100644 --- a/generated-protos/spectrum-msgs.pb.go +++ b/generated-protos/spectrum-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: spectrum-msgs.proto package protos diff --git a/generated-protos/spectrum.pb.go b/generated-protos/spectrum.pb.go index 18f5adef..6913d8fb 100644 --- a/generated-protos/spectrum.pb.go +++ b/generated-protos/spectrum.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: spectrum.proto package protos diff --git a/generated-protos/tag-msgs.pb.go b/generated-protos/tag-msgs.pb.go index 79946b98..39f47461 100644 --- a/generated-protos/tag-msgs.pb.go +++ b/generated-protos/tag-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: tag-msgs.proto package protos diff --git a/generated-protos/tags.pb.go b/generated-protos/tags.pb.go index 6fcc8ebb..a233f8c2 100644 --- a/generated-protos/tags.pb.go +++ b/generated-protos/tags.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: tags.proto package protos diff --git a/generated-protos/test-msgs.pb.go b/generated-protos/test-msgs.pb.go index c6e2e75a..44f5de72 100644 --- a/generated-protos/test-msgs.pb.go +++ b/generated-protos/test-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: test-msgs.proto package protos diff --git a/generated-protos/user-db-items.pb.go b/generated-protos/user-db-items.pb.go index 5e0fd88e..a30f4e4b 100644 --- a/generated-protos/user-db-items.pb.go +++ b/generated-protos/user-db-items.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-db-items.proto package protos diff --git a/generated-protos/user-group-admins-msgs.pb.go b/generated-protos/user-group-admins-msgs.pb.go index 4d0062ab..98a63b86 100644 --- a/generated-protos/user-group-admins-msgs.pb.go +++ b/generated-protos/user-group-admins-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group-admins-msgs.proto package protos diff --git a/generated-protos/user-group-joining-msgs.pb.go b/generated-protos/user-group-joining-msgs.pb.go index 80731bba..2af8766b 100644 --- a/generated-protos/user-group-joining-msgs.pb.go +++ b/generated-protos/user-group-joining-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group-joining-msgs.proto package protos diff --git a/generated-protos/user-group-management-msgs.pb.go b/generated-protos/user-group-management-msgs.pb.go index fec608b2..c3a480fa 100644 --- a/generated-protos/user-group-management-msgs.pb.go +++ b/generated-protos/user-group-management-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group-management-msgs.proto package protos diff --git a/generated-protos/user-group-membership-msgs.pb.go b/generated-protos/user-group-membership-msgs.pb.go index 0679f2f4..5b48e29c 100644 --- a/generated-protos/user-group-membership-msgs.pb.go +++ b/generated-protos/user-group-membership-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group-membership-msgs.proto package protos diff --git a/generated-protos/user-group-retrieval-msgs.pb.go b/generated-protos/user-group-retrieval-msgs.pb.go index 37b865a9..e6e1d7db 100644 --- a/generated-protos/user-group-retrieval-msgs.pb.go +++ b/generated-protos/user-group-retrieval-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group-retrieval-msgs.proto package protos diff --git a/generated-protos/user-group.pb.go b/generated-protos/user-group.pb.go index dbbd9876..44f8bf32 100644 --- a/generated-protos/user-group.pb.go +++ b/generated-protos/user-group.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-group.proto package protos diff --git a/generated-protos/user-management-msgs.pb.go b/generated-protos/user-management-msgs.pb.go index 1d76081d..49c0d317 100644 --- a/generated-protos/user-management-msgs.pb.go +++ b/generated-protos/user-management-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-management-msgs.proto package protos diff --git a/generated-protos/user-msgs.pb.go b/generated-protos/user-msgs.pb.go index 6bb811e8..4ec68913 100644 --- a/generated-protos/user-msgs.pb.go +++ b/generated-protos/user-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-msgs.proto package protos diff --git a/generated-protos/user-notification-setting-msgs.pb.go b/generated-protos/user-notification-setting-msgs.pb.go index 28249f5e..d83fe206 100644 --- a/generated-protos/user-notification-setting-msgs.pb.go +++ b/generated-protos/user-notification-setting-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-notification-setting-msgs.proto package protos diff --git a/generated-protos/user-notification-settings.pb.go b/generated-protos/user-notification-settings.pb.go index 76a55c97..caf4b6ae 100644 --- a/generated-protos/user-notification-settings.pb.go +++ b/generated-protos/user-notification-settings.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user-notification-settings.proto package protos diff --git a/generated-protos/user.pb.go b/generated-protos/user.pb.go index fc680e5b..82598223 100644 --- a/generated-protos/user.pb.go +++ b/generated-protos/user.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: user.proto package protos diff --git a/generated-protos/version.pb.go b/generated-protos/version.pb.go index 58100287..3f1a73d4 100644 --- a/generated-protos/version.pb.go +++ b/generated-protos/version.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: version.proto package protos diff --git a/generated-protos/viewstate-msgs.pb.go b/generated-protos/viewstate-msgs.pb.go index 68fb77e5..1723140c 100644 --- a/generated-protos/viewstate-msgs.pb.go +++ b/generated-protos/viewstate-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: viewstate-msgs.proto package protos diff --git a/generated-protos/viewstate.pb.go b/generated-protos/viewstate.pb.go index 0f848bf0..8d212158 100644 --- a/generated-protos/viewstate.pb.go +++ b/generated-protos/viewstate.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: viewstate.proto package protos diff --git a/generated-protos/websocket.pb.go b/generated-protos/websocket.pb.go index c5f1400c..3755977c 100644 --- a/generated-protos/websocket.pb.go +++ b/generated-protos/websocket.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: websocket.proto package protos diff --git a/generated-protos/widget-data-msgs.pb.go b/generated-protos/widget-data-msgs.pb.go index 8438a1a9..cc100e37 100644 --- a/generated-protos/widget-data-msgs.pb.go +++ b/generated-protos/widget-data-msgs.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: widget-data-msgs.proto package protos diff --git a/generated-protos/widget-data.pb.go b/generated-protos/widget-data.pb.go index eb5be8f2..ddd0b90e 100644 --- a/generated-protos/widget-data.pb.go +++ b/generated-protos/widget-data.pb.go @@ -1,7 +1,7 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.30.0 -// protoc v3.19.4 +// protoc-gen-go v1.28.1 +// protoc v3.20.3 // source: widget-data.proto package protos @@ -1607,11 +1607,15 @@ type SingleAxisRGBUWidgetState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Minerals []string `protobuf:"bytes,1,rep,name=minerals,proto3" json:"minerals,omitempty"` - ChannelA string `protobuf:"bytes,2,opt,name=channelA,proto3" json:"channelA,omitempty"` - ChannelB string `protobuf:"bytes,3,opt,name=channelB,proto3" json:"channelB,omitempty"` - RoiStackedOverlap bool `protobuf:"varint,4,opt,name=roiStackedOverlap,proto3" json:"roiStackedOverlap,omitempty"` - ImageName string `protobuf:"bytes,5,opt,name=imageName,proto3" json:"imageName,omitempty"` + Minerals []string `protobuf:"bytes,1,rep,name=minerals,proto3" json:"minerals,omitempty"` + ChannelA string `protobuf:"bytes,2,opt,name=channelA,proto3" json:"channelA,omitempty"` + ChannelB string `protobuf:"bytes,3,opt,name=channelB,proto3" json:"channelB,omitempty"` + RoiStackedOverlap bool `protobuf:"varint,4,opt,name=roiStackedOverlap,proto3" json:"roiStackedOverlap,omitempty"` + ImageName string `protobuf:"bytes,5,opt,name=imageName,proto3" json:"imageName,omitempty"` + SelectedMinValue float32 `protobuf:"fixed32,6,opt,name=selectedMinValue,proto3" json:"selectedMinValue,omitempty"` + SelectedMaxValue float32 `protobuf:"fixed32,7,opt,name=selectedMaxValue,proto3" json:"selectedMaxValue,omitempty"` + RoiIds []string `protobuf:"bytes,8,rep,name=roiIds,proto3" json:"roiIds,omitempty"` + ShowAllMineralLabels bool `protobuf:"varint,9,opt,name=showAllMineralLabels,proto3" json:"showAllMineralLabels,omitempty"` } func (x *SingleAxisRGBUWidgetState) Reset() { @@ -1681,6 +1685,34 @@ func (x *SingleAxisRGBUWidgetState) GetImageName() string { return "" } +func (x *SingleAxisRGBUWidgetState) GetSelectedMinValue() float32 { + if x != nil { + return x.SelectedMinValue + } + return 0 +} + +func (x *SingleAxisRGBUWidgetState) GetSelectedMaxValue() float32 { + if x != nil { + return x.SelectedMaxValue + } + return 0 +} + +func (x *SingleAxisRGBUWidgetState) GetRoiIds() []string { + if x != nil { + return x.RoiIds + } + return nil +} + +func (x *SingleAxisRGBUWidgetState) GetShowAllMineralLabels() bool { + if x != nil { + return x.ShowAllMineralLabels + } + return false +} + type RGBUImagesWidgetState struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -1743,8 +1775,9 @@ type ParallelogramWidgetState struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Regions []string `protobuf:"bytes,1,rep,name=regions,proto3" json:"regions,omitempty"` - Channels []string `protobuf:"bytes,2,rep,name=channels,proto3" json:"channels,omitempty"` + Regions []string `protobuf:"bytes,1,rep,name=regions,proto3" json:"regions,omitempty"` + Channels []string `protobuf:"bytes,2,rep,name=channels,proto3" json:"channels,omitempty"` + ImageName string `protobuf:"bytes,3,opt,name=imageName,proto3" json:"imageName,omitempty"` } func (x *ParallelogramWidgetState) Reset() { @@ -1793,6 +1826,13 @@ func (x *ParallelogramWidgetState) GetChannels() []string { return nil } +func (x *ParallelogramWidgetState) GetImageName() string { + if x != nil { + return x.ImageName + } + return "" +} + type WidgetData struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -2206,7 +2246,7 @@ var file_widget_data_proto_rawDesc = []byte{ 0x75, 0x65, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x02, 0x52, 0x11, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x4d, 0x61, 0x78, 0x59, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, - 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xbb, 0x01, 0x0a, 0x19, 0x53, 0x69, + 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0xdf, 0x02, 0x0a, 0x19, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, 0x42, 0x55, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x1a, 0x0a, 0x08, 0x6d, 0x69, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x6d, 0x69, 0x6e, 0x65, 0x72, @@ -2218,61 +2258,73 @@ var file_widget_data_proto_rawDesc = []byte{ 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x11, 0x72, 0x6f, 0x69, 0x53, 0x74, 0x61, 0x63, 0x6b, 0x65, 0x64, 0x4f, 0x76, 0x65, 0x72, 0x6c, 0x61, 0x70, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x05, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, - 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x55, 0x0a, 0x15, 0x52, 0x47, 0x42, 0x55, 0x49, - 0x6d, 0x61, 0x67, 0x65, 0x73, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, - 0x12, 0x1e, 0x0a, 0x0a, 0x62, 0x72, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x65, 0x73, 0x73, 0x18, 0x02, - 0x20, 0x01, 0x28, 0x02, 0x52, 0x0a, 0x62, 0x72, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x65, 0x73, 0x73, - 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x18, 0x03, 0x20, - 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x50, - 0x0a, 0x18, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x57, - 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, 0x0a, 0x07, 0x72, 0x65, - 0x67, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, 0x07, 0x72, 0x65, 0x67, - 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, - 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x73, - 0x22, 0xa1, 0x05, 0x0a, 0x0a, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, 0x61, 0x12, - 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, - 0x30, 0x0a, 0x08, 0x73, 0x70, 0x65, 0x63, 0x74, 0x72, 0x75, 0x6d, 0x18, 0x02, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x14, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x74, 0x72, 0x75, 0x6d, 0x57, 0x69, 0x64, 0x67, - 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x08, 0x73, 0x70, 0x65, 0x63, 0x74, 0x72, 0x75, - 0x6d, 0x12, 0x24, 0x0a, 0x06, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x0c, 0x2e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x06, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x27, 0x0a, 0x07, 0x74, 0x65, 0x72, 0x6e, 0x61, - 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x54, 0x65, 0x72, 0x6e, 0x61, - 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x07, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x72, 0x79, - 0x12, 0x2d, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x05, 0x20, - 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x53, - 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, - 0x36, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x18, - 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, - 0x6d, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, - 0x78, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x63, 0x68, 0x6f, 0x72, 0x64, - 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x43, 0x68, 0x6f, 0x72, 0x64, 0x53, 0x74, - 0x61, 0x74, 0x65, 0x52, 0x05, 0x63, 0x68, 0x6f, 0x72, 0x64, 0x12, 0x21, 0x0a, 0x05, 0x74, 0x61, - 0x62, 0x6c, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x54, 0x61, 0x62, 0x6c, - 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x39, 0x0a, - 0x0d, 0x72, 0x6f, 0x69, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x09, - 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x52, 0x4f, 0x49, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x54, - 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, 0x72, 0x6f, 0x69, 0x51, 0x75, - 0x61, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x2d, 0x0a, 0x09, 0x76, 0x61, 0x72, 0x69, - 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x56, 0x61, - 0x72, 0x69, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x76, 0x61, - 0x72, 0x69, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x30, 0x0a, 0x08, 0x72, 0x67, 0x62, 0x75, 0x50, - 0x6c, 0x6f, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x52, 0x47, 0x42, 0x55, - 0x50, 0x6c, 0x6f, 0x74, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, - 0x08, 0x72, 0x67, 0x62, 0x75, 0x50, 0x6c, 0x6f, 0x74, 0x12, 0x42, 0x0a, 0x0e, 0x73, 0x69, 0x6e, - 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, 0x42, 0x55, 0x18, 0x0c, 0x20, 0x01, 0x28, - 0x0b, 0x32, 0x1a, 0x2e, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, - 0x42, 0x55, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0e, 0x73, - 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, 0x42, 0x55, 0x12, 0x34, 0x0a, - 0x09, 0x72, 0x67, 0x62, 0x75, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x0d, 0x20, 0x01, 0x28, 0x0b, - 0x32, 0x16, 0x2e, 0x52, 0x47, 0x42, 0x55, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x57, 0x69, 0x64, - 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x72, 0x67, 0x62, 0x75, 0x49, 0x6d, - 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, - 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x50, 0x61, 0x72, - 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, - 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, - 0x67, 0x72, 0x61, 0x6d, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, - 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, + 0x74, 0x65, 0x64, 0x4d, 0x69, 0x6e, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, + 0x02, 0x52, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x4d, 0x69, 0x6e, 0x56, 0x61, + 0x6c, 0x75, 0x65, 0x12, 0x2a, 0x0a, 0x10, 0x73, 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x4d, + 0x61, 0x78, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x07, 0x20, 0x01, 0x28, 0x02, 0x52, 0x10, 0x73, + 0x65, 0x6c, 0x65, 0x63, 0x74, 0x65, 0x64, 0x4d, 0x61, 0x78, 0x56, 0x61, 0x6c, 0x75, 0x65, 0x12, + 0x16, 0x0a, 0x06, 0x72, 0x6f, 0x69, 0x49, 0x64, 0x73, 0x18, 0x08, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x06, 0x72, 0x6f, 0x69, 0x49, 0x64, 0x73, 0x12, 0x32, 0x0a, 0x14, 0x73, 0x68, 0x6f, 0x77, 0x41, + 0x6c, 0x6c, 0x4d, 0x69, 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x18, + 0x09, 0x20, 0x01, 0x28, 0x08, 0x52, 0x14, 0x73, 0x68, 0x6f, 0x77, 0x41, 0x6c, 0x6c, 0x4d, 0x69, + 0x6e, 0x65, 0x72, 0x61, 0x6c, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73, 0x22, 0x55, 0x0a, 0x15, 0x52, + 0x47, 0x42, 0x55, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, + 0x74, 0x61, 0x74, 0x65, 0x12, 0x1e, 0x0a, 0x0a, 0x62, 0x72, 0x69, 0x67, 0x68, 0x74, 0x6e, 0x65, + 0x73, 0x73, 0x18, 0x02, 0x20, 0x01, 0x28, 0x02, 0x52, 0x0a, 0x62, 0x72, 0x69, 0x67, 0x68, 0x74, + 0x6e, 0x65, 0x73, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x22, 0x6e, 0x0a, 0x18, 0x50, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, 0x67, + 0x72, 0x61, 0x6d, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x18, + 0x0a, 0x07, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x09, 0x52, + 0x07, 0x72, 0x65, 0x67, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x63, 0x68, 0x61, 0x6e, + 0x6e, 0x65, 0x6c, 0x73, 0x12, 0x1c, 0x0a, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, 0x6d, + 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x09, 0x69, 0x6d, 0x61, 0x67, 0x65, 0x4e, 0x61, + 0x6d, 0x65, 0x22, 0xa1, 0x05, 0x0a, 0x0a, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x44, 0x61, 0x74, + 0x61, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, + 0x64, 0x12, 0x30, 0x0a, 0x08, 0x73, 0x70, 0x65, 0x63, 0x74, 0x72, 0x75, 0x6d, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x53, 0x70, 0x65, 0x63, 0x74, 0x72, 0x75, 0x6d, 0x57, 0x69, + 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x08, 0x73, 0x70, 0x65, 0x63, 0x74, + 0x72, 0x75, 0x6d, 0x12, 0x24, 0x0a, 0x06, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x18, 0x03, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x0c, 0x2e, 0x42, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x06, 0x62, 0x69, 0x6e, 0x61, 0x72, 0x79, 0x12, 0x27, 0x0a, 0x07, 0x74, 0x65, 0x72, + 0x6e, 0x61, 0x72, 0x79, 0x18, 0x04, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0d, 0x2e, 0x54, 0x65, 0x72, + 0x6e, 0x61, 0x72, 0x79, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x07, 0x74, 0x65, 0x72, 0x6e, 0x61, + 0x72, 0x79, 0x12, 0x2d, 0x0a, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, + 0x05, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, + 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x68, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, + 0x6d, 0x12, 0x36, 0x0a, 0x0c, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74, 0x49, 0x6d, 0x61, 0x67, + 0x65, 0x18, 0x06, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, + 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0c, 0x63, 0x6f, 0x6e, + 0x74, 0x65, 0x78, 0x74, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x21, 0x0a, 0x05, 0x63, 0x68, 0x6f, + 0x72, 0x64, 0x18, 0x07, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x43, 0x68, 0x6f, 0x72, 0x64, + 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x63, 0x68, 0x6f, 0x72, 0x64, 0x12, 0x21, 0x0a, 0x05, + 0x74, 0x61, 0x62, 0x6c, 0x65, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x54, 0x61, + 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x05, 0x74, 0x61, 0x62, 0x6c, 0x65, 0x12, + 0x39, 0x0a, 0x0d, 0x72, 0x6f, 0x69, 0x51, 0x75, 0x61, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, + 0x18, 0x09, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x13, 0x2e, 0x52, 0x4f, 0x49, 0x51, 0x75, 0x61, 0x6e, + 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, 0x72, 0x6f, 0x69, + 0x51, 0x75, 0x61, 0x6e, 0x74, 0x54, 0x61, 0x62, 0x6c, 0x65, 0x12, 0x2d, 0x0a, 0x09, 0x76, 0x61, + 0x72, 0x69, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0f, 0x2e, + 0x56, 0x61, 0x72, 0x69, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, + 0x76, 0x61, 0x72, 0x69, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x12, 0x30, 0x0a, 0x08, 0x72, 0x67, 0x62, + 0x75, 0x50, 0x6c, 0x6f, 0x74, 0x18, 0x0b, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x52, 0x47, + 0x42, 0x55, 0x50, 0x6c, 0x6f, 0x74, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, + 0x65, 0x52, 0x08, 0x72, 0x67, 0x62, 0x75, 0x50, 0x6c, 0x6f, 0x74, 0x12, 0x42, 0x0a, 0x0e, 0x73, + 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, 0x42, 0x55, 0x18, 0x0c, 0x20, + 0x01, 0x28, 0x0b, 0x32, 0x1a, 0x2e, 0x53, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, + 0x52, 0x47, 0x42, 0x55, 0x57, 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, + 0x0e, 0x73, 0x69, 0x6e, 0x67, 0x6c, 0x65, 0x41, 0x78, 0x69, 0x73, 0x52, 0x47, 0x42, 0x55, 0x12, + 0x34, 0x0a, 0x09, 0x72, 0x67, 0x62, 0x75, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x18, 0x0d, 0x20, 0x01, + 0x28, 0x0b, 0x32, 0x16, 0x2e, 0x52, 0x47, 0x42, 0x55, 0x49, 0x6d, 0x61, 0x67, 0x65, 0x73, 0x57, + 0x69, 0x64, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x09, 0x72, 0x67, 0x62, 0x75, + 0x49, 0x6d, 0x61, 0x67, 0x65, 0x12, 0x3f, 0x0a, 0x0d, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, + 0x6c, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x18, 0x0e, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x19, 0x2e, 0x50, + 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, 0x6c, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x57, 0x69, 0x64, 0x67, + 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x52, 0x0d, 0x70, 0x61, 0x72, 0x61, 0x6c, 0x6c, 0x65, + 0x6c, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, + 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From 50ae499cb6e3d1cc8487143fbefb3a1bf0ea3c56 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 06:29:34 +1000 Subject: [PATCH 13/19] Pass version number into lambda on build --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 12e3781c..09e223a8 100644 --- a/Makefile +++ b/Makefile @@ -40,7 +40,7 @@ build-linux: echo "sha: ${GITHUB_SHA}" GOOS=linux GOARCH=amd64 go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/ GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/pixlise-api-linux ./internal/api - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -o ./_out/bootstrap ./internal/lambdas/data-import + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/bootstrap ./internal/lambdas/data-import # GOOS=linux GOARCH=amd64 go build -v -o ./_out/importtest-linux ./internal/cmdline-tools/import-integration-test # GOOS=linux GOARCH=amd64 go build -v -o ./_out/integrationtest-linux ./internal/cmdline-tools/api-integration-test From ec464d5436a13730534fbdbb5fd398abd30f4b8e Mon Sep 17 00:00:00 2001 From: Ryan Stonebraker Date: Tue, 26 Mar 2024 14:00:15 -0700 Subject: [PATCH 14/19] Stops returning user icon with ownership summary --- api/ws/wsHelpers/ownership.go | 16 +++++++++++++++- api/ws/wsHelpers/userDBCache.go | 8 ++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/api/ws/wsHelpers/ownership.go b/api/ws/wsHelpers/ownership.go index 7f220c62..5635c646 100644 --- a/api/ws/wsHelpers/ownership.go +++ b/api/ws/wsHelpers/ownership.go @@ -185,7 +185,7 @@ func ListGroupAccessibleIDs(requireEdit bool, objectType protos.ObjectType, grou return result, nil } -func MakeOwnerSummary(ownership *protos.OwnershipItem, sessionUser SessionUser, db *mongo.Database, ts timestamper.ITimeStamper) *protos.OwnershipSummary { +func FetchOwnershipSummary(ownership *protos.OwnershipItem, sessionUser SessionUser, db *mongo.Database, ts timestamper.ITimeStamper, fullDetails bool) *protos.OwnershipSummary { user, err := getUserInfo(ownership.CreatorUserId, db, ts) result := &protos.OwnershipSummary{ CreatedUnixSec: ownership.CreatedUnixSec, @@ -237,9 +237,23 @@ func MakeOwnerSummary(ownership *protos.OwnershipItem, sessionUser SessionUser, } result.SharedWithOthers = result.ViewerUserCount > 0 || result.ViewerGroupCount > 0 || result.EditorUserCount > 0 || result.EditorGroupCount > 0 + + // Hide more data intensive fields if we don't care + if !fullDetails { + result.CreatorUser.IconURL = "" + } + return result } +func MakeOwnerSummary(ownership *protos.OwnershipItem, sessionUser SessionUser, db *mongo.Database, ts timestamper.ITimeStamper) *protos.OwnershipSummary { + return FetchOwnershipSummary(ownership, sessionUser, db, ts, false) +} + +func MakeFullOwnerSummary(ownership *protos.OwnershipItem, sessionUser SessionUser, db *mongo.Database, ts timestamper.ITimeStamper) *protos.OwnershipSummary { + return FetchOwnershipSummary(ownership, sessionUser, db, ts, true) +} + func FindUserIdsFor(objectId string, mongoDB *mongo.Database) ([]string, error) { filter := bson.M{"_id": objectId} opts := options.FindOne() diff --git a/api/ws/wsHelpers/userDBCache.go b/api/ws/wsHelpers/userDBCache.go index d80e7119..19477870 100644 --- a/api/ws/wsHelpers/userDBCache.go +++ b/api/ws/wsHelpers/userDBCache.go @@ -12,6 +12,14 @@ import ( ) func GetDBUser(userId string, db *mongo.Database) (*protos.UserDBItem, error) { + // opts := options.FindOne().SetProjection(bson.D{ + // {Key: "_id", Value: true}, + // {Key: "info.id", Value: true}, + // {Key: "info.name", Value: true}, + // {Key: "info.email", Value: true}, + // {Key: "datacollectionversion", Value: true}, + // {Key: "notificationsettings", Value: true}, + // }) userResult := db.Collection(dbCollections.UsersName).FindOne(context.TODO(), bson.M{"_id": userId}) if userResult.Err() != nil { return nil, userResult.Err() From 5429eea7e4cec8d6f6604372d683e4dbe4fc9c90 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 08:07:05 +1000 Subject: [PATCH 15/19] More logging to diagnose why beam locations are not saved correctly for uploaded datasets. Fixed some error handling too --- api/dataimport/internal/output/output.go | 14 +++++++++++--- api/quantification/create.go | 7 +------ api/quantification/importCSV.go | 5 +---- api/ws/handlers/element-set.go | 6 ++---- api/ws/handlers/expression-group.go | 5 +---- api/ws/handlers/expression.go | 5 +---- api/ws/handlers/module.go | 5 +---- api/ws/handlers/roi.go | 5 +---- api/ws/handlers/screen-configuration.go | 5 +---- api/ws/wsHelpers/ownership.go | 4 ++-- core/beamLocation/experimentFileToDB.go | 2 ++ 11 files changed, 24 insertions(+), 39 deletions(-) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 0d56d79d..909aa498 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -318,6 +318,10 @@ func (s *PIXLISEDataSaver) Save( // We work out the default file name when copying output images now... because if there isn't one, we may pick one during that process. defaultContextImage, err := copyImagesToOutput(contextImageSrcPath, []string{data.DatasetID}, data.DatasetID, outputImagesPath, data, db, jobLog) + if err != nil { + return err + } + exp.MainContextImage = defaultContextImage // Set any matched aligned images - this happens after copyImagesToOutput because file names may be modified by it depending on formats @@ -359,7 +363,7 @@ func (s *PIXLISEDataSaver) Save( summaryData := makeSummaryFileContent(&exp, data.DatasetID, data.Instrument, data.Meta, int(fi.Size()), creationUnixTimeSec, data.CreatorUserId) - jobLog.Infof("Writing summary to DB...") + jobLog.Infof("Writing summary to DB for %v...", summaryData.Id) coll = db.Collection(dbCollections.ScansName) opt := options.Update().SetUpsert(true) @@ -374,7 +378,7 @@ func (s *PIXLISEDataSaver) Save( } // Set ownership - ownerItem, err := wsHelpers.MakeOwnerForWrite(summaryData.Id, protos.ObjectType_OT_SCAN, summaryData.CreatorUserId, creationUnixTimeSec) + ownerItem := wsHelpers.MakeOwnerForWrite(summaryData.Id, protos.ObjectType_OT_SCAN, summaryData.CreatorUserId, creationUnixTimeSec) ownerItem.Viewers = autoShare.Viewers ownerItem.Editors = autoShare.Editors @@ -382,6 +386,7 @@ func (s *PIXLISEDataSaver) Save( coll = db.Collection(dbCollections.OwnershipName) opt = options.Update().SetUpsert(true) + jobLog.Infof("Writing ownership to DB for scan %v...", ownerItem.Id) result, err = coll.UpdateOne(context.TODO(), bson.D{{Key: "_id", Value: ownerItem.Id}}, bson.D{{Key: "$set", Value: ownerItem}}, opt) if err != nil { jobLog.Errorf("Failed to write ownership item to DB: %v", err) @@ -408,10 +413,13 @@ func insertDefaultImage(db *mongo.Database, scanId string, defaultImage string, coll := db.Collection(dbCollections.ScanDefaultImagesName) opt := options.InsertOne() - defaultImageResult, err := coll.InsertOne(context.TODO(), &protos.ScanImageDefaultDB{ScanId: scanId, DefaultImageFileName: path.Join(scanId, defaultImage)}, opt) + imgPath := path.Join(scanId, defaultImage) + jobLog.Infof("Writing default image %v to DB for scan %v...", imgPath, scanId) + defaultImageResult, err := coll.InsertOne(context.TODO(), &protos.ScanImageDefaultDB{ScanId: scanId, DefaultImageFileName: imgPath}, opt) if err != nil { if mongo.IsDuplicateKeyError(err) { // Don't overwrite, so we're OK with this + jobLog.Infof("Default image for scan %v already exists, not overwriting existing one in case of user edit.", scanId) return nil } return err diff --git a/api/quantification/create.go b/api/quantification/create.go index f40dc4a3..2e36f6e3 100644 --- a/api/quantification/create.go +++ b/api/quantification/create.go @@ -382,12 +382,7 @@ func (r *quantNodeRunner) triggerPiquantNodes(wg *sync.WaitGroup) { }, } - ownerItem, err := wsHelpers.MakeOwnerForWrite(r.jobId, protos.ObjectType_OT_QUANTIFICATION, r.quantStartSettings.RequestorUserId, now) - if err != nil { - msg := fmt.Sprintf("Failed to create ownership info for quant job %v. Error was: %v", r.jobId, err) - r.completeJobState(false, msg, quantOutPath, piquantLogList) - return - } + ownerItem := wsHelpers.MakeOwnerForWrite(r.jobId, protos.ObjectType_OT_QUANTIFICATION, r.quantStartSettings.RequestorUserId, now) err = writeQuantAndOwnershipToDB(summary, ownerItem, svcs.MongoDB) if err != nil { diff --git a/api/quantification/importCSV.go b/api/quantification/importCSV.go index bc86cb84..c9278741 100644 --- a/api/quantification/importCSV.go +++ b/api/quantification/importCSV.go @@ -66,10 +66,7 @@ func ImportQuantCSV( } // Finally, write the summary data to DB along with ownership entry - ownerItem, err := wsHelpers.MakeOwnerForWrite(quantId, protos.ObjectType_OT_QUANTIFICATION, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return quantId, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(quantId, protos.ObjectType_OT_QUANTIFICATION, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) summary := protos.QuantificationSummary{ Id: quantId, diff --git a/api/ws/handlers/element-set.go b/api/ws/handlers/element-set.go index c79a7c70..b8aa244d 100644 --- a/api/ws/handlers/element-set.go +++ b/api/ws/handlers/element-set.go @@ -101,10 +101,8 @@ func createElementSet(elementSet *protos.ElementSet, hctx wsHelpers.HandlerConte elementSet.Id = id // We need to create an ownership item along with it - ownerItem, err := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_ELEMENT_SET, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_ELEMENT_SET, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) + elementSet.ModifiedUnixSec = ownerItem.CreatedUnixSec wc := writeconcern.New(writeconcern.WMajority()) diff --git a/api/ws/handlers/expression-group.go b/api/ws/handlers/expression-group.go index 88813af6..1c188dc3 100644 --- a/api/ws/handlers/expression-group.go +++ b/api/ws/handlers/expression-group.go @@ -106,10 +106,7 @@ func createExpressionGroup(egroup *protos.ExpressionGroup, hctx wsHelpers.Handle egroup.Id = id // We need to create an ownership item along with it - ownerItem, err := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_EXPRESSION_GROUP, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_EXPRESSION_GROUP, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) egroup.ModifiedUnixSec = ownerItem.CreatedUnixSec diff --git a/api/ws/handlers/expression.go b/api/ws/handlers/expression.go index 53a0b4ee..2464dcd3 100644 --- a/api/ws/handlers/expression.go +++ b/api/ws/handlers/expression.go @@ -110,10 +110,7 @@ func createExpression(expr *protos.DataExpression, hctx wsHelpers.HandlerContext expr.Id = id // We need to create an ownership item along with it - ownerItem, err := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_EXPRESSION, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_EXPRESSION, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) expr.ModifiedUnixSec = ownerItem.CreatedUnixSec diff --git a/api/ws/handlers/module.go b/api/ws/handlers/module.go index c06a3a19..87346ffb 100644 --- a/api/ws/handlers/module.go +++ b/api/ws/handlers/module.go @@ -262,10 +262,7 @@ func createModule(name string, comments string, intialSourceCode string, tags [] } // We need to create an ownership item along with it - ownerItem, err := wsHelpers.MakeOwnerForWrite(modId, protos.ObjectType_OT_DATA_MODULE, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(modId, protos.ObjectType_OT_DATA_MODULE, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) module.ModifiedUnixSec = ownerItem.CreatedUnixSec diff --git a/api/ws/handlers/roi.go b/api/ws/handlers/roi.go index 8f83d88d..337decd3 100644 --- a/api/ws/handlers/roi.go +++ b/api/ws/handlers/roi.go @@ -180,10 +180,7 @@ func createROI(roi *protos.ROIItem, hctx wsHelpers.HandlerContext, needMistEntry roi.Id = id // We need to create an ownership item along with it - ownerItem, err := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_ROI, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + ownerItem := wsHelpers.MakeOwnerForWrite(id, protos.ObjectType_OT_ROI, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) roi.ModifiedUnixSec = ownerItem.CreatedUnixSec diff --git a/api/ws/handlers/screen-configuration.go b/api/ws/handlers/screen-configuration.go index 7759a090..96e9625e 100644 --- a/api/ws/handlers/screen-configuration.go +++ b/api/ws/handlers/screen-configuration.go @@ -156,10 +156,7 @@ func writeScreenConfiguration(screenConfig *protos.ScreenConfiguration, hctx wsH } // We need to create an ownership item along with it - owner, err = wsHelpers.MakeOwnerForWrite(screenConfig.Id, protos.ObjectType_OT_SCREEN_CONFIG, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) - if err != nil { - return nil, err - } + owner = wsHelpers.MakeOwnerForWrite(screenConfig.Id, protos.ObjectType_OT_SCREEN_CONFIG, hctx.SessUser.User.Id, hctx.Svcs.TimeStamper.GetTimeNowSec()) screenConfig.ModifiedUnixSec = owner.CreatedUnixSec configuration = screenConfig diff --git a/api/ws/wsHelpers/ownership.go b/api/ws/wsHelpers/ownership.go index 7f220c62..15c2bbdd 100644 --- a/api/ws/wsHelpers/ownership.go +++ b/api/ws/wsHelpers/ownership.go @@ -15,7 +15,7 @@ import ( "go.mongodb.org/mongo-driver/mongo/options" ) -func MakeOwnerForWrite(objectId string, objectType protos.ObjectType, creatorUserId string, createTimeUnixSec int64) (*protos.OwnershipItem, error) { +func MakeOwnerForWrite(objectId string, objectType protos.ObjectType, creatorUserId string, createTimeUnixSec int64) *protos.OwnershipItem { ownerItem := &protos.OwnershipItem{ Id: objectId, ObjectType: objectType, @@ -30,7 +30,7 @@ func MakeOwnerForWrite(objectId string, objectType protos.ObjectType, creatorUse } } - return ownerItem, nil + return ownerItem } // Checks object access - if requireEdit is true, it checks for edit access diff --git a/core/beamLocation/experimentFileToDB.go b/core/beamLocation/experimentFileToDB.go index 27b3b22d..7c49fff9 100644 --- a/core/beamLocation/experimentFileToDB.go +++ b/core/beamLocation/experimentFileToDB.go @@ -47,6 +47,8 @@ func ImportBeamLocationToDB(imgName string, instrument protos.ScanInstrument, fo }) opt := options.Replace().SetUpsert(true) + logger.Infof("Writing beam location to DB for image: %v, and scan: %v, instrument: %v, beamVersion: %v", imgName, forScanId, instrument, beamVersion) + result, err := imagesColl.ReplaceOne(context.TODO(), bson.M{"_id": imgName}, beams, opt) if err != nil { return err From 09babd7dbaa36d0a31259a1735b8ec7c7450904d Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 10:57:13 +1000 Subject: [PATCH 16/19] Fixing all situations where importer needs width/height of image being read, and that image is a RGBU TIF, which failed to be read by Go tif lib. We now detect it's an RGBU TIF and use a hard-coded width/height --- api/coreg/import.go | 17 ++++++++--- .../internal/converterSelector/selector.go | 4 +-- api/dataimport/internal/output/output.go | 25 ++++++++++++++--- api/ws/handlers/image.go | 8 +++--- core/utils/images.go | 28 +++++++++++-------- 5 files changed, 56 insertions(+), 26 deletions(-) diff --git a/api/coreg/import.go b/api/coreg/import.go index 52dab56a..699da0b7 100644 --- a/api/coreg/import.go +++ b/api/coreg/import.go @@ -257,10 +257,15 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp return "", nil, nil, err } + imgWidth := uint32(0) + imgHeight := uint32(0) + var img image.Image if imageExt == ".IMG" { w, h, d, err := imgFormat.ReadIMGFile(imgData) + imgWidth = uint32(w) + imgHeight = uint32(h) if err != nil { return "", nil, nil, fmt.Errorf("Failed to read IMG file: %v. Error: %v", imageUrl, err) } @@ -280,7 +285,7 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp imgData = buf.Bytes() } else { - img, _, err = image.Decode(bytes.NewReader(imgData)) + imgWidth, imgHeight, err = utils.ReadImageDimensions(path.Base(imgPath), imgData) if err != nil { return "", nil, nil, fmt.Errorf("Failed to read image file: %v. Error: %v", imageUrl, err) } @@ -296,7 +301,9 @@ func importNewImage(jobId string, imageUrl string, baseRTT string, marsViewerExp "", //baseRTT, "", nil, - img) + imgWidth, + imgHeight, + ) ctx := context.TODO() coll := hctx.Svcs.MongoDB.Collection(dbCollections.ImagesName) @@ -523,7 +530,7 @@ func importWarpedImage(warpedImageUrl string, rttWarpedTo string, baseImage stri return err } - img, _, err := image.Decode(bytes.NewReader(imgData)) + imgWidth, imgHeight, err := utils.ReadImageDimensions(path.Base(warpedSrcPath), imgData) if err != nil { return err } @@ -545,7 +552,9 @@ func importWarpedImage(warpedImageUrl string, rttWarpedTo string, baseImage stri rttWarpedTo, "", matchInfo, - img) + imgWidth, + imgHeight, + ) ctx := context.TODO() coll := hctx.Svcs.MongoDB.Collection(dbCollections.ImagesName) diff --git a/api/dataimport/internal/converterSelector/selector.go b/api/dataimport/internal/converterSelector/selector.go index c15c6356..ab239afb 100644 --- a/api/dataimport/internal/converterSelector/selector.go +++ b/api/dataimport/internal/converterSelector/selector.go @@ -92,9 +92,9 @@ func SelectDataConverter(localFS fileaccess.FileAccess, remoteFS fileaccess.File // Log the paths to help us diagnose issues... // Print it in one log message - logMsg := "SelectDataConverter path listing:" + logMsg := "SelectDataConverter path listing:\n" for c, item := range items { - logMsg += fmt.Sprintf("\n %v. %v\n", c+1, item) + logMsg += fmt.Sprintf(" %v. %v\n", c+1, item) } log.Infof(logMsg) diff --git a/api/dataimport/internal/output/output.go b/api/dataimport/internal/output/output.go index 909aa498..e67d6e2a 100644 --- a/api/dataimport/internal/output/output.go +++ b/api/dataimport/internal/output/output.go @@ -319,7 +319,7 @@ func (s *PIXLISEDataSaver) Save( // We work out the default file name when copying output images now... because if there isn't one, we may pick one during that process. defaultContextImage, err := copyImagesToOutput(contextImageSrcPath, []string{data.DatasetID}, data.DatasetID, outputImagesPath, data, db, jobLog) if err != nil { - return err + return fmt.Errorf("Error copying images: %v", err) } exp.MainContextImage = defaultContextImage @@ -664,19 +664,36 @@ func insertImageDBEntryForImage( matchInfo *protos.ImageMatchTransform, jobLog logger.ILogger) error { // Read the image - we used to only copy files around but here we need to open it for meta data - imgFile, err := utils.ReadImageFile(imagePath) + imgbytes, err := os.ReadFile(imagePath) if err != nil { return err } - //defer imgFile.Close() + imgWidth, imgHeight, err := utils.ReadImageDimensions(imagePath, imgbytes) + if err != nil { + return err + } stats, err := os.Stat(imagePath) + if err != nil { + return err + } saveName := filepath.Base(imagePath) savePath := path.Join(originScanId, saveName) - img := utils.MakeScanImage(savePath, uint32(stats.Size()), source, purpose, associatedScanIds, originScanId, originImageURL, matchInfo, imgFile) + img := utils.MakeScanImage( + savePath, + uint32(stats.Size()), + source, + purpose, + associatedScanIds, + originScanId, + originImageURL, + matchInfo, + imgWidth, + imgHeight, + ) return insertImageDBEntry(db, img, jobLog) } diff --git a/api/ws/handlers/image.go b/api/ws/handlers/image.go index 4ab7adc7..b3bdafa4 100644 --- a/api/ws/handlers/image.go +++ b/api/ws/handlers/image.go @@ -1,11 +1,9 @@ package wsHandler import ( - "bytes" "context" "errors" "fmt" - "image" "net/http" "path" "strings" @@ -338,7 +336,7 @@ func HandleImageUploadReq(req *protos.ImageUploadReq, hctx wsHelpers.HandlerCont db := hctx.Svcs.MongoDB // Save image meta in collection - img, _, err := image.Decode(bytes.NewReader(req.ImageData)) + imgWidth, imgHeight, err := utils.ReadImageDimensions(req.Name, req.ImageData) if err != nil { return nil, err } @@ -364,7 +362,9 @@ func HandleImageUploadReq(req *protos.ImageUploadReq, hctx wsHelpers.HandlerCont req.OriginScanId, "", req.GetBeamImageRef(), - img) + imgWidth, + imgHeight, + ) coll = db.Collection(dbCollections.ImagesName) diff --git a/core/utils/images.go b/core/utils/images.go index 21f60c33..35812483 100644 --- a/core/utils/images.go +++ b/core/utils/images.go @@ -28,18 +28,21 @@ import ( protos "github.com/pixlise/core/v4/generated-protos" ) -func ReadImageFile(path string) (image.Image, error) { - // Load the full context image from test data - imgbytes, err := os.ReadFile(path) +// Returns width, height and error +func ReadImageDimensions(imageName string, imgBytes []byte) (uint32, uint32, error) { + // Try to read the image + img, _, err := image.Decode(bytes.NewReader(imgBytes)) if err != nil { - return nil, err - } + // Failed, maybe it's a RGBU TIF image + upperName := strings.ToUpper(imageName) + if strings.Contains(err.Error(), "sample format") && (strings.Contains(upperName, "VIS_") || strings.Contains(upperName, "MSA_")) && strings.HasSuffix(upperName, ".TIF") { + // We can't read these tif files, but it's an RGBU image, and they have a known resolution - the same as our MCC images + return 752, 580, nil + } - img, _, err := image.Decode(bytes.NewReader(imgbytes)) - if err != nil { - return nil, err + return 0, 0, err } - return img, nil + return uint32(img.Bounds().Dx()), uint32(img.Bounds().Dy()), nil } func ImagesEqual(aPath, bPath string) error { @@ -111,13 +114,14 @@ func MakeScanImage( originScanId string, originImageURL string, matchInfo *protos.ImageMatchTransform, - img image.Image) *protos.ScanImage { + width uint32, + height uint32) *protos.ScanImage { result := &protos.ScanImage{ ImagePath: imgPath, Source: source, - Width: uint32(img.Bounds().Dx()), - Height: uint32(img.Bounds().Dy()), + Width: width, + Height: height, FileSize: fileSize, Purpose: purpose, From c886a7f66ec6131337337858b43b2a49bdd10d96 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 12:06:10 +1000 Subject: [PATCH 17/19] Log meta extracted from FM dataset creation --- api/dataimport/internal/converters/pixlfm/import.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/api/dataimport/internal/converters/pixlfm/import.go b/api/dataimport/internal/converters/pixlfm/import.go index acb117f7..145fcb3f 100644 --- a/api/dataimport/internal/converters/pixlfm/import.go +++ b/api/dataimport/internal/converters/pixlfm/import.go @@ -272,6 +272,8 @@ func (p PIXLFM) Import(importPath string, pseudoIntensityRangesPath string, data return nil, "", err } // Stop after first file + log.Infof("Extracting metadata from file name: %v", file) + log.Infof("Meta extracted: %v", housekeepingFileNameMeta.ToString()) break } case "pseudoIntensityDir": From 63abf402e3784d1357710738e37551a04c91dce6 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 12:38:12 +1000 Subject: [PATCH 18/19] Bring imported dataset meta in line with migrated dataset meta --- api/dataimport/internal/output/summary.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/api/dataimport/internal/output/summary.go b/api/dataimport/internal/output/summary.go index 1f155d1f..10721293 100644 --- a/api/dataimport/internal/output/summary.go +++ b/api/dataimport/internal/output/summary.go @@ -67,12 +67,12 @@ func makeSummaryFileContent( } saveMeta := map[string]string{ - "TargetID": exp.TargetId, - "SiteID": fmt.Sprintf("%v", meta.SiteID), - "DriveID": fmt.Sprintf("%v", meta.DriveID), + "TargetId": exp.TargetId, + "SiteId": fmt.Sprintf("%v", meta.SiteID), + "DriveId": fmt.Sprintf("%v", meta.DriveID), "Target": meta.Target, "Site": meta.Site, - "SOL": meta.SOL, + "Sol": meta.SOL, "RTT": meta.RTT, "SCLK": fmt.Sprintf("%v", meta.SCLK), } From 2ccd6eec3f644f55d52f5a73c4953a46cd18956d Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 27 Mar 2024 13:21:34 +1000 Subject: [PATCH 19/19] Fixing example tests which rely on json output which was not deterministic --- api/coreg/import_test.go | 13 ++----------- api/dataimport/for-trigger_test.go | 24 ++++++++++++++---------- core/utils/jsonprint.go | 29 +++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 21 deletions(-) create mode 100644 core/utils/jsonprint.go diff --git a/api/coreg/import_test.go b/api/coreg/import_test.go index 3a18d29e..85fd0429 100644 --- a/api/coreg/import_test.go +++ b/api/coreg/import_test.go @@ -1,10 +1,10 @@ package coreg import ( - "encoding/json" "fmt" "log" + "github.com/pixlise/core/v4/core/utils" protos "github.com/pixlise/core/v4/generated-protos" "google.golang.org/protobuf/encoding/protojson" ) @@ -16,16 +16,7 @@ func printWarpXform(xform *protos.ImageMatchTransform, name string, err error) { log.Fatalln(err) } else { // Proto isn't deterministic, so we read as JSON and write it back out again - var anyJson map[string]interface{} - err = json.Unmarshal(b, &anyJson) - if err != nil { - log.Fatalln(err) - } - b, err = json.MarshalIndent(anyJson, "", " ") - if err != nil { - log.Fatalln(err) - } - fmt.Printf("%v\n", string(b)) + fmt.Printf("%v\n", utils.MakeDeterministicJSON(b, false)) } } diff --git a/api/dataimport/for-trigger_test.go b/api/dataimport/for-trigger_test.go index 784933e2..81d49531 100644 --- a/api/dataimport/for-trigger_test.go +++ b/api/dataimport/for-trigger_test.go @@ -20,6 +20,7 @@ package dataimport import ( "context" "fmt" + "log" "os" "strings" @@ -28,6 +29,7 @@ import ( "github.com/pixlise/core/v4/core/fileaccess" "github.com/pixlise/core/v4/core/logger" "github.com/pixlise/core/v4/core/scan" + "github.com/pixlise/core/v4/core/utils" "github.com/pixlise/core/v4/core/wstestlib" protos "github.com/pixlise/core/v4/generated-protos" "go.mongodb.org/mongo-driver/mongo" @@ -205,7 +207,7 @@ func Example_importForTrigger_OCS_Archive_Exists() { // Log shows exists in archive: true } -func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database) { +func printArchiveOKLogOutput(logger *logger.StdOutLoggerForTest, db *mongo.Database) { // Ensure these log msgs appeared... requiredLogs := []string{ "Downloading archived zip files...", @@ -221,7 +223,7 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database } for _, msg := range requiredLogs { - fmt.Printf("Logged \"%v\": %v\n", msg, log.LogContains(msg)) + fmt.Printf("Logged \"%v\": %v\n", msg, logger.LogContains(msg)) } // Dump contents of summary file, this verifies most things imported as expected @@ -234,8 +236,10 @@ func printArchiveOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database summary.TimestampUnixSec = 0 b, err := protojson.Marshal(summary) - s := strings.ReplaceAll(string(b), " ", "") - fmt.Printf("%v|%v\n", err, s) + if err != nil { + log.Fatalf("%v", err) + } + fmt.Printf("%v|%v\n", err, utils.MakeDeterministicJSON(b, true)) } // Import FM-style (simulate trigger by OCS pipeline), file goes to archive, then all files downloaded from archive and dataset created @@ -301,7 +305,7 @@ func Example_importForTrigger_OCS_Archive_OK() { // Logged "Diffraction db saved successfully": true // Logged "Applying custom title: Naltsos": true // Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true - // |{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"} } // Import FM-style (simulate trigger by dataset edit screen), should create dataset with custom name+image @@ -354,7 +358,7 @@ func Example_importForTrigger_OCS_DatasetEdit() { // Logged "Diffraction db saved successfully": true // Logged "Applying custom title: Naltsos": true // Logged "Matched aligned image: PCCR0577_0718181212_000MSA_N029000020073728500030LUD01.tif, offset(0, 0), scale(1, 1). Match for aligned index: 0": true - // |{"id":"048300551","title":"Naltsos","dataTypes":[{"dataType":"SD_IMAGE","count":5},{"dataType":"SD_RGBU","count":1},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_FM","instrumentConfig":"PIXL","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"contentCounts": {"BulkSpectra": 2,"DwellSpectra": 0,"MaxSpectra": 2,"NormalSpectra": 242,"PseudoIntensities": 121},"creatorUserId": "PIXLISEImport","dataTypes": [{"count": 5,"dataType": "SD_IMAGE"},{"count": 1,"dataType": "SD_RGBU"},{"count": 242,"dataType": "SD_XRF"}],"id": "048300551","instrument": "PIXL_FM","instrumentConfig": "PIXL","meta": {"DriveId": "1712","RTT": "048300551","SCLK": "678031418","Site": "","SiteId": "4","Sol": "0125","Target": "","TargetId": "?"},"title": "Naltsos"} } func printManualOKLogOutput(log *logger.StdOutLoggerForTest, db *mongo.Database, datasetId string, fileCount uint32) { @@ -423,7 +427,7 @@ func Example_importForTrigger_Manual_JPL() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": true // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"} + // |{"id":"test1234","title":"test1234","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"JPL_BREADBOARD","instrumentConfig":"Breadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"JPLImport"} } // Import a breadboard dataset from manual uploaded zip file @@ -456,7 +460,7 @@ func Example_importForTrigger_Manual_SBU() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} + // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } // Import a breadboard dataset from manual uploaded zip file @@ -489,7 +493,7 @@ func Example_ImportForTrigger_Manual_SBU_NoAutoShare() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": true - // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveID":"0","RTT":"","SCLK":"0","SOL":"","Site":"","SiteID":"0","Target":"","TargetID":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} + // |{"id":"test1234sbu","title":"test1234sbu","dataTypes":[{"dataType":"SD_XRF","count":2520}],"instrument":"SBU_BREADBOARD","instrumentConfig":"StonyBrookBreadboard","meta":{"DriveId":"0","RTT":"","SCLK":"0","Site":"","SiteId":"0","Sol":"","Target":"","TargetId":"0"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":2520,"PseudoIntensities":0},"creatorUserId":"SBUImport"} } /* Didnt get this working when the above was changed. Problem is this still generates the user name: SBUImport, so the @@ -540,7 +544,7 @@ func Example_importForTrigger_Manual_EM() { // Logged "Diffraction db saved successfully": true // Logged "Warning: No import.json found, defaults will be used": false // Logged "No auto-share destination found, so only importing user will be able to access this dataset.": false - // |{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveID":"1712","RTT":"048300551","SCLK":"678031418","SOL":"0125","Site":"","SiteID":"4","Target":"","TargetID":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} + // |{"id":"048300551","title":"048300551","dataTypes":[{"dataType":"SD_IMAGE","count":4},{"dataType":"SD_XRF","count":242}],"instrument":"PIXL_EM","instrumentConfig":"PIXL-EM-E2E","meta":{"DriveId":"1712","RTT":"048300551","SCLK":"678031418","Site":"","SiteId":"4","Sol":"0125","Target":"","TargetId":"?"},"contentCounts":{"BulkSpectra":2,"DwellSpectra":0,"MaxSpectra":2,"NormalSpectra":242,"PseudoIntensities":121},"creatorUserId":"PIXLISEImport"} } /* NOT TESTED YET, because it's not done yet! diff --git a/core/utils/jsonprint.go b/core/utils/jsonprint.go new file mode 100644 index 00000000..1e3cff47 --- /dev/null +++ b/core/utils/jsonprint.go @@ -0,0 +1,29 @@ +package utils + +import ( + "encoding/json" + "log" + "strings" +) + +func MakeDeterministicJSON(b []byte, flat bool) string { + var anyJson map[string]interface{} + err := json.Unmarshal(b, &anyJson) + if err != nil { + log.Fatalln(err) + } + indent := "" + if !flat { + indent = " " + } + b2, err := json.MarshalIndent(anyJson, "", indent) + if err != nil { + log.Fatalln(err) + } + + result := string(b2) + if flat { + result = strings.ReplaceAll(result, "\n", "") + } + return result +}