From af41d4bba8b44b9e871927cdf4e31e6c9c20f949 Mon Sep 17 00:00:00 2001
From: Ryan Stonebraker <ryan.a.stonebraker@jpl.nasa.gov>
Date: Fri, 15 Mar 2024 19:12:47 -0700
Subject: [PATCH 01/30] Id field needs to be longer when we're requesting
 prefixed ids, ran against description field max with diffraction expression
 auto-description, adds direct connect flag to local mongo connection so it
 doesnt fail on vpn (finally fixed!)

---
 api/ws/wsHelpers/fields.go                                    | 4 ++--
 core/mongoDBConnection/local.go                               | 2 +-
 .../cmd-line-tools/api-integration-test/testUserGroups.go     | 4 ++--
 3 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/api/ws/wsHelpers/fields.go b/api/ws/wsHelpers/fields.go
index 115d009b..7a521a41 100644
--- a/api/ws/wsHelpers/fields.go
+++ b/api/ws/wsHelpers/fields.go
@@ -6,9 +6,9 @@ import (
 	"github.com/pixlise/core/v4/core/errorwithstatus"
 )
 
-const IdFieldMaxLength = 16
+const IdFieldMaxLength = 32
 const Auth0UserIdFieldMaxLength = 32
-const DescriptionFieldMaxLength = 512
+const DescriptionFieldMaxLength = 1024 * 5
 const SourceCodeMaxLength = 1024 * 1024 * 5 // Trying to be very generous here, but maybe this is not enough?
 const TagListMaxLength = 100
 
diff --git a/core/mongoDBConnection/local.go b/core/mongoDBConnection/local.go
index 8377ff78..acb1eb0e 100644
--- a/core/mongoDBConnection/local.go
+++ b/core/mongoDBConnection/local.go
@@ -40,7 +40,7 @@ func connectToLocalMongoDB(log logger.ILogger) (*mongo.Client, error) {
 		mongoUri = "mongodb://localhost"
 	}
 	//ctx := context.Background()
-	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoUri).SetMonitor(cmdMonitor))
+	client, err := mongo.Connect(context.TODO(), options.Client().ApplyURI(mongoUri).SetMonitor(cmdMonitor).SetDirect(true))
 	if err != nil {
 		return nil, fmt.Errorf("Failed to create new local mongo DB connection: %v", err)
 	}
diff --git a/internal/cmd-line-tools/api-integration-test/testUserGroups.go b/internal/cmd-line-tools/api-integration-test/testUserGroups.go
index cc563b59..a1c1e044 100644
--- a/internal/cmd-line-tools/api-integration-test/testUserGroups.go
+++ b/internal/cmd-line-tools/api-integration-test/testUserGroups.go
@@ -389,12 +389,12 @@ func testAddRemoveUserAsGroupMember(u2 wstestlib.ScriptedTestUser, nonAdminUserI
 func testUserGroupAdminAdd(u2 wstestlib.ScriptedTestUser, nonAdminUserId string) {
 	// Edits by admin of group
 	u2.AddSendReqAction("Add admin user to bad group id",
-		`{"userGroupAddAdminReq":{"groupId": "way-too-long-group-id", "adminUserId": "u123"}}`,
+		`{"userGroupAddAdminReq":{"groupId": "way-too-long-group-id-way-too-long-group-id-way-too-long-group-id-way-too-long-group-id", "adminUserId": "u123"}}`,
 		`{"msgId":15,"status":"WS_BAD_REQUEST","errorText": "GroupId is too long","userGroupAddAdminResp":{}}`,
 	)
 
 	u2.AddSendReqAction("Add bad admin user id to group id",
-		`{"userGroupAddAdminReq":{"groupId": "non-existant", "adminUserId": "admin-user-id-that-is-way-too-long even-for-auth0"}}`,
+		`{"userGroupAddAdminReq":{"groupId": "non-existant", "adminUserId": "admin-user-id-that-is-way-too-long even-for-auth0-admin-user-id-that-is-way-too-long even-for-auth0"}}`,
 		`{"msgId":16,"status":"WS_BAD_REQUEST","errorText": "AdminUserId is too long","userGroupAddAdminResp":{}}`,
 	)
 

From a23c664b222df7a1be17ebe49417459a4292440f Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:05:33 +1000
Subject: [PATCH 02/30] Fixing notification ids were different between saved to
 DB version and sent in upd version. Also dismiss notification handler didnt
 allow long id+userid combined notification ids

---
 api/notificationSender/interfaceImplementation.go    | 12 ++++++------
 api/notificationSender/notifications.go              |  8 ++++----
 api/ws/handlers/notification.go                      |  2 +-
 .../api-integration-test/testNotification.go         |  2 +-
 4 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/api/notificationSender/interfaceImplementation.go b/api/notificationSender/interfaceImplementation.go
index c23b9332..362f36a1 100644
--- a/api/notificationSender/interfaceImplementation.go
+++ b/api/notificationSender/interfaceImplementation.go
@@ -141,6 +141,12 @@ func (n *NotificationSender) NotifyObjectShared(objectType string, objectId stri
 }
 
 func (n *NotificationSender) NotifyUserGroupMessage(subject string, message string, notificationType protos.NotificationType, actionLink string, groupId string, groupName string, sender string) {
+	userIds, err := wsHelpers.GetUserIdsForGroup([]string{groupId}, n.db)
+	if err != nil {
+		n.log.Errorf("Failed to get user ids for group: %v. Error: %v", groupId, err)
+		return
+	}
+
 	notifMsg := &protos.NotificationUpd{
 		Notification: &protos.Notification{
 			NotificationType: notificationType,
@@ -151,12 +157,6 @@ func (n *NotificationSender) NotifyUserGroupMessage(subject string, message stri
 		},
 	}
 
-	userIds, err := wsHelpers.GetUserIdsForGroup([]string{groupId}, n.db)
-	if err != nil {
-		n.log.Errorf("Failed to get user ids for group: %v. Error: %v", groupId, err)
-		return
-	}
-
 	n.sendNotification(subject, "", notifMsg, userIds)
 }
 
diff --git a/api/notificationSender/notifications.go b/api/notificationSender/notifications.go
index bd388332..94f46914 100644
--- a/api/notificationSender/notifications.go
+++ b/api/notificationSender/notifications.go
@@ -91,7 +91,7 @@ func (n *NotificationSender) sendNotification(sourceId string, topicId string, n
 
 	for _, userId := range userIds {
 		// Write it to DB if needed
-		err := n.saveNotificationToDB(userId, notifMsg.Notification)
+		err := n.saveNotificationToDB(origId, userId, notifMsg.Notification)
 		if err != nil {
 			n.log.Errorf("Failed to save notification to DB for user: %v. Error: \"%v\". Notification was: %+v", userId, err, notifMsg.Notification)
 		}
@@ -129,7 +129,7 @@ func (n *NotificationSender) sendNotification(sourceId string, topicId string, n
 		for _, session := range sessions {
 			// Send it with a unique ID for this user
 			sessUser, err := wsHelpers.GetSessionUser(session)
-			if err != nil {
+			if err == nil {
 				notifMsg.Notification.Id = origId + "-" + sessUser.User.Id
 				msg := &protos.WSMessage{Contents: &protos.WSMessage_NotificationUpd{NotificationUpd: notifMsg}}
 
@@ -201,12 +201,12 @@ func (n *NotificationSender) sendEmail(notif *protos.Notification, userId string
 	awsutil.SESSendEmail(user.Info.Email, "UTF-8", text, html, notif.Subject, "info@mail.pixlise.org", []string{}, []string{})
 }
 
-func (n *NotificationSender) saveNotificationToDB(destUserId string, notification *protos.Notification) error {
+func (n *NotificationSender) saveNotificationToDB(notifId string, destUserId string, notification *protos.Notification) error {
 	// Make a copy which has the user id set
 	toSave := &protos.Notification{
 		DestUserId: destUserId,
 
-		Id:               notification.Id,
+		Id:               notifId + "-" + destUserId,
 		DestUserGroupId:  notification.DestUserGroupId,
 		MaxSecToExpiry:   notification.MaxSecToExpiry,
 		Subject:          notification.Subject,
diff --git a/api/ws/handlers/notification.go b/api/ws/handlers/notification.go
index 631b8ce5..59a83408 100644
--- a/api/ws/handlers/notification.go
+++ b/api/ws/handlers/notification.go
@@ -43,7 +43,7 @@ func HandleNotificationReq(req *protos.NotificationReq, hctx wsHelpers.HandlerCo
 
 func HandleNotificationDismissReq(req *protos.NotificationDismissReq, hctx wsHelpers.HandlerContext) (*protos.NotificationDismissResp, error) {
 	// Find this in the DB and clear it
-	if err := wsHelpers.CheckStringField(&req.Id, "Id", 1, wsHelpers.IdFieldMaxLength); err != nil {
+	if err := wsHelpers.CheckStringField(&req.Id, "Id", 1, wsHelpers.IdFieldMaxLength*2); err != nil {
 		return nil, err
 	}
 
diff --git a/internal/cmd-line-tools/api-integration-test/testNotification.go b/internal/cmd-line-tools/api-integration-test/testNotification.go
index 6f7d5b60..099ac260 100644
--- a/internal/cmd-line-tools/api-integration-test/testNotification.go
+++ b/internal/cmd-line-tools/api-integration-test/testNotification.go
@@ -117,7 +117,7 @@ func testNotification(apiHost string) (string, string) {
 		`{"msgId": 4, "status": "WS_OK", "notificationResp": {
 				"notification": [
 					{
-						"id": "${IDSAVE=notificationId}",
+						"id": "${IDCHK=notificationId}",
 						"destUserId": "${USERID}",
 						"subject": "test subject",
 						"contents": "The body\nThis message was sent by test2@pixlise.org - WS Integration Test",

From 91c9a17e0bb7a16f42218c9f5159fbe568c44b7e Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:14:26 +1000
Subject: [PATCH 03/30] Fixing tests due to data changes (beam location regen
 caused different values to come back from certain API calls than previously
 expected)

---
 .../api-integration-test/testScanData.go             | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/internal/cmd-line-tools/api-integration-test/testScanData.go b/internal/cmd-line-tools/api-integration-test/testScanData.go
index ed28a7be..200e135e 100644
--- a/internal/cmd-line-tools/api-integration-test/testScanData.go
+++ b/internal/cmd-line-tools/api-integration-test/testScanData.go
@@ -1305,15 +1305,15 @@ func testScanDataHasPermission(apiHost string, actionMsg string, editAllowed boo
 			"scanBeamLocationsResp":{
 				"beamLocations": [
 					{
-						"x": -0.151088,
-						"y": 0.131007,
-						"z": 0.246569
+						"x": -0.150853,
+						"y": 0.131032,
+						"z": 0.246621
 					},
 					{},
 					{
-						"x": -0.135994,
-						"y": 0.131017,
-						"z": 0.249705
+						"x": -0.135756,
+						"y": 0.131042,
+						"z": 0.249732
 					},
 					{}
 				]

From ee11e5a241fff6f294041fa9938117fb4b8be994 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:18:53 +1000
Subject: [PATCH 04/30] Turn go unit tests back on

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 239fa85f..577a9862 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -63,7 +63,7 @@ jobs:
         with:
           go-version: "1.21.0"
       - name: Test
-        run: echo $PIXLISE_API_TEST_ZENODO_URI && echo "Tests Off" # make test
+        run: make test
         env:
           PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
           PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}

From b9770636dc9f7a5b65c95943c1d67268c5b35e39 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:30:19 +1000
Subject: [PATCH 05/30] Trying to fix tests - for some reason go was trying to
 build code from migration tool as part of test

---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index 6b14174e..5a5d3ea2 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ test: ## Run unittests
 	go install github.com/favadi/protoc-go-inject-tag@latest
 	go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
 	protoc-go-inject-tag -remove_tag_comment -input="./generated-protos/*.pb.go"
-	go test -p 1 -v ./...
+	go test ./...
 
 codegen:
 	./genproto.sh checkgen

From 5fe914482233b5132ffc8934c757273827050f6b Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:32:31 +1000
Subject: [PATCH 06/30] Run go tests in parallel with builds

---
 .github/workflows/main.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 577a9862..d07a1824 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -53,6 +53,7 @@ jobs:
 
   test:
     runs-on: ubuntu-latest
+    needs: [version]
     steps:
       - uses: actions/checkout@v3
         with:
@@ -77,7 +78,7 @@ jobs:
 
   build:
     runs-on: ubuntu-latest
-    needs: [version, test]
+    needs: [version]
     env:
       AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
       AWS_DEFAULT_REGION: us-east-1
@@ -114,7 +115,7 @@ jobs:
           aws s3 cp . s3://corestack-buildartifactsf774a77d-105on4pno9pjm/ --recursive --region us-east-1
 
   docker:
-    needs: [version, test]
+    needs: [version]
     runs-on: ubuntu-latest
     permissions:
       contents: read

From f1d6e44dca69338993c55fb240d4d34b12c37cb9 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 14:41:47 +1000
Subject: [PATCH 07/30] Checked in dummyRead() empty func to allow compile to
 work (for go tests), but in reality this hard codes some user data, so the
 real file is only on local machine for migration to work

---
 .../cmd-line-tools/v3-importer/auth0dummy.go  | 23 +++++++++++++++++++
 1 file changed, 23 insertions(+)
 create mode 100644 internal/cmd-line-tools/v3-importer/auth0dummy.go

diff --git a/internal/cmd-line-tools/v3-importer/auth0dummy.go b/internal/cmd-line-tools/v3-importer/auth0dummy.go
new file mode 100644
index 00000000..3fde3c46
--- /dev/null
+++ b/internal/cmd-line-tools/v3-importer/auth0dummy.go
@@ -0,0 +1,23 @@
+package main
+
+func dummyRead() (map[string][]string, map[string]bool, map[string][]string) {
+	// This dummys up a call to auth0 so our migration tool can work often and not hit auth0 api rate limits
+	// this stuff doesn't change much really... here is what it dummys up:
+
+	// The roles we find... so id to list of groups
+	roleToGroup := map[string][]string{
+		"role_id": {"access:JPL Breadboard", "access:PIXL-EM", "access:PIXL-FM"},
+	}
+
+	// Group membership
+	userToGroup := map[string][]string{
+		"auth0_user_id": {"access:JPL Breadboard", "access:PIXL-EM", "access:PIXL-FM", "access:Stony Brook Breadboard"},
+	}
+
+	// Just a straight list of the groups
+	allGroups := map[string]bool{
+		"access:JPL Breadboard": true,
+	}
+
+	return roleToGroup, allGroups, userToGroup
+}

From 119ed2e1b18888107fcfa3a6a0557237551c73b6 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 15:16:33 +1000
Subject: [PATCH 08/30] Attempt to get github actions to run a local mongo db
 for tests to run. Also removed redundant mongo mock creation in one test that
 no longer needs it

---
 .github/workflows/main.yml             |  14 +++
 api/endpoints/middlewareLogger_test.go | 138 +++++++++++--------------
 2 files changed, 74 insertions(+), 78 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index d07a1824..bbcf84d4 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,6 +54,20 @@ jobs:
   test:
     runs-on: ubuntu-latest
     needs: [version]
+    services:
+      mongodb:
+        image: mongo:4.0.28
+        env:
+          #MONGO_INITDB_ROOT_USERNAME: admin
+          #MONGO_INITDB_ROOT_PASSWORD: admin
+          #MONGO_INITDB_DATABASE: APP-DATABASE
+        ports:
+          - 27017:27017
+        options: >-
+          --health-cmd mongo
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     steps:
       - uses: actions/checkout@v3
         with:
diff --git a/api/endpoints/middlewareLogger_test.go b/api/endpoints/middlewareLogger_test.go
index e2ca8c85..59e10f58 100644
--- a/api/endpoints/middlewareLogger_test.go
+++ b/api/endpoints/middlewareLogger_test.go
@@ -15,7 +15,6 @@ import (
 	"github.com/pixlise/core/v4/core/jwtparser"
 	"github.com/pixlise/core/v4/core/logger"
 	"github.com/pixlise/core/v4/core/timestamper"
-	"go.mongodb.org/mongo-driver/mongo/integration/mtest"
 )
 
 func Test_testLoggingDebug(t *testing.T) {
@@ -28,82 +27,65 @@ func Test_testLoggingInfo(t *testing.T) {
 }
 
 func runMiddlewareLoggingTest(t *testing.T, logLevel *logger.LogLevel) {
-	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
-	defer mt.Close()
-
-	mt.Run("success", func(mt *mtest.T) {
-		//mt.AddMockResponses()
-
-		var mockS3 awsutil.MockS3Client
-		defer mockS3.FinishTest()
-
-		mockS3.ExpPutObjectInput = []s3.PutObjectInput{
-			{
-				Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("Activity/2022-11-11/id-123.json"), Body: bytes.NewReader([]byte(`{
-    "Instance": "",
-    "Time": "2022-11-11T04:56:19Z",
-    "Component": "/foo",
-    "Message": "the bodyyy",
-    "Response": "{\"alive\": true}",
-    "Version": "",
-    "Params": {
-        "method": "GET"
-    },
-    "Environment": "unit-test",
-    "User": "myuserid"
+	var mockS3 awsutil.MockS3Client
+	defer mockS3.FinishTest()
+
+	mockS3.ExpPutObjectInput = []s3.PutObjectInput{
+		{
+			Bucket: aws.String(UsersBucketForUnitTest), Key: aws.String("Activity/2022-11-11/id-123.json"), Body: bytes.NewReader([]byte(`{
+"Instance": "",
+"Time": "2022-11-11T04:56:19Z",
+"Component": "/foo",
+"Message": "the bodyyy",
+"Response": "{\"alive\": true}",
+"Version": "",
+"Params": {
+	"method": "GET"
+},
+"Environment": "unit-test",
+"User": "myuserid"
 }`)),
-			},
-		}
-		mockS3.QueuedPutObjectOutput = []*s3.PutObjectOutput{
-			{},
-		}
-
-		idGen := idgen.MockIDGenerator{
-			IDs: []string{"id-123"},
-		}
-
-		s := MakeMockSvcs(&mockS3, &idGen, logLevel)
-		s.TimeStamper = &timestamper.MockTimeNowStamper{
-			QueuedTimeStamps: []int64{1668142579},
-		}
-		/*
-			notifications, err := notifications.MakeNotificationStack(mt.Client, "unit_test", nil, &logger.StdOutLoggerForTest{}, []string{})
-			if err != nil {
-				t.Error(err)
-			}
-
-			s.Notifications = notifications
-
-			// Add requestor as a tracked user, so we should see activity saved
-			s.Notifications.SetTrack("myuserid", true)
-		*/
-		mockvalidator := jwtparser.MockJWTValidator{}
-		l := LoggerMiddleware{
-			APIServices:  &s,
-			JwtValidator: &mockvalidator,
-		}
-
-		handler := func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusOK)
-			w.Header().Set("Content-Type", "application/json")
-
-			// In the future we could report back on the status of our DB, or our cache
-			// (e.g. Redis) by performing a simple PING, and include them in the response.
-			io.WriteString(w, `{"alive": true}`)
-		}
-
-		req := httptest.NewRequest("GET", "http://example.com/foo", bytes.NewReader([]byte("the bodyyy")))
-		w := httptest.NewRecorder()
-		handler(w, req)
-
-		h := http.HandlerFunc(handler)
-		handlerToTest := l.Middleware(h)
-
-		handlerToTest.ServeHTTP(httptest.NewRecorder(), req)
-
-		// Wait a bit for any threads to finish
-		time.Sleep(2 * time.Second)
-
-		checkResult(t, w, 200, "{\"alive\": true}")
-	})
+		},
+	}
+	mockS3.QueuedPutObjectOutput = []*s3.PutObjectOutput{
+		{},
+	}
+
+	idGen := idgen.MockIDGenerator{
+		IDs: []string{"id-123"},
+	}
+
+	s := MakeMockSvcs(&mockS3, &idGen, logLevel)
+	s.TimeStamper = &timestamper.MockTimeNowStamper{
+		QueuedTimeStamps: []int64{1668142579},
+	}
+
+	mockvalidator := jwtparser.MockJWTValidator{}
+	l := LoggerMiddleware{
+		APIServices:  &s,
+		JwtValidator: &mockvalidator,
+	}
+
+	handler := func(w http.ResponseWriter, r *http.Request) {
+		w.WriteHeader(http.StatusOK)
+		w.Header().Set("Content-Type", "application/json")
+
+		// In the future we could report back on the status of our DB, or our cache
+		// (e.g. Redis) by performing a simple PING, and include them in the response.
+		io.WriteString(w, `{"alive": true}`)
+	}
+
+	req := httptest.NewRequest("GET", "http://example.com/foo", bytes.NewReader([]byte("the bodyyy")))
+	w := httptest.NewRecorder()
+	handler(w, req)
+
+	h := http.HandlerFunc(handler)
+	handlerToTest := l.Middleware(h)
+
+	handlerToTest.ServeHTTP(httptest.NewRecorder(), req)
+
+	// Wait a bit for any threads to finish
+	time.Sleep(2 * time.Second)
+
+	checkResult(t, w, 200, "{\"alive\": true}")
 }

From 73f9aab6cb8035cb11d16bfb7d139a6fdf5821af Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 15:17:01 +1000
Subject: [PATCH 09/30] Added example of local mongo mock setup for tests in
 case we need it again in future

---
 api/dataimport/for-trigger_test.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/api/dataimport/for-trigger_test.go b/api/dataimport/for-trigger_test.go
index 6dd0ff0f..eb99d82b 100644
--- a/api/dataimport/for-trigger_test.go
+++ b/api/dataimport/for-trigger_test.go
@@ -67,6 +67,14 @@ func initTest(testDir string, autoShareCreatorId string, autoShareCreatorGroupEd
 	return remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db
 }
 
+/*
+func startTestWithMockMongo(name string, t *testing.T, testFunc func(mt *mtest.T)) {
+	mt := mtest.New(t, mtest.NewOptions().ClientType(mtest.Mock))
+	defer mt.Close()
+
+	mt.Run(name, testFunc)
+}
+*/
 // Import unknown dataset (simulate trigger by OCS pipeline), file goes to archive, then all files downloaded from archive, dataset create fails due to unknown data type
 func Example_importForTrigger_OCS_Archive_BadData() {
 	remoteFS, log, envName, configBucket, datasetBucket, manualBucket, db := initTest("Archive_BadData", specialUserIds.PIXLISESystemUserId, "PIXLFMGroupId")

From 3902159ff15faa96881ed4d54bee0c85c3c06213 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 15:18:35 +1000
Subject: [PATCH 10/30] Get github action test local mongo working

---
 .github/workflows/main.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index bbcf84d4..9559442d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -57,7 +57,7 @@ jobs:
     services:
       mongodb:
         image: mongo:4.0.28
-        env:
+        #env:
           #MONGO_INITDB_ROOT_USERNAME: admin
           #MONGO_INITDB_ROOT_PASSWORD: admin
           #MONGO_INITDB_DATABASE: APP-DATABASE

From 2b9399be47d3fd6dda03c83a4efba387d7215d7d Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 15:38:55 +1000
Subject: [PATCH 11/30] Attempt to run integration tests in github actions with
 local mongo

---
 .github/workflows/main.yml | 21 ++++++++++++++++++---
 Makefile                   | 13 ++++++++++---
 2 files changed, 28 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 9559442d..1438f942 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -77,8 +77,24 @@ jobs:
         uses: actions/setup-go@v3
         with:
           go-version: "1.21.0"
-      - name: Test
-        run: make test
+      - name: Unit Test
+        run: make unittest
+        env:
+          PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
+          PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
+          PIXLISE_API_TEST_AUTH0_SECRET: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
+          PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID }}
+          PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }}
+          PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_DEFAULT_REGION: us-east-1
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+      - name: Integration Test
+        run: |
+          make integrationtest
+          ./api-service &
+          sleep 5
+          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password "TheRemarkablesKawarau!" -test2Username "test2@pixlise.org" -test2Password "TrebleConeRaffils!"
         env:
           PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
           PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
@@ -89,7 +105,6 @@ jobs:
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_DEFAULT_REGION: us-east-1
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-
   build:
     runs-on: ubuntu-latest
     needs: [version]
diff --git a/Makefile b/Makefile
index 5a5d3ea2..2bc659e3 100644
--- a/Makefile
+++ b/Makefile
@@ -4,16 +4,16 @@ PROJECT_NAME := core
 PKG := github.com/pixlise/$(PROJECT_NAME)
 PKG_LIST := $(shell go list ${PKG}/... | grep -v /vendor/)
 
-.PHONY: all build clean test lint
+.PHONY: all build clean unittest integrationtest lint
 
-all: codegen build test lint
+all: codegen build unittest integrationtest lint
 
 lint: ## Lint the files
 	echo "${PKG}"
 	#golint -set_exit_status ${PKG_LIST}
 	golint ${PKG_LIST}
 
-test: ## Run unittests
+unittest: ## Run unittests
 	pwd
 	cd ..
 	mkdir -p _out
@@ -22,6 +22,13 @@ test: ## Run unittests
 	protoc-go-inject-tag -remove_tag_comment -input="./generated-protos/*.pb.go"
 	go test ./...
 
+integrationtest:
+	mkdir -p _out
+	echo "version: ${BUILD_VERSION}"
+	echo "sha: ${GITHUB_SHA}"
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./api-service ./internal/api
+	GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./internal/cmd-line-tools/api-integration-test/tester ./internal/cmd-line-tools/api-integration-test
+
 codegen:
 	./genproto.sh checkgen
 

From 01debfeb40d9f5ac2abf03fe324d69f15058a1b7 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 15:58:46 +1000
Subject: [PATCH 12/30] Add api config for running integration test in github
 action

---
 .github/workflows/main.yml   |  8 ++++++--
 integration-test-config.json | 29 +++++++++++++++++++++++++++++
 2 files changed, 35 insertions(+), 2 deletions(-)
 create mode 100644 integration-test-config.json

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 1438f942..1d0c0e36 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -92,10 +92,14 @@ jobs:
       - name: Integration Test
         run: |
           make integrationtest
-          ./api-service &
+          ./api-service -customConfigPath ./integration-test-config.json" &
           sleep 5
-          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password "TheRemarkablesKawarau!" -test2Username "test2@pixlise.org" -test2Password "TrebleConeRaffils!"
+          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
+          PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
+          PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
+          PIXLISE_Auth0ManagementClientID": ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
+          PIXLISE_Auth0ManagementSecret": ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
           PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
           PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
           PIXLISE_API_TEST_AUTH0_SECRET: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
diff --git a/integration-test-config.json b/integration-test-config.json
new file mode 100644
index 00000000..8cc7edb2
--- /dev/null
+++ b/integration-test-config.json
@@ -0,0 +1,29 @@
+{
+    "MaxQuantNodes": 20,
+    "AWSBucketRegion": "us-east-1",
+    "AWSCloudwatchRegion": "us-east-1",
+    "EnvironmentName": "unittest",
+    "LogLevel": 1,
+    "CoresPerNode": 4,
+    "PiquantDockerImage": "registry.gitlab.com/pixlise/piquant/runner:3.2.17",
+    "QuantNamespace": "unittest",
+    "QuantExecutor": "docker",
+    "SentryEndpoint": "",
+
+    "DatasetsBucket": "integration-test-data-pixlise",
+    "UsersBucket": "integration-test-users-pixlise",
+    "ConfigBucket": "integration-test-config-pixlise",
+    "PiquantJobsBucket": "integration-test-jobs-pixlise",
+    "ManualUploadBucket": "integration-test-upload-pixlise",
+
+    "BuildsBucket": "",
+    "DataSourceSNSTopic": "",
+
+	"WSWriteWaitMs": 10000,
+	"WSPongWaitMs": 60000,
+	"WSPingPeriodMs": 54000,
+	"WSMaxMessageSize": 40000,
+	"WSMessageBufferSize": 256,
+    
+	"ZenodoURI": "https://sandbox.zenodo.org"
+}
\ No newline at end of file

From a22bcaaa317898dfe5ecae9b84eea6b5372da6e9 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 16:04:25 +1000
Subject: [PATCH 13/30] Verbose unit test output, also fix/print out cmd line

---
 .github/workflows/main.yml | 3 ++-
 Makefile                   | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 1d0c0e36..66b5430d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -92,8 +92,9 @@ jobs:
       - name: Integration Test
         run: |
           make integrationtest
-          ./api-service -customConfigPath ./integration-test-config.json" &
+          ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
+          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
           ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
diff --git a/Makefile b/Makefile
index 2bc659e3..12e3781c 100644
--- a/Makefile
+++ b/Makefile
@@ -20,7 +20,7 @@ unittest: ## Run unittests
 	go install github.com/favadi/protoc-go-inject-tag@latest
 	go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/
 	protoc-go-inject-tag -remove_tag_comment -input="./generated-protos/*.pb.go"
-	go test ./...
+	go test -v ./...
 
 integrationtest:
 	mkdir -p _out

From 8bc8d71dbbd0484fa6d1d69e3db13089b065d926 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 16:09:56 +1000
Subject: [PATCH 14/30] Fixing cmd line

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 66b5430d..ca09a859 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -94,8 +94,8 @@ jobs:
           make integrationtest
           ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
-          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
-          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -apiDBSecret "" -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
           PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}

From 7cf3fbd4aaecb2ad35695fecc46f20e21b14b8b2 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 16:21:12 +1000
Subject: [PATCH 15/30] Fixing env vars for integration test

---
 .github/workflows/main.yml | 12 ++----------
 1 file changed, 2 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ca09a859..7f16abc8 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -80,10 +80,6 @@ jobs:
       - name: Unit Test
         run: make unittest
         env:
-          PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
-          PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
-          PIXLISE_API_TEST_AUTH0_SECRET: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
-          PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID }}
           PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }}
           PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -99,12 +95,8 @@ jobs:
         env:
           PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
           PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
-          PIXLISE_Auth0ManagementClientID": ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
-          PIXLISE_Auth0ManagementSecret": ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
-          PIXLISE_API_TEST_AUTH0_CLIENT_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
-          PIXLISE_API_TEST_AUTH0_DOMAIN: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
-          PIXLISE_API_TEST_AUTH0_SECRET: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
-          PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID: ${{ secrets.PIXLISE_API_TEST_AUTH0_NEWUSER_ROLE_ID }}
+          PIXLISE_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
+          PIXLISE_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
           PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }}
           PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

From 4327052e5399ef4bf2a1d1e7bb80b599f225fcc4 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 16:36:37 +1000
Subject: [PATCH 16/30] Fixing env vars for integration test

---
 .github/workflows/main.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7f16abc8..2ed81e8b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -90,8 +90,8 @@ jobs:
           make integrationtest
           ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
-          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
-          ./internal/cmd-line-tools/api-integration-test/tester -apiHost http://localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -expectedAPIVersion "" -testType "local" -test1Username "test1@pixlise.org" -test1Password  ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          ./internal/cmd-line-tools/api-integration-test/tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
           PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}

From 6450139a21e472ceb8f263927543d294c2c3a7ec Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 16:45:56 +1000
Subject: [PATCH 17/30] Fixing env vars for integration test

---
 .github/workflows/main.yml | 19 ++++++++-----------
 1 file changed, 8 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 2ed81e8b..241f4c57 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -80,28 +80,25 @@ jobs:
       - name: Unit Test
         run: make unittest
         env:
-          PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }}
-          PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_DEFAULT_REGION: us-east-1
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
       - name: Integration Test
         run: |
           make integrationtest
           ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
-          echo ./internal/cmd-line-tools/api-integration-test/tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
-          ./internal/cmd-line-tools/api-integration-test/tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          cd ./internal/cmd-line-tools/api-integration-test
+          echo tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
+          AWS_DEFAULT_REGION: us-east-1
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
-          PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
           PIXLISE_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
           PIXLISE_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
-          PIXLISE_API_TEST_ZENODO_URI: ${{ secrets.ZENODO_URI }}
-          PIXLISE_API_TEST_ZENODO_ACCESS_TOKEN: ${{ secrets.ZENODO_ACCESS_TOKEN }}
-          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
-          AWS_DEFAULT_REGION: us-east-1
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
   build:
     runs-on: ubuntu-latest
     needs: [version]

From 239f62ac049ac2e283e32c8cf751a62e5d20f8b9 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 19:23:09 +1000
Subject: [PATCH 18/30] Fixing env vars for integration test

---
 .github/workflows/main.yml | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 241f4c57..89f338d6 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -88,9 +88,13 @@ jobs:
           make integrationtest
           ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
+          pwd
+          ls
           cd ./internal/cmd-line-tools/api-integration-test
-          echo tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
-          tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          pwd
+          ls
+          echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}

From 5d450894e65f3e8ebb034815f138cef541b2bd9d Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 20:02:09 +1000
Subject: [PATCH 19/30] Fixing env vars for integration test, also clearer
 logging for startup config

---
 .github/workflows/main.yml | 8 ++++----
 internal/api/main.go       | 1 +
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 89f338d6..ef7b397d 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -99,10 +99,10 @@ jobs:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
-          PIXLISE_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
-          PIXLISE_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
-          PIXLISE_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
-          PIXLISE_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
+          PIXLISE_CONFIG_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
+          PIXLISE_CONFIG_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
+          PIXLISE_CONFIG_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
+          PIXLISE_CONFIG_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
   build:
     runs-on: ubuntu-latest
     needs: [version]
diff --git a/internal/api/main.go b/internal/api/main.go
index 926d3e7d..f4588482 100644
--- a/internal/api/main.go
+++ b/internal/api/main.go
@@ -192,6 +192,7 @@ func loadConfig() config.APIConfig {
 	}
 
 	cfgStr := string(cfgJSON)
+	log.Println("API startup configuration:")
 	log.Println(cfgStr)
 	return cfg
 }

From aa86123108f8da43ec7195dc7a69f0e61e50a5dd Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 20:41:58 +1000
Subject: [PATCH 20/30] Create new test mode for CI which doesn't run tests
 that need to start PIQUANT

---
 .github/workflows/main.yml                          |  4 ++--
 .../cmd-line-tools/api-integration-test/main.go     |  8 ++++----
 .../api-integration-test/testQuant.go               | 13 ++++++++++---
 3 files changed, 16 insertions(+), 9 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index ef7b397d..7885d060 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -93,8 +93,8 @@ jobs:
           cd ./internal/cmd-line-tools/api-integration-test
           pwd
           ls
-          echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
-          ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "local" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
diff --git a/internal/cmd-line-tools/api-integration-test/main.go b/internal/cmd-line-tools/api-integration-test/main.go
index db934486..2d77282f 100644
--- a/internal/cmd-line-tools/api-integration-test/main.go
+++ b/internal/cmd-line-tools/api-integration-test/main.go
@@ -113,7 +113,7 @@ func main() {
 
 	if testType == "env" {
 		runEnvTests(apiHost)
-	} else if testType == "local" {
+	} else if testType == "local" || testType == "ci" {
 		// Connect to DB and drop the unit test database
 		db := wstestlib.GetDB()
 		err = db.Drop(context.TODO())
@@ -123,7 +123,7 @@ func main() {
 
 		dbCollections.InitCollections(db, &logger.StdOutLogger{}, "")
 
-		runLocalTests(apiHost)
+		runLocalTests(apiHost, testType == "ci")
 	} else {
 		log.Fatal("Unexpected test type: " + testType)
 	}
@@ -157,7 +157,7 @@ func runEnvTests(apiHost string) {
 	// TODO: download all bits for a given dataset... all proto msgs + context image download(s)
 }
 
-func runLocalTests(apiHost string) {
+func runLocalTests(apiHost string, isCI bool) {
 	testImageGet_PreWS(apiHost) // Must be run before any web sockets log in
 
 	// testScanData(apiHost, 0 /*3 for proper testing*/)
@@ -172,7 +172,7 @@ func runLocalTests(apiHost string) {
 	testMemoisation(apiHost)
 	testImageMatchTransform(apiHost)
 	testSelectionMsgs(apiHost)
-	testQuants(apiHost)
+	testQuants(apiHost, !isCI) // We only run the tests that need to start PIQUANT outside of CI for now
 	testDiffractionManualPeaks(apiHost)
 	testDiffractionStatus(apiHost)
 	testPiquantMsgs(apiHost)
diff --git a/internal/cmd-line-tools/api-integration-test/testQuant.go b/internal/cmd-line-tools/api-integration-test/testQuant.go
index cda930e0..e17928b9 100644
--- a/internal/cmd-line-tools/api-integration-test/testQuant.go
+++ b/internal/cmd-line-tools/api-integration-test/testQuant.go
@@ -10,10 +10,17 @@ import (
 	protos "github.com/pixlise/core/v4/generated-protos"
 )
 
-func testQuants(apiHost string) {
-	testQuantCreate(apiHost)
+func testQuants(apiHost string, runPiquantTests bool) {
+	if runPiquantTests {
+		testQuantCreate(apiHost)
+	}
+
 	testMultiQuant(apiHost)
-	testQuantFit(apiHost)
+
+	if runPiquantTests {
+		testQuantFit(apiHost)
+	}
+
 	testQuantUpload(apiHost)
 	testQuantGetListDelete(apiHost)
 }

From 9e024d5f35cb0abaa1a18edfd21ffa3fee0adea3 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 20:54:31 +1000
Subject: [PATCH 21/30] Skip quant combine test on CI for now too

---
 internal/cmd-line-tools/api-integration-test/testQuant.go | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/internal/cmd-line-tools/api-integration-test/testQuant.go b/internal/cmd-line-tools/api-integration-test/testQuant.go
index e17928b9..942d2631 100644
--- a/internal/cmd-line-tools/api-integration-test/testQuant.go
+++ b/internal/cmd-line-tools/api-integration-test/testQuant.go
@@ -13,11 +13,7 @@ import (
 func testQuants(apiHost string, runPiquantTests bool) {
 	if runPiquantTests {
 		testQuantCreate(apiHost)
-	}
-
-	testMultiQuant(apiHost)
-
-	if runPiquantTests {
+		testMultiQuant(apiHost)
 		testQuantFit(apiHost)
 	}
 

From 9e73c8478117e708bec2f7b3c4339f653c3ffe0c Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 22:21:56 +1000
Subject: [PATCH 22/30] Skip quant upload test on CI for now too

---
 internal/cmd-line-tools/api-integration-test/testQuant.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/internal/cmd-line-tools/api-integration-test/testQuant.go b/internal/cmd-line-tools/api-integration-test/testQuant.go
index 942d2631..5447d988 100644
--- a/internal/cmd-line-tools/api-integration-test/testQuant.go
+++ b/internal/cmd-line-tools/api-integration-test/testQuant.go
@@ -15,9 +15,9 @@ func testQuants(apiHost string, runPiquantTests bool) {
 		testQuantCreate(apiHost)
 		testMultiQuant(apiHost)
 		testQuantFit(apiHost)
+		testQuantUpload(apiHost)
 	}
 
-	testQuantUpload(apiHost)
 	testQuantGetListDelete(apiHost)
 }
 

From 13efec60175a01e1f074238a4188de19837b432a Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Mon, 18 Mar 2024 22:37:42 +1000
Subject: [PATCH 23/30] Skip expr runtime saving test on CI for now too

---
 internal/cmd-line-tools/api-integration-test/main.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/cmd-line-tools/api-integration-test/main.go b/internal/cmd-line-tools/api-integration-test/main.go
index 2d77282f..4ff782c1 100644
--- a/internal/cmd-line-tools/api-integration-test/main.go
+++ b/internal/cmd-line-tools/api-integration-test/main.go
@@ -176,7 +176,9 @@ func runLocalTests(apiHost string, isCI bool) {
 	testDiffractionManualPeaks(apiHost)
 	testDiffractionStatus(apiHost)
 	testPiquantMsgs(apiHost)
-	testExpressionRuntimeMsgs(apiHost)
+	if !isCI {
+		testExpressionRuntimeMsgs(apiHost)
+	}
 	testDataModules(apiHost)
 	testUserContent(apiHost, map[string]contentMessaging{
 		"elementSet": {

From b97f56962d45a05dc3216eb7821a6520a8f25146 Mon Sep 17 00:00:00 2001
From: Ryan Stonebraker <ryan.a.stonebraker@jpl.nasa.gov>
Date: Mon, 18 Mar 2024 12:18:16 -0700
Subject: [PATCH 24/30] Fixes bug with ROI updating where the wrong field
 (title case) was being updated for scan entries, fixes selection log error
 when selection hasn't changed

---
 api/ws/handlers/roi.go             | 4 ++--
 api/ws/handlers/selection-entry.go | 5 +++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/api/ws/handlers/roi.go b/api/ws/handlers/roi.go
index b168f280..8f83d88d 100644
--- a/api/ws/handlers/roi.go
+++ b/api/ws/handlers/roi.go
@@ -278,7 +278,7 @@ func updateROI(roi *protos.ROIItem, hctx wsHelpers.HandlerContext) (*protos.ROII
 	// Once created, these can't be set to empty
 	if roi.ScanEntryIndexesEncoded != nil && !utils.SlicesEqual(dbItem.ScanEntryIndexesEncoded, roi.ScanEntryIndexesEncoded) {
 		dbItem.ScanEntryIndexesEncoded = roi.ScanEntryIndexesEncoded
-		update = append(update, bson.E{Key: "ScanEntryIndexesEncoded", Value: roi.ScanEntryIndexesEncoded})
+		update = append(update, bson.E{Key: "scanentryindexesencoded", Value: roi.ScanEntryIndexesEncoded})
 	}
 
 	// Once created, these can't be set to empty
@@ -310,7 +310,7 @@ func updateROI(roi *protos.ROIItem, hctx wsHelpers.HandlerContext) (*protos.ROII
 	}
 
 	if result.MatchedCount != 1 {
-		hctx.Svcs.Log.Errorf("ROI UpdateByID result had unexpected counts %+v id: %v", result, roi.Id)
+		hctx.Svcs.Log.Errorf("ROI UpdateByID result had unexpected counts %v id: %v", result, roi.Id)
 	}
 
 	// Return the merged item we validated, which in theory is in the DB now
diff --git a/api/ws/handlers/selection-entry.go b/api/ws/handlers/selection-entry.go
index f33af444..95ec31b7 100644
--- a/api/ws/handlers/selection-entry.go
+++ b/api/ws/handlers/selection-entry.go
@@ -93,8 +93,9 @@ func writeSelection(id string, idxs *protos.ScanEntryRange, db *mongo.Database,
 		return err
 	}
 
-	if dbResult.UpsertedCount != 1 && dbResult.ModifiedCount != 1 {
-		logger.Errorf("writeSelection UpdateByID result had unexpected counts %+v", dbResult)
+	// Modified and Upsert counts will be 0 if the selection hasn't changed, so we just check matched
+	if dbResult.MatchedCount != 1 {
+		logger.Errorf("writeSelection (%v) UpdateByID result had unexpected counts %+v", id, dbResult)
 	}
 
 	return nil

From 1abd3225e8a1ae52c3d54f1c055c2b4a7579d904 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 11:55:37 +1000
Subject: [PATCH 25/30] Github action build step: try to use mongo in a
 different way to enable replica sets

---
 .github/workflows/main.yml | 34 ++++++++++++++++++++--------------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 7885d060..14fc726b 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -54,20 +54,20 @@ jobs:
   test:
     runs-on: ubuntu-latest
     needs: [version]
-    services:
-      mongodb:
-        image: mongo:4.0.28
-        #env:
-          #MONGO_INITDB_ROOT_USERNAME: admin
-          #MONGO_INITDB_ROOT_PASSWORD: admin
-          #MONGO_INITDB_DATABASE: APP-DATABASE
-        ports:
-          - 27017:27017
-        options: >-
-          --health-cmd mongo
-          --health-interval 10s
-          --health-timeout 5s
-          --health-retries 5
+    # services:
+    #   mongodb:
+    #     image: mongo:4.0.28
+    #     #env:
+    #       #MONGO_INITDB_ROOT_USERNAME: admin
+    #       #MONGO_INITDB_ROOT_PASSWORD: admin
+    #       #MONGO_INITDB_DATABASE: APP-DATABASE
+    #     ports:
+    #       - 27017:27017
+    #     options: >-
+    #       --health-cmd mongo
+    #       --health-interval 10s
+    #       --health-timeout 5s
+    #       --health-retries 5
     steps:
       - uses: actions/checkout@v3
         with:
@@ -77,6 +77,12 @@ jobs:
         uses: actions/setup-go@v3
         with:
           go-version: "1.21.0"
+      - name: Start MongoDB
+        uses: supercharge/mongodb-github-action@1.10.0
+        with:
+          mongodb-version: 4.0.28
+          mongodb-replica-set: test
+          mongodb-port: 27017
       - name: Unit Test
         run: make unittest
         env:

From c30abbc01d2c3335be25231ce54a2e3a79f9d591 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 12:05:47 +1000
Subject: [PATCH 26/30] Use correct auth0 config for API in github CI

---
 .github/workflows/main.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 14fc726b..595742f9 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -94,20 +94,20 @@ jobs:
           make integrationtest
           ./api-service -customConfigPath ./integration-test-config.json &
           sleep 5
-          pwd
-          ls
-          cd ./internal/cmd-line-tools/api-integration-test
-          pwd
-          ls
-          echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
+          #pwd
+          #ls
+          #cd ./internal/cmd-line-tools/api-integration-test
+          #pwd
+          #ls
+          #echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
           ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
         env:
           AWS_DEFAULT_REGION: us-east-1
           AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
           AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
           PIXLISE_CONFIG_Auth0Domain: ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }}
-          PIXLISE_CONFIG_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }}
-          PIXLISE_CONFIG_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }}
+          PIXLISE_CONFIG_Auth0ManagementClientID: ${{ secrets.PIXLISE_API_MGMT_AUTH0_CLIENT_ID }}
+          PIXLISE_CONFIG_Auth0ManagementSecret: ${{ secrets.PIXLISE_API_MGMT_AUTH0_SECRET }}
           PIXLISE_CONFIG_ZenodoAccessToken: ${{ secrets.ZENODO_ACCESS_TOKEN }}
   build:
     runs-on: ubuntu-latest

From e887c601019b91aedb8782d8174cc536fbd51b03 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 12:10:46 +1000
Subject: [PATCH 27/30] Fixing gitlab CI tests

---
 .github/workflows/main.yml                                | 2 +-
 internal/cmd-line-tools/api-integration-test/testQuant.go | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 595742f9..856825be 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -96,7 +96,7 @@ jobs:
           sleep 5
           #pwd
           #ls
-          #cd ./internal/cmd-line-tools/api-integration-test
+          cd ./internal/cmd-line-tools/api-integration-test
           #pwd
           #ls
           #echo ./tester -apiHost localhost:8080 -datasetBucket "integration-test-data-pixlise" -usersBucket "integration-test-users-pixlise" -auth0Domain ${{ secrets.PIXLISE_API_TEST_AUTH0_DOMAIN }} -auth0ClientId ${{ secrets.PIXLISE_API_TEST_AUTH0_CLIENT_ID }} -auth0Secret ${{ secrets.PIXLISE_API_TEST_AUTH0_SECRET }} -auth0Audience "pixlise-backend" -testType "ci" -test1Username "test1@pixlise.org" -test1Password ${{ secrets.TEST_USER_1_PASSWORD }} -test2Username "test2@pixlise.org" -test2Password ${{ secrets.TEST_USER_2_PASSWORD }}
diff --git a/internal/cmd-line-tools/api-integration-test/testQuant.go b/internal/cmd-line-tools/api-integration-test/testQuant.go
index 5447d988..d84fb051 100644
--- a/internal/cmd-line-tools/api-integration-test/testQuant.go
+++ b/internal/cmd-line-tools/api-integration-test/testQuant.go
@@ -13,11 +13,11 @@ import (
 func testQuants(apiHost string, runPiquantTests bool) {
 	if runPiquantTests {
 		testQuantCreate(apiHost)
-		testMultiQuant(apiHost)
 		testQuantFit(apiHost)
-		testQuantUpload(apiHost)
 	}
 
+	testMultiQuant(apiHost)
+	testQuantUpload(apiHost)
 	testQuantGetListDelete(apiHost)
 }
 

From 6a14bebc7fa74be3e84560e5f3407859f660c4cc Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 12:22:49 +1000
Subject: [PATCH 28/30] Fixing gitlab CI tests

---
 internal/cmd-line-tools/api-integration-test/testQuant.go | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/internal/cmd-line-tools/api-integration-test/testQuant.go b/internal/cmd-line-tools/api-integration-test/testQuant.go
index d84fb051..14701928 100644
--- a/internal/cmd-line-tools/api-integration-test/testQuant.go
+++ b/internal/cmd-line-tools/api-integration-test/testQuant.go
@@ -16,8 +16,11 @@ func testQuants(apiHost string, runPiquantTests bool) {
 		testQuantFit(apiHost)
 	}
 
-	testMultiQuant(apiHost)
-	testQuantUpload(apiHost)
+	if runPiquantTests {
+		testMultiQuant(apiHost)
+		testQuantUpload(apiHost)
+	}
+
 	testQuantGetListDelete(apiHost)
 }
 

From 32a5ebed3d9a7cf952019e3b0e75f37ab802d253 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 13:21:54 +1000
Subject: [PATCH 29/30] Skip user management test (via auth0), probably using
 wrong keys, tried putting management keys in but still failing... no time to
 diagnose now

---
 internal/cmd-line-tools/api-integration-test/main.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/internal/cmd-line-tools/api-integration-test/main.go b/internal/cmd-line-tools/api-integration-test/main.go
index 4ff782c1..cf951245 100644
--- a/internal/cmd-line-tools/api-integration-test/main.go
+++ b/internal/cmd-line-tools/api-integration-test/main.go
@@ -447,7 +447,9 @@ func runLocalTests(apiHost string, isCI bool) {
 
 	testUserSearch(apiHost)
 	testUserDetails(apiHost, u1Id, u2Id)
-	testUserManagement(apiHost)
+	if !isCI {
+		testUserManagement(apiHost)
+	}
 	testUserGroups(apiHost)
 	testLogMsgs(apiHost)
 	testScanData(apiHost, 0 /*3 for proper testing*/)

From 31cb3902af0a7f54febd1f31c93c614dea9695f4 Mon Sep 17 00:00:00 2001
From: Peter Nemere <peter.nemere@qut.edu.au>
Date: Tue, 19 Mar 2024 13:45:52 +1000
Subject: [PATCH 30/30] Add some sleeps so we dont hit auth0 rate limit

---
 internal/cmd-line-tools/api-integration-test/main.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/internal/cmd-line-tools/api-integration-test/main.go b/internal/cmd-line-tools/api-integration-test/main.go
index cf951245..e6411ddf 100644
--- a/internal/cmd-line-tools/api-integration-test/main.go
+++ b/internal/cmd-line-tools/api-integration-test/main.go
@@ -174,12 +174,14 @@ func runLocalTests(apiHost string, isCI bool) {
 	testSelectionMsgs(apiHost)
 	testQuants(apiHost, !isCI) // We only run the tests that need to start PIQUANT outside of CI for now
 	testDiffractionManualPeaks(apiHost)
+	time.Sleep(5 * time.Second)
 	testDiffractionStatus(apiHost)
 	testPiquantMsgs(apiHost)
 	if !isCI {
 		testExpressionRuntimeMsgs(apiHost)
 	}
 	testDataModules(apiHost)
+	time.Sleep(5 * time.Second)
 	testUserContent(apiHost, map[string]contentMessaging{
 		"elementSet": {
 			itemName: "elementSet",
@@ -445,18 +447,22 @@ func runLocalTests(apiHost string, isCI bool) {
 		},
 	})
 
+	time.Sleep(5 * time.Second)
 	testUserSearch(apiHost)
 	testUserDetails(apiHost, u1Id, u2Id)
 	if !isCI {
 		testUserManagement(apiHost)
 	}
 	testUserGroups(apiHost)
+	time.Sleep(5 * time.Second)
 	testLogMsgs(apiHost)
 	testScanData(apiHost, 0 /*3 for proper testing*/)
+	time.Sleep(5 * time.Second)
 	testDetectorConfig(apiHost)
 
 	testTags(apiHost)
 	testROIUserConfiguration(apiHost)
+	time.Sleep(5 * time.Second)
 	testScreenConfiguration(apiHost)
 	testWidgetData(apiHost)