From 2493fdc12cec5c3697bc9e36ee3c26d92217ce7c Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 13 Mar 2024 15:11:41 +1000 Subject: [PATCH 1/3] Fixing quant job error outputs --- api/quantification/quantRunner/runnerKubernetes.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/api/quantification/quantRunner/runnerKubernetes.go b/api/quantification/quantRunner/runnerKubernetes.go index a08fe4a1..246beef8 100644 --- a/api/quantification/quantRunner/runnerKubernetes.go +++ b/api/quantification/quantRunner/runnerKubernetes.go @@ -266,14 +266,14 @@ func (r *kubernetesRunner) runQuantJob(params PiquantParams, jobId, namespace, d defer close(status) paramsJSON, err := json.Marshal(params) if err != nil { - r.kubeHelper.Log.Errorf("Failed to serialise JSON params for node: %v", params.PMCListName) + r.kubeHelper.Log.Errorf("Failed to serialise JSON params for node: %v", jobId) return } paramsStr := string(paramsJSON) jobSpec := getJobObject(params, paramsStr, dockerImage, jobId, namespace, requestorUserId, count) job, err := r.kubeHelper.Clientset.BatchV1().Jobs(jobSpec.Namespace).Create(context.Background(), jobSpec, metav1.CreateOptions{}) if err != nil { - r.kubeHelper.Log.Errorf("Job create failed for: %v. namespace: %v, count: %v", params.PMCListName, namespace, count) + r.kubeHelper.Log.Errorf("Job create failed for: %v. namespace: %v, count: %v", jobId, namespace, count) r.fatalErrors <- err return } @@ -283,7 +283,7 @@ func (r *kubernetesRunner) runQuantJob(params PiquantParams, jobId, namespace, d time.Sleep(5 * time.Second) jobStatus, err := r.getJobStatus(job.Namespace, job.Name) if err != nil { - r.kubeHelper.Log.Errorf("Failed to get job status for: %v. namespace: %v, count: %v", params.PMCListName, namespace, count) + r.kubeHelper.Log.Errorf("Failed to get job status for: %v. namespace: %v, count: %v", jobId, namespace, count) r.fatalErrors <- err return } From e4c2d083331e95166b2155845dc553928d7f92b7 Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Wed, 13 Mar 2024 15:12:22 +1000 Subject: [PATCH 2/3] Use latest data formats --- data-formats | 2 +- generated-protos/screen-configuration.pb.go | 67 ++++++++++++--------- 2 files changed, 39 insertions(+), 30 deletions(-) diff --git a/data-formats b/data-formats index a5595cd5..159a4b73 160000 --- a/data-formats +++ b/data-formats @@ -1 +1 @@ -Subproject commit a5595cd585b79dc15f82da6fba71535abe6241b5 +Subproject commit 159a4b730c42f1953f17e311aae616154d3fe471 diff --git a/generated-protos/screen-configuration.pb.go b/generated-protos/screen-configuration.pb.go index ffa1288b..d2ebb60d 100644 --- a/generated-protos/screen-configuration.pb.go +++ b/generated-protos/screen-configuration.pb.go @@ -398,6 +398,7 @@ type ScanConfiguration struct { Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty" bson:"_id,omitempty"` QuantId string `protobuf:"bytes,2,opt,name=quantId,proto3" json:"quantId,omitempty"` Calibrations []*ScanCalibrationConfiguration `protobuf:"bytes,3,rep,name=calibrations,proto3" json:"calibrations,omitempty"` + Colour string `protobuf:"bytes,4,opt,name=colour,proto3" json:"colour,omitempty"` } func (x *ScanConfiguration) Reset() { @@ -453,6 +454,13 @@ func (x *ScanConfiguration) GetCalibrations() []*ScanCalibrationConfiguration { return nil } +func (x *ScanConfiguration) GetColour() string { + if x != nil { + return x.Colour + } + return "" +} + type ScreenConfiguration struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -608,7 +616,7 @@ var file_screen_configuration_proto_rawDesc = []byte{ 0x6e, 0x65, 0x6c, 0x18, 0x02, 0x20, 0x01, 0x28, 0x01, 0x52, 0x0c, 0x65, 0x56, 0x70, 0x65, 0x72, 0x43, 0x68, 0x61, 0x6e, 0x6e, 0x65, 0x6c, 0x12, 0x1a, 0x0a, 0x08, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x08, 0x64, 0x65, 0x74, 0x65, 0x63, - 0x74, 0x6f, 0x72, 0x22, 0x80, 0x01, 0x0a, 0x11, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x74, 0x6f, 0x72, 0x22, 0x98, 0x01, 0x0a, 0x11, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x18, 0x0a, 0x07, 0x71, 0x75, 0x61, 0x6e, 0x74, 0x49, 0x64, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, 0x71, 0x75, 0x61, 0x6e, @@ -616,35 +624,36 @@ var file_screen_configuration_proto_rawDesc = []byte{ 0x6f, 0x6e, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x61, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x0c, 0x63, 0x61, 0x6c, 0x69, 0x62, 0x72, - 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x22, 0xa8, 0x03, 0x0a, 0x13, 0x53, 0x63, 0x72, 0x65, 0x65, - 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, - 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, - 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, - 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, - 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, - 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x07, 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x73, 0x18, - 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x63, 0x72, 0x65, - 0x65, 0x6e, 0x4c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x52, 0x07, 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, - 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, - 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x28, 0x0a, 0x0f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, - 0x64, 0x55, 0x6e, 0x69, 0x78, 0x53, 0x65, 0x63, 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, - 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x53, 0x65, 0x63, 0x12, - 0x5c, 0x0a, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, - 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x53, 0x63, - 0x72, 0x65, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, - 0x6e, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, - 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x43, - 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, - 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x0a, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x4f, - 0x77, 0x6e, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x52, - 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x1a, 0x59, 0x0a, 0x17, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, + 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x16, 0x0a, 0x06, 0x63, 0x6f, 0x6c, 0x6f, 0x75, 0x72, + 0x18, 0x04, 0x20, 0x01, 0x28, 0x09, 0x52, 0x06, 0x63, 0x6f, 0x6c, 0x6f, 0x75, 0x72, 0x22, 0xa8, + 0x03, 0x0a, 0x13, 0x53, 0x63, 0x72, 0x65, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, + 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x0e, 0x0a, 0x02, 0x69, 0x64, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x02, 0x69, 0x64, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, + 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x20, 0x0a, 0x0b, 0x64, 0x65, + 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x18, 0x03, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x0b, 0x64, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x12, 0x2b, 0x0a, 0x07, + 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x11, 0x2e, + 0x46, 0x75, 0x6c, 0x6c, 0x53, 0x63, 0x72, 0x65, 0x65, 0x6e, 0x4c, 0x61, 0x79, 0x6f, 0x75, 0x74, + 0x52, 0x07, 0x6c, 0x61, 0x79, 0x6f, 0x75, 0x74, 0x73, 0x12, 0x12, 0x0a, 0x04, 0x74, 0x61, 0x67, + 0x73, 0x18, 0x07, 0x20, 0x03, 0x28, 0x09, 0x52, 0x04, 0x74, 0x61, 0x67, 0x73, 0x12, 0x28, 0x0a, + 0x0f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, 0x55, 0x6e, 0x69, 0x78, 0x53, 0x65, 0x63, + 0x18, 0x08, 0x20, 0x01, 0x28, 0x0d, 0x52, 0x0f, 0x6d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64, + 0x55, 0x6e, 0x69, 0x78, 0x53, 0x65, 0x63, 0x12, 0x5c, 0x0a, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x43, + 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x09, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x2c, 0x2e, 0x53, 0x63, 0x72, 0x65, 0x65, 0x6e, 0x43, 0x6f, 0x6e, 0x66, + 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, - 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, - 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, - 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, - 0x01, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x3b, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, - 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x79, 0x52, 0x12, 0x73, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, + 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x12, 0x27, 0x0a, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x18, 0x0a, + 0x20, 0x01, 0x28, 0x0b, 0x32, 0x11, 0x2e, 0x4f, 0x77, 0x6e, 0x65, 0x72, 0x73, 0x68, 0x69, 0x70, + 0x53, 0x75, 0x6d, 0x6d, 0x61, 0x72, 0x79, 0x52, 0x05, 0x6f, 0x77, 0x6e, 0x65, 0x72, 0x1a, 0x59, + 0x0a, 0x17, 0x53, 0x63, 0x61, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, + 0x69, 0x6f, 0x6e, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x28, 0x0a, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x53, 0x63, 0x61, + 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x75, 0x72, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x52, 0x05, + 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x42, 0x0a, 0x5a, 0x08, 0x2e, 0x3b, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x73, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( From a62c88e506ca46ba07152bb6e2808d583f983dac Mon Sep 17 00:00:00 2001 From: Peter Nemere Date: Fri, 15 Mar 2024 09:06:44 +1000 Subject: [PATCH 3/3] Cleaned up quant starting code, better error logging, and made data importer exe name be bootstrap so it works with amazon linux 2 Go lambda env --- .github/workflows/main.yml | 10 +- Makefile | 6 +- .../quantRunner/runnerKubernetes.go | 137 +----------------- 3 files changed, 10 insertions(+), 143 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d61b3115..239fa85f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -102,17 +102,13 @@ jobs: cd _out rm pixlise-api-linux wget https://s3.amazonaws.com/rds-downloads/rds-combined-ca-bundle.pem - mkdir jobupdater datasourceupdater integrationtest dataimport + mkdir dataimport integrationtest + zip -q dataimport/dataimport-linux-${{ needs.version.outputs.version }}.zip rds-combined-ca-bundle.pem bootstrap + rm bootstrap # mkdir test-data # cp ../internal/cmdline-tools/import-integration-test/test-data/000000001-21-10-2022-15-37-00.zip ./test-data - # zip -q jobupdater/jobupdater-linux-${{ needs.version.outputs.version }}.zip rds-combined-ca-bundle.pem jobupdater-linux - # rm jobupdater-linux - # zip -q datasourceupdater/datasourceupdater-linux-${{ needs.version.outputs.version }}.zip rds-combined-ca-bundle.pem datasourceupdater-linux - # rm datasourceupdater-linux # zip -q integrationtest-linux-${{ needs.version.outputs.version }}.zip rds-combined-ca-bundle.pem integrationtest-linux # rm integrationtest-linux - zip -q dataimport/dataimport-linux-${{ needs.version.outputs.version }}.zip rds-combined-ca-bundle.pem dataimport-linux - rm dataimport-linux # zip -q importtest/importtest-linux-${{ needs.version.outputs.version }}.zip importtest-linux test-data/000000001-21-10-2022-15-37-00.zip # rm importtest-linux aws s3 cp . s3://corestack-buildartifactsf774a77d-105on4pno9pjm/ --recursive --region us-east-1 diff --git a/Makefile b/Makefile index 256f6c6a..6b14174e 100644 --- a/Makefile +++ b/Makefile @@ -33,11 +33,9 @@ build-linux: echo "sha: ${GITHUB_SHA}" GOOS=linux GOARCH=amd64 go run ./data-formats/codegen/main.go -protoPath ./data-formats/api-messages/ -goOutPath ./api/ws/ GOOS=linux GOARCH=amd64 go build -ldflags "-X 'github.com/pixlise/core/v4/api/services.ApiVersion=${BUILD_VERSION}' -X 'github.com/pixlise/core/v4/api/services.GitHash=${GITHUB_SHA}'" -v -o ./_out/pixlise-api-linux ./internal/api -# GOOS=linux GOARCH=amd64 go build -v -o ./_out/jobupdater-linux ./internal/lambdas/quant-job-updater -# GOOS=linux GOARCH=amd64 go build -v -o ./_out/datasourceupdater-linux ./internal/lambdas/dataset-tile-updater -# GOOS=linux GOARCH=amd64 go build -v -o ./_out/integrationtest-linux ./internal/cmdline-tools/api-integration-test - GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -o ./_out/dataimport-linux ./internal/lambdas/data-import + GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -v -o ./_out/bootstrap ./internal/lambdas/data-import # GOOS=linux GOARCH=amd64 go build -v -o ./_out/importtest-linux ./internal/cmdline-tools/import-integration-test +# GOOS=linux GOARCH=amd64 go build -v -o ./_out/integrationtest-linux ./internal/cmdline-tools/api-integration-test # build-mac: # GOPRIVATE=github.com/pixlise GOOS=darwin GOARCH=amd64 go build -ldflags "-X services.ApiVersion=${BUILD_VERSION} -X services.GitHash=${GITHUB_SHA}" -v -o ./_out/pixlise-api-mac ./internal/api diff --git a/api/quantification/quantRunner/runnerKubernetes.go b/api/quantification/quantRunner/runnerKubernetes.go index 246beef8..40d1f93c 100644 --- a/api/quantification/quantRunner/runnerKubernetes.go +++ b/api/quantification/quantRunner/runnerKubernetes.go @@ -23,7 +23,6 @@ import ( "fmt" "strconv" "strings" - "sync" "time" "github.com/pixlise/core/v4/api/config" @@ -95,74 +94,6 @@ func (r *kubernetesRunner) RunPiquant(piquantDockerImage string, params PiquantP } } -func getPodObject(paramsStr string, params PiquantParams, dockerImage string, jobid, namespace string, requestorUserId string, length int) *apiv1.Pod { - sec := apiv1.LocalObjectReference{Name: "api-auth"} - application := "piquant-runner" - parts := strings.Split(params.PMCListName, ".") - node := parts[0] - name := fmt.Sprintf("piquant-%s", params.Command) - instance := fmt.Sprintf("%s-%s", name, node) - // Set the serviceaccount for the piquant pods based on namespace - // Piquant Fit commands will run in the same namespace and share a service account - // Piquant Map commands (jobs) will run in the piquant-map namespace with a more limited service account - san := "pixlise-api" - cpu := "250m" - if params.Command == "map" { - san = "piquant-map" - // PiQuant Map Commands will need much more CPU (and can safely request it since they are running on Fargate nodes) - cpu = "3500m" - } - - // Kubernetes doesn't like | in owner name, so we swap it for a _ here - safeUserId := strings.ReplaceAll(requestorUserId, "|", "_") - - return &apiv1.Pod{ - ObjectMeta: metav1.ObjectMeta{ - Name: jobid + "-" + parts[0], - Namespace: namespace, - Labels: map[string]string{ - "pixlise.org/application": application, - "pixlise.org/environment": params.RunTimeEnv, - "app.kubernetes.io/name": name, - "app.kubernetes.io/instance": instance, - "app.kubernetes.io/component": application, - "piquant/command": params.Command, - "app": node, - "owner": safeUserId, - "jobid": jobid, - "numberofpods": strconv.Itoa(length), - }, - }, - Spec: apiv1.PodSpec{ - ImagePullSecrets: []apiv1.LocalObjectReference{sec}, - RestartPolicy: apiv1.RestartPolicyNever, - ServiceAccountName: san, - Containers: []apiv1.Container{ - { - Name: parts[0], - Image: dockerImage, - ImagePullPolicy: apiv1.PullAlways, - Resources: apiv1.ResourceRequirements{ - Requests: apiv1.ResourceList{ - // The request determines how much cpu is reserved on the Node and will affect scheduling - "cpu": resource.MustParse(cpu), - }, - Limits: apiv1.ResourceList{ - // Allow the pod to use up to 3500m cpu if it's available on the node - "cpu": resource.MustParse("3500m"), - }, - }, - - Env: []apiv1.EnvVar{ - {Name: "QUANT_PARAMS", Value: paramsStr}, - {Name: "PYTHONUNBUFFERED", Value: "TRUE"}, - }, - }, - }, - }, - } -} - // getJobObject generates a Kubernetes Job Manifest for running piquant. // It takes in the following parameters: // - params: a PiquantParams struct containing all parameters needed by piquant @@ -244,6 +175,11 @@ func getJobObject(params PiquantParams, paramsStr, dockerImage, jobId, namespace Env: []apiv1.EnvVar{ {Name: "QUANT_PARAMS", Value: paramsStr}, {Name: "PYTHONUNBUFFERED", Value: "TRUE"}, + /*{Name: "NODE_INDEX", ValueFrom: &apiv1.EnvVarSource{ + FieldRef: &apiv1.ObjectFieldSelector{ + FieldPath: "metadata.annotations['batch.kubernetes.io/job-completion-index']", + }, + }},*/ }, }, }, @@ -294,66 +230,3 @@ func (r *kubernetesRunner) runQuantJob(params PiquantParams, jobId, namespace, d } } } - -func (r *kubernetesRunner) runQuantPod(wg *sync.WaitGroup, params PiquantParams, jobid string, namespace string, dockerImage string, requestorUserId string, count int) { - defer wg.Done() - - // Make a JSON string out of params so it can be passed in - paramsJSON, err := json.Marshal(params) - if err != nil { - r.kubeHelper.Log.Errorf("Failed to serialise JSON params for node: %v", params.PMCListName) - return - } - paramsStr := string(paramsJSON) - - //log.Debugf("getPodObject for: %v. namespace: %v, count: %v", params.PMCListName, namespace, count) - pod := getPodObject(paramsStr, params, dockerImage, jobid, namespace, requestorUserId, count) - - co := metav1.CreateOptions{} - pod, err = r.kubeHelper.Clientset.CoreV1().Pods(pod.Namespace).Create(context.TODO(), pod, co) - if err != nil { - r.kubeHelper.Log.Errorf("Pod create failed for: %v. namespace: %v, count: %v", params.PMCListName, namespace, count) - r.fatalErrors <- err - return - } - - // Create Deployment - r.kubeHelper.Log.Infof("Creating pod for %v in namespace %v...", params.PMCListName, namespace) - - // Now wait for it to finish - startUnix := time.Now().Unix() - maxEndUnix := startUnix + config.KubernetesMaxTimeoutSec - - lastPhase := "" - - for currUnix := time.Now().Unix(); currUnix < maxEndUnix; currUnix = time.Now().Unix() { - // Check kubernetes pod status - pod, _ := r.kubeHelper.Clientset.CoreV1().Pods(pod.Namespace).Get(context.TODO(), pod.Name, metav1.GetOptions{}) - - // TODO: is this needed, now that we log? - //fmt.Println(pod.Status.Phase) - //log.Infof("%v phase: %v, pod name: %v, namespace: %v", params.PMCListName, pod.Status.Phase, pod.Name, pod.Namespace) - - phase := string(pod.Status.Phase) - if lastPhase != phase { - r.kubeHelper.Log.Infof("%v phase: %v, pod name: %v, namespace: %v", params.PMCListName, pod.Status.Phase, pod.Name, pod.Namespace) - lastPhase = phase - } - - if pod.Status.Phase != apiv1.PodRunning && pod.Status.Phase != apiv1.PodPending { - r.kubeHelper.Log.Infof("Deleting pod: %v from namespace: %v", pod.Name, pod.Namespace) - - deletePolicy := metav1.DeletePropagationForeground - do := &metav1.DeleteOptions{ - PropagationPolicy: &deletePolicy, - } - err := r.kubeHelper.Clientset.CoreV1().Pods(pod.Namespace).Delete(context.TODO(), pod.Name, *do) - if err != nil { - r.kubeHelper.Log.Errorf("Failed to remove pod: %v, namespace: %v\n", pod.Name, pod.Namespace) - } - break - } - - time.Sleep(5 * time.Second) - } -}