From ab637383faea841bab6b0cccf4de7fb223563576 Mon Sep 17 00:00:00 2001 From: Ian McGinnis <67600557+ian-noaa@users.noreply.github.com> Date: Wed, 27 Mar 2024 22:02:16 -0600 Subject: [PATCH 1/2] Update the path to cb-metar on the gsl-dev cluster Our webdriver tests expect the path to end in the appname so "cb-metar-capella" wasn't valid. Add another level to the URL so that we can include the DB location information before the appname. --- kubernetes/overlays/gsl-dev/ingress-apps.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kubernetes/overlays/gsl-dev/ingress-apps.yaml b/kubernetes/overlays/gsl-dev/ingress-apps.yaml index 0e57479411..34c21533f1 100644 --- a/kubernetes/overlays/gsl-dev/ingress-apps.yaml +++ b/kubernetes/overlays/gsl-dev/ingress-apps.yaml @@ -18,14 +18,14 @@ spec: name: scorecard port: name: http - - path: /mats-dev/cb-metar + - path: /mats-dev/on-prem/cb-metar pathType: Prefix backend: service: name: cb-metar port: number: 80 - - path: /mats-dev/cb-metar-capella + - path: /mats-dev/capella/cb-metar pathType: Prefix backend: service: From 62f4afaf54fd01d424d6ddc100ebece16f40f615 Mon Sep 17 00:00:00 2001 From: Ian McGinnis <67600557+ian-noaa@users.noreply.github.com> Date: Fri, 29 Mar 2024 14:51:32 -0600 Subject: [PATCH 2/2] Initial k8s "kustomizations" for the EKS cluster We don't have a CSI driver in EKS yet, so I ended up disabling the PVC for mongo and using an on-node "emptyDir". I also disabled resource requests and limits in all our deployments while we were performance testing. We should re-enable both those features when possible. --- .../overlays/aws-dev/cb-metar/deployment.yaml | 32 +++++++++++++++ .../aws-dev/cb-metar/kustomization.yaml | 19 +++++++++ .../overlays/aws-dev/home/deployment.yaml | 29 ++++++++++++++ .../overlays/aws-dev/home/kustomization.yaml | 14 +++++++ kubernetes/overlays/aws-dev/ingress-apps.yaml | 20 ++++++++++ kubernetes/overlays/aws-dev/ingress-home.yaml | 25 ++++++++++++ .../overlays/aws-dev/kustomization.yaml | 8 ++++ .../overlays/aws-dev/mongo/delete-pvc.yaml | 6 +++ .../overlays/aws-dev/mongo/deployment.yaml | 39 +++++++++++++++++++ .../overlays/aws-dev/mongo/kustomization.yaml | 22 +++++++++++ kubernetes/overlays/aws-dev/mongo/mongod.conf | 14 +++++++ 11 files changed, 228 insertions(+) create mode 100644 kubernetes/overlays/aws-dev/cb-metar/deployment.yaml create mode 100644 kubernetes/overlays/aws-dev/cb-metar/kustomization.yaml create mode 100644 kubernetes/overlays/aws-dev/home/deployment.yaml create mode 100644 kubernetes/overlays/aws-dev/home/kustomization.yaml create mode 100644 kubernetes/overlays/aws-dev/ingress-apps.yaml create mode 100644 kubernetes/overlays/aws-dev/ingress-home.yaml create mode 100644 kubernetes/overlays/aws-dev/kustomization.yaml create mode 100644 kubernetes/overlays/aws-dev/mongo/delete-pvc.yaml create mode 100644 kubernetes/overlays/aws-dev/mongo/deployment.yaml create mode 100644 kubernetes/overlays/aws-dev/mongo/kustomization.yaml create mode 100644 kubernetes/overlays/aws-dev/mongo/mongod.conf diff --git a/kubernetes/overlays/aws-dev/cb-metar/deployment.yaml b/kubernetes/overlays/aws-dev/cb-metar/deployment.yaml new file mode 100644 index 0000000000..6f4698e373 --- /dev/null +++ b/kubernetes/overlays/aws-dev/cb-metar/deployment.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cb-metar +spec: + template: + spec: + containers: + - name: cb-metar + # FIXME: disable resource limits while we do performance tuning + # resources: + # requests: + # memory: "1Gi" + # cpu: "0.25" + # limits: + # memory: "8Gi" + # cpu: "4" + envFrom: + - secretRef: + name: cb-metar-secret + volumeMounts: + - name: cb-metar-settings-file + mountPath: /usr/app/settings/cb-metar/settings.json + subPath: settings.json + readOnly: true + imagePullPolicy: Always # Since we track a long-lived tag + imagePullSecrets: + - name: mats-ghcr + volumes: + - name: cb-metar-settings-file + configMap: + name: cb-metar-config diff --git a/kubernetes/overlays/aws-dev/cb-metar/kustomization.yaml b/kubernetes/overlays/aws-dev/cb-metar/kustomization.yaml new file mode 100644 index 0000000000..3167e0767c --- /dev/null +++ b/kubernetes/overlays/aws-dev/cb-metar/kustomization.yaml @@ -0,0 +1,19 @@ +resources: + - ../../../base/cb-metar + +patchesStrategicMerge: + - deployment.yaml + +images: + - name: ghcr.io/noaa-gsl/mats/development/cb-metar + newTag: development + +configMapGenerator: + - name: cb-metar-config + files: + - settings.json # Should mirror the appropriate settings.json file in mats-settings + +secretGenerator: + - name: cb-metar-secret + envs: + - .env # Should contain mongo_url, root_url, and delay diff --git a/kubernetes/overlays/aws-dev/home/deployment.yaml b/kubernetes/overlays/aws-dev/home/deployment.yaml new file mode 100644 index 0000000000..ba187d8b8f --- /dev/null +++ b/kubernetes/overlays/aws-dev/home/deployment.yaml @@ -0,0 +1,29 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: home +spec: + template: + spec: + containers: + - name: home + # FIXME: disable resource limits while we do performance tuning + # resources: + # requests: + # memory: "1Gi" + # cpu: "0.25" + # limits: + # memory: "8Gi" + # cpu: "4" + volumeMounts: + - name: home-settings-file + mountPath: /app/settings.json + subPath: settings.json + readOnly: true + imagePullPolicy: Always # Since we track a long-lived tag + imagePullSecrets: + - name: mats-ghcr + volumes: + - name: home-settings-file + configMap: + name: home-config diff --git a/kubernetes/overlays/aws-dev/home/kustomization.yaml b/kubernetes/overlays/aws-dev/home/kustomization.yaml new file mode 100644 index 0000000000..761bb03548 --- /dev/null +++ b/kubernetes/overlays/aws-dev/home/kustomization.yaml @@ -0,0 +1,14 @@ +resources: + - ../../../base/home + +patchesStrategicMerge: + - deployment.yaml + +images: + - name: ghcr.io/noaa-gsl/mats/development/home + newTag: development + +configMapGenerator: + - name: home-config + files: + - settings.json # Should mirror the appropriate settings.json file in mats-settings diff --git a/kubernetes/overlays/aws-dev/ingress-apps.yaml b/kubernetes/overlays/aws-dev/ingress-apps.yaml new file mode 100644 index 0000000000..098596a654 --- /dev/null +++ b/kubernetes/overlays/aws-dev/ingress-apps.yaml @@ -0,0 +1,20 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + # Omit the rewrite-target annotation as it causes problems with Meteor's ROOT_URL + nginx.ingress.kubernetes.io/use-regex: "true" + name: mats-apps + # namespace: mats-dev +spec: + ingressClassName: nginx + rules: + - http: + paths: + - path: /mats-dev/cb-metar + pathType: Prefix + backend: + service: + name: cb-metar + port: + number: 80 diff --git a/kubernetes/overlays/aws-dev/ingress-home.yaml b/kubernetes/overlays/aws-dev/ingress-home.yaml new file mode 100644 index 0000000000..8a8a29e52c --- /dev/null +++ b/kubernetes/overlays/aws-dev/ingress-home.yaml @@ -0,0 +1,25 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + # Use nginx's rewrite-target to strip the path prefix before sending the request to the home service + nginx.ingress.kubernetes.io/use-regex: "true" + nginx.ingress.kubernetes.io/rewrite-target: /$1 + name: mats-home + # namespace: mats-dev +spec: + ingressClassName: nginx + rules: + - http: + paths: + # Note - this rewrite rule can cause issues if the "path" with the regex capture group + # for the nginx rewrite-target gets longer than the paths in ingress-mats.yaml. + # This has happened before with the rule for `/mats-dev/ptype`. If multiple paths match, + # Kubernetes will prefer the longer path. + - path: /mats-dev/(.*) + pathType: Prefix + backend: + service: + name: home + port: + number: 80 diff --git a/kubernetes/overlays/aws-dev/kustomization.yaml b/kubernetes/overlays/aws-dev/kustomization.yaml new file mode 100644 index 0000000000..7c0d871000 --- /dev/null +++ b/kubernetes/overlays/aws-dev/kustomization.yaml @@ -0,0 +1,8 @@ +resources: + - cb-metar + - home + - mongo + - ingress-apps.yaml + - ingress-home.yaml +commonLabels: + environment: dev diff --git a/kubernetes/overlays/aws-dev/mongo/delete-pvc.yaml b/kubernetes/overlays/aws-dev/mongo/delete-pvc.yaml new file mode 100644 index 0000000000..1e4db2ae35 --- /dev/null +++ b/kubernetes/overlays/aws-dev/mongo/delete-pvc.yaml @@ -0,0 +1,6 @@ +$patch: delete +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: mongo-pv-claim + diff --git a/kubernetes/overlays/aws-dev/mongo/deployment.yaml b/kubernetes/overlays/aws-dev/mongo/deployment.yaml new file mode 100644 index 0000000000..ebac049fca --- /dev/null +++ b/kubernetes/overlays/aws-dev/mongo/deployment.yaml @@ -0,0 +1,39 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: mongodb +spec: + template: + spec: + containers: + - name: mongodb + # FIXME: disable resource limits while we do performance tuning + # resources: + # requests: + # memory: "2Gi" + # cpu: "1" + # limits: + # memory: "2Gi" + # cpu: "1" + envFrom: + - secretRef: + name: mongo-secret + volumeMounts: + - name: mongo-config-file + mountPath: /etc/mongod.conf + readOnly: true + # FIXME: Switch to an in-memory store until we have a CSI driver + - name: mongo-ephemeral-storage + mountPath: /data/db + - $patch: delete + name: mongo-persistent-storage + volumes: + - name: mongo-config-file + configMap: + name: mongo-config + # FIXME: Switch to an in-memory store until we have a CSI driver + - $patch: delete + name: mongo-persistent-storage + - name: mongo-ephemeral-storage + emptyDir: + sizeLimit: "2Gi" \ No newline at end of file diff --git a/kubernetes/overlays/aws-dev/mongo/kustomization.yaml b/kubernetes/overlays/aws-dev/mongo/kustomization.yaml new file mode 100644 index 0000000000..26106370ec --- /dev/null +++ b/kubernetes/overlays/aws-dev/mongo/kustomization.yaml @@ -0,0 +1,22 @@ +resources: + - ../../../base/mongo + +patchesStrategicMerge: + - deployment.yaml + +patches: + # FIXME: We want to use an in-memory store until we have a CSI driver + - path: delete-pvc.yaml + target: + kind: PersistentVolumeClaim + name: mongo-pv-claim + +configMapGenerator: + - name: mongo-config + files: + - mongod.conf + +secretGenerator: + - name: mongo-secret + envs: + - .env.mongo.secret # Should contain user and password diff --git a/kubernetes/overlays/aws-dev/mongo/mongod.conf b/kubernetes/overlays/aws-dev/mongo/mongod.conf new file mode 100644 index 0000000000..f3758d6349 --- /dev/null +++ b/kubernetes/overlays/aws-dev/mongo/mongod.conf @@ -0,0 +1,14 @@ +systemLog: + destination: file + path: "/var/log/mongodb/mongod.log" + logAppend: true +storage: + journal: + enabled: true +processManagement: + fork: true +net: + bindIp: 127.0.0.1 + port: 27017 +setParameter: + enableLocalhostAuthBypass: false