From c09c485f900a750925888c1f74247bac167f5971 Mon Sep 17 00:00:00 2001 From: "Reuven V. Gonzales" Date: Thu, 12 Dec 2024 21:44:03 +0000 Subject: [PATCH] Helm + Flux setup for MCS --- .../metrics-calculation-service/Chart.yaml | 7 ++ .../templates/_helpers.tpl | 82 ++++++++++++++++ .../templates/app.yaml | 93 +++++++++++++++++++ .../templates/configmap.yaml | 9 ++ .../templates/rbac.yaml | 26 ++++++ .../templates/service-account.yaml | 5 + .../templates/service.yaml | 12 +++ .../metrics-calculation-service/values.yaml | 47 ++++++++++ .../kustomization.yaml | 5 + .../base/metrics-calculation-service/mcs.yaml | 38 ++++++++ 10 files changed, 324 insertions(+) create mode 100644 ops/helm-charts/metrics-calculation-service/Chart.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/templates/_helpers.tpl create mode 100644 ops/helm-charts/metrics-calculation-service/templates/app.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/templates/configmap.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/templates/rbac.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/templates/service-account.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/templates/service.yaml create mode 100644 ops/helm-charts/metrics-calculation-service/values.yaml create mode 100644 ops/k8s-apps/base/metrics-calculation-service/kustomization.yaml create mode 100644 ops/k8s-apps/base/metrics-calculation-service/mcs.yaml diff --git a/ops/helm-charts/metrics-calculation-service/Chart.yaml b/ops/helm-charts/metrics-calculation-service/Chart.yaml new file mode 100644 index 000000000..ef2ae68c5 --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/Chart.yaml @@ -0,0 +1,7 @@ +apiVersion: v2 +name: metrics-calculation-service +description: The metrics calculation service + +type: application +version: 0.1.0 +appVersion: 0.1.0 \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/_helpers.tpl b/ops/helm-charts/metrics-calculation-service/templates/_helpers.tpl new file mode 100644 index 000000000..776463bb8 --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/_helpers.tpl @@ -0,0 +1,82 @@ +{{/* +Expand the name of the chart. +*/}} + +# Disable the pgisready check due to our use of cloudsql proxy injected into the +pod. +{{- define "mcs.logging.config" }} +# Default log configuration for the metrics calculation service. This can be +# used by uvicorn Thanks to: +# https://gist.github.com/liviaerxin/d320e33cbcddcc5df76dd92948e5be3b for a +# starting point. +version: 1 +disable_existing_loggers: False +formatters: + default: + # "()": uvicorn.logging.DefaultFormatter + format: '{{ .Values.mcs.logging.format }}' + access: + # "()": uvicorn.logging.AccessFormatter + format: '{{ .Values.mcs.logging.format }}' +handlers: + default: + formatter: default + class: logging.StreamHandler + stream: ext://sys.stderr + access: + formatter: access + class: logging.StreamHandler + stream: ext://sys.stdout +loggers: + uvicorn.error: + level: {{ .Values.mcs.logging.uvicorn.level }} + handlers: + - default + propagate: no + uvicorn.access: + level: {{ .Values.mcs.logging.uvicorn.level }} + handlers: + - access + propagate: no + metrics_tools: + level: {{ .Values.mcs.logging.metrics_tools.level }} + handlers: + - default + propagate: no +root: + level: {{ .Values.mcs.logging.root.level }} + handlers: + - default + propagate: no +{{- end }} + +{{/* +This is copied due to some kind of error with helm and flux when overriding +portions of this +*/}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "mcs.fullname" -}} +{{- if .Values.global.fullnameOverride -}} +{{- .Values.global.fullnameOverride | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- $name := "mcs" -}} +{{- if contains $name .Release.Name -}} +{{- .Release.Name | trunc 63 | trimSuffix "-" -}} +{{- else -}} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} +{{- end -}} +{{- end -}} +{{- end -}} + +{{- define "mcs.labels" -}} +app.kubernetes.io/name: {{ include "mcs.fullname" . }} +{{- end -}} + +{{- define "mcs.selectorLabels" -}} +{{ include "mcs.labels" . }} +{{- end -}} \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/app.yaml b/ops/helm-charts/metrics-calculation-service/templates/app.yaml new file mode 100644 index 000000000..4efb2dd3b --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/app.yaml @@ -0,0 +1,93 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "mcs.fullname" . }} + labels: + {{- include "mcs.labels" . | nindent 4 }} +spec: + # for now this application is a single process + thread + replicas: 1 + selector: + matchLabels: + app: mcs + template: + metadata: + labels: + app: mcs + spec: + serviceAccountName: {{ include "mcs.fullname" . }}-sa + containers: + - name: metrics-calculation-service + # TODO: Remove latest tag and use a specific version + image: {{ .Values.image.repo }}:{{ .Values.image.tag }} + command: ["uvicorn"] + args: + - "metrics_tools.compute.server:app" + - "--host" + - "0.0.0.0" + - "--port" + - "8000" + - "--log-config" + - "/config/log_config.yaml" + imagePullPolicy: Always + ports: + - containerPort: 8000 + volumeMounts: + - name: config + mountPath: /config + env: + - name: METRICS_CLUSTER_NAMESPACE + value: {{ .Values.mcs.cluster.namespace }} + - name: METRICS_CLUSTER_NAME + value: {{ .Values.mcs.cluster.name }} + - name: METRICS_CLUSTER_IMAGE_REPO + value: {{ .Values.mcs.cluster.image.repo }} + - name: METRICS_CLUSTER_IMAGE_TAG + value: {{ .Values.mcs.cluster.image.tag }} + - name: METRICS_SCHEDULER_MEMORY_LIMIT + value: {{ .Values.mcs.cluster.scheduler.memory.limit }} + - name: METRICS_SCHEDULER_MEMORY_REQUEST + value: {{ .Values.mcs.cluster.scheduler.memory.request }} + - name: METRICS_CLUSTER_WORKER_THREADS + value: {{ .Values.mcs.cluster.worker.threads }} + - name: METRICS_WORKER_MEMORY_LIMIT + value: {{ .Values.mcs.cluster.worker.memory.limit }} + - name: METRICS_WORKER_MEMORY_REQUEST + value: {{ .Values.mcs.cluster.worker.memory.request }} + - name: METRICS_WORKER_DUCKDB_PATH + value: {{ .Values.mcs.cluster.worker.duckdb_path }} + - name: METRICS_GCS_BUCKET + value: {{ .Values.mcs.gcs.bucket }} + - name: METRICS_GCS_KEY_ID + value: {{ .Values.mcs.gcs.key_id }} + - name: METRICS_GCS_SECRET + value: {{ .Values.mcs.gcs.secret }} + - name: METRICS_GCS_RESULTS_PATH_PREFIX + value: {{ .Values.mcs.gcs.results_path_prefix }} + - name: METRICS_TRINO_HOST + value: {{ .Values.mcs.trino.host }} + - name: METRICS_TRINO_PORT + value: {{ .Values.mcs.trino.port }} + - name: METRICS_TRINO_USER + value: {{ .Values.mcs.trino.user }} + - name: METRICS_TRINO_CATALOG + value: {{ .Values.mcs.trino.catalog }} + - name: METRICS_HIVE_CATALOG + value: {{ .Values.mcs.hive.catalog }} + - name: METRICS_HIVE_SCHEMA + value: {{ .Values.mcs.hive.schema }} + - name: METRICS_DEBUG_ALL + value: {{ .Values.mcs.debug.all }} + - name: METRICS_DEBUG_CACHE + value: {{ .Values.mcs.debug.cache }} + - name: METRICS_DEBUG_CLUSTER + value: {{ .Values.mcs.debug.cluster }} + - name: METRICS_DEBUG_CLUSTER_NO_SHUTDOWN + value: {{ .Values.mcs.debug.cluster_no_shutdown }} + volumes: + - name: config + configMap: + # Provide the name of the ConfigMap containing the files you want + # to add to the container + name: {{ include "mcs.fullname" . }}-config \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/configmap.yaml b/ops/helm-charts/metrics-calculation-service/templates/configmap.yaml new file mode 100644 index 000000000..52a953764 --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/configmap.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "mcs.fullname" . }}-config + labels: + {{- include "mcs.labels" . | nindent 4 }} +data: + log_config.yaml: | + {{- include "mcs.logging.config" . | nindent 4 }} \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/rbac.yaml b/ops/helm-charts/metrics-calculation-service/templates/rbac.yaml new file mode 100644 index 000000000..93764cd3e --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/rbac.yaml @@ -0,0 +1,26 @@ +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "mcs.fullname" . }}-role +rules: +- apiGroups: [""] + resources: ["pods", "services"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] +- apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "list", "watch", "create", "update", "patch", "delete"] + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "mcs.fullname" . }}-role-binding +subjects: +- kind: ServiceAccount + name: {{ include "mcs.fullname" . }}-sa + namespace: {{ .Release.Namespace }} +roleRef: + kind: ClusterRole + name: {{ include "mcs.fullname" . }}-role + apiGroup: rbac.authorization.k8s.io \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/service-account.yaml b/ops/helm-charts/metrics-calculation-service/templates/service-account.yaml new file mode 100644 index 000000000..505c7dafc --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/service-account.yaml @@ -0,0 +1,5 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "mcs.fullname" . }}-sa \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/templates/service.yaml b/ops/helm-charts/metrics-calculation-service/templates/service.yaml new file mode 100644 index 000000000..4a9b28c38 --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/templates/service.yaml @@ -0,0 +1,12 @@ +--- +apiVersion: v1 +kind: Service +metadata: + name: {{ include "mcs.fullname" . }} +spec: + selector: + {{ include "mcs.selectorLabels" . | nindent 4 }} + ports: + - protocol: TCP + port: {{ .Values.service.port }} + targetPort: 8000 \ No newline at end of file diff --git a/ops/helm-charts/metrics-calculation-service/values.yaml b/ops/helm-charts/metrics-calculation-service/values.yaml new file mode 100644 index 000000000..ed93e2aa7 --- /dev/null +++ b/ops/helm-charts/metrics-calculation-service/values.yaml @@ -0,0 +1,47 @@ +mcs: + service: + port: 8000 + logging: + format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s" + root: + level: "ERROR" + metrics_tools: + level: "DEBUG" + uvicorn: + level: "INFO" + image: + repo: ghcr.io/opensource-observer/dagster-dask + tag: latest + cluster: + namespace: "default" + name: "default" + image: + repo: "ghcr.io/opensource-observer/dagster-dask" + tag: "latest" + scheduler: + memory: + limit: "2Gi" + request: "2Gi" + worker: + threads: "4" + memory: + limit: "2Gi" + request: "2Gi" + duckdb_path: "/scratch/mcs-local.db" + gcs: + bucket: "oso-playground-dataset-transfer-bucket" + key_id: "gcp:secretmanager:mcs-gcs-key-id/versions/latest" + secret: "gcp:secretmanager:mcs-gcs-secret/versions/latest" + trino: + host: "trino" + port: "8080" + user: "trino" + catalog: "metrics" + hive: + catalog: "source" + schema: "export" + debug: + all: "false" + cache: "false" + cluster: "false" + cluster_no_shutdown: "false" \ No newline at end of file diff --git a/ops/k8s-apps/base/metrics-calculation-service/kustomization.yaml b/ops/k8s-apps/base/metrics-calculation-service/kustomization.yaml new file mode 100644 index 000000000..93ae67a37 --- /dev/null +++ b/ops/k8s-apps/base/metrics-calculation-service/kustomization.yaml @@ -0,0 +1,5 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: base-mcs +resources: + - mcs.yaml \ No newline at end of file diff --git a/ops/k8s-apps/base/metrics-calculation-service/mcs.yaml b/ops/k8s-apps/base/metrics-calculation-service/mcs.yaml new file mode 100644 index 000000000..7f4d42fcf --- /dev/null +++ b/ops/k8s-apps/base/metrics-calculation-service/mcs.yaml @@ -0,0 +1,38 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: base-mcs + labels: + toolkit.fluxcd.io/tenant: apps + ops.opensource.observer/environment: base + kube-secrets-init.doit-intl.com/enable-mutation: "true" +--- +apiVersion: helm.toolkit.fluxcd.io/v2 +kind: HelmRelease +metadata: + name: mcs + namespace: base-mcs +spec: + chart: + spec: + chart: ./ops/helm-charts/metrics-calculation-service + sourceRef: + kind: GitRepository + name: flux-system + namespace: flux-system + interval: 50m + install: + remediation: + retries: 3 + values: + mcs: + cluster: + scheduler: + memory: + limit: "90Gi" + request: "85Gi" + worker: + threads: "16" + memory: + limit: "90Gi" + request: "90Gi" \ No newline at end of file