Helm + Flux setup for MCS

ops/helm-charts/metrics-calculation-service/Chart.yaml

@@ -0,0 +1,7 @@
+apiVersion: v2
+name: metrics-calculation-service
+description: The metrics calculation service
+
+type: application
+version: 0.1.0
+appVersion: 0.1.0

ops/helm-charts/metrics-calculation-service/templates/_helpers.tpl

@@ -0,0 +1,82 @@
+{{/*
+Expand the name of the chart.
+*/}}
+
+# Disable the pgisready check due to our use of cloudsql proxy injected into the
+pod.
+{{- define "mcs.logging.config" }}
+# Default log configuration for the metrics calculation service. This can be
+# used by uvicorn Thanks to:
+# https://gist.github.com/liviaerxin/d320e33cbcddcc5df76dd92948e5be3b for a
+# starting point.
+version: 1
+disable_existing_loggers: False
+formatters:
+  default:
+    # "()": uvicorn.logging.DefaultFormatter
+    format: '{{ .Values.mcs.logging.format }}'
+  access:
+    # "()": uvicorn.logging.AccessFormatter
+    format: '{{ .Values.mcs.logging.format }}'
+handlers:
+  default:
+    formatter: default
+    class: logging.StreamHandler
+    stream: ext://sys.stderr
+  access:
+    formatter: access
+    class: logging.StreamHandler
+    stream: ext://sys.stdout
+loggers:
+  uvicorn.error:
+    level: {{ .Values.mcs.logging.uvicorn.level }}
+    handlers:
+      - default
+    propagate: no
+  uvicorn.access:
+    level: {{ .Values.mcs.logging.uvicorn.level }}
+    handlers:
+      - access
+    propagate: no
+  metrics_tools:
+    level: {{ .Values.mcs.logging.metrics_tools.level }}
+    handlers:
+      - default
+    propagate: no
+root:
+  level: {{ .Values.mcs.logging.root.level }}
+  handlers:
+    - default
+  propagate: no
+{{- end }}
+
+{{/* 
+This is copied due to some kind of error with helm and flux when overriding
+portions of this
+*/}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mcs.fullname" -}}
+{{- if .Values.global.fullnameOverride -}}
+{{- .Values.global.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := "mcs" -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{- define "mcs.labels" -}}
+app.kubernetes.io/name: {{ include "mcs.fullname" . }}
+{{- end -}}
+
+{{- define "mcs.selectorLabels" -}}
+{{ include "mcs.labels" . }}
+{{- end -}}

ops/helm-charts/metrics-calculation-service/templates/app.yaml

@@ -0,0 +1,93 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mcs.fullname" . }}
+  labels:
+    {{- include "mcs.labels" . | nindent 4 }}
+spec:
+  # for now this application is a single process + thread
+  replicas: 1
+  selector:
+    matchLabels:
+      app: mcs
+  template:
+    metadata:
+      labels:
+        app: mcs
+    spec:
+      serviceAccountName: {{ include "mcs.fullname" . }}-sa
+      containers:
+      - name: metrics-calculation-service
+        # TODO: Remove latest tag and use a specific version
+        image: {{ .Values.image.repo }}:{{ .Values.image.tag }}
+        command: ["uvicorn"] 
+        args: 
+          - "metrics_tools.compute.server:app"
+          - "--host"
+          - "0.0.0.0"
+          - "--port"
+          - "8000"
+          - "--log-config"
+          - "/config/log_config.yaml"
+        imagePullPolicy: Always
+        ports:
+          - containerPort: 8000
+        volumeMounts:
+          - name: config
+            mountPath: /config
+        env:
+          - name: METRICS_CLUSTER_NAMESPACE
+            value: {{ .Values.mcs.cluster.namespace }}
+          - name: METRICS_CLUSTER_NAME
+            value: {{ .Values.mcs.cluster.name }}
+          - name: METRICS_CLUSTER_IMAGE_REPO
+            value: {{ .Values.mcs.cluster.image.repo }}
+          - name: METRICS_CLUSTER_IMAGE_TAG
+            value: {{ .Values.mcs.cluster.image.tag }}
+          - name: METRICS_SCHEDULER_MEMORY_LIMIT
+            value: {{ .Values.mcs.cluster.scheduler.memory.limit }}
+          - name: METRICS_SCHEDULER_MEMORY_REQUEST
+            value: {{ .Values.mcs.cluster.scheduler.memory.request }}
+          - name: METRICS_CLUSTER_WORKER_THREADS
+            value: {{ .Values.mcs.cluster.worker.threads }}
+          - name: METRICS_WORKER_MEMORY_LIMIT
+            value: {{ .Values.mcs.cluster.worker.memory.limit }}
+          - name: METRICS_WORKER_MEMORY_REQUEST
+            value: {{ .Values.mcs.cluster.worker.memory.request }}
+          - name: METRICS_WORKER_DUCKDB_PATH
+            value: {{ .Values.mcs.cluster.worker.duckdb_path }}
+          - name: METRICS_GCS_BUCKET
+            value: {{ .Values.mcs.gcs.bucket }}
+          - name: METRICS_GCS_KEY_ID
+            value: {{ .Values.mcs.gcs.key_id }}
+          - name: METRICS_GCS_SECRET
+            value: {{ .Values.mcs.gcs.secret }}
+          - name: METRICS_GCS_RESULTS_PATH_PREFIX
+            value: {{ .Values.mcs.gcs.results_path_prefix }}
+          - name: METRICS_TRINO_HOST
+            value: {{ .Values.mcs.trino.host }}
+          - name: METRICS_TRINO_PORT
+            value: {{ .Values.mcs.trino.port }}
+          - name: METRICS_TRINO_USER
+            value: {{ .Values.mcs.trino.user }}
+          - name: METRICS_TRINO_CATALOG
+            value: {{ .Values.mcs.trino.catalog }}
+          - name: METRICS_HIVE_CATALOG
+            value: {{ .Values.mcs.hive.catalog }}
+          - name: METRICS_HIVE_SCHEMA
+            value: {{ .Values.mcs.hive.schema }}
+          - name: METRICS_DEBUG_ALL
+            value: {{ .Values.mcs.debug.all }}
+          - name: METRICS_DEBUG_CACHE
+            value: {{ .Values.mcs.debug.cache }}
+          - name: METRICS_DEBUG_CLUSTER
+            value: {{ .Values.mcs.debug.cluster }}
+          - name: METRICS_DEBUG_CLUSTER_NO_SHUTDOWN
+            value: {{ .Values.mcs.debug.cluster_no_shutdown }}
+      volumes:
+        - name: config
+          configMap: 
+            # Provide the name of the ConfigMap containing the files you want
+            # to add to the container
+            name: {{ include "mcs.fullname" . }}-config

ops/helm-charts/metrics-calculation-service/templates/configmap.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: {{ include "mcs.fullname" . }}-config
+  labels:
+    {{- include "mcs.labels" . | nindent 4 }}
+data:
+  log_config.yaml: |
+    {{- include "mcs.logging.config" . | nindent 4 }}

ops/helm-charts/metrics-calculation-service/templates/rbac.yaml

@@ -0,0 +1,26 @@
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "mcs.fullname" . }}-role
+rules:
+- apiGroups: [""]
+  resources: ["pods", "services"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+- apiGroups: ["apps"]
+  resources: ["deployments"]
+  verbs: ["get", "list", "watch", "create", "update", "patch", "delete"]
+
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "mcs.fullname" . }}-role-binding
+subjects:
+- kind: ServiceAccount
+  name: {{ include "mcs.fullname" . }}-sa 
+  namespace: {{ .Release.Namespace }}
+roleRef:
+  kind: ClusterRole
+  name: {{ include "mcs.fullname" . }}-role
+  apiGroup: rbac.authorization.k8s.io

ops/helm-charts/metrics-calculation-service/templates/service-account.yaml

@@ -0,0 +1,5 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "mcs.fullname" . }}-sa

ops/helm-charts/metrics-calculation-service/templates/service.yaml

@@ -0,0 +1,12 @@
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: {{ include "mcs.fullname" . }}
+spec:
+  selector:
+    {{ include "mcs.selectorLabels" . | nindent 4 }}
+  ports:
+    - protocol: TCP
+      port: {{ .Values.service.port }}
+      targetPort: 8000

ops/helm-charts/metrics-calculation-service/values.yaml

@@ -0,0 +1,47 @@
+mcs:
+  service:
+    port: 8000
+  logging:
+    format: "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
+    root:
+      level: "ERROR"
+    metrics_tools:
+      level: "DEBUG"
+    uvicorn:
+      level: "INFO"
+  image:
+    repo: ghcr.io/opensource-observer/dagster-dask
+    tag: latest
+  cluster:
+    namespace: "default"
+    name: "default"
+    image:
+      repo: "ghcr.io/opensource-observer/dagster-dask"
+      tag: "latest"
+    scheduler:
+      memory:
+        limit: "2Gi"
+        request: "2Gi"
+    worker:
+      threads: "4"
+      memory:
+        limit: "2Gi"
+        request: "2Gi"
+      duckdb_path: "/scratch/mcs-local.db"
+  gcs:
+    bucket: "oso-playground-dataset-transfer-bucket"
+    key_id: "gcp:secretmanager:mcs-gcs-key-id/versions/latest"
+    secret: "gcp:secretmanager:mcs-gcs-secret/versions/latest"
+  trino:
+    host: "trino"
+    port: "8080"
+    user: "trino"
+    catalog: "metrics"
+  hive:
+    catalog: "source"
+    schema: "export"
+  debug:
+    all: "false"
+    cache: "false"
+    cluster: "false"
+    cluster_no_shutdown: "false"

ops/k8s-apps/base/metrics-calculation-service/kustomization.yaml

@@ -0,0 +1,5 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: base-mcs
+resources:
+  - mcs.yaml

ops/k8s-apps/base/metrics-calculation-service/mcs.yaml

@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: base-mcs
+  labels:
+    toolkit.fluxcd.io/tenant: apps
+    ops.opensource.observer/environment: base
+    kube-secrets-init.doit-intl.com/enable-mutation: "true"
+---
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: mcs
+  namespace: base-mcs
+spec:
+  chart:
+    spec:
+      chart: ./ops/helm-charts/metrics-calculation-service
+      sourceRef:
+        kind: GitRepository
+        name: flux-system
+        namespace: flux-system
+  interval: 50m
+  install:
+    remediation:
+      retries: 3
+  values:
+    mcs:
+      cluster:
+        scheduler:
+          memory:
+            limit: "90Gi"
+            request: "85Gi"
+        worker:
+          threads: "16"
+          memory:
+            limit: "90Gi"
+            request: "90Gi"

ops/k8s-apps/production/metrics-calculation-service/custom-helm-values.yaml

@@ -0,0 +1,15 @@
+apiVersion: helm.toolkit.fluxcd.io/v2
+kind: HelmRelease
+metadata:
+  name: production-mcs
+spec:
+  values:
+    mcs:
+      trino:
+        host: production-trino-trino.production-trino.svc.cluster.local
+        port: 8080
+        user: sqlmesh
+      gcs:
+        bucket: oso-dataset-transfer-bucket
+        key_id: gcp:secretmanager:production-mcs-gcs-key-id/versions/latest
+        secret: gcp:secretmanager:production-mcs-gcs-secret/versions/latest

ops/k8s-apps/production/metrics-calculation-service/kustomization.yaml

@@ -0,0 +1,11 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - ../../base/metrics-calculation-service
+namespace: production-mcs
+patches:
+  - path: ./custom-helm-values.yaml
+    target:
+      kind: HelmRelease
+    options:
+      allowNameChange: true