Add metrics and deployment related code (#2)

* Restructure directories and code
* Add helm chart for kubernetes deoplyment
* Add Dockerfile with multistage build
* Add prometheus metrics exporter and checks

.dockerignore

@@ -0,0 +1,23 @@
+### Go template
+# If you prefer the allow list template instead of the deny list, see community template:
+# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore
+#
+# Binaries for programs and plugins
+*.exe
+*.exe~
+*.dll
+*.so
+*.dylib
+
+# Test binary, built with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Dependency directories (remove the comment below to include it)
+# vendor/
+
+# Go workspace file
+go.work
+

.gitignore

@@ -16,3 +16,6 @@
 
 # Dependency directories (remove the comment below to include it)
 # vendor/
+
+# Project binaries
+cmd/controller/controller

.pre-commit-config.yaml

@@ -1,10 +1,8 @@
 repos:
-- repo: https://github.com/dnephin/pre-commit-golang
-  rev: v0.5.1
+- repo: https://github.com/tekwizely/pre-commit-golang
+  rev: v1.0.0-rc.1
   hooks:
+  - id: go-mod-tidy
   - id: go-fmt
-    exclude: templates/
-  - id: go-vet
-    exclude: templates/
+  - id: go-vet-mod
   - id: go-lint
-    exclude: templates/

build/Dockerfile

@@ -0,0 +1,17 @@
+# syntax=docker/dockerfile:1.2.1
+FROM golang:1.19.4-alpine AS builder
+WORKDIR /go/src/github.com/healthjoy/mimir-rules-controller
+COPY go.mod go.sum ./
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    go mod download
+COPY . .
+RUN --mount=type=cache,target=/root/.cache/go-build \
+    --mount=type=cache,target=/go/pkg/mod \
+    cd cmd/controller \
+ && CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -o /go/bin/mimir-rules-controller
+
+FROM alpine:3.17.0
+COPY --from=builder /go/bin/mimir-rules-controller /bin/mimir-rules-controller
+ENTRYPOINT ["/bin/mimir-rules-controller"]
+CMD ["--help"]

main.go → cmd/controller/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"context"
 	"flag"
+	"fmt"
 	"os"
 	"os/signal"
 	"syscall"
@@ -16,14 +17,17 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/klog/v2"
 
+	"github.com/healthjoy/mimir-rules-controller/pkg/controller"
 	rulesclientset "github.com/healthjoy/mimir-rules-controller/pkg/generated/clientset/versioned"
 	rulesinformers "github.com/healthjoy/mimir-rules-controller/pkg/generated/informers/externalversions"
+	"github.com/healthjoy/mimir-rules-controller/pkg/metrics"
 )
 
 var (
 	masterURL  string
 	kubeconfig string
-	config     ControllerConfig
+	address    string
+	config     controller.Config
 	mmConf     client.Config
 )
 
@@ -33,6 +37,7 @@ func init() {
 	flag.StringVar(&masterURL, "master", "", "The address of the Kubernetes API server. Overrides any value in kubeconfig. Only required if out-of-cluster.")
 
 	// Controller config
+	flag.StringVar(&address, "address", ":9000", "The address to expose prometheus metrics and service endpoints.")
 	flag.StringVar(&config.ClusterName, "cluster-name", getEnv("CLUSTER_NAME", "default"), "The name of the cluster. Used to identify the cluster in the Mimir API.")
 	flag.StringVar(&config.PodName, "pod-name", getEnv("POD_NAME", ""), "The name of the pod")
 	flag.StringVar(&config.PodNamespace, "pod-namespace", getEnv("POD_NAMESPACE", ""), "The namespace of the pod")
@@ -71,7 +76,6 @@ func main() {
 
 	// set up signals, so we handle the first shutdown signal gracefully
 	ctx := contextWithSigterm(context.Background())
-	ctx, cancel := context.WithCancel(ctx)
 
 	// creates the connection
 	cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
@@ -100,12 +104,15 @@ func main() {
 	// Create the rules informer factory
 	rulesInformerFactory := rulesinformers.NewSharedInformerFactory(rulesClient, time.Second*30)
 
-	// Create the controller
-	controller := NewController(config,
+	metricServer := metrics.New()
+	// Create the ruleController
+	ruleController := controller.NewController(config,
 		kubeClient, rulesClient, mimirClient,
-		rulesInformerFactory.Rulescontroller().V1alpha1().MimirRules())
+		rulesInformerFactory.Rulescontroller().V1alpha1().MimirRules(),
+		metricServer.Registry,
+	)
 
-	// Start the informer factories to begin populating the informer caches
+	// runServer the informer factories to begin populating the informer caches
 	rulesInformerFactory.Start(ctx.Done())
 
 	lock := &resourcelock.LeaseLock{
@@ -120,17 +127,20 @@ func main() {
 		},
 	}
 
+	metricServer.Start(ctx, address)
+
 	leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+		Name:          fmt.Sprintf("%s/%s", config.LeaseLockNamespace, config.LeaseLockName),
 		Lock:          lock,
 		LeaseDuration: 15 * time.Second,
 		RenewDeadline: 10 * time.Second,
 		RetryPeriod:   2 * time.Second,
 		Callbacks: leaderelection.LeaderCallbacks{
 			OnStartedLeading: func(ctx context.Context) {
-				// Start the controller
+				// runServer the ruleController
 				klog.Info("Started leading")
-				if err = controller.Run(ctx, 2); err != nil {
-					klog.Fatalf("Error running controller: %s", err.Error())
+				if err = ruleController.Run(ctx, 2); err != nil {
+					klog.Fatalf("Error running ruleController: %s", err.Error())
 				}
 			},
 			OnStoppedLeading: func() {
@@ -140,7 +150,6 @@ func main() {
 				if identity == config.Identity() {
 					return
 				}
-				cancel()
 				klog.Infof("New leader elected: %s", identity)
 			},
 		},

deployments/charts/mimir-rules-controller/.helmignore

@@ -0,0 +1,23 @@
+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/

deployments/charts/mimir-rules-controller/Chart.yaml

@@ -0,0 +1,6 @@
+apiVersion: v2
+name: mimir-rules-controller
+description: A Helm chart for Mimir Rules Controller
+type: application
+version: 0.1.0
+appVersion: latest

deployments/charts/mimir-rules-controller/templates/NOTES.txt

@@ -0,0 +1,16 @@
+1. Get the application URL by running these commands:
+{{- if contains "NodePort" .Values.service.type }}
+  export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "mimir-rules-controller.fullname" . }})
+  export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}")
+  echo http://$NODE_IP:$NODE_PORT
+{{- else if contains "LoadBalancer" .Values.service.type }}
+     NOTE: It may take a few minutes for the LoadBalancer IP to be available.
+           You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "mimir-rules-controller.fullname" . }}'
+  export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "mimir-rules-controller.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}")
+  echo http://$SERVICE_IP:{{ .Values.service.port }}
+{{- else if contains "ClusterIP" .Values.service.type }}
+  export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "mimir-rules-controller.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
+  export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}")
+  echo "Visit http://127.0.0.1:8080 to use your application"
+  kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT
+{{- end }}

deployments/charts/mimir-rules-controller/templates/_helpers.tpl

@@ -0,0 +1,62 @@
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "mimir-rules-controller.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "mimir-rules-controller.fullname" -}}
+{{- if .Values.fullnameOverride }}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- $name := default .Chart.Name .Values.nameOverride }}
+{{- if contains $name .Release.Name }}
+{{- .Release.Name | trunc 63 | trimSuffix "-" }}
+{{- else }}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
+{{- end }}
+{{- end }}
+{{- end }}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "mimir-rules-controller.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
+{{- end }}
+
+{{/*
+Common labels
+*/}}
+{{- define "mimir-rules-controller.labels" -}}
+helm.sh/chart: {{ include "mimir-rules-controller.chart" . }}
+{{ include "mimir-rules-controller.selectorLabels" . }}
+{{- if .Chart.AppVersion }}
+app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
+{{- end }}
+app.kubernetes.io/managed-by: {{ .Release.Service }}
+{{- end }}
+
+{{/*
+Selector labels
+*/}}
+{{- define "mimir-rules-controller.selectorLabels" -}}
+app.kubernetes.io/name: {{ include "mimir-rules-controller.name" . }}
+app.kubernetes.io/instance: {{ .Release.Name }}
+{{- end }}
+
+{{/*
+Create the name of the service account to use
+*/}}
+{{- define "mimir-rules-controller.serviceAccountName" -}}
+{{- if .Values.serviceAccount.create }}
+{{- default (include "mimir-rules-controller.fullname" .) .Values.serviceAccount.name }}
+{{- else }}
+{{- default "default" .Values.serviceAccount.name }}
+{{- end }}
+{{- end }}

deployments/charts/mimir-rules-controller/templates/clusterrole.yaml

@@ -0,0 +1,24 @@
+{{- if .Values.rbac.create }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: {{ include "mimir-rules-controller.fullname" . }}
+  labels:
+    {{- include "mimir-rules-controller.labels" . | nindent 4 }}
+rules:
+- apiGroups:
+  - rulescontroller.k8s.healthjoy.com
+  resources:
+  - mimirrules
+  - mimirrules/status
+  - mimirrules/finalizers
+  verbs:
+  - get
+  - list
+  - watch
+  - update
+  - patch
+  - create
+  - delete
+{{- end}}

deployments/charts/mimir-rules-controller/templates/clusterrolebinding.yaml

@@ -0,0 +1,16 @@
+{{- if .Values.rbac.create }}
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: {{ include "mimir-rules-controller.fullname" . }}
+  labels:
+    {{- include "mimir-rules-controller.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: {{ include "mimir-rules-controller.fullname" . }}
+subjects:
+- kind: ServiceAccount
+  name: {{ include "mimir-rules-controller.serviceAccountName" . }}
+  namespace: {{ .Release.Namespace }}
+{{- end }}

deployments/charts/mimir-rules-controller/templates/deployment.yaml

@@ -0,0 +1,78 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: {{ include "mimir-rules-controller.fullname" . }}
+  labels:
+    {{- include "mimir-rules-controller.labels" . | nindent 4 }}
+spec:
+  {{- if not .Values.autoscaling.enabled }}
+  replicas: {{ .Values.replicaCount }}
+  {{- end }}
+  selector:
+    matchLabels:
+      {{- include "mimir-rules-controller.selectorLabels" . | nindent 6 }}
+  template:
+    metadata:
+      {{- with .Values.podAnnotations }}
+      annotations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      labels:
+        {{- include "mimir-rules-controller.selectorLabels" . | nindent 8 }}
+    spec:
+      {{- with .Values.imagePullSecrets }}
+      imagePullSecrets:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      serviceAccountName: {{ include "mimir-rules-controller.serviceAccountName" . }}
+      securityContext:
+        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+      containers:
+        - name: {{ .Chart.Name }}
+          securityContext:
+            {{- toYaml .Values.securityContext | nindent 12 }}
+          image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          ports:
+            - name: http
+              containerPort: 9000
+              protocol: TCP
+          livenessProbe:
+            httpGet:
+              path: /healthz
+              port: http
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: http
+          resources:
+            {{- toYaml .Values.resources | nindent 12 }}
+          env:
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: MIMIR_ADDRESS
+            value: {{ required "A valid Mimir address is required" .Values.mimir.address }}
+          - name: CLUSTER_NAME
+            value: {{ required "A valid cluster name is required" .Values.mimir.clusterName }}
+      {{- with .Values.nodeSelector }}
+      nodeSelector:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.affinity }}
+      affinity:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.tolerations }}
+      tolerations:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}
+      {{- with .Values.topologySpreadConstraints }}
+      topologySpreadConstraints:
+        {{- toYaml . | nindent 8 }}
+      {{- end }}