feat(kubernetes): add health endpoints (#1602)

* feat(backend): add health endpoint

* feat(helm): health probe path

* feat(frontend): add health endpoints

* feat(backend): remove unused app state fields

* fix(active): rewrote active boards logic

* feat(backend): add endpoint for resetting active boards

backend/helm/backend/values.yaml

@@ -1,45 +1,46 @@
 common:
-    app: backend
-    shortname: backend
-    team: selvbetjent
-    ingress:
-        enabled: true
-        trafficType: public
-    service:
-        internalPort: 3000
-    container:
-        image: <+artifacts.primary.image>
-        cpu: 0.3
-        memory: 512
-        env:
-          - name: REDIS_PASSWORD
-            valueFrom:
-              secretKeyRef:
-                name: redis
-                key: redis-password
-          - name: BACKEND_API_KEY
-            valueFrom:
-              secretKeyRef:
-                name: backend
-                key: api-key
-        probes:
-            enabled: true
-            spec:
-                startupProbe:
-                    tcpSocket:
-                        port: 3000
-                    periodSeconds: 1
-                    timeoutSeconds: 1
-                    failureThreshold: 300
-                livenessProbe:
-                    tcpSocket:
-                        port: 3000
-                    periodSeconds: 5
-                    successThreshold: 1
-                    timeoutSeconds: 1
-                readinessProbe:
-                    tcpSocket:
-                        port: 3000
-                    periodSeconds: 5
-                    successThreshold: 1
-                    timeoutSeconds: 1
+  app: backend
+  shortname: backend
+  team: selvbetjent
+  ingress:
+    enabled: true
+    trafficType: public
+  service:
+    internalPort: 3000
+  container:
+    image: <+artifacts.primary.image>
+    cpu: 0.3
+    memory: 512
+    env:
+      - name: REDIS_PASSWORD
+        valueFrom:
+          secretKeyRef:
+            name: redis
+            key: redis-password
+      - name: BACKEND_API_KEY
+        valueFrom:
+          secretKeyRef:
+            name: backend
+            key: api-key
+    probes:
+      enabled: true
+      spec:
+        startupProbe:
+          tcpSocket:
+            port: 3000
+          periodSeconds: 1
+          timeoutSeconds: 1
+          failureThreshold: 300
+        livenessProbe:
+          tcpSocket:
+            port: 3000
+          periodSeconds: 5
+          successThreshold: 1
+          timeoutSeconds: 1
+          path: "/health"
+        readinessProbe:
+          tcpSocket:
+            port: 3000
+          periodSeconds: 5
+          successThreshold: 1
+          timeoutSeconds: 1

backend/src/main.rs

@@ -3,8 +3,7 @@ use std::time::Duration;
 use axum::{
     body::Body,
     extract::{Path, State},
-    http::{HeaderValue, Method, StatusCode},
-    response::Response,
+    http::{HeaderValue, Method, Response, StatusCode},
     routing::{get, post},
     Json, Router,
 };
@@ -13,7 +12,7 @@ use axum_auth::AuthBearer;
 use serde_json::{from_str, Value};
 use tokio::{net::TcpListener, time};
 
-use redis::AsyncCommands;
+use redis::{AsyncCommands, ConnectionLike};
 
 use tokio_stream::StreamExt;
 use tokio_util::{sync::CancellationToken, task::TaskTracker};
@@ -25,10 +24,12 @@ use tower_http::cors::CorsLayer;
 use types::{AppError, AppState, Message};
 use utils::{graceful_shutdown, setup_redis};
 
+use crate::types::Guard;
+
 #[tokio::main]
 async fn main() {
     let host = std::env::var("HOST").unwrap_or("0.0.0.0".to_string());
-    let port = std::env::var("PORT").unwrap_or("3000".to_string());
+    let port = std::env::var("PORT").unwrap_or("3001".to_string());
     let key = std::env::var("BACKEND_API_KEY").expect("Expected to find api key");
 
     let listener = TcpListener::bind(format!("{}:{}", host, port))
@@ -43,8 +44,6 @@ async fn main() {
     let redis_clients = AppState {
         master,
         replicas,
-        runtime_status: runtime_status.clone(),
-        task_tracker: task_tracker.clone(),
         key,
     };
 
@@ -62,6 +61,8 @@ async fn main() {
         .route("/subscribe/:bid", get(subscribe))
         .route("/refresh/:bid", post(trigger))
         .route("/update", post(update))
+        .route("/health", get(check_health))
+        .route("/reset", post(reset_active))
         .with_state(redis_clients)
         .layer(cors);
 
@@ -71,6 +72,25 @@ async fn main() {
         .unwrap()
 }
 
+async fn reset_active(
+    AuthBearer(token): AuthBearer,
+    State(mut state): State<AppState>,
+) -> Result<StatusCode, AppError> {
+    if token != state.key {
+        return Ok(StatusCode::UNAUTHORIZED);
+    }
+
+    state.master.set("active_boards", 0).await?;
+    Ok(StatusCode::OK)
+}
+
+async fn check_health(State(mut state): State<AppState>) -> StatusCode {
+    if state.replicas.check_connection() {
+        return StatusCode::OK;
+    }
+    StatusCode::INTERNAL_SERVER_ERROR
+}
+
 async fn active_boards(
     AuthBearer(token): AuthBearer,
     State(mut state): State<AppState>,
@@ -111,21 +131,18 @@ async fn update(
 
 async fn subscribe(
     Path(bid): Path<String>,
-    State(mut state): State<AppState>,
+    State(state): State<AppState>,
 ) -> Result<Message, AppError> {
     let mut pubsub = state.replicas.get_async_pubsub().await?;
     pubsub.subscribe(bid).await?;
     pubsub.subscribe("update").await?;
 
-    state
-        .master
-        .incr::<&str, i32, i32>("active_boards", 1)
-        .await?;
+    let _guard = Guard::new(state.master.clone());
 
     let mut msg_stream = pubsub.on_message();
     let res = tokio::select! {
             Some(msg) = msg_stream.next() => {
-    let channel = msg.get_channel_name();
+                let channel = msg.get_channel_name();
                 if channel == "update" {
                     Message::Update
                 } else {
@@ -139,10 +156,5 @@ async fn subscribe(
         }
     };
 
-    state
-        .master
-        .decr::<&str, i32, i32>("active_boards", 1)
-        .await?;
-
     Ok(res)
 }

backend/src/types.rs

@@ -3,17 +3,14 @@ use axum::{
     http::{Response, StatusCode},
     response::IntoResponse,
 };
-use redis::{aio::MultiplexedConnection, Client};
+use redis::{aio::MultiplexedConnection, AsyncCommands, Client};
 use serde::Serialize;
 use serde_json::{to_string, Value};
-use tokio_util::{sync::CancellationToken, task::TaskTracker};
 
 #[derive(Clone)]
 pub struct AppState {
     pub master: MultiplexedConnection,
     pub replicas: Client,
-    pub runtime_status: CancellationToken,
-    pub task_tracker: TaskTracker,
     pub key: String,
 }
 
@@ -25,6 +22,29 @@ pub enum Message {
     Timeout,
 }
 
+pub struct Guard {
+    pub master: MultiplexedConnection,
+}
+
+impl Guard {
+    pub fn new(master: MultiplexedConnection) -> Self {
+        let mut m = master.clone();
+        tokio::spawn(async move {
+            let _ = m.incr::<&str, i32, i32>("active_boards", 1).await;
+        });
+        Guard { master }
+    }
+}
+
+impl Drop for Guard {
+    fn drop(&mut self) {
+        let mut m = self.master.clone();
+        tokio::spawn(async move {
+            let _ = m.decr::<&str, i32, i32>("active_boards", 1).await;
+        });
+    }
+}
+
 impl IntoResponse for Message {
     fn into_response(self) -> axum::response::Response {
         if let Ok(json_string) = to_string(&self) {

next-tavla/app/health/route.ts

@@ -0,0 +1,5 @@
+import { NextResponse } from 'next/server'
+
+export function GET() {
+    return NextResponse.json({ status: 'OK' })
+}

next-tavla/helm/tavla/values.yaml

@@ -32,6 +32,7 @@ common:
                     periodSeconds: 5
                     successThreshold: 1
                     timeoutSeconds: 1
+                    path: '/health'
                 readinessProbe:
                     tcpSocket:
                         port: 3000