fix: add readiness probe for lms/cms (#133) (#137)

* fix: add readiness probe for lms/cms * fix: add readiness probe for lms/cms * fix: add affinity to spread lms/cms to multiple nodes * chore: remove readiness probe * fix: reduce startup probe period seconds * fix: gracefully kill uwsgi workers * fix: disable local file loggers * fix: disable logging * fix: reduce max unavailable to 0 * fix: add liveness probe for cms and lms * fix: fail early on tracking logger removal * chore: remove rolling update options * fix: restore preStop hook * fix: use right host for cms livenessProbe * fix: use lms/cms host only * chore: restore prestopHook (cherry picked from commit f970e01)
eduNEXT · Sep 11, 2024 · 917032f · 917032f
1 parent d492218
commit 917032f
Show file tree

Hide file tree

Showing 4 changed files with 57 additions and 10 deletions.
diff --git a/drydock/patches/openedx-common-settings b/drydock/patches/openedx-common-settings
@@ -18,3 +18,6 @@ XBLOCK_SETTINGS["ScormXBlock"] = {
     "STORAGE_FUNC": scorm_xblock_storage,
 }
 {% endif %}
+
+LOGGING["loggers"].pop("tracking")
+LOGGING["loggers"][""]["handlers"] = ["console"]
diff --git a/drydock/patches/uwsgi-config b/drydock/patches/uwsgi-config
@@ -12,3 +12,5 @@ no-defer-accept = true
 master = true
 py-call-osafterfork = true
 vacuum = true
+hook-master-start = unix_signal:1 gracefully_kill_them_all
+disable-logging = true
diff --git a/drydock/templates/drydock/k8s/lifecycle/cms.yml b/drydock/templates/drydock/k8s/lifecycle/cms.yml
@@ -12,16 +12,37 @@ spec:
               exec:
                 command:
                 - "sleep"
-                - "30"
+                - "15"
           startupProbe:
             httpGet:
               httpHeaders:
                 - name: Host
-                  value: {{ CMS_HOST }}
+                  value: cms
               path: /heartbeat
               port: 8000
-            initialDelaySeconds: 5
+            initialDelaySeconds: 1
             timeoutSeconds: 3
-            periodSeconds: 5
-            failureThreshold: 5
+            periodSeconds: 1
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              httpHeaders:
+                - name: Host
+                  value: lms
+              path: /heartbeat
+              port: 8000
+            initialDelaySeconds: 3
+            timeoutSeconds: 30
+            periodSeconds: 60
+            failureThreshold: 2
+            successThreshold: 1
       terminationGracePeriodSeconds: 60
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: cms
+              topologyKey: kubernetes.io/hostname
diff --git a/drydock/templates/drydock/k8s/lifecycle/lms.yml b/drydock/templates/drydock/k8s/lifecycle/lms.yml
@@ -12,16 +12,37 @@ spec:
               exec:
                 command:
                 - "sleep"
-                - "30"
+                - "15"
           startupProbe:
             httpGet:
               httpHeaders:
                 - name: Host
-                  value: {{ LMS_HOST }}
+                  value: lms
               path: /heartbeat
               port: 8000
-            initialDelaySeconds: 5
+            initialDelaySeconds: 1
             timeoutSeconds: 3
-            periodSeconds: 5
-            failureThreshold: 5
+            periodSeconds: 1
+            failureThreshold: 30
+          livenessProbe:
+            httpGet:
+              httpHeaders:
+                - name: Host
+                  value: lms
+              path: /heartbeat
+              port: 8000
+            initialDelaySeconds: 3
+            timeoutSeconds: 30
+            periodSeconds: 60
+            failureThreshold: 2
+            successThreshold: 1
       terminationGracePeriodSeconds: 60
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchLabels:
+                  app.kubernetes.io/name: lms
+              topologyKey: kubernetes.io/hostname