From 5b2da02acae9dde7c08c3bff383ebeaac53d75f6 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 22 Aug 2023 16:21:12 +0200 Subject: [PATCH 1/9] Remove tini usage in backup-restore-sidecar and use post-exec-cmds. --- .../templates/postgres.yaml | 37 ++++++------------- .../templates/rethinkdb.yaml | 27 ++++---------- 2 files changed, 19 insertions(+), 45 deletions(-) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 62d5dca2e..55e5fa07c 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -45,18 +45,8 @@ spec: image: {{ postgres_image_name }}:{{ postgres_image_tag }} imagePullPolicy: {{ postgres_image_pull_policy }} command: - - tini - - -- - args: - - sh - - -c - - backup-restore-sidecar wait && docker-entrypoint.sh postgres - {% if postgres_shared_libraries_preload %}-c shared_preload_libraries={{ postgres_shared_libraries_preload | join(',') }}{% endif %} - {% if postgres_maintenance_work_mem %}-c maintenance_work_mem={{ postgres_maintenance_work_mem }}{% endif %} - {% if postgres_shared_buffers %}-c shared_buffers={{ postgres_shared_buffers }}{% endif %} - {% if postgres_effective_cache_size %}-c effective_cache_size={{ postgres_effective_cache_size }}{% endif %} - {% if postgres_work_mem %}-c work_mem={{ postgres_work_mem }}{% endif %} - -c max_connections={{ postgres_max_connections }} + - backup-restore-sidecar + - wait ports: - containerPort: 5432 env: @@ -89,19 +79,12 @@ spec: mountPath: /usr/local/bin/backup-restore-sidecar - name: backup-restore-sidecar-config mountPath: /etc/backup-restore-sidecar - - name: bin-provision - subPath: tini - mountPath: /usr/local/bin/tini - name: backup-restore-sidecar image: {{ postgres_image_name }}:{{ postgres_image_tag }} imagePullPolicy: {{ postgres_image_pull_policy }} command: - - tini - - -- - args: - - sh - - -c - - mkdir -p /data/postgres && backup-restore-sidecar start + - backup-restore-sidecar + - start ports: - containerPort: 2112 env: @@ -142,9 +125,6 @@ spec: mountPath: /usr/local/bin/backup-restore-sidecar - name: backup-restore-sidecar-config mountPath: /etc/backup-restore-sidecar - - name: bin-provision - subPath: tini - mountPath: /usr/local/bin/tini {% if postgres_backup_restore_sidecar_provider == "gcp" %} - name: gcp-credentials mountPath: /gcp/credentials @@ -157,7 +137,6 @@ spec: command: - cp - /backup-restore-sidecar - - /sbin/tini - /bin-provision volumeMounts: - name: bin-provision @@ -230,6 +209,14 @@ data: projectID: {{ postgres_backup_restore_sidecar_gcp_project_id | b64encode }} serviceaccount.json: {{ postgres_backup_restore_sidecar_gcp_serviceaccount_json | to_json | b64encode }} {% endif %} + post-exec-cmds: + - docker-entrypoint.sh postgres + {% if postgres_shared_libraries_preload %}-c shared_preload_libraries={{ postgres_shared_libraries_preload | join(',') }}{% endif %} + {% if postgres_maintenance_work_mem %}-c maintenance_work_mem={{ postgres_maintenance_work_mem }}{% endif %} + {% if postgres_shared_buffers %}-c shared_buffers={{ postgres_shared_buffers }}{% endif %} + {% if postgres_effective_cache_size %}-c effective_cache_size={{ postgres_effective_cache_size }}{% endif %} + {% if postgres_work_mem %}-c work_mem={{ postgres_work_mem }}{% endif %} + -c max_connections={{ postgres_max_connections }} --- apiVersion: v1 kind: Service diff --git a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml index 3b8cb2895..19ae3d754 100644 --- a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml +++ b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml @@ -37,13 +37,8 @@ spec: image: {{ rethinkdb_image_name }}:{{ rethinkdb_image_tag }} imagePullPolicy: {{ rethinkdb_image_pull_policy }} command: - - tini - - -- - args: - - sh - - -c - # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location - - backup-restore-sidecar wait && rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} + - backup-restore-sidecar + - wait env: - name: RETHINKDB_PASSWORD valueFrom: @@ -62,19 +57,12 @@ spec: mountPath: /usr/local/bin/backup-restore-sidecar - name: backup-restore-sidecar-config mountPath: /etc/backup-restore-sidecar - - name: bin-provision - subPath: tini - mountPath: /usr/local/bin/tini - name: backup-restore-sidecar image: {{ rethinkdb_image_name }}:{{ rethinkdb_image_tag }} imagePullPolicy: {{ rethinkdb_image_pull_policy }} command: - - tini - - -- - args: - - sh - - -c - - mkdir -p /data/rethinkdb && backup-restore-sidecar start + - backup-restore-sidecar + - start ports: - containerPort: 2112 env: @@ -107,9 +95,6 @@ spec: - name: bin-provision subPath: backup-restore-sidecar mountPath: /usr/local/bin/backup-restore-sidecar - - name: bin-provision - subPath: tini - mountPath: /usr/local/bin/tini - name: bin-provision subPath: rethinkdb-dump mountPath: /usr/local/bin/rethinkdb-dump @@ -128,7 +113,6 @@ spec: command: - cp - /backup-restore-sidecar - - /ubuntu/tini - /rethinkdb/rethinkdb-dump - /rethinkdb/rethinkdb-restore - /bin-provision @@ -184,6 +168,9 @@ data: rethinkdb-passwordfile: /rethinkdb-secret/rethinkdb-password.txt backup-cron-schedule: "{{ rethinkdb_backup_restore_sidecar_backup_cron_schedule }}" object-prefix: rethinkdb-{{ metal_control_plane_stage_name }} + post-exec-cmds: + # IMPORTANT: the --directory needs to point to the exact sidecar data dir, otherwise the database will be restored to the wrong location + - rethinkdb --bind all --directory /data/rethinkdb --initial-password ${RETHINKDB_PASSWORD} {% if rethinkdb_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ rethinkdb_backup_restore_sidecar_object_max_keep }} {% endif %} From 73cd9c21de3ce71cdc14dec8128f7ffb5a14baeb Mon Sep 17 00:00:00 2001 From: Gerrit Date: Thu, 24 Aug 2023 14:57:57 +0200 Subject: [PATCH 2/9] Fix. --- .../postgres-backup-restore/templates/postgres.yaml | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 55e5fa07c..813aa585d 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -186,6 +186,8 @@ data: {% if postgres_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ postgres_backup_restore_sidecar_object_max_keep }} {% endif %} + post-exec-cmds: + - docker-entrypoint.sh postgres {% if postgres_shared_libraries_preload %} -c shared_preload_libraries={{ postgres_shared_libraries_preload | join(',') }}{% endif %}{% if postgres_maintenance_work_mem %} -c maintenance_work_mem={{ postgres_maintenance_work_mem }}{% endif %}{% if postgres_shared_buffers %} -c shared_buffers={{ postgres_shared_buffers }}{% endif %}{% if postgres_effective_cache_size %} -c effective_cache_size={{ postgres_effective_cache_size }}{% endif %}{% if postgres_work_mem %} -c work_mem={{ postgres_work_mem }}{% endif %} -c max_connections={{ postgres_max_connections }} --- apiVersion: v1 kind: Secret @@ -209,14 +211,6 @@ data: projectID: {{ postgres_backup_restore_sidecar_gcp_project_id | b64encode }} serviceaccount.json: {{ postgres_backup_restore_sidecar_gcp_serviceaccount_json | to_json | b64encode }} {% endif %} - post-exec-cmds: - - docker-entrypoint.sh postgres - {% if postgres_shared_libraries_preload %}-c shared_preload_libraries={{ postgres_shared_libraries_preload | join(',') }}{% endif %} - {% if postgres_maintenance_work_mem %}-c maintenance_work_mem={{ postgres_maintenance_work_mem }}{% endif %} - {% if postgres_shared_buffers %}-c shared_buffers={{ postgres_shared_buffers }}{% endif %} - {% if postgres_effective_cache_size %}-c effective_cache_size={{ postgres_effective_cache_size }}{% endif %} - {% if postgres_work_mem %}-c work_mem={{ postgres_work_mem }}{% endif %} - -c max_connections={{ postgres_max_connections }} --- apiVersion: v1 kind: Service From 87de871ce9f714162bfaac16966627d759bc7934 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 4 Sep 2023 11:50:04 +0200 Subject: [PATCH 3/9] Add postgres liveliness and readiness probes. --- .../templates/postgres.yaml | 37 +++++++++++++++++++ 1 file changed, 37 insertions(+) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 813aa585d..6ca5cb0de 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -70,6 +70,43 @@ spec: secretKeyRef: key: POSTGRES_DATA name: {{ postgres_name }} + livenessProbe: + exec: + command: + - /bin/sh + - -c + - exec + - pg_isready + - -U + - {{ postgres_user }} + - -h + - 127.0.0.1 + - -p + - "5432" + failureThreshold: 6 + initialDelaySeconds: 30 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 5 + name: postgres + ports: + - containerPort: 5432 + readinessProbe: + exec: + command: + - /bin/sh + - -c + - exec + - pg_isready + - -U + - {{ postgres_user }} + - -h + - 127.0.0.1 + - -p + - "5432" + initialDelaySeconds: 5 + periodSeconds: 10 + timeoutSeconds: 5 resources: {{ postgres_resources | to_json }} volumeMounts: - name: {{ postgres_name }} From 4c000542f70e9315c2474ad00ac568b6779f7bb0 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 4 Sep 2023 11:50:33 +0200 Subject: [PATCH 4/9] backup-restore-sidecar now exposes grpc port. --- .../roles/postgres-backup-restore/templates/postgres.yaml | 3 ++- .../roles/rethinkdb-backup-restore/templates/rethinkdb.yaml | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 6ca5cb0de..585fe25af 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -123,7 +123,8 @@ spec: - backup-restore-sidecar - start ports: - - containerPort: 2112 + - containerPort: 8000 + name: grpc env: - name: BACKUP_RESTORE_SIDECAR_POSTGRES_PASSWORD valueFrom: diff --git a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml index 19ae3d754..1cc13af9d 100644 --- a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml +++ b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml @@ -64,7 +64,8 @@ spec: - backup-restore-sidecar - start ports: - - containerPort: 2112 + - containerPort: 8000 + name: grpc env: {% if rethinkdb_backup_restore_sidecar_provider == "gcp" %} - name: BACKUP_RESTORE_SIDECAR_GCP_PROJECT From 18b83227fc59ddef6e9206689c489c33231ca3a9 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 4 Sep 2023 11:51:16 +0200 Subject: [PATCH 5/9] Add separate backup volume to prevent writing backups to var/lib/docker. --- .../templates/postgres.yaml | 17 +++++++++++++++++ .../templates/rethinkdb.yaml | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 585fe25af..7f73974d4 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -158,6 +158,9 @@ spec: volumeMounts: - name: {{ postgres_name }} mountPath: /data + - mountPath: /backup + name: backup + - mountPath: /data - name: bin-provision subPath: backup-restore-sidecar mountPath: /usr/local/bin/backup-restore-sidecar @@ -183,6 +186,9 @@ spec: - name: {{ postgres_name }} persistentVolumeClaim: claimName: {{ postgres_name }} + - name: backup + persistentVolumeClaim: + claimName: backup - name: backup-restore-sidecar-config configMap: name: backup-restore-sidecar-config-{{ postgres_name }} @@ -207,6 +213,17 @@ spec: storage: {{ postgres_storage_size }} {% if postgres_storage_class %} storageClassName: {{ postgres_storage_class }} +{% endif %} + - metadata: + name: backup + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ postgres_storage_size }} +{% if postgres_storage_class %} + storageClassName: {{ postgres_storage_class }} {% endif %} --- apiVersion: v1 diff --git a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml index 1cc13af9d..035a4289c 100644 --- a/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml +++ b/control-plane/roles/rethinkdb-backup-restore/templates/rethinkdb.yaml @@ -89,6 +89,8 @@ spec: volumeMounts: - mountPath: /data name: {{ rethinkdb_name }} + - mountPath: /backup + name: backup - name: rethinkdb-credentials mountPath: /rethinkdb-secret - name: backup-restore-sidecar-config @@ -124,6 +126,9 @@ spec: - name: {{ rethinkdb_name }} persistentVolumeClaim: claimName: {{ rethinkdb_name }} + - name: backup + persistentVolumeClaim: + claimName: backup - name: rethinkdb-credentials secret: secretName: {{ rethinkdb_name }} @@ -154,6 +159,17 @@ spec: storage: {{ rethinkdb_storage_size }} {% if rethinkdb_storage_class %} storageClassName: {{ rethinkdb_storage_class }} +{% endif %} + - metadata: + name: backup + spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: {{ rethinkdb_storage_size }} +{% if rethinkdb_storage_class %} + storageClassName: {{ rethinkdb_storage_class }} {% endif %} --- apiVersion: v1 From 343b23389dddda051410e53a76a5fe23924d2249 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Mon, 4 Sep 2023 11:51:34 +0200 Subject: [PATCH 6/9] Explicitly define compression method for postgres. --- .../roles/postgres-backup-restore/templates/postgres.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index 7f73974d4..ee3bf9f58 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -238,6 +238,7 @@ data: backup-provider: {{ postgres_backup_restore_sidecar_provider }} backup-cron-schedule: "{{ postgres_backup_restore_sidecar_backup_cron_schedule }}" object-prefix: {{ postgres_backup_restore_sidecar_object_prefix }} + compression-method: targz {% if postgres_backup_restore_sidecar_object_max_keep %} object-max-keep: {{ postgres_backup_restore_sidecar_object_max_keep }} {% endif %} From 0421ab317d34d8823ea73e46e6adcf7d422c52b7 Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 5 Sep 2023 17:29:39 +0200 Subject: [PATCH 7/9] Fix data volume mount. --- .../roles/postgres-backup-restore/templates/postgres.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml index ee3bf9f58..566a8325d 100644 --- a/control-plane/roles/postgres-backup-restore/templates/postgres.yaml +++ b/control-plane/roles/postgres-backup-restore/templates/postgres.yaml @@ -160,7 +160,6 @@ spec: mountPath: /data - mountPath: /backup name: backup - - mountPath: /data - name: bin-provision subPath: backup-restore-sidecar mountPath: /usr/local/bin/backup-restore-sidecar From 61012a5e689ea52c24c26cce2d02f6c3e5e46a8a Mon Sep 17 00:00:00 2001 From: Gerrit Date: Tue, 5 Sep 2023 17:29:44 +0200 Subject: [PATCH 8/9] Add migration step. --- .../roles/postgres-backup-restore/tasks/main.yml | 12 ++++++++++++ .../roles/rethinkdb-backup-restore/tasks/main.yml | 12 ++++++++++++ 2 files changed, 24 insertions(+) diff --git a/control-plane/roles/postgres-backup-restore/tasks/main.yml b/control-plane/roles/postgres-backup-restore/tasks/main.yml index 922449e4b..5498d6682 100644 --- a/control-plane/roles/postgres-backup-restore/tasks/main.yml +++ b/control-plane/roles/postgres-backup-restore/tasks/main.yml @@ -12,7 +12,19 @@ - postgres_backup_restore_sidecar_image_name is defined - postgres_backup_restore_sidecar_image_tag is defined +- name: Migration to separate backup volume (since pre- and post-cmds) + k8s: + definition: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: "{{ postgres_name }}" + namespace: "{{ postgres_namespace }}" + state: absent + when: "(lookup('k8s', api_version='apps/v1', kind='StatefulSet', resource_name=postgres_name, namespace=postgres_namespace) | default({}, true)).get('spec', {}).get('volumeClaimTemplates', []) | length == 1" + - name: Deploy postgres (backup-restore) k8s: definition: "{{ lookup('template', 'postgres.yaml') }}" namespace: "{{ postgres_namespace }}" + apply: yes diff --git a/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml b/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml index 9ec845aef..cfb05b5f6 100644 --- a/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml +++ b/control-plane/roles/rethinkdb-backup-restore/tasks/main.yml @@ -19,7 +19,19 @@ that: - rethinkdb_backup_restore_sidecar_image_tag is defined +- name: Migration to separate backup volume (since pre- and post-cmds) + k8s: + definition: + apiVersion: apps/v1 + kind: StatefulSet + metadata: + name: "{{ rethinkdb_name }}" + namespace: "{{ rethinkdb_namespace }}" + state: absent + when: "(lookup('k8s', api_version='apps/v1', kind='StatefulSet', resource_name=rethinkdb_name, namespace=rethinkdb_namespace) | default({}, true)).get('spec', {}).get('volumeClaimTemplates', []) | length == 1" + - name: Deploy rethinkdb (backup-restore) k8s: definition: "{{ lookup('template', 'rethinkdb.yaml') }}" namespace: "{{ rethinkdb_namespace }}" + apply: yes From 6a9207fbe8332994f68674a562f37672de1b838d Mon Sep 17 00:00:00 2001 From: Gerrit Date: Wed, 6 Sep 2023 13:55:50 +0200 Subject: [PATCH 9/9] Remove unnecessary newline from README. --- control-plane/README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/control-plane/README.md b/control-plane/README.md index f42f8072f..0f2e04aac 100644 --- a/control-plane/README.md +++ b/control-plane/README.md @@ -26,7 +26,6 @@ The `control-plane-defaults` folder contains defaults that are used by multiple | metal_control_plane_namespace | | The target namespace of all deployed kubernetes resources of the metal-control-plane | | metal_control_plane_image_pull_policy | | Global value for an ImagePullPolicy that will be used for Kubernetes entities | - ## Roles | Role Name | Description |