From 3cfcff648d89cf4e26c20937e28991add78079ee Mon Sep 17 00:00:00 2001 From: imjasmeet Date: Fri, 29 Sep 2023 17:38:30 +0530 Subject: [PATCH] [pipelines] 1.44.5 release --- stable/pipelines/CHANGELOG.md | 12 ++ stable/pipelines/Chart.lock | 6 +- stable/pipelines/Chart.yaml | 6 +- stable/pipelines/ci/default-values.yaml | 36 ---- .../pipelines/ci/global-section-values.yaml | 36 ---- stable/pipelines/ci/ha-values.yaml | 36 ---- stable/pipelines/ci/hpa-values.yaml | 36 ---- stable/pipelines/templates/_helpers.tpl | 58 +++--- .../templates/pipelines-cron-statefulset.yaml | 60 ++++++ .../pipelines-hookhandler-statefulset.yaml | 60 ++++++ .../pipelines-internalapi-statefulset.yaml | 7 + .../templates/pipelines-statefulset.yaml | 180 +++-------------- .../pipelines-steptrigger-statefulset.yaml | 98 +++++---- .../templates/pipelines-sync-statefulset.yaml | 60 ++++++ .../templates/pipelines-trigger-hpa.yaml | 8 +- .../pipelines-trigger-statefulset.yaml | 189 ++++++------------ stable/pipelines/values.yaml | 184 +++++++---------- 17 files changed, 462 insertions(+), 610 deletions(-) diff --git a/stable/pipelines/CHANGELOG.md b/stable/pipelines/CHANGELOG.md index 9169f4904..63b168ed0 100644 --- a/stable/pipelines/CHANGELOG.md +++ b/stable/pipelines/CHANGELOG.md @@ -1,6 +1,18 @@ # JFrog Pipelines Chart Changelog All changes to this chart to be documented in this file. +## [101.44.5] - Aug 7, 2023 +* Upadate chart version of vault to 0.25.0 to work with 1.25 of kubernetes +* Added option to stream logs in json +* Add support to work without vault on modifying corresponding flags +* Remove steptrigger from pipelines +* Remove logup from pipelines +* Add ability to pass filebeat metric configuration +* Updated nodePollerInterval from 15 seconds to 5 seconds +* Fixed #adding colon in image registry breaks deployment with meta label error +* Add observability container to non api pods +* Add terminationGracePeriodSeconds for all the pods + ## [101.41.3] - Jun 16, 2023 * Add ability to work with redis.fullnameOverride * Add support to pass db metric parameters in system.yaml diff --git a/stable/pipelines/Chart.lock b/stable/pipelines/Chart.lock index f0d1bdbec..2a1dd2970 100644 --- a/stable/pipelines/Chart.lock +++ b/stable/pipelines/Chart.lock @@ -10,6 +10,6 @@ dependencies: version: 12.10.1 - name: vault repository: https://charts.jfrog.io/ - version: 0.16.1 -digest: sha256:70447f57bec8971f2c77adbf8acced07cbba97acc00c063b994f801be853cf5c -generated: "2022-04-21T12:53:31.324192+05:30" + version: 0.25.0 +digest: sha256:9d9b9eaa7258294872a09b31aae9d39fb4f0970ce5c17220d2d6ab889562e3b5 +generated: "2023-08-07T15:59:25.511241+05:30" diff --git a/stable/pipelines/Chart.yaml b/stable/pipelines/Chart.yaml index 4063e2de7..eabb61289 100644 --- a/stable/pipelines/Chart.yaml +++ b/stable/pipelines/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v2 -appVersion: 1.41.3 +appVersion: 1.44.5 dependencies: - condition: postgresql.enabled name: postgresql @@ -16,7 +16,7 @@ dependencies: - condition: vault.enabled name: vault repository: https://charts.jfrog.io/ - version: 0.16.1 + version: 0.25.0 description: A Helm chart for JFrog Pipelines home: https://jfrog.com/pipelines/ icon: https://raw.githubusercontent.com/jfrog/charts/master/stable/pipelines/icon/pipelines-logo.png @@ -32,4 +32,4 @@ name: pipelines sources: - https://github.com/jfrog/charts type: application -version: 101.41.3 +version: 101.44.5 diff --git a/stable/pipelines/ci/default-values.yaml b/stable/pipelines/ci/default-values.yaml index 27faf0311..392cc8e0a 100644 --- a/stable/pipelines/ci/default-values.yaml +++ b/stable/pipelines/ci/default-values.yaml @@ -47,24 +47,6 @@ pipelines: cpu: 25m memory: 40Mi - runTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - - stepTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - cron: resources: limits: @@ -92,24 +74,6 @@ pipelines: cpu: 25m memory: 40Mi - marshaller: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 60Mi - - logup: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - extensionSync: resources: limits: diff --git a/stable/pipelines/ci/global-section-values.yaml b/stable/pipelines/ci/global-section-values.yaml index 34edfd943..3d99a0191 100644 --- a/stable/pipelines/ci/global-section-values.yaml +++ b/stable/pipelines/ci/global-section-values.yaml @@ -88,24 +88,6 @@ pipelines: cpu: 25m memory: 40Mi - runTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - - stepTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - cron: resources: limits: @@ -133,24 +115,6 @@ pipelines: cpu: 25m memory: 40Mi - marshaller: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 60Mi - - logup: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - extensionSync: resources: limits: diff --git a/stable/pipelines/ci/ha-values.yaml b/stable/pipelines/ci/ha-values.yaml index 461ebce43..f99dfd8e2 100644 --- a/stable/pipelines/ci/ha-values.yaml +++ b/stable/pipelines/ci/ha-values.yaml @@ -41,24 +41,6 @@ pipelines: cpu: 25m memory: 40Mi - runTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - - stepTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - cron: resources: limits: @@ -86,24 +68,6 @@ pipelines: cpu: 25m memory: 40Mi - marshaller: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 60Mi - - logup: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - extensionSync: resources: limits: diff --git a/stable/pipelines/ci/hpa-values.yaml b/stable/pipelines/ci/hpa-values.yaml index d574ee9da..b13a93992 100644 --- a/stable/pipelines/ci/hpa-values.yaml +++ b/stable/pipelines/ci/hpa-values.yaml @@ -39,24 +39,6 @@ pipelines: cpu: 25m memory: 40Mi - runTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - - stepTrigger: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - cron: resources: limits: @@ -84,24 +66,6 @@ pipelines: cpu: 25m memory: 40Mi - marshaller: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 60Mi - - logup: - resources: - limits: - cpu: 500m - memory: 500Mi - requests: - cpu: 5m - memory: 40Mi - extensionSync: resources: limits: diff --git a/stable/pipelines/templates/_helpers.tpl b/stable/pipelines/templates/_helpers.tpl index 1a3f7ac36..b1b36a8f1 100644 --- a/stable/pipelines/templates/_helpers.tpl +++ b/stable/pipelines/templates/_helpers.tpl @@ -42,11 +42,6 @@ The services name {{- printf "%s-%s-trigger" $name .Chart.Name | trunc 63 | trimSuffix "-" -}} {{- end -}} -{{- define "pipelines.steptrigger.name" -}} -{{- $name := .Release.Name | trunc 29 -}} -{{- printf "%s-%s-steptrigger" $name .Chart.Name | trunc 63 | trimSuffix "-" -}} -{{- end -}} - {{- define "pipelines.extensionsync.name" -}} {{- $name := .Release.Name | trunc 29 -}} {{- printf "%s-%s-extensionsync" $name .Chart.Name | trunc 63 | trimSuffix "-" -}} @@ -224,7 +219,7 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} {{/* -Common labels for runtrigger +Common labels for runservice */}} {{- define "pipelines.trigger.labels" -}} helm.sh/chart: {{ include "pipelines.chart" . }} @@ -236,11 +231,11 @@ app.kubernetes.io/managed-by: {{ .Release.Service }} {{- end -}} {{/* -Common labels for steptrigger +Common labels for stepservice */}} -{{- define "pipelines.steptrigger.labels" -}} +{{- define "pipelines.stepservice.labels" -}} helm.sh/chart: {{ include "pipelines.chart" . }} -{{ include "pipelines.steptrigger.selectorLabels" . }} +{{ include "pipelines.stepservice.selectorLabels" . }} {{- if .Chart.AppVersion }} app.kubernetes.io/version: {{ include "pipelines.app.version" . | quote }} {{- end }} @@ -348,10 +343,10 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} {{/* -Selector labels for steptrigger pod +Selector labels for stepservice pod */}} -{{- define "pipelines.steptrigger.selectorLabels" -}} -app.kubernetes.io/name: {{ include "pipelines.steptrigger.name" . }} +{{- define "pipelines.stepservice.selectorLabels" -}} +app.kubernetes.io/name: {{ include "pipelines.stepservice.name" . }} app.kubernetes.io/instance: {{ .Release.Name }} {{- end -}} @@ -560,24 +555,24 @@ Resolve customInitContainers value for internalapi {{- end -}} {{/* -Resolve customInitContainersBegin value for steptrigger +Resolve customInitContainersBegin value for stepservice */}} -{{- define "pipelines.steptrigger.customInitContainersBegin" -}} +{{- define "pipelines.stepservice.customInitContainersBegin" -}} {{- if .Values.global.customInitContainersBegin -}} {{- .Values.global.customInitContainersBegin -}} -{{- else if .Values.pipelines.stepTrigger.customInitContainersBegin -}} -{{- .Values.pipelines.stepTrigger.customInitContainersBegin -}} +{{- else if .Values.pipelines.stepservice.customInitContainersBegin -}} +{{- .Values.pipelines.stepservice.customInitContainersBegin -}} {{- end -}} {{- end -}} {{/* -Resolve customInitContainers value for steptrigger +Resolve customInitContainers value for stepservice */}} -{{- define "pipelines.steptrigger.customInitContainers" -}} +{{- define "pipelines.stepservice.customInitContainers" -}} {{- if .Values.global.customInitContainers -}} {{- .Values.global.customInitContainers -}} -{{- else if .Values.pipelines.stepTrigger.customInitContainers -}} -{{- .Values.pipelines.stepTrigger.customInitContainers -}} +{{- else if .Values.pipelines.stepservice.customInitContainers -}} +{{- .Values.pipelines.stepservice.customInitContainers -}} {{- end -}} {{- end -}} @@ -609,8 +604,8 @@ Resolve customInitContainersBegin value for trigger {{- define "pipelines.trigger.customInitContainersBegin" -}} {{- if .Values.global.customInitContainersBegin -}} {{- .Values.global.customInitContainersBegin -}} -{{- else if .Values.pipelines.runTrigger.customInitContainersBegin -}} -{{- .Values.pipelines.runTrigger.customInitContainersBegin -}} +{{- else if .Values.pipelines.runservice.customInitContainersBegin -}} +{{- .Values.pipelines.runservice.customInitContainersBegin -}} {{- end -}} {{- end -}} @@ -620,8 +615,8 @@ Resolve customInitContainers value for trigger {{- define "pipelines.trigger.customInitContainers" -}} {{- if .Values.global.customInitContainers -}} {{- .Values.global.customInitContainers -}} -{{- else if .Values.pipelines.runTrigger.customInitContainers -}} -{{- .Values.pipelines.runTrigger.customInitContainers -}} +{{- else if .Values.pipelines.runservice.customInitContainers -}} +{{- .Values.pipelines.runservice.customInitContainers -}} {{- end -}} {{- end -}} @@ -698,14 +693,14 @@ Resolve customSidecarContainers value for internalapi {{- end -}} {{/* -Resolve customSidecarContainers value for steptrigger +Resolve customSidecarContainers value for stepservice */}} -{{- define "pipelines.steptrigger.customSidecarContainers" -}} +{{- define "pipelines.stepservice.customSidecarContainers" -}} {{- if .Values.global.customSidecarContainers -}} {{- .Values.global.customSidecarContainers -}} {{- end -}} -{{- if .Values.pipelines.stepTrigger.customSidecarContainers -}} -{{- .Values.pipelines.stepTrigger.customSidecarContainers -}} +{{- if .Values.pipelines.stepservice.customSidecarContainers -}} +{{- .Values.pipelines.stepservice.customSidecarContainers -}} {{- end -}} {{- end -}} @@ -728,8 +723,8 @@ Resolve customSidecarContainers value for trigger {{- if .Values.global.customSidecarContainers -}} {{- .Values.global.customSidecarContainers -}} {{- end -}} -{{- if .Values.pipelines.runTrigger.customSidecarContainers -}} -{{- .Values.pipelines.runTrigger.customSidecarContainers -}} +{{- if .Values.pipelines.runservice.customSidecarContainers -}} +{{- .Values.pipelines.runservice.customSidecarContainers -}} {{- end -}} {{- end -}} @@ -797,8 +792,7 @@ Return the proper vault image name Return the proper pipelines app version */}} {{- define "pipelines.app.version" -}} -{{- $image := split ":" ((include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelinesInit" )) | toString) -}} -{{- $tag := $image._1 -}} +{{- $tag := (splitList ":" ((include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelinesInit" )))) | last | toString -}} {{- printf "%s" $tag -}} {{- end -}} diff --git a/stable/pipelines/templates/pipelines-cron-statefulset.yaml b/stable/pipelines/templates/pipelines-cron-statefulset.yaml index af95689c7..c92f19c70 100644 --- a/stable/pipelines/templates/pipelines-cron-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-cron-statefulset.yaml @@ -56,6 +56,7 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: {{- if or .Values.pipelines.cron.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.cron.customInitContainersBegin" .) . | indent 8 }} @@ -170,11 +171,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -366,6 +369,63 @@ spec: {{- if .Values.pipelines.router.readinessProbe.enabled }} readinessProbe: {{ tpl .Values.pipelines.router.readinessProbe.config . | indent 12 }} + {{- end }} + - name: observability + image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "observability" ) }} + imagePullPolicy: {{ .Values.pipelines.observability.image.pullPolicy }} + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + capabilities: + drop: + - NET_RAW + env: + - name: JF_OBSERVABILITY_PARENT_SERVICE + value: "cron" + - name: JF_OBSERVABILITY_PARENT_PRODUCT + value: "pipelines" + - name: JF_SHARED_SECURITY_MASTERKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.masterKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: master-key + - name: JF_SHARED_SECURITY_JOINKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.joinKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: join-key +{{- if .Values.pipelines.extraEnvironmentVariables }} +{{- with .Values.pipelines.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pipelines.observability.extraEnvironmentVariables }} +{{- with .Values.pipelines.observability.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} + resources: + {{ toYaml .Values.pipelines.observability.resources | nindent 12 }} + {{- if .Values.pipelines.observability.startupProbe.enabled }} + volumeMounts: + - name: jfrog-pipelines-logs + mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} + startupProbe: +{{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} + {{- end }} + {{- if .Values.pipelines.observability.livenessProbe.enabled }} + livenessProbe: +{{ tpl .Values.pipelines.observability.livenessProbe.config . | indent 12 }} {{- end }} - name: cron image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "cron" ) }} diff --git a/stable/pipelines/templates/pipelines-hookhandler-statefulset.yaml b/stable/pipelines/templates/pipelines-hookhandler-statefulset.yaml index 84cbf0f16..e401a333d 100644 --- a/stable/pipelines/templates/pipelines-hookhandler-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-hookhandler-statefulset.yaml @@ -60,6 +60,7 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: {{- if or .Values.pipelines.hookHandler.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.hookhandler.customInitContainersBegin" .) . | indent 8 }} @@ -174,11 +175,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -370,6 +373,63 @@ spec: {{- if .Values.pipelines.router.readinessProbe.enabled }} readinessProbe: {{ tpl .Values.pipelines.router.readinessProbe.config . | indent 12 }} + {{- end }} + - name: observability + image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "observability" ) }} + imagePullPolicy: {{ .Values.pipelines.observability.image.pullPolicy }} + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + capabilities: + drop: + - NET_RAW + env: + - name: JF_OBSERVABILITY_PARENT_SERVICE + value: "hookhandler" + - name: JF_OBSERVABILITY_PARENT_PRODUCT + value: "pipelines" + - name: JF_SHARED_SECURITY_MASTERKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.masterKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: master-key + - name: JF_SHARED_SECURITY_JOINKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.joinKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: join-key +{{- if .Values.pipelines.extraEnvironmentVariables }} +{{- with .Values.pipelines.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pipelines.observability.extraEnvironmentVariables }} +{{- with .Values.pipelines.observability.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} + resources: + {{ toYaml .Values.pipelines.observability.resources | nindent 12 }} + {{- if .Values.pipelines.observability.startupProbe.enabled }} + volumeMounts: + - name: jfrog-pipelines-logs + mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} + startupProbe: +{{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} + {{- end }} + {{- if .Values.pipelines.observability.livenessProbe.enabled }} + livenessProbe: +{{ tpl .Values.pipelines.observability.livenessProbe.config . | indent 12 }} {{- end }} - name: hookhandler image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "hookHandler" ) }} diff --git a/stable/pipelines/templates/pipelines-internalapi-statefulset.yaml b/stable/pipelines/templates/pipelines-internalapi-statefulset.yaml index 582345d71..925a5c4fd 100644 --- a/stable/pipelines/templates/pipelines-internalapi-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-internalapi-statefulset.yaml @@ -62,6 +62,7 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: {{- if or .Values.pipelines.internalapi.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.internalapi.customInitContainersBegin" .) . | indent 8 }} @@ -168,6 +169,7 @@ spec: done; {{- end }} {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: wait-for-vault image: "{{ .Values.initContainer.image }}" imagePullPolicy: {{ .Values.initContainer.pullPolicy }} @@ -216,6 +218,7 @@ spec: sleep 1; done; {{- end }} +{{- end }} - name: pipelines-installer image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelinesInit" ) }} imagePullPolicy: {{ .Values.pipelines.pipelinesInit.image.pullPolicy }} @@ -233,11 +236,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -496,6 +501,8 @@ spec: volumeMounts: - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} startupProbe: {{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} {{- end }} diff --git a/stable/pipelines/templates/pipelines-statefulset.yaml b/stable/pipelines/templates/pipelines-statefulset.yaml index 6508d7afb..a11dcd2e1 100644 --- a/stable/pipelines/templates/pipelines-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-statefulset.yaml @@ -62,6 +62,7 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: {{- if or .Values.pipelines.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.customInitContainersBegin" .) . | indent 8 }} @@ -125,6 +126,27 @@ spec: - name: ca-certs mountPath: "/tmp/certs" {{- end }} + - name: change-ownership + image: "{{ .Values.initContainer.image }}" + imagePullPolicy: {{ .Values.initContainer.pullPolicy }} + securityContext: + allowPrivilegeEscalation: false + capabilities: + add: + - CHOWN + drop: + - NET_RAW + resources: +{{ toYaml .Values.initContainers.resources | nindent 12 }} + command: + - '/bin/sh' + - '-c' + - > + echo "change ownership of {{ .Values.pipelines.mountPath }} directory"; + chown -R 1066:1066 {{ .Values.pipelines.mountPath }}; + volumeMounts: + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.mountPath }} {{- if .Values.vault.enabled }} {{- if .Values.waitForDatabase }} - name: wait-for-db @@ -195,6 +217,7 @@ spec: {{- end }} {{- end }} {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: wait-for-vault image: "{{ .Values.initContainer.image }}" imagePullPolicy: {{ .Values.initContainer.pullPolicy }} @@ -243,6 +266,7 @@ spec: sleep 2; done; {{- end }} +{{- end }} - name: pipelines-installer image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelinesInit" ) }} imagePullPolicy: {{ .Values.pipelines.pipelinesInit.image.pullPolicy }} @@ -260,11 +284,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -525,6 +551,8 @@ spec: volumeMounts: - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} startupProbe: {{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} {{- end }} @@ -804,8 +832,6 @@ spec: {{ tpl . $ | nindent 10 }} {{- end }} {{- end }} - -{{- if .Values.pipelines.stepservice.enabled }} - name: stepservice image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "stepservice" ) }} imagePullPolicy: {{ .Values.pipelines.stepservice.image.pullPolicy }} @@ -843,8 +869,6 @@ spec: {{- with .Values.pipelines.customVolumeMounts }} {{ tpl . $ | nindent 10 }} {{- end }} -{{- end }} - - name: pipelinesync image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelineSync" ) }} imagePullPolicy: {{ .Values.pipelines.pipelineSync.image.pullPolicy }} @@ -882,80 +906,6 @@ spec: mountPath: {{ .Values.pipelines.mountPath }} - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.logPath }} - - name: runtrigger - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "runTrigger" ) }} - imagePullPolicy: {{ .Values.pipelines.runTrigger.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/runTrigger - env: - - name: COMPONENT - value: runtrigger - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.runTrigger.extraEnvironmentVariables }} -{{- with .Values.pipelines.runTrigger.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.runTrigger.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - - name: steptrigger - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "stepTrigger" ) }} - imagePullPolicy: {{ .Values.pipelines.stepTrigger.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/stepTrigger - env: - - name: COMPONENT - value: steptrigger - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.stepTrigger.extraEnvironmentVariables }} -{{- with .Values.pipelines.stepTrigger.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.stepTrigger.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - name: cron image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "cron" ) }} imagePullPolicy: {{ .Values.pipelines.cron.image.pullPolicy }} @@ -1069,80 +1019,6 @@ spec: mountPath: {{ .Values.pipelines.mountPath }} - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.logPath }} - - name: marshaller - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "marshaller" ) }} - imagePullPolicy: {{ .Values.pipelines.marshaller.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/marshaller - env: - - name: COMPONENT - value: marshaller - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.marshaller.extraEnvironmentVariables }} -{{- with .Values.pipelines.marshaller.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.marshaller.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - - name: logup - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "logup" ) }} - imagePullPolicy: {{ .Values.pipelines.logup.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/logup - env: - - name: COMPONENT - value: logup - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.logup.extraEnvironmentVariables }} -{{- with .Values.pipelines.logup.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.logup.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - name: extensionsync image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "extensionSync" ) }} imagePullPolicy: {{ .Values.pipelines.extensionSync.image.pullPolicy }} diff --git a/stable/pipelines/templates/pipelines-steptrigger-statefulset.yaml b/stable/pipelines/templates/pipelines-steptrigger-statefulset.yaml index 7542c9315..676ecbc6b 100644 --- a/stable/pipelines/templates/pipelines-steptrigger-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-steptrigger-statefulset.yaml @@ -2,10 +2,10 @@ apiVersion: apps/v1 kind: StatefulSet metadata: - name: {{ include "pipelines.steptrigger.name" . }} + name: {{ include "pipelines.stepservice.name" . }} labels: - {{- include "pipelines.steptrigger.labels" . | nindent 4 }} - {{- with .Values.pipelines.stepTrigger.labels }} + {{- include "pipelines.stepservice.labels" . | nindent 4 }} + {{- with .Values.pipelines.stepservice.labels }} {{ toYaml . | indent 4 }} {{- end }} {{- include "pipelines.common.labels" . | nindent 4 }} @@ -14,18 +14,18 @@ metadata: {{- end }} spec: serviceName: {{ include "pipelines.services.name" . }}-headless - replicas: {{ .Values.pipelines.stepTrigger.replicaCount }} + replicas: {{ .Values.pipelines.stepservice.replicaCount }} updateStrategy: type: {{ .Values.pipelines.updateStrategy }} selector: matchLabels: - {{- include "pipelines.steptrigger.selectorLabels" . | nindent 6 }} - component: {{ include "pipelines.steptrigger.name" . }} + {{- include "pipelines.stepservice.selectorLabels" . | nindent 6 }} + component: {{ include "pipelines.stepservice.name" . }} template: metadata: labels: - {{- include "pipelines.steptrigger.selectorLabels" . | nindent 8 }} - component: {{ include "pipelines.steptrigger.name" . }} + {{- include "pipelines.stepservice.selectorLabels" . | nindent 8 }} + component: {{ include "pipelines.stepservice.name" . }} {{- include "pipelines.common.labels" . | nindent 8 }} {{- with .Values.pipelines.labels }} {{ toYaml . | indent 8 }} @@ -56,9 +56,10 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: - {{- if or .Values.pipelines.stepTrigger.customInitContainersBegin .Values.global.customInitContainersBegin }} -{{ tpl (include "pipelines.steptrigger.customInitContainersBegin" .) . | indent 8 }} + {{- if or .Values.pipelines.stepservice.customInitContainersBegin .Values.global.customInitContainersBegin }} +{{ tpl (include "pipelines.stepservice.customInitContainersBegin" .) . | indent 8 }} {{- end }} - name: wait-for-pipelines-internalapi image: '{{ .Values.initContainer.image }}' @@ -170,11 +171,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -258,8 +261,8 @@ spec: mountPath: {{ .Values.pipelines.mountPath }}/buildplane-config readOnly: true {{- end }} - {{- if or .Values.pipelines.stepTrigger.customInitContainers .Values.global.customInitContainers }} -{{ tpl (include "pipelines.steptrigger.customInitContainers" .) . | indent 8 }} + {{- if or .Values.pipelines.stepservice.customInitContainers .Values.global.customInitContainers }} +{{ tpl (include "pipelines.stepservice.customInitContainers" .) . | indent 8 }} {{- end }} {{- if .Values.hostAliases }} hostAliases: @@ -367,7 +370,6 @@ spec: readinessProbe: {{ tpl .Values.pipelines.router.readinessProbe.config . | indent 12 }} {{- end }} -{{- if .Values.pipelines.stepservice.enabled }} - name: stepservice image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "stepservice" ) }} imagePullPolicy: {{ .Values.pipelines.stepservice.image.pullPolicy }} @@ -405,49 +407,63 @@ spec: {{- with .Values.pipelines.customVolumeMounts }} {{ tpl . $ | nindent 10 }} {{- end }} -{{- end }} - - name: steptrigger - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "stepTrigger" ) }} - imagePullPolicy: {{ .Values.pipelines.stepTrigger.image.pullPolicy }} + - name: observability + image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "observability" ) }} + imagePullPolicy: {{ .Values.pipelines.observability.image.pullPolicy }} securityContext: + runAsNonRoot: false allowPrivilegeEscalation: false capabilities: drop: - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/stepTrigger env: - - name: PIPELINES_INTERNAL_API - value: "true" - - name: COMPONENT - value: steptrigger - - name: PIPELINES_NODE_ID + - name: JF_OBSERVABILITY_PARENT_SERVICE + value: "stepservice" + - name: JF_OBSERVABILITY_PARENT_PRODUCT + value: "pipelines" + - name: JF_SHARED_SECURITY_MASTERKEY valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.masterKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: master-key + - name: JF_SHARED_SECURITY_JOINKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.joinKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: join-key {{- if .Values.pipelines.extraEnvironmentVariables }} {{- with .Values.pipelines.extraEnvironmentVariables }} {{ tpl (toYaml .) $ | indent 12 }} {{- end }} {{- end }} -{{- if .Values.pipelines.stepTrigger.extraEnvironmentVariables }} -{{- with .Values.pipelines.stepTrigger.extraEnvironmentVariables }} +{{- if .Values.pipelines.observability.extraEnvironmentVariables }} +{{- with .Values.pipelines.observability.extraEnvironmentVariables }} {{ tpl (toYaml .) $ | indent 12 }} {{- end }} {{- end }} resources: - {{- toYaml .Values.pipelines.stepTrigger.resources | nindent 12 }} + {{ toYaml .Values.pipelines.observability.resources | nindent 12 }} + {{- if .Values.pipelines.observability.startupProbe.enabled }} volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - {{- if or .Values.pipelines.stepTrigger.customSidecarContainers .Values.global.customSidecarContainers }} -{{ tpl (include "pipelines.steptrigger.customSidecarContainers" .) . | nindent 8 }} - {{- end }} + mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} + startupProbe: +{{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} + {{- end }} + {{- if .Values.pipelines.observability.livenessProbe.enabled }} + livenessProbe: +{{ tpl .Values.pipelines.observability.livenessProbe.config . | indent 12 }} + {{- end }} {{- if or .Values.pipelines.nodeSelector .Values.global.nodeSelector }} {{ tpl (include "pipelines.nodeSelector" .) . | indent 6 }} {{- end }} @@ -465,7 +481,7 @@ spec: topologyKey: {{ .Values.pipelines.podAntiAffinity.topologyKey }} labelSelector: matchLabels: - app: {{ template "pipelines.steptrigger.name" . }} + app: {{ template "pipelines.stepservice.name" . }} release: {{ .Release.Name }} {{- else if eq .Values.pipelines.podAntiAffinity.type "hard" }} affinity: @@ -474,7 +490,7 @@ spec: - topologyKey: {{ .Values.pipelines.podAntiAffinity.topologyKey }} labelSelector: matchLabels: - app: {{ template "pipelines.steptrigger.name" . }} + app: {{ template "pipelines.stepservice.name" . }} release: {{ .Release.Name }} {{- end }} {{- with .Values.pipelines.tolerations }} @@ -573,4 +589,4 @@ spec: persistentVolumeClaim: claimName: {{ .Values.pipelines.customPersistentVolumeClaim.name }} {{- end }} -{{- end -}} +{{- end -}} \ No newline at end of file diff --git a/stable/pipelines/templates/pipelines-sync-statefulset.yaml b/stable/pipelines/templates/pipelines-sync-statefulset.yaml index 3063ff276..a80d00ca6 100644 --- a/stable/pipelines/templates/pipelines-sync-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-sync-statefulset.yaml @@ -58,6 +58,7 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: {{- if or .Values.pipelines.pipelineSync.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.sync.customInitContainersBegin" .) . | indent 8 }} @@ -172,11 +173,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -372,6 +375,63 @@ spec: {{- if .Values.pipelines.router.readinessProbe.enabled }} readinessProbe: {{ tpl .Values.pipelines.router.readinessProbe.config . | indent 12 }} + {{- end }} + - name: observability + image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "observability" ) }} + imagePullPolicy: {{ .Values.pipelines.observability.image.pullPolicy }} + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + capabilities: + drop: + - NET_RAW + env: + - name: JF_OBSERVABILITY_PARENT_SERVICE + value: "sync" + - name: JF_OBSERVABILITY_PARENT_PRODUCT + value: "pipelines" + - name: JF_SHARED_SECURITY_MASTERKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.masterKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: master-key + - name: JF_SHARED_SECURITY_JOINKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.joinKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: join-key +{{- if .Values.pipelines.extraEnvironmentVariables }} +{{- with .Values.pipelines.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pipelines.observability.extraEnvironmentVariables }} +{{- with .Values.pipelines.observability.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} + resources: + {{ toYaml .Values.pipelines.observability.resources | nindent 12 }} + {{- if .Values.pipelines.observability.startupProbe.enabled }} + volumeMounts: + - name: jfrog-pipelines-logs + mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} + startupProbe: +{{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} + {{- end }} + {{- if .Values.pipelines.observability.livenessProbe.enabled }} + livenessProbe: +{{ tpl .Values.pipelines.observability.livenessProbe.config . | indent 12 }} {{- end }} - name: pipelinesync image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "pipelineSync" ) }} diff --git a/stable/pipelines/templates/pipelines-trigger-hpa.yaml b/stable/pipelines/templates/pipelines-trigger-hpa.yaml index 01ee677a1..0d562d4df 100644 --- a/stable/pipelines/templates/pipelines-trigger-hpa.yaml +++ b/stable/pipelines/templates/pipelines-trigger-hpa.yaml @@ -1,4 +1,4 @@ -{{- if and (.Values.splitServicesToPods) (.Values.pipelines.runTrigger.autoscaling.enabled) -}} +{{- if and (.Values.splitServicesToPods) (.Values.pipelines.runservice.autoscaling.enabled) -}} {{ if semverCompare ">=v1.23.0-0" .Capabilities.KubeVersion.Version }} apiVersion: autoscaling/v2 {{ else }} @@ -14,13 +14,13 @@ spec: apiVersion: apps/v1 kind: StatefulSet name: {{ include "pipelines.trigger.name" . }} - minReplicas: {{ .Values.pipelines.runTrigger.autoscaling.minReplicas }} - maxReplicas: {{ .Values.pipelines.runTrigger.autoscaling.maxReplicas }} + minReplicas: {{ .Values.pipelines.runservice.autoscaling.minReplicas }} + maxReplicas: {{ .Values.pipelines.runservice.autoscaling.maxReplicas }} metrics: - type: Resource resource: name: cpu target: type: Utilization - averageUtilization: {{ .Values.pipelines.runTrigger.autoscaling.targetCPUUtilizationPercentage }} + averageUtilization: {{ .Values.pipelines.runservice.autoscaling.targetCPUUtilizationPercentage }} {{- end -}} diff --git a/stable/pipelines/templates/pipelines-trigger-statefulset.yaml b/stable/pipelines/templates/pipelines-trigger-statefulset.yaml index eb46b6abb..347224ec7 100644 --- a/stable/pipelines/templates/pipelines-trigger-statefulset.yaml +++ b/stable/pipelines/templates/pipelines-trigger-statefulset.yaml @@ -5,7 +5,7 @@ metadata: name: {{ include "pipelines.trigger.name" . }} labels: {{- include "pipelines.trigger.labels" . | nindent 4 }} - {{- with .Values.pipelines.runTrigger.labels }} + {{- with .Values.pipelines.runservice.labels }} {{ toYaml . | indent 4 }} {{- end }} {{- include "pipelines.common.labels" . | nindent 4 }} @@ -14,8 +14,8 @@ metadata: {{- end }} spec: serviceName: {{ include "pipelines.services.name" . }}-headless -{{- if not .Values.pipelines.runTrigger.autoscaling.enabled }} - replicas: {{ .Values.pipelines.runTrigger.replicaCount }} +{{- if not .Values.pipelines.runservice.autoscaling.enabled }} + replicas: {{ .Values.pipelines.runservice.replicaCount }} {{- end }} updateStrategy: type: {{ .Values.pipelines.updateStrategy }} @@ -58,8 +58,9 @@ spec: {{- if or .Values.imagePullSecrets .Values.global.imagePullSecrets }} {{- include "pipelines.imagePullSecrets" . | nindent 6 }} {{- end }} + terminationGracePeriodSeconds: {{ .Values.terminationGracePeriodSeconds }} initContainers: - {{- if or .Values.pipelines.runTrigger.customInitContainersBegin .Values.global.customInitContainersBegin }} + {{- if or .Values.pipelines.runservice.customInitContainersBegin .Values.global.customInitContainersBegin }} {{ tpl (include "pipelines.trigger.customInitContainersBegin" .) . | indent 8 }} {{- end }} - name: wait-for-pipelines-internalapi @@ -172,11 +173,13 @@ spec: - name: NODE_EXTRA_CA_CERTS value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" {{- end }} +{{- if and .Values.vault.enabled (.Values.access.shouldReadFromVault) (not .Values.access.shouldJustUpdateAccess) }} - name: VAULT_TOKEN valueFrom: secretKeyRef: name: {{ .Values.global.vault.existingSecret | default (printf "%s" "root-vault-secret") }} key: token +{{- end }} - name: PIPELINES_SHARED_DB_CONNECTIONSTRING valueFrom: secretKeyRef: @@ -260,7 +263,7 @@ spec: mountPath: {{ .Values.pipelines.mountPath }}/buildplane-config readOnly: true {{- end }} - {{- if or .Values.pipelines.runTrigger.customInitContainers .Values.global.customInitContainers }} + {{- if or .Values.pipelines.runservice.customInitContainers .Values.global.customInitContainers }} {{ tpl (include "pipelines.trigger.customInitContainers" .) . | indent 8 }} {{- end }} {{- if .Values.hostAliases }} @@ -370,6 +373,63 @@ spec: {{ tpl .Values.pipelines.router.readinessProbe.config . | indent 12 }} {{- end }} {{- if .Values.pipelines.nodepoolservice.enabled }} + - name: observability + image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "observability" ) }} + imagePullPolicy: {{ .Values.pipelines.observability.image.pullPolicy }} + securityContext: + runAsNonRoot: false + allowPrivilegeEscalation: false + capabilities: + drop: + - NET_RAW + env: + - name: JF_OBSERVABILITY_PARENT_SERVICE + value: "trigger" + - name: JF_OBSERVABILITY_PARENT_PRODUCT + value: "pipelines" + - name: JF_SHARED_SECURITY_MASTERKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.masterKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: master-key + - name: JF_SHARED_SECURITY_JOINKEY + valueFrom: + secretKeyRef: + {{- if not .Values.pipelines.unifiedSecretInstallation }} + name: "{{ include "pipelines.joinKeySecretName" . }}" + {{- else }} + name: "{{ template "pipelines.name" . }}-unified-secret" + {{- end }} + key: join-key +{{- if .Values.pipelines.extraEnvironmentVariables }} +{{- with .Values.pipelines.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} +{{- if .Values.pipelines.observability.extraEnvironmentVariables }} +{{- with .Values.pipelines.observability.extraEnvironmentVariables }} +{{ tpl (toYaml .) $ | indent 12 }} +{{- end }} +{{- end }} + resources: + {{ toYaml .Values.pipelines.observability.resources | nindent 12 }} + {{- if .Values.pipelines.observability.startupProbe.enabled }} + volumeMounts: + - name: jfrog-pipelines-logs + mountPath: {{ .Values.pipelines.observability.logPath }} + - name: jfrog-pipelines-folder + mountPath: {{ .Values.pipelines.observability.etcPath }} + startupProbe: +{{ tpl .Values.pipelines.observability.startupProbe.config . | indent 12 }} + {{- end }} + {{- if .Values.pipelines.observability.livenessProbe.enabled }} + livenessProbe: +{{ tpl .Values.pipelines.observability.livenessProbe.config . | indent 12 }} + {{- end }} - name: nodepoolservice image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "nodepoolservice" ) }} imagePullPolicy: {{ .Values.pipelines.nodepoolservice.image.pullPolicy }} @@ -532,84 +592,6 @@ spec: - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.logPath }} {{- end }} - - name: marshaller - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "marshaller" ) }} - imagePullPolicy: {{ .Values.pipelines.marshaller.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/marshaller - env: - - name: PIPELINES_INTERNAL_API - value: "true" - - name: COMPONENT - value: marshaller - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.marshaller.extraEnvironmentVariables }} -{{- with .Values.pipelines.marshaller.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.marshaller.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - - name: runtrigger - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "runTrigger" ) }} - imagePullPolicy: {{ .Values.pipelines.runTrigger.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/runTrigger - env: - - name: PIPELINES_INTERNAL_API - value: "true" - - name: COMPONENT - value: runtrigger - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.runTrigger.extraEnvironmentVariables }} -{{- with .Values.pipelines.runTrigger.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.runTrigger.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - name: reqsealer image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "reqSealer" ) }} imagePullPolicy: {{ .Values.pipelines.reqSealer.image.pullPolicy }} @@ -649,46 +631,7 @@ spec: mountPath: {{ .Values.pipelines.mountPath }} - name: jfrog-pipelines-logs mountPath: {{ .Values.pipelines.logPath }} - - name: logup - image: {{ include "pipelines.getImageInfoByValue" (list . "pipelines" "logup" ) }} - imagePullPolicy: {{ .Values.pipelines.logup.image.pullPolicy }} - securityContext: - allowPrivilegeEscalation: false - capabilities: - drop: - - NET_RAW - workingDir: /opt/jfrog/pipelines/app/micro/logup - env: - - name: PIPELINES_INTERNAL_API - value: "true" - - name: COMPONENT - value: logup - - name: PIPELINES_NODE_ID - valueFrom: - fieldRef: - fieldPath: "metadata.name" - {{- if or .Values.pipelines.customCertificates.enabled .Values.global.customCertificates.enabled }} - - name: NODE_EXTRA_CA_CERTS - value: "{{ .Values.pipelines.mountPath }}/security/keys/trusted/pipelines_custom_certs.crt" - {{- end }} -{{- if .Values.pipelines.extraEnvironmentVariables }} -{{- with .Values.pipelines.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} -{{- if .Values.pipelines.logup.extraEnvironmentVariables }} -{{- with .Values.pipelines.logup.extraEnvironmentVariables }} -{{ tpl (toYaml .) $ | indent 12 }} -{{- end }} -{{- end }} - resources: - {{- toYaml .Values.pipelines.logup.resources | nindent 12 }} - volumeMounts: - - name: jfrog-pipelines-folder - mountPath: {{ .Values.pipelines.mountPath }} - - name: jfrog-pipelines-logs - mountPath: {{ .Values.pipelines.logPath }} - {{- if or .Values.pipelines.runTrigger.customSidecarContainers .Values.global.customSidecarContainers }} + {{- if or .Values.pipelines.runservice.customSidecarContainers .Values.global.customSidecarContainers }} {{ tpl (include "pipelines.trigger.customSidecarContainers" .) . | nindent 8 }} {{- end }} {{- if or .Values.pipelines.nodeSelector .Values.global.nodeSelector }} diff --git a/stable/pipelines/values.yaml b/stable/pipelines/values.yaml index 3dd1d00e3..b33c6ef48 100644 --- a/stable/pipelines/values.yaml +++ b/stable/pipelines/values.yaml @@ -139,6 +139,11 @@ securityContext: splitServicesToPods: false +## @param terminationGracePeriodSeconds Default duration in seconds k8s waits for container to exit before sending kill signal. +## Any time in excess of 10 seconds will be spent waiting for any synchronization necessary for cluster not to lose data. +## +terminationGracePeriodSeconds: 60 + ## Pipelines components pipelines: @@ -213,13 +218,17 @@ pipelines: # PostgressDB health check timeout in seconds dbHealthCheckTimeoutInSeconds: 2 # BuildPlane polling interval - nodePollerIntervalMS: 15000 + nodePollerIntervalMS: 5000 # Auto sync pipelineSource when resource is outdated autoSyncResourceIfOutdated: false + # Retrigger the pipeline after syncing outdated resources + reTriggerOnSyncFailure: true # Allow build badges for pipelines allowBuildBadges: false # Allow static nodes allowCustomNodes: true + # Enable anti-affinity spec for k8s node pool to run one pod per one node + runOnePodInOneNode: true # Enforce non root node pools enforceNonRootNodes: false # maximum step timeout value @@ -264,8 +273,10 @@ pipelines: metrics: ## if enabled, metrics will be logged enabled: false + ## Logging settings logging: + enableJsonConsoleLogAppenders: false ## Livelog settings view: ## If enabled, livelogs will be enabled @@ -771,10 +782,10 @@ pipelines: # customInitContainers: | # customSidecarContainers: | - runTrigger: + runservice: image: # registry: - repository: jfrog/pipelines-micro + repository: jfrog/pipelines-run-service # tag: pullPolicy: IfNotPresent @@ -784,30 +795,6 @@ pipelines: maxReplicas: 3 targetCPUUtilizationPercentage: 200 - resources: {} - # limits: - # cpu: 500m - # memory: 500Mi - # requests: - # cpu: 5m - # memory: 40Mi - - extraEnvironmentVariables: - # - name: MY_ENV_VAR - # value: "example_value" - - ## Add custom init begin containers - first init container to run - # customInitContainersBegin: | - ## Add custom init containers - last init container to run - # customInitContainers: | - # customSidecarContainers: | - - runservice: - image: - # registry: - repository: jfrog/pipelines-run-service - # tag: - pullPolicy: IfNotPresent resources: {} ## This service will be enabled in split deployment design (splitServicesToPods) as a mandatory service enabled: true @@ -822,6 +809,12 @@ pipelines: # - name: MY_ENV_VAR # value: "example_value" + ## Add custom init begin containers - first init container to run + # customInitContainersBegin: | + ## Add custom init containers - last init container to run + # customInitContainers: | + # customSidecarContainers: | + logservice: image: # registry: @@ -842,31 +835,6 @@ pipelines: # - name: MY_ENV_VAR # value: "example_value" - stepTrigger: - image: - # registry: - repository: jfrog/pipelines-micro - # tag: - pullPolicy: IfNotPresent - replicaCount: 1 - resources: {} - # limits: - # cpu: 500m - # memory: 500Mi - # requests: - # cpu: 5m - # memory: 40Mi - - extraEnvironmentVariables: - # - name: MY_ENV_VAR - # value: "example_value" - - ## Add custom init begin containers - first init container to run - # customInitContainersBegin: | - ## Add custom init containers - last init container to run - # customInitContainers: | - # customSidecarContainers: | - stepservice: image: # registry: @@ -973,44 +941,6 @@ pipelines: # customInitContainers: | # customSidecarContainers: | - marshaller: - image: - # registry: - repository: jfrog/pipelines-micro - # tag: - pullPolicy: IfNotPresent - - resources: {} - # limits: - # cpu: 500m - # memory: 500Mi - # requests: - # cpu: 5m - # memory: 60Mi - - extraEnvironmentVariables: - # - name: MY_ENV_VAR - # value: "example_value" - - logup: - image: - # registry: - repository: jfrog/pipelines-micro - # tag: - pullPolicy: IfNotPresent - - resources: {} - # limits: - # cpu: 500m - # memory: 500Mi - # requests: - # cpu: 5m - # memory: 40Mi - - extraEnvironmentVariables: - # - name: MY_ENV_VAR - # value: "example_value" - extensionSync: image: # registry: @@ -1075,6 +1005,7 @@ pipelines: # name: observability internalPort: 8036 logPath: "/opt/jfrog/observability/var/log" + etcPath: "/opt/jfrog/observability/var/etc" resources: {} # requests: @@ -1114,6 +1045,10 @@ pipelines: # - name: MY_ENV_VAR # value: "example_value" + filebeat: + enabled: false + + ## Pipelines installer pipelinesInit: image: @@ -1243,6 +1178,23 @@ pipelines: systemYaml: | + observability: + logging: + application: + level: info + {{- if .Values.pipelines.observability.filebeat.enabled }} + filebeat: + enabled: true + elasticsearch: + url: {{ .Values.pipelines.observability.filebeat.elasticsearch.url }} + username: {{ .Values.pipelines.observability.filebeat.elasticsearch.username }} + password: {{ .Values.pipelines.observability.filebeat.elasticsearch.password }} + {{- if .Values.pipelines.observability.filebeat.elasticsearch.ssl }} + ssl: + verification_mode: {{ .Values.pipelines.observability.filebeat.elasticsearch.ssl.verification_mode }} + {{- end }} + {{- end }} + {{- if .Values.router.routerConfiguration }} router: ## Router configuration @@ -1360,11 +1312,7 @@ pipelines: {{- end }} queues: - "core.pipelineSync" - - "core.runTrigger" - - "core.stepTrigger" - - "core.marshaller" - "cluster.init" - - "core.logup" - "www.signals" {{- if .Values.pipelines.nexec.enabled }} - "core.nexec" @@ -1451,7 +1399,9 @@ pipelines: ## Metrics logging metrics: enabled: {{ .Values.pipelines.metrics.enabled }} + logging: + enableJsonConsoleLogAppenders: {{ .Values.pipelines.logging.enableJsonConsoleLogAppenders }} view: enabled: {{ .Values.pipelines.logging.view.enabled }} @@ -1526,6 +1476,7 @@ pipelines: dbHealthCheckIntervalInMins: {{ .Values.pipelines.dbHealthCheckIntervalInMins }} dbHealthCheckTimeoutInSeconds: {{ .Values.pipelines.dbHealthCheckTimeoutInSeconds }} autoSyncResourceIfOutdated: {{ .Values.pipelines.autoSyncResourceIfOutdated}} + reTriggerOnSyncFailure: {{ .Values.pipelines.reTriggerOnSyncFailure }} allowBuildBadges: {{ .Values.pipelines.allowBuildBadges }} ## Global proxy settings, to be applied to all services ## @@ -1606,14 +1557,10 @@ pipelines: sessionSecret: "{{ .Values.pipelines.authToken }}" pipelineSync: name: pipelineSync - runTrigger: - name: runTrigger runservice: name: runservice logservice: name: logservice - stepTrigger: - name: stepTrigger stepservice: name: stepservice cron: @@ -1624,16 +1571,12 @@ pipelines: {{- end }} hookHandler: name: hookHandler - marshaller: - name: marshaller extensionSync: name: extensionSync templateSync: name: templateSync reqSealer: name: reqSealer - logup: - name: logup ## Runtime configuration ## @@ -1652,6 +1595,7 @@ pipelines: defaultMinionInstanceSize: "c4.large" allowDynamicNodes: true allowCustomNodes: {{ .Values.pipelines.allowCustomNodes }} + runOnePodInOneNode: {{ .Values.pipelines.runOnePodInOneNode }} enforceNonRootNodes: {{ .Values.pipelines.enforceNonRootNodes }} {{- range $key, $value := .Values.runtimeOverride }} {{ $key }}: {{ $value | quote }} @@ -1686,9 +1630,9 @@ pipelines: os: Ubuntu_20.04 language: node registryUrl: releases-docker.jfrog.io - image: jfrog/pipelines-u20node + image: jfrog/pipelines-u20node-arm64 isDefault: true - defaultVersion: 16 + defaultVersion: 18 - architecture: x86_64 os: Ubuntu_20.04 language: node @@ -1769,7 +1713,7 @@ pipelines: language: node registryUrl: releases-docker.jfrog.io image: jfrog/pipelines-w19node - defaultVersion: 16 + defaultVersion: 18 - architecture: x86_64 os: WindowsServer_2019 language: java @@ -1787,7 +1731,7 @@ pipelines: language: go registryUrl: releases-docker.jfrog.io image: jfrog/pipelines-w19go - defaultVersion: 1.19 + defaultVersion: 1.21 - architecture: x86_64 os: WindowsServer_2019 language: dotnet @@ -1851,7 +1795,31 @@ pipelines: registryUrl: releases-docker.jfrog.io image: jfrog/pipelines-c8go defaultVersion: 1.19 - + - architecture: x86_64 + os: AmazonLinux_2 + language: node + registryUrl: releases-docker.jfrog.io + image: jfrog/pipelines-c8node + isDefault: true + defaultVersion: 16 + - architecture: x86_64 + os: AmazonLinux_2 + language: java + registryUrl: releases-docker.jfrog.io + image: jfrog/pipelines-c8java + defaultVersion: 17 + - architecture: x86_64 + os: AmazonLinux_2 + language: cpp + registryUrl: releases-docker.jfrog.io + image: jfrog/pipelines-c8cpp + defaultVersion: 9 + - architecture: x86_64 + os: AmazonLinux_2 + language: go + registryUrl: releases-docker.jfrog.io + image: jfrog/pipelines-c8go + defaultVersion: 1.19 ## Runtime Override Properties Section runtimeOverride: {} @@ -1963,7 +1931,7 @@ rabbitmq: access: enableVaultToAccessMigration: false shouldJustUpdateAccess: false - shouldReadFromVault: false + shouldReadFromVault: true ## Redis ## Configuration values for the redis dependency ## ref: https://github.com/bitnami/charts/tree/master/bitnami/redis