From ced6a4e1b6040785b6a2ac6f4a6ee5e619729f15 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 10 Oct 2024 16:37:16 +0200 Subject: [PATCH 01/52] [openetelemetry] bump charts --- system/opentelemetry/Chart.lock | 6 +++--- system/opentelemetry/Chart.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/Chart.lock b/system/opentelemetry/Chart.lock index cd1e554b9ca..1b5009bb27a 100644 --- a/system/opentelemetry/Chart.lock +++ b/system/opentelemetry/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.3.7 + version: 0.3.11 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 - name: linkerd-support repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.1.4 -digest: sha256:04dbbe63ef041e41b5738f242fc78dd9889d20c33240eadf668eccd725d35f85 -generated: "2024-09-19T09:51:19.623153+02:00" +digest: sha256:cf8b09743c88a6a5a3e18af5102bd598ee3ff57796e1fae9bfb6870b3dfcd6e3 +generated: "2024-10-10T16:36:21.088893+02:00" diff --git a/system/opentelemetry/Chart.yaml b/system/opentelemetry/Chart.yaml index dc608d9f037..8e56cda5ca0 100644 --- a/system/opentelemetry/Chart.yaml +++ b/system/opentelemetry/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: opentelemetry -version: 0.0.3 +version: 0.0.4 description: OpenTelemetry Operator Helm chart for Kubernetes type: application home: https://github.com/sapcc/helm-charts/tree/master/system/opentelemetry-operator @@ -10,7 +10,7 @@ appVersion: 0.98.0 dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.3.7 + version: 0.3.11 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 From 94b2cb71caa36d73de4994b032886c30c13d3df1 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 19 Nov 2024 15:28:34 +0100 Subject: [PATCH 02/52] [opentelemetry] bump chart --- system/opentelemetry/Chart.lock | 6 +++--- system/opentelemetry/Chart.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/Chart.lock b/system/opentelemetry/Chart.lock index 1b5009bb27a..c3acee244fe 100644 --- a/system/opentelemetry/Chart.lock +++ b/system/opentelemetry/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.3.11 + version: 0.4.4 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 - name: linkerd-support repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.1.4 -digest: sha256:cf8b09743c88a6a5a3e18af5102bd598ee3ff57796e1fae9bfb6870b3dfcd6e3 -generated: "2024-10-10T16:36:21.088893+02:00" +digest: sha256:d184fa56eb74a9425933a6540d8ae59fe58d9ec3f242cfe4a66859a5f0bc8f1f +generated: "2024-11-19T13:43:56.744556+01:00" diff --git a/system/opentelemetry/Chart.yaml b/system/opentelemetry/Chart.yaml index 8e56cda5ca0..27b20aee029 100644 --- a/system/opentelemetry/Chart.yaml +++ b/system/opentelemetry/Chart.yaml @@ -6,11 +6,11 @@ type: application home: https://github.com/sapcc/helm-charts/tree/master/system/opentelemetry-operator sources: - https://github.com/open-telemetry/opentelemetry-operator -appVersion: 0.98.0 +appVersion: 0.113.0 dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.3.11 + version: 0.4.4 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 From 8c127fb1853f4d5465756a9468d4de1070df36b6 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 19 Nov 2024 16:23:33 +0100 Subject: [PATCH 03/52] [opentelemetry] bump to latest 0.5.0 release --- system/opentelemetry/Chart.lock | 6 +++--- system/opentelemetry/Chart.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/Chart.lock b/system/opentelemetry/Chart.lock index c3acee244fe..81becd89256 100644 --- a/system/opentelemetry/Chart.lock +++ b/system/opentelemetry/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.4.4 + version: 0.5.0 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 - name: linkerd-support repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.1.4 -digest: sha256:d184fa56eb74a9425933a6540d8ae59fe58d9ec3f242cfe4a66859a5f0bc8f1f -generated: "2024-11-19T13:43:56.744556+01:00" +digest: sha256:cf531764203012b55352e6b40e3bf5e2ad099f329860dacbac0917b6245af545 +generated: "2024-11-19T16:23:03.241148+01:00" diff --git a/system/opentelemetry/Chart.yaml b/system/opentelemetry/Chart.yaml index 27b20aee029..986143a1841 100644 --- a/system/opentelemetry/Chart.yaml +++ b/system/opentelemetry/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: opentelemetry -version: 0.0.4 +version: 0.5.0 description: OpenTelemetry Operator Helm chart for Kubernetes type: application home: https://github.com/sapcc/helm-charts/tree/master/system/opentelemetry-operator @@ -10,7 +10,7 @@ appVersion: 0.113.0 dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.4.4 + version: 0.5.0 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 From 010196162c03ef8c2154464d36c4b67168e43f5f Mon Sep 17 00:00:00 2001 From: I504010 Date: Wed, 20 Nov 2024 11:42:42 +0100 Subject: [PATCH 04/52] [opentelemetry] bump chart to 5.1 for bugfix --- system/opentelemetry/Chart.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/opentelemetry/Chart.yaml b/system/opentelemetry/Chart.yaml index 986143a1841..28c1f24ccee 100644 --- a/system/opentelemetry/Chart.yaml +++ b/system/opentelemetry/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: opentelemetry -version: 0.5.0 +version: 0.5.1 description: OpenTelemetry Operator Helm chart for Kubernetes type: application home: https://github.com/sapcc/helm-charts/tree/master/system/opentelemetry-operator @@ -10,7 +10,7 @@ appVersion: 0.113.0 dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.5.0 + version: 0.5.1 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 From 2684c4174c025a9f18adc62194c20bdfa24dab21 Mon Sep 17 00:00:00 2001 From: I504010 Date: Wed, 20 Nov 2024 11:46:17 +0100 Subject: [PATCH 05/52] [opentelemetry] add helm dep up --- system/opentelemetry/Chart.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/opentelemetry/Chart.lock b/system/opentelemetry/Chart.lock index 81becd89256..79c36424bdc 100644 --- a/system/opentelemetry/Chart.lock +++ b/system/opentelemetry/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.5.0 + version: 0.5.1 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 - name: linkerd-support repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.1.4 -digest: sha256:cf531764203012b55352e6b40e3bf5e2ad099f329860dacbac0917b6245af545 -generated: "2024-11-19T16:23:03.241148+01:00" +digest: sha256:1021ed62c4eae0359b5d5abd0209e4094b447ef588ee751077a09e792994a6b6 +generated: "2024-11-20T11:45:40.324326+01:00" From 546317bcc922d2899b6117e8286457bd672b387f Mon Sep 17 00:00:00 2001 From: I504010 Date: Wed, 20 Nov 2024 15:09:51 +0100 Subject: [PATCH 06/52] [opentelemetry] bump chart --- system/opentelemetry/Chart.lock | 6 +++--- system/opentelemetry/Chart.yaml | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/Chart.lock b/system/opentelemetry/Chart.lock index 79c36424bdc..0653dda75a1 100644 --- a/system/opentelemetry/Chart.lock +++ b/system/opentelemetry/Chart.lock @@ -1,12 +1,12 @@ dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.5.1 + version: 0.5.2 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 - name: linkerd-support repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.1.4 -digest: sha256:1021ed62c4eae0359b5d5abd0209e4094b447ef588ee751077a09e792994a6b6 -generated: "2024-11-20T11:45:40.324326+01:00" +digest: sha256:ceaefd207a2f73f99e192f85837a62355a21d68e4bf845f043446f1b8de69232 +generated: "2024-11-20T15:09:11.898263+01:00" diff --git a/system/opentelemetry/Chart.yaml b/system/opentelemetry/Chart.yaml index 28c1f24ccee..0ac7dd3e76e 100644 --- a/system/opentelemetry/Chart.yaml +++ b/system/opentelemetry/Chart.yaml @@ -1,6 +1,6 @@ apiVersion: v2 name: opentelemetry -version: 0.5.1 +version: 0.5.2 description: OpenTelemetry Operator Helm chart for Kubernetes type: application home: https://github.com/sapcc/helm-charts/tree/master/system/opentelemetry-operator @@ -10,7 +10,7 @@ appVersion: 0.113.0 dependencies: - name: opentelemetry-operator repository: oci://ghcr.io/cloudoperators/greenhouse-extensions/charts - version: 0.5.1 + version: 0.5.2 - name: owner-info repository: oci://keppel.eu-de-1.cloud.sap/ccloud-helm version: 0.2.0 From 74037475953cacfc69a4da2173ca622aa34d4574 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 21 Nov 2024 16:03:51 +0100 Subject: [PATCH 07/52] [opentelemetry] testing overwriting --- .../opentelemetry/templates/pmon-filelog.yaml | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 system/opentelemetry/templates/pmon-filelog.yaml diff --git a/system/opentelemetry/templates/pmon-filelog.yaml b/system/opentelemetry/templates/pmon-filelog.yaml new file mode 100644 index 00000000000..e926571aab4 --- /dev/null +++ b/system/opentelemetry/templates/pmon-filelog.yaml @@ -0,0 +1,26 @@ +{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.openTelemetry.podMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + annotations: + name: opentelemetry-collector-logs + labels: + {{- include "plugin.labels" . | nindent 4 }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- include "plugin.prometheusLabels" . | nindent 4 }} +spec: + podMetricsEndpoints: + - interval: 60s + scrapeTimeout: 50s + path: /metrics + scheme: http + port: prometheus + selector: + matchLabels: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: {{ .Release.Namespace }}.logs + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry +{{- end }} From bd7f07cee05437c81a9412243ff9f7a7a3a4cf10 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 10:01:10 +0100 Subject: [PATCH 08/52] Update pmon-filelog.yaml --- system/opentelemetry/templates/pmon-filelog.yaml | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/system/opentelemetry/templates/pmon-filelog.yaml b/system/opentelemetry/templates/pmon-filelog.yaml index e926571aab4..470ae32f958 100644 --- a/system/opentelemetry/templates/pmon-filelog.yaml +++ b/system/opentelemetry/templates/pmon-filelog.yaml @@ -5,11 +5,7 @@ metadata: annotations: name: opentelemetry-collector-logs labels: - {{- include "plugin.labels" . | nindent 4 }} - {{- if .Values.customLabels }} - {{ toYaml .Values.customLabels | nindent 4 }} - {{- end }} - {{- include "plugin.prometheusLabels" . | nindent 4 }} + prometheus: infra-monitoring spec: podMetricsEndpoints: - interval: 60s From 4df58848779e824f367ae42e5bad05764fd14b2d Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 10:27:37 +0100 Subject: [PATCH 09/52] Update pmon-filelog.yaml --- system/opentelemetry/templates/pmon-filelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/pmon-filelog.yaml b/system/opentelemetry/templates/pmon-filelog.yaml index 470ae32f958..72876f4d29e 100644 --- a/system/opentelemetry/templates/pmon-filelog.yaml +++ b/system/opentelemetry/templates/pmon-filelog.yaml @@ -1,4 +1,4 @@ -{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.openTelemetry.podMonitor.enabled }} +{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.podMonitor.enabled }} apiVersion: monitoring.coreos.com/v1 kind: PodMonitor metadata: From fdba411290f0fa9bd3cc8e34d0f2f53d4e1691b0 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 10:31:12 +0100 Subject: [PATCH 10/52] Update pmon-filelog.yaml --- system/opentelemetry/templates/pmon-filelog.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/pmon-filelog.yaml b/system/opentelemetry/templates/pmon-filelog.yaml index 72876f4d29e..061c5303c13 100644 --- a/system/opentelemetry/templates/pmon-filelog.yaml +++ b/system/opentelemetry/templates/pmon-filelog.yaml @@ -5,7 +5,7 @@ metadata: annotations: name: opentelemetry-collector-logs labels: - prometheus: infra-monitoring + prometheus: infra-frontend spec: podMetricsEndpoints: - interval: 60s From 05841012f9d7d015c764be57df3403148370dcab Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 10:35:04 +0100 Subject: [PATCH 11/52] Create logs-collector.yaml --- .../templates/logs-collector.yaml | 449 ++++++++++++++++++ 1 file changed, 449 insertions(+) create mode 100644 system/opentelemetry/templates/logs-collector.yaml diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml new file mode 100644 index 00000000000..d0d5acb902e --- /dev/null +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -0,0 +1,449 @@ +{{- if .Values.logsCollector.enabled }} +apiVersion: opentelemetry.io/v1beta1 +kind: OpenTelemetryCollector +metadata: + name: logs + labels: + {{- include "plugin.labels" . | nindent 4 }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 4 }} + {{- end }} +spec: + mode: daemonset + tolerations: + - operator: Exists + env: + - name: NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: MY_POD_IP + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: status.podIP + - name: cluster + value: "{{ .Values.openTelemetry.cluster }}" + - name: region + value: "{{ .Values.openTelemetry.region }}" + envFrom: + - secretRef: + name: otel-basic-auth +{{- if .Values.podMonitor.enabled }} + ports: + - name: prometheus + port: 9999 +{{- end }} + image: {{ index .Values "opentelemetry-operator" "manager" "collectorImage" "repository" }}:{{ index .Values "opentelemetry-operator" "manager" "collectorImage" "tag" }} + volumeMounts: + - mountPath: /var/log + name: varlog + readOnly: true + volumes: + - name: varlog + hostPath: + path: /var/log + config: + receivers: + journald: + directory: /var/log/journal + operators: + - id: journal-label + type: add + field: attributes["log.type"] + value: "journald" + + k8s_events: + auth_type: serviceAccount + + filelog/containerd: + include_file_path: true + include: [ /var/log/pods/*/*/*.log ] + exclude: [ /var/log/pods/otel_logs-*, /var/log/pods/logs_* ] + operators: + - id: container-parser + type: container + - id: parser-containerd + type: add + field: resource["container.runtime"] + value: "containerd" + - id: container-label + type: add + field: attributes["log.type"] + value: "containerd" + + filelog/files: + include: [ /var/log/libvirt/qemu/*.log, /var/log/openvswitch/*.log ] + include_file_path: true + start_at: beginning + multiline: + line_start_pattern: ^\d{4}-\d{2}-\d{2} + operators: + - type: regex_parser + regex: (?P\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}) + - type: time_parser + parse_from: attributes.logtime + layout: '%Y-%m-%dT%H:%M:%S.%L' + layout_type: strptime + - id: file-label + type: add + field: attributes["log.type"] + value: "files" + +{{- if .Values.openTelemetry.prometheus.podMonitor.enabled }} + prometheus/internal: + config: + scrape_configs: + - job_name: 'internal' + scrape_interval: 5s + static_configs: + - targets: ['127.0.0.1:8888'] +{{- end }} + + processors: + batch: + send_batch_max_size: 500 + timeout: 5s + send_batch_size : 10 + +{{- if .Values.openTelemetry.prometheus.podMonitor.enabled }} + attributes/prometheus: + actions: + - action: insert + key: k8s_node_name + value: ${KUBE_NODE_NAME} + - action: insert + key: k8s_cluster_name + value: ${cluster} + - action: insert + key: region + value: ${region} + - action: delete + key: service_instance_id + - action: delete + key: pod +{{- end }} + + attributes/cluster: + actions: + - action: insert + key: k8s.cluster.name + value: ${cluster} + - action: insert + key: region + value: ${region} + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + + attributes/k8sevents: + actions: + - action: insert + key: k8s.node.name + value: ${KUBE_NODE_NAME} + - key: k8s.namespace.name + from_attribute: k8s.namespace.name + action: insert + - action: insert + key: k8s.cluster.name + value: ${cluster} + - action: insert + key: region + value: ${region} + - action: insert + key: log.type + value: "k8sevents" + + transform/journal: + error_mode: ignore + log_statements: + - context: log + statements: + - merge_maps(cache, body, "upsert") where IsMatch(body, "^\\{") + - set(attributes["message"], cache["MESSAGE"]) + - set(attributes["code_file"], cache["CODE_FILE"]) + - set(attributes["code_func"], cache["CODE_FUNC"]) + - set(attributes["code_line"], cache["CODE_LINE"]) + - set(attributes["incovation_id"], cache["INVOCATION_ID"]) + - set(attributes["message_id"], cache["MESSAGE_ID"]) + - set(attributes["priority"], cache["PRIORITY"]) + - set(attributes["syslog_facility"], cache["SYSLOG_FACILITY"]) + - set(attributes["syslog_identifier"], cache["SYSLOG_IDENTIFIER"]) + - set(attributes["tid"], cache["TID"]) + - set(attributes["unit"], cache["UNIT"]) + - set(attributes["boot_id"], cache["_BOOT_ID"]) + - set(attributes["cao_effective"], cache["_CAP_EFFECTIVE"]) + - set(attributes["cmdline"], cache["_CMDLINE"]) + - set(attributes["exe"], cache["_EXE"]) + - set(attributes["gid"], cache["_GID"]) + - set(attributes["hostname"], cache["_HOSTNAME"]) + - set(attributes["machine_id"], cache["_MACHINE_ID"]) + - set(attributes["pid"], cache["_PID"]) + - set(attributes["runtime_scope"], cache["_RUNTIME_SCOPE"]) + - set(attributes["selinux_context"], cache["_SELINUX_CONTEXT"]) + - set(attributes["source_realtime_timestamp"], cache["_SOURCE_REALTIME_TIMESTAMP"]) + - set(attributes["systemd_cgroup"], cache["_SYSTEMD_CGROUP"]) + - set(attributes["systemd_slice"], cache["_SYSTEMD_SLICE"]) + - set(attributes["systemd_unit"], cache["_SYSTEMD_UNIT"]) + - set(attributes["transport"], cache["_TRANSPORT"]) + - set(attributes["uid"], cache["_UID"]) + - set(attributes["cursor"], cache["__CURSOR"]) + - set(attributes["monotonic_timestamp"], cache["__MONOTONIC_TIMESTAMP"]) + - delete_key(attributes, "cache") + + transform/ingress: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["app.label.name"] == "ingress-nginx" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:request_method} %{NOTSPACE:request_path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent}\" %{NUMBER:request_length:int} %{BASE10NUM:request_time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream_response_length:int} %{BASE10NUM:upstream_response_time:float} %{NOTSPACE:upstream_status} %{NOTSPACE:request_id}", true),"upsert") + - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) + + transform/ceph_rgw: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["app.label.component"] == "cephobjectstores.ceph.rook.io" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:debug_level} %{TIMESTAMP_ISO8601:log_timestamp}(%{SPACE})?%{NOTSPACE}(%{SPACE})?%{NOTSPACE}(%{SPACE})?%{WORD}\\:(%{SPACE})?%{WORD}\\:(%{SPACE})?%{IP:client.address}(%{SPACE})?%{NOTSPACE}(%{SPACE})%{PROJECT_ID:project.id}(\\$%{NOTSPACE})?(%{SPACE})?\\[%{HTTPDATE:request.timestamp}\\] \"%{WORD:request.method} \\/(?[a-zA-Z0-9._+-]+)?(\\/)?(%{NOTSPACE:request.path})? %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content.length:int} %{NOTSPACE} \"%{GREEDYDATA:user.agent}\" %{NOTSPACE} latency=%{NUMBER:latency:float}", true, ["PROJECT_ID=([A-Za-z0-9-]+)"]),"upsert") + - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:log_level}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE:process.name}", true),"upsert") + - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) where cache["network.protocol.name"] != nil + - set(attributes["config.parsed"], "ceph_rgw") where attributes["project.id"] != nil + - set(attributes["config.parsed"], "ceph_rgw") where attributes["log_level"] != nil + + transform/ceph_osd: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["app.label.component"] == "cephclusters.ceph.rook.io" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:osd.stats.level}%{SPACE}%{NOTSPACE:osd.stats.files}%{SPACE}%{NUMBER:osd.stats.osd.stats.size:float}%{SPACE}%{WORD}%{SPACE}%{NUMBER:osd.stats.size_unit:float}%{SPACE}%{NUMBER:osd.stats.score:float}%{SPACE}%{NUMBER:osd.stats.read_gb:float}%{SPACE}%{NUMBER:osd.stats.rn_gb:float}%{SPACE}%{NUMBER:osd.stats.rnp1_gb:float}%{SPACE}%{NUMBER:osd.stats.write_gb:float}%{SPACE}%{NUMBER:osd.stats.wnew_gb:float}%{SPACE}%{NUMBER:osd.stats.moved:float}%{SPACE}%{NUMBER:osd.stats.w_amp:float}%{SPACE}%{NUMBER:osd.stats.rd_mb_s:float}%{SPACE}%{NUMBER:osd.stats.wr_mb_s:float}%{SPACE}%{NUMBER:osd.stats.comp_sec:float}%{SPACE}%{NUMBER:osd.stats.comp_merge_cpu_sec:float}%{SPACE}%{NUMBER:osd.stats.cpmp_cnt:float}%{SPACE}%{NUMBER:osd.stats.av_sec:float}%{SPACE}%{NUMBER:osd.stats.keyin:float}%{SPACE}%{NUMBER:osd.stats.keydrop:float}%{SPACE}%{NUMBER:osd.stats.rblob_gb:float}%{SPACE}%{NUMBER:osd.stats.wblol_gb:float}", true),"upsert") + - merge_maps(attributes, ExtractGrokPatterns(body, "%{GREEDYDATA:osd.wall.type}\\:%{SPACE}%{NUMBER:osd.wall.writes:float}(K|M|G)?%{SPACE}writes,%{SPACE}%{NUMBER:osd.wall.syncs:float}(K|M|G)?%{SPACE}syncs,%{SPACE}%{NUMBER:osd.wall.writes_per_sync:float}%{SPACE}writes per sync, written\\:%{SPACE}%{NUMBER:osd.wall.written_gb:float}%{SPACE}(GB|MB|TB)?,%{SPACE}%{NUMBER:osd.written_mb_sec:float}", true),"upsert") + - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:log_level}%{SPACE}%{NOTSPACE}%{SPACE}%{SPACE}%{NOTSPACE:process.name}", true),"upsert") + - set(attributes["config.parsed"], "ceph_osd") where attributes["osd.stats.level"] != nil + - set(attributes["config.parsed"], "ceph_osd") where attributes["log_level"] != nil + + transform/files: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["log.type"] == "files" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{TIMESTAMP_ISO8601:timestamp}%{SPACE}%{GREEDYDATA:log}", true), "upsert") + - set(attributes["config.parsed"], "files") where attributes["log_level"] != nil + + transform/kvm_openvswitch: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["k8s.daemonset.name"] == "neutron-openvswitch-agent" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{NUMBER:process.id}%{SPACE}%{WORD:log_level}%{SPACE}%{NOTSPACE:process.name}%{SPACE}\\[%{REQUEST_ID:request.id}%{SPACE}%{REQUEST_ID:request.global_id}", true, ["REQUEST_ID=([A-Za-z0-9-]+)"]), "upsert") + - set(attributes["config.parsed"], "kvm_openvswitch") where attributes["log_level"] != nil + + transform/kvm_nova_agent: + error_mode: ignore + log_statements: + - context: log + conditions: + - resource.attributes["k8s.daemonset.name"] == "nova-hypervisor-agents-compute-kvm" + statements: + - merge_maps(attributes, ExtractGrokPatterns(body, "%{TIMESTAMP_ISO8601:logtime}%{SPACE}%{NUMBER:process.id}%{SPACE}%{WORD:log_level}%{SPACE}%{NOTSPACE:process.name}%{SPACE}\\[%{REQUEST_ID:request.id}%{SPACE}%{REQUEST_ID:request.global_id}", true, ["REQUEST_ID=([A-Za-z0-9-]+)"]), "upsert") + - set(attributes["config.parsed"], "kvm_nova_agent") where attributes["log_level"] != nil + + k8sattributes: + auth_type: "serviceAccount" + passthrough: false + filter: + node_from_env_var: KUBE_NODE_NAME + extract: + metadata: + - k8s.namespace.name + - k8s.pod.name + - k8s.pod.uid + - k8s.pod.ip + - k8s.pod.start_time + - k8s.deployment.name + - k8s.deployment.uid + - k8s.node.name + - k8s.replicaset.name + - k8s.replicaset.uid + - k8s.statefulset.name + - k8s.statefulset.uid + - k8s.daemonset.name + - k8s.daemonset.uid + - k8s.job.name + - k8s.job.uid + - k8s.cronjob.name + - k8s.node.uid + - k8s.cluster.uid + - k8s.container.name + - container.image.name + - container.image.tag + labels: + - tag_name: app.label.ingress-nginx + key: ingress-nginx + from: pod + - tag_name: app.label.support-group + key: ccloud/support-group + from: pod + - tag_name: app.label.app_name + key: app.kubernetes.io/app_name + from: pod + - tag_name: app.label.component + key: app.kubernetes.io/component + from: pod + - tag_name: app.label.name + key: app.kubernetes.io/name + from: pod + - tag_name: app.label.instance + key: app.kubernetes.io/instance + from: pod + - tag_name: app.label.pod-template-hash + key: app.kubernetes.io/pod-template-hash + from: pod + #ceph + - tag_name: app.label.component + key: app.kubernetes.io/component + from: pod + - tag_name: app.label.created-by + key: app.kubernetes.io/created-by + from: pod + - tag_name: app.label.managed-by + key: app.kubernetes.io/managed-by + from: pod + - tag_name: app.label.part-of + key: app.kubernetes.io/part-of + from: pod + - tag_name: app.label.ceph-osd-id + key: ceph-osd-id + from: pod + - tag_name: app.label.ceph_daemon_id + key: ceph_daemon_id + from: pod + - tag_name: app.label.ceph_daemon_type + key: ceph_daemon_type + from: pod + - tag_name: app.label.device-class + key: device-class + from: pod + - tag_name: app.label.failure-domain + key: failure-domain + from: pod + - tag_name: app.label.osd + key: osd + from: pod + - tag_name: app.label.osd-store + key: osd-store + from: pod + - tag_name: app.label.portable + key: portable + from: pod + - tag_name: app.label.rook_cluster + key: rook_cluster + from: pod + - tag_name: app.label.rook_io.operator-namespace + key: rook_io/operator-namespace + from: pod + - tag_name: app.label.topology-location-host + key: topology-location-host + from: pod + - tag_name: app.label.topology-location-region + key: topology-location-region + from: pod + - tag_name: app.label.topology-location-region + key: topology-location-region + from: pod + - tag_name: app.label.topology-location-root + key: topology-location-root + from: pod + - tag_name: app.label.topology-location-zone + key: topology-location-zone + from: pod + pod_association: + - sources: + - from: resource_attribute + name: k8s.pod.ip + - sources: + - from: resource_attribute + name: k8s.pod.uid + - sources: + - from: resource_attribute + name: k8s.pod.name + - sources: + - from: connection + memory_limiter: + check_interval: 5s + limit_percentage: 80 + spike_limit_percentage: 30 + + exporters: + debug: + verbosity: basic + opensearch/logs: + http: + auth: + authenticator: basicauth + endpoint: {{ .Values.openTelemetry.openSearchLogs.endpoint }} + logs_index: ${username}-datastream + + prometheus: + endpoint: 0.0.0.0:9999 + + extensions: + basicauth: + client_auth: + username: ${username} + password: ${password} + + connectors: + routing: + default_pipelines: [logs/containerd] + error_mode: ignore + table: + - statement: route() where attributes["container.runtime"] =="containerd" + pipelines: [logs/containerd] + + service: + extensions: + - basicauth +{{- if .Values.podMonitor.enabled }} + telemetry: + metrics: + address: 127.0.0.1:8888 + level: detailed +{{- end }} + pipelines: + logs/containerd: + receivers: [filelog/containerd] + processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] + exporters: [opensearch/logs] + logs/files: + receivers: [filelog/files] + processors: [k8sattributes,attributes/cluster,transform/files,batch] + exporters: [opensearch/logs] + logs/k8sevents: + receivers: [k8s_events] + processors: [attributes/k8sevents,batch] + exporters: [opensearch/logs] + logs/journald: + receivers: [journald] + processors: [attributes/cluster,transform/journal,batch] + exporters: [opensearch/logs] +{{- if .Values.podMonitor.enabled }} + metrics/prometheus: + receivers: [prometheus/internal] + processors: [attributes/prometheus] + exporters: [prometheus] +{{- end }} +{{- end }} From 941bf8fdc3427294c623d3377d7591cdd692d76b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 11:12:23 +0100 Subject: [PATCH 12/52] switch to local variables for testing --- system/opentelemetry/templates/logs-collector.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index d0d5acb902e..34aadf9163a 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -27,9 +27,9 @@ spec: apiVersion: v1 fieldPath: status.podIP - name: cluster - value: "{{ .Values.openTelemetry.cluster }}" + value: "{{ .Values.cluster }}" - name: region - value: "{{ .Values.openTelemetry.region }}" + value: "{{ .Values.region }}" envFrom: - secretRef: name: otel-basic-auth @@ -38,7 +38,7 @@ spec: - name: prometheus port: 9999 {{- end }} - image: {{ index .Values "opentelemetry-operator" "manager" "collectorImage" "repository" }}:{{ index .Values "opentelemetry-operator" "manager" "collectorImage" "tag" }} + image: {{ index .Values "manager" "collectorImage" "repository" }}:{{ index .Values "manager" "collectorImage" "tag" }} volumeMounts: - mountPath: /var/log name: varlog @@ -94,7 +94,7 @@ spec: field: attributes["log.type"] value: "files" -{{- if .Values.openTelemetry.prometheus.podMonitor.enabled }} +{{- if .Values.podMonitor.enabled }} prometheus/internal: config: scrape_configs: @@ -110,7 +110,7 @@ spec: timeout: 5s send_batch_size : 10 -{{- if .Values.openTelemetry.prometheus.podMonitor.enabled }} +{{- if .Values.podMonitor.enabled }} attributes/prometheus: actions: - action: insert @@ -394,7 +394,7 @@ spec: http: auth: authenticator: basicauth - endpoint: {{ .Values.openTelemetry.openSearchLogs.endpoint }} + endpoint: {{ .Values.openSearchLogs.endpoint }} logs_index: ${username}-datastream prometheus: From bcb0e856da5b504053c9b1b51dae9070557427b3 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 12:06:02 +0100 Subject: [PATCH 13/52] Create smon.yaml --- system/opentelemetry/templates/smon.yaml | 29 ++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 system/opentelemetry/templates/smon.yaml diff --git a/system/opentelemetry/templates/smon.yaml b/system/opentelemetry/templates/smon.yaml new file mode 100644 index 00000000000..27657d10656 --- /dev/null +++ b/system/opentelemetry/templates/smon.yaml @@ -0,0 +1,29 @@ +{{- if and (.Capabilities.APIVersions.Has "monitoring.coreos.com/v1") .Values.serviceMonitor.enabled }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + annotations: + name: opentelemetry-collector-metrics + labels: + {{- include "plugin.labels" . | nindent 4 }} + {{- if .Values.customLabels }} + {{ toYaml .Values.customLabels | nindent 4 }} + {{- end }} + {{- include "plugin.prometheusLabels" . | nindent 4 }} +spec: + endpoints: + - port: prometheus + relabelings: + - regex: pod + action: labeldrop + - port: monitoring + relabelings: + - regex: pod + action: labeldrop + selector: + matchLabels: + app.kubernetes.io/component: opentelemetry-collector + app.kubernetes.io/instance: {{ .Release.Namespace }}.opentelemetry.metrics + app.kubernetes.io/managed-by: opentelemetry-operator + app.kubernetes.io/part-of: opentelemetry +{{- end }} From a52505d30710553b7eeb16c62c88483908bdf19f Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 13:26:33 +0100 Subject: [PATCH 14/52] testing recreation --- system/opentelemetry/templates/logs-collector.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 34aadf9163a..bac208fce9e 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -63,7 +63,6 @@ spec: filelog/containerd: include_file_path: true include: [ /var/log/pods/*/*/*.log ] - exclude: [ /var/log/pods/otel_logs-*, /var/log/pods/logs_* ] operators: - id: container-parser type: container From 430da621299e0cf21ab8cd5a1aed1a070d15b34b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 22 Nov 2024 14:06:00 +0100 Subject: [PATCH 15/52] remove non table value --- system/opentelemetry/values.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/system/opentelemetry/values.yaml b/system/opentelemetry/values.yaml index 304f9b95577..b569e2ce9b7 100644 --- a/system/opentelemetry/values.yaml +++ b/system/opentelemetry/values.yaml @@ -45,7 +45,6 @@ opentelemetry-operator: region: DEFINED-IN-SECRETS logsCollector: enabled: false - prometheus: DEFINED-IN-SECRETS podMonitor: enabled: false serviceMonitor: From 5a5f6e6f01db103801562f2a02c6dcabe62ea12b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 15:46:17 +0100 Subject: [PATCH 16/52] [opentelemetry] testing counter and connectors --- .../templates/logs-collector.yaml | 34 +++++++++---------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index bac208fce9e..26a9a1cc9b8 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -406,43 +406,43 @@ spec: password: ${password} connectors: - routing: - default_pipelines: [logs/containerd] - error_mode: ignore - table: - - statement: route() where attributes["container.runtime"] =="containerd" - pipelines: [logs/containerd] + forward: + count: service: extensions: - basicauth -{{- if .Values.podMonitor.enabled }} telemetry: metrics: address: 127.0.0.1:8888 level: detailed -{{- end }} pipelines: logs/containerd: receivers: [filelog/containerd] - processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] - exporters: [opensearch/logs] + processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] + exporters: [forward, count] logs/files: receivers: [filelog/files] - processors: [k8sattributes,attributes/cluster,transform/files,batch] - exporters: [opensearch/logs] + processors: [k8sattributes,attributes/cluster,transform/files] + exporters: [forward, count] logs/k8sevents: receivers: [k8s_events] - processors: [attributes/k8sevents,batch] - exporters: [opensearch/logs] + processors: [attributes/k8sevents] + exporters: [forward, count] logs/journald: receivers: [journald] - processors: [attributes/cluster,transform/journal,batch] + processors: [attributes/cluster,transform/journal] + exporters: [forward, count] + logs: + receivers: [forward] + processors: [batch] + exporters: [opensearch/logs] + metrics/count: + receivers: [count] + processors: [prometheus/internal] exporters: [opensearch/logs] -{{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] exporters: [prometheus] {{- end }} -{{- end }} From 42012b9b424016b30b9d95fedac4008a2386a892 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 15:59:27 +0100 Subject: [PATCH 17/52] [opentelemetry] testing counter and connectors --- system/opentelemetry/templates/logs-collector.yaml | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 26a9a1cc9b8..97e6291abf6 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -395,20 +395,19 @@ spec: authenticator: basicauth endpoint: {{ .Values.openSearchLogs.endpoint }} logs_index: ${username}-datastream - prometheus: endpoint: 0.0.0.0:9999 + connectors: + forward: + count: + extensions: basicauth: client_auth: username: ${username} password: ${password} - connectors: - forward: - count: - service: extensions: - basicauth @@ -439,8 +438,7 @@ spec: exporters: [opensearch/logs] metrics/count: receivers: [count] - processors: [prometheus/internal] - exporters: [opensearch/logs] + receivers: [prometheus/internal] metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] From 290bf2224ff4e298a0b367388f0bc08d9768ed20 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 16:03:49 +0100 Subject: [PATCH 18/52] [opentelemetry] testing counter and connectors --- system/opentelemetry/templates/logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 97e6291abf6..99059d35cb3 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -438,7 +438,7 @@ spec: exporters: [opensearch/logs] metrics/count: receivers: [count] - receivers: [prometheus/internal] + exporters: [prometheus/internal] metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] From c9f7e83db49c2b993786acc6ed1fb4b7dddd0f4d Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 17:05:40 +0100 Subject: [PATCH 19/52] [opentelemetry] testing counter and connectors --- system/opentelemetry/templates/logs-collector.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 99059d35cb3..fc1439e895d 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -419,20 +419,20 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward, count] + exporters: [logs/forward, count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [forward, count] + exporters: [logs/forward, count] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents] - exporters: [forward, count] + exporters: [logs/forward, count] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal] - exporters: [forward, count] - logs: + exporters: [logs/forward, count] + logs/forward: receivers: [forward] processors: [batch] exporters: [opensearch/logs] From f826af73ec6db8027fb32683600e9a5b4872e9db Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 17:10:36 +0100 Subject: [PATCH 20/52] [opentelemetry] remove counter and connectors --- .../templates/logs-collector.yaml | 27 ++++++------------- 1 file changed, 8 insertions(+), 19 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index fc1439e895d..aab7d994c06 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -398,10 +398,6 @@ spec: prometheus: endpoint: 0.0.0.0:9999 - connectors: - forward: - count: - extensions: basicauth: client_auth: @@ -418,27 +414,20 @@ spec: pipelines: logs/containerd: receivers: [filelog/containerd] - processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [logs/forward, count] + processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent, batch] + exporters: [opensearch/logs] logs/files: receivers: [filelog/files] - processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [logs/forward, count] + processors: [k8sattributes,attributes/cluster,transform/files, batch] + exporters: [opensearch/logs] logs/k8sevents: receivers: [k8s_events] - processors: [attributes/k8sevents] - exporters: [logs/forward, count] + processors: [attributes/k8sevents,batch] + exporters: [opensearch/logs] logs/journald: receivers: [journald] - processors: [attributes/cluster,transform/journal] - exporters: [logs/forward, count] - logs/forward: - receivers: [forward] - processors: [batch] - exporters: [opensearch/logs] - metrics/count: - receivers: [count] - exporters: [prometheus/internal] + processors: [attributes/cluster,transform/journal,batch] + receivers: [opensearch/logs] metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] From f72324612b48ad5d7e8c48f066a8a839cd7c814b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Tue, 26 Nov 2024 17:26:06 +0100 Subject: [PATCH 21/52] [opentelemetry] remove counter and connectors --- system/opentelemetry/templates/logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index aab7d994c06..5cbd1bf7c95 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -427,7 +427,7 @@ spec: logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal,batch] - receivers: [opensearch/logs] + exporters:: [opensearch/logs] metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] From 4c5b4dba3fbab1fa16062720c317939da46f7008 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 09:09:24 +0100 Subject: [PATCH 22/52] [opentelemetry] revert --- .../templates/logs-collector.yaml | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 5cbd1bf7c95..bac208fce9e 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -395,6 +395,7 @@ spec: authenticator: basicauth endpoint: {{ .Values.openSearchLogs.endpoint }} logs_index: ${username}-datastream + prometheus: endpoint: 0.0.0.0:9999 @@ -404,21 +405,31 @@ spec: username: ${username} password: ${password} + connectors: + routing: + default_pipelines: [logs/containerd] + error_mode: ignore + table: + - statement: route() where attributes["container.runtime"] =="containerd" + pipelines: [logs/containerd] + service: extensions: - basicauth +{{- if .Values.podMonitor.enabled }} telemetry: metrics: address: 127.0.0.1:8888 level: detailed +{{- end }} pipelines: logs/containerd: receivers: [filelog/containerd] - processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent, batch] + processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] exporters: [opensearch/logs] logs/files: receivers: [filelog/files] - processors: [k8sattributes,attributes/cluster,transform/files, batch] + processors: [k8sattributes,attributes/cluster,transform/files,batch] exporters: [opensearch/logs] logs/k8sevents: receivers: [k8s_events] @@ -427,9 +438,11 @@ spec: logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal,batch] - exporters:: [opensearch/logs] + exporters: [opensearch/logs] +{{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] processors: [attributes/prometheus] exporters: [prometheus] {{- end }} +{{- end }} From d4c6ad41a2f0cff04549663348589849d021c0c8 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 15:32:46 +0100 Subject: [PATCH 23/52] [count] testing --- system/opentelemetry/templates/logs-collector.yaml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index bac208fce9e..9930d36cde0 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -412,6 +412,7 @@ spec: table: - statement: route() where attributes["container.runtime"] =="containerd" pipelines: [logs/containerd] + count: service: extensions: @@ -426,19 +427,22 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] - exporters: [opensearch/logs] + exporters: [opensearch/logs,count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files,batch] - exporters: [opensearch/logs] + exporters: [opensearch/logs,count] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents,batch] - exporters: [opensearch/logs] + exporters: [opensearch/logs,count] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal,batch] - exporters: [opensearch/logs] + exporters: [opensearch/logs,count] + metrics: + receivers: [count] + exporters: [prometheus/internal] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] From 2740fcd7b94a9cae4811d674b323286ca5137d5e Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 15:54:08 +0100 Subject: [PATCH 24/52] [count] testing --- system/opentelemetry/templates/logs-collector.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 9930d36cde0..37a1b2da48e 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -406,13 +406,13 @@ spec: password: ${password} connectors: + count: routing: default_pipelines: [logs/containerd] error_mode: ignore table: - statement: route() where attributes["container.runtime"] =="containerd" pipelines: [logs/containerd] - count: service: extensions: @@ -427,20 +427,20 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] - exporters: [opensearch/logs,count] + exporters: [opensearch/logs, count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files,batch] - exporters: [opensearch/logs,count] + exporters: [opensearch/logs, count] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents,batch] - exporters: [opensearch/logs,count] + exporters: [opensearch/logs, count] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal,batch] - exporters: [opensearch/logs,count] - metrics: + exporters: [opensearch/logs, count] + metrics/count: receivers: [count] exporters: [prometheus/internal] {{- if .Values.podMonitor.enabled }} From 320d47108f2442cf08a37d5c19b13ea758b2824b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 16:06:54 +0100 Subject: [PATCH 25/52] [count] testing --- system/opentelemetry/templates/logs-collector.yaml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 37a1b2da48e..f787ad72ccd 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -407,12 +407,13 @@ spec: connectors: count: - routing: - default_pipelines: [logs/containerd] - error_mode: ignore - table: - - statement: route() where attributes["container.runtime"] =="containerd" - pipelines: [logs/containerd] + + #routing: + # default_pipelines: [logs/containerd] + # error_mode: ignore + # table: + # - statement: route() where attributes["container.runtime"] =="containerd" + # pipelines: [logs/containerd] service: extensions: From 155c523727c232493c25a32205fc4cef43b4f34e Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 16:14:55 +0100 Subject: [PATCH 26/52] [count] testing --- system/opentelemetry/templates/logs-collector.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index f787ad72ccd..75219f503d1 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -406,8 +406,7 @@ spec: password: ${password} connectors: - count: - + count: {} #routing: # default_pipelines: [logs/containerd] # error_mode: ignore From a9a1922b1aeeac0934ea761b901b5d30d7ace3c2 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Wed, 27 Nov 2024 16:20:44 +0100 Subject: [PATCH 27/52] [count] testing --- system/opentelemetry/templates/logs-collector.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 75219f503d1..1a817b4fda2 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -442,7 +442,8 @@ spec: exporters: [opensearch/logs, count] metrics/count: receivers: [count] - exporters: [prometheus/internal] + processors: [attributes/prometheus] + exporters: [prometheus] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] From 6cb721e294f39788df2b09ebf75e1b233d30f3cd Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 09:52:35 +0100 Subject: [PATCH 28/52] [count] disable for testing --- .../templates/logs-collector.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 1a817b4fda2..278cf5b419d 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -406,7 +406,7 @@ spec: password: ${password} connectors: - count: {} + #count: {} #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -427,23 +427,23 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] - exporters: [opensearch/logs, count] + exporters: [opensearch/logs] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files,batch] - exporters: [opensearch/logs, count] + exporters: [opensearch/logs] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents,batch] - exporters: [opensearch/logs, count] + exporters: [opensearch/logs] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal,batch] - exporters: [opensearch/logs, count] - metrics/count: - receivers: [count] - processors: [attributes/prometheus] - exporters: [prometheus] + exporters: [opensearch/logs] + #metrics/count: + # receivers: [count] + # processors: [attributes/prometheus] + # exporters: [prometheus] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] From bf32ebaa108cdd34bf67f885c127b10de752a5a1 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 13:17:33 +0100 Subject: [PATCH 29/52] [forward] testing --- .../templates/logs-collector.yaml | 20 ++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 278cf5b419d..528994f1c96 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -407,6 +407,7 @@ spec: connectors: #count: {} + forward: {} #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -426,20 +427,25 @@ spec: pipelines: logs/containerd: receivers: [filelog/containerd] - processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent,batch] - exporters: [opensearch/logs] + processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] + exporters: [forward] logs/files: receivers: [filelog/files] - processors: [k8sattributes,attributes/cluster,transform/files,batch] - exporters: [opensearch/logs] + processors: [k8sattributes,attributes/cluster,transform/files] + exporters: [forward] logs/k8sevents: receivers: [k8s_events] - processors: [attributes/k8sevents,batch] - exporters: [opensearch/logs] + processors: [attributes/k8sevents] + exporters: [forward] logs/journald: receivers: [journald] - processors: [attributes/cluster,transform/journal,batch] + processors: [attributes/cluster,transform/journal] + exporters: [forward] + logs/forward: + receivers: [forward] + processors: [batch] exporters: [opensearch/logs] + ] #metrics/count: # receivers: [count] # processors: [attributes/prometheus] From 4399d5f8318f52368db001f8f7fa34e9473ac953 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 13:26:50 +0100 Subject: [PATCH 30/52] [forward] syntax foo --- system/opentelemetry/templates/logs-collector.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 528994f1c96..ee89491483d 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -406,8 +406,8 @@ spec: password: ${password} connectors: - #count: {} forward: {} + #count: {} #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -445,7 +445,6 @@ spec: receivers: [forward] processors: [batch] exporters: [opensearch/logs] - ] #metrics/count: # receivers: [count] # processors: [attributes/prometheus] From 7f5d8ca061027f9c165f5453e6cdf3dbbfb6efe6 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:10:09 +0100 Subject: [PATCH 31/52] [count] adding specific counter for filters --- .../templates/logs-collector.yaml | 39 +++++++++++++++---- 1 file changed, 32 insertions(+), 7 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index ee89491483d..3b34a612018 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -203,6 +203,7 @@ spec: statements: - merge_maps(attributes, ExtractGrokPatterns(body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:request_method} %{NOTSPACE:request_path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent}\" %{NUMBER:request_length:int} %{BASE10NUM:request_time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream_response_length:int} %{BASE10NUM:upstream_response_time:float} %{NOTSPACE:upstream_status} %{NOTSPACE:request_id}", true),"upsert") - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) + - set(attributes["config.parsed"], "ingress-nginx") where attributes["client.address"] != nil transform/ceph_rgw: error_mode: ignore @@ -215,7 +216,6 @@ spec: - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:log_level}%{SPACE}%{NOTSPACE}%{SPACE}%{NOTSPACE:process.name}", true),"upsert") - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) where cache["network.protocol.name"] != nil - set(attributes["config.parsed"], "ceph_rgw") where attributes["project.id"] != nil - - set(attributes["config.parsed"], "ceph_rgw") where attributes["log_level"] != nil transform/ceph_osd: error_mode: ignore @@ -228,7 +228,7 @@ spec: - merge_maps(attributes, ExtractGrokPatterns(body, "%{GREEDYDATA:osd.wall.type}\\:%{SPACE}%{NUMBER:osd.wall.writes:float}(K|M|G)?%{SPACE}writes,%{SPACE}%{NUMBER:osd.wall.syncs:float}(K|M|G)?%{SPACE}syncs,%{SPACE}%{NUMBER:osd.wall.writes_per_sync:float}%{SPACE}writes per sync, written\\:%{SPACE}%{NUMBER:osd.wall.written_gb:float}%{SPACE}(GB|MB|TB)?,%{SPACE}%{NUMBER:osd.written_mb_sec:float}", true),"upsert") - merge_maps(attributes, ExtractGrokPatterns(body, "%{WORD:log_level}%{SPACE}%{NOTSPACE}%{SPACE}%{SPACE}%{NOTSPACE:process.name}", true),"upsert") - set(attributes["config.parsed"], "ceph_osd") where attributes["osd.stats.level"] != nil - - set(attributes["config.parsed"], "ceph_osd") where attributes["log_level"] != nil + - set(attributes["config.parsed"], "ceph_osd") where attributes["osd.wall.type"] != nil transform/files: error_mode: ignore @@ -407,7 +407,32 @@ spec: connectors: forward: {} - #count: {} + count: + logs: + otelcol_filter_ingress_count: + description: Counter for ingress filtered events + conditions: + - 'attributes["config.parsed"], "ingress-nginx"' + otelcol_filter_ceph_rgw_count: + description: Counter for ceph_rgw filtered events + conditions: + - 'attributes["config.parsed"], "ceph_rgw"' + otelcol_filter_ceph_osd_count: + description: Counter for ceph_osd filtered events + conditions: + - 'attributes["config.parsed"], "ceph_osd"' + otelcol_filter_source_files_count: + description: Counter for extra files source events + conditions: + - 'attributes["config.parsed"], "files"' + otelcol_filter_kvm_openvswitch_count: + description: Counter for kvm_openvswitch filtered events + conditions: + - 'attributes["config.parsed"], "kvm_openvswitch"' + otelcol_filter_kvm_nova_agent_count: + description: Counter for kvm nova agent filtered events + conditions: + - 'attributes["config.parsed"], "kvm_nova_agent"' #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -445,10 +470,10 @@ spec: receivers: [forward] processors: [batch] exporters: [opensearch/logs] - #metrics/count: - # receivers: [count] - # processors: [attributes/prometheus] - # exporters: [prometheus] + metrics/count: + receivers: [count] + processors: [attributes/prometheus] + exporters: [prometheus] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] From 9b1e56b8b724522301792c0cccc893bb37cb14c3 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:23:47 +0100 Subject: [PATCH 32/52] [count] different syntax, as docs describe --- .../templates/logs-collector.yaml | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 3b34a612018..5f8fb5c5b31 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -410,29 +410,35 @@ spec: count: logs: otelcol_filter_ingress_count: - description: Counter for ingress filtered events + description: + - 'Counter for ingress filtered events' conditions: - - 'attributes["config.parsed"], "ingress-nginx"' + - 'attributes["config.parsed"] == "ingress-nginx"' otelcol_filter_ceph_rgw_count: - description: Counter for ceph_rgw filtered events + description: + - 'Counter for ceph_rgw filtered events' conditions: - - 'attributes["config.parsed"], "ceph_rgw"' + - 'attributes["config.parsed"] == "ceph_rgw"' otelcol_filter_ceph_osd_count: - description: Counter for ceph_osd filtered events + description: + - 'Counter for ceph_osd filtered events' conditions: - - 'attributes["config.parsed"], "ceph_osd"' + - 'attributes["config.parsed"] == "ceph_osd"' otelcol_filter_source_files_count: - description: Counter for extra files source events + description: + - 'Counter for extra files source events' conditions: - - 'attributes["config.parsed"], "files"' + - 'attributes["config.parsed"] == "files"' otelcol_filter_kvm_openvswitch_count: - description: Counter for kvm_openvswitch filtered events + description: + - 'Counter for kvm_openvswitch filtered events' conditions: - - 'attributes["config.parsed"], "kvm_openvswitch"' + - 'attributes["config.parsed"] == "kvm_openvswitch"' otelcol_filter_kvm_nova_agent_count: - description: Counter for kvm nova agent filtered events + description: + - 'Counter for kvm nova agent filtered events' conditions: - - 'attributes["config.parsed"], "kvm_nova_agent"' + - 'attributes["config.parsed"] == "kvm_nova_agent"' #routing: # default_pipelines: [logs/containerd] # error_mode: ignore From 0d870669ceb3e0457adc9bbad5a13adaa7b2b0fc Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:27:44 +0100 Subject: [PATCH 33/52] [count] syntax foo --- .../templates/logs-collector.yaml | 48 +++++++++---------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 5f8fb5c5b31..e6e7fa0be75 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -410,35 +410,35 @@ spec: count: logs: otelcol_filter_ingress_count: - description: - - 'Counter for ingress filtered events' - conditions: - - 'attributes["config.parsed"] == "ingress-nginx"' + description: + - 'Counter for ingress filtered events' + conditions: + - 'attributes["config.parsed"] == "ingress-nginx"' otelcol_filter_ceph_rgw_count: - description: - - 'Counter for ceph_rgw filtered events' - conditions: - - 'attributes["config.parsed"] == "ceph_rgw"' + description: + - 'Counter for ceph_rgw filtered events' + conditions: + - 'attributes["config.parsed"] == "ceph_rgw"' otelcol_filter_ceph_osd_count: - description: - - 'Counter for ceph_osd filtered events' - conditions: - - 'attributes["config.parsed"] == "ceph_osd"' + description: + - 'Counter for ceph_osd filtered events' + conditions: + - 'attributes["config.parsed"] == "ceph_osd"' otelcol_filter_source_files_count: - description: - - 'Counter for extra files source events' - conditions: - - 'attributes["config.parsed"] == "files"' + description: + - 'Counter for extra files source events' + conditions: + - 'attributes["config.parsed"] == "files"' otelcol_filter_kvm_openvswitch_count: - description: - - 'Counter for kvm_openvswitch filtered events' - conditions: - - 'attributes["config.parsed"] == "kvm_openvswitch"' + description: + - 'Counter for kvm_openvswitch filtered events' + conditions: + - 'attributes["config.parsed"] == "kvm_openvswitch"' otelcol_filter_kvm_nova_agent_count: - description: - - 'Counter for kvm nova agent filtered events' - conditions: - - 'attributes["config.parsed"] == "kvm_nova_agent"' + description: + - 'Counter for kvm nova agent filtered events' + conditions: + - 'attributes["config.parsed"] == "kvm_nova_agent"' #routing: # default_pipelines: [logs/containerd] # error_mode: ignore From 048865ff745446696b32a89c4294ee86fae13831 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:32:13 +0100 Subject: [PATCH 34/52] [count] syntax foo --- .../opentelemetry/templates/logs-collector.yaml | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index e6e7fa0be75..e53e0dc17c3 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -410,28 +410,23 @@ spec: count: logs: otelcol_filter_ingress_count: - description: - - 'Counter for ingress filtered events' + description: Counter for ingress filtered events. conditions: - 'attributes["config.parsed"] == "ingress-nginx"' otelcol_filter_ceph_rgw_count: - description: - - 'Counter for ceph_rgw filtered events' + description: Counter for ceph_rgw filtered events. conditions: - 'attributes["config.parsed"] == "ceph_rgw"' otelcol_filter_ceph_osd_count: - description: - - 'Counter for ceph_osd filtered events' + description: Counter for ceph_osd filtered events. conditions: - 'attributes["config.parsed"] == "ceph_osd"' otelcol_filter_source_files_count: - description: - - 'Counter for extra files source events' + description: Counter for extra files source events. conditions: - 'attributes["config.parsed"] == "files"' otelcol_filter_kvm_openvswitch_count: - description: - - 'Counter for kvm_openvswitch filtered events' + description: Counter for kvm_openvswitch filtered events. conditions: - 'attributes["config.parsed"] == "kvm_openvswitch"' otelcol_filter_kvm_nova_agent_count: From 442ba048011968e92d7aabd56697d71e53132d0a Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:37:43 +0100 Subject: [PATCH 35/52] [count] syntax foo --- system/opentelemetry/templates/logs-collector.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index e53e0dc17c3..be256846090 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -430,8 +430,7 @@ spec: conditions: - 'attributes["config.parsed"] == "kvm_openvswitch"' otelcol_filter_kvm_nova_agent_count: - description: - - 'Counter for kvm nova agent filtered events' + description: Counter for kvm nova agent filtered events. conditions: - 'attributes["config.parsed"] == "kvm_nova_agent"' #routing: From c9595dbf647e7c06ebf64d58e887bad16224e071 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 14:56:01 +0100 Subject: [PATCH 36/52] [count] used as receiver in metrics pipeline but not used in any supported exporter pipeline --- system/opentelemetry/templates/logs-collector.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index be256846090..1399615b6ff 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -453,24 +453,24 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward] + exporters: [forward,count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [forward] + exporters: [forward,count] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents] - exporters: [forward] + exporters: [forward,count] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal] - exporters: [forward] + exporters: [forward,count] logs/forward: receivers: [forward] processors: [batch] exporters: [opensearch/logs] - metrics/count: + metrics: receivers: [count] processors: [attributes/prometheus] exporters: [prometheus] From fadfca1c01f639f9237f98ac8c20dfe1c30a78ca Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 15:25:12 +0100 Subject: [PATCH 37/52] [count] configure prometheus --- system/opentelemetry/templates/logs-collector.yaml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 1399615b6ff..d9c7c8b9735 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -397,7 +397,10 @@ spec: logs_index: ${username}-datastream prometheus: - endpoint: 0.0.0.0:9999 + endpoint: "0.0.0.0:9999" + metric_expiration: 5m + resource_to_telemetry_conversion: + enabled: true extensions: basicauth: From f4a6939941e8ea4b663b8e3ee3fb0ff8e1e47203 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 15:28:51 +0100 Subject: [PATCH 38/52] [count] configure prometheus --- system/opentelemetry/templates/logs-collector.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index d9c7c8b9735..7abfff6f593 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -399,8 +399,6 @@ spec: prometheus: endpoint: "0.0.0.0:9999" metric_expiration: 5m - resource_to_telemetry_conversion: - enabled: true extensions: basicauth: From 9c18a8bc93889333e6029ecd8bef816aa1c1b349 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 15:50:11 +0100 Subject: [PATCH 39/52] [count] disable internal metrics --- system/opentelemetry/templates/logs-collector.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 7abfff6f593..bd0a926d201 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -476,9 +476,9 @@ spec: processors: [attributes/prometheus] exporters: [prometheus] {{- if .Values.podMonitor.enabled }} - metrics/prometheus: - receivers: [prometheus/internal] - processors: [attributes/prometheus] - exporters: [prometheus] + #metrics/prometheus: + # receivers: [prometheus/internal] + # processors: [attributes/prometheus] + # exporters: [prometheus] {{- end }} {{- end }} From 6b782b5b6791d4c9b2dbbbcd0864d0fe43b7b5a9 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 16:58:13 +0100 Subject: [PATCH 40/52] [count] remove count out --- system/opentelemetry/templates/logs-collector.yaml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index bd0a926d201..6d7986ac164 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -454,26 +454,25 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward,count] + exporters: [forward] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [forward,count] + exporters: [forward] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents] - exporters: [forward,count] + exporters: [forward] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal] - exporters: [forward,count] + exporters: [forward] logs/forward: receivers: [forward] processors: [batch] exporters: [opensearch/logs] metrics: receivers: [count] - processors: [attributes/prometheus] exporters: [prometheus] {{- if .Values.podMonitor.enabled }} #metrics/prometheus: From 76f67d7f4f79211dd476700e28aaba35a998aaf9 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 17:20:19 +0100 Subject: [PATCH 41/52] [count] adding debug out --- system/opentelemetry/templates/logs-collector.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 6d7986ac164..ca776ca676b 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -395,10 +395,10 @@ spec: authenticator: basicauth endpoint: {{ .Values.openSearchLogs.endpoint }} logs_index: ${username}-datastream - prometheus: endpoint: "0.0.0.0:9999" - metric_expiration: 5m + resource_to_telemetry_conversion: + enabled: true extensions: basicauth: @@ -473,7 +473,7 @@ spec: exporters: [opensearch/logs] metrics: receivers: [count] - exporters: [prometheus] + exporters: [prometheus, debug] {{- if .Values.podMonitor.enabled }} #metrics/prometheus: # receivers: [prometheus/internal] From f4a9f3b1c3585dadb1f0bac38cc5c127412ce0c0 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Thu, 28 Nov 2024 17:34:23 +0100 Subject: [PATCH 42/52] [count] adding debug out --- system/opentelemetry/templates/logs-collector.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index ca776ca676b..1a96c1ac264 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -454,19 +454,19 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward] + exporters: [forward, count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [forward] + exporters: [forward, count] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents] - exporters: [forward] + exporters: [forward, count] logs/journald: receivers: [journald] processors: [attributes/cluster,transform/journal] - exporters: [forward] + exporters: [forward, count] logs/forward: receivers: [forward] processors: [batch] From 0a0dfc23c98e412e5ec89480cafaf967f6c12bef Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:01:40 +0100 Subject: [PATCH 43/52] [count] remove debug out --- system/opentelemetry/templates/logs-collector.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 1a96c1ac264..2b992360333 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -458,7 +458,7 @@ spec: logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] - exporters: [forward, count] + exporters: [forward] logs/k8sevents: receivers: [k8s_events] processors: [attributes/k8sevents] @@ -473,11 +473,11 @@ spec: exporters: [opensearch/logs] metrics: receivers: [count] - exporters: [prometheus, debug] + exporters: [prometheus] {{- if .Values.podMonitor.enabled }} - #metrics/prometheus: - # receivers: [prometheus/internal] - # processors: [attributes/prometheus] - # exporters: [prometheus] + metrics/prometheus: + receivers: [prometheus/internal] + processors: [attributes/prometheus] + exporters: [prometheus] {{- end }} {{- end }} From e7f45d8f31afb6e2141ca9f8441ef445600f09d4 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:13:54 +0100 Subject: [PATCH 44/52] [count] remove debug out --- system/opentelemetry/templates/logs-collector.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 2b992360333..949994988b3 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -202,7 +202,7 @@ spec: - resource.attributes["app.label.name"] == "ingress-nginx" statements: - merge_maps(attributes, ExtractGrokPatterns(body, "%{IP:client.address} %{NOTSPACE:client.ident} %{NOTSPACE:client.auth} \\[%{HTTPDATE:timestamp}\\] \"%{WORD:request_method} %{NOTSPACE:request_path} %{WORD:network.protocol.name}/%{NOTSPACE:network.protocol.version}\" %{NUMBER:response} %{NUMBER:content_length:int} %{QUOTEDSTRING} \"%{GREEDYDATA:user_agent}\" %{NUMBER:request_length:int} %{BASE10NUM:request_time:float}( \\[%{NOTSPACE:service}\\])? ?(\\[\\])? %{IP:server.address}\\:%{NUMBER:server.port} %{NUMBER:upstream_response_length:int} %{BASE10NUM:upstream_response_time:float} %{NOTSPACE:upstream_status} %{NOTSPACE:request_id}", true),"upsert") - - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) + - set(attributes["network.protocol.name"], ConvertCase(attributes["network.protocol.name"], "lower")) where attributes["network.protocol.name"] != nil - set(attributes["config.parsed"], "ingress-nginx") where attributes["client.address"] != nil transform/ceph_rgw: @@ -454,7 +454,7 @@ spec: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward, count] + exporters: [forward] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] From 2612b8bb69b709aff29a7807f23723c301e66d2b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:23:27 +0100 Subject: [PATCH 45/52] [count] remove debug out --- system/opentelemetry/templates/logs-collector.yaml | 2 -- 1 file changed, 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 949994988b3..829ac545f09 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -397,8 +397,6 @@ spec: logs_index: ${username}-datastream prometheus: endpoint: "0.0.0.0:9999" - resource_to_telemetry_conversion: - enabled: true extensions: basicauth: From 429a9839b4ba9dfcf922ac07520377f8e21a9f04 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:33:38 +0100 Subject: [PATCH 46/52] [count] disable otel metrics --- .../templates/logs-collector.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 829ac545f09..58a3b938d0e 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -94,13 +94,13 @@ spec: value: "files" {{- if .Values.podMonitor.enabled }} - prometheus/internal: - config: - scrape_configs: - - job_name: 'internal' - scrape_interval: 5s - static_configs: - - targets: ['127.0.0.1:8888'] + #prometheus/internal: + # config: + # scrape_configs: + # - job_name: 'internal' + # scrape_interval: 5s + # static_configs: + # - targets: ['127.0.0.1:8888'] {{- end }} processors: @@ -443,16 +443,16 @@ spec: extensions: - basicauth {{- if .Values.podMonitor.enabled }} - telemetry: - metrics: - address: 127.0.0.1:8888 - level: detailed + #telemetry: + # metrics: + # address: 127.0.0.1:8888 + # level: detailed {{- end }} pipelines: logs/containerd: receivers: [filelog/containerd] processors: [k8sattributes,attributes/cluster,transform/ingress,transform/ceph_rgw,transform/ceph_osd,transform/kvm_openvswitch,transform/kvm_nova_agent] - exporters: [forward] + exporters: [forward, count] logs/files: receivers: [filelog/files] processors: [k8sattributes,attributes/cluster,transform/files] From 9d524b92ebad41f5be1b1d79ef28ab1c3d06ec50 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:36:51 +0100 Subject: [PATCH 47/52] [count] disable internal metrics --- system/opentelemetry/templates/logs-collector.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 58a3b938d0e..fc1c4c95b76 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -473,9 +473,9 @@ spec: receivers: [count] exporters: [prometheus] {{- if .Values.podMonitor.enabled }} - metrics/prometheus: - receivers: [prometheus/internal] - processors: [attributes/prometheus] - exporters: [prometheus] + #metrics/prometheus: + # receivers: [prometheus/internal] + # processors: [attributes/prometheus] + # exporters: [prometheus] {{- end }} {{- end }} From d031f3951ebc21bebc0ca99ded8157876ae814b4 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 10:48:59 +0100 Subject: [PATCH 48/52] [count] enable internal metrics --- .../templates/logs-collector.yaml | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index fc1c4c95b76..f76cf847613 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -94,13 +94,13 @@ spec: value: "files" {{- if .Values.podMonitor.enabled }} - #prometheus/internal: - # config: - # scrape_configs: - # - job_name: 'internal' - # scrape_interval: 5s - # static_configs: - # - targets: ['127.0.0.1:8888'] + prometheus/internal: + config: + scrape_configs: + - job_name: 'internal' + scrape_interval: 5s + static_configs: + - targets: ['127.0.0.1:8888'] {{- end }} processors: @@ -443,10 +443,10 @@ spec: extensions: - basicauth {{- if .Values.podMonitor.enabled }} - #telemetry: - # metrics: - # address: 127.0.0.1:8888 - # level: detailed + telemetry: + metrics: + address: 127.0.0.1:8888 + level: detailed {{- end }} pipelines: logs/containerd: @@ -473,9 +473,9 @@ spec: receivers: [count] exporters: [prometheus] {{- if .Values.podMonitor.enabled }} - #metrics/prometheus: - # receivers: [prometheus/internal] - # processors: [attributes/prometheus] - # exporters: [prometheus] + metrics/prometheus: + receivers: [prometheus/internal] + processors: [attributes/prometheus] + exporters: [prometheus] {{- end }} {{- end }} From 553014cef706907a6e992abe35dff08cb6469ef3 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 11:10:00 +0100 Subject: [PATCH 49/52] [count] enable counter for all container logs --- system/opentelemetry/templates/logs-collector.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index f76cf847613..ff400c02397 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -432,6 +432,10 @@ spec: description: Counter for kvm nova agent filtered events. conditions: - 'attributes["config.parsed"] == "kvm_nova_agent"' + otelcol_all_container_records: + description: all container log records. + conditions: + - 'attributes["log.type"] == "containerd"' #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -471,6 +475,7 @@ spec: exporters: [opensearch/logs] metrics: receivers: [count] + processors: [attributes/prometheus] exporters: [prometheus] {{- if .Values.podMonitor.enabled }} metrics/prometheus: From a90e1b73f322cd8eb341d56567dd448661c200ca Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 11:17:12 +0100 Subject: [PATCH 50/52] [count]switch condition --- system/opentelemetry/templates/logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index ff400c02397..8b4fd3919b4 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -411,7 +411,7 @@ spec: otelcol_filter_ingress_count: description: Counter for ingress filtered events. conditions: - - 'attributes["config.parsed"] == "ingress-nginx"' + - IsMatch(attributes["config.parsed"], "ingress-nginx") otelcol_filter_ceph_rgw_count: description: Counter for ceph_rgw filtered events. conditions: From 49da65a8df1adba1ac1c01ecd8ca99accf17774b Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 11:37:24 +0100 Subject: [PATCH 51/52] [count]send metrics to OpenSearch --- system/opentelemetry/templates/logs-collector.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 8b4fd3919b4..5f39b6ec1dd 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -435,7 +435,7 @@ spec: otelcol_all_container_records: description: all container log records. conditions: - - 'attributes["log.type"] == "containerd"' + - IsMatch(attributes["log.type"], "containerd") #routing: # default_pipelines: [logs/containerd] # error_mode: ignore @@ -476,7 +476,7 @@ spec: metrics: receivers: [count] processors: [attributes/prometheus] - exporters: [prometheus] + exporters: [forward] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal] From 1d9772fb3a3df63c1431f53db78698676e33a2f8 Mon Sep 17 00:00:00 2001 From: Olaf Heydorn Date: Fri, 29 Nov 2024 11:40:28 +0100 Subject: [PATCH 52/52] [count]send metrics to OpenSearch --- system/opentelemetry/templates/logs-collector.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/system/opentelemetry/templates/logs-collector.yaml b/system/opentelemetry/templates/logs-collector.yaml index 5f39b6ec1dd..3260802a9de 100644 --- a/system/opentelemetry/templates/logs-collector.yaml +++ b/system/opentelemetry/templates/logs-collector.yaml @@ -476,7 +476,7 @@ spec: metrics: receivers: [count] processors: [attributes/prometheus] - exporters: [forward] + exporters: [opensearch/logs] {{- if .Values.podMonitor.enabled }} metrics/prometheus: receivers: [prometheus/internal]