From 0579fe80edd965a266687e996f6f7861d21d6b6a Mon Sep 17 00:00:00 2001
From: Doug Szumski <doug@stackhpc.com>
Date: Mon, 3 Feb 2020 16:52:12 +0000
Subject: [PATCH] Extract useful information from HAProxy logs

The HAProxy logs contain detailed information about HTTP and TCP
connections which are not available from the HAProxy exporter. For
example, the information from these logs can be used to measure the
response time for an HTTP request as a function of status code or
'tail latency' as a function of service and backend server.

Change-Id: I716e0d6d42a26c53aae3519db5b26b05ed60b253
(cherry picked from commit 2abd7d44a52a958f99ca7429db9252683dc0a33f)
---
 ansible/roles/common/defaults/main.yml        |  1 +
 .../parse-haproxy-logs-5f550fbdee432aaa.yaml  |  5 +++++
 .../conf/filter/01-rewrite-0.12.conf.j2       |  7 +++++++
 .../conf/filter/01-rewrite-0.14.conf.j2       | 15 ++++++++++++++
 .../templates/conf/filter/02-parser.conf.j2   | 20 +++++++++++++++++++
 .../templates/conf/output/00-local.conf.j2    |  2 +-
 6 files changed, 49 insertions(+), 1 deletion(-)
 create mode 100644 ansible/roles/common/releasenotes/notes/parse-haproxy-logs-5f550fbdee432aaa.yaml

diff --git a/ansible/roles/common/defaults/main.yml b/ansible/roles/common/defaults/main.yml
index 31f4aaa3f1..9d217d8629 100644
--- a/ansible/roles/common/defaults/main.yml
+++ b/ansible/roles/common/defaults/main.yml
@@ -79,6 +79,7 @@ syslog_facilities:
     output_tag: true
     output_time: true
   - name: "haproxy"
+    match: "infra.haproxy.**"
     enabled: "{{ enable_haproxy | bool and inventory_hostname in groups['haproxy'] }}"
     facility: "{{ syslog_haproxy_facility }}"
     logdir: "haproxy"
diff --git a/ansible/roles/common/releasenotes/notes/parse-haproxy-logs-5f550fbdee432aaa.yaml b/ansible/roles/common/releasenotes/notes/parse-haproxy-logs-5f550fbdee432aaa.yaml
new file mode 100644
index 0000000000..2d5c77afd0
--- /dev/null
+++ b/ansible/roles/common/releasenotes/notes/parse-haproxy-logs-5f550fbdee432aaa.yaml
@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    HAProxy logs are now parsed to extract detailed information about HTTP
+    and TCP connections. The original log message is retained.
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
index 473b7a692c..a58b9b1af3 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2
@@ -43,3 +43,10 @@
     rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe
     rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld
 </match>
+
+# Retag log messages from HAProxy according to log format
+<match syslog.{{ syslog_haproxy_facility }}.**>
+    @type rewrite_tag_filter
+    rewriterule1 Payload ^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S{2}[^\/]+.*$ infra.haproxy.tcp
+    rewriterule2 Payload ^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S \S \S{4}[^\/]+.*$ infra.haproxy.http
+</match>
diff --git a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2 b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
index 4337b877f2..cd882fd477 100644
--- a/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2
@@ -192,3 +192,18 @@
     tag infra.mariadb.mysqld
   </rule>
 </match>
+
+# Retag log messages from HAProxy according to log format
+<match syslog.{{ syslog_haproxy_facility }}.**>
+  @type rewrite_tag_filter
+  <rule>
+    key Payload
+    pattern /^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S{2}[^\/]+.*$/
+    tag infra.haproxy.tcp
+  </rule>
+  <rule>
+    key Payload
+    pattern /^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S \S \S{4}[^\/]+.*$/
+    tag infra.haproxy.http
+  </rule>
+</match>
diff --git a/ansible/roles/common/templates/conf/filter/02-parser.conf.j2 b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2
index 5936a3cf54..be852a7cd5 100644
--- a/ansible/roles/common/templates/conf/filter/02-parser.conf.j2
+++ b/ansible/roles/common/templates/conf/filter/02-parser.conf.j2
@@ -25,3 +25,23 @@
     timestamp ${time}
   </record>
 </filter>
+
+# Parse HAProxy TCP logs
+<filter infra.haproxy.tcp>
+    @type parser
+    format /^.*haproxy.*\]: (?<client_ip>[^ ]+):(?<client_port>[^ ]+) \[(?<accept_date>[^ ]+)\] (?<frontend_name>[^ ]+) (?<backend_name>[^ ]+)\/(?<server_name>[^ ]+) ((?<tw>[^\/]+)\/(?<tc>[^\/]+)\/(?<tt>[^ ]+)) (?<bytes_read>[^ ]+) \S{2} (?<actconn>[^\/]+)\/(?<feconn>[^\/]+)\/(?<beconn>[^\/]+)\/(?<srv_conn>[^\/]+)\/(?<retries>[^ ]+) (?<srv_queue>[^\/]+)\/(?<backend_queue>[^ ]+)$/
+    time_format %d/%b/%Y:%k:%M:%S.%N
+    time_key accept_date
+    key_name Payload
+    reserve_data true
+</filter>
+
+# Parse HAProxy HTTP logs
+<filter infra.haproxy.http>
+    @type parser
+    format /^.*haproxy.*\]: (?<client_ip>[^ ]+):(?<client_port>[^ ]+) \[(?<accept_date>[^ ]+)\] (?<frontend_name>[^ ]+) (?<backend_name>[^ ]+)\/(?<server_name>[^ ]+) (((?<tq>[^\/]+)\/(?<tw>[^\/]+)\/(?<tc>[^\/]+)\/(?<tr>[^\/]+)\/(?<tt>[^ ]+) (?<status_code>[^ ]+))) (?<bytes_read>[^ ]+) (\S \S \S{4}|\S{2}) (?<actconn>[^\/]+)\/(?<feconn>[^\/]+)\/(?<beconn>[^\/]+)\/(?<srv_conn>[^\/]+)\/(?<retries>[^ ]+) (?<srv_queue>[^\/]+)\/(?<backend_queue>[^ ]+) "(?<http_req>[^ ]+) (?<http_path>[^ ]+) (?<http_version>[^ ]+)"$/
+    time_format %d/%b/%Y:%k:%M:%S.%N
+    time_key accept_date
+    key_name Payload
+    reserve_data true
+</filter>
diff --git a/ansible/roles/common/templates/conf/output/00-local.conf.j2 b/ansible/roles/common/templates/conf/output/00-local.conf.j2
index 2f628f8111..f2ccae3f9d 100644
--- a/ansible/roles/common/templates/conf/output/00-local.conf.j2
+++ b/ansible/roles/common/templates/conf/output/00-local.conf.j2
@@ -1,5 +1,5 @@
 {% for item in syslog_facilities | selectattr('enabled') %}
-<match syslog.{{ item.facility }}.**>
+<match {{ item.match | default('syslog.' ~ item.facility ~ '.**') }}>
   @type copy
   <store>
     @type file