Extract useful information from HAProxy logs

ansible/roles/common/defaults/main.yml

@@ -79,6 +79,7 @@ syslog_facilities:
     output_tag: true
     output_time: true
   - name: "haproxy"
+    match: "infra.haproxy.**"
     enabled: "{{ enable_haproxy | bool and inventory_hostname in groups['haproxy'] }}"
     facility: "{{ syslog_haproxy_facility }}"
     logdir: "haproxy"

ansible/roles/common/releasenotes/notes/parse-haproxy-logs-5f550fbdee432aaa.yaml

@@ -0,0 +1,5 @@
+---
+features:
+  - |
+    HAProxy logs are now parsed to extract detailed information about HTTP
+    and TCP connections. The original log message is retained.

ansible/roles/common/templates/conf/filter/01-rewrite-0.12.conf.j2

@@ -43,3 +43,10 @@
     rewriterule1 Payload ^\d{6} infra.mariadb.mysqld_safe
     rewriterule2 Payload ^\d{4}-\d{2}-\d{2} infra.mariadb.mysqld
 </match>
+
+# Retag log messages from HAProxy according to log format
+<match syslog.{{ syslog_haproxy_facility }}.**>
+    @type rewrite_tag_filter
+    rewriterule1 Payload ^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S{2}[^\/]+.*$ infra.haproxy.tcp
+    rewriterule2 Payload ^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S \S \S{4}[^\/]+.*$ infra.haproxy.http
+</match>

ansible/roles/common/templates/conf/filter/01-rewrite-0.14.conf.j2

@@ -192,3 +192,18 @@
     tag infra.mariadb.mysqld
   </rule>
 </match>
+
+# Retag log messages from HAProxy according to log format
+<match syslog.{{ syslog_haproxy_facility }}.**>
+  @type rewrite_tag_filter
+  <rule>
+    key Payload
+    pattern /^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S{2}[^\/]+.*$/
+    tag infra.haproxy.tcp
+  </rule>
+  <rule>
+    key Payload
+    pattern /^.*haproxy.*\]: [^ ]+:[^ ]+ [^ ]+ [^ ]+ [^ ]+\/[^ ]+ [^\/]+\/[^\/]+\/[^ ]+ [^ ]+ [^ ]+ \S \S \S{4}[^\/]+.*$/
+    tag infra.haproxy.http
+  </rule>
+</match>

ansible/roles/common/templates/conf/filter/02-parser.conf.j2

@@ -25,3 +25,23 @@
     timestamp ${time}
   </record>
 </filter>
+
+# Parse HAProxy TCP logs
+<filter infra.haproxy.tcp>
+    @type parser
+    format /^.*haproxy.*\]: (?<client_ip>[^ ]+):(?<client_port>[^ ]+) \[(?<accept_date>[^ ]+)\] (?<frontend_name>[^ ]+) (?<backend_name>[^ ]+)\/(?<server_name>[^ ]+) ((?<tw>[^\/]+)\/(?<tc>[^\/]+)\/(?<tt>[^ ]+)) (?<bytes_read>[^ ]+) \S{2} (?<actconn>[^\/]+)\/(?<feconn>[^\/]+)\/(?<beconn>[^\/]+)\/(?<srv_conn>[^\/]+)\/(?<retries>[^ ]+) (?<srv_queue>[^\/]+)\/(?<backend_queue>[^ ]+)$/
+    time_format %d/%b/%Y:%k:%M:%S.%N
+    time_key accept_date
+    key_name Payload
+    reserve_data true
+</filter>
+
+# Parse HAProxy HTTP logs
+<filter infra.haproxy.http>
+    @type parser
+    format /^.*haproxy.*\]: (?<client_ip>[^ ]+):(?<client_port>[^ ]+) \[(?<accept_date>[^ ]+)\] (?<frontend_name>[^ ]+) (?<backend_name>[^ ]+)\/(?<server_name>[^ ]+) (((?<tq>[^\/]+)\/(?<tw>[^\/]+)\/(?<tc>[^\/]+)\/(?<tr>[^\/]+)\/(?<tt>[^ ]+) (?<status_code>[^ ]+))) (?<bytes_read>[^ ]+) (\S \S \S{4}|\S{2}) (?<actconn>[^\/]+)\/(?<feconn>[^\/]+)\/(?<beconn>[^\/]+)\/(?<srv_conn>[^\/]+)\/(?<retries>[^ ]+) (?<srv_queue>[^\/]+)\/(?<backend_queue>[^ ]+) "(?<http_req>[^ ]+) (?<http_path>[^ ]+) (?<http_version>[^ ]+)"$/
+    time_format %d/%b/%Y:%k:%M:%S.%N
+    time_key accept_date
+    key_name Payload
+    reserve_data true
+</filter>

ansible/roles/common/templates/conf/output/00-local.conf.j2

@@ -1,5 +1,5 @@
 {% for item in syslog_facilities | selectattr('enabled') %}
-<match syslog.{{ item.facility }}.**>
+<match {{ item.match | default('syslog.' ~ item.facility ~ '.**') }}>
   @type copy
   <store>
     @type file