merge container-exporter into metrics-pusher

metrics-pusher/Dockerfile

@@ -2,8 +2,9 @@ FROM debian:bookworm-slim
 
 WORKDIR /app
 
-RUN apt-get update && apt-get upgrade --yes python3-requests
+RUN apt-get update && apt-get install --yes python3-requests python3-docker python3-prometheus-client
 
 COPY ./metrics-pusher.py .
+COPY ./monitor.py .
 
 CMD [ "python3", "/app/metrics-pusher.py" ]

metrics-pusher/README.md

@@ -5,17 +5,20 @@
 Run using docker:
 
 ```sh
-docker run -v /var/run/docker.sock:/var/run/docker.sock \
+docker run --privileged \
+    -v /var/run/docker.sock:/var/run/docker.sock \
+    -v /proc:/host/proc:ro \
     -e INSTANCE_NAME="foo" \
     -e PUSHGATEWAY_URL="https://<domain>/path/to/pushgateway" \
     -e SCRAPE_INTERVAL=60 \
     -e AUTH_USER="<user>" \
     -e AUTH_PASSWORD="<password>" \
     -e ENDPOINTS="http://<domain1>/metrics,http://<domain2>/metrics" \
+    -e HOST_PROC_PATH="/host/proc" \
     scalableminds/metrics-pusher
 ```
 
-This will scrape all specified endpoints.
+This will scrape all specified endpoints and containers using the internal [Container Exporter](#container-exporter).
 
 ## Configuration
 
@@ -29,4 +32,22 @@ Environment Variables:
 | `AUTH_USER` | User for Basic Auth |
 | `AUTH_PASSWORD` | Password for Basic Auth |
 | `ENDPOINTS` | Comma separated list of URLs. Each endpoint will be scraped once per interval. Allows at most one URL per hostname (e.g. `http://node_exporter:9100/metrics`) |
+| `HOST_PROC_PATH` | Path to the mounted `/proc` directory |
+| `DOCKER_HOST` | Path to docker socket. Defaults to `unix:///var/run/docker.sock` |
 
+## Container Exporter
+
+In addition to scraping multiple endpoints, this script also scrapes the container performance metrics.
+Therefore, following metrics are generated:
+
+- `system_cpu_total` All system jiffies spend, including idle.
+- `container_cpu_user` Number of jiffies a container spend in user mode. 
+- `container_cpu_kernel` Number of jiffies a container spend in kernel mode. 
+- `container_memory_used` Number of Memory pages allocated to this container. 
+- `container_number_processes` Number of processes running inside a container.
+- `container_number_threads` Number of threads created by the processes.
+- `container_disk_read` Number of bytes read from disk.
+- `container_disk_write` Number of bytes written to disk.
+
+All metrics will be aggregated over all processes running inside a container.
+In case a container is restarted these metrics reset to 0.

metrics-pusher/metrics-pusher.py

@@ -6,6 +6,7 @@
 import os
 import traceback
 import urllib.parse
+import prometheus_client
 
 
 def push_metrics(
@@ -53,13 +54,43 @@ def push_metrics(
             traceback.print_exception(e)
 
 
+def push_container_metrics(
+    name, pushgateway_url, proc_path, interval, username, password
+):
+    print("(container) starting exporter")
+    import monitor
+
+    def auth_handler(url, method, timeout, headers, data):
+        from prometheus_client.exposition import basic_auth_handler
+
+        return basic_auth_handler(
+            url, method, timeout, headers, data, username, password
+        )
+
+    while True:
+        time_until_next_fetch = interval - (time.time() % interval)
+        time.sleep(time_until_next_fetch)
+
+        try:
+            monitor.scrape(proc_path)
+            prometheus_client.push_to_gateway(
+                pushgateway_url,
+                job=f"{name}.container",
+                handler=auth_handler,
+                registry=monitor.registry,
+            )
+        except Exception as e:
+            traceback.print_exception(e)
+
+
 if __name__ == "__main__":
     name = os.environ.get("INSTANCE_NAME")
     pushgateway_url = os.environ.get("PUSHGATEWAY_URL")
     scrape_interval = int(os.getenv("SCRAPE_INTERVAL", "60"))
     auth_user = os.environ.get("AUTH_USER")
     auth_pass = os.environ.get("AUTH_PASSWORD")
     urls = os.environ.get("ENDPOINTS")
+    proc_path = os.environ.get("HOST_PROC_PATH", "/host/proc")
 
     if name is None or name == "":
         print("No INSTANCE_NAME provided")
@@ -100,6 +131,15 @@ def push_metrics(
         p.start()
         processes.append(p)
 
+    push_container_metrics(
+        name,
+        pushgateway_url,
+        proc_path,
+        scrape_interval,
+        auth_user,
+        auth_pass,
+    )
+
     for p in processes:
         p.join()
         print("warning:", p.name, "closed")