Add support for usage metrics + improve metrics performance + improve…

… iptables mode performance (#365)

.github/workflows/build-binary-package.yml

@@ -27,7 +27,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: 1.22.5
+        go-version: '1.22'
 
     - name: Build all platforms
       run: |

.github/workflows/lint.yml

@@ -25,7 +25,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: 1.22.5
+        go-version: '1.22'
 
     - name: Initialize CodeQL
       uses: github/codeql-action/init@v3

.github/workflows/tests.yml

@@ -24,7 +24,12 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: 1.22.5
+        go-version: '1.22'
+
+    - name: mod tidy
+      run: |
+        go mod tidy
+        git diff
 
     - name: Build
       run: |

.github/workflows/tests_deb.yml

@@ -24,7 +24,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v5
       with:
-        go-version: 1.22.5
+        go-version: '1.22'
 
     - name: Cache virtualenvs
       id: cache-pipenv

cmd/root.go

@@ -9,18 +9,21 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"slices"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 
 	csbouncer "github.com/crowdsecurity/go-cs-bouncer"
 	"github.com/crowdsecurity/go-cs-lib/csdaemon"
 	"github.com/crowdsecurity/go-cs-lib/csstring"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -30,7 +33,11 @@ import (
 	"github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics"
 )
 
-const name = "crowdsec-firewall-bouncer"
+const bouncerType = "crowdsec-firewall-bouncer"
+
+type metricsHandler struct {
+	backend *backend.BackendCTX
+}
 
 func backendCleanup(backend *backend.BackendCTX) {
 	log.Info("Shutting down backend")
@@ -136,6 +143,134 @@ func addDecisions(backend *backend.BackendCTX, decisions []*models.Decision, con
 	}
 }
 
+func getLabelValue(labels []*io_prometheus_client.LabelPair, key string) string {
+
+	for _, label := range labels {
+		if label.GetName() == key {
+			return label.GetValue()
+		}
+	}
+
+	return ""
+}
+
+// metricsUpdater receives a metrics struct with basic data and populates it with the current metrics.
+func (m metricsHandler) metricsUpdater(met *models.RemediationComponentsMetrics, updateInterval time.Duration) {
+	log.Debugf("Updating metrics")
+
+	m.backend.CollectMetrics()
+
+	//Most of the common fields are set automatically by the metrics provider
+	//We only need to care about the metrics themselves
+
+	promMetrics, err := prometheus.DefaultGatherer.Gather()
+
+	if err != nil {
+		log.Errorf("unable to gather prometheus metrics: %s", err)
+		return
+	}
+
+	met.Metrics = append(met.Metrics, &models.DetailedMetrics{
+		Meta: &models.MetricsMeta{
+			UtcNowTimestamp:   ptr.Of(time.Now().Unix()),
+			WindowSizeSeconds: ptr.Of(int64(updateInterval.Seconds())),
+		},
+		Items: make([]*models.MetricsDetailItem, 0),
+	})
+
+	for _, metricFamily := range promMetrics {
+		for _, metric := range metricFamily.GetMetric() {
+			switch metricFamily.GetName() {
+			case metrics.ActiveBannedIPsMetricName:
+				//We send the absolute value, as it makes no sense to try to sum them crowdsec side
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				log.Debugf("Sending active decisions for %s %s | current value: %f", origin, ipType, value)
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("active_decisions"),
+					Value: ptr.Of(value),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("ip"),
+				})
+			case metrics.DroppedBytesMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				key := origin + ipType
+				log.Debugf("Sending dropped bytes for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedBytesValue[key], value, metrics.LastDroppedBytesValue[key])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("dropped"),
+					Value: ptr.Of(value - metrics.LastDroppedBytesValue[key]),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("byte"),
+				})
+				metrics.LastDroppedBytesValue[key] = value
+			case metrics.DroppedPacketsMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				key := origin + ipType
+				log.Debugf("Sending dropped packets for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedPacketsValue[key], value, metrics.LastDroppedPacketsValue[key])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("dropped"),
+					Value: ptr.Of(value - metrics.LastDroppedPacketsValue[key]),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("packet"),
+				})
+				metrics.LastDroppedPacketsValue[key] = value
+			case metrics.ProcessedBytesMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				ipType := getLabelValue(labels, "ip_type")
+				log.Debugf("Sending processed bytes for %s %f | current value: %f | previous value: %f\n", ipType, value-metrics.LastProcessedBytesValue[ipType], value, metrics.LastProcessedBytesValue[ipType])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("processed"),
+					Value: ptr.Of(value - metrics.LastProcessedBytesValue[ipType]),
+					Labels: map[string]string{
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("byte"),
+				})
+				metrics.LastProcessedBytesValue[ipType] = value
+			case metrics.ProcessedPacketsMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				ipType := getLabelValue(labels, "ip_type")
+				log.Debugf("Sending processed packets for %s %f | current value: %f | previous value: %f\n", ipType, value-metrics.LastProcessedPacketsValue[ipType], value, metrics.LastProcessedPacketsValue[ipType])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("processed"),
+					Value: ptr.Of(value - metrics.LastProcessedPacketsValue[ipType]),
+					Labels: map[string]string{
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("packet"),
+				})
+				metrics.LastProcessedPacketsValue[ipType] = value
+			}
+		}
+	}
+}
+
+func (m metricsHandler) computeMetricsHandler(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		m.backend.CollectMetrics()
+		next.ServeHTTP(w, r)
+	})
+}
+
 func Execute() error {
 	configPath := flag.String("c", "", "path to crowdsec-firewall-bouncer.yaml")
 	verbose := flag.Bool("v", false, "set verbose mode")
@@ -176,7 +311,7 @@ func Execute() error {
 		log.SetLevel(log.DebugLevel)
 	}
 
-	log.Infof("Starting crowdsec-firewall-bouncer %s", version.String())
+	log.Infof("Starting %s %s", bouncerType, version.String())
 
 	backend, err := backend.NewBackend(config)
 	if err != nil {
@@ -196,7 +331,7 @@ func Execute() error {
 		return err
 	}
 
-	bouncer.UserAgent = fmt.Sprintf("%s/%s", name, version.String())
+	bouncer.UserAgent = fmt.Sprintf("%s/%s", bouncerType, version.String())
 	if err := bouncer.Init(); err != nil {
 		return fmt.Errorf("unable to configure bouncer: %w", err)
 	}
@@ -217,21 +352,27 @@ func Execute() error {
 		return errors.New("bouncer stream halted")
 	})
 
-	if config.PrometheusConfig.Enabled {
-		if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode {
-			go backend.CollectMetrics()
+	mHandler := metricsHandler{
+		backend: backend,
+	}
 
-			if config.Mode == cfg.IpsetMode {
-				prometheus.MustRegister(metrics.TotalActiveBannedIPs)
-			} else {
-				prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs)
-			}
-		}
+	metricsProvider, err := csbouncer.NewMetricsProvider(bouncer.APIClient, bouncerType, mHandler.metricsUpdater, log.StandardLogger())
+	if err != nil {
+		return fmt.Errorf("unable to create metrics provider: %w", err)
+	}
 
-		prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError)
+	g.Go(func() error {
+		return metricsProvider.Run(ctx)
+	})
+
+	if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode {
+		prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs, metrics.TotalProcessedBytes, metrics.TotalProcessedPackets)
+	}
 
+	prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError)
+	if config.PrometheusConfig.Enabled {
 		go func() {
-			http.Handle("/metrics", promhttp.Handler())
+			http.Handle("/metrics", mHandler.computeMetricsHandler(promhttp.Handler()))
 
 			listenOn := net.JoinHostPort(
 				config.PrometheusConfig.ListenAddress,

go.mod

@@ -1,61 +1,63 @@
 module github.com/crowdsecurity/cs-firewall-bouncer
 
-go 1.21
+go 1.22
 
 require (
-	github.com/crowdsecurity/crowdsec v1.6.1
-	github.com/crowdsecurity/go-cs-bouncer v0.0.13
-	github.com/crowdsecurity/go-cs-lib v0.0.10
-	github.com/google/nftables v0.1.1-0.20230710063801-8a10f689006b
-	github.com/prometheus/client_golang v1.17.0
+	github.com/crowdsecurity/crowdsec v1.6.3-rc3
+	github.com/crowdsecurity/go-cs-bouncer v0.0.14-0.20240819095913-4521d8ddc0c6
+	github.com/crowdsecurity/go-cs-lib v0.0.13
+	github.com/google/nftables v0.2.0
+	github.com/prometheus/client_golang v1.20.0
+	github.com/prometheus/client_model v0.6.1
 	github.com/sirupsen/logrus v1.9.3
-	github.com/stretchr/testify v1.8.4
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/sync v0.6.0
-	golang.org/x/sys v0.19.0
+	github.com/stretchr/testify v1.9.0
+	golang.org/x/sync v0.8.0
+	golang.org/x/sys v0.24.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
 )
 
 require (
-	github.com/antonmedv/expr v1.15.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/blackfireio/osinfo v1.0.5 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
-	github.com/go-openapi/analysis v0.21.4 // indirect
-	github.com/go-openapi/errors v0.20.4 // indirect
-	github.com/go-openapi/jsonpointer v0.20.0 // indirect
-	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/loads v0.21.2 // indirect
-	github.com/go-openapi/spec v0.20.9 // indirect
-	github.com/go-openapi/strfmt v0.21.7 // indirect
-	github.com/go-openapi/swag v0.22.4 // indirect
-	github.com/go-openapi/validate v0.22.1 // indirect
-	github.com/goccy/go-yaml v1.11.2 // indirect
+	github.com/expr-lang/expr v1.16.9 // indirect
+	github.com/fatih/color v1.17.0 // indirect
+	github.com/go-openapi/analysis v0.23.0 // indirect
+	github.com/go-openapi/errors v0.22.0 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
+	github.com/go-openapi/jsonreference v0.21.0 // indirect
+	github.com/go-openapi/loads v0.22.0 // indirect
+	github.com/go-openapi/spec v0.21.0 // indirect
+	github.com/go-openapi/strfmt v0.23.0 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-openapi/validate v0.24.0 // indirect
+	github.com/goccy/go-yaml v1.12.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
+	github.com/google/uuid v1.6.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/josharian/native v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
-	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
-	github.com/mdlayher/netlink v1.7.1 // indirect
-	github.com/mdlayher/socket v0.4.0 // indirect
+	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
-	github.com/prometheus/common v0.45.0 // indirect
-	github.com/prometheus/procfs v0.12.0 // indirect
-	go.mongodb.org/mongo-driver v1.12.1 // indirect
-	golang.org/x/net v0.24.0 // indirect
-	golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	github.com/prometheus/common v0.55.0 // indirect
+	github.com/prometheus/procfs v0.15.1 // indirect
+	go.mongodb.org/mongo-driver v1.16.1 // indirect
+	golang.org/x/net v0.28.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
+	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )