From 3f592b52075a80734b4fc291d5a08043d433c8fe Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 17 Sep 2024 13:51:58 +0200 Subject: [PATCH] Add support for usage metrics + improve metrics performance + improve iptables mode performance (#365) --- .github/workflows/build-binary-package.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/tests.yml | 7 +- .github/workflows/tests_deb.yml | 2 +- cmd/root.go | 171 +++++++++- go.mod | 72 ++--- go.sum | 333 ++++++-------------- pkg/ipsetcmd/ipset.go | 273 ++++++++++++++++ pkg/iptables/iptables.go | 169 +++++----- pkg/iptables/iptables_context.go | 344 ++++++++++++++------- pkg/iptables/metrics.go | 302 ++++++++++++++---- pkg/metrics/metrics.go | 40 ++- pkg/nftables/metrics.go | 202 ++++++------ pkg/nftables/nftables.go | 106 +++++-- pkg/nftables/nftables_context.go | 127 ++++---- pkg/pf/metrics.go | 46 ++- test/backends/iptables/test_iptables.py | 56 ++-- test/backends/nftables/test_nftables.py | 25 +- test/bouncer/test_iptables_deny_action.py | 10 +- 19 files changed, 1472 insertions(+), 817 deletions(-) create mode 100644 pkg/ipsetcmd/ipset.go diff --git a/.github/workflows/build-binary-package.yml b/.github/workflows/build-binary-package.yml index 32d62231..87add9d8 100644 --- a/.github/workflows/build-binary-package.yml +++ b/.github/workflows/build-binary-package.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.22.5 + go-version: '1.22' - name: Build all platforms run: | diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 5b9d3ca3..194b4502 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -25,7 +25,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.22.5 + go-version: '1.22' - name: Initialize CodeQL uses: github/codeql-action/init@v3 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 3ce73bd4..7ec5130d 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,7 +24,12 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.22.5 + go-version: '1.22' + + - name: mod tidy + run: | + go mod tidy + git diff - name: Build run: | diff --git a/.github/workflows/tests_deb.yml b/.github/workflows/tests_deb.yml index 89cf9ff2..902ec0c4 100644 --- a/.github/workflows/tests_deb.yml +++ b/.github/workflows/tests_deb.yml @@ -24,7 +24,7 @@ jobs: - name: Set up Go uses: actions/setup-go@v5 with: - go-version: 1.22.5 + go-version: '1.22' - name: Cache virtualenvs id: cache-pipenv diff --git a/cmd/root.go b/cmd/root.go index 6e891324..f77af44c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -9,18 +9,21 @@ import ( "net/http" "os" "os/signal" + "slices" "strings" "syscall" + "time" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + io_prometheus_client "github.com/prometheus/client_model/go" log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" "golang.org/x/sync/errgroup" csbouncer "github.com/crowdsecurity/go-cs-bouncer" "github.com/crowdsecurity/go-cs-lib/csdaemon" "github.com/crowdsecurity/go-cs-lib/csstring" + "github.com/crowdsecurity/go-cs-lib/ptr" "github.com/crowdsecurity/go-cs-lib/version" "github.com/crowdsecurity/crowdsec/pkg/models" @@ -30,7 +33,11 @@ import ( "github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics" ) -const name = "crowdsec-firewall-bouncer" +const bouncerType = "crowdsec-firewall-bouncer" + +type metricsHandler struct { + backend *backend.BackendCTX +} func backendCleanup(backend *backend.BackendCTX) { log.Info("Shutting down backend") @@ -136,6 +143,134 @@ func addDecisions(backend *backend.BackendCTX, decisions []*models.Decision, con } } +func getLabelValue(labels []*io_prometheus_client.LabelPair, key string) string { + + for _, label := range labels { + if label.GetName() == key { + return label.GetValue() + } + } + + return "" +} + +// metricsUpdater receives a metrics struct with basic data and populates it with the current metrics. +func (m metricsHandler) metricsUpdater(met *models.RemediationComponentsMetrics, updateInterval time.Duration) { + log.Debugf("Updating metrics") + + m.backend.CollectMetrics() + + //Most of the common fields are set automatically by the metrics provider + //We only need to care about the metrics themselves + + promMetrics, err := prometheus.DefaultGatherer.Gather() + + if err != nil { + log.Errorf("unable to gather prometheus metrics: %s", err) + return + } + + met.Metrics = append(met.Metrics, &models.DetailedMetrics{ + Meta: &models.MetricsMeta{ + UtcNowTimestamp: ptr.Of(time.Now().Unix()), + WindowSizeSeconds: ptr.Of(int64(updateInterval.Seconds())), + }, + Items: make([]*models.MetricsDetailItem, 0), + }) + + for _, metricFamily := range promMetrics { + for _, metric := range metricFamily.GetMetric() { + switch metricFamily.GetName() { + case metrics.ActiveBannedIPsMetricName: + //We send the absolute value, as it makes no sense to try to sum them crowdsec side + labels := metric.GetLabel() + value := metric.GetGauge().GetValue() + origin := getLabelValue(labels, "origin") + ipType := getLabelValue(labels, "ip_type") + log.Debugf("Sending active decisions for %s %s | current value: %f", origin, ipType, value) + met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{ + Name: ptr.Of("active_decisions"), + Value: ptr.Of(value), + Labels: map[string]string{ + "origin": origin, + "ip_type": ipType, + }, + Unit: ptr.Of("ip"), + }) + case metrics.DroppedBytesMetricName: + labels := metric.GetLabel() + value := metric.GetGauge().GetValue() + origin := getLabelValue(labels, "origin") + ipType := getLabelValue(labels, "ip_type") + key := origin + ipType + log.Debugf("Sending dropped bytes for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedBytesValue[key], value, metrics.LastDroppedBytesValue[key]) + met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{ + Name: ptr.Of("dropped"), + Value: ptr.Of(value - metrics.LastDroppedBytesValue[key]), + Labels: map[string]string{ + "origin": origin, + "ip_type": ipType, + }, + Unit: ptr.Of("byte"), + }) + metrics.LastDroppedBytesValue[key] = value + case metrics.DroppedPacketsMetricName: + labels := metric.GetLabel() + value := metric.GetGauge().GetValue() + origin := getLabelValue(labels, "origin") + ipType := getLabelValue(labels, "ip_type") + key := origin + ipType + log.Debugf("Sending dropped packets for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedPacketsValue[key], value, metrics.LastDroppedPacketsValue[key]) + met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{ + Name: ptr.Of("dropped"), + Value: ptr.Of(value - metrics.LastDroppedPacketsValue[key]), + Labels: map[string]string{ + "origin": origin, + "ip_type": ipType, + }, + Unit: ptr.Of("packet"), + }) + metrics.LastDroppedPacketsValue[key] = value + case metrics.ProcessedBytesMetricName: + labels := metric.GetLabel() + value := metric.GetGauge().GetValue() + ipType := getLabelValue(labels, "ip_type") + log.Debugf("Sending processed bytes for %s %f | current value: %f | previous value: %f\n", ipType, value-metrics.LastProcessedBytesValue[ipType], value, metrics.LastProcessedBytesValue[ipType]) + met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{ + Name: ptr.Of("processed"), + Value: ptr.Of(value - metrics.LastProcessedBytesValue[ipType]), + Labels: map[string]string{ + "ip_type": ipType, + }, + Unit: ptr.Of("byte"), + }) + metrics.LastProcessedBytesValue[ipType] = value + case metrics.ProcessedPacketsMetricName: + labels := metric.GetLabel() + value := metric.GetGauge().GetValue() + ipType := getLabelValue(labels, "ip_type") + log.Debugf("Sending processed packets for %s %f | current value: %f | previous value: %f\n", ipType, value-metrics.LastProcessedPacketsValue[ipType], value, metrics.LastProcessedPacketsValue[ipType]) + met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{ + Name: ptr.Of("processed"), + Value: ptr.Of(value - metrics.LastProcessedPacketsValue[ipType]), + Labels: map[string]string{ + "ip_type": ipType, + }, + Unit: ptr.Of("packet"), + }) + metrics.LastProcessedPacketsValue[ipType] = value + } + } + } +} + +func (m metricsHandler) computeMetricsHandler(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m.backend.CollectMetrics() + next.ServeHTTP(w, r) + }) +} + func Execute() error { configPath := flag.String("c", "", "path to crowdsec-firewall-bouncer.yaml") verbose := flag.Bool("v", false, "set verbose mode") @@ -176,7 +311,7 @@ func Execute() error { log.SetLevel(log.DebugLevel) } - log.Infof("Starting crowdsec-firewall-bouncer %s", version.String()) + log.Infof("Starting %s %s", bouncerType, version.String()) backend, err := backend.NewBackend(config) if err != nil { @@ -196,7 +331,7 @@ func Execute() error { return err } - bouncer.UserAgent = fmt.Sprintf("%s/%s", name, version.String()) + bouncer.UserAgent = fmt.Sprintf("%s/%s", bouncerType, version.String()) if err := bouncer.Init(); err != nil { return fmt.Errorf("unable to configure bouncer: %w", err) } @@ -217,21 +352,27 @@ func Execute() error { return errors.New("bouncer stream halted") }) - if config.PrometheusConfig.Enabled { - if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode { - go backend.CollectMetrics() + mHandler := metricsHandler{ + backend: backend, + } - if config.Mode == cfg.IpsetMode { - prometheus.MustRegister(metrics.TotalActiveBannedIPs) - } else { - prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs) - } - } + metricsProvider, err := csbouncer.NewMetricsProvider(bouncer.APIClient, bouncerType, mHandler.metricsUpdater, log.StandardLogger()) + if err != nil { + return fmt.Errorf("unable to create metrics provider: %w", err) + } - prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError) + g.Go(func() error { + return metricsProvider.Run(ctx) + }) + + if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode { + prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs, metrics.TotalProcessedBytes, metrics.TotalProcessedPackets) + } + prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError) + if config.PrometheusConfig.Enabled { go func() { - http.Handle("/metrics", promhttp.Handler()) + http.Handle("/metrics", mHandler.computeMetricsHandler(promhttp.Handler())) listenOn := net.JoinHostPort( config.PrometheusConfig.ListenAddress, diff --git a/go.mod b/go.mod index 55b5176c..d807ebb7 100644 --- a/go.mod +++ b/go.mod @@ -1,61 +1,63 @@ module github.com/crowdsecurity/cs-firewall-bouncer -go 1.21 +go 1.22 require ( - github.com/crowdsecurity/crowdsec v1.6.1 - github.com/crowdsecurity/go-cs-bouncer v0.0.13 - github.com/crowdsecurity/go-cs-lib v0.0.10 - github.com/google/nftables v0.1.1-0.20230710063801-8a10f689006b - github.com/prometheus/client_golang v1.17.0 + github.com/crowdsecurity/crowdsec v1.6.3-rc3 + github.com/crowdsecurity/go-cs-bouncer v0.0.14-0.20240819095913-4521d8ddc0c6 + github.com/crowdsecurity/go-cs-lib v0.0.13 + github.com/google/nftables v0.2.0 + github.com/prometheus/client_golang v1.20.0 + github.com/prometheus/client_model v0.6.1 github.com/sirupsen/logrus v1.9.3 - github.com/stretchr/testify v1.8.4 - golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 - golang.org/x/sync v0.6.0 - golang.org/x/sys v0.19.0 + github.com/stretchr/testify v1.9.0 + golang.org/x/sync v0.8.0 + golang.org/x/sys v0.24.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v2 v2.4.0 ) require ( - github.com/antonmedv/expr v1.15.3 // indirect github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/blackfireio/osinfo v1.0.5 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/fatih/color v1.15.0 // indirect - github.com/go-openapi/analysis v0.21.4 // indirect - github.com/go-openapi/errors v0.20.4 // indirect - github.com/go-openapi/jsonpointer v0.20.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/loads v0.21.2 // indirect - github.com/go-openapi/spec v0.20.9 // indirect - github.com/go-openapi/strfmt v0.21.7 // indirect - github.com/go-openapi/swag v0.22.4 // indirect - github.com/go-openapi/validate v0.22.1 // indirect - github.com/goccy/go-yaml v1.11.2 // indirect + github.com/expr-lang/expr v1.16.9 // indirect + github.com/fatih/color v1.17.0 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/goccy/go-yaml v1.12.0 // indirect github.com/golang-jwt/jwt/v4 v4.5.0 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/josharian/native v1.0.0 // indirect + github.com/josharian/native v1.1.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect - github.com/mdlayher/netlink v1.7.1 // indirect - github.com/mdlayher/socket v0.4.0 // indirect + github.com/mdlayher/netlink v1.7.2 // indirect + github.com/mdlayher/socket v0.5.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.45.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - go.mongodb.org/mongo-driver v1.12.1 // indirect - golang.org/x/net v0.24.0 // indirect - golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect - google.golang.org/protobuf v1.33.0 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect + go.mongodb.org/mongo-driver v1.16.1 // indirect + golang.org/x/net v0.28.0 // indirect + golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index 5c36430b..2ec62e9f 100644 --- a/go.sum +++ b/go.sum @@ -1,305 +1,148 @@ -github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= -github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= -github.com/antonmedv/expr v1.15.3 h1:q3hOJZNvLvhqE8OHBs1cFRdbXFNKuA+bHmRaI+AmRmI= -github.com/antonmedv/expr v1.15.3/go.mod h1:0E/6TxnOlRNp81GMzX9QfDPAmHo2Phg00y4JUv1ihsE= -github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= -github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= -github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/blackfireio/osinfo v1.0.5 h1:6hlaWzfcpb87gRmznVf7wSdhysGqLRz9V/xuSdCEXrA= +github.com/blackfireio/osinfo v1.0.5/go.mod h1:Pd987poVNmd5Wsx6PRPw4+w7kLlf9iJxoRKPtPAjOrA= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/crowdsecurity/crowdsec v1.6.1 h1:L0b/gV1eSOcuz5fhjGpzQJP+tmoNpUqAZjC7KAfBxTc= -github.com/crowdsecurity/crowdsec v1.6.1/go.mod h1:2zt1/+yOzTZU3En9cAtfMMhAQASmFRup2604vOg/usQ= -github.com/crowdsecurity/go-cs-bouncer v0.0.13 h1:BndYyRr7NtATbrbU9ju43kfIESfkdsq2wmIptxkyzB0= -github.com/crowdsecurity/go-cs-bouncer v0.0.13/go.mod h1:CQrs7Al1ORcdDtY/sMv/ps1LjxFDCiM2Kvlamn3uJx0= -github.com/crowdsecurity/go-cs-lib v0.0.10 h1:Twt/y/rYCUspGY1zxDnGurL2svRSREAz+2+puLepd9c= -github.com/crowdsecurity/go-cs-lib v0.0.10/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k= +github.com/crowdsecurity/crowdsec v1.6.3-rc3 h1:Hlb4CIHI7oB6DptjhTjdLZXBiWgm5ufCFdekIEnKh+A= +github.com/crowdsecurity/crowdsec v1.6.3-rc3/go.mod h1:o1M2A0sNqch8D5RwiXpmiLj5JPznZGwTFalXskXhkPY= +github.com/crowdsecurity/go-cs-bouncer v0.0.14-0.20240819095913-4521d8ddc0c6 h1:fLJyC+uzlUEufSg9vBPHNZ74X9y4pWvE/877DLjDK2o= +github.com/crowdsecurity/go-cs-bouncer v0.0.14-0.20240819095913-4521d8ddc0c6/go.mod h1:qnxo6SrLoU3BwU+v9vKDosVHQnhoNDGss37wseoGhRk= +github.com/crowdsecurity/go-cs-lib v0.0.13 h1:asmtjIEPOibUK8eaYQCIR7XIBU/EX5vyAp1EbKFQJtY= +github.com/crowdsecurity/go-cs-lib v0.0.13/go.mod h1:ePyQyJBxp1W/1bq4YpVAilnLSz7HkzmtI7TRhX187EU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= -github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= -github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU= -github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA= -github.com/go-openapi/analysis v0.21.2/go.mod h1:HZwRk4RRisyG8vx2Oe6aqeSQcoxRp47Xkp3+K6q+LdY= -github.com/go-openapi/analysis v0.21.4 h1:ZDFLvSNxpDaomuCueM0BlSXxpANBlFYiBvr+GXrvIHc= -github.com/go-openapi/analysis v0.21.4/go.mod h1:4zQ35W4neeZTqh3ol0rv/O8JBbka9QyAgQRPp9y3pfo= -github.com/go-openapi/errors v0.19.8/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.19.9/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.2/go.mod h1:cM//ZKUKyO06HSwqAelJ5NsEMMcpa6VpXe8DOa1Mi1M= -github.com/go-openapi/errors v0.20.4 h1:unTcVm6PispJsMECE3zWgvG4xTiKda1LIR5rCRWLG6M= -github.com/go-openapi/errors v0.20.4/go.mod h1:Z3FlZ4I8jEGxjUK+bugx3on2mIAk4txuAOhlsB1FSgk= -github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.20.0 h1:ESKJdU9ASRfaPNOPRx12IUyA1vn3R9GiE3KYD14BXdQ= -github.com/go-openapi/jsonpointer v0.20.0/go.mod h1:6PGzBjjIIumbLYysB73Klnms1mwnU4G3YHOECG3CedA= -github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= -github.com/go-openapi/jsonreference v0.20.0/go.mod h1:Ag74Ico3lPc+zR+qjn4XBUmXymS4zJbYVCZmcgkasdo= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/loads v0.21.1/go.mod h1:/DtAMXXneXFjbQMGEtbamCZb+4x7eGwkvZCvBmwUG+g= -github.com/go-openapi/loads v0.21.2 h1:r2a/xFIYeZ4Qd2TnGpWDIQNcP80dIaZgf704za8enro= -github.com/go-openapi/loads v0.21.2/go.mod h1:Jq58Os6SSGz0rzh62ptiu8Z31I+OTHqmULx5e/gJbNw= -github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= -github.com/go-openapi/spec v0.20.6/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/spec v0.20.9 h1:xnlYNQAwKd2VQRRfwTEI0DcK+2cbuvI/0c7jx3gA8/8= -github.com/go-openapi/spec v0.20.9/go.mod h1:2OpW+JddWPrpXSCIX8eOx7lZ5iyuWj3RYR6VaaBKcWA= -github.com/go-openapi/strfmt v0.21.0/go.mod h1:ZRQ409bWMj+SOgXofQAGTIo2Ebu72Gs+WaRADcS5iNg= -github.com/go-openapi/strfmt v0.21.1/go.mod h1:I/XVKeLc5+MM5oPNN7P6urMOpuLXEcNrCX/rPGuWb0k= -github.com/go-openapi/strfmt v0.21.3/go.mod h1:k+RzNO0Da+k3FrrynSNN8F7n/peCmQQqbbXjtDfvmGg= -github.com/go-openapi/strfmt v0.21.7 h1:rspiXgNWgeUzhjo1YU01do6qsahtJNByjLVbPLNHb8k= -github.com/go-openapi/strfmt v0.21.7/go.mod h1:adeGTkxE44sPyLk0JV235VQAO/ZXUr8KAzYjclFs3ew= -github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= -github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.21.1/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/validate v0.22.1 h1:G+c2ub6q47kfX1sOBLwIQwzBVt8qmOAARyo/9Fqs9NU= -github.com/go-openapi/validate v0.22.1/go.mod h1:rjnrwK57VJ7A8xqfpAOEKRH8yQSGUriMu5/zuPSQ1hg= +github.com/expr-lang/expr v1.16.9 h1:WUAzmR0JNI9JCiF0/ewwHB1gmcGw5wW7nWt8gc6PpCI= +github.com/expr-lang/expr v1.16.9/go.mod h1:8/vRC7+7HBzESEqt5kKpYXxrxkr31SaO8r40VO/1IT4= +github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= +github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= +github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0= +github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA= github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY= github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY= github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= -github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= -github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= -github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= -github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= -github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg= -github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= -github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= -github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/flect v0.1.3/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= -github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= -github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= -github.com/gobuffalo/genny v0.1.0/go.mod h1:XidbUqzak3lHdS//TPu2OgiFB+51Ur5f7CSnXZ/JDvo= -github.com/gobuffalo/genny v0.1.1/go.mod h1:5TExbEyY48pfunL4QSXxlDOmdsD44RRq4mVZ0Ex28Xk= -github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= -github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= -github.com/gobuffalo/gogen v0.1.0/go.mod h1:8NTelM5qd8RZ15VjQTFkAW6qOMx5wBbW4dSCS3BY8gg= -github.com/gobuffalo/gogen v0.1.1/go.mod h1:y8iBtmHmGc4qa3urIyo1shvOD8JftTtfcKi+71xfDNE= -github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= -github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= -github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packd v0.1.0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= -github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= -github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/VCm/3ptBN+0= -github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= -github.com/goccy/go-yaml v1.11.2 h1:joq77SxuyIs9zzxEjgyLBugMQ9NEgTWxXfz2wVqwAaQ= -github.com/goccy/go-yaml v1.11.2/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= +github.com/go-playground/validator/v10 v10.17.0 h1:SmVVlfAOtlZncTxRuinDPomC2DkXJ4E5T9gDA0AIH74= +github.com/go-playground/validator/v10 v10.17.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= +github.com/goccy/go-yaml v1.12.0 h1:/1WHjnMsI1dlIBQutrvSMGZRQufVO3asrHfTwfACoPM= +github.com/goccy/go-yaml v1.12.0/go.mod h1:wKnAMd44+9JAAnGQpWVEgBzGt3YuTaQ4uXoHvE4m7WU= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg= github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= -github.com/google/nftables v0.1.1-0.20230710063801-8a10f689006b h1:efO6FAh/47nbHQfPqLThkv3XZCche98okS85TZxD96o= -github.com/google/nftables v0.1.1-0.20230710063801-8a10f689006b/go.mod h1:7OLL+86wZKfBnAJxNxmdcZ0ebbgdp/A28fcagx9oJqA= -github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/google/nftables v0.2.0 h1:PbJwaBmbVLzpeldoeUKGkE2RjstrjPKMl6oLrfEJ6/8= +github.com/google/nftables v0.2.0/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= -github.com/josharian/native v1.0.0 h1:Ts/E8zCSEsG17dUqv7joXJFybuMLjQfWE04tsBODTxk= -github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= -github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= -github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA= -github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= +github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q= -github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4= -github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= -github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/leodido/go-urn v1.3.0 h1:jX8FDLfW4ThVXctBNZ+3cIWnCSnrACDV73r76dy0aQQ= +github.com/leodido/go-urn v1.3.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= -github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= -github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= -github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= -github.com/mdlayher/netlink v1.7.1 h1:FdUaT/e33HjEXagwELR8R3/KL1Fq5x3G5jgHLp/BTmg= -github.com/mdlayher/netlink v1.7.1/go.mod h1:nKO5CSjE/DJjVhk/TNp6vCE1ktVxEA8VEh8drhZzxsQ= -github.com/mdlayher/socket v0.4.0 h1:280wsy40IC9M9q1uPGcLBwXpcTQDtoGwVt+BNoITxIw= -github.com/mdlayher/socket v0.4.0/go.mod h1:xxFqz5GRCUN3UEOm9CZqEJsAbe1C8OwSK46NlmWuVoc= -github.com/mitchellh/mapstructure v1.3.3/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mdlayher/netlink v1.7.2 h1:/UtM3ofJap7Vl4QWCPDGXY8d3GIY2UGSDbK+QWmY8/g= +github.com/mdlayher/netlink v1.7.2/go.mod h1:xraEF7uJbxLhc5fpHL4cPe221LI2bdttWlU+ZGLfQSw= +github.com/mdlayher/socket v0.5.1 h1:VZaqt6RkGkt2OE9l3GcC6nZkqD3xKeQLyfleW/uBcos= +github.com/mdlayher/socket v0.5.1/go.mod h1:TjPLHI1UgwEv5J1B5q0zTZq12A/6H7nKmtTanQE37IQ= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= -github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= -github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= -github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= -github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= -github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= -github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/prometheus/client_golang v1.20.0 h1:jBzTZ7B099Rg24tny+qngoynol8LtVYlA2bqx3vEloI= +github.com/prometheus/client_golang v1.20.0/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G1dc= +github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= +github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= +github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= -github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= -github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= -github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc h1:R83G5ikgLMxrBvLh22JhdfI8K6YXEPHx5P03Uu3DRs4= github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI= -github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= -github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs= -github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= -github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4= -github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM= -github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= -github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM= -github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= -github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -go.mongodb.org/mongo-driver v1.7.3/go.mod h1:NqaYOwnXWr5Pm7AOpO5QFxKJ503nbMse/R79oO62zWg= -go.mongodb.org/mongo-driver v1.7.5/go.mod h1:VXEWRZ6URJIkUq2SCAyapmhH0ZLRBP+FT4xhp5Zvxng= -go.mongodb.org/mongo-driver v1.10.0/go.mod h1:wsihk0Kdgv8Kqu1Anit4sfK+22vSFbUrAVEYRhCXrA8= -go.mongodb.org/mongo-driver v1.12.1 h1:nLkghSU8fQNaK7oUmDhQFsnrtcoNy7Z6LVFKsEecqgE= -go.mongodb.org/mongo-driver v1.12.1/go.mod h1:/rGBTebI3XYboVmgz+Wv3Bcbl3aD0QF9zl6kDDw18rQ= -golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20190422162423-af44ce270edf/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= -golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= -golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= -golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= -golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= -golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= -golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= -golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190419153524-e8e3143a4f4a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190531175056-4c3a928424d2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +go.mongodb.org/mongo-driver v1.16.1 h1:rIVLL3q0IHM39dvE+z2ulZLp9ENZKThVfuvN/IiN4l8= +go.mongodb.org/mongo-driver v1.16.1/go.mod h1:oB6AhJQvFQL4LEHyXi6aJzQJtBiTQHiAd83l0GdFaiw= +golang.org/x/crypto v0.26.0 h1:RrRspgV4mU+YwB4FYnuBoKsUapNIL5cohGAmSH3azsw= +golang.org/x/crypto v0.26.0/go.mod h1:GY7jblb9wI+FOo5y8/S2oY4zWP07AkOJ4+jxCqdqn54= +golang.org/x/net v0.28.0 h1:a9JDOJc5GMUJ0+UDqmLT86WiEy7iWyIhz8gz8E4e5hE= +golang.org/x/net v0.28.0/go.mod h1:yqtgsTWOOnlGLG9GFRrK3++bGOUEkNBoHZc8MEDWPNg= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= -golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/sys v0.24.0 h1:Twjiwq9dn6R1fQcyiK+wQyHWfaz/BJB+YIpzU/Cv3Xg= +golang.org/x/sys v0.24.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.17.0 h1:XtiM5bkSOt+ewxlOE/aE/AKEHibwj/6gvWMl9Rsh0Qc= +golang.org/x/text v0.17.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= -golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= -google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= -google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 h1:LLhsEBxRTBLuKlQxFBYUOU8xyFgXv6cOTp2HASDlsDk= +golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/natefinch/lumberjack.v2 v2.2.1 h1:bBRl1b0OH9s/DuPhuXpNl+VtCaJXFZ5/uEFST95x9zc= gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYskCTPBJVb9jqSc= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 h1:yiW+nvdHb9LVqSHQBXfZCieqV4fzYhNBql77zY0ykqs= gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637/go.mod h1:BHsqpu/nsuzkT5BpiH1EMZPLyqSMM8JbIavyFACoFNk= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/ipsetcmd/ipset.go b/pkg/ipsetcmd/ipset.go new file mode 100644 index 00000000..6a24b637 --- /dev/null +++ b/pkg/ipsetcmd/ipset.go @@ -0,0 +1,273 @@ +package ipsetcmd + +import ( + "errors" + "fmt" + "os/exec" + "strconv" + "strings" + + log "github.com/sirupsen/logrus" +) + +type IPSet struct { + binaryPath string + setName string +} + +type CreateOptions struct { + Timeout string + MaxElem string + Family string + Type string +} + +const ipsetBinary = "ipset" + +func NewIPSet(setName string) (*IPSet, error) { + ipsetBin, err := exec.LookPath(ipsetBinary) + if err != nil { + return nil, errors.New("unable to find ipset") + } + return &IPSet{ + binaryPath: ipsetBin, + setName: setName, + }, nil +} + +//Wraps all the ipset commands + +func (i *IPSet) Create(opts CreateOptions) error { + cmdArgs := []string{"create", i.setName} + + if opts.Type != "" { + cmdArgs = append(cmdArgs, opts.Type) + } + + if opts.Timeout != "" { + cmdArgs = append(cmdArgs, "timeout", opts.Timeout) + } + + if opts.MaxElem != "" { + cmdArgs = append(cmdArgs, "maxelem", opts.MaxElem) + } + + if opts.Family != "" { + cmdArgs = append(cmdArgs, "family", opts.Family) + } + + cmd := exec.Command(i.binaryPath, cmdArgs...) + + log.Debugf("ipset create command: %v", cmd.String()) + + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error creating ipset: %s", out) + } + + return nil +} + +func (i *IPSet) Add(entry string) error { + cmd := exec.Command(i.binaryPath, "add", i.setName, entry) + + log.Debugf("ipset add command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error creating ipset: %s", out) + } + + return nil +} + +func (i *IPSet) DeleteEntry(entry string) error { + cmd := exec.Command(i.binaryPath, "del", i.setName, entry) + + log.Debugf("ipset delete entry command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error creating ipset: %s", out) + } + + return nil +} + +func (i *IPSet) List() ([]string, error) { + cmd := exec.Command(i.binaryPath, "list", i.setName) + + log.Debugf("ipset list command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return nil, fmt.Errorf("error listing ipset: %s", out) + } + + return strings.Split(string(out), "\n"), nil +} + +func (i *IPSet) Flush() error { + cmd := exec.Command(i.binaryPath, "flush", i.setName) + + log.Debugf("ipset flush command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error flushing ipset: %s", out) + } + + return nil +} + +func (i *IPSet) Destroy() error { + cmd := exec.Command(i.binaryPath, "destroy", i.setName) + + log.Debugf("ipset destroy command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error destroying ipset: %s", out) + } + + return nil +} + +func (i *IPSet) Rename(toSetName string) error { + cmd := exec.Command(i.binaryPath, "rename", i.setName, toSetName) + + log.Debugf("ipset rename command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error renaming ipset: %s", out) + } + + i.setName = toSetName + + return nil +} + +func (i *IPSet) Test(entry string) error { + cmd := exec.Command(i.binaryPath, "test", i.setName, entry) + + log.Debugf("ipset test command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error testing ipset: %s", out) + } + + return nil +} + +func (i *IPSet) Save() ([]string, error) { + cmd := exec.Command(i.binaryPath, "save", i.setName) + + log.Debugf("ipset save command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return nil, fmt.Errorf("error saving ipset: %s", out) + } + return strings.Split(string(out), "\n"), nil +} + +func (i *IPSet) Restore(filename string) error { + cmd := exec.Command(i.binaryPath, "restore", "-file", filename) + + log.Debugf("ipset restore command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error restoring ipset: %s", out) + } + + return nil +} + +func (i *IPSet) Swap(toSetName string) error { + cmd := exec.Command(i.binaryPath, "swap", i.setName, toSetName) + + log.Debugf("ipset swap command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return fmt.Errorf("error swapping ipset: %s", out) + } + + i.setName = toSetName + + return nil +} + +func (i *IPSet) Name() string { + return i.setName +} + +func (i *IPSet) Exists() bool { + cmd := exec.Command(i.binaryPath, "list", i.setName) + + err := cmd.Run() + + return err == nil +} + +func (i *IPSet) Len() int { + cmd := exec.Command(i.binaryPath, "list", i.setName) + + log.Debugf("ipset list command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return 0 + } + + for _, line := range strings.Split(string(out), "\n") { + if strings.Contains(strings.ToLower(line), "number of entries:") { + fields := strings.Split(line, ":") + if len(fields) != 2 { + continue + } + count, err := strconv.Atoi(strings.TrimSpace(fields[1])) + if err != nil { + return 0 + } + return count + } + } + + return 0 +} + +//Helpers + +func GetSetsStartingWith(name string) (map[string]*IPSet, error) { + cmd := exec.Command(ipsetBinary, "list", "-name") + + log.Debugf("ipset list command: %v", cmd.String()) + out, err := cmd.CombinedOutput() + + if err != nil { + return nil, fmt.Errorf("error listing ipset: %s", out) + } + + sets := make(map[string]*IPSet, 0) + + for _, line := range strings.Split(string(out), "\n") { + if strings.HasPrefix(line, name) { + fields := strings.Fields(line) + if len(fields) != 1 { + continue + } + set, err := NewIPSet(fields[0]) + if err != nil { + return nil, err + } + sets[fields[0]] = set + } + } + + return sets, nil +} diff --git a/pkg/iptables/iptables.go b/pkg/iptables/iptables.go index 8541333d..febba9d9 100644 --- a/pkg/iptables/iptables.go +++ b/pkg/iptables/iptables.go @@ -7,14 +7,15 @@ import ( "errors" "fmt" "os/exec" + "slices" "strings" log "github.com/sirupsen/logrus" - "golang.org/x/exp/slices" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/cs-firewall-bouncer/pkg/cfg" + "github.com/crowdsecurity/cs-firewall-bouncer/pkg/ipsetcmd" "github.com/crowdsecurity/cs-firewall-bouncer/pkg/types" ) @@ -29,108 +30,104 @@ type iptables struct { } func NewIPTables(config *cfg.BouncerConfig) (types.Backend, error) { + var err error ret := &iptables{} - ipv4Ctx := &ipTablesContext{ - Name: "ipset", - version: "v4", - SetName: config.BlacklistsIpv4, - SetType: config.SetType, - SetSize: config.SetSize, - StartupCmds: [][]string{}, - ShutdownCmds: [][]string{}, - CheckIptableCmds: [][]string{}, - Chains: []string{}, - } - ipv6Ctx := &ipTablesContext{ - Name: "ipset", - version: "v6", - SetName: config.BlacklistsIpv6, - SetType: config.SetType, - SetSize: config.SetSize, - StartupCmds: [][]string{}, - ShutdownCmds: [][]string{}, - CheckIptableCmds: [][]string{}, - Chains: []string{}, + defaultSet, err := ipsetcmd.NewIPSet("") + + if err != nil { + return nil, err } - allowedActions := []string{"DROP", "REJECT", "TARPIT"} + allowedActions := []string{"DROP", "REJECT", "TARPIT", "LOG"} target := strings.ToUpper(config.DenyAction) if target == "" { target = "DROP" } + log.Infof("using '%s' as deny_action", target) + if !slices.Contains(allowedActions, target) { return nil, fmt.Errorf("invalid deny_action '%s', must be one of %s", config.DenyAction, strings.Join(allowedActions, ", ")) } - log.Tracef("using '%s' as deny_action", target) + v4Sets := make(map[string]*ipsetcmd.IPSet) + v6Sets := make(map[string]*ipsetcmd.IPSet) - ipsetBin, err := exec.LookPath("ipset") + ipv4Ctx := &ipTablesContext{ + version: "v4", + SetName: config.BlacklistsIpv4, + SetType: config.SetType, + SetSize: config.SetSize, + Chains: []string{}, + defaultSet: defaultSet, + target: target, + } + ipv6Ctx := &ipTablesContext{ + version: "v6", + SetName: config.BlacklistsIpv6, + SetType: config.SetType, + SetSize: config.SetSize, + Chains: []string{}, + defaultSet: defaultSet, + target: target, + } + + ipv4Ctx.iptablesSaveBin, err = exec.LookPath("iptables-save") if err != nil { - return nil, errors.New("unable to find ipset") + return nil, errors.New("unable to find iptables-save") } - ipv4Ctx.ipsetBin = ipsetBin if config.Mode == cfg.IpsetMode { ipv4Ctx.ipsetContentOnly = true + set, err := ipsetcmd.NewIPSet(config.BlacklistsIpv4) + if err != nil { + return nil, err + } + v4Sets["ipset"] = set } else { ipv4Ctx.iptablesBin, err = exec.LookPath("iptables") if err != nil { return nil, errors.New("unable to find iptables") } + + //Try to "adopt" any leftover sets from a previous run if we crashed + //They will get flushed/deleted just after + v4Sets, _ = ipsetcmd.GetSetsStartingWith(config.BlacklistsIpv4) + v6Sets, _ = ipsetcmd.GetSetsStartingWith(config.BlacklistsIpv6) + ipv4Ctx.Chains = config.IptablesChains - for _, v := range config.IptablesChains { - ipv4Ctx.StartupCmds = append(ipv4Ctx.StartupCmds, - []string{"-I", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", target}) - ipv4Ctx.ShutdownCmds = append(ipv4Ctx.ShutdownCmds, - []string{"-D", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", target}) - ipv4Ctx.CheckIptableCmds = append(ipv4Ctx.CheckIptableCmds, - []string{"-C", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", target}) - if config.DenyLog { - ipv4Ctx.StartupCmds = append(ipv4Ctx.StartupCmds, - []string{"-I", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - ipv4Ctx.ShutdownCmds = append(ipv4Ctx.ShutdownCmds, - []string{"-D", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - ipv4Ctx.CheckIptableCmds = append(ipv4Ctx.CheckIptableCmds, - []string{"-C", v, "-m", "set", "--match-set", ipv4Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - } - } } + ipv4Ctx.ipsets = v4Sets ret.v4 = ipv4Ctx if config.DisableIPV6 { return ret, nil } - ipv6Ctx.ipsetBin = ipsetBin + ipv6Ctx.iptablesSaveBin, err = exec.LookPath("ip6tables-save") + if err != nil { + return nil, errors.New("unable to find ip6tables-save") + } + if config.Mode == cfg.IpsetMode { ipv6Ctx.ipsetContentOnly = true + set, err := ipsetcmd.NewIPSet(config.BlacklistsIpv6) + if err != nil { + return nil, err + } + v6Sets["ipset"] = set } else { ipv6Ctx.iptablesBin, err = exec.LookPath("ip6tables") if err != nil { return nil, errors.New("unable to find ip6tables") } + ipv6Ctx.Chains = config.IptablesChains - for _, v := range config.IptablesChains { - ipv6Ctx.StartupCmds = append(ipv6Ctx.StartupCmds, - []string{"-I", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", target}) - ipv6Ctx.ShutdownCmds = append(ipv6Ctx.ShutdownCmds, - []string{"-D", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", target}) - ipv6Ctx.CheckIptableCmds = append(ipv6Ctx.CheckIptableCmds, - []string{"-C", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", target}) - if config.DenyLog { - ipv6Ctx.StartupCmds = append(ipv6Ctx.StartupCmds, - []string{"-I", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - ipv6Ctx.ShutdownCmds = append(ipv6Ctx.ShutdownCmds, - []string{"-D", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - ipv6Ctx.CheckIptableCmds = append(ipv6Ctx.CheckIptableCmds, - []string{"-C", v, "-m", "set", "--match-set", ipv6Ctx.SetName, "src", "-j", "LOG", "--log-prefix", config.DenyLogPrefix}) - } - } } + ipv6Ctx.ipsets = v6Sets ret.v6 = ipv6Ctx return ret, nil @@ -146,9 +143,8 @@ func (ipt *iptables) Init() error { return fmt.Errorf("iptables shutdown failed: %w", err) } - // Create iptable to rule to attach the set - if err = ipt.v4.CheckAndCreate(); err != nil { - return fmt.Errorf("iptables init failed: %w", err) + if !ipt.v4.ipsetContentOnly { + ipt.v4.setupChain() } if ipt.v6 != nil { @@ -159,9 +155,8 @@ func (ipt *iptables) Init() error { return fmt.Errorf("iptables shutdown failed: %w", err) } - // Create iptable to rule to attach the set - if err := ipt.v6.CheckAndCreate(); err != nil { - return fmt.Errorf("iptables init failed: %w", err) + if !ipt.v6.ipsetContentOnly { + ipt.v6.setupChain() } } @@ -169,46 +164,38 @@ func (ipt *iptables) Init() error { } func (ipt *iptables) Commit() error { + if ipt.v4 != nil { + err := ipt.v4.commit() + if err != nil { + return fmt.Errorf("ipset for ipv4 commit failed: %w", err) + } + } + + if ipt.v6 != nil { + err := ipt.v6.commit() + if err != nil { + return fmt.Errorf("ipset for ipv6 commit failed: %w", err) + } + } + return nil } func (ipt *iptables) Add(decision *models.Decision) error { - done := false - if strings.HasPrefix(*decision.Type, "simulation:") { log.Debugf("measure against '%s' is in simulation mode, skipping it", *decision.Value) return nil } - // we now have to know if ba is for an ipv4 or ipv6 the obvious way - // would be to get the len of net.ParseIp(ba) but this is 16 internally - // even for ipv4. so we steal the ugly hack from - // https://github.com/asaskevich/govalidator/blob/3b2665001c4c24e3b076d1ca8c428049ecbb925b/validator.go#L501 if strings.Contains(*decision.Value, ":") { if ipt.v6 == nil { log.Debugf("not adding '%s' because ipv6 is disabled", *decision.Value) return nil } - - if err := ipt.v6.add(decision); err != nil { - return fmt.Errorf("failed inserting ban ip '%s' for iptables ipv4 rule", *decision.Value) - } - - done = true - } - - if strings.Contains(*decision.Value, ".") { - if err := ipt.v4.add(decision); err != nil { - return fmt.Errorf("failed inserting ban ip '%s' for iptables ipv6 rule", *decision.Value) - } - - done = true - } - - if !done { - return fmt.Errorf("failed inserting ban: ip %s was not recognized", *decision.Value) + ipt.v6.add(decision) + } else { + ipt.v4.add(decision) } - return nil } diff --git a/pkg/iptables/iptables_context.go b/pkg/iptables/iptables_context.go index 2f23148f..c1a704b0 100644 --- a/pkg/iptables/iptables_context.go +++ b/pkg/iptables/iptables_context.go @@ -5,7 +5,9 @@ package iptables import ( "fmt" + "os" "os/exec" + "slices" "strconv" "strings" "time" @@ -13,159 +15,291 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/cs-firewall-bouncer/pkg/ipsetcmd" ) +const chainName = "CROWDSEC_CHAIN" + type ipTablesContext struct { - Name string version string - ipsetBin string iptablesBin string + iptablesSaveBin string SetName string // crowdsec-netfilter SetType string SetSize int - StartupCmds [][]string // -I INPUT -m set --match-set myset src -j DROP - ShutdownCmds [][]string // -D INPUT -m set --match-set myset src -j DROP - CheckIptableCmds [][]string ipsetContentOnly bool Chains []string + + target string + + ipsets map[string]*ipsetcmd.IPSet + defaultSet *ipsetcmd.IPSet //This one is only used to restore the content, as the file will contain the name of the set for each decision + + toAdd []*models.Decision + toDel []*models.Decision + + //To avoid issues with set name length (ipsest name length is limited to 31 characters) + //Store the origin of the decisions, and use the index in the slice as the name + //This is not stable (ie, between two runs, the index of a set can change), but it's (probably) not an issue + originSetMapping []string } -func (ctx *ipTablesContext) CheckAndCreate() error { - log.Infof("Checking existing set") - /* check if the set already exist */ - cmd := exec.Command(ctx.ipsetBin, "-L", ctx.SetName) - if _, err := cmd.CombinedOutput(); err != nil { // it doesn't exist - if ctx.ipsetContentOnly { - /*if we manage ipset content only, error*/ - log.Errorf("set %s doesn't exist, can't manage content", ctx.SetName) - return fmt.Errorf("set %s doesn't exist: %w", ctx.SetName, err) - } +func (ctx *ipTablesContext) setupChain() { + cmd := []string{"-N", chainName, "-t", "filter"} - if ctx.version == "v6" { - cmd = exec.Command(ctx.ipsetBin, "-exist", "create", ctx.SetName, ctx.SetType, "timeout", "300", "family", - "inet6", "maxelem", strconv.Itoa(ctx.SetSize)) - } else { - cmd = exec.Command(ctx.ipsetBin, "-exist", "create", ctx.SetName, ctx.SetType, "timeout", "300", - "maxelem", strconv.Itoa(ctx.SetSize)) - } + c := exec.Command(ctx.iptablesBin, cmd...) - log.Infof("ipset set-up : %s", cmd.String()) + log.Infof("Creating chain : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) - if out, err := cmd.CombinedOutput(); err != nil { - return fmt.Errorf("error while creating set : %w --> %s", err, string(out)) - } + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while creating chain : %v --> %s", err, string(out)) + return } - // waiting for propagation - time.Sleep(1 * time.Second) + for _, chain := range ctx.Chains { - checkOk := true - - // checking if iptables rules exist - for _, checkCmd := range ctx.CheckIptableCmds { - cmd = exec.Command(ctx.iptablesBin, checkCmd...) - if stdout, err := cmd.CombinedOutput(); err != nil { - checkOk = false - /*rule doesn't exist, avoid alarming error messages*/ - if strings.Contains(string(stdout), "iptables: Bad rule") { - log.Infof("Rule doesn't exist (%s)", cmd.String()) - } else { - log.Warningf("iptables check command (%s) failed : %s", cmd.String(), err) - log.Debugf("output: %s", string(stdout)) - } + cmd = []string{"-I", chain, "-j", chainName} + + c = exec.Command(ctx.iptablesBin, cmd...) + + log.Infof("Adding rule : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) + + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while adding rule : %v --> %s", err, string(out)) + continue } } - /*if any of the check command error'ed, exec the setup command*/ - if !checkOk { - // if doesn't exist, create it - for _, startCmd := range ctx.StartupCmds { - cmd = exec.Command(ctx.iptablesBin, startCmd...) - log.Infof("iptables set-up : %s", cmd.String()) - - if out, err := cmd.CombinedOutput(); err != nil { - log.Warningf("Error inserting set in iptables (%s): %v : %s", cmd.String(), err, string(out)) - return fmt.Errorf("while inserting set in iptables: %w", err) - } +} + +func (ctx *ipTablesContext) deleteChain() { + + for _, chain := range ctx.Chains { + + cmd := []string{"-D", chain, "-j", chainName} + + c := exec.Command(ctx.iptablesBin, cmd...) + + log.Infof("Deleting rule : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) + + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while removing rule : %v --> %s", err, string(out)) } } - return nil + cmd := []string{"-F", chainName} + + c := exec.Command(ctx.iptablesBin, cmd...) + + log.Infof("Flushing chain : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) + + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while flushing chain : %v --> %s", err, string(out)) + } + + cmd = []string{"-X", chainName} + + c = exec.Command(ctx.iptablesBin, cmd...) + + log.Infof("Deleting chain : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) + + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while deleting chain : %v --> %s", err, string(out)) + } } -func (ctx *ipTablesContext) add(decision *models.Decision) error { - /*Create our set*/ - banDuration, err := time.ParseDuration(*decision.Duration) +func (ctx *ipTablesContext) createRule(setName string) { + cmd := []string{"-I", chainName, "-m", "set", "--match-set", setName, "src", "-j", ctx.target} + + c := exec.Command(ctx.iptablesBin, cmd...) + + log.Infof("Creating rule : %s %s", ctx.iptablesBin, strings.Join(cmd, " ")) + + if out, err := c.CombinedOutput(); err != nil { + log.Errorf("error while inserting set entry in iptables : %v --> %s", err, string(out)) + } +} + +func (ctx *ipTablesContext) commit() error { + + tmpFile, err := os.CreateTemp("", "cs-firewall-bouncer-ipset-") + if err != nil { return err } - log.Debugf("ipset add ban [%s] (for %d seconds)", *decision.Value, int(banDuration.Seconds())) + defer func() { + tmpFile.Close() + os.Remove(tmpFile.Name()) - if banDuration.Seconds() > 2147483 { - log.Warnf("Ban duration too long (%d seconds), maximum for ipset is 2147483, setting duration to 2147482", int(banDuration.Seconds())) - banDuration = time.Duration(2147482) * time.Second - } + ctx.toAdd = nil + ctx.toDel = nil + }() + + for _, decision := range ctx.toDel { + + var set *ipsetcmd.IPSet + var ok bool + + //Decisions coming from lists will have "lists" as origin, and the scenario will be the list name + //We use those to build a custom origin because we want to track metrics per list + //In case of other origin (crowdsec, cscli, ...), we do not really care about the scenario, it would be too noisy + origin := *decision.Origin + if origin == "lists" { + origin = origin + ":" + *decision.Scenario + } + + if ctx.ipsetContentOnly { + set = ctx.ipsets["ipset"] + } else { + set, ok = ctx.ipsets[origin] + if !ok { + //No set for this origin, skip, as there's nothing to delete + continue + } + } + + delCmd := fmt.Sprintf("del %s %s -exist\n", set.Name(), *decision.Value) - cmd := exec.Command(ctx.ipsetBin, "-exist", "add", ctx.SetName, *decision.Value, "timeout", strconv.Itoa(int(banDuration.Seconds()))) - log.Debugf("ipset add : %s", cmd.String()) + log.Debugf("%s", delCmd) - if out, err := cmd.CombinedOutput(); err != nil { - log.Infof("Error while inserting in set (%s): %v --> %s", cmd.String(), err, string(out)) + _, err = tmpFile.WriteString(delCmd) + + if err != nil { + log.Errorf("error while writing to temp file : %s", err) + continue + } } - // ipset -exist add test 192.168.0.1 timeout 600 - return nil -} -func (ctx *ipTablesContext) shutDown() error { - /*clean iptables rules*/ - var cmd *exec.Cmd - // if doesn't exist, create it - for _, startCmd := range ctx.ShutdownCmds { - cmd = exec.Command(ctx.iptablesBin, startCmd...) - log.Infof("iptables clean-up : %s", cmd.String()) - - if out, err := cmd.CombinedOutput(); err != nil { - if strings.Contains(string(out), "Set "+ctx.SetName+" doesn't exist.") { - log.Infof("ipset '%s' doesn't exist, skip", ctx.SetName) - } else { - log.Errorf("error while removing set entry in iptables : %v --> %s", err, string(out)) + for _, decision := range ctx.toAdd { + banDuration, err := time.ParseDuration(*decision.Duration) + if err != nil { + log.Errorf("error while parsing ban duration : %s", err) + continue + } + + var set *ipsetcmd.IPSet + var ok bool + + if banDuration.Seconds() > 2147483 { + log.Warnf("Ban duration too long (%d seconds), maximum for ipset is 2147483, setting duration to 2147482", int(banDuration.Seconds())) + banDuration = time.Duration(2147482) * time.Second + } + + origin := *decision.Origin + + if origin == "lists" { + origin = origin + ":" + *decision.Scenario + } + + if ctx.ipsetContentOnly { + set = ctx.ipsets["ipset"] + } else { + set, ok = ctx.ipsets[origin] + + if !ok { + + idx := slices.Index(ctx.originSetMapping, origin) + + if idx == -1 { + ctx.originSetMapping = append(ctx.originSetMapping, origin) + idx = len(ctx.originSetMapping) - 1 + } + + setName := fmt.Sprintf("%s-%d", ctx.SetName, idx) + + log.Infof("Using %s as set for origin %s", setName, origin) + + set, err = ipsetcmd.NewIPSet(setName) + + if err != nil { + log.Errorf("error while creating ipset : %s", err) + continue + } + + family := "inet" + + if ctx.version == "v6" { + family = "inet6" + } + + err = set.Create(ipsetcmd.CreateOptions{ + Family: family, + Timeout: "300", + MaxElem: strconv.Itoa(ctx.SetSize), + Type: ctx.SetType, + }) + + //Ignore errors if the set already exists + if err != nil { + log.Errorf("error while creating ipset : %s", err) + continue + } + + ctx.ipsets[origin] = set + + if !ctx.ipsetContentOnly { + //Create the rule to use the set + ctx.createRule(set.Name()) + } } } + + addCmd := fmt.Sprintf("add %s %s timeout %d -exist\n", set.Name(), *decision.Value, int(banDuration.Seconds())) + + log.Debugf("%s", addCmd) + + _, err = tmpFile.WriteString(addCmd) + + if err != nil { + log.Errorf("error while writing to temp file : %s", err) + continue + } } - /*clean ipset set*/ - var ipsetCmd string - if ctx.ipsetContentOnly { - ipsetCmd = "flush" - } else { - ipsetCmd = "destroy" + if len(ctx.toAdd) == 0 && len(ctx.toDel) == 0 { + return nil } - cmd = exec.Command(ctx.ipsetBin, "-exist", ipsetCmd, ctx.SetName) - log.Infof("ipset clean-up : %s", cmd.String()) + return ctx.defaultSet.Restore(tmpFile.Name()) +} + +func (ctx *ipTablesContext) add(decision *models.Decision) { + ctx.toAdd = append(ctx.toAdd, decision) +} - if out, err := cmd.CombinedOutput(); err != nil { - if strings.Contains(string(out), "The set with the given name does not exist") { - log.Infof("ipset '%s' doesn't exist, skip", ctx.SetName) +func (ctx *ipTablesContext) shutDown() error { + + //Remove rules + if !ctx.ipsetContentOnly { + ctx.deleteChain() + } + + time.Sleep(1 * time.Second) + + //Clean sets + for _, set := range ctx.ipsets { + if ctx.ipsetContentOnly { + err := set.Flush() + if err != nil { + log.Errorf("error while flushing ipset : %s", err) + } } else { - log.Errorf("set %s error : %v - %s", ipsetCmd, err, string(out)) + err := set.Destroy() + if err != nil { + log.Errorf("error while destroying set %s : %s", set.Name(), err) + } } } + if !ctx.ipsetContentOnly { + //In case we are starting, just reset the map + ctx.ipsets = make(map[string]*ipsetcmd.IPSet) + } + return nil } func (ctx *ipTablesContext) delete(decision *models.Decision) error { - /* - ipset -exist delete test 192.168.0.1 timeout 600 - ipset -exist add test 192.168.0.1 timeout 600 - */ - log.Debugf("ipset del ban for [%s]", *decision.Value) - - cmd := exec.Command(ctx.ipsetBin, "-exist", "del", ctx.SetName, *decision.Value) - if out, err := cmd.CombinedOutput(); err != nil { - log.Infof("Error while deleting from set (%s): %v --> %s", cmd.String(), err, string(out)) - } - // ipset -exist add test 192.168.0.1 timeout 600 + ctx.toDel = append(ctx.toDel, decision) return nil } diff --git a/pkg/iptables/metrics.go b/pkg/iptables/metrics.go index 07b33a55..a29c4e83 100644 --- a/pkg/iptables/metrics.go +++ b/pkg/iptables/metrics.go @@ -4,111 +4,295 @@ package iptables import ( - "encoding/xml" + "bufio" "os/exec" + "regexp" "strconv" "strings" - "time" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics" + + "github.com/prometheus/client_golang/prometheus" ) -type Ipsets struct { - Ipset []struct { - Name string `xml:"name,attr"` - Header struct { - Numentries string `xml:"numentries"` - } `xml:"header"` - } `xml:"ipset"` -} +// iptables does not provide a "nice" way to get the counters for a rule, so we have to parse the output of iptables-save +// chainRegexp is just used to get the counters for the chain CROWDSEC_CHAIN (the chain managed by the bouncer that will contains our rules) from the JUMP rule +// ruleRegexp is used to get the counters for the rules we have added that will actually block the traffic +// Example output of iptables-save : +// [2080:13210403] -A INPUT -j CROWDSEC_CHAIN +// ... +// [0:0] -A CROWDSEC_CHAIN -m set --match-set test-set-ipset-mode-0 src -j DROP +// First number is the number of packets, second is the number of bytes +// In case of a jump, the counters represent the number of packets and bytes that have been processed by the chain (ie, whether the packets have been accepted or dropped) +// In case of a rule, the counters represent the number of packets and bytes that have been matched by the rule (ie, the packets that have been dropped). -func collectDroppedPackets(binaryPath string, chains []string, setName string) (float64, float64) { - var droppedPackets, droppedBytes float64 +var chainRegexp = regexp.MustCompile(`^\[(\d+):(\d+)\]`) +var ruleRegexp = regexp.MustCompile(`^\[(\d+):(\d+)\] -A [0-9A-Za-z_-]+ -m set --match-set (.*) src -j \w+`) + +// In ipset mode, we have to track the numbers of processed bytes/packets at the chain level +// This is not really accurate, as a rule *before* the crowdsec rule could impact the numbers, but we don't have any other way. + +var ipsetChainDeclaration = regexp.MustCompile(`^:([0-9A-Za-z_-]+) ([0-9A-Za-z_-]+) \[(\d+):(\d+)\]`) +var ipsetRule = regexp.MustCompile(`^\[(\d+):(\d+)\] -A ([0-9A-Za-z_-]+)`) + +func (ctx *ipTablesContext) collectMetricsIptables(scanner *bufio.Scanner) (map[string]int, map[string]int, int, int) { + processedBytes := 0 + processedPackets := 0 + + droppedBytes := make(map[string]int) + droppedPackets := make(map[string]int) + + for scanner.Scan() { + line := scanner.Text() + + //Ignore chain declaration + if line[0] == ':' { + continue + } + + //Jump to our chain, we can get the processed packets and bytes + if strings.Contains(line, "-j "+chainName) { + matches := chainRegexp.FindStringSubmatch(line) + if len(matches) != 3 { + log.Errorf("error while parsing counters : %s | not enough matches", line) + continue + } + val, err := strconv.Atoi(matches[1]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + processedPackets += val + + val, err = strconv.Atoi(matches[2]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + processedBytes += val - for _, chain := range chains { - out, err := exec.Command(binaryPath, "-L", chain, "-v", "-x").CombinedOutput() - if err != nil { - log.Error(string(out), err) continue } - for _, line := range strings.Split(string(out), "\n") { - if !strings.Contains(line, setName) || strings.Contains(line, "LOG") { + //This is a rule + if strings.Contains(line, "-A "+chainName) { + matches := ruleRegexp.FindStringSubmatch(line) + if len(matches) != 4 { + log.Errorf("error while parsing counters : %s | not enough matches", line) continue } - parts := strings.Fields(line) + originIDStr, found := strings.CutPrefix(matches[3], ctx.SetName+"-") + if !found { + log.Errorf("error while parsing counters : %s | no origin found", line) + continue + } + originID, err := strconv.Atoi(originIDStr) - tdp, err := strconv.ParseFloat(parts[IPTablesDroppedPacketIdx], 64) if err != nil { - log.Error(err.Error()) + log.Errorf("error while parsing counters : %s | %s", line, err) + continue } - droppedPackets += tdp + if len(ctx.originSetMapping) < originID { + log.Errorf("Found unknown origin id : %d", originID) + continue + } + + origin := ctx.originSetMapping[originID] + + val, err := strconv.Atoi(matches[1]) + if err != nil { + log.Errorf("error while parsing counters : %s | %s", line, err) + continue + } + droppedPackets[origin] += val - tdb, err := strconv.ParseFloat(parts[IPTablesDroppedByteIdx], 64) + val, err = strconv.Atoi(matches[2]) if err != nil { - log.Error(err.Error()) + log.Errorf("error while parsing counters : %s | %s", line, err) + continue } - droppedBytes += tdb + droppedBytes[origin] += val } } - return droppedPackets, droppedBytes + return droppedPackets, droppedBytes, processedPackets, processedBytes + } -func (ipt *iptables) CollectMetrics() { - var ip4DroppedPackets, ip4DroppedBytes, ip6DroppedPackets, ip6DroppedBytes float64 +type chainCounters struct { + bytes int + packets int +} + +// In ipset mode, we only get dropped packets and bytes by matching on the set name in the rule +// It's probably not perfect, but good enough for most users. +func (ctx *ipTablesContext) collectMetricsIpset(scanner *bufio.Scanner) (map[string]int, map[string]int, int, int) { + processedBytes := 0 + processedPackets := 0 - t := time.NewTicker(metrics.MetricCollectionInterval) - for range t.C { - if ipt.v4 != nil && !ipt.v4.ipsetContentOnly { - ip4DroppedPackets, ip4DroppedBytes = collectDroppedPackets(ipt.v4.iptablesBin, ipt.v4.Chains, ipt.v4.SetName) + droppedBytes := make(map[string]int) + droppedPackets := make(map[string]int) + + // We need to store the counters for all chains + // As we don't know in which chain the user has setup the rules + // We'll resolve the value laters. + chainsCounter := make(map[string]chainCounters) + + // Hardcode the origin to ipset as we cannot know it based on the rule. + droppedBytes["ipset"] = 0 + droppedPackets["ipset"] = 0 + + for scanner.Scan() { + line := scanner.Text() + + //Chain declaration + if line[0] == ':' { + matches := ipsetChainDeclaration.FindStringSubmatch(line) + if len(matches) != 5 { + log.Errorf("error while parsing counters : %s | not enough matches", line) + continue + } + + log.Debugf("Found chain %s with matches %+v", matches[1], matches) + + c, ok := chainsCounter[matches[1]] + if !ok { + c = chainCounters{} + } + + val, err := strconv.Atoi(matches[3]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + c.packets += val + + val, err = strconv.Atoi(matches[4]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + c.bytes += val + + chainsCounter[matches[1]] = c + continue } - if ipt.v6 != nil && !ipt.v6.ipsetContentOnly { - ip6DroppedPackets, ip6DroppedBytes = collectDroppedPackets(ipt.v6.iptablesBin, ipt.v6.Chains, ipt.v6.SetName) + // Assume that if a line contains the set name, it's a rule we are interested in. + if strings.Contains(line, ctx.SetName) { + matches := ipsetRule.FindStringSubmatch(line) + if len(matches) != 4 { + log.Errorf("error while parsing counters : %s | not enough matches", line) + continue + } + + val, err := strconv.Atoi(matches[1]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + droppedPackets["ipset"] += val + + val, err = strconv.Atoi(matches[2]) + if err != nil { + log.Errorf("error while parsing counters : %s", line) + continue + } + + droppedBytes["ipset"] += val + + //Resolve the chain counters + c, ok := chainsCounter[matches[3]] + if !ok { + log.Errorf("error while parsing counters : %s | chain not found", line) + continue + } + + processedPackets += c.packets + processedBytes += c.bytes } + } + + return droppedPackets, droppedBytes, processedPackets, processedBytes +} + +func (ctx *ipTablesContext) collectMetrics() (map[string]int, map[string]int, int, int, error) { + //-c is required to get the counters + cmd := []string{ctx.iptablesSaveBin, "-c", "-t", "filter"} + saveCmd := exec.Command(cmd[0], cmd[1:]...) + out, err := saveCmd.CombinedOutput() + if err != nil { + log.Errorf("error while getting iptables rules with cmd %+v : %v --> %s", cmd, err, string(out)) + return nil, nil, 0, 0, err + } + + var processedBytes int + var processedPackets int + var droppedBytes map[string]int + var droppedPackets map[string]int + + scanner := bufio.NewScanner(strings.NewReader(string(out))) + + if !ctx.ipsetContentOnly { + droppedPackets, droppedBytes, processedPackets, processedBytes = ctx.collectMetricsIptables(scanner) + } else { + droppedPackets, droppedBytes, processedPackets, processedBytes = ctx.collectMetricsIpset(scanner) + } + + log.Debugf("Processed %d packets and %d bytes", processedPackets, processedBytes) + log.Debugf("Dropped packets : %v", droppedPackets) + log.Debugf("Dropped bytes : %v", droppedBytes) + + return droppedPackets, droppedBytes, processedPackets, processedBytes, nil +} - if (ipt.v4 != nil && !ipt.v4.ipsetContentOnly) || (ipt.v6 != nil && !ipt.v6.ipsetContentOnly) { - metrics.TotalDroppedPackets.Set(ip4DroppedPackets + ip6DroppedPackets) - metrics.TotalDroppedBytes.Set(ip6DroppedBytes + ip4DroppedBytes) +func (ipt *iptables) CollectMetrics() { + if ipt.v4 != nil { + for origin, set := range ipt.v4.ipsets { + metrics.TotalActiveBannedIPs.With(prometheus.Labels{"ip_type": "ipv4", "origin": origin}).Set(float64(set.Len())) } + ipv4DroppedPackets, ipv4DroppedBytes, ipv4ProcessedPackets, ipv4ProcessedBytes, err := ipt.v4.collectMetrics() - out, err := exec.Command(ipt.v4.ipsetBin, "list", "-o", "xml").CombinedOutput() if err != nil { - log.Error(err) - continue - } + log.Errorf("can't collect dropped packets for ipv4 from iptables: %s", err) + } else { + metrics.TotalProcessedPackets.With(prometheus.Labels{"ip_type": "ipv4"}).Set(float64(ipv4ProcessedPackets)) + metrics.TotalProcessedBytes.With(prometheus.Labels{"ip_type": "ipv4"}).Set(float64(ipv4ProcessedBytes)) - ipsets := Ipsets{} + for origin, count := range ipv4DroppedPackets { + metrics.TotalDroppedPackets.With(prometheus.Labels{"ip_type": "ipv4", "origin": origin}).Set(float64(count)) + } - if err := xml.Unmarshal(out, &ipsets); err != nil { - log.Error(err) - continue + for origin, count := range ipv4DroppedBytes { + metrics.TotalDroppedBytes.With(prometheus.Labels{"ip_type": "ipv4", "origin": origin}).Set(float64(count)) + } } + } - newCount := float64(0) + if ipt.v6 != nil { + for origin, set := range ipt.v6.ipsets { + metrics.TotalActiveBannedIPs.With(prometheus.Labels{"ip_type": "ipv6", "origin": origin}).Set(float64(set.Len())) + } + ipv6DroppedPackets, ipv6DroppedBytes, ipv6ProcessedPackets, ipv6ProcessedBytes, err := ipt.v6.collectMetrics() - for _, ipset := range ipsets.Ipset { - if ipset.Name == ipt.v4.SetName || (ipt.v6 != nil && ipset.Name == ipt.v6.SetName) { - if ipset.Header.Numentries == "" { - continue - } + if err != nil { + log.Errorf("can't collect dropped packets for ipv6 from iptables: %s", err) + } else { + metrics.TotalProcessedPackets.With(prometheus.Labels{"ip_type": "ipv6"}).Set(float64(ipv6ProcessedPackets)) + metrics.TotalProcessedBytes.With(prometheus.Labels{"ip_type": "ipv6"}).Set(float64(ipv6ProcessedBytes)) - count, err := strconv.ParseFloat(ipset.Header.Numentries, 64) - if err != nil { - log.Errorf("error while parsing Numentries from ipsets: %s", err) - continue - } + for origin, count := range ipv6DroppedPackets { + metrics.TotalDroppedPackets.With(prometheus.Labels{"ip_type": "ipv6", "origin": origin}).Set(float64(count)) + } - newCount += count + for origin, count := range ipv6DroppedBytes { + metrics.TotalDroppedBytes.With(prometheus.Labels{"ip_type": "ipv6", "origin": origin}).Set(float64(count)) } } - - metrics.TotalActiveBannedIPs.Set(newCount) } } diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go index 6482744b..ec3acf8a 100644 --- a/pkg/metrics/metrics.go +++ b/pkg/metrics/metrics.go @@ -8,17 +8,39 @@ import ( const MetricCollectionInterval = time.Second * 10 -var TotalDroppedPackets = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "fw_bouncer_dropped_packets", +const ( + DroppedPacketsMetricName = "fw_bouncer_dropped_packets" + DroppedBytesMetricName = "fw_bouncer_dropped_bytes" + ProcessedPacketsMetricName = "fw_bouncer_processed_packets" + ProcessedBytesMetricName = "fw_bouncer_processed_bytes" + ActiveBannedIPsMetricName = "fw_bouncer_banned_ips" +) + +var TotalDroppedPackets = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: DroppedPacketsMetricName, Help: "Denotes the number of total dropped packets because of rule(s) created by crowdsec", -}) +}, []string{"origin", "ip_type"}) +var LastDroppedPacketsValue map[string]float64 = make(map[string]float64) -var TotalDroppedBytes = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "fw_bouncer_dropped_bytes", +var TotalDroppedBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: DroppedBytesMetricName, Help: "Denotes the number of total dropped bytes because of rule(s) created by crowdsec", -}) +}, []string{"origin", "ip_type"}) +var LastDroppedBytesValue map[string]float64 = make(map[string]float64) -var TotalActiveBannedIPs = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "fw_bouncer_banned_ips", +var TotalActiveBannedIPs = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: ActiveBannedIPsMetricName, Help: "Denotes the number of IPs which are currently banned", -}) +}, []string{"origin", "ip_type"}) + +var TotalProcessedPackets = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: ProcessedPacketsMetricName, + Help: "Denotes the number of total processed packets by the rules created by crowdsec", +}, []string{"ip_type"}) +var LastProcessedPacketsValue map[string]float64 = make(map[string]float64) + +var TotalProcessedBytes = prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: ProcessedBytesMetricName, + Help: "Denotes the number of total processed bytes by the rules created by crowdsec", +}, []string{"ip_type"}) +var LastProcessedBytesValue map[string]float64 = make(map[string]float64) diff --git a/pkg/nftables/metrics.go b/pkg/nftables/metrics.go index 81faefdd..a688f7f0 100644 --- a/pkg/nftables/metrics.go +++ b/pkg/nftables/metrics.go @@ -4,150 +4,142 @@ package nftables import ( - "encoding/json" "fmt" - "os/exec" + "strings" "time" - log "github.com/sirupsen/logrus" - "github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics" + "github.com/google/nftables/expr" + "github.com/prometheus/client_golang/prometheus" + + log "github.com/sirupsen/logrus" ) -type Counter struct { - Nftables []struct { - Rule struct { - Expr []struct { - Counter *struct { - Packets int `json:"packets"` - Bytes int `json:"bytes"` - } `json:"counter,omitempty"` - } `json:"expr"` - } `json:"rule,omitempty"` - } `json:"nftables"` -} +func (c *nftContext) collectDroppedPackets() (map[string]int, map[string]int, int, int, error) { + droppedPackets := make(map[string]int) + droppedBytes := make(map[string]int) + processedPackets := 0 + processedBytes := 0 + //setName := "" + for chainName, chain := range c.chains { + rules, err := c.conn.GetRules(c.table, chain) + if err != nil { + log.Errorf("can't get rules for chain %s: %s", chainName, err) + continue + } + for _, rule := range rules { + for _, xpr := range rule.Exprs { + obj, ok := xpr.(*expr.Counter) + if ok { + log.Debugf("rule %d (%s): packets %d, bytes %d (%s)", rule.Position, rule.Table.Name, obj.Packets, obj.Bytes, rule.UserData) + if string(rule.UserData) == "processed" { + processedPackets += int(obj.Packets) + processedBytes += int(obj.Bytes) + continue + } + origin, _ := strings.CutPrefix(string(rule.UserData), c.blacklists+"-") + if origin == "" { + continue + } + droppedPackets[origin] += int(obj.Packets) + droppedBytes[origin] += int(obj.Bytes) + } + } + } + } -type Set struct { - Nftables []struct { - Set struct { - Elem []struct { - Elem struct{} `json:"elem"` - } `json:"elem"` - } `json:"set,omitempty"` - } `json:"nftables"` + return droppedPackets, droppedBytes, processedPackets, processedBytes, nil } -func (c *nftContext) collectDroppedPackets(path string, chain string) (int, int, error) { - cmd := exec.Command(path, "-j", "list", "chain", c.ipFamily(), c.tableName, chain) - - out, err := cmd.CombinedOutput() - if err != nil { - return 0, 0, fmt.Errorf("while running %s: %w", cmd.String(), err) - } - - parsedOut := Counter{} - if err := json.Unmarshal(out, &parsedOut); err != nil { - return 0, 0, err - } +func (c *nftContext) collectActiveBannedIPs() (map[string]int, error) { + //Find the size of the set we have created + ret := make(map[string]int) - for _, r := range parsedOut.Nftables { - for _, expr := range r.Rule.Expr { - if expr.Counter != nil { - return expr.Counter.Packets, expr.Counter.Bytes, nil - } + for origin, set := range c.sets { + setContent, err := c.conn.GetSetElements(set) + if err != nil { + return nil, fmt.Errorf("can't get set elements for %s: %w", set.Name, err) } + if c.setOnly { + ret[c.blacklists] = len(setContent) + } else { + ret[origin] = len(setContent) + } + return ret, nil } - return 0, 0, nil + return ret, nil } -func (c *nftContext) ipFamily() string { - if c.version == "v4" { - return "ip" +func (c *nftContext) collectDropped() (map[string]int, map[string]int, int, int, map[string]int) { + if c.conn == nil { + return nil, nil, 0, 0, nil } - return "ip6" -} - -func (c *nftContext) collectActiveBannedIPs(path string) (int, error) { - cmd := exec.Command(path, "-j", "list", "set", c.ipFamily(), c.tableName, c.blacklists) + droppedPackets, droppedBytes, processedPackets, processedBytes, err := c.collectDroppedPackets() - out, err := cmd.CombinedOutput() if err != nil { - return 0, fmt.Errorf("while running %s: %w", cmd.String(), err) + log.Errorf("can't collect dropped packets for ip%s from nft: %s", c.version, err) } - set := Set{} - if err := json.Unmarshal(out, &set); err != nil { - return 0, err + banned, err := c.collectActiveBannedIPs() + if err != nil { + log.Errorf("can't collect total banned IPs for ip%s from nft: %s", c.version, err) } - ret := 0 - for _, r := range set.Nftables { - ret += len(r.Set.Elem) + return droppedPackets, droppedBytes, processedPackets, processedBytes, banned +} + +func getOriginForList(origin string) string { + if !strings.HasPrefix(origin, "lists-") { + return origin } - return ret, nil + return strings.Replace(origin, "-", ":", 1) } -func (c *nftContext) collectDropped(path string, hooks []string) (int, int, int) { - if c.conn == nil { - return 0, 0, 0 - } +func (n *nft) CollectMetrics() { + startTime := time.Now() + ip4DroppedPackets, ip4DroppedBytes, ip4ProcessedPackets, ip4ProcessedBytes, bannedIP4 := n.v4.collectDropped() + ip6DroppedPackets, ip6DroppedBytes, ip6ProcessedPackets, ip6ProcessedBytes, bannedIP6 := n.v6.collectDropped() - var droppedPackets, droppedBytes, banned int + log.Debugf("metrics collection took %s", time.Since(startTime)) + log.Debugf("ip4: dropped packets: %+v, dropped bytes: %+v, banned IPs: %+v, proccessed packets: %d, processed bytes: %d", ip4DroppedPackets, ip4DroppedBytes, bannedIP4, ip4ProcessedPackets, ip4ProcessedBytes) + log.Debugf("ip6: dropped packets: %+v, dropped bytes: %+v, banned IPs: %+v, proccessed packets: %d, processed bytes: %d", ip6DroppedPackets, ip6DroppedBytes, bannedIP6, ip6ProcessedPackets, ip6ProcessedBytes) - if c.setOnly { - pkt, byt, err := c.collectDroppedPackets(path, c.chainName) - if err != nil { - log.Errorf("can't collect dropped packets for ip%s from nft: %s", c.version, err) - } + metrics.TotalProcessedPackets.With(prometheus.Labels{"ip_type": "ipv4"}).Set(float64(ip4ProcessedPackets)) + metrics.TotalProcessedBytes.With(prometheus.Labels{"ip_type": "ipv4"}).Set(float64(ip4ProcessedBytes)) - droppedPackets += pkt - droppedBytes += byt - } else { - for _, hook := range hooks { - pkt, byt, err := c.collectDroppedPackets(path, c.chainName+"-"+hook) - if err != nil { - log.Errorf("can't collect dropped packets for ip%s from nft: %s", c.version, err) - } + metrics.TotalProcessedPackets.With(prometheus.Labels{"ip_type": "ipv6"}).Set(float64(ip6ProcessedPackets)) + metrics.TotalProcessedBytes.With(prometheus.Labels{"ip_type": "ipv6"}).Set(float64(ip6ProcessedBytes)) - droppedPackets += pkt - droppedBytes += byt - } + for origin, count := range bannedIP4 { + origin = getOriginForList(origin) + metrics.TotalActiveBannedIPs.With(prometheus.Labels{"origin": origin, "ip_type": "ipv4"}).Set(float64(count)) } - banned, err := c.collectActiveBannedIPs(path) - if err != nil { - log.Errorf("can't collect total banned IPs for ip%s from nft: %s", c.version, err) + for origin, count := range bannedIP6 { + origin = getOriginForList(origin) + metrics.TotalActiveBannedIPs.With(prometheus.Labels{"origin": origin, "ip_type": "ipv6"}).Set(float64(count)) } - return droppedPackets, droppedBytes, banned -} - -func (n *nft) CollectMetrics() { - path, err := exec.LookPath("nft") - if err != nil { - log.Error("can't monitor dropped packets: ", err) - return + for origin, count := range ip4DroppedPackets { + origin = getOriginForList(origin) + metrics.TotalDroppedPackets.With(prometheus.Labels{"origin": origin, "ip_type": "ipv4"}).Set(float64(count)) } - cmd := exec.Command(path, "-j", "list", "tables") - - _, err = cmd.CombinedOutput() - if err != nil { - log.Warningf("nft -j is not supported (requires 0.9.7), nftables metrics are disabled") - return + for origin, count := range ip6DroppedPackets { + origin = getOriginForList(origin) + metrics.TotalDroppedPackets.With(prometheus.Labels{"origin": origin, "ip_type": "ipv6"}).Set(float64(count)) } - t := time.NewTicker(metrics.MetricCollectionInterval) - - for range t.C { - ip4DroppedPackets, ip4DroppedBytes, bannedIP4 := n.v4.collectDropped(path, n.Hooks) - ip6DroppedPackets, ip6DroppedBytes, bannedIP6 := n.v6.collectDropped(path, n.Hooks) + for origin, count := range ip4DroppedBytes { + origin = getOriginForList(origin) + metrics.TotalDroppedBytes.With(prometheus.Labels{"origin": origin, "ip_type": "ipv4"}).Set(float64(count)) + } - metrics.TotalDroppedPackets.Set(float64(ip4DroppedPackets + ip6DroppedPackets)) - metrics.TotalDroppedBytes.Set(float64(ip6DroppedBytes + ip4DroppedBytes)) - metrics.TotalActiveBannedIPs.Set(float64(bannedIP4 + bannedIP6)) + for origin, count := range ip6DroppedBytes { + origin = getOriginForList(origin) + metrics.TotalDroppedBytes.With(prometheus.Labels{"origin": origin, "ip_type": "ipv6"}).Set(float64(count)) } } diff --git a/pkg/nftables/nftables.go b/pkg/nftables/nftables.go index 69678b6d..4233e018 100644 --- a/pkg/nftables/nftables.go +++ b/pkg/nftables/nftables.go @@ -10,6 +10,7 @@ import ( "time" "github.com/google/nftables" + "github.com/google/nftables/binaryutil" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" @@ -49,11 +50,11 @@ func NewNFTables(config *cfg.BouncerConfig) (*nft, error) { func (n *nft) Init() error { log.Debug("nftables: Init()") - if err := n.v4.init(n.Hooks, n.DenyLog, n.DenyLogPrefix, n.DenyAction); err != nil { + if err := n.v4.init(n.Hooks); err != nil { return err } - if err := n.v6.init(n.Hooks, n.DenyLog, n.DenyLogPrefix, n.DenyAction); err != nil { + if err := n.v6.init(n.Hooks); err != nil { return err } @@ -139,14 +140,42 @@ func (n *nft) commitDeletedDecisions() error { return nil } +func (n *nft) createSetAndRuleForOrigin(ctx *nftContext, origin string) error { + if _, ok := ctx.sets[origin]; !ok { + //First time we see this origin, create the rule/set for all hooks + set := &nftables.Set{ + Name: fmt.Sprintf("%s-%s", ctx.blacklists, origin), + Table: ctx.table, + KeyType: ctx.typeIPAddr, + KeyByteOrder: binaryutil.BigEndian, + HasTimeout: true, + } + + ctx.sets[origin] = set + + if err := ctx.conn.AddSet(set, []nftables.SetElement{}); err != nil { + return err + } + for _, chain := range ctx.chains { + rule, err := ctx.createRule(chain, set, n.DenyLog, n.DenyLogPrefix, n.DenyAction) + if err != nil { + return err + } + ctx.conn.AddRule(rule) + log.Infof("Created set and rule for origin %s and type %s in chain %s", origin, ctx.typeIPAddr.Name, chain.Name) + } + } + return nil +} + func (n *nft) commitAddedDecisions() error { banned, err := n.getBannedState() if err != nil { return fmt.Errorf("failed to get current state: %w", err) } - ip4 := []nftables.SetElement{} - ip6 := []nftables.SetElement{} + ip4 := make(map[string][]nftables.SetElement, 0) + ip6 := make(map[string][]nftables.SetElement, 0) n.decisionsToAdd = normalizedDecisions(n.decisionsToAdd) @@ -159,20 +188,47 @@ func (n *nft) commitAddedDecisions() error { t, _ := time.ParseDuration(*decision.Duration) + origin := *decision.Origin + + if origin == "lists" { + origin = origin + "-" + *decision.Scenario + } + if strings.Contains(ip.String(), ":") { if n.v6.conn != nil { + if n.v6.setOnly { + origin = n.v6.blacklists + } log.Tracef("adding %s to buffer", ip) - - ip6 = append(ip6, nftables.SetElement{Timeout: t, Key: ip.To16()}) + if _, ok := ip6[origin]; !ok { + ip6[origin] = make([]nftables.SetElement, 0) + } + ip6[origin] = append(ip6[origin], nftables.SetElement{Timeout: t, Key: ip.To16()}) + if !n.v6.setOnly { + err := n.createSetAndRuleForOrigin(n.v6, origin) + if err != nil { + return err + } + } } - continue } if n.v4.conn != nil { + if n.v4.setOnly { + origin = n.v4.blacklists + } log.Tracef("adding %s to buffer", ip) - - ip4 = append(ip4, nftables.SetElement{Timeout: t, Key: ip.To4()}) + if _, ok := ip4[origin]; !ok { + ip4[origin] = make([]nftables.SetElement, 0) + } + ip4[origin] = append(ip4[origin], nftables.SetElement{Timeout: t, Key: ip.To4()}) + if !n.v4.setOnly { + err := n.createSetAndRuleForOrigin(n.v4, origin) + if err != nil { + return err + } + } } } @@ -201,9 +257,15 @@ func (n *nft) Commit() error { return nil } +type tmpDecisions struct { + duration time.Duration + origin string + scenario string +} + // remove duplicates, normalize decision timeouts, keep the longest decision when dups are present. func normalizedDecisions(decisions []*models.Decision) []*models.Decision { - vals := make(map[string]time.Duration) + vals := make(map[string]tmpDecisions) finalDecisions := make([]*models.Decision, 0) for _, d := range decisions { @@ -213,16 +275,26 @@ func normalizedDecisions(decisions []*models.Decision) []*models.Decision { } *d.Value = strings.Split(*d.Value, "/")[0] - vals[*d.Value] = maxTime(t, vals[*d.Value]) + if max, ok := vals[*d.Value]; !ok || t > max.duration { + vals[*d.Value] = tmpDecisions{ + duration: t, + origin: *d.Origin, + scenario: *d.Scenario, + } + } } - for ip, duration := range vals { - d := duration.String() + for ip, decision := range vals { + d := decision.duration.String() i := ip // copy it because we don't same value for all decisions as `ip` is same pointer :) + origin := decision.origin + scenario := decision.scenario finalDecisions = append(finalDecisions, &models.Decision{ Duration: &d, Value: &i, + Origin: &origin, + Scenario: &scenario, }) } @@ -245,11 +317,3 @@ func (n *nft) ShutDown() error { return nil } - -func maxTime(a time.Duration, b time.Duration) time.Duration { - if a > b { - return a - } - - return b -} diff --git a/pkg/nftables/nftables_context.go b/pkg/nftables/nftables_context.go index 4d45421b..f12bb9d2 100644 --- a/pkg/nftables/nftables_context.go +++ b/pkg/nftables/nftables_context.go @@ -29,8 +29,9 @@ var HookNameToHookID = map[string]nftables.ChainHook{ } type nftContext struct { + chains map[string]*nftables.Chain conn *nftables.Conn - set *nftables.Set + sets map[string]*nftables.Set table *nftables.Table tableFamily nftables.TableFamily typeIPAddr nftables.SetDatatype @@ -70,6 +71,7 @@ func NewNFTV4Context(config *cfg.BouncerConfig) *nftContext { blacklists: config.BlacklistsIpv4, setOnly: config.Nftables.Ipv4.SetOnly, priority: config.Nftables.Ipv4.Priority, + sets: make(map[string]*nftables.Set), } log.Debugf("nftables: ipv4: %t, table: %s, chain: %s, blacklist: %s, set-only: %t", @@ -99,6 +101,7 @@ func NewNFTV6Context(config *cfg.BouncerConfig) *nftContext { blacklists: config.BlacklistsIpv6, setOnly: config.Nftables.Ipv6.SetOnly, priority: config.Nftables.Ipv6.Priority, + sets: make(map[string]*nftables.Set), } log.Debugf("nftables: ipv6: %t, table6: %s, chain6: %s, blacklist: %s, set-only6: %t", @@ -113,13 +116,16 @@ func (c *nftContext) setBanned(banned map[string]struct{}) error { return nil } - elements, err := c.conn.GetSetElements(c.set) - if err != nil { - return err - } + for _, set := range c.sets { - for _, el := range elements { - banned[net.IP(el.Key).String()] = struct{}{} + elements, err := c.conn.GetSetElements(set) + if err != nil { + return err + } + + for _, el := range elements { + banned[net.IP(el.Key).String()] = struct{}{} + } } return nil @@ -157,13 +163,13 @@ func (c *nftContext) initSetOnly() error { } } - c.set = set + c.sets[c.blacklists] = set log.Debugf("nftables: ip%s set '%s' configured", c.version, c.blacklists) return nil } -func (c *nftContext) initOwnTable(hooks []string, denyLog bool, denyLogPrefix string, denyAction string) error { +func (c *nftContext) initOwnTable(hooks []string) error { log.Debugf("nftables: ip%s own table", c.version) c.table = c.conn.AddTable(&nftables.Table{ @@ -171,20 +177,6 @@ func (c *nftContext) initOwnTable(hooks []string, denyLog bool, denyLogPrefix st Name: c.tableName, }) - set := &nftables.Set{ - Name: c.blacklists, - Table: c.table, - KeyType: c.typeIPAddr, - KeyByteOrder: binaryutil.BigEndian, - HasTimeout: true, - } - - if err := c.conn.AddSet(set, []nftables.SetElement{}); err != nil { - return err - } - - c.set = set - for _, hook := range hooks { hooknum := HookNameToHookID[hook] priority := nftables.ChainPriority(c.priority) @@ -196,14 +188,21 @@ func (c *nftContext) initOwnTable(hooks []string, denyLog bool, denyLogPrefix st Priority: &priority, }) - log.Debugf("nftables: ip%s chain '%s' created", c.version, chain.Name) - - r, err := c.createRule(chain, set, denyLog, denyLogPrefix, denyAction) - if err != nil { - return err + r := &nftables.Rule{ + Table: c.table, + Chain: chain, + Exprs: []expr.Any{ + &expr.Counter{}, + }, + UserData: []byte("processed"), } c.conn.AddRule(r) + + c.chains[hook] = chain + + log.Debugf("nftables: ip%s chain '%s' created", c.version, chain.Name) + //Rules and sets are created on the fly when we detect a new origin } if err := c.conn.Flush(); err != nil { @@ -215,11 +214,15 @@ func (c *nftContext) initOwnTable(hooks []string, denyLog bool, denyLogPrefix st return nil } -func (c *nftContext) init(hooks []string, denyLog bool, denyLogPrefix string, denyAction string) error { +func (c *nftContext) init(hooks []string) error { if c.conn == nil { return nil } + if c.chains == nil { + c.chains = make(map[string]*nftables.Chain) + } + log.Debugf("nftables: ip%s init starting", c.version) var err error @@ -227,7 +230,7 @@ func (c *nftContext) init(hooks []string, denyLog bool, denyLogPrefix string, de if c.setOnly { err = c.initSetOnly() } else { - err = c.initOwnTable(hooks, denyLog, denyLogPrefix, denyAction) + err = c.initOwnTable(hooks) } if err != nil && strings.Contains(err.Error(), "out of range") { @@ -258,9 +261,10 @@ func (c *nftContext) createRule(chain *nftables.Chain, set *nftables.Set, denyLog bool, denyLogPrefix string, denyAction string, ) (*nftables.Rule, error) { r := &nftables.Rule{ - Table: c.table, - Chain: chain, - Exprs: []expr.Any{}, + Table: c.table, + Chain: chain, + Exprs: []expr.Any{}, + UserData: []byte(set.Name), } // [ payload load 4b @ network header + 16 => reg 1 ] r.Exprs = append(r.Exprs, &expr.Payload{ @@ -310,21 +314,26 @@ func (c *nftContext) createRule(chain *nftables.Chain, set *nftables.Set, } func (c *nftContext) deleteElementChunk(els []nftables.SetElement) error { - if err := c.conn.SetDeleteElements(c.set, els); err != nil { - return fmt.Errorf("failed to remove ip%s elements from set: %w", c.version, err) - } - - if err := c.conn.Flush(); err != nil { - if len(els) == 1 { - log.Debugf("deleting %s, failed to flush: %s", reprIP(els[0].Key), err) - return nil + //FIXME: only delete IPs from the set they are in + //But this could lead to strange behavior if we have duplicate decisions with different origins + for _, set := range c.sets { + log.Debugf("removing %d ip%s elements from set %s", len(els), c.version, set.Name) + if err := c.conn.SetDeleteElements(set, els); err != nil { + return fmt.Errorf("failed to remove ip%s elements from set: %w", c.version, err) } - log.Debugf("failed to flush chunk of %d elements, will retry each one: %s", len(els), err) + if err := c.conn.Flush(); err != nil { + if len(els) == 1 { + log.Debugf("deleting %s, failed to flush: %s", reprIP(els[0].Key), err) + continue + } - for _, el := range els { - if err := c.deleteElementChunk([]nftables.SetElement{el}); err != nil { - return err + log.Debugf("failed to flush chunk of %d elements, will retry each one: %s", len(els), err) + + for _, el := range els { + if err := c.deleteElementChunk([]nftables.SetElement{el}); err != nil { + return err + } } } } @@ -348,16 +357,26 @@ func (c *nftContext) deleteElements(els []nftables.SetElement) error { return nil } -func (c *nftContext) addElements(els []nftables.SetElement) error { - for _, chunk := range slicetools.Chunks(els, chunkSize) { - log.Debugf("adding %d ip%s elements to set", len(chunk), c.version) +func (c *nftContext) addElements(els map[string][]nftables.SetElement) error { + var setName string - if err := c.conn.SetAddElements(c.set, chunk); err != nil { - return fmt.Errorf("failed to add ip%s elements to set: %w", c.version, err) + for origin, set := range c.sets { + if c.setOnly { + setName = c.blacklists + } else { + setName = fmt.Sprintf("%s-%s", c.blacklists, origin) } + log.Debugf("Using %s as origin | len of IPs: %d | set name is %s", origin, len(els[origin]), setName) + for _, chunk := range slicetools.Chunks(els[origin], chunkSize) { + log.Debugf("adding %d ip%s elements to set %s", len(chunk), c.version, setName) - if err := c.conn.Flush(); err != nil { - return fmt.Errorf("failed to flush ip%s conn: %w", c.version, err) + if err := c.conn.SetAddElements(set, chunk); err != nil { + return fmt.Errorf("failed to add ip%s elements to set: %w", c.version, err) + } + + if err := c.conn.Flush(); err != nil { + return fmt.Errorf("failed to flush ip%s conn: %w", c.version, err) + } } } @@ -371,8 +390,8 @@ func (c *nftContext) shutDown() error { if c.setOnly { // Flush blacklist4 set empty - log.Infof("flushing '%s' set in '%s' table", c.set.Name, c.table.Name) - c.conn.FlushSet(c.set) + log.Infof("flushing '%s' set in '%s' table", c.sets[c.blacklists].Name, c.table.Name) + c.conn.FlushSet(c.sets[c.blacklists]) } else { log.Infof("removing '%s' table", c.table.Name) c.conn.DelTable(c.table) diff --git a/pkg/pf/metrics.go b/pkg/pf/metrics.go index cbe56f43..96988c53 100644 --- a/pkg/pf/metrics.go +++ b/pkg/pf/metrics.go @@ -6,8 +6,8 @@ import ( "slices" "strconv" "strings" - "time" + "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics" @@ -110,35 +110,31 @@ func (pf *pf) CollectMetrics() { tables = append(tables, pf.inet6.table) } - t := time.NewTicker(metrics.MetricCollectionInterval) + cmd := execPfctl("", "-v", "-sr") - for range t.C { - cmd := execPfctl("", "-v", "-sr") + out, err := cmd.Output() + if err != nil { + log.Errorf("failed to run 'pfctl -v -sr': %s", err) + return + } - out, err := cmd.Output() - if err != nil { - log.Errorf("failed to run 'pfctl -v -sr': %s", err) + reader := strings.NewReader(string(out)) + stats := parseMetrics(reader, tables) + bannedIPs := 0 + + for _, table := range tables { + st, ok := stats[table] + if !ok { continue } - reader := strings.NewReader(string(out)) - stats := parseMetrics(reader, tables) - bannedIPs := 0 + droppedPackets += float64(st.packets) + droppedBytes += float64(st.bytes) - for _, table := range tables { - st, ok := stats[table] - if !ok { - continue - } - - droppedPackets += float64(st.packets) - droppedBytes += float64(st.bytes) - - bannedIPs += pf.countIPs(table) - } - - metrics.TotalDroppedPackets.Set(droppedPackets) - metrics.TotalDroppedBytes.Set(droppedBytes) - metrics.TotalActiveBannedIPs.Set(float64(bannedIPs)) + bannedIPs += pf.countIPs(table) } + + metrics.TotalDroppedPackets.With(prometheus.Labels{"ip_type": "ipv4", "origin": ""}).Set(droppedPackets) + metrics.TotalDroppedBytes.With(prometheus.Labels{"ip_type": "ipv4", "origin": ""}).Set(droppedBytes) + metrics.TotalActiveBannedIPs.With(prometheus.Labels{"ip_type": "ipv4", "origin": ""}).Set(float64(bannedIPs)) } diff --git a/test/backends/iptables/test_iptables.py b/test/backends/iptables/test_iptables.py index 8f986036..9edda8d2 100644 --- a/test/backends/iptables/test_iptables.py +++ b/test/backends/iptables/test_iptables.py @@ -15,12 +15,12 @@ BINARY_PATH = PROJECT_ROOT.joinpath("crowdsec-firewall-bouncer") CONFIG_PATH = SCRIPT_DIR.joinpath("crowdsec-firewall-bouncer.yaml") -SET_NAME_IPV4 = "crowdsec-blacklists" -SET_NAME_IPV6 = "crowdsec6-blacklists" +SET_NAME_IPV4 = "crowdsec-blacklists-0" +SET_NAME_IPV6 = "crowdsec6-blacklists-0" +RULES_CHAIN_NAME = "CROWDSEC_CHAIN" CHAIN_NAME = "INPUT" - class TestIPTables(unittest.TestCase): def setUp(self): self.fb = subprocess.Popen([BINARY_PATH, "-c", CONFIG_PATH]) @@ -32,52 +32,42 @@ def tearDown(self): self.fb.kill() self.fb.wait() self.lapi.stop() - run_cmd( - "iptables", - "-D", - CHAIN_NAME, - "-m", - "set", - "--match-set", - SET_NAME_IPV4, - "src", - "-j", - "DROP", - ignore_error=True, - ) - run_cmd( - "ip6tables", - "-D", - CHAIN_NAME, - "-m", - "set", - "--match-set", - SET_NAME_IPV6, - "src", - "-j", - "DROP", - ignore_error=True, - ) - run_cmd("ipset", "destroy", SET_NAME_IPV4, ignore_error=True) - run_cmd("ipset", "destroy", SET_NAME_IPV6, ignore_error=True) def test_table_rule_set_are_created(self): + d1 = generate_n_decisions(3) + d2 = generate_n_decisions(1, ipv4=False) + self.lapi.ds.insert_decisions(d1 + d2) sleep(3) # IPV4 Chain - output = run_cmd("iptables", "-L", CHAIN_NAME) + # Check the rules with the sets + output = run_cmd("iptables", "-L", RULES_CHAIN_NAME) rules = [line for line in output.split("\n") if SET_NAME_IPV4 in line] self.assertEqual(len(rules), 1) assert f"match-set {SET_NAME_IPV4} src" in rules[0] + # Check the JUMP to CROWDSEC_CHAIN + output = run_cmd("iptables", "-L", CHAIN_NAME) + rules = [line for line in output.split("\n") if RULES_CHAIN_NAME in line] + + self.assertEqual(len(rules), 1) + assert f"{RULES_CHAIN_NAME}" in rules[0] + # IPV6 Chain - output = run_cmd("ip6tables", "-L", CHAIN_NAME) + output = run_cmd("ip6tables", "-L", RULES_CHAIN_NAME) rules = [line for line in output.split("\n") if SET_NAME_IPV6 in line] self.assertEqual(len(rules), 1) assert f"match-set {SET_NAME_IPV6} src" in rules[0] + # Check the JUMP to CROWDSEC_CHAIN + output = run_cmd("ip6tables", "-L", CHAIN_NAME) + rules = [line for line in output.split("\n") if RULES_CHAIN_NAME in line] + + self.assertEqual(len(rules), 1) + assert f"{RULES_CHAIN_NAME}" in rules[0] + output = run_cmd("ipset", "list") assert SET_NAME_IPV6 in output diff --git a/test/backends/nftables/test_nftables.py b/test/backends/nftables/test_nftables.py index 92ab7f04..b7f00f7f 100644 --- a/test/backends/nftables/test_nftables.py +++ b/test/backends/nftables/test_nftables.py @@ -31,6 +31,9 @@ def tearDown(self): run_cmd("nft", "delete", "table", "ip6", "crowdsec6", ignore_error=True) def test_table_rule_set_are_created(self): + d1 = generate_n_decisions(3) + d2 = generate_n_decisions(1, ipv4=False) + self.lapi.ds.insert_decisions(d1 + d2) sleep(1) output = json.loads(run_cmd("nft", "-j", "list", "tables")) tables = { @@ -48,7 +51,7 @@ def test_table_rule_set_are_created(self): for node in output["nftables"] if "set" in node } - assert ("ip", "crowdsec-blacklists", "ipv4_addr") in sets + assert ("ip", "crowdsec-blacklists-script", "ipv4_addr") in sets rules = { node["rule"]["chain"] for node in output["nftables"] if "rule" in node } # maybe stricter check ? @@ -62,7 +65,7 @@ def test_table_rule_set_are_created(self): for node in output["nftables"] if "set" in node } - assert ("ip6", "crowdsec6-blacklists", "ipv6_addr") in sets + assert ("ip6", "crowdsec6-blacklists-script", "ipv6_addr") in sets rules = { node["rule"]["chain"] for node in output["nftables"] if "rule" in node @@ -74,24 +77,24 @@ def test_duplicate_decisions_across_decision_stream(self): d1, d2, d3 = generate_n_decisions(3, dup_count=1) self.lapi.ds.insert_decisions([d1]) sleep(1) - self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists"), {"0.0.0.0"}) + self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script"), {"0.0.0.0"}) self.lapi.ds.insert_decisions([d2, d3]) sleep(1) assert self.fb.poll() is None self.assertEqual( - get_set_elements("ip", "crowdsec", "crowdsec-blacklists"), {"0.0.0.0", "0.0.0.1"} + get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script"), {"0.0.0.0", "0.0.0.1"} ) self.lapi.ds.delete_decision_by_id(d1["id"]) self.lapi.ds.delete_decision_by_id(d2["id"]) sleep(1) - self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists"), set()) + self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script"), set()) assert self.fb.poll() is None self.lapi.ds.delete_decision_by_id(d3["id"]) sleep(1) - self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists"), set()) + self.assertEqual(get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script"), set()) assert self.fb.poll() is None def test_decision_insertion_deletion_ipv4(self): @@ -100,7 +103,7 @@ def test_decision_insertion_deletion_ipv4(self): self.lapi.ds.insert_decisions(decisions) sleep(1) # let the bouncer insert the decisions - set_elements = get_set_elements("ip", "crowdsec", "crowdsec-blacklists") + set_elements = get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script") self.assertEqual(len(set_elements), total_decisions - duplicate_decisions) assert {i["value"] for i in decisions} == set_elements assert "0.0.0.0" in set_elements @@ -108,7 +111,7 @@ def test_decision_insertion_deletion_ipv4(self): self.lapi.ds.delete_decisions_by_ip("0.0.0.0") sleep(1) - set_elements = get_set_elements("ip", "crowdsec", "crowdsec-blacklists") + set_elements = get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script") assert {i["value"] for i in decisions if i["value"] != "0.0.0.0"} == set_elements assert len(set_elements) == total_decisions - duplicate_decisions - 1 assert "0.0.0.0" not in set_elements @@ -119,7 +122,7 @@ def test_decision_insertion_deletion_ipv6(self): self.lapi.ds.insert_decisions(decisions) sleep(1) - set_elements = get_set_elements("ip6", "crowdsec6", "crowdsec6-blacklists") + set_elements = get_set_elements("ip6", "crowdsec6", "crowdsec6-blacklists-script") set_elements = set(map(ip_address, set_elements)) assert len(set_elements) == total_decisions - duplicate_decisions assert {ip_address(i["value"]) for i in decisions} == set_elements @@ -128,7 +131,7 @@ def test_decision_insertion_deletion_ipv6(self): self.lapi.ds.delete_decisions_by_ip("::1:0:3") sleep(1) - set_elements = get_set_elements("ip6", "crowdsec6", "crowdsec6-blacklists") + set_elements = get_set_elements("ip6", "crowdsec6", "crowdsec6-blacklists-script") set_elements = set(map(ip_address, set_elements)) self.assertEqual(len(set_elements), total_decisions - duplicate_decisions - 1) assert ( @@ -154,7 +157,7 @@ def test_longest_decision_insertion(self): ] self.lapi.ds.insert_decisions(decisions) sleep(1) - elems = get_set_elements("ip", "crowdsec", "crowdsec-blacklists", with_timeout=True) + elems = get_set_elements("ip", "crowdsec", "crowdsec-blacklists-script", with_timeout=True) assert len(elems) == 1 elems = list(elems) assert elems[0][0] == "123.45.67.12" diff --git a/test/bouncer/test_iptables_deny_action.py b/test/bouncer/test_iptables_deny_action.py index 1e93df10..c905ebbb 100644 --- a/test/bouncer/test_iptables_deny_action.py +++ b/test/bouncer/test_iptables_deny_action.py @@ -9,7 +9,7 @@ def test_iptables_deny_action(bouncer, fw_cfg_factory): fw.wait_for_lines_fnmatch([ "*using 'DROP' as deny_action*", ]) - fw.proc.wait(timeout=0.2) + fw.proc.wait(timeout=5) assert not fw.proc.is_running() cfg['deny_action'] = 'drop' @@ -18,7 +18,7 @@ def test_iptables_deny_action(bouncer, fw_cfg_factory): fw.wait_for_lines_fnmatch([ "*using 'DROP' as deny_action*", ]) - fw.proc.wait(timeout=0.2) + fw.proc.wait(timeout=5) assert not fw.proc.is_running() cfg['deny_action'] = 'reject' @@ -27,7 +27,7 @@ def test_iptables_deny_action(bouncer, fw_cfg_factory): fw.wait_for_lines_fnmatch([ "*using 'REJECT' as deny_action*", ]) - fw.proc.wait(timeout=0.2) + fw.proc.wait(timeout=5) assert not fw.proc.is_running() cfg['deny_action'] = 'tarpit' @@ -36,7 +36,7 @@ def test_iptables_deny_action(bouncer, fw_cfg_factory): fw.wait_for_lines_fnmatch([ "*using 'TARPIT' as deny_action*", ]) - fw.proc.wait(timeout=0.2) + fw.proc.wait(timeout=5) assert not fw.proc.is_running() cfg['deny_action'] = 'somethingelse' @@ -45,5 +45,5 @@ def test_iptables_deny_action(bouncer, fw_cfg_factory): fw.wait_for_lines_fnmatch([ "*invalid deny_action 'somethingelse', must be one of DROP, REJECT, TARPIT*", ]) - fw.proc.wait(timeout=0.2) + fw.proc.wait(timeout=5) assert not fw.proc.is_running()