Usage metrics

cmd/root.go

@@ -9,18 +9,21 @@
 	"net/http"
 	"os"
 	"os/signal"
+	"slices"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 	log "github.com/sirupsen/logrus"
-	"golang.org/x/exp/slices"
 	"golang.org/x/sync/errgroup"
 
 	csbouncer "github.com/crowdsecurity/go-cs-bouncer"
 	"github.com/crowdsecurity/go-cs-lib/csdaemon"
 	"github.com/crowdsecurity/go-cs-lib/csstring"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -30,7 +33,9 @@
 	"github.com/crowdsecurity/cs-firewall-bouncer/pkg/metrics"
 )
 
-const name = "crowdsec-firewall-bouncer"
+const bouncerType = "crowdsec-firewall-bouncer"
+
+var metricsInterval time.Duration
 
 func backendCleanup(backend *backend.BackendCTX) {
 	log.Info("Shutting down backend")
@@ -136,6 +141,98 @@
 	}
 }
 
+func getLabelValue(labels []*io_prometheus_client.LabelPair, key string) string {
+
+	for _, label := range labels {
+		if label.GetName() == key {
+			return label.GetValue()
+		}
+	}
+
+	return ""
+}
+
+// metricsUpdater receives a metrics struct with basic data and populates it with the current metrics.
+func metricsUpdater(met *models.RemediationComponentsMetrics) {
+	log.Debugf("Updating metrics")
+
+	//Most of the common fields are set automatically by the metrics provider
+	//We only need to care about the metrics themselves
+
+	promMetrics, err := prometheus.DefaultGatherer.Gather()
+
+	if err != nil {
+		log.Errorf("unable to gather prometheus metrics: %s", err)
+		return
+	}
+
+	met.Metrics = append(met.Metrics, &models.DetailedMetrics{
+		Meta: &models.MetricsMeta{
+			UtcNowTimestamp:   ptr.Of(time.Now().Unix()),
+			WindowSizeSeconds: ptr.Of(int64(metricsInterval.Seconds())),
+		},
+		Items: make([]*models.MetricsDetailItem, 0),
+	})
+
+	for _, metricFamily := range promMetrics {
+		for _, metric := range metricFamily.GetMetric() {
+			switch metricFamily.GetName() {
+			case metrics.ActiveBannedIPsMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				key := origin + ipType
+				log.Debugf("Sending active decisions for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastActiveBannedIPsValue[key], value, metrics.LastActiveBannedIPsValue[key])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("active_decisions"),
+					Value: ptr.Of(value - metrics.LastActiveBannedIPsValue[key]),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("ip"),
+				})
+				metrics.LastActiveBannedIPsValue[key] = value
+			case metrics.DroppedBytesMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				key := origin + ipType
+				log.Debugf("Sending dropped bytes for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedBytesValue[key], value, metrics.LastDroppedBytesValue[key])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("dropped"),
+					Value: ptr.Of(value - metrics.LastDroppedBytesValue[key]),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("byte"),
+				})
+				metrics.LastDroppedBytesValue[key] = value
+			case metrics.DroppedPacketsMetricName:
+				labels := metric.GetLabel()
+				value := metric.GetGauge().GetValue()
+				origin := getLabelValue(labels, "origin")
+				ipType := getLabelValue(labels, "ip_type")
+				key := origin + ipType
+				log.Debugf("Sending dropped packets for %s %s %f | current value: %f | previous value: %f\n", origin, ipType, value-metrics.LastDroppedPacketsValue[key], value, metrics.LastDroppedPacketsValue[key])
+				met.Metrics[0].Items = append(met.Metrics[0].Items, &models.MetricsDetailItem{
+					Name:  ptr.Of("dropped"),
+					Value: ptr.Of(value - metrics.LastDroppedPacketsValue[key]),
+					Labels: map[string]string{
+						"origin":  origin,
+						"ip_type": ipType,
+					},
+					Unit: ptr.Of("packet"),
+				})
+				metrics.LastDroppedPacketsValue[key] = value
+			}
+		}
+	}
+}
+
 func Execute() error {
 	configPath := flag.String("c", "", "path to crowdsec-firewall-bouncer.yaml")
 	verbose := flag.Bool("v", false, "set verbose mode")
@@ -176,7 +273,7 @@
 		log.SetLevel(log.DebugLevel)
 	}
 
-	log.Infof("Starting crowdsec-firewall-bouncer %s", version.String())
+	log.Infof("Starting %s %s", bouncerType, version.String())
 
 	backend, err := backend.NewBackend(config)
 	if err != nil {
@@ -196,7 +293,7 @@
 		return err
 	}
 
-	bouncer.UserAgent = fmt.Sprintf("%s/%s", name, version.String())
+	bouncer.UserAgent = fmt.Sprintf("%s/%s", bouncerType, version.String())
 	if err := bouncer.Init(); err != nil {
 		return fmt.Errorf("unable to configure bouncer: %w", err)
 	}
@@ -217,19 +314,28 @@
 		return errors.New("bouncer stream halted")
 	})
 
-	if config.PrometheusConfig.Enabled {
-		if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode {
-			go backend.CollectMetrics()
+	metricsInterval = *bouncer.MetricsInterval
 
-			if config.Mode == cfg.IpsetMode {
-				prometheus.MustRegister(metrics.TotalActiveBannedIPs)
-			} else {
-				prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs)
-			}
-		}
+	metricsProvider, err := csbouncer.NewMetricsProvider(bouncer.APIClient, bouncerType, metricsInterval, metricsUpdater, log.StandardLogger())
+	if err != nil {
+		return fmt.Errorf("unable to create metrics provider: %w", err)
+	}
+
+	g.Go(func() error {
+		return metricsProvider.Run(ctx)
+	})
 
-		prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError)
+	if config.Mode == cfg.IptablesMode || config.Mode == cfg.NftablesMode || config.Mode == cfg.IpsetMode || config.Mode == cfg.PfMode {
+		go backend.CollectMetrics()
+		if config.Mode == cfg.IpsetMode {
+			prometheus.MustRegister(metrics.TotalActiveBannedIPs)
+		} else {
+			prometheus.MustRegister(metrics.TotalDroppedBytes, metrics.TotalDroppedPackets, metrics.TotalActiveBannedIPs)
+		}
+	}
 
+	prometheus.MustRegister(csbouncer.TotalLAPICalls, csbouncer.TotalLAPIError)
+	if config.PrometheusConfig.Enabled {
 		go func() {
 			http.Handle("/metrics", promhttp.Handler())
 

go.mod

@@ -1,17 +1,19 @@
 module github.com/crowdsecurity/cs-firewall-bouncer
 
-go 1.21
+go 1.22
+
+toolchain go1.22.2
 
 require (
-	github.com/crowdsecurity/crowdsec v1.6.1
-	github.com/crowdsecurity/go-cs-bouncer v0.0.13
-	github.com/crowdsecurity/go-cs-lib v0.0.10
-	github.com/google/nftables v0.1.1-0.20230710063801-8a10f689006b
+	github.com/crowdsecurity/crowdsec v1.6.3-0.20240625082021-0bfbd3e2bcd7
+	github.com/crowdsecurity/go-cs-bouncer v0.0.14-0.20240625082322-2398d05359c7
+	github.com/crowdsecurity/go-cs-lib v0.0.11
+	github.com/google/nftables v0.2.0
 	github.com/prometheus/client_golang v1.17.0
+	github.com/prometheus/client_model v0.5.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/stretchr/testify v1.8.4
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	golang.org/x/sync v0.6.0
+	golang.org/x/sync v0.7.0
 	golang.org/x/sys v0.19.0
 	gopkg.in/natefinch/lumberjack.v2 v2.2.1
 	gopkg.in/yaml.v2 v2.4.0
@@ -21,10 +23,11 @@ require (
 	github.com/antonmedv/expr v1.15.3 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/blackfireio/osinfo v1.0.5 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/fatih/color v1.15.0 // indirect
+	github.com/fatih/color v1.16.0 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
 	github.com/go-openapi/errors v0.20.4 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
@@ -36,20 +39,19 @@ require (
 	github.com/go-openapi/validate v0.22.1 // indirect
 	github.com/goccy/go-yaml v1.11.2 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
-	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
-	github.com/josharian/native v1.0.0 // indirect
+	github.com/josharian/native v1.1.0 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mattn/go-colorable v0.1.13 // indirect
 	github.com/mattn/go-isatty v0.0.20 // indirect
 	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
-	github.com/mdlayher/netlink v1.7.1 // indirect
-	github.com/mdlayher/socket v0.4.0 // indirect
+	github.com/mdlayher/netlink v1.7.2 // indirect
+	github.com/mdlayher/socket v0.5.1 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
 	github.com/prometheus/common v0.45.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
 	go.mongodb.org/mongo-driver v1.12.1 // indirect