feat: add prometheus metrics extension

README.md

@@ -4,7 +4,7 @@
 
 # hoglet
 
-Simple low-overhead, circuit breaker library.
+Simple low-overhead circuit breaker library.
 
 ## Usage
 

extensions/prometheus/go.mod

@@ -0,0 +1,26 @@
+module github.com/exaring/hoglet/extensions/prometheus
+
+go 1.21.3
+
+require (
+	github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec
+	github.com/prometheus/client_golang v1.17.0
+	github.com/stretchr/testify v1.8.2
+)
+
+require (
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/kr/text v0.2.0 // indirect
+	github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect
+	github.com/prometheus/common v0.44.0 // indirect
+	github.com/prometheus/procfs v0.11.1 // indirect
+	golang.org/x/sync v0.5.0 // indirect
+	golang.org/x/sys v0.11.0 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

extensions/prometheus/go.sum

@@ -0,0 +1,58 @@
+github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec h1:4ITVxT5N5odnIn5f8qJlFLHu5KnHl87vfiOIOVSGhuY=
+github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec/go.mod h1:05lKY5CjhK3UHc9hr6VZdU6PAdmXumJksCVPdD6yrEQ=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
+github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo=
+github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q=
+github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM=
+github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU=
+github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY=
+github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY=
+github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI=
+github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY=
+github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8=
+github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE=
+golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8=
+google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

extensions/prometheus/prometheus.go

@@ -0,0 +1,140 @@
+package hogprom
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"strconv"
+	"time"
+
+	"github.com/exaring/hoglet"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "hoglet"
+)
+
+// WithPrometheusMetrics returns a [hoglet.BreakerMiddleware] that registers prometheus metrics for the circuit breaker.
+//
+// ⚠️ Note: the provided name must be unique across all hoglet instances using the same registerer.
+func WithPrometheusMetrics(name string, reg prometheus.Registerer) hoglet.BreakerMiddleware {
+	return func(next hoglet.ObserverFactory) (hoglet.ObserverFactory, error) {
+		callDurations := prometheus.NewHistogramVec(
+			prometheus.HistogramOpts{
+				Namespace: namespace,
+				Subsystem: "circuit",
+				Name:      "call_durations_seconds",
+				Help:      "Call durations in seconds",
+				ConstLabels: prometheus.Labels{
+					"circuit": name,
+				},
+			},
+			[]string{"error"},
+		)
+
+		droppedCalls := prometheus.NewCounterVec(
+			prometheus.CounterOpts{
+				Namespace: namespace,
+				Subsystem: "circuit",
+				Name:      "dropped_calls_total",
+				Help:      "Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)",
+				ConstLabels: prometheus.Labels{
+					"circuit": name,
+				},
+			},
+			[]string{"cause"},
+		)
+
+		inflightCalls := prometheus.NewGauge(
+			prometheus.GaugeOpts{
+				Namespace: namespace,
+				Subsystem: "circuit",
+				Name:      "inflight_calls_current",
+				Help:      "Current number of calls in-flight",
+				ConstLabels: prometheus.Labels{
+					"circuit": name,
+				},
+			},
+		)
+
+		for _, c := range []prometheus.Collector{
+			callDurations,
+			droppedCalls,
+			inflightCalls,
+		} {
+			if err := reg.Register(c); err != nil {
+				return nil, fmt.Errorf("hoglet: registering collector: %w", err)
+			}
+		}
+
+		return &prometheusObserverFactory{
+			next: next,
+
+			timesource: wallclock{},
+
+			callDurations: callDurations,
+			droppedCalls:  droppedCalls,
+			inflightCalls: inflightCalls,
+		}, nil
+	}
+}
+
+type prometheusObserverFactory struct {
+	next hoglet.ObserverFactory
+
+	timesource timesource
+
+	callDurations *prometheus.HistogramVec
+	droppedCalls  *prometheus.CounterVec
+	inflightCalls prometheus.Gauge
+}
+
+func (pos *prometheusObserverFactory) ObserverForCall(ctx context.Context, state hoglet.State) (hoglet.Observer, error) {
+	o, err := pos.next.ObserverForCall(ctx, state)
+	if err != nil {
+		pos.droppedCalls.WithLabelValues(errToCause(err)).Inc()
+		return nil, err
+	}
+	start := pos.timesource.Now()
+	pos.inflightCalls.Inc()
+	return hoglet.ObserverFunc(func(b bool) {
+		pos.callDurations.WithLabelValues(strconv.FormatBool(b)).Observe(pos.timesource.Since(start).Seconds())
+		pos.inflightCalls.Dec()
+		o.Observe(b)
+	}), nil
+}
+
+// errToCause converts known circuit errors to metric labels.
+func errToCause(err error) string {
+	switch err {
+	case hoglet.ErrCircuitOpen:
+		return "circuit_open"
+	case hoglet.ErrConcurrencyLimitReached:
+		return "concurrency_limit"
+	default:
+		// leave the errors.Is check as last, since it carries a performance penalty
+		if errors.Is(err, context.Canceled) {
+			return "context_canceled"
+		} else if errors.Is(err, context.DeadlineExceeded) {
+			return "deadline_exceeded"
+		}
+		return "other"
+	}
+}
+
+type timesource interface {
+	Now() time.Time
+	Since(time.Time) time.Duration
+}
+
+// wallclock wraps time.Now/time.Since to allow mocking
+type wallclock struct{}
+
+func (wallclock) Now() time.Time {
+	return time.Now()
+}
+
+func (wallclock) Since(t time.Time) time.Duration {
+	return time.Since(t)
+}

extensions/prometheus/prometheus_test.go

@@ -0,0 +1,124 @@
+package hogprom
+
+import (
+	"context"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/exaring/hoglet"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/testutil"
+	"github.com/stretchr/testify/require"
+)
+
+type mockObserverFactory struct{}
+
+// ObserverForCall implements hoglet.ObserverFactory.
+func (*mockObserverFactory) ObserverForCall(_ context.Context, state hoglet.State) (hoglet.Observer, error) {
+	// this simple factory abuses the state argument to directly control the result of the call
+	switch state {
+	case hoglet.StateClosed:
+		return mockObserver{}, nil
+	case hoglet.StateOpen:
+		return nil, hoglet.ErrCircuitOpen
+	default:
+		panic("not implemented")
+	}
+}
+
+type mockObserver struct{}
+
+func (mockObserver) Observe(bool) {}
+
+type mockTimesource struct {
+	t time.Time
+}
+
+func (m mockTimesource) Now() time.Time {
+	return m.t
+}
+
+func (m mockTimesource) Since(t time.Time) time.Duration {
+	return m.t.Sub(t)
+}
+
+func TestWithPrometheusMetrics(t *testing.T) {
+	reg := prometheus.NewPedanticRegistry()
+	m := WithPrometheusMetrics("test", reg)
+	of, err := m(&mockObserverFactory{})
+	require.NoError(t, err)
+
+	mt := &mockTimesource{time.Now()}
+
+	of.(*prometheusObserverFactory).timesource = mt
+
+	inflightOut0 := `# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight
+                     # TYPE hoglet_circuit_inflight_calls_current gauge
+                     hoglet_circuit_inflight_calls_current{circuit="test"} 0
+                    `
+
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut0)); err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = of.ObserverForCall(context.Background(), hoglet.StateOpen)
+	require.ErrorIs(t, err, hoglet.ErrCircuitOpen)
+
+	droppedOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)
+                    # TYPE hoglet_circuit_dropped_calls_total counter
+                    hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1
+                    # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight
+                    # TYPE hoglet_circuit_inflight_calls_current gauge
+                    hoglet_circuit_inflight_calls_current{circuit="test"} 0
+				   `
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(droppedOut1)); err != nil {
+		t.Fatal(err)
+	}
+
+	o, err := of.ObserverForCall(context.Background(), hoglet.StateClosed)
+	require.NoError(t, err)
+
+	inflightOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)
+                     # TYPE hoglet_circuit_dropped_calls_total counter
+                     hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1
+                     # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight
+                     # TYPE hoglet_circuit_inflight_calls_current gauge
+                     hoglet_circuit_inflight_calls_current{circuit="test"} 1
+				   `
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut1)); err != nil {
+		t.Fatal(err)
+	}
+
+	mt.t = mt.t.Add(time.Second) // move the clock 1 second forward
+
+	o.Observe(true)
+
+	durationsOut1 := `# HELP hoglet_circuit_call_durations_seconds Call durations in seconds
+	                  # TYPE hoglet_circuit_call_durations_seconds histogram
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.005"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.01"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.025"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.05"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.1"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.25"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.5"} 0
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="1"} 1
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="2.5"} 1
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="5"} 1
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="10"} 1
+	                  hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="+Inf"} 1
+	                  hoglet_circuit_call_durations_seconds_sum{circuit="test",error="true"} 1
+	                  hoglet_circuit_call_durations_seconds_count{circuit="test",error="true"} 1
+	                  # HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)
+                      # TYPE hoglet_circuit_dropped_calls_total counter
+                      hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1
+                      # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight
+                      # TYPE hoglet_circuit_inflight_calls_current gauge
+                      hoglet_circuit_inflight_calls_current{circuit="test"} 0
+                     `
+
+	if err := testutil.GatherAndCompare(reg, strings.NewReader(durationsOut1)); err != nil {
+		t.Fatal(err)
+	}
+}