From ae3723c437b59017c3e4e00c633789e1138123e5 Mon Sep 17 00:00:00 2001 From: Leo Antunes Date: Wed, 8 Nov 2023 23:50:20 +0100 Subject: [PATCH] feat: add prometheus metrics extension --- README.md | 2 +- extensions/prometheus/go.mod | 26 +++++ extensions/prometheus/go.sum | 58 ++++++++++ extensions/prometheus/prometheus.go | 140 +++++++++++++++++++++++ extensions/prometheus/prometheus_test.go | 124 ++++++++++++++++++++ 5 files changed, 349 insertions(+), 1 deletion(-) create mode 100644 extensions/prometheus/go.mod create mode 100644 extensions/prometheus/go.sum create mode 100644 extensions/prometheus/prometheus.go create mode 100644 extensions/prometheus/prometheus_test.go diff --git a/README.md b/README.md index 3d7be48..8550037 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ # hoglet -Simple low-overhead, circuit breaker library. +Simple low-overhead circuit breaker library. ## Usage diff --git a/extensions/prometheus/go.mod b/extensions/prometheus/go.mod new file mode 100644 index 0000000..ad76496 --- /dev/null +++ b/extensions/prometheus/go.mod @@ -0,0 +1,26 @@ +module github.com/exaring/hoglet/extensions/prometheus + +go 1.21.3 + +require ( + github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec + github.com/prometheus/client_golang v1.17.0 + github.com/stretchr/testify v1.8.2 +) + +require ( + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.11.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/extensions/prometheus/go.sum b/extensions/prometheus/go.sum new file mode 100644 index 0000000..015a088 --- /dev/null +++ b/extensions/prometheus/go.sum @@ -0,0 +1,58 @@ +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec h1:4ITVxT5N5odnIn5f8qJlFLHu5KnHl87vfiOIOVSGhuY= +github.com/exaring/hoglet v0.0.0-20231028194910-dc253709e0ec/go.mod h1:05lKY5CjhK3UHc9hr6VZdU6PAdmXumJksCVPdD6yrEQ= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/extensions/prometheus/prometheus.go b/extensions/prometheus/prometheus.go new file mode 100644 index 0000000..51af463 --- /dev/null +++ b/extensions/prometheus/prometheus.go @@ -0,0 +1,140 @@ +package hogprom + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/exaring/hoglet" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "hoglet" +) + +// WithPrometheusMetrics returns a [hoglet.BreakerMiddleware] that registers prometheus metrics for the circuit breaker. +// +// ⚠️ Note: the provided name must be unique across all hoglet instances using the same registerer. +func WithPrometheusMetrics(name string, reg prometheus.Registerer) hoglet.BreakerMiddleware { + return func(next hoglet.ObserverFactory) (hoglet.ObserverFactory, error) { + callDurations := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: namespace, + Subsystem: "circuit", + Name: "call_durations_seconds", + Help: "Call durations in seconds", + ConstLabels: prometheus.Labels{ + "circuit": name, + }, + }, + []string{"error"}, + ) + + droppedCalls := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: "circuit", + Name: "dropped_calls_total", + Help: "Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function)", + ConstLabels: prometheus.Labels{ + "circuit": name, + }, + }, + []string{"cause"}, + ) + + inflightCalls := prometheus.NewGauge( + prometheus.GaugeOpts{ + Namespace: namespace, + Subsystem: "circuit", + Name: "inflight_calls_current", + Help: "Current number of calls in-flight", + ConstLabels: prometheus.Labels{ + "circuit": name, + }, + }, + ) + + for _, c := range []prometheus.Collector{ + callDurations, + droppedCalls, + inflightCalls, + } { + if err := reg.Register(c); err != nil { + return nil, fmt.Errorf("hoglet: registering collector: %w", err) + } + } + + return &prometheusObserverFactory{ + next: next, + + timesource: wallclock{}, + + callDurations: callDurations, + droppedCalls: droppedCalls, + inflightCalls: inflightCalls, + }, nil + } +} + +type prometheusObserverFactory struct { + next hoglet.ObserverFactory + + timesource timesource + + callDurations *prometheus.HistogramVec + droppedCalls *prometheus.CounterVec + inflightCalls prometheus.Gauge +} + +func (pos *prometheusObserverFactory) ObserverForCall(ctx context.Context, state hoglet.State) (hoglet.Observer, error) { + o, err := pos.next.ObserverForCall(ctx, state) + if err != nil { + pos.droppedCalls.WithLabelValues(errToCause(err)).Inc() + return nil, err + } + start := pos.timesource.Now() + pos.inflightCalls.Inc() + return hoglet.ObserverFunc(func(b bool) { + pos.callDurations.WithLabelValues(strconv.FormatBool(b)).Observe(pos.timesource.Since(start).Seconds()) + pos.inflightCalls.Dec() + o.Observe(b) + }), nil +} + +// errToCause converts known circuit errors to metric labels. +func errToCause(err error) string { + switch err { + case hoglet.ErrCircuitOpen: + return "circuit_open" + case hoglet.ErrConcurrencyLimitReached: + return "concurrency_limit" + default: + // leave the errors.Is check as last, since it carries a performance penalty + if errors.Is(err, context.Canceled) { + return "context_canceled" + } else if errors.Is(err, context.DeadlineExceeded) { + return "deadline_exceeded" + } + return "other" + } +} + +type timesource interface { + Now() time.Time + Since(time.Time) time.Duration +} + +// wallclock wraps time.Now/time.Since to allow mocking +type wallclock struct{} + +func (wallclock) Now() time.Time { + return time.Now() +} + +func (wallclock) Since(t time.Time) time.Duration { + return time.Since(t) +} diff --git a/extensions/prometheus/prometheus_test.go b/extensions/prometheus/prometheus_test.go new file mode 100644 index 0000000..6f740af --- /dev/null +++ b/extensions/prometheus/prometheus_test.go @@ -0,0 +1,124 @@ +package hogprom + +import ( + "context" + "strings" + "testing" + "time" + + "github.com/exaring/hoglet" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" + "github.com/stretchr/testify/require" +) + +type mockObserverFactory struct{} + +// ObserverForCall implements hoglet.ObserverFactory. +func (*mockObserverFactory) ObserverForCall(_ context.Context, state hoglet.State) (hoglet.Observer, error) { + // this simple factory abuses the state argument to directly control the result of the call + switch state { + case hoglet.StateClosed: + return mockObserver{}, nil + case hoglet.StateOpen: + return nil, hoglet.ErrCircuitOpen + default: + panic("not implemented") + } +} + +type mockObserver struct{} + +func (mockObserver) Observe(bool) {} + +type mockTimesource struct { + t time.Time +} + +func (m mockTimesource) Now() time.Time { + return m.t +} + +func (m mockTimesource) Since(t time.Time) time.Duration { + return m.t.Sub(t) +} + +func TestWithPrometheusMetrics(t *testing.T) { + reg := prometheus.NewPedanticRegistry() + m := WithPrometheusMetrics("test", reg) + of, err := m(&mockObserverFactory{}) + require.NoError(t, err) + + mt := &mockTimesource{time.Now()} + + of.(*prometheusObserverFactory).timesource = mt + + inflightOut0 := `# HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight + # TYPE hoglet_circuit_inflight_calls_current gauge + hoglet_circuit_inflight_calls_current{circuit="test"} 0 + ` + + if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut0)); err != nil { + t.Fatal(err) + } + + _, err = of.ObserverForCall(context.Background(), hoglet.StateOpen) + require.ErrorIs(t, err, hoglet.ErrCircuitOpen) + + droppedOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) + # TYPE hoglet_circuit_dropped_calls_total counter + hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 + # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight + # TYPE hoglet_circuit_inflight_calls_current gauge + hoglet_circuit_inflight_calls_current{circuit="test"} 0 + ` + if err := testutil.GatherAndCompare(reg, strings.NewReader(droppedOut1)); err != nil { + t.Fatal(err) + } + + o, err := of.ObserverForCall(context.Background(), hoglet.StateClosed) + require.NoError(t, err) + + inflightOut1 := `# HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) + # TYPE hoglet_circuit_dropped_calls_total counter + hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 + # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight + # TYPE hoglet_circuit_inflight_calls_current gauge + hoglet_circuit_inflight_calls_current{circuit="test"} 1 + ` + if err := testutil.GatherAndCompare(reg, strings.NewReader(inflightOut1)); err != nil { + t.Fatal(err) + } + + mt.t = mt.t.Add(time.Second) // move the clock 1 second forward + + o.Observe(true) + + durationsOut1 := `# HELP hoglet_circuit_call_durations_seconds Call durations in seconds + # TYPE hoglet_circuit_call_durations_seconds histogram + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.005"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.01"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.025"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.05"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.1"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.25"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="0.5"} 0 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="1"} 1 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="2.5"} 1 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="5"} 1 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="10"} 1 + hoglet_circuit_call_durations_seconds_bucket{circuit="test",error="true",le="+Inf"} 1 + hoglet_circuit_call_durations_seconds_sum{circuit="test",error="true"} 1 + hoglet_circuit_call_durations_seconds_count{circuit="test",error="true"} 1 + # HELP hoglet_circuit_dropped_calls_total Total number of calls with an open circuit (i.e.: calls that did not reach the wrapped function) + # TYPE hoglet_circuit_dropped_calls_total counter + hoglet_circuit_dropped_calls_total{cause="circuit_open",circuit="test"} 1 + # HELP hoglet_circuit_inflight_calls_current Current number of calls in-flight + # TYPE hoglet_circuit_inflight_calls_current gauge + hoglet_circuit_inflight_calls_current{circuit="test"} 0 + ` + + if err := testutil.GatherAndCompare(reg, strings.NewReader(durationsOut1)); err != nil { + t.Fatal(err) + } +}