From ecd1932b0290c30dbcee8682cdea0ffa76e4bb7f Mon Sep 17 00:00:00 2001 From: jake-shin0 Date: Mon, 22 Apr 2024 21:31:28 +0900 Subject: [PATCH] [FEED-8337] feat: add goroutine watcher --- autopprof.go | 105 +++++++- autopprof_test.go | 255 +++++++++++++++---- cgroups.go | 31 --- cgroups_mock.go | 78 ------ error.go | 1 - examples/go.mod | 2 +- examples/go.sum | 1 + go.mod | 4 +- go.sum | 17 +- option.go | 7 + profile.go | 18 +- profile_mock.go | 23 +- cgroupv1.go => queryer/cgroupv1.go | 71 +++--- cgroupv1_test.go => queryer/cgroupv1_test.go | 36 +-- cgroupv2.go => queryer/cgroupv2.go | 75 +++--- cgroupv2_test.go => queryer/cgroupv2_test.go | 36 +-- queryer/error.go | 8 + queryer/queryer.go | 39 +++ queryer/queryer_mock.go | 119 +++++++++ cgroups_test.go => queryer/queryer_test.go | 6 +- queue.go => queryer/queue.go | 6 +- queue_test.go => queryer/queue_test.go | 2 +- queryer/runtime.go | 14 + queryer/runtime_test.go | 39 +++ report/report.go | 12 + report/report_mock.go | 26 +- report/slack.go | 27 +- 27 files changed, 746 insertions(+), 312 deletions(-) delete mode 100644 cgroups.go delete mode 100644 cgroups_mock.go rename cgroupv1.go => queryer/cgroupv1.go (90%) rename cgroupv1_test.go => queryer/cgroupv1_test.go (56%) rename cgroupv2.go => queryer/cgroupv2.go (88%) rename cgroupv2_test.go => queryer/cgroupv2_test.go (59%) create mode 100644 queryer/error.go create mode 100644 queryer/queryer.go create mode 100644 queryer/queryer_mock.go rename cgroups_test.go => queryer/queryer_test.go (76%) rename queue.go => queryer/queue.go (93%) rename queue_test.go => queryer/queue_test.go (99%) create mode 100644 queryer/runtime.go create mode 100644 queryer/runtime_test.go diff --git a/autopprof.go b/autopprof.go index 1c45524..37038f7 100644 --- a/autopprof.go +++ b/autopprof.go @@ -7,7 +7,9 @@ import ( "bytes" "context" "fmt" + "github.com/daangn/autopprof/queryer" "log" + "runtime/pprof" "time" "github.com/daangn/autopprof/report" @@ -34,13 +36,22 @@ type autoPprof struct { // Default: 0.75. (mean 75%) memThreshold float64 + // goroutineThreshold is the goroutine count threshold to trigger profile. + // If the goroutine count is over the threshold, the autopprof will + // report the goroutine profile. + // Default: 10000. + goroutineThreshold int + // minConsecutiveOverThreshold is the minimum consecutive // number of over a threshold for reporting profile again. // Default: 12. minConsecutiveOverThreshold int - // queryer is used to query the quota and the cgroup stat. - queryer queryer + // cgroupQueryer is used to query the quota and the cgroup stat. + cgroupQueryer queryer.CgroupsQueryer + + // runtimeQueryer is used to query the runtime stat. + runtimeQueryer queryer.RuntimeQueryer // profiler is used to profile the cpu and the heap memory. profiler profiler @@ -53,8 +64,9 @@ type autoPprof struct { reportBoth bool // Flags to disable the profiling. - disableCPUProf bool - disableMemProf bool + disableCPUProf bool + disableMemProf bool + disableGoroutineProf bool // stopC is the signal channel to stop the watch processes. stopC chan struct{} @@ -65,7 +77,7 @@ var globalAp *autoPprof // Start configures and runs the autopprof process. func Start(opt Option) error { - qryer, err := newQueryer() + cgroupQryer, err := queryer.NewCgroupQueryer() if err != nil { return err } @@ -73,13 +85,18 @@ func Start(opt Option) error { return err } + pprof.Profiles() + runtimeQryer, err := queryer.NewRuntimeQueryer() + profr := newDefaultProfiler(defaultCPUProfilingDuration) ap := &autoPprof{ watchInterval: defaultWatchInterval, cpuThreshold: defaultCPUThreshold, memThreshold: defaultMemThreshold, + goroutineThreshold: defaultGoroutineThreshold, minConsecutiveOverThreshold: defaultMinConsecutiveOverThreshold, - queryer: qryer, + cgroupQueryer: cgroupQryer, + runtimeQueryer: runtimeQryer, profiler: profr, reporter: opt.Reporter, reportBoth: opt.ReportBoth, @@ -93,6 +110,9 @@ func Start(opt Option) error { if opt.MemThreshold != 0 { ap.memThreshold = opt.MemThreshold } + if opt.GoroutineThreshold != 0 { + ap.goroutineThreshold = opt.GoroutineThreshold + } if !ap.disableCPUProf { if err := ap.loadCPUQuota(); err != nil { return err @@ -112,7 +132,7 @@ func Stop() { } func (ap *autoPprof) loadCPUQuota() error { - err := ap.queryer.setCPUQuota() + err := ap.cgroupQueryer.SetCPUQuota() if err == nil { return nil } @@ -134,6 +154,7 @@ func (ap *autoPprof) loadCPUQuota() error { func (ap *autoPprof) watch() { go ap.watchCPUUsage() go ap.watchMemUsage() + go ap.watchGoroutineCount() <-ap.stopC } @@ -149,7 +170,7 @@ func (ap *autoPprof) watchCPUUsage() { for { select { case <-ticker.C: - usage, err := ap.queryer.cpuUsage() + usage, err := ap.cgroupQueryer.CpuUsage() if err != nil { log.Println(err) return @@ -170,7 +191,7 @@ func (ap *autoPprof) watchCPUUsage() { )) } if ap.reportBoth && !ap.disableMemProf { - memUsage, err := ap.queryer.memUsage() + memUsage, err := ap.cgroupQueryer.MemUsage() if err != nil { log.Println(err) return @@ -226,7 +247,7 @@ func (ap *autoPprof) watchMemUsage() { for { select { case <-ticker.C: - usage, err := ap.queryer.memUsage() + usage, err := ap.cgroupQueryer.MemUsage() if err != nil { log.Println(err) return @@ -247,7 +268,7 @@ func (ap *autoPprof) watchMemUsage() { )) } if ap.reportBoth && !ap.disableCPUProf { - cpuUsage, err := ap.queryer.cpuUsage() + cpuUsage, err := ap.cgroupQueryer.CpuUsage() if err != nil { log.Println(err) return @@ -291,6 +312,68 @@ func (ap *autoPprof) reportHeapProfile(memUsage float64) error { return nil } +func (ap *autoPprof) watchGoroutineCount() { + if ap.disableGoroutineProf { + return + } + + ticker := time.NewTicker(ap.watchInterval) + defer ticker.Stop() + + var consecutiveOverThresholdCnt int + for { + select { + case <-ticker.C: + count := ap.runtimeQueryer.GoroutineCount() + + if count < ap.goroutineThreshold { + // Reset the count if the goroutine count goes under the threshold. + consecutiveOverThresholdCnt = 0 + continue + } + + // If goroutine count remains high for a short period of time, no + // duplicate reports are sent. + // This is to prevent the autopprof from sending too many reports. + if consecutiveOverThresholdCnt == 0 { + if err := ap.reportGoroutineProfile(count); err != nil { + log.Println(fmt.Errorf( + "autopprof: failed to report the goroutine profile: %w", err, + )) + } + } + + consecutiveOverThresholdCnt++ + if consecutiveOverThresholdCnt >= ap.minConsecutiveOverThreshold { + // Reset the count and ready to report the cpu profile again. + consecutiveOverThresholdCnt = 0 + } + case <-ap.stopC: + return + } + } +} + +func (ap *autoPprof) reportGoroutineProfile(goroutineCount int) error { + b, err := ap.profiler.profileGoroutine() + if err != nil { + return fmt.Errorf("autopprof: failed to profile the goroutine: %w", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), reportTimeout) + defer cancel() + + ci := report.GoroutineInfo{ + ThresholdCount: ap.goroutineThreshold, + Count: goroutineCount, + } + bReader := bytes.NewReader(b) + if err := ap.reporter.ReportGoroutineProfile(ctx, bReader, ci); err != nil { + return err + } + return nil +} + func (ap *autoPprof) stop() { close(ap.stopC) } diff --git a/autopprof_test.go b/autopprof_test.go index 548cf91..7f34359 100644 --- a/autopprof_test.go +++ b/autopprof_test.go @@ -6,12 +6,12 @@ package autopprof import ( "context" "errors" + "github.com/daangn/autopprof/queryer" + "go.uber.org/mock/gomock" "io" "testing" "time" - "github.com/golang/mock/gomock" - "github.com/daangn/autopprof/report" ) @@ -158,13 +158,13 @@ func TestAutoPprof_loadCPUQuota(t *testing.T) { newAp: func() *autoPprof { ctrl := gomock.NewController(t) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - setCPUQuota(). + SetCPUQuota(). Return(nil) // Means that the quota is set correctly. return &autoPprof{ - queryer: mockQueryer, + cgroupQueryer: mockQueryer, disableCPUProf: false, disableMemProf: false, } @@ -177,13 +177,13 @@ func TestAutoPprof_loadCPUQuota(t *testing.T) { newAp: func() *autoPprof { ctrl := gomock.NewController(t) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - setCPUQuota(). + SetCPUQuota(). Return(ErrV2CPUQuotaUndefined) return &autoPprof{ - queryer: mockQueryer, + cgroupQueryer: mockQueryer, disableCPUProf: false, disableMemProf: false, } @@ -196,13 +196,13 @@ func TestAutoPprof_loadCPUQuota(t *testing.T) { newAp: func() *autoPprof { ctrl := gomock.NewController(t) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - setCPUQuota(). + SetCPUQuota(). Return(ErrV2CPUQuotaUndefined) return &autoPprof{ - queryer: mockQueryer, + cgroupQueryer: mockQueryer, disableCPUProf: false, disableMemProf: true, } @@ -233,9 +233,9 @@ func TestAutoPprof_watchCPUUsage(t *testing.T) { reported bool ) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). DoAndReturn( func() (float64, error) { @@ -269,7 +269,7 @@ func TestAutoPprof_watchCPUUsage(t *testing.T) { disableMemProf: true, watchInterval: 1 * time.Second, cpuThreshold: 0.5, // 50%. - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, stopC: make(chan struct{}), @@ -296,9 +296,9 @@ func TestAutoPprof_watchCPUUsage_consecutive(t *testing.T) { reportedCnt int ) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). DoAndReturn( func() (float64, error) { @@ -333,7 +333,7 @@ func TestAutoPprof_watchCPUUsage_consecutive(t *testing.T) { watchInterval: 1 * time.Second, cpuThreshold: 0.5, // 50%. minConsecutiveOverThreshold: 3, - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, stopC: make(chan struct{}), @@ -390,7 +390,7 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { testCases := []struct { name string fields fields - mockFunc func(*Mockqueryer, *Mockprofiler, *report.MockReporter) + mockFunc func(*queryer.MockCgroupsQueryer, *Mockprofiler, *report.MockReporter) }{ { name: "reportBoth: true", @@ -401,10 +401,10 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { disableMemProf: false, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). Return(0.6, nil), @@ -422,7 +422,7 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { Return(nil), mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). Return(0.2, nil), @@ -450,10 +450,10 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { disableMemProf: true, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). Return(0.6, nil), @@ -481,10 +481,10 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { disableMemProf: false, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). Return(0.6, nil), @@ -508,7 +508,7 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockProfiler := NewMockprofiler(ctrl) mockReporter := report.NewMockReporter(ctrl) @@ -516,7 +516,7 @@ func TestAutoPprof_watchCPUUsage_reportBoth(t *testing.T) { watchInterval: tc.fields.watchInterval, cpuThreshold: tc.fields.cpuThreshold, memThreshold: 0.5, // 50%. - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, reportBoth: tc.fields.reportBoth, @@ -543,9 +543,9 @@ func TestAutoPprof_watchMemUsage(t *testing.T) { reported bool ) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). DoAndReturn( func() (float64, error) { @@ -577,7 +577,7 @@ func TestAutoPprof_watchMemUsage(t *testing.T) { disableCPUProf: true, watchInterval: 1 * time.Second, memThreshold: 0.2, // 20%. - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, stopC: make(chan struct{}), @@ -604,9 +604,9 @@ func TestAutoPprof_watchMemUsage_consecutive(t *testing.T) { reportedCnt int ) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). DoAndReturn( func() (float64, error) { @@ -641,7 +641,7 @@ func TestAutoPprof_watchMemUsage_consecutive(t *testing.T) { watchInterval: 1 * time.Second, memThreshold: 0.2, // 20%. minConsecutiveOverThreshold: 3, - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, stopC: make(chan struct{}), @@ -698,7 +698,7 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { testCases := []struct { name string fields fields - mockFunc func(*Mockqueryer, *Mockprofiler, *report.MockReporter) + mockFunc func(*queryer.MockCgroupsQueryer, *Mockprofiler, *report.MockReporter) }{ { name: "reportBoth: true", @@ -709,10 +709,10 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { disableCPUProf: false, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). Return(0.6, nil), @@ -730,7 +730,7 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { Return(nil), mockQueryer.EXPECT(). - cpuUsage(). + CpuUsage(). AnyTimes(). Return(0.2, nil), @@ -758,10 +758,10 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { disableCPUProf: true, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). Return(0.6, nil), @@ -789,10 +789,10 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { disableCPUProf: false, stopC: make(chan struct{}), }, - mockFunc: func(mockQueryer *Mockqueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { + mockFunc: func(mockQueryer *queryer.MockCgroupsQueryer, mockProfiler *Mockprofiler, mockReporter *report.MockReporter) { gomock.InOrder( mockQueryer.EXPECT(). - memUsage(). + MemUsage(). AnyTimes(). Return(0.6, nil), @@ -816,7 +816,7 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { t.Run(tc.name, func(t *testing.T) { ctrl := gomock.NewController(t) - mockQueryer := NewMockqueryer(ctrl) + mockQueryer := queryer.NewMockCgroupsQueryer(ctrl) mockProfiler := NewMockprofiler(ctrl) mockReporter := report.NewMockReporter(ctrl) @@ -824,7 +824,7 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { watchInterval: tc.fields.watchInterval, cpuThreshold: 0.5, // 50%. memThreshold: tc.fields.memThreshold, - queryer: mockQueryer, + cgroupQueryer: mockQueryer, profiler: mockProfiler, reporter: mockReporter, reportBoth: tc.fields.reportBoth, @@ -843,6 +843,159 @@ func TestAutoPprof_watchMemUsage_reportBoth(t *testing.T) { } } +func TestAutoPprof_watchGoroutineCount(t *testing.T) { + ctrl := gomock.NewController(t) + + var ( + profiled bool + reported bool + ) + + mockQueryer := queryer.NewMockRuntimeQueryer(ctrl) + mockQueryer.EXPECT(). + GoroutineCount(). + AnyTimes(). + DoAndReturn( + func() (int, error) { + return 200, nil + }, + ) + + mockProfiler := NewMockprofiler(ctrl) + mockProfiler.EXPECT(). + profileGoroutine(). + DoAndReturn( + func() ([]byte, error) { + profiled = true + return []byte("prof"), nil + }, + ) + + mockReporter := report.NewMockReporter(ctrl) + mockReporter.EXPECT(). + ReportGoroutineProfile(gomock.Any(), gomock.Any(), gomock.Any()). + DoAndReturn( + func(_ context.Context, _ io.Reader, _ report.MemInfo) error { + reported = true + return nil + }, + ) + + ap := &autoPprof{ + disableCPUProf: true, + watchInterval: 1 * time.Second, + goroutineThreshold: 100, + runtimeQueryer: mockQueryer, + profiler: mockProfiler, + reporter: mockReporter, + stopC: make(chan struct{}), + } + + go ap.watchGoroutineCount() + t.Cleanup(func() { ap.stop() }) + + // Wait for profiling and reporting. + time.Sleep(1050 * time.Millisecond) + if !profiled { + t.Errorf("goroutine count is not profiled") + } + if !reported { + t.Errorf("goroutine count is not reported") + } +} + +func TestAutoPprof_watchGoroutineCount_consecutive(t *testing.T) { + ctrl := gomock.NewController(t) + + var ( + profiledCnt int + reportedCnt int + ) + + mockQueryer := queryer.NewMockRuntimeQueryer(ctrl) + mockQueryer.EXPECT(). + GoroutineCount(). + AnyTimes(). + DoAndReturn( + func() (float64, error) { + return 200, nil + }, + ) + + mockProfiler := NewMockprofiler(ctrl) + mockProfiler.EXPECT(). + profileGoroutine(). + AnyTimes(). + DoAndReturn( + func() ([]byte, error) { + profiledCnt++ + return []byte("prof"), nil + }, + ) + + mockReporter := report.NewMockReporter(ctrl) + mockReporter.EXPECT(). + ReportGoroutineProfile(gomock.Any(), gomock.Any(), gomock.Any()). + AnyTimes(). + DoAndReturn( + func(_ context.Context, _ io.Reader, _ report.MemInfo) error { + reportedCnt++ + return nil + }, + ) + + ap := &autoPprof{ + disableCPUProf: true, + watchInterval: 1 * time.Second, + memThreshold: 0.2, // 20%. + goroutineThreshold: 100, + minConsecutiveOverThreshold: 3, + runtimeQueryer: mockQueryer, + profiler: mockProfiler, + reporter: mockReporter, + stopC: make(chan struct{}), + } + + go ap.watchGoroutineCount() + t.Cleanup(func() { ap.stop() }) + + // Wait for profiling and reporting. + time.Sleep(1050 * time.Millisecond) + if profiledCnt != 1 { + t.Errorf("goroutine count is profiled %d times, want 1", profiledCnt) + } + if reportedCnt != 1 { + t.Errorf("goroutine count is reported %d times, want 1", reportedCnt) + } + + time.Sleep(1050 * time.Millisecond) + // 2nd time. It shouldn't be profiled and reported. + if profiledCnt != 1 { + t.Errorf("goroutine count is profiled %d times, want 1", profiledCnt) + } + if reportedCnt != 1 { + t.Errorf("goroutine count is reported %d times, want 1", reportedCnt) + } + + time.Sleep(1050 * time.Millisecond) + // 3rd time. It shouldn't be profiled and reported. + if profiledCnt != 1 { + t.Errorf("goroutine count is profiled %d times, want 1", profiledCnt) + } + if reportedCnt != 1 { + t.Errorf("goroutine count is reported %d times, want 1", reportedCnt) + } + + time.Sleep(1050 * time.Millisecond) + // 4th time. Now it should be profiled and reported. + if profiledCnt != 2 { + t.Errorf("goroutine count is profiled %d times, want 2", profiledCnt) + } + if reportedCnt != 2 { + t.Errorf("goroutine count is reported %d times, want 2", reportedCnt) + } +} + func fib(n int) int64 { if n <= 1 { return int64(n) @@ -858,13 +1011,13 @@ func BenchmarkLightJob(b *testing.B) { func BenchmarkLightJobWithWatchCPUUsage(b *testing.B) { var ( - qryer, _ = newQueryer() + qryer, _ = queryer.NewCgroupQueryer() ticker = time.NewTicker(defaultWatchInterval) ) for i := 0; i < b.N; i++ { select { case <-ticker.C: - _, _ = qryer.cpuUsage() + _, _ = qryer.CpuUsage() default: fib(10) } @@ -873,13 +1026,13 @@ func BenchmarkLightJobWithWatchCPUUsage(b *testing.B) { func BenchmarkLightJobWithWatchMemUsage(b *testing.B) { var ( - qryer, _ = newQueryer() + qryer, _ = queryer.NewCgroupQueryer() ticker = time.NewTicker(defaultWatchInterval) ) for i := 0; i < b.N; i++ { select { case <-ticker.C: - _, _ = qryer.memUsage() + _, _ = qryer.MemUsage() default: fib(10) } @@ -894,13 +1047,13 @@ func BenchmarkHeavyJob(b *testing.B) { func BenchmarkHeavyJobWithWatchCPUUsage(b *testing.B) { var ( - qryer, _ = newQueryer() + qryer, _ = queryer.NewCgroupQueryer() ticker = time.NewTicker(defaultWatchInterval) ) for i := 0; i < b.N; i++ { select { case <-ticker.C: - _, _ = qryer.cpuUsage() + _, _ = qryer.CpuUsage() default: fib(24) } @@ -909,13 +1062,13 @@ func BenchmarkHeavyJobWithWatchCPUUsage(b *testing.B) { func BenchmarkHeavyJobWithWatchMemUsage(b *testing.B) { var ( - qryer, _ = newQueryer() + qryer, _ = queryer.NewCgroupQueryer() ticker = time.NewTicker(defaultWatchInterval) ) for i := 0; i < b.N; i++ { select { case <-ticker.C: - _, _ = qryer.memUsage() + _, _ = qryer.MemUsage() default: fib(24) } diff --git a/cgroups.go b/cgroups.go deleted file mode 100644 index 25d1043..0000000 --- a/cgroups.go +++ /dev/null @@ -1,31 +0,0 @@ -//go:build linux -// +build linux - -package autopprof - -import ( - "github.com/containerd/cgroups" -) - -//go:generate mockgen -source=cgroups.go -destination=cgroups_mock.go -package=autopprof - -const ( - cpuUsageSnapshotQueueSize = 24 // 24 * 5s = 2 minutes. -) - -type queryer interface { - cpuUsage() (float64, error) - memUsage() (float64, error) - - setCPUQuota() error -} - -func newQueryer() (queryer, error) { - switch cgroups.Mode() { - case cgroups.Legacy: - return newCgroupsV1(), nil - case cgroups.Hybrid, cgroups.Unified: - return newCgroupsV2(), nil - } - return nil, ErrCgroupsUnavailable -} diff --git a/cgroups_mock.go b/cgroups_mock.go deleted file mode 100644 index 2f51c06..0000000 --- a/cgroups_mock.go +++ /dev/null @@ -1,78 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: cgroups.go - -// Package autopprof is a generated GoMock package. -package autopprof - -import ( - reflect "reflect" - - gomock "github.com/golang/mock/gomock" -) - -// Mockqueryer is a mock of queryer interface. -type Mockqueryer struct { - ctrl *gomock.Controller - recorder *MockqueryerMockRecorder -} - -// MockqueryerMockRecorder is the mock recorder for Mockqueryer. -type MockqueryerMockRecorder struct { - mock *Mockqueryer -} - -// NewMockqueryer creates a new mock instance. -func NewMockqueryer(ctrl *gomock.Controller) *Mockqueryer { - mock := &Mockqueryer{ctrl: ctrl} - mock.recorder = &MockqueryerMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *Mockqueryer) EXPECT() *MockqueryerMockRecorder { - return m.recorder -} - -// cpuUsage mocks base method. -func (m *Mockqueryer) cpuUsage() (float64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "cpuUsage") - ret0, _ := ret[0].(float64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// cpuUsage indicates an expected call of cpuUsage. -func (mr *MockqueryerMockRecorder) cpuUsage() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "cpuUsage", reflect.TypeOf((*Mockqueryer)(nil).cpuUsage)) -} - -// memUsage mocks base method. -func (m *Mockqueryer) memUsage() (float64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "memUsage") - ret0, _ := ret[0].(float64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// memUsage indicates an expected call of memUsage. -func (mr *MockqueryerMockRecorder) memUsage() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "memUsage", reflect.TypeOf((*Mockqueryer)(nil).memUsage)) -} - -// setCPUQuota mocks base method. -func (m *Mockqueryer) setCPUQuota() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "setCPUQuota") - ret0, _ := ret[0].(error) - return ret0 -} - -// setCPUQuota indicates an expected call of setCPUQuota. -func (mr *MockqueryerMockRecorder) setCPUQuota() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "setCPUQuota", reflect.TypeOf((*Mockqueryer)(nil).setCPUQuota)) -} diff --git a/error.go b/error.go index edba05f..72e4a71 100644 --- a/error.go +++ b/error.go @@ -7,7 +7,6 @@ var ( ErrUnsupportedPlatform = fmt.Errorf( "autopprof: unsupported platform (only Linux is supported)", ) - ErrCgroupsUnavailable = fmt.Errorf("autopprof: cgroups is unavailable") ErrInvalidCPUThreshold = fmt.Errorf( "autopprof: cpu threshold value must be between 0 and 1", ) diff --git a/examples/go.mod b/examples/go.mod index d5a531e..647ae4f 100644 --- a/examples/go.mod +++ b/examples/go.mod @@ -17,5 +17,5 @@ require ( github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/sirupsen/logrus v1.8.1 // indirect github.com/slack-go/slack v0.11.3 // indirect - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect + golang.org/x/sys v0.1.0 // indirect ) diff --git a/examples/go.sum b/examples/go.sum index feeb5c6..ff95b39 100644 --- a/examples/go.sum +++ b/examples/go.sum @@ -60,6 +60,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= diff --git a/go.mod b/go.mod index 4935b5c..ed94c5a 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.19 require ( github.com/containerd/cgroups v1.0.4 - github.com/golang/mock v1.6.0 github.com/slack-go/slack v0.11.3 + go.uber.org/mock v0.4.0 ) require ( @@ -17,5 +17,5 @@ require ( github.com/gorilla/websocket v1.4.2 // indirect github.com/opencontainers/runtime-spec v1.0.2 // indirect github.com/sirupsen/logrus v1.8.1 // indirect - golang.org/x/sys v0.0.0-20210510120138-977fb7262007 // indirect + golang.org/x/sys v0.1.0 // indirect ) diff --git a/go.sum b/go.sum index df937ab..373ee32 100644 --- a/go.sum +++ b/go.sum @@ -4,7 +4,6 @@ github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= @@ -26,7 +25,6 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -40,23 +38,20 @@ github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/slack-go/slack v0.11.3 h1:GN7revxEMax4amCc3El9a+9SGnjmBvSUobs0QnO6ZO8= github.com/slack-go/slack v0.11.3/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= -go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/mock v0.4.0 h1:VcM4ZOtdbR4f6VXfiOpwpVJDL6lCReaZ6mw31wqh7KU= +go.uber.org/mock v0.4.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -73,24 +68,20 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210510120138-977fb7262007 h1:gG67DSER+11cZvqIMb8S8bt0vZtiN6xWYARwirrOSfE= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= -golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/option.go b/option.go index 1dd45d6..5d87e06 100644 --- a/option.go +++ b/option.go @@ -9,6 +9,7 @@ import ( const ( defaultCPUThreshold = 0.75 defaultMemThreshold = 0.75 + defaultGoroutineThreshold = 10000 defaultWatchInterval = 5 * time.Second defaultCPUProfilingDuration = 10 * time.Second defaultMinConsecutiveOverThreshold = 12 // min 1 minute. (12*5s) @@ -33,6 +34,12 @@ type Option struct { // is higher than this threshold. MemThreshold float64 + // GoroutineThreshold is the goroutine count threshold to trigger the goroutine profiling. + // to trigger the goroutine profiling. + // Autopprof will start the goroutine profiling when the goroutine count + // is higher than this threshold. + GoroutineThreshold int + // ReportBoth sets whether to trigger reports for both CPU and memory when either threshold is exceeded. // If some profiling is disabled, exclude it. ReportBoth bool diff --git a/profile.go b/profile.go index 8f88d28..6a3b25e 100644 --- a/profile.go +++ b/profile.go @@ -14,6 +14,8 @@ type profiler interface { profileCPU() ([]byte, error) // profileHeap profiles the heap usage. profileHeap() ([]byte, error) + // profileGoroutine profiles the goroutine usage. + profileGoroutine() ([]byte, error) } type defaultProfiler struct { @@ -51,7 +53,21 @@ func (p *defaultProfiler) profileHeap() ([]byte, error) { buf bytes.Buffer w = bufio.NewWriter(&buf) ) - if err := pprof.WriteHeapProfile(w); err != nil { + if err := pprof.Lookup("heap").WriteTo(w, 0); err != nil { + return nil, err + } + if err := w.Flush(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func (p *defaultProfiler) profileGoroutine() ([]byte, error) { + var ( + buf bytes.Buffer + w = bufio.NewWriter(&buf) + ) + if err := pprof.Lookup("goroutine").WriteTo(w, 0); err != nil { return nil, err } if err := w.Flush(); err != nil { diff --git a/profile_mock.go b/profile_mock.go index 290bfe2..2b8cd6f 100644 --- a/profile_mock.go +++ b/profile_mock.go @@ -1,13 +1,17 @@ // Code generated by MockGen. DO NOT EDIT. // Source: profile.go - +// +// Generated by this command: +// +// mockgen -source=profile.go -destination=profile_mock.go -package=autopprof +// // Package autopprof is a generated GoMock package. package autopprof import ( reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // Mockprofiler is a mock of profiler interface. @@ -48,6 +52,21 @@ func (mr *MockprofilerMockRecorder) profileCPU() *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "profileCPU", reflect.TypeOf((*Mockprofiler)(nil).profileCPU)) } +// profileGoroutine mocks base method. +func (m *Mockprofiler) profileGoroutine() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "profileGoroutine") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// profileGoroutine indicates an expected call of profileGoroutine. +func (mr *MockprofilerMockRecorder) profileGoroutine() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "profileGoroutine", reflect.TypeOf((*Mockprofiler)(nil).profileGoroutine)) +} + // profileHeap mocks base method. func (m *Mockprofiler) profileHeap() ([]byte, error) { m.ctrl.T.Helper() diff --git a/cgroupv1.go b/queryer/cgroupv1.go similarity index 90% rename from cgroupv1.go rename to queryer/cgroupv1.go index 74bcab7..d2869f1 100644 --- a/cgroupv1.go +++ b/queryer/cgroupv1.go @@ -1,10 +1,11 @@ //go:build linux // +build linux -package autopprof +package queryer import ( "bufio" + "github.com/daangn/autopprof" "os" "path" "strconv" @@ -30,7 +31,7 @@ type cgroupV1 struct { cpuQuota float64 - q cpuUsageSnapshotQueuer + q CpuUsageSnapshotQueuer } func newCgroupsV1() *cgroupV1 { @@ -45,7 +46,38 @@ func newCgroupsV1() *cgroupV1 { } } -func (c *cgroupV1) setCPUQuota() error { +func (c *cgroupV1) CpuUsage() (float64, error) { + stat, err := c.stat() + if err != nil { + return 0, err + } + c.snapshotCPUUsage(stat.CPU.Usage.Total) // In nanoseconds. + + // Calculate the usage only if there are enough snapshots. + if !c.q.isFull() { + return 0, nil + } + + s1, s2 := c.q.head(), c.q.tail() + delta := time.Duration(s2.usage-s1.usage) * cgroupV1UsageUnit + duration := s2.timestamp.Sub(s1.timestamp) + return (float64(delta) / float64(duration)) / c.cpuQuota, nil +} + +func (c *cgroupV1) MemUsage() (float64, error) { + stat, err := c.stat() + if err != nil { + return 0, err + } + var ( + sm = stat.Memory + usage = sm.Usage.Usage - sm.InactiveFile + limit = sm.HierarchicalMemoryLimit + ) + return float64(usage) / float64(limit), nil +} + +func (c *cgroupV1) SetCPUQuota() error { quota, err := c.parseCPU(cgroupV1CPUQuotaFile) if err != nil { return err @@ -80,37 +112,6 @@ func (c *cgroupV1) stat() (*v1.Metrics, error) { return stat, nil } -func (c *cgroupV1) cpuUsage() (float64, error) { - stat, err := c.stat() - if err != nil { - return 0, err - } - c.snapshotCPUUsage(stat.CPU.Usage.Total) // In nanoseconds. - - // Calculate the usage only if there are enough snapshots. - if !c.q.isFull() { - return 0, nil - } - - s1, s2 := c.q.head(), c.q.tail() - delta := time.Duration(s2.usage-s1.usage) * cgroupV1UsageUnit - duration := s2.timestamp.Sub(s1.timestamp) - return (float64(delta) / float64(duration)) / c.cpuQuota, nil -} - -func (c *cgroupV1) memUsage() (float64, error) { - stat, err := c.stat() - if err != nil { - return 0, err - } - var ( - sm = stat.Memory - usage = sm.Usage.Usage - sm.InactiveFile - limit = sm.HierarchicalMemoryLimit - ) - return float64(usage) / float64(limit), nil -} - func (c *cgroupV1) parseCPU(filename string) (int, error) { f, err := os.Open( path.Join(c.mountPoint, c.cpuSubsystem, filename), @@ -129,5 +130,5 @@ func (c *cgroupV1) parseCPU(filename string) (int, error) { if err := scanner.Err(); err != nil { return 0, err } - return 0, ErrV1CPUSubsystemEmpty + return 0, autopprof.ErrV1CPUSubsystemEmpty } diff --git a/cgroupv1_test.go b/queryer/cgroupv1_test.go similarity index 56% rename from cgroupv1_test.go rename to queryer/cgroupv1_test.go index aaf1d45..dcde3b9 100644 --- a/cgroupv1_test.go +++ b/queryer/cgroupv1_test.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package autopprof +package queryer import ( "testing" @@ -10,7 +10,7 @@ import ( "github.com/containerd/cgroups" ) -func TestCgroupV1_cpuUsage(t *testing.T) { +func TestCgroupV1_CpuUsage(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Legacy { t.Skip("cgroup v1 is not available") @@ -19,57 +19,57 @@ func TestCgroupV1_cpuUsage(t *testing.T) { cgv1.cpuQuota = 2 cgv1.q = newCPUUsageSnapshotQueue(3) - usage, err := cgv1.cpuUsage() + usage, err := cgv1.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage != 0 { // The cpu usage is 0 until the queue is full. - t.Errorf("cpuUsage() = %f, want 0", usage) + t.Errorf("CpuUsage() = %f, want 0", usage) } time.Sleep(1050 * time.Millisecond) - usage, err = cgv1.cpuUsage() + usage, err = cgv1.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage != 0 { // The cpu usage is 0 until the queue is full. - t.Errorf("cpuUsage() = %f, want 0", usage) + t.Errorf("CpuUsage() = %f, want 0", usage) } time.Sleep(1050 * time.Millisecond) - usage, err = cgv1.cpuUsage() + usage, err = cgv1.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage < 0 || usage > 1 { - t.Errorf("cpuUsage() = %f, want between 0 and 1", usage) + t.Errorf("CpuUsage() = %f, want between 0 and 1", usage) } } -func TestCgroupV1_memUsage(t *testing.T) { +func TestCgroupV1_MemUsage(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Legacy { t.Skip("cgroup v1 is not available") } - usage, err := newCgroupsV1().memUsage() + usage, err := newCgroupsV1().MemUsage() if err != nil { - t.Errorf("memUsage() = %v, want nil", err) + t.Errorf("MemUsage() = %v, want nil", err) } if usage < 0 || usage > 1 { - t.Errorf("memUsage() = %f, want between 0 and 1", usage) + t.Errorf("MemUsage() = %f, want between 0 and 1", usage) } } -func TestCgroupV1_setCPUQuota(t *testing.T) { +func TestCgroupV1_SetCPUQuota(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Legacy { t.Skip("cgroup v1 is not available") } cgv1 := newCgroupsV1() - if err := cgv1.setCPUQuota(); err != nil { - t.Errorf("setCPUQuota() = %v, want nil", err) + if err := cgv1.SetCPUQuota(); err != nil { + t.Errorf("SetCPUQuota() = %v, want nil", err) } // The cpu quota of test docker container is 1.5. if cgv1.cpuQuota != 1.5 { diff --git a/cgroupv2.go b/queryer/cgroupv2.go similarity index 88% rename from cgroupv2.go rename to queryer/cgroupv2.go index 9f7da9a..4a8d5e9 100644 --- a/cgroupv2.go +++ b/queryer/cgroupv2.go @@ -1,11 +1,12 @@ //go:build linux // +build linux -package autopprof +package queryer import ( "bufio" "fmt" + "github.com/daangn/autopprof" "os" "path" "strconv" @@ -34,7 +35,7 @@ type cgroupV2 struct { cpuQuota float64 - q cpuUsageSnapshotQueuer + q CpuUsageSnapshotQueuer } func newCgroupsV2() *cgroupV2 { @@ -49,12 +50,43 @@ func newCgroupsV2() *cgroupV2 { } } -func (c *cgroupV2) setCPUQuota() error { +func (c *cgroupV2) CpuUsage() (float64, error) { + stat, err := c.stat() + if err != nil { + return 0, err + } + c.snapshotCPUUsage(stat.CPU.UsageUsec) // In microseconds. + + // Calculate the usage only if there are enough snapshots. + if !c.q.isFull() { + return 0, nil + } + + s1, s2 := c.q.head(), c.q.tail() + delta := time.Duration(s2.usage-s1.usage) * cgroupV2UsageUnit + duration := s2.timestamp.Sub(s1.timestamp) + return (float64(delta) / float64(duration)) / c.cpuQuota, nil +} + +func (c *cgroupV2) MemUsage() (float64, error) { + stat, err := c.stat() + if err != nil { + return 0, err + } + var ( + sm = stat.Memory + usage = sm.Usage - sm.InactiveFile + limit = sm.UsageLimit + ) + return float64(usage) / float64(limit), nil +} + +func (c *cgroupV2) SetCPUQuota() error { f, err := os.Open( path.Join(c.mountPoint, c.cpuMaxFile), ) if os.IsNotExist(err) { - return ErrV2CPUQuotaUndefined + return autopprof.ErrV2CPUQuotaUndefined } if err != nil { return err @@ -69,7 +101,7 @@ func (c *cgroupV2) setCPUQuota() error { ) } if fields[0] == cgroupV2CPUMaxQuotaMax { - return ErrV2CPUQuotaUndefined + return autopprof.ErrV2CPUQuotaUndefined } max, err := strconv.Atoi(fields[0]) @@ -90,7 +122,7 @@ func (c *cgroupV2) setCPUQuota() error { if err := scanner.Err(); err != nil { return err } - return ErrV2CPUMaxEmpty + return autopprof.ErrV2CPUMaxEmpty } func (c *cgroupV2) snapshotCPUUsage(usage uint64) { @@ -115,34 +147,3 @@ func (c *cgroupV2) stat() (*stats.Metrics, error) { } return stat, nil } - -func (c *cgroupV2) cpuUsage() (float64, error) { - stat, err := c.stat() - if err != nil { - return 0, err - } - c.snapshotCPUUsage(stat.CPU.UsageUsec) // In microseconds. - - // Calculate the usage only if there are enough snapshots. - if !c.q.isFull() { - return 0, nil - } - - s1, s2 := c.q.head(), c.q.tail() - delta := time.Duration(s2.usage-s1.usage) * cgroupV2UsageUnit - duration := s2.timestamp.Sub(s1.timestamp) - return (float64(delta) / float64(duration)) / c.cpuQuota, nil -} - -func (c *cgroupV2) memUsage() (float64, error) { - stat, err := c.stat() - if err != nil { - return 0, err - } - var ( - sm = stat.Memory - usage = sm.Usage - sm.InactiveFile - limit = sm.UsageLimit - ) - return float64(usage) / float64(limit), nil -} diff --git a/cgroupv2_test.go b/queryer/cgroupv2_test.go similarity index 59% rename from cgroupv2_test.go rename to queryer/cgroupv2_test.go index cedcd78..1ad1b97 100644 --- a/cgroupv2_test.go +++ b/queryer/cgroupv2_test.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package autopprof +package queryer import ( "testing" @@ -10,7 +10,7 @@ import ( "github.com/containerd/cgroups" ) -func TestCgroupV2_cpuUsage(t *testing.T) { +func TestCgroupV2_CpuUsage(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Hybrid && mode != cgroups.Unified { t.Skip("cgroup v2 is not available") @@ -19,58 +19,58 @@ func TestCgroupV2_cpuUsage(t *testing.T) { cgv2.cpuQuota = 2 cgv2.q = newCPUUsageSnapshotQueue(3) - usage, err := cgv2.cpuUsage() + usage, err := cgv2.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage != 0 { // The cpu usage is 0 until the queue is full. - t.Errorf("cpuUsage() = %f, want 0", usage) + t.Errorf("CpuUsage() = %f, want 0", usage) } time.Sleep(1050 * time.Millisecond) - usage, err = cgv2.cpuUsage() + usage, err = cgv2.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage != 0 { // The cpu usage is 0 until the queue is full. - t.Errorf("cpuUsage() = %f, want 0", usage) + t.Errorf("CpuUsage() = %f, want 0", usage) } time.Sleep(1050 * time.Millisecond) - usage, err = cgv2.cpuUsage() + usage, err = cgv2.CpuUsage() if err != nil { - t.Errorf("cpuUsage() = %v, want nil", err) + t.Errorf("CpuUsage() = %v, want nil", err) } if usage < 0 || usage > 1 { - t.Errorf("cpuUsage() = %f, want between 0 and 1", usage) + t.Errorf("CpuUsage() = %f, want between 0 and 1", usage) } } -func TestCgroupV2_memUsage(t *testing.T) { +func TestCgroupV2_MemUsage(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Hybrid && mode != cgroups.Unified { t.Skip("cgroup v2 is not available") } cgv2 := newCgroupsV2() - usage, err := cgv2.memUsage() + usage, err := cgv2.MemUsage() if err != nil { - t.Errorf("memUsage() = %v, want nil", err) + t.Errorf("MemUsage() = %v, want nil", err) } if usage < 0 || usage > 1 { - t.Errorf("memUsage() = %f, want between 0 and 1", usage) + t.Errorf("MemUsage() = %f, want between 0 and 1", usage) } } -func TestCgroupV2_setCPUQuota(t *testing.T) { +func TestCgroupV2_SetCPUQuota(t *testing.T) { mode := cgroups.Mode() if mode != cgroups.Hybrid && mode != cgroups.Unified { t.Skip("cgroup v2 is not available") } cgv2 := newCgroupsV2() - if err := cgv2.setCPUQuota(); err != nil { - t.Errorf("setCPUQuota() = %v, want nil", err) + if err := cgv2.SetCPUQuota(); err != nil { + t.Errorf("SetCPUQuota() = %v, want nil", err) } // The cpu quota of test docker container is 1.5. if cgv2.cpuQuota != 1.5 { diff --git a/queryer/error.go b/queryer/error.go new file mode 100644 index 0000000..68298d4 --- /dev/null +++ b/queryer/error.go @@ -0,0 +1,8 @@ +package queryer + +import "fmt" + +// Errors. +var ( + ErrCgroupsUnavailable = fmt.Errorf("autopprof: cgroups is unavailable") +) diff --git a/queryer/queryer.go b/queryer/queryer.go new file mode 100644 index 0000000..5572de9 --- /dev/null +++ b/queryer/queryer.go @@ -0,0 +1,39 @@ +//go:build linux +// +build linux + +package queryer + +import ( + "github.com/containerd/cgroups" +) + +//go:generate mockgen -source=queryer.go -destination=queryer_mock.go -package=queryer + +const ( + cpuUsageSnapshotQueueSize = 24 // 24 * 5s = 2 minutes. +) + +type CgroupsQueryer interface { + CpuUsage() (float64, error) + MemUsage() (float64, error) + + SetCPUQuota() error +} + +type RuntimeQueryer interface { + GoroutineCount() int +} + +func NewCgroupQueryer() (CgroupsQueryer, error) { + switch cgroups.Mode() { + case cgroups.Legacy: + return newCgroupsV1(), nil + case cgroups.Hybrid, cgroups.Unified: + return newCgroupsV2(), nil + } + return nil, ErrCgroupsUnavailable +} + +func NewRuntimeQueryer() (RuntimeQueryer, error) { + return newRuntimeQuery(), nil +} diff --git a/queryer/queryer_mock.go b/queryer/queryer_mock.go new file mode 100644 index 0000000..41fa231 --- /dev/null +++ b/queryer/queryer_mock.go @@ -0,0 +1,119 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: queryer.go +// +// Generated by this command: +// +// mockgen -source=queryer.go -destination=queryer_mock.go -package=queryer +// +// Package queryer is a generated GoMock package. +package queryer + +import ( + reflect "reflect" + + gomock "go.uber.org/mock/gomock" +) + +// MockCgroupsQueryer is a mock of CgroupsQueryer interface. +type MockCgroupsQueryer struct { + ctrl *gomock.Controller + recorder *MockCgroupsQueryerMockRecorder +} + +// MockCgroupsQueryerMockRecorder is the mock recorder for MockCgroupsQueryer. +type MockCgroupsQueryerMockRecorder struct { + mock *MockCgroupsQueryer +} + +// NewMockCgroupsQueryer creates a new mock instance. +func NewMockCgroupsQueryer(ctrl *gomock.Controller) *MockCgroupsQueryer { + mock := &MockCgroupsQueryer{ctrl: ctrl} + mock.recorder = &MockCgroupsQueryerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockCgroupsQueryer) EXPECT() *MockCgroupsQueryerMockRecorder { + return m.recorder +} + +// CpuUsage mocks base method. +func (m *MockCgroupsQueryer) CpuUsage() (float64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CpuUsage") + ret0, _ := ret[0].(float64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CpuUsage indicates an expected call of CpuUsage. +func (mr *MockCgroupsQueryerMockRecorder) CpuUsage() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CpuUsage", reflect.TypeOf((*MockCgroupsQueryer)(nil).CpuUsage)) +} + +// MemUsage mocks base method. +func (m *MockCgroupsQueryer) MemUsage() (float64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MemUsage") + ret0, _ := ret[0].(float64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MemUsage indicates an expected call of MemUsage. +func (mr *MockCgroupsQueryerMockRecorder) MemUsage() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MemUsage", reflect.TypeOf((*MockCgroupsQueryer)(nil).MemUsage)) +} + +// SetCPUQuota mocks base method. +func (m *MockCgroupsQueryer) SetCPUQuota() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "SetCPUQuota") + ret0, _ := ret[0].(error) + return ret0 +} + +// SetCPUQuota indicates an expected call of SetCPUQuota. +func (mr *MockCgroupsQueryerMockRecorder) SetCPUQuota() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SetCPUQuota", reflect.TypeOf((*MockCgroupsQueryer)(nil).SetCPUQuota)) +} + +// MockRuntimeQueryer is a mock of RuntimeQueryer interface. +type MockRuntimeQueryer struct { + ctrl *gomock.Controller + recorder *MockRuntimeQueryerMockRecorder +} + +// MockRuntimeQueryerMockRecorder is the mock recorder for MockRuntimeQueryer. +type MockRuntimeQueryerMockRecorder struct { + mock *MockRuntimeQueryer +} + +// NewMockRuntimeQueryer creates a new mock instance. +func NewMockRuntimeQueryer(ctrl *gomock.Controller) *MockRuntimeQueryer { + mock := &MockRuntimeQueryer{ctrl: ctrl} + mock.recorder = &MockRuntimeQueryerMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockRuntimeQueryer) EXPECT() *MockRuntimeQueryerMockRecorder { + return m.recorder +} + +// GoroutineCount mocks base method. +func (m *MockRuntimeQueryer) GoroutineCount() int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GoroutineCount") + ret0, _ := ret[0].(int) + return ret0 +} + +// GoroutineCount indicates an expected call of GoroutineCount. +func (mr *MockRuntimeQueryerMockRecorder) GoroutineCount() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GoroutineCount", reflect.TypeOf((*MockRuntimeQueryer)(nil).GoroutineCount)) +} diff --git a/cgroups_test.go b/queryer/queryer_test.go similarity index 76% rename from cgroups_test.go rename to queryer/queryer_test.go index 2eb29ef..1386009 100644 --- a/cgroups_test.go +++ b/queryer/queryer_test.go @@ -1,7 +1,7 @@ //go:build linux // +build linux -package autopprof +package queryer import ( "testing" @@ -9,9 +9,9 @@ import ( "github.com/containerd/cgroups" ) -func TestNewQueryer(t *testing.T) { +func TestNewCgroupQueryer(t *testing.T) { mode := cgroups.Mode() - _, err := newQueryer() + _, err := NewCgroupQueryer() if mode == cgroups.Unavailable && err == nil { t.Errorf("newQueryer() = nil, want error") } else if err != nil { diff --git a/queue.go b/queryer/queue.go similarity index 93% rename from queue.go rename to queryer/queue.go index 23f886e..9c6483b 100644 --- a/queue.go +++ b/queryer/queue.go @@ -1,10 +1,10 @@ -package autopprof +package queryer import "time" -// cpuUsageSnapshotQueue is a circular queue of cpuUsageSnapshot. +// CpuUsageSnapshotQueue is a circular queue of cpuUsageSnapshot. // It doesn't implement dequeue() method because it's not needed. -type cpuUsageSnapshotQueuer interface { +type CpuUsageSnapshotQueuer interface { // Enqueue adds an element to the queue. // If the queue is full, the oldest element is overwritten. enqueue(snapshot *cpuUsageSnapshot) diff --git a/queue_test.go b/queryer/queue_test.go similarity index 99% rename from queue_test.go rename to queryer/queue_test.go index c03cbeb..df34312 100644 --- a/queue_test.go +++ b/queryer/queue_test.go @@ -1,4 +1,4 @@ -package autopprof +package queryer import ( "testing" diff --git a/queryer/runtime.go b/queryer/runtime.go new file mode 100644 index 0000000..2f0669b --- /dev/null +++ b/queryer/runtime.go @@ -0,0 +1,14 @@ +package queryer + +import "runtime/pprof" + +type runtimeQuery struct { +} + +func newRuntimeQuery() *runtimeQuery { + return &runtimeQuery{} +} + +func (r runtimeQuery) GoroutineCount() int { + return pprof.Lookup("goroutine").Count() +} diff --git a/queryer/runtime_test.go b/queryer/runtime_test.go new file mode 100644 index 0000000..06ab749 --- /dev/null +++ b/queryer/runtime_test.go @@ -0,0 +1,39 @@ +package queryer + +import ( + "sync" + "testing" + "time" +) + +func Test_runtimeQuery_GoroutineCount(t *testing.T) { + r := newRuntimeQuery() + + initGoroutineCnt := r.GoroutineCount() + if initGoroutineCnt < 1 { + t.Errorf("GoroutineCount() = %d; want is > 0", initGoroutineCnt) + } + + wg := sync.WaitGroup{} + + goroutineCnt := 1000 + for i := 0; i < goroutineCnt; i++ { + wg.Add(1) + go func() { + time.Sleep(500 * time.Millisecond) + wg.Done() + }() + } + + addedGoroutineCnt := r.GoroutineCount() + if addedGoroutineCnt != initGoroutineCnt+goroutineCnt { + t.Errorf("GoroutineCount() = %d; want is %d", addedGoroutineCnt, initGoroutineCnt+1) + } + + wg.Wait() + + endGoroutineCnt := r.GoroutineCount() + if endGoroutineCnt != initGoroutineCnt { + t.Errorf("GoroutineCount() = %d; want is %d", endGoroutineCnt, initGoroutineCnt) + } +} diff --git a/report/report.go b/report/report.go index 9e43cfc..ee7513e 100644 --- a/report/report.go +++ b/report/report.go @@ -15,6 +15,10 @@ const ( // HeapProfileFilenameFmt is the filename format for the heap profile. // pprof...alloc_objects.alloc_space.inuse_objects.inuse_space..pprof. HeapProfileFilenameFmt = "pprof.%s.%s.alloc_objects.alloc_space.inuse_objects.inuse_space.%s.pprof" + + // GoroutineProfileFilenameFmt is the filename format for the goroutine profile. + // pprof...goroutine..pprof. + GoroutineProfileFilenameFmt = "pprof.%s.%s.goroutine.%s.pprof" ) // Reporter is responsible for reporting the profiling report to the destination. @@ -24,6 +28,9 @@ type Reporter interface { // ReportHeapProfile sends the heap profiling data to the specific destination. ReportHeapProfile(ctx context.Context, r io.Reader, mi MemInfo) error + + // ReportGoroutineProfile sends the goroutine profiling data to the specific destination. + ReportGoroutineProfile(ctx context.Context, r io.Reader, gi GoroutineInfo) error } // CPUInfo is the CPU usage information. @@ -37,3 +44,8 @@ type MemInfo struct { ThresholdPercentage float64 UsagePercentage float64 } + +type GoroutineInfo struct { + ThresholdCount int + Count int +} diff --git a/report/report_mock.go b/report/report_mock.go index 873ba39..27ea438 100644 --- a/report/report_mock.go +++ b/report/report_mock.go @@ -1,6 +1,10 @@ // Code generated by MockGen. DO NOT EDIT. // Source: report.go - +// +// Generated by this command: +// +// mockgen -source=report.go -destination=report_mock.go -package=report +// // Package report is a generated GoMock package. package report @@ -9,7 +13,7 @@ import ( io "io" reflect "reflect" - gomock "github.com/golang/mock/gomock" + gomock "go.uber.org/mock/gomock" ) // MockReporter is a mock of Reporter interface. @@ -44,11 +48,25 @@ func (m *MockReporter) ReportCPUProfile(ctx context.Context, r io.Reader, ci CPU } // ReportCPUProfile indicates an expected call of ReportCPUProfile. -func (mr *MockReporterMockRecorder) ReportCPUProfile(ctx, r, ci interface{}) *gomock.Call { +func (mr *MockReporterMockRecorder) ReportCPUProfile(ctx, r, ci any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportCPUProfile", reflect.TypeOf((*MockReporter)(nil).ReportCPUProfile), ctx, r, ci) } +// ReportGoroutineProfile mocks base method. +func (m *MockReporter) ReportGoroutineProfile(ctx context.Context, r io.Reader, gi GoroutineInfo) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ReportGoroutineProfile", ctx, r, gi) + ret0, _ := ret[0].(error) + return ret0 +} + +// ReportGoroutineProfile indicates an expected call of ReportGoroutineProfile. +func (mr *MockReporterMockRecorder) ReportGoroutineProfile(ctx, r, gi any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportGoroutineProfile", reflect.TypeOf((*MockReporter)(nil).ReportGoroutineProfile), ctx, r, gi) +} + // ReportHeapProfile mocks base method. func (m *MockReporter) ReportHeapProfile(ctx context.Context, r io.Reader, mi MemInfo) error { m.ctrl.T.Helper() @@ -58,7 +76,7 @@ func (m *MockReporter) ReportHeapProfile(ctx context.Context, r io.Reader, mi Me } // ReportHeapProfile indicates an expected call of ReportHeapProfile. -func (mr *MockReporterMockRecorder) ReportHeapProfile(ctx, r, mi interface{}) *gomock.Call { +func (mr *MockReporterMockRecorder) ReportHeapProfile(ctx, r, mi any) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ReportHeapProfile", reflect.TypeOf((*MockReporter)(nil).ReportHeapProfile), ctx, r, mi) } diff --git a/report/slack.go b/report/slack.go index 0769b20..27ab319 100644 --- a/report/slack.go +++ b/report/slack.go @@ -13,8 +13,9 @@ import ( const ( reportTimeLayout = "2006-01-02T150405.MST" - cpuCommentFmt = ":rotating_light:[CPU] usage (*%.2f%%*) > threshold (*%.2f%%*)" - memCommentFmt = ":rotating_light:[MEM] usage (*%.2f%%*) > threshold (*%.2f%%*)" + cpuCommentFmt = ":rotating_light:[CPU] usage (*%.2f%%*) > threshold (*%.2f%%*)" + memCommentFmt = ":rotating_light:[MEM] usage (*%.2f%%*) > threshold (*%.2f%%*)" + goroutineCommentFmt = ":rotating_light:[GOROUTINE] count (*%d*) > threshold (*%d*)" ) // SlackReporter is the reporter to send the profiling report to the @@ -85,3 +86,25 @@ func (s *SlackReporter) ReportHeapProfile( } return nil } + +// ReportGoroutineProfile sends the goroutine profiling data to the Slack. +func (s *SlackReporter) ReportGoroutineProfile( + ctx context.Context, r io.Reader, gi GoroutineInfo, +) error { + hostname, _ := os.Hostname() // Don't care about this error. + var ( + now = time.Now().Format(reportTimeLayout) + filename = fmt.Sprintf(GoroutineProfileFilenameFmt, s.app, hostname, now) + comment = fmt.Sprintf(goroutineCommentFmt, gi.Count, gi.ThresholdCount) + ) + if _, err := s.client.UploadFileContext(ctx, slack.FileUploadParameters{ + Reader: r, + Filename: filename, + Title: filename, + InitialComment: comment, + Channels: []string{s.channel}, + }); err != nil { + return fmt.Errorf("autopprof: failed to upload a file to Slack channel: %w", err) + } + return nil +}