feat: support free OS memory at intervals

server/Makefile

@@ -117,7 +117,7 @@ test: vendor $(proto) $(generated_libs)
 
 .PHONY: server
 server: vendor $(proto) $(generated_libs)
-	go build -mod vendor ${FLAGS} -o bin/deepflow-server${BINARY_SUFFIX} cmd/server/main.go cmd/server/config.go
+	go build -mod vendor ${FLAGS} -o bin/deepflow-server${BINARY_SUFFIX} cmd/server/main.go cmd/server/config.go cmd/server/free_os_memory_handler.go
 
 .PHONY: querier
 querier: vendor $(proto) $(generated_libs)

server/cmd/server/config.go

@@ -31,12 +31,18 @@ import (
 )
 
 type Config struct {
-	LogFile           string            `default:"/var/log/deepflow/server.log" yaml:"log-file"`
-	LogLevel          string            `default:"info" yaml:"log-level"`
-	ContinuousProfile ContinuousProfile `yaml:"continuous-profile"`
-	Profiler          bool              `yaml:"profiler"`
-	MaxCPUs           int               `yaml:"max-cpus"`
-	MonitorPaths      []string          `yaml:"monitor-paths"`
+	LogFile             string              `default:"/var/log/deepflow/server.log" yaml:"log-file"`
+	LogLevel            string              `default:"info" yaml:"log-level"`
+	ContinuousProfile   ContinuousProfile   `yaml:"continuous-profile"`
+	Profiler            bool                `yaml:"profiler"`
+	MaxCPUs             int                 `yaml:"max-cpus"`
+	MonitorPaths        []string            `yaml:"monitor-paths"`
+	FreeOSMemoryManager FreeOSMemoryManager `yaml:"free-os-memory-manager"`
+}
+
+type FreeOSMemoryManager struct {
+	Enabled  bool `yaml:"enabled"`
+	Interval int  `yaml:"interval"`
 }
 
 type ContinuousProfile struct {
@@ -60,7 +66,8 @@ func loadConfig(path string) *Config {
 			BlockRate:     5,
 			LogEnabled:    true,
 		},
-		MonitorPaths: []string{"/", "/mnt", "/var/log"},
+		MonitorPaths:        []string{"/", "/mnt", "/var/log"},
+		FreeOSMemoryManager: FreeOSMemoryManager{false, DEFAULT_FREE_INTERVAL_SECOND},
 	}
 	configBytes, err := ioutil.ReadFile(path)
 	if err != nil {

server/cmd/server/free_os_memory_handler.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"fmt"
+	"runtime"
+	"runtime/debug"
+	"strconv"
+	"time"
+
+	"github.com/deepflowio/deepflow/server/ingester/ingesterctl"
+	debugcmd "github.com/deepflowio/deepflow/server/libs/debug"
+)
+
+const (
+	DEFAULT_FREE_INTERVAL_SECOND = 3600
+)
+
+func getMemUsage(tag string) string {
+	var m runtime.MemStats
+	runtime.ReadMemStats(&m)
+	return fmt.Sprintf("%s Memory Usage: Alloc=%d MB, Sys=%d MB, NumGC=%d",
+		tag, bToMb(m.Alloc), bToMb(m.Sys), m.NumGC)
+}
+
+func bToMb(b uint64) uint64 {
+	return b / 1024 / 1024
+}
+
+type FreeOSMemoryHandler struct {
+	enabled  bool
+	interval int
+	ticker   *time.Ticker
+	running  bool
+}
+
+func NewFreeOSMemoryHandler(cfg *FreeOSMemoryManager) *FreeOSMemoryHandler {
+	f := &FreeOSMemoryHandler{
+		enabled:  cfg.Enabled,
+		interval: cfg.Interval,
+	}
+	if f.interval == 0 {
+		f.interval = DEFAULT_FREE_INTERVAL_SECOND
+	}
+
+	debugcmd.ServerRegisterSimple(ingesterctl.CMD_FREE_OS_MEMORY, f)
+	return f
+}
+
+func (f *FreeOSMemoryHandler) Run() {
+	f.running = true
+	f.ticker = time.NewTicker(time.Duration(f.interval) * time.Second)
+	for f.running {
+		select {
+		case <-f.ticker.C:
+			log.Info(getMemUsage("Before FreeOSMemory(),"))
+			debug.FreeOSMemory()
+			log.Info(getMemUsage("After FreeOSMemory(),"))
+		}
+	}
+	f.ticker.Stop()
+	f.ticker = nil
+}
+
+func (f *FreeOSMemoryHandler) Start(force bool) string {
+	if !f.enabled && !force {
+		return logInfo("FreeOSMemory() interval execution disabled")
+	}
+	if f.running {
+		return logInfo("FreeOSMemory() interval execution already running")
+	}
+	go f.Run()
+	return logInfo(fmt.Sprintf("starting FreeOSMemory() interval execution with interval %ds", f.interval))
+}
+
+func (f *FreeOSMemoryHandler) Stop() string {
+	if !f.running {
+		return logInfo("FreeOSMemory() interval execution already stopped")
+	}
+
+	f.running = false
+	return logInfo("FreeOSMemory() interval execution is stopping")
+}
+
+func (f *FreeOSMemoryHandler) CallFreeOSMemoryOnce() string {
+	before := getMemUsage("\nBefore FreeOSMemory(), ")
+	debug.FreeOSMemory()
+	after := getMemUsage("\nAfter FreeOSMemory(), ")
+	return before + after
+}
+
+func (f *FreeOSMemoryHandler) SetInterval(interval int) string {
+	if interval <= 0 {
+		return logInfo(fmt.Sprintf("invalid interval value %d, should > 0", interval))
+	}
+	if f.ticker != nil {
+		f.interval = interval
+		f.ticker.Reset(time.Duration(f.interval) * time.Second)
+		return logInfo(fmt.Sprintf("set interval to %d second", f.interval))
+	} else {
+		return logInfo("FreeOSMemory() interval execution is not running and interval cannot be set")
+	}
+}
+
+const (
+	CMD_FREE_OS_MEMORY_ON uint16 = iota
+	CMD_FREE_OS_MEMORY_OFF
+	CMD_FREE_OS_MEMORY_ONCE
+	CMD_FREE_OS_MEMORY_STATUS
+	CMD_FREE_OS_MEMORY_SET_INTERVAL
+)
+
+func (f *FreeOSMemoryHandler) HandleSimpleCommand(op uint16, arg string) string {
+	switch op {
+	case CMD_FREE_OS_MEMORY_ON:
+		return f.Start(true)
+	case CMD_FREE_OS_MEMORY_OFF:
+		return f.Stop()
+	case CMD_FREE_OS_MEMORY_ONCE:
+		return f.CallFreeOSMemoryOnce()
+	case CMD_FREE_OS_MEMORY_STATUS:
+		if f.running {
+			return fmt.Sprintf("running with interval %d", f.interval)
+		}
+		return "stopped"
+	case CMD_FREE_OS_MEMORY_SET_INTERVAL:
+		interval := 0
+		if arg != "" {
+			interval, _ = strconv.Atoi(arg)
+		}
+		return f.SetInterval(interval)
+	}
+
+	return logInfo("invalid op")
+}

server/cmd/server/main.go

@@ -94,6 +94,7 @@ func main() {
 	}
 
 	NewContinuousProfiler(&cfg.ContinuousProfile).Start(false)
+	NewFreeOSMemoryHandler(&cfg.FreeOSMemoryManager).Start(false)
 
 	ctx, cancel := utils.NewWaitGroupCtx()
 	defer func() {

server/ingester/ingesterctl/cmd/cmd.go

@@ -87,6 +87,17 @@ func RegisterIngesterCommand(root *cobra.Command) {
 			{Cmd: "set-profile-types [cpu][,inuse_objects][,alloc_objects][,inuse_space][,alloc_space][,goroutines][,mutex_count][,mutex_duration][,block_count][,block_duration]", Helper: "default continuous profiler profile types: cpu,inuse_objects,alloc_objects,inuse_space,alloc_space. need to restart continuous profiler to take effect"},
 		},
 	))
+	ingesterCmd.AddCommand(debug.ClientRegisterSimple(
+		ingesterctl.CMD_FREE_OS_MEMORY,
+		debug.CmdHelper{Cmd: "free-os-memory", Helper: "free os memory commands"},
+		[]debug.CmdHelper{
+			{Cmd: "on", Helper: "start free os memory at intervals"},
+			{Cmd: "off", Helper: "stop free os memory"},
+			{Cmd: "once", Helper: "start free os memory once"},
+			{Cmd: "status", Helper: "free os memory status"},
+			{Cmd: "set-interval [second]", Helper: "set free os memory interval"},
+		},
+	))
 	ingesterCmd.AddCommand(debug.ClientRegisterSimple(
 		ingesterctl.CMD_ORG_SWITCH,
 		debug.CmdHelper{Cmd: "switch-to-debug-org [org-id]", Helper: "the debugging command switches to the specified organization"},

server/ingester/ingesterctl/const.go

@@ -58,6 +58,7 @@ const (
 	CMD_EXPORTER_PLATFORMDATA
 	CMD_CONTINUOUS_PROFILER
 	CMD_ORG_SWITCH
+	CMD_FREE_OS_MEMORY
 )
 
 const (

server/server.yaml

@@ -18,6 +18,11 @@ log-level: info
 #  block-rate: 5 # valid when ProfileTypes contains 'block_count' or 'block_duration'
 #  log-enabled: true # whether record profile debug logs
 
+## every `interval`, call debug.FreeOSMemory()(https://pkg.go.dev/runtime/debug#FreeOSMemory) to release OS memory.
+#free-os-memory-manager:
+#  enabled: false
+#  interval: 3600  # uint: second
+
 ## extract an integer (generally used timestamp) from traceId as an additional index to speed up traceId queries.
 #trace-id-with-index:
 #  disabled: false