From 48b5ed428b165af727995b3e873660116945b2fa Mon Sep 17 00:00:00 2001 From: zhuofeng Date: Tue, 10 Sep 2024 19:39:48 +0800 Subject: [PATCH] feat: support profile result filter --- server/querier/profile/config/config.go | 12 +++- server/querier/profile/model/model.go | 1 + server/querier/profile/service/profile.go | 80 ++++++++++++++++++++++- server/server.yaml | 5 ++ 4 files changed, 95 insertions(+), 3 deletions(-) diff --git a/server/querier/profile/config/config.go b/server/querier/profile/config/config.go index 5fa37a5cd8c..59af76f99c2 100644 --- a/server/querier/profile/config/config.go +++ b/server/querier/profile/config/config.go @@ -22,9 +22,10 @@ import ( "reflect" "regexp" - "github.com/op/go-logging" - "gopkg.in/yaml.v2" "strings" + + logging "github.com/op/go-logging" + yaml "gopkg.in/yaml.v2" ) var log = logging.MustGetLogger("profile") @@ -40,6 +41,7 @@ type ProfileConfig struct { ListenPort int `default:"20419" yaml:"listen-port"` FlameQueryLimit int `default:"1000000" yaml:"flame_query_limit"` Querier Querier `yaml:"querier"` + Filter Filter `yaml:"filter"` } type Querier struct { @@ -47,6 +49,12 @@ type Querier struct { Port int `default:"20416" yaml:"port"` } +type Filter struct { + Disabled bool `default:"false" yaml:"disabled"` + TotalValuePercent float64 `default:"0.1" yaml:"total-value-percent"` + Depth int `default:"16" yaml:"depth"` +} + func (c *Config) expendEnv() { reConfig := reflect.ValueOf(&c.ProfileConfig) reConfig = reConfig.Elem() diff --git a/server/querier/profile/model/model.go b/server/querier/profile/model/model.go index 8871b3cbc78..4edc2ff310d 100644 --- a/server/querier/profile/model/model.go +++ b/server/querier/profile/model/model.go @@ -43,6 +43,7 @@ type ProfileTreeNode struct { ParentNodeID int SelfValue int TotalValue int + Depth int } type Debug struct { diff --git a/server/querier/profile/service/profile.go b/server/querier/profile/service/profile.go index f54b9e83092..041a179a0ff 100644 --- a/server/querier/profile/service/profile.go +++ b/server/querier/profile/service/profile.go @@ -236,6 +236,7 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string for i := range result.FunctionValues.Values { result.FunctionValues.Values[i] = []int{0, 0} } + result.NodeValues.Values = make([][]int, 0, len(nodes)) for _, node := range nodes { locationID := node.LocationID @@ -244,6 +245,11 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string result.NodeValues.Values = append(result.NodeValues.Values, []int{locationID, node.ParentNodeID, node.SelfValue, node.TotalValue}) } + if !cfg.Profile.Filter.Disabled { + totalValueThreshold := int(float64(nodes[0].TotalValue) * cfg.Profile.Filter.TotalValuePercent / 100) + filterResults(nodes, &result, totalValueThreshold, cfg.Profile.Filter.Depth) + } + result.Functions = locations locationTypes := GetLocationType(locations, result.FunctionValues.Values, args.ProfileEventType) result.FunctionTypes = locationTypes @@ -256,6 +262,60 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string return } +func isFilterDiscard(node *model.ProfileTreeNode, totalValueThreshold, depthThreshold int) bool { + return node.TotalValue < totalValueThreshold && node.Depth > depthThreshold +} + +func filterResults(nodes []model.ProfileTreeNode, result *model.ProfileTree, totalValueThreshold, depthThreshold int) { + if len(nodes) < 10000 { + return + } + // get the nodeid that needs to be deleted, + deleteNodeIDs := []int{} + for i, node := range nodes { + calculateDepth(nodes, i) + if !isFilterDiscard(&node, totalValueThreshold, depthThreshold) { + continue + } + if node.ParentNodeID >= 0 { + nodes[node.ParentNodeID].SelfValue = nodes[node.ParentNodeID].SelfValue + node.TotalValue + } + deleteNodeIDs = append(deleteNodeIDs, i) + } + + // 获取删除 nodeId 后的id和删除前的id的映射表，用于更新ParentNodeID + // obtain the mapping table between the id after nodeId is deleted and the id before deletion, used to update ParentNodeID + mapping := getMapping(len(nodes), deleteNodeIDs) + + maxDepth := 0 + result.NodeValues.Values = result.NodeValues.Values[:0] + for _, node := range nodes { + if isFilterDiscard(&node, totalValueThreshold, depthThreshold) { + continue + } + parentNodeID := node.ParentNodeID + if node.ParentNodeID != -1 { + parentNodeID = mapping[node.ParentNodeID] + } + result.NodeValues.Values = append(result.NodeValues.Values, []int{node.LocationID, parentNodeID, node.SelfValue, node.TotalValue}) + + if node.Depth > maxDepth { + maxDepth = node.Depth + } + } + log.Infof("profile total nodes count %d, total value is %d, value threshold is %d, depth threshold is %d, valid node count %d, maxDepth=%d", len(nodes), nodes[0].TotalValue, totalValueThreshold, depthThreshold, len(result.NodeValues.Values), maxDepth) +} + +func calculateDepth(nodes []model.ProfileTreeNode, thisNodeID int) { + thisNode := &nodes[thisNodeID] + depth := 1 + for thisNode.ParentNodeID >= 0 { + depth++ + thisNode = &nodes[thisNode.ParentNodeID] + } + nodes[thisNodeID].Depth = depth +} + func newProfileTreeNode(locationID, selfValue, totalValue int) model.ProfileTreeNode { node := model.ProfileTreeNode{} node.LocationID = locationID @@ -275,7 +335,6 @@ func updateAllParentNodes(nodes []model.ProfileTreeNode, thisNodeID, selfValue, thisNode = &nodes[thisNode.ParentNodeID] thisNode.TotalValue += totalValue } - } func CutKernelFunction(profileLocationByteSlice []byte, maxKernelStackDepth int, sep string) ([]byte, bool) { @@ -343,3 +402,22 @@ func GetLocationType(locations []string, locationValues [][]int, profileEventTyp } return locationTypes } + +func getMapping(arrLen int, indicesToRemove []int) map[int]int { + removed := make(map[int]struct{}) + for _, index := range indicesToRemove { + removed[index] = struct{}{} + } + + mapping := make(map[int]int) + newIndex := 0 + + for i := 0; i < arrLen; i++ { + if _, found := removed[i]; !found { + mapping[i] = newIndex + newIndex++ + } + } + + return mapping +} diff --git a/server/server.yaml b/server/server.yaml index 8b6d7198f5b..9782c4ce02c 100644 --- a/server/server.yaml +++ b/server/server.yaml @@ -360,6 +360,11 @@ querier: # profile相关配置 profile: flame_query_limit: 1000000 + filter: + disabled: false + # nodes that exceed the `depth` layer and have a total-value ratio less than `total-value-percent` are not displayed. + total-value-percent: 0.1 + depth: 16 # trace-map 相关配置 trace-map: