Skip to content

Commit

Permalink
feat: support profile result filter
Browse files Browse the repository at this point in the history
  • Loading branch information
lzf575 committed Sep 13, 2024
1 parent 68c04e4 commit 72eb71c
Show file tree
Hide file tree
Showing 4 changed files with 107 additions and 10 deletions.
22 changes: 15 additions & 7 deletions server/querier/profile/config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,10 @@ import (
"reflect"
"regexp"

"github.com/op/go-logging"
"gopkg.in/yaml.v2"
"strings"

logging "github.com/op/go-logging"
yaml "gopkg.in/yaml.v2"
)

var log = logging.MustGetLogger("profile")
Expand All @@ -35,18 +36,25 @@ type Config struct {
}

type ProfileConfig struct {
LogFile string `default:"/var/log/profile.log" yaml:"log-file"`
LogLevel string `default:"info" yaml:"log-level"`
ListenPort int `default:"20419" yaml:"listen-port"`
FlameQueryLimit int `default:"1000000" yaml:"flame_query_limit"`
Querier Querier `yaml:"querier"`
LogFile string `default:"/var/log/profile.log" yaml:"log-file"`
LogLevel string `default:"info" yaml:"log-level"`
ListenPort int `default:"20419" yaml:"listen-port"`
FlameQueryLimit int `default:"1000000" yaml:"flame_query_limit"`
Querier Querier `yaml:"querier"`
ResultFilter ResultFilter `yaml:"result-filter"`
}

type Querier struct {
Host string `default:"127.0.0.1" yaml:"host"`
Port int `default:"20416" yaml:"port"`
}

type ResultFilter struct {
Disabled bool `default:"false" yaml:"disabled"`
TotalValuePercent float64 `default:"0.1" yaml:"total-value-percent"`
Depth int `default:"12" yaml:"depth"`
}

func (c *Config) expendEnv() {
reConfig := reflect.ValueOf(&c.ProfileConfig)
reConfig = reConfig.Elem()
Expand Down
1 change: 1 addition & 0 deletions server/querier/profile/model/model.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ type ProfileTreeNode struct {
ParentNodeID int
SelfValue int
TotalValue int
Depth int
}

type Debug struct {
Expand Down
89 changes: 86 additions & 3 deletions server/querier/profile/service/profile.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ var log = logging.MustGetLogger("profile")
var InstanceProfileEventType = []string{"inuse_objects", "inuse_space", "goroutines", "mem-inuse"}

const (
initLocationCapacity = 1024
initNodeCapacity = 8192
initLocationCapacity = 1024
initNodeCapacity = 8192
FILTER_TRIGGER_THRESHOLD = 10000
)

func Profile(args model.Profile, cfg *config.QuerierConfig) (result model.ProfileTree, debug interface{}, err error) {
Expand Down Expand Up @@ -236,6 +237,7 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string
for i := range result.FunctionValues.Values {
result.FunctionValues.Values[i] = []int{0, 0}
}

result.NodeValues.Values = make([][]int, 0, len(nodes))
for _, node := range nodes {
locationID := node.LocationID
Expand All @@ -244,6 +246,11 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string
result.NodeValues.Values = append(result.NodeValues.Values, []int{locationID, node.ParentNodeID, node.SelfValue, node.TotalValue})
}

if !cfg.Profile.ResultFilter.Disabled {
totalValueThreshold := int(float64(nodes[0].TotalValue) * cfg.Profile.ResultFilter.TotalValuePercent / 100)
filterResults(nodes, &result, totalValueThreshold, cfg.Profile.ResultFilter.Depth)
}

result.Functions = locations
locationTypes := GetLocationType(locations, result.FunctionValues.Values, args.ProfileEventType)
result.FunctionTypes = locationTypes
Expand All @@ -256,6 +263,64 @@ func GenerateProfile(args model.Profile, cfg *config.QuerierConfig, where string
return
}

func isFilterDiscard(node *model.ProfileTreeNode, totalValueThreshold, depthThreshold int) bool {
return node.TotalValue < totalValueThreshold && node.Depth > depthThreshold
}

func filterResults(nodes []model.ProfileTreeNode, result *model.ProfileTree, totalValueThreshold, depthThreshold int) {
if len(nodes) < FILTER_TRIGGER_THRESHOLD {
return
}
// get the nodeid that needs to be deleted,
deleteNodeIDs := []int{}
for i, node := range nodes {
calculateDepth(nodes, i)
if !isFilterDiscard(&nodes[i], totalValueThreshold, depthThreshold) {
continue
}
if node.ParentNodeID >= 0 {
nodes[node.ParentNodeID].SelfValue = nodes[node.ParentNodeID].SelfValue + node.TotalValue
}
deleteNodeIDs = append(deleteNodeIDs, i)
}

// 获取删除 nodeId 后的id和删除前的id的映射表,用于更新 parentNodeID
// obtain the mapping table between the id after nodeId is deleted and the id before deletion, used to update parentNodeID
mapping := getMapping(len(nodes), deleteNodeIDs)

maxDepth := 0
result.NodeValues.Values = result.NodeValues.Values[:0]
for i, node := range nodes {
if isFilterDiscard(&nodes[i], totalValueThreshold, depthThreshold) {
continue
}
parentNodeID := node.ParentNodeID
if node.ParentNodeID != -1 {
parentNodeID = mapping[node.ParentNodeID]
}
result.NodeValues.Values = append(result.NodeValues.Values, []int{node.LocationID, parentNodeID, node.SelfValue, node.TotalValue})

if node.Depth > maxDepth {
maxDepth = node.Depth
}
}
log.Infof("profile total nodes count %d, total value is %d, value threshold is %d, depth threshold is %d, valid node count %d, maxDepth=%d", len(nodes), nodes[0].TotalValue, totalValueThreshold, depthThreshold, len(result.NodeValues.Values), maxDepth)
}

func calculateDepth(nodes []model.ProfileTreeNode, thisNodeID int) {
thisNode := &nodes[thisNodeID]
depth := 1
for thisNode.ParentNodeID >= 0 {
thisNode = &nodes[thisNode.ParentNodeID]
if thisNode.Depth > 0 {
depth += thisNode.Depth
break
}
depth++
}
nodes[thisNodeID].Depth = depth
}

func newProfileTreeNode(locationID, selfValue, totalValue int) model.ProfileTreeNode {
node := model.ProfileTreeNode{}
node.LocationID = locationID
Expand All @@ -275,7 +340,6 @@ func updateAllParentNodes(nodes []model.ProfileTreeNode, thisNodeID, selfValue,
thisNode = &nodes[thisNode.ParentNodeID]
thisNode.TotalValue += totalValue
}

}

func CutKernelFunction(profileLocationByteSlice []byte, maxKernelStackDepth int, sep string) ([]byte, bool) {
Expand Down Expand Up @@ -343,3 +407,22 @@ func GetLocationType(locations []string, locationValues [][]int, profileEventTyp
}
return locationTypes
}

func getMapping(arrLen int, indicesToRemove []int) map[int]int {
removed := make(map[int]struct{})
for _, index := range indicesToRemove {
removed[index] = struct{}{}
}

mapping := make(map[int]int)
newIndex := 0

for i := 0; i < arrLen; i++ {
if _, found := removed[i]; !found {
mapping[i] = newIndex
newIndex++
}
}

return mapping
}
5 changes: 5 additions & 0 deletions server/server.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -360,6 +360,11 @@ querier:
# profile相关配置
profile:
flame_query_limit: 1000000
result_filter:
disabled: false
# nodes that exceed the `depth` layer and have a total-value ratio less than `total-value-percent` are not displayed.
total-value-percent: 0.1
depth: 12

# trace-map 相关配置
trace-map:
Expand Down

0 comments on commit 72eb71c

Please sign in to comment.