[ingester] add severity_number enum

deepflowio · May 20, 2024 · fa16c26 · fa16c26
1 parent 5e4c60a
commit fa16c26
Show file tree

Hide file tree

Showing 4 changed files with 41 additions and 18 deletions.
diff --git a/server/ingester/app_log/dbwriter/log.go b/server/ingester/app_log/dbwriter/log.go
@@ -66,7 +66,7 @@ type ApplicationLogStore struct {
 	SpanID     string
 	TraceFlags uint32
 
-	SeverityText   string // numerical value of the severity(also known as log level id)
+	SeverityText   string // value of the severity(also known as log level)
 	SeverityNumber uint8  // numerical value of the severity(also known as log level id)
 
 	Body string

diff --git a/server/ingester/app_log/decoder/decoder.go b/server/ingester/app_log/decoder/decoder.go
@@ -20,9 +20,9 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
-	"log/syslog"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	logging "github.com/op/go-logging"
@@ -48,6 +48,34 @@ const (
 	SEPARATOR   = ", "
 )
 
+const (
+	SEVERITY_FATAL uint8 = 2 + iota // value as log/syslog.LOG_CRIT
+	SEVERITY_ERROR                  // value as log/syslog.LOG_ERR
+	SEVERITY_WARN                   // value as log/syslog.LOG_WARN
+	SEVERITY_INFO                   // value as log/syslog.LOG_INFO
+	SEVERITY_DEBUG                  // value as log/syslog.LOG_DEBUG
+	SEVERITY_TRACE
+	SEVERITY_UNKNOWN
+)
+
+func StringToSeverity(str string) uint8 {
+	switch strings.ToUpper(str) {
+	case "FATAL", "CRITICAL":
+		return SEVERITY_FATAL
+	case "ERROR":
+		return SEVERITY_ERROR
+	case "WARNING", "WARN":
+		return SEVERITY_WARN
+	case "INFO":
+		return SEVERITY_INFO
+	case "DEBUG":
+		return SEVERITY_DEBUG
+	case "TRACE":
+		return SEVERITY_TRACE
+	}
+	return SEVERITY_UNKNOWN
+}
+
 type Counter struct {
 	InCount    int64 `statsd:"in-count"`
 	OutCount   int64 `statsd:"out-count"`
@@ -173,23 +201,19 @@ func (d *Decoder) WriteAgentLog(agentId uint16, bs []byte) error {
 	s.AttributeValues = append(s.AttributeValues, host)
 	s.AppInstance = host
 
-	severity := syslog.Priority(0)
 	severityText := ""
 	switch string(columns[3]) {
 	case "[INFO]":
-		severity = syslog.LOG_INFO
 		severityText = "INFO"
 	case "[WARN]":
-		severity = syslog.LOG_WARNING
 		severityText = "WARN"
 	case "[ERRO]", "[ERROR]":
-		severity = syslog.LOG_ERR
 		severityText = "ERROR"
 	default:
 		return fmt.Errorf("ignored log level: %s", string(columns[3]))
 	}
 	s.SeverityText = severityText
-	s.SeverityNumber = uint8(severity)
+	s.SeverityNumber = StringToSeverity(severityText)
 	s.Body = string(columns[5])
 
 	s.AttributeNames = append(s.AttributeNames, "module")
@@ -235,16 +259,7 @@ func (d *Decoder) WriteAppLog(agentId uint16, l *AppLogEntry) error {
 
 	s.Body = l.Message
 	s.SeverityText = l.Level
-	severity := syslog.Priority(0)
-	switch l.Level {
-	case "INFO":
-		severity = syslog.LOG_INFO
-	case "WARN":
-		severity = syslog.LOG_WARNING
-	case "ERRO", "ERROR":
-		severity = syslog.LOG_ERR
-	}
-	s.SeverityNumber = uint8(severity)
+	s.SeverityNumber = StringToSeverity(l.Level)
 	s.AppInstance = l.Kubernetes.PodName
 
 	if l.Kubernetes.PodIp != "" {

diff --git a/server/querier/db_descriptions/clickhouse/tag/application_log/log b/server/querier/db_descriptions/clickhouse/tag/application_log/log
@@ -52,5 +52,5 @@ app_instance               , app_instance              , app_instance
 trace_id                  , trace_id                  , trace_id                   , string         ,                       , Tracing Info     , 111         , 0
 span_id                   , span_id                   , span_id                    , string         ,                       , Tracing Info     , 111         , 0
 severity_text             , severity_text             , severity_text              , string         ,                       , Tracing Info     , 111         , 0
-severity_number           , severity_number           , severity_number            , int            ,                       , Tracing Info     , 111         , 0
+severity_number           , severity_number           , severity_number            , int_enum       , severity_number       , Tracing Info     , 111         , 0
 body                      , body                      , body                       , string         ,                       , Tracing Info     , 111         , 0
diff --git a/server/querier/db_descriptions/clickhouse/tag/enum/severity_number b/server/querier/db_descriptions/clickhouse/tag/enum/severity_number
@@ -0,0 +1,8 @@
+# Value         , DisplayName    , Description
+2               , FATEL
+3               , ERROR
+4               , WARN
+6               , INFO
+7               , DEBUG
+8               , TRACE
+9               , UNKNOWN