Skip to content

Commit

Permalink
[FIX] handle nil on dashboard info collection (#40)
Browse files Browse the repository at this point in the history
  • Loading branch information
nicolastakashi committed May 15, 2023
1 parent 8a69828 commit eb80fe5
Show file tree
Hide file tree
Showing 6 changed files with 226 additions and 30 deletions.
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.2.2
1.2.3
4 changes: 2 additions & 2 deletions charts/cole/Chart.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,10 @@ type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 1.4.1
version: 1.4.2

# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "1.16.0"
appVersion: "1.2.3"
25 changes: 13 additions & 12 deletions charts/cole/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ image:
repository: ntakashi/cole
pullPolicy: IfNotPresent
# Overrides the image tag whose default is the chart appVersion.
tag: "1.2.0"
tag: "1.2.3"

imagePullSecrets: []
nameOverride: ""
Expand All @@ -23,17 +23,19 @@ serviceAccount:

podAnnotations: {}

podSecurityContext: {}
podSecurityContext:
{}
# fsGroup: 2000

securityContext:
securityContext:
readOnlyRootFilesystem: true

service:
type: ClusterIP
port: 80

resources: {}
resources:
{}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
Expand All @@ -53,10 +55,9 @@ affinity: {}

# Prometheus Operator ServiceMonitor configuration
serviceMonitor:

# if `true`, creates a Prometheus Operator ServiceMonitor
enabled: false

# Interval at which metrics should be scraped.
# ref: https://github.com/coreos/prometheus-operator/blob/master/Documentation/api.md#endpoint
interval: ""
Expand All @@ -76,7 +77,7 @@ grafanaApiSecret:
#
# grafanaApiSecret.data and grafanaApiSecret.secretKeyReference should be mutually exclusive.
# The secret key must be a yaml file with the following content:
#
#
# address: <address of grafana>
# apiKey: <key used to authenticate against grafana>
#
Expand All @@ -103,19 +104,19 @@ flags:
grafana:
# namespace where Grafana is running
namespace: grafana

# Grafana container name
containerName: grafana

# Grafana pod label selector
podLabelselector:
- name: app.kubernetes.io/name
value: grafana

log:
# Grafana pod log format
format: "console"

metrics:
# Include user name to metrics (disabled by default due to PII information)
includeUname: false
includeUname: false
11 changes: 10 additions & 1 deletion internal/cole/cole.go
Original file line number Diff line number Diff line change
Expand Up @@ -61,9 +61,18 @@ func (cole *Cole) Start() error {
case <-cole.GrafanaConfig.GrafanaApiPoolTime.C:
if cole.Scmd.GrafanaApiConfigFile != "" {
logrus.Info("starting pool grafana api")
dashboardinfos, err := grafana.GetDashboardInfo(cole.GrafanaConfig)

gc, err := cole.GrafanaConfig.NewClient()

if err != nil {
logrus.Error(err)
return err
}

dashboardinfos, err := gc.GetDashboardsInfo()
if err != nil {
logrus.Error(err)
return err
}

dm := metrics.DashboardMetrics{
Expand Down
47 changes: 33 additions & 14 deletions internal/grafana/client.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package grafana

import (
"io/ioutil"
"fmt"
"net/url"
"os"
"strings"
"time"

Expand All @@ -26,6 +27,10 @@ type GrafanaConfig struct {
ApiKey string `yaml:"apiKey"`
}

type GrafanaClient struct {
Api *gapi.Client
}

var search_dashboard_or_folder_latency = prometheus.NewHistogram(
prometheus.HistogramOpts{
Namespace: "cole",
Expand Down Expand Up @@ -56,7 +61,7 @@ var get_dashboard_error_total = prometheus.NewCounter(prometheus.CounterOpts{

func (gc *GrafanaConfig) ReadConfigFile(grafanaApiConfigFile string) error {
if grafanaApiConfigFile != "" {
file, err := ioutil.ReadFile(grafanaApiConfigFile)
file, err := os.ReadFile(grafanaApiConfigFile)
if err != nil {
logrus.Error("error to read grafana api config file")
return err
Expand All @@ -71,18 +76,23 @@ func (gc *GrafanaConfig) ReadConfigFile(grafanaApiConfigFile string) error {
return nil
}

func GetDashboardInfo(config GrafanaConfig) ([]DashboardInfo, error) {
c, err := gapi.New(config.Address, gapi.Config{
func (config GrafanaConfig) NewClient() (GrafanaClient, error) {
client, err := gapi.New(config.Address, gapi.Config{
APIKey: config.ApiKey,
})

if err != nil {
logrus.Error(err)
return nil, err
return GrafanaClient{}, err
}

return GrafanaClient{
Api: client,
}, nil
}

func (gc GrafanaClient) GetDashboardsInfo() ([]DashboardInfo, error) {
start := time.Now()
dashboards, err := c.FolderDashboardSearch(url.Values{
dashboards, err := gc.Api.FolderDashboardSearch(url.Values{
"type": []string{"dash-db"},
})

Expand All @@ -100,7 +110,7 @@ func GetDashboardInfo(config GrafanaConfig) ([]DashboardInfo, error) {
for _, dashboardSearchResponse := range dashboards {
start := time.Now()

dashboard, err := c.DashboardByUID(dashboardSearchResponse.UID)
dashboard, err := gc.Api.DashboardByUID(dashboardSearchResponse.UID)

if err != nil {

Expand All @@ -110,19 +120,28 @@ func GetDashboardInfo(config GrafanaConfig) ([]DashboardInfo, error) {
}

get_dashboard_error_total.Inc()
logrus.Error(err)
logrus.Error(fmt.Printf("%s %s", err, dashboardSearchResponse.UID))
continue
}

elapsedSeconds := time.Since(start).Seconds()
get_dashboard_latency.Observe(elapsedSeconds)

di := DashboardInfo{
UID: dashboardSearchResponse.UID,
IsStared: dashboard.Meta.IsStarred,
Version: dashboard.Model["version"].(float64),
SchemaVersion: dashboard.Model["schemaVersion"].(float64),
Timezone: dashboard.Model["timezone"].(string),
UID: dashboardSearchResponse.UID,
IsStared: dashboard.Meta.IsStarred,
}

if version, ok := dashboard.Model["version"].(float64); ok {
di.Version = version
}

if schemaVersion, ok := dashboard.Model["schemaVersion"].(float64); ok {
di.SchemaVersion = schemaVersion
}

if timezone, ok := dashboard.Model["timezone"].(string); ok {
di.Timezone = timezone
}

dashboardsInfos = append(dashboardsInfos, di)
Expand Down
167 changes: 167 additions & 0 deletions internal/grafana/client_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,167 @@
package grafana_test

import (
"fmt"
"net/http"
"net/http/httptest"
"net/url"
"testing"

gapi "github.com/grafana/grafana-api-golang-client"
"github.com/nicolastakashi/cole/internal/grafana"
"github.com/stretchr/testify/assert"
)

type mockServerCall struct {
code int
body string
}

type mockServer struct {
upcomingCalls []mockServerCall
executedCalls []mockServerCall
server *httptest.Server
}

func (m *mockServer) Close() {
m.server.Close()
}

func gapiTestToolsFromCalls(t *testing.T, calls []mockServerCall) *gapi.Client {
t.Helper()

mock := &mockServer{
upcomingCalls: calls,
}

mock.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
call := mock.upcomingCalls[0]
if len(calls) > 1 {
mock.upcomingCalls = mock.upcomingCalls[1:]
} else {
mock.upcomingCalls = nil
}
w.WriteHeader(call.code)
w.Header().Set("Content-Type", "application/json")
fmt.Fprint(w, call.body)
mock.executedCalls = append(mock.executedCalls, call)
}))

tr := &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
return url.Parse(mock.server.URL)
},
}

httpClient := &http.Client{Transport: tr}

client, err := gapi.New("http://my-grafana.com", gapi.Config{APIKey: "my-key", Client: httpClient})
if err != nil {
t.Fatal(err)
}

t.Cleanup(func() {
mock.Close()
})

return client
}

func TestGetDashboardInfoWithModelProperty(t *testing.T) {
const getFolderDashboardSearchResponse = `[
{
"id":1,
"uid": "cIBgcSjkk",
"title":"Production Overview",
"url": "/d/cIBgcSjkk/production-overview",
"type":"dash-db",
"tags":["prod"],
"isStarred":true,
"folderId": 2,
"folderUid": "000000163",
"folderTitle": "Folder",
"folderUrl": "/dashboards/f/000000163/folder",
"uri":"db/production-overview"
}
]`

const dashboard = `
{
"id":1,
"uid": "cIBgcSjkk",
"title":"Production Overview",
"url": "/d/cIBgcSjkk/production-overview",
"type":"dash-db",
"tags":["prod"],
"isStarred":true,
"folderId": 2,
"folderUid": "000000163",
"folderTitle": "Folder",
"folderUrl": "/dashboards/f/000000163/folder",
"uri":"db/production-overview",
"dashboard": {
"version": 1,
"schemaVersion": 36,
"timezone": "utc"
}
}`

gc := grafana.GrafanaClient{
Api: gapiTestToolsFromCalls(t, []mockServerCall{{200, getFolderDashboardSearchResponse}, {200, dashboard}}),
}

dashboardInfos, err := gc.GetDashboardsInfo()

assert.Nil(t, err)
assert.Len(t, dashboardInfos, 1)
assert.NotNil(t, dashboardInfos[0].Version)
assert.NotNil(t, dashboardInfos[0].SchemaVersion)
assert.NotNil(t, dashboardInfos[0].Timezone)
}

func TestGetDashboardInfoWithoutModelProperty(t *testing.T) {
const getFolderDashboardSearchResponse = `[
{
"id":1,
"uid": "cIBgcSjkk",
"title":"Production Overview",
"url": "/d/cIBgcSjkk/production-overview",
"type":"dash-db",
"tags":["prod"],
"isStarred":true,
"folderId": 2,
"folderUid": "000000163",
"folderTitle": "Folder",
"folderUrl": "/dashboards/f/000000163/folder",
"uri":"db/production-overview"
}
]`

const dashboard = `
{
"id":1,
"uid": "cIBgcSjkk",
"title":"Production Overview",
"url": "/d/cIBgcSjkk/production-overview",
"type":"dash-db",
"tags":["prod"],
"isStarred":true,
"folderId": 2,
"folderUid": "000000163",
"folderTitle": "Folder",
"folderUrl": "/dashboards/f/000000163/folder",
"uri":"db/production-overview"
}`

gc := grafana.GrafanaClient{
Api: gapiTestToolsFromCalls(t, []mockServerCall{{200, getFolderDashboardSearchResponse}, {200, dashboard}}),
}

dashboardInfos, err := gc.GetDashboardsInfo()

assert.Nil(t, err)
assert.Len(t, dashboardInfos, 1)
assert.Equal(t, dashboardInfos[0].Version, float64(0))
assert.Equal(t, dashboardInfos[0].SchemaVersion, float64(0))
assert.Equal(t, dashboardInfos[0].Timezone, "")
}

0 comments on commit eb80fe5

Please sign in to comment.