diff --git a/go.mod b/go.mod index cea64cc3..dc157dd6 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,7 @@ require ( go.opentelemetry.io/otel/metric v1.21.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect golang.org/x/crypto v0.16.0 // indirect + golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/sync v0.5.0 // indirect diff --git a/go.sum b/go.sum index fa5b3d92..62f8c347 100644 --- a/go.sum +++ b/go.sum @@ -227,6 +227,8 @@ golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq golang.org/x/exp v0.0.0-20231127185646-65229373498e h1:Gvh4YaCaXNs6dKTlfgismwWZKyjVZXwOPfIyUaqU3No= golang.org/x/exp v0.0.0-20231127185646-65229373498e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= diff --git a/pkg/controller/metadata.go b/pkg/controller/metadata.go new file mode 100644 index 00000000..f2485c2f --- /dev/null +++ b/pkg/controller/metadata.go @@ -0,0 +1,24 @@ +package controller + +import ( + "context" + + goGitlab "github.com/xanzy/go-gitlab" + + "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab" +) + +func (c *Controller) GetGitLabMetadata(ctx context.Context) error { + options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)} + + metadata, _, err := c.Gitlab.Metadata.GetMetadata(options...) + if err != nil { + return err + } + + if metadata.Version != "" { + c.Gitlab.UpdateVersion(gitlab.NewGitLabVersion(metadata.Version)) + } + + return nil +} diff --git a/pkg/controller/metadata_test.go b/pkg/controller/metadata_test.go new file mode 100644 index 00000000..5fe830f0 --- /dev/null +++ b/pkg/controller/metadata_test.go @@ -0,0 +1,61 @@ +package controller + +import ( + "fmt" + "net/http" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config" + "github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab" +) + +func TestGetGitLabMetadataSuccess(t *testing.T) { + tests := []struct { + name string + data string + expectedVersion gitlab.GitLabVersion + }{ + { + name: "successful parse", + data: ` +{ +"version":"16.7.0-pre", +"revision":"3fe364fe754", +"kas":{ + "enabled":true, + "externalUrl":"wss://kas.gitlab.com", + "version":"v16.7.0-rc2" +}, +"enterprise":true +} +`, + expectedVersion: gitlab.NewGitLabVersion("v16.7.0-pre"), + }, + { + name: "unsuccessful parse", + data: ` +{ +"revision":"3fe364fe754" +} +`, + expectedVersion: gitlab.NewGitLabVersion(""), + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, c, mux, srv := newTestController(config.Config{}) + defer srv.Close() + + mux.HandleFunc("/api/v4/metadata", + func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, tc.data) + }) + + assert.NoError(t, c.GetGitLabMetadata(ctx)) + assert.Equal(t, tc.expectedVersion, c.Gitlab.Version()) + }) + } +} diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 7a3ee763..37852af6 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -307,6 +307,10 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule") defer span.End() + go func() { + c.GetGitLabMetadata(ctx) + }() + for tt, cfg := range map[schemas.TaskType]config.SchedulerConfig{ schemas.TaskTypePullProjectsFromWildcards: config.SchedulerConfig(pull.ProjectsFromWildcards), schemas.TaskTypePullEnvironmentsFromProjects: config.SchedulerConfig(pull.EnvironmentsFromProjects), diff --git a/pkg/gitlab/client.go b/pkg/gitlab/client.go index 64cefc69..2d979e28 100644 --- a/pkg/gitlab/client.go +++ b/pkg/gitlab/client.go @@ -6,6 +6,7 @@ import ( "fmt" "net/http" "strconv" + "sync" "sync/atomic" "time" @@ -36,6 +37,9 @@ type Client struct { RequestsCounter atomic.Uint64 RequestsLimit int RequestsRemaining int + + version GitLabVersion + mutex sync.RWMutex } // ClientConfig .. @@ -140,6 +144,19 @@ func (c *Client) rateLimit(ctx context.Context) { c.RequestsCounter.Add(1) } +func (c *Client) UpdateVersion(version GitLabVersion) { + c.mutex.Lock() + defer c.mutex.Unlock() + c.version = version +} + +func (c *Client) Version() GitLabVersion { + c.mutex.RLock() + defer c.mutex.RUnlock() + + return c.version +} + func (c *Client) requestsRemaining(response *goGitlab.Response) { if response == nil { return diff --git a/pkg/gitlab/jobs.go b/pkg/gitlab/jobs.go index e728db31..156caac5 100644 --- a/pkg/gitlab/jobs.go +++ b/pkg/gitlab/jobs.go @@ -231,13 +231,24 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo var ( foundJobs []*goGitlab.Job resp *goGitlab.Response + opt *goGitlab.ListJobsOptions ) - opt := &goGitlab.ListJobsOptions{ - ListOptions: goGitlab.ListOptions{ - Pagination: "keyset", - PerPage: 100, - }, + keysetPagination := c.Version().PipelineJobsKeysetPaginationSupported() + if keysetPagination { + opt = &goGitlab.ListJobsOptions{ + ListOptions: goGitlab.ListOptions{ + Pagination: "keyset", + PerPage: 100, + }, + } + } else { + opt = &goGitlab.ListJobsOptions{ + ListOptions: goGitlab.ListOptions{ + Page: 1, + PerPage: 100, + }, + } } options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)} @@ -274,7 +285,8 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo } } - if resp.NextLink == "" { + if keysetPagination && resp.NextLink == "" || + (!keysetPagination && resp.CurrentPage >= resp.NextPage) { var notFoundJobs []string for k := range jobsToRefresh { @@ -295,9 +307,11 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo break } - options = []goGitlab.RequestOptionFunc{ - goGitlab.WithContext(ctx), - goGitlab.WithKeysetPaginationParameters(resp.NextLink), + if keysetPagination { + options = []goGitlab.RequestOptionFunc{ + goGitlab.WithContext(ctx), + goGitlab.WithKeysetPaginationParameters(resp.NextLink), + } } } diff --git a/pkg/gitlab/jobs_test.go b/pkg/gitlab/jobs_test.go index a7a45e4a..93b28813 100644 --- a/pkg/gitlab/jobs_test.go +++ b/pkg/gitlab/jobs_test.go @@ -132,64 +132,93 @@ func TestListPipelineBridges(t *testing.T) { } func TestListRefMostRecentJobs(t *testing.T) { - ctx, mux, server, c := getMockedClient() - defer server.Close() - - ref := schemas.Ref{ - Project: schemas.NewProject("foo"), - Name: "yay", + tests := []struct { + name string + keysetPagination bool + expectedQueryParams url.Values + }{ + { + name: "offset pagination", + keysetPagination: false, + expectedQueryParams: url.Values{ + "page": []string{"1"}, + "per_page": []string{"100"}, + }, + }, + { + name: "keyset pagination", + keysetPagination: true, + expectedQueryParams: url.Values{ + "pagination": []string{"keyset"}, + "per_page": []string{"100"}, + }, + }, } - jobs, err := c.ListRefMostRecentJobs(ctx, ref) - assert.NoError(t, err) - assert.Len(t, jobs, 0) + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + ctx, mux, server, c := getMockedClient() + defer server.Close() - mux.HandleFunc("/api/v4/projects/foo/jobs", - func(w http.ResponseWriter, r *http.Request) { - assert.Equal(t, "GET", r.Method) - expectedQueryParams := url.Values{ - "pagination": []string{"keyset"}, - "per_page": []string{"100"}, + if tc.keysetPagination { + c.UpdateVersion(NewGitLabVersion("16.0.0")) + } else { + c.UpdateVersion(NewGitLabVersion("15.0.0")) } - assert.Equal(t, expectedQueryParams, r.URL.Query()) - fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`) - }) - mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"), - func(w http.ResponseWriter, r *http.Request) { - w.WriteHeader(http.StatusNotFound) - }) + ref := schemas.Ref{ + Project: schemas.NewProject("foo"), + Name: "yay", + } - ref.LatestJobs = schemas.Jobs{ - "foo": { - ID: 1, - Name: "foo", - }, - "bar": { - ID: 2, - Name: "bar", - }, - } + jobs, err := c.ListRefMostRecentJobs(ctx, ref) + assert.NoError(t, err) + assert.Len(t, jobs, 0) + + mux.HandleFunc("/api/v4/projects/foo/jobs", + func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "GET", r.Method) + assert.Equal(t, tc.expectedQueryParams, r.URL.Query()) + fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`) + }) + + mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"), + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusNotFound) + }) + + ref.LatestJobs = schemas.Jobs{ + "foo": { + ID: 1, + Name: "foo", + }, + "bar": { + ID: 2, + Name: "bar", + }, + } - jobs, err = c.ListRefMostRecentJobs(ctx, ref) - assert.NoError(t, err) - assert.Len(t, jobs, 2) - assert.Equal(t, 3, jobs[0].ID) - assert.Equal(t, 4, jobs[1].ID) + jobs, err = c.ListRefMostRecentJobs(ctx, ref) + assert.NoError(t, err) + assert.Len(t, jobs, 2) + assert.Equal(t, 3, jobs[0].ID) + assert.Equal(t, 4, jobs[1].ID) - ref.LatestJobs["baz"] = schemas.Job{ - ID: 5, - Name: "baz", - } + ref.LatestJobs["baz"] = schemas.Job{ + ID: 5, + Name: "baz", + } - jobs, err = c.ListRefMostRecentJobs(ctx, ref) - assert.NoError(t, err) - assert.Len(t, jobs, 2) - assert.Equal(t, 3, jobs[0].ID) - assert.Equal(t, 4, jobs[1].ID) + jobs, err = c.ListRefMostRecentJobs(ctx, ref) + assert.NoError(t, err) + assert.Len(t, jobs, 2) + assert.Equal(t, 3, jobs[0].ID) + assert.Equal(t, 4, jobs[1].ID) - // Test invalid project id - ref.Project.Name = "bar" - _, err = c.ListRefMostRecentJobs(ctx, ref) - assert.Error(t, err) + // Test invalid project id + ref.Project.Name = "bar" + _, err = c.ListRefMostRecentJobs(ctx, ref) + assert.Error(t, err) + }) + } } diff --git a/pkg/gitlab/version.go b/pkg/gitlab/version.go new file mode 100644 index 00000000..efb5dcfb --- /dev/null +++ b/pkg/gitlab/version.go @@ -0,0 +1,32 @@ +package gitlab + +import ( + "strings" + + "golang.org/x/mod/semver" +) + +type GitLabVersion struct { + Version string +} + +func NewGitLabVersion(version string) GitLabVersion { + ver := "" + if strings.HasPrefix(version, "v") { + ver = version + } else if version != "" { + ver = "v" + version + } + + return GitLabVersion{Version: ver} +} + +// PipelineJobsKeysetPaginationSupported returns true if the GitLab instance +// is running 15.9 or later. +func (v GitLabVersion) PipelineJobsKeysetPaginationSupported() bool { + if v.Version == "" { + return false + } + + return semver.Compare(v.Version, "v15.9.0") >= 0 +} diff --git a/pkg/gitlab/version_test.go b/pkg/gitlab/version_test.go new file mode 100644 index 00000000..d12febf4 --- /dev/null +++ b/pkg/gitlab/version_test.go @@ -0,0 +1,64 @@ +package gitlab + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestPipelineJobsKeysetPaginationSupported(t *testing.T) { + tests := []struct { + name string + version GitLabVersion + expectedResult bool + }{ + { + name: "unknown", + version: NewGitLabVersion(""), + expectedResult: false, + }, + { + name: "15.8.0", + version: NewGitLabVersion("15.8.0"), + expectedResult: false, + }, + { + name: "v15.8.0", + version: NewGitLabVersion("v15.8.0"), + expectedResult: false, + }, + { + name: "15.9.0", + version: NewGitLabVersion("15.9.0"), + expectedResult: true, + }, + { + name: "v15.9.0", + version: NewGitLabVersion("v15.9.0"), + expectedResult: true, + }, + { + name: "15.9.1", + version: NewGitLabVersion("15.9.1"), + expectedResult: true, + }, + { + name: "15.10.2", + version: NewGitLabVersion("15.10.2"), + expectedResult: true, + }, + { + name: "16.0.0", + version: NewGitLabVersion("16.0.0"), + expectedResult: true, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + result := tc.version.PipelineJobsKeysetPaginationSupported() + + assert.Equal(t, tc.expectedResult, result) + }) + } +}