fix: use keyset pagination only when it is supported

pkg/controller/metadata.go

@@ -0,0 +1,53 @@
+package controller
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+
+	goGitlab "github.com/xanzy/go-gitlab"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
+)
+
+func parseVersionRegex(version string) (int, int, int, string) {
+	regexPattern := `^(\d+)\.(\d+)\.(\d+)(?:-(.*))?$`
+	r := regexp.MustCompile(regexPattern)
+
+	matches := r.FindStringSubmatch(version)
+
+	if matches == nil {
+		return 0, 0, 0, "Invalid version format"
+	}
+
+	major, _ := strconv.Atoi(matches[1])
+	minor, _ := strconv.Atoi(matches[2])
+	patch, _ := strconv.Atoi(matches[3])
+
+	suffix := matches[4]
+
+	return major, minor, patch, suffix
+}
+
+func (c *Controller) GetGitLabMetadata(ctx context.Context) error {
+	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
+
+	metadata, _, err := c.Gitlab.Metadata.GetMetadata(options...)
+	if err != nil {
+		return err
+	}
+
+	if metadata.Version != "" {
+		major, minor, patch, suffix := parseVersionRegex(metadata.Version)
+		c.Gitlab.UpdateVersion(
+			gitlab.GitLabVersion{
+				Major:  major,
+				Minor:  minor,
+				Patch:  patch,
+				Suffix: suffix,
+			},
+		)
+	}
+
+	return nil
+}

pkg/controller/metadata_test.go

@@ -0,0 +1,61 @@
+package controller
+
+import (
+	"fmt"
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/config"
+	"github.com/mvisonneau/gitlab-ci-pipelines-exporter/pkg/gitlab"
+)
+
+func TestGetGitLabMetadataSuccess(t *testing.T) {
+	tests := []struct {
+		name            string
+		data            string
+		expectedVersion gitlab.GitLabVersion
+	}{
+		{
+			name: "successful parse",
+			data: `
+{
+"version":"16.7.0-pre",
+"revision":"3fe364fe754",
+"kas":{
+	"enabled":true,
+	"externalUrl":"wss://kas.gitlab.com",
+	"version":"v16.7.0-rc2"
+},
+"enterprise":true
+}
+`,
+			expectedVersion: gitlab.GitLabVersion{Major: 16, Minor: 7, Patch: 0, Suffix: "pre"},
+		},
+		{
+			name: "unsuccessful parse",
+			data: `
+{
+"version":"16.7"
+}
+`,
+			expectedVersion: gitlab.GitLabVersion{Major: 0, Minor: 0, Patch: 0, Suffix: "Invalid version format"},
+		},
+	}
+
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, c, mux, srv := newTestController(config.Config{})
+			defer srv.Close()
+
+			mux.HandleFunc("/api/v4/metadata",
+				func(w http.ResponseWriter, r *http.Request) {
+					fmt.Fprint(w, tc.data)
+				})
+
+			assert.NoError(t, c.GetGitLabMetadata(ctx))
+			assert.Equal(t, tc.expectedVersion, c.Gitlab.Version())
+		})
+	}
+}

pkg/controller/scheduler.go

@@ -307,6 +307,10 @@ func (c *Controller) Schedule(ctx context.Context, pull config.Pull, gc config.G
 	ctx, span := otel.Tracer(tracerName).Start(ctx, "controller:Schedule")
 	defer span.End()
 
+	go func() {
+		c.GetGitLabMetadata(ctx)
+	}()
+
 	for tt, cfg := range map[schemas.TaskType]config.SchedulerConfig{
 		schemas.TaskTypePullProjectsFromWildcards:    config.SchedulerConfig(pull.ProjectsFromWildcards),
 		schemas.TaskTypePullEnvironmentsFromProjects: config.SchedulerConfig(pull.EnvironmentsFromProjects),

pkg/gitlab/client.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 	"strconv"
+	"sync"
 	"sync/atomic"
 	"time"
 
@@ -36,6 +37,9 @@ type Client struct {
 	RequestsCounter   atomic.Uint64
 	RequestsLimit     int
 	RequestsRemaining int
+
+	version GitLabVersion
+	mutex   sync.RWMutex
 }
 
 // ClientConfig ..
@@ -140,6 +144,19 @@ func (c *Client) rateLimit(ctx context.Context) {
 	c.RequestsCounter.Add(1)
 }
 
+func (c *Client) UpdateVersion(version GitLabVersion) {
+	c.mutex.Lock()
+	defer c.mutex.Unlock()
+	c.version = version
+}
+
+func (c *Client) Version() GitLabVersion {
+	c.mutex.RLock()
+	defer c.mutex.RUnlock()
+
+	return c.version
+}
+
 func (c *Client) requestsRemaining(response *goGitlab.Response) {
 	if response == nil {
 		return

pkg/gitlab/jobs.go

@@ -231,13 +231,24 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 	var (
 		foundJobs []*goGitlab.Job
 		resp      *goGitlab.Response
+		opt       *goGitlab.ListJobsOptions
 	)
 
-	opt := &goGitlab.ListJobsOptions{
-		ListOptions: goGitlab.ListOptions{
-			Pagination: "keyset",
-			PerPage:    100,
-		},
+	keysetPagination := c.Version().PipelineJobsKeysetPaginationSupported()
+	if keysetPagination {
+		opt = &goGitlab.ListJobsOptions{
+			ListOptions: goGitlab.ListOptions{
+				Pagination: "keyset",
+				PerPage:    100,
+			},
+		}
+	} else {
+		opt = &goGitlab.ListJobsOptions{
+			ListOptions: goGitlab.ListOptions{
+				Page:    1,
+				PerPage: 100,
+			},
+		}
 	}
 
 	options := []goGitlab.RequestOptionFunc{goGitlab.WithContext(ctx)}
@@ -274,7 +285,8 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			}
 		}
 
-		if resp.NextLink == "" {
+		if keysetPagination && resp.NextLink == "" ||
+			(!keysetPagination && resp.CurrentPage >= resp.NextPage) {
 			var notFoundJobs []string
 
 			for k := range jobsToRefresh {
@@ -295,9 +307,11 @@ func (c *Client) ListRefMostRecentJobs(ctx context.Context, ref schemas.Ref) (jo
 			break
 		}
 
-		options = []goGitlab.RequestOptionFunc{
-			goGitlab.WithContext(ctx),
-			goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+		if keysetPagination {
+			options = []goGitlab.RequestOptionFunc{
+				goGitlab.WithContext(ctx),
+				goGitlab.WithKeysetPaginationParameters(resp.NextLink),
+			}
 		}
 	}
 

pkg/gitlab/jobs_test.go

@@ -132,64 +132,93 @@ func TestListPipelineBridges(t *testing.T) {
 }
 
 func TestListRefMostRecentJobs(t *testing.T) {
-	ctx, mux, server, c := getMockedClient()
-	defer server.Close()
-
-	ref := schemas.Ref{
-		Project: schemas.NewProject("foo"),
-		Name:    "yay",
+	tests := []struct {
+		name                string
+		keysetPagination    bool
+		expectedQueryParams url.Values
+	}{
+		{
+			name:             "offset pagination",
+			keysetPagination: false,
+			expectedQueryParams: url.Values{
+				"page":     []string{"1"},
+				"per_page": []string{"100"},
+			},
+		},
+		{
+			name:             "keyset pagination",
+			keysetPagination: true,
+			expectedQueryParams: url.Values{
+				"pagination": []string{"keyset"},
+				"per_page":   []string{"100"},
+			},
+		},
 	}
 
-	jobs, err := c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 0)
+	for _, tc := range tests {
+		t.Run(tc.name, func(t *testing.T) {
+			ctx, mux, server, c := getMockedClient()
+			defer server.Close()
 
-	mux.HandleFunc("/api/v4/projects/foo/jobs",
-		func(w http.ResponseWriter, r *http.Request) {
-			assert.Equal(t, "GET", r.Method)
-			expectedQueryParams := url.Values{
-				"pagination": []string{"keyset"},
-				"per_page":   []string{"100"},
+			if tc.keysetPagination {
+				c.UpdateVersion(GitLabVersion{Major: 16, Minor: 0})
+			} else {
+				c.UpdateVersion(GitLabVersion{Major: 15, Minor: 0})
 			}
-			assert.Equal(t, expectedQueryParams, r.URL.Query())
-			fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
-		})
 
-	mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
-		func(w http.ResponseWriter, r *http.Request) {
-			w.WriteHeader(http.StatusNotFound)
-		})
+			ref := schemas.Ref{
+				Project: schemas.NewProject("foo"),
+				Name:    "yay",
+			}
 
-	ref.LatestJobs = schemas.Jobs{
-		"foo": {
-			ID:   1,
-			Name: "foo",
-		},
-		"bar": {
-			ID:   2,
-			Name: "bar",
-		},
-	}
+			jobs, err := c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 0)
+
+			mux.HandleFunc("/api/v4/projects/foo/jobs",
+				func(w http.ResponseWriter, r *http.Request) {
+					assert.Equal(t, "GET", r.Method)
+					assert.Equal(t, tc.expectedQueryParams, r.URL.Query())
+					fmt.Fprint(w, `[{"id":3,"name":"foo","ref":"yay"},{"id":4,"name":"bar","ref":"yay"}]`)
+				})
+
+			mux.HandleFunc(fmt.Sprintf("/api/v4/projects/bar/jobs"),
+				func(w http.ResponseWriter, r *http.Request) {
+					w.WriteHeader(http.StatusNotFound)
+				})
+
+			ref.LatestJobs = schemas.Jobs{
+				"foo": {
+					ID:   1,
+					Name: "foo",
+				},
+				"bar": {
+					ID:   2,
+					Name: "bar",
+				},
+			}
 
-	jobs, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 2)
-	assert.Equal(t, 3, jobs[0].ID)
-	assert.Equal(t, 4, jobs[1].ID)
+			jobs, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 2)
+			assert.Equal(t, 3, jobs[0].ID)
+			assert.Equal(t, 4, jobs[1].ID)
 
-	ref.LatestJobs["baz"] = schemas.Job{
-		ID:   5,
-		Name: "baz",
-	}
+			ref.LatestJobs["baz"] = schemas.Job{
+				ID:   5,
+				Name: "baz",
+			}
 
-	jobs, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.NoError(t, err)
-	assert.Len(t, jobs, 2)
-	assert.Equal(t, 3, jobs[0].ID)
-	assert.Equal(t, 4, jobs[1].ID)
+			jobs, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.NoError(t, err)
+			assert.Len(t, jobs, 2)
+			assert.Equal(t, 3, jobs[0].ID)
+			assert.Equal(t, 4, jobs[1].ID)
 
-	// Test invalid project id
-	ref.Project.Name = "bar"
-	_, err = c.ListRefMostRecentJobs(ctx, ref)
-	assert.Error(t, err)
+			// Test invalid project id
+			ref.Project.Name = "bar"
+			_, err = c.ListRefMostRecentJobs(ctx, ref)
+			assert.Error(t, err)
+		})
+	}
 }

pkg/gitlab/version.go

@@ -0,0 +1,22 @@
+package gitlab
+
+type GitLabVersion struct {
+	Major  int
+	Minor  int
+	Patch  int
+	Suffix string
+}
+
+// PipelineJobsKeysetPaginationSupported returns true if the GitLab instance
+// is running 15.9 or later.
+func (v GitLabVersion) PipelineJobsKeysetPaginationSupported() bool {
+	if v.Major == 0 {
+		return false
+	} else if v.Major < 15 {
+		return false
+	} else if v.Major > 15 {
+		return true
+	} else {
+		return v.Minor >= 9
+	}
+}