Feat/tm gitlab pagination

pkg/sparrow/targets/remote/gitlab/gitlab.go

@@ -26,6 +26,7 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strconv"
 	"strings"
 	"time"
 
@@ -35,6 +36,9 @@ import (
 	"github.com/caas-team/sparrow/internal/logger"
 )
 
+// The amount of items the paginated request to gitlab should return
+const paginationPerPage = 30
+
 var _ remote.Interactor = (*client)(nil)
 
 // client is the implementation of the remote.Interactor for gitlab
@@ -143,26 +147,42 @@ func (c *client) fetchFile(ctx context.Context, f string) (checks.GlobalTarget,
 // so they may be fetched individually
 func (c *client) fetchFileList(ctx context.Context) ([]string, error) {
 	log := logger.FromContext(ctx)
-	log.DebugContext(ctx, "Fetching file list from gitlab")
+	log.DebugContext(ctx, "Preparing to fetch file list from gitlab")
+
+	rawUrl := fmt.Sprintf("%s/api/v4/projects/%d/repository/tree", c.config.BaseURL, c.config.ProjectID)
+	reqUrl, err := url.Parse(rawUrl)
+	if err != nil {
+		log.ErrorContext(ctx, "Could not parse GitLab API repository URL", "url", rawUrl, "error", err)
+	}
+	query := reqUrl.Query()
+	query.Set("pagination", "keyset")
+	query.Set("per_page", strconv.Itoa(paginationPerPage))
+	query.Set("order_by", "id")
+	query.Set("sort", "asc")
+
+	query.Set("ref", c.config.Branch)
+	reqUrl.RawQuery = query.Encode()
+
+	return c.fetchNextFileList(ctx, reqUrl.String())
+}
+
+// fetchNextFileList is fetching files from GitLab.
+// Gitlab pagination is handled recursively.
+func (c *client) fetchNextFileList(ctx context.Context, reqUrl string) ([]string, error) {
+	log := logger.FromContext(ctx)
+	log.DebugContext(ctx, "Fetching file list page from gitlab")
+
 	type file struct {
 		Name string `json:"name"`
 	}
 
-	req, err := http.NewRequestWithContext(ctx,
-		http.MethodGet,
-		fmt.Sprintf("%s/api/v4/projects/%d/repository/tree", c.config.BaseURL, c.config.ProjectID),
-		http.NoBody,
-	)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqUrl, http.NoBody)
 	if err != nil {
 		log.ErrorContext(ctx, "Failed to create request", "error", err)
 		return nil, err
 	}
-
 	req.Header.Add("PRIVATE-TOKEN", c.config.Token)
 	req.Header.Add("Content-Type", "application/json")
-	query := req.URL.Query()
-	query.Add("ref", c.config.Branch)
-	req.URL.RawQuery = query.Encode()
 
 	resp, err := c.client.Do(req)
 	if err != nil {
@@ -193,7 +213,16 @@ func (c *client) fetchFileList(ctx context.Context) ([]string, error) {
 		}
 	}
 
-	log.DebugContext(ctx, "Successfully fetched file list", "files", len(files))
+	if nextLink := getNextLink(resp.Header); nextLink != "" {
+		nextFiles, err := c.fetchNextFileList(ctx, nextLink)
+		if err != nil {
+			return nil, err
+		}
+		log.DebugContext(ctx, "Successfully fetched next file page, adding to file list")
+		files = append(files, nextFiles...)
+	}
+
+	log.DebugContext(ctx, "Successfully fetched file list recursively", "files", len(files))
 	return files, nil
 }
 

pkg/sparrow/targets/remote/gitlab/gitlab_test.go

@@ -28,7 +28,6 @@ import (
 
 	"github.com/caas-team/sparrow/pkg/checks"
 	"github.com/caas-team/sparrow/pkg/sparrow/targets/remote"
-
 	"github.com/jarcoal/httpmock"
 )
 
@@ -108,7 +107,7 @@ func Test_gitlab_fetchFileList(t *testing.T) {
 			if err != nil {
 				t.Fatalf("error creating mock response: %v", err)
 			}
-			httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp)
+			httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/tree?order_by=id&pagination=keyset&per_page=%d&ref=main&sort=asc", paginationPerPage), resp)
 
 			g := &client{
 				config: Config{
@@ -218,7 +217,7 @@ func Test_gitlab_FetchFiles(t *testing.T) {
 			if err != nil {
 				t.Fatalf("error creating mock response: %v", err)
 			}
-			httpmock.RegisterResponder("GET", "http://test/api/v4/projects/1/repository/tree?ref=main", resp)
+			httpmock.RegisterResponder("GET", fmt.Sprintf("http://test/api/v4/projects/1/repository/tree?order_by=id&pagination=keyset&per_page=%d&ref=main&sort=asc", paginationPerPage), resp)
 
 			got, err := g.FetchFiles(context.Background())
 			if (err != nil) != tt.wantErr {
@@ -612,3 +611,134 @@ func TestClient_fetchDefaultBranch(t *testing.T) {
 		})
 	}
 }
+
+func Test_client_fetchNextFileList(t *testing.T) {
+	type mockRespFile struct {
+		Name string `json:"name"`
+	}
+	type mockResponder struct {
+		reqUrl     string
+		linkHeader string
+		statusCode int
+		response   []mockRespFile
+	}
+
+	tests := []struct {
+		name    string
+		mock    []mockResponder
+		want    []string
+		wantErr bool
+	}{
+		{
+			name: "success - no pagination",
+			mock: []mockResponder{
+				{
+					reqUrl:     "https://test.de/pagination",
+					linkHeader: "",
+					statusCode: http.StatusOK,
+					response:   []mockRespFile{{Name: "file1.json"}, {Name: "file2.json"}},
+				},
+			},
+			want: []string{
+				"file1.json",
+				"file2.json",
+			},
+			wantErr: false,
+		},
+		{
+			name: "success - with pagination",
+			mock: []mockResponder{
+				{
+					reqUrl:     "https://test.de/pagination",
+					linkHeader: "<https://test.de/pagination?page=2>; rel=\"next\"",
+					statusCode: http.StatusOK,
+					response:   []mockRespFile{{Name: "file1.json"}},
+				},
+				{
+					reqUrl:     "https://test.de/pagination?page=2",
+					linkHeader: "<https://test.de/pagination?page=3>; rel=\"next\"",
+					statusCode: http.StatusOK,
+					response:   []mockRespFile{{Name: "file2.json"}},
+				},
+				{
+					reqUrl:     "https://test.de/pagination?page=3",
+					linkHeader: "",
+					statusCode: http.StatusOK,
+					response:   []mockRespFile{{Name: "file3.json"}},
+				},
+			},
+			want: []string{
+				"file1.json",
+				"file2.json",
+				"file3.json",
+			},
+			wantErr: false,
+		},
+		{
+			name: "fail - status code nok while paginated requests",
+			mock: []mockResponder{
+				{
+					reqUrl:     "https://test.de/pagination",
+					linkHeader: "<https://test.de/pagination?page=2>; rel=\"next\"",
+					statusCode: http.StatusOK,
+					response:   []mockRespFile{{Name: "file1.json"}},
+				},
+				{
+					reqUrl:     "https://test.de/pagination?page=2",
+					linkHeader: "",
+					statusCode: http.StatusBadRequest,
+					response:   []mockRespFile{{Name: "file2.json"}},
+				},
+			},
+			want:    []string{},
+			wantErr: true,
+		},
+	}
+
+	httpmock.Activate()
+	defer httpmock.DeactivateAndReset()
+	c := &client{
+		config: Config{
+			BaseURL:   "https://test.de",
+			ProjectID: 1,
+			Token:     "test",
+		},
+		client: http.DefaultClient,
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			// prepare http mock responder for paginated requests
+			for _, responder := range tt.mock {
+				httpmock.RegisterResponder(http.MethodGet, responder.reqUrl, func(req *http.Request) (*http.Response, error) {
+					// Check if header are properly set
+					token := req.Header.Get("PRIVATE-TOKEN")
+					cType := req.Header.Get("Content-Type")
+					if token == "" || cType == "" {
+						t.Error("Some header not properly set", "PRIVATE-TOKEN", token != "", "Content-Type", cType != "")
+					}
+
+					resp, err := httpmock.NewJsonResponse(responder.statusCode, responder.response)
+
+					// Add link header for next page (pagination)
+					resp.Header.Set(linkHeader, responder.linkHeader)
+					return resp, err
+				})
+			}
+
+			got, err := c.fetchNextFileList(context.Background(), tt.mock[0].reqUrl)
+			if err != nil {
+				if !tt.wantErr {
+					t.Fatalf("fetchNextFileList() error = %v, wantErr %v", err, tt.wantErr)
+				}
+				return
+			}
+			if tt.wantErr {
+				t.Fatalf("fetchNextFileList() error = %v, wantErr %v", err, tt.wantErr)
+			}
+			if !reflect.DeepEqual(got, tt.want) {
+				t.Fatalf("fetchNextFileList() got = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

pkg/sparrow/targets/remote/gitlab/pagination.go

@@ -0,0 +1,52 @@
+// sparrow
+// (C) 2024, Deutsche Telekom IT GmbH
+//
+// Deutsche Telekom IT GmbH and all other contributors /
+// copyright owners license this file to you under the Apache
+// License, Version 2.0 (the "License"); you may not use this
+// file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package gitlab
+
+import (
+	"net/http"
+	"strings"
+)
+
+const (
+	linkHeader = "Link"
+	linkNext   = "next"
+)
+
+// getNextLink returns the url to the next page of
+// a paginated http response provided in the passed response header.
+func getNextLink(header http.Header) string {
+	link := header.Get(linkHeader)
+	if link == "" {
+		return ""
+	}
+
+	for _, link := range strings.Split(link, ",") {
+		linkParts := strings.Split(link, ";")
+		if len(linkParts) != 2 {
+			continue
+		}
+		linkType := strings.Trim(strings.Split(linkParts[1], "=")[1], "\"")
+
+		if linkType != linkNext {
+			continue
+		}
+		return strings.Trim(linkParts[0], "< >")
+	}
+	return ""
+}

pkg/sparrow/targets/remote/gitlab/pagination_test.go

@@ -0,0 +1,79 @@
+// sparrow
+// (C) 2024, Deutsche Telekom IT GmbH
+//
+// Deutsche Telekom IT GmbH and all other contributors /
+// copyright owners license this file to you under the Apache
+// License, Version 2.0 (the "License"); you may not use this
+// file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package gitlab
+
+import (
+	"net/http"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestGetNextLink(t *testing.T) {
+	type header struct {
+		noLinkHeader bool
+		key          string
+		value        string
+	}
+	tests := []struct {
+		name   string
+		header header
+		want   string
+	}{
+		{
+			"no link header present",
+			header{
+				noLinkHeader: true,
+			},
+			"",
+		},
+		{
+			"no next link in link header present",
+			header{
+				key:   "link",
+				value: "<https://link.first.de>; rel=\"first\", <https://link.last.de>; rel=\"last\"",
+			},
+			"",
+		},
+		{
+			"link header syntax not valid",
+			header{
+				key:   "link",
+				value: "no link here",
+			},
+			"",
+		},
+		{
+			"valid next link",
+			header{
+				key:   "link",
+				value: "<https://link.next.de>; rel=\"next\", <https://link.first.de>; rel=\"first\", <https://link.last.de>; rel=\"last\"",
+			},
+			"https://link.next.de",
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			testHeader := http.Header{}
+			testHeader.Add(tt.header.key, tt.header.value)
+
+			assert.Equal(t, tt.want, getNextLink(testHeader))
+		})
+	}
+}