diff --git a/backend/plugins/gitlab/tasks/account_collector.go b/backend/plugins/gitlab/tasks/account_collector.go index 747c5d6d286..8b61a72032d 100644 --- a/backend/plugins/gitlab/tasks/account_collector.go +++ b/backend/plugins/gitlab/tasks/account_collector.go @@ -36,6 +36,7 @@ func init() { } const RAW_USER_TABLE = "gitlab_api_users" +const KEYSET_MIN_VERSION = "v16.5.0" var CollectAccountsMeta = plugin.SubTaskMeta{ Name: "Collect Users", @@ -62,6 +63,18 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { urlTemplate = "/users" } + apiVersion := data.ApiClient.GetData(models.GitlabApiClientData_ApiVersion).(string) + + useKeyset := false + if urlTemplate == "/users" && semver.IsValid(apiVersion) && semver.Compare(apiVersion, KEYSET_MIN_VERSION) >= 0 { + useKeyset = true + } else if urlTemplate == "/users" && !semver.IsValid(apiVersion) { + // If version unknown, be conservative for CE 11–16.4: default to offset + logger.Debug("GitLab version is unknown/invalid; falling back to offset pagination for /users") + } + + var lastID int + collector, err := api.NewApiCollector(api.ApiCollectorArgs{ RawDataSubTaskArgs: *rawDataSubTaskArgs, ApiClient: data.ApiClient, @@ -69,6 +82,17 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { PageSize: 100, Query: func(reqData *api.RequestData) (url.Values, errors.Error) { query := url.Values{} + // Use keyset only when gated true and only on /users endpoint + if useKeyset && urlTemplate == "/users" { + query.Set("pagination", "keyset") + query.Set("order_by", "id") + query.Set("sort", "asc") + query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) + if lastID > 0 { + query.Set("id_after", fmt.Sprintf("%d", lastID)) + } + return query, nil + } query.Set("page", fmt.Sprintf("%v", reqData.Pager.Page)) query.Set("per_page", fmt.Sprintf("%v", reqData.Pager.Size)) return query, nil @@ -80,6 +104,15 @@ func CollectAccounts(taskCtx plugin.SubTaskContext) errors.Error { if err != nil { return nil, err } + if len(items) > 0 && urlTemplate == "/users" { + var tail struct { + ID int `json:"id"` + } + _ = json.Unmarshal(items[len(items)-1], &tail) + if tail.ID > 0 { + lastID = tail.ID + } + } return items, nil }, })