commits: add incremental git log limit

The existing git log logic fetched the first 300 commits of a repo and
displayed them in the local and sub-commit views. Once a user selected a
commit beyond a threshold of 200 commits the whole repository was
loaded. This is problematic with large repos e.g. the linux kernel with
currently ~138k commits as lazygit slows down substantially with such a
large number of commits in memory.

This commit replaces the current all or only the first 300 commits logic
with an incremental fetching approach:

1. The first 400 commits of repo are loaded by default.
2. If the user selects a commit beyond a threshold (current limit-100)
   the git log limit is increased by 400 if there are more commits
   available. If there are more commits available is currently checked
   by comparing the previous log limit with the real commit count in the
   model, if the commit count is less then the limit it is assumed that
   we reached the end of the commit log. Ideally it would be better to
   call `git rev-list --count xyz` in the right places and compare with
   this result, but this requires more changes.

Adding a "paginated implementation" by utilizing `git log --skip=x
--max-count=y` and appending commits to the model instead of replacing
the whole collection would be nice, but this requires deeper changes to
keep everything consistent.

Signed-off-by: Stefan Kerkmann <[email protected]>

Author: KarlK90

pkg/commands/git_commands/commit_loader.go

@@ -55,8 +55,29 @@ func NewCommitLoader(
 	}
 }
 
+const (
+	GIT_LOG_FETCH_COUNT     = 400
+	GIT_LOG_FETCH_THRESHOLD = GIT_LOG_FETCH_COUNT / 4
+)
+
+type GitLogLimit struct {
+	Limit int
+}
+
+func DefaultGitLogLimit() *GitLogLimit {
+	return &GitLogLimit{Limit: GIT_LOG_FETCH_COUNT}
+}
+
+func (self *GitLogLimit) CanFetchMoreCommits(lineIdx, logCommitCount int) bool {
+	return lineIdx >= logCommitCount-GIT_LOG_FETCH_THRESHOLD && logCommitCount >= self.Limit
+}
+
+func (self *GitLogLimit) Increase(realCommitCount int) {
+	self.Limit = realCommitCount + GIT_LOG_FETCH_COUNT
+}
+
 type GetCommitsOptions struct {
-	Limit                bool
+	LogLimit             *GitLogLimit
 	FilterPath           string
 	FilterAuthor         string
 	IncludeRebaseCommits bool
@@ -592,7 +613,7 @@ func (self *CommitLoader) getLogCmd(opts GetCommitsOptions) *oscommands.CmdObj {
 		Arg(prettyFormat).
 		Arg("--abbrev=40").
 		ArgIf(opts.FilterAuthor != "", "--author="+opts.FilterAuthor).
-		ArgIf(opts.Limit, "-300").
+		ArgIf(opts.LogLimit != nil, fmt.Sprintf("--max-count=%d", opts.LogLimit.Limit)).
 		ArgIf(opts.FilterPath != "", "--follow", "--name-status").
 		Arg("--no-show-signature").
 		ArgIf(opts.RefToShowDivergenceFrom != "", "--left-right").

pkg/gui/context/local_commits_context.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/jesseduffield/gocui"
+	"github.com/jesseduffield/lazygit/pkg/commands/git_commands"
 	"github.com/jesseduffield/lazygit/pkg/commands/models"
 	"github.com/jesseduffield/lazygit/pkg/gui/presentation"
 	"github.com/jesseduffield/lazygit/pkg/gui/types"
@@ -144,7 +145,7 @@ type LocalCommitsViewModel struct {
 
 	// If this is true we limit the amount of commits we load, for the sake of keeping things fast.
 	// If the user attempts to scroll past the end of the list, we will load more commits.
-	limitCommits bool
+	gitLogLimit *git_commands.GitLogLimit
 
 	// If this is true we'll use git log --all when fetching the commits.
 	showWholeGitGraph bool
@@ -153,7 +154,7 @@ type LocalCommitsViewModel struct {
 func NewLocalCommitsViewModel(getModel func() []*models.Commit, c *ContextCommon) *LocalCommitsViewModel {
 	self := &LocalCommitsViewModel{
 		ListViewModel:     NewListViewModel(getModel),
-		limitCommits:      true,
+		gitLogLimit:       git_commands.DefaultGitLogLimit(),
 		showWholeGitGraph: c.UserConfig().Git.Log.ShowWholeGraph,
 	}
 
@@ -226,12 +227,12 @@ func (self *LocalCommitsContext) ModelSearchResults(searchStr string, caseSensit
 	return searchModelCommits(caseSensitive, self.GetCommits(), self.ColumnPositions(), self.ModelIndexToViewIndex, searchStr)
 }
 
-func (self *LocalCommitsViewModel) SetLimitCommits(value bool) {
-	self.limitCommits = value
+func (self *LocalCommitsViewModel) SetGitLogLimit(limit *git_commands.GitLogLimit) {
+	self.gitLogLimit = limit
 }
 
-func (self *LocalCommitsViewModel) GetLimitCommits() bool {
-	return self.limitCommits
+func (self *LocalCommitsViewModel) GetGitLogLimit() *git_commands.GitLogLimit {
+	return self.gitLogLimit
 }
 
 func (self *LocalCommitsViewModel) SetShowWholeGitGraph(value bool) {

pkg/gui/context/sub_commits_context.go

@@ -34,8 +34,8 @@ func NewSubCommitsContext(
 		ListViewModel: NewListViewModel(
 			func() []*models.Commit { return c.Model().SubCommits },
 		),
-		ref:          nil,
-		limitCommits: true,
+		ref:         nil,
+		gitLogLimit: git_commands.DefaultGitLogLimit(),
 	}
 
 	getDisplayStrings := func(startIdx int, endIdx int) [][]string {
@@ -145,7 +145,7 @@ type SubCommitsViewModel struct {
 	refToShowDivergenceFrom string
 	*ListViewModel[*models.Commit]
 
-	limitCommits    bool
+	gitLogLimit     *git_commands.GitLogLimit
 	showBranchHeads bool
 }
 
@@ -202,12 +202,12 @@ func (self *SubCommitsContext) GetCommits() []*models.Commit {
 	return self.getModel()
 }
 
-func (self *SubCommitsContext) SetLimitCommits(value bool) {
-	self.limitCommits = value
+func (self *SubCommitsContext) SetGitLogLimit(limit *git_commands.GitLogLimit) {
+	self.gitLogLimit = limit
 }
 
-func (self *SubCommitsContext) GetLimitCommits() bool {
-	return self.limitCommits
+func (self *SubCommitsContext) GetGitLogLimit() *git_commands.GitLogLimit {
+	return self.gitLogLimit
 }
 
 func (self *SubCommitsContext) GetDiffTerminals() []string {

pkg/gui/controllers/helpers/refresh_helper.go

@@ -321,7 +321,7 @@ func (self *RefreshHelper) refreshCommitsWithLimit() error {
 	checkedOutRef := self.determineCheckedOutRef()
 	commits, err := self.c.Git().Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                self.c.Contexts().LocalCommits.GetLimitCommits(),
+			LogLimit:             self.c.Contexts().LocalCommits.GetGitLogLimit(),
 			FilterPath:           self.c.Modes().Filtering.GetPath(),
 			FilterAuthor:         self.c.Modes().Filtering.GetAuthor(),
 			IncludeRebaseCommits: true,
@@ -358,7 +358,7 @@ func (self *RefreshHelper) refreshSubCommitsWithLimit() error {
 
 	commits, err := self.c.Git().Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                   self.c.Contexts().SubCommits.GetLimitCommits(),
+			LogLimit:                self.c.Contexts().SubCommits.GetGitLogLimit(),
 			FilterPath:              self.c.Modes().Filtering.GetPath(),
 			FilterAuthor:            self.c.Modes().Filtering.GetAuthor(),
 			IncludeRebaseCommits:    false,

pkg/gui/controllers/helpers/refs_helper.go

@@ -44,7 +44,7 @@ func (self *RefsHelper) CheckoutRef(ref string, options types.CheckoutRefOptions
 		self.c.Contexts().ReflogCommits.SetSelection(0)
 		self.c.Contexts().LocalCommits.SetSelection(0)
 		// loading a heap of commits is slow so we limit them whenever doing a reset
-		self.c.Contexts().LocalCommits.SetLimitCommits(true)
+		self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 
 		self.c.Refresh(types.RefreshOptions{Mode: types.BLOCK_UI, KeepBranchSelectionIndex: true})
 	}
@@ -179,7 +179,7 @@ func (self *RefsHelper) ResetToRef(ref string, strength string, envVars []string
 	self.c.Contexts().LocalCommits.SetSelection(0)
 	self.c.Contexts().ReflogCommits.SetSelection(0)
 	// loading a heap of commits is slow so we limit them whenever doing a reset
-	self.c.Contexts().LocalCommits.SetLimitCommits(true)
+	self.c.Contexts().LocalCommits.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 
 	self.c.Refresh(types.RefreshOptions{Scope: []types.RefreshableView{types.FILES, types.BRANCHES, types.REFLOG, types.COMMITS}})
 

pkg/gui/controllers/helpers/sub_commits_helper.go

@@ -34,7 +34,7 @@ type ViewSubCommitsOpts struct {
 func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error {
 	commits, err := self.c.Git().Loaders.CommitLoader.GetCommits(
 		git_commands.GetCommitsOptions{
-			Limit:                   true,
+			LogLimit:                git_commands.DefaultGitLogLimit(),
 			FilterPath:              self.c.Modes().Filtering.GetPath(),
 			FilterAuthor:            self.c.Modes().Filtering.GetAuthor(),
 			IncludeRebaseCommits:    false,
@@ -59,7 +59,7 @@ func (self *SubCommitsHelper) ViewSubCommits(opts ViewSubCommitsOpts) error {
 	subCommitsContext.SetTitleRef(utils.TruncateWithEllipsis(opts.TitleRef, 50))
 	subCommitsContext.SetRef(opts.Ref)
 	subCommitsContext.SetRefToShowDivergenceFrom(opts.RefToShowDivergenceFrom)
-	subCommitsContext.SetLimitCommits(true)
+	subCommitsContext.SetGitLogLimit(git_commands.DefaultGitLogLimit())
 	subCommitsContext.SetShowBranchHeads(opts.ShowBranchHeads)
 	subCommitsContext.ClearSearchString()
 	subCommitsContext.GetView().ClearSearch()

pkg/gui/controllers/local_commits_controller.go

@@ -18,9 +18,6 @@ import (
 	"github.com/stefanhaller/git-todo-parser/todo"
 )
 
-// after selecting the 200th commit, we'll load in all the rest
-const COMMIT_THRESHOLD = 200
-
 type (
 	PullFilesFn func() error
 )
@@ -1142,8 +1139,8 @@ func (self *LocalCommitsController) createTag(commit *models.Commit) error {
 
 func (self *LocalCommitsController) openSearch() error {
 	// we usually lazyload these commits but now that we're searching we need to load them now
-	if self.context().GetLimitCommits() {
-		self.context().SetLimitCommits(false)
+	if self.context().GetGitLogLimit() != nil {
+		self.context().SetGitLogLimit(nil)
 		self.c.Refresh(types.RefreshOptions{Mode: types.ASYNC, Scope: []types.RefreshableView{types.COMMITS}})
 	}
 
@@ -1160,7 +1157,9 @@ func (self *LocalCommitsController) handleOpenLogMenu() error {
 					self.context().SetShowWholeGitGraph(!self.context().GetShowWholeGitGraph())
 
 					if self.context().GetShowWholeGitGraph() {
-						self.context().SetLimitCommits(false)
+						self.context().SetGitLogLimit(nil)
+					} else {
+						self.context().SetGitLogLimit(git_commands.DefaultGitLogLimit())
 					}
 
 					return self.c.WithWaitingStatus(self.c.Tr.LoadingCommits, func(gocui.Task) error {
@@ -1262,8 +1261,30 @@ func (self *LocalCommitsController) handleOpenLogMenu() error {
 func (self *LocalCommitsController) GetOnFocus() func(types.OnFocusOpts) {
 	return func(types.OnFocusOpts) {
 		context := self.context()
-		if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() {
-			context.SetLimitCommits(false)
+		limit := context.GetGitLogLimit()
+
+		if limit == nil {
+			return
+		}
+
+		lineIdx := context.GetSelectedLineIdx()
+		logCommitCount := len(self.c.Model().Commits)
+
+		if self.isRebasing() {
+			rebaseCommitCount := lo.CountBy(self.c.Model().Commits, func(c *models.Commit) bool {
+				return c.IsTODO()
+			})
+
+			if lineIdx < rebaseCommitCount {
+				lineIdx = 0
+			} else {
+				lineIdx -= rebaseCommitCount
+			}
+			logCommitCount -= rebaseCommitCount
+		}
+
+		if limit.CanFetchMoreCommits(lineIdx, logCommitCount) {
+			limit.Increase(logCommitCount)
 			self.c.Refresh(types.RefreshOptions{Mode: types.ASYNC, Scope: []types.RefreshableView{types.COMMITS}})
 		}
 	}

pkg/gui/controllers/sub_commits_controller.go

@@ -64,8 +64,16 @@ func (self *SubCommitsController) GetOnRenderToMain() func() {
 func (self *SubCommitsController) GetOnFocus() func(types.OnFocusOpts) {
 	return func(types.OnFocusOpts) {
 		context := self.context()
-		if context.GetSelectedLineIdx() > COMMIT_THRESHOLD && context.GetLimitCommits() {
-			context.SetLimitCommits(false)
+		limit := context.GetGitLogLimit()
+
+		if limit == nil {
+			return
+		}
+
+		logCommitCount := len(self.c.Model().SubCommits)
+
+		if limit.CanFetchMoreCommits(context.GetSelectedLineIdx(), logCommitCount) {
+			limit.Increase(logCommitCount)
 			self.c.Refresh(types.RefreshOptions{Mode: types.ASYNC, Scope: []types.RefreshableView{types.SUB_COMMITS}})
 		}
 	}