Skip to content

do not try another peer on a local error #440

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
freemans13 wants to merge 3 commits into
base: main
Choose a base branch
from
Open

do not try another peer on a local error #440

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
aac32ed
do not try another peer on a local error
freemans13 Jan 28, 2026
a305cda
Move isLocalErr to errors package
freemans13 Jan 28, 2026
b8a9d63
Merge branch 'main' into stu/do-not-try-another-peer-on-local-error
freemans13 Feb 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 28 additions & 0 deletions errors/error_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,34 @@ func IsContextError(err error) bool {
return false
}

// IsLocalError checks if an error is a local resource error (not peer-related).
// Local errors include context cancellation, semaphore exhaustion, and storage errors
// that are caused by local resource constraints rather than peer failures.
// These errors should not trigger peer failover since trying another peer won't help.
//
// Parameters:
// - err: Error to check
//
// Returns:
// - bool: true if error is a local resource error
func IsLocalError(err error) bool {
if err == nil {
return false
}

// Context errors are local (includes semaphore wait timeouts)
if IsContextError(err) {
return true
}

// Storage errors indicate local resource issues
if Is(err, ErrStorageError) {
return true
}

return false
}

// GetErrorCategory returns a string representing the category of the error.
// This is useful for logging and metrics.
//
Expand Down
56 changes: 56 additions & 0 deletions errors/error_utils_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -222,6 +222,62 @@ func TestIsContextError(t *testing.T) {
}
}

func TestIsLocalError(t *testing.T) {
tests := []struct {
name string
err error
expected bool
}{
{
name: "nil error",
err: nil,
expected: false,
},
{
name: "context canceled",
err: context.Canceled,
expected: true,
},
{
name: "context deadline exceeded",
err: context.DeadlineExceeded,
expected: true,
},
{
name: "wrapped context canceled",
err: NewContextCanceledError("test"),
expected: true,
},
{
name: "storage error",
err: NewStorageError("test"),
expected: true,
},
{
name: "network error - should retry with other peers",
err: NewNetworkTimeoutError("test"),
expected: false,
},
{
name: "service error - should retry with other peers",
err: NewServiceError("test"),
expected: false,
},
{
name: "processing error - should retry with other peers",
err: NewProcessingError("test"),
expected: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result := IsLocalError(tt.err)
assert.Equal(t, tt.expected, result)
})
}
}

func TestGetErrorCategory(t *testing.T) {
tests := []struct {
name string
Expand Down
16 changes: 16 additions & 0 deletions services/blockvalidation/get_blocks.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -567,8 +567,16 @@ func (u *Server) fetchAndStoreSubtreeAndSubtreeData(ctx context.Context, block *
if err = u.fetchAndStoreSubtreeData(ctx, block, subtreeHash, subtree, peerID, baseURL); err == nil {
return nil // Success
}
// Check if error is local (not peer-related) - don't retry with other peers
if errors.IsLocalError(err) {
return errors.NewServiceError("[catchup:fetchAndStoreSubtreeAndSubtreeData] Local error fetching subtreeData for %s (not retrying with other peers)", subtreeHash.String(), err)
Copy link
Contributor

@github-actions github-actions bot Feb 2, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Format string bug: has one %s placeholder but passes two arguments (hash and err). The error will be ignored. Need to add %v placeholder for the error.

}
u.logger.Warnf("[catchup:fetchAndStoreSubtreeAndSubtreeData] Primary peer %s failed to fetch subtreeData for %s: %v, trying alternatives", peerID, subtreeHash.String(), err)
} else {
// Check if error is local (not peer-related) - don't retry with other peers
if errors.IsLocalError(err) {
return errors.NewServiceError("[catchup:fetchAndStoreSubtreeAndSubtreeData] Local error fetching subtree for %s (not retrying with other peers)", subtreeHash.String(), err)
}
u.logger.Warnf("[catchup:fetchAndStoreSubtreeAndSubtreeData] Primary peer %s failed to fetch subtree for %s: %v, trying alternatives", peerID, subtreeHash.String(), err)
}

Expand Down Expand Up @@ -596,13 +604,21 @@ func (u *Server) fetchAndStoreSubtreeAndSubtreeData(ctx context.Context, block *
if err != nil {
u.logger.Debugf("[catchup:fetchAndStoreSubtreeAndSubtreeData] Alternative peer %s failed for subtree %s: %v", altPeerID, subtreeHash.String(), err)
lastErr = err
// Don't continue trying other peers if it's a local error
if errors.IsLocalError(err) {
return errors.NewServiceError("[catchup:fetchAndStoreSubtreeAndSubtreeData] Local error fetching subtree %s (aborting peer retry)", subtreeHash.String(), err)
}
continue
}

// Subtree succeeded, try subtreeData
if err = u.fetchAndStoreSubtreeData(ctx, block, subtreeHash, subtree, altPeerID, altBaseURL); err != nil {
u.logger.Debugf("[catchup:fetchAndStoreSubtreeAndSubtreeData] Alternative peer %s failed for subtreeData %s: %v", altPeerID, subtreeHash.String(), err)
lastErr = err
// Don't continue trying other peers if it's a local error
if errors.IsLocalError(err) {
return errors.NewServiceError("[catchup:fetchAndStoreSubtreeAndSubtreeData] Local error fetching subtreeData %s (aborting peer retry)", subtreeHash.String(), err)
}
continue
}

Expand Down
Loading