Skip to content

fix: make WebSocket disconnects recoverable and actionable #217

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Razzmatazzz merged 1 commit into from
Aug 17, 2026
Merged

fix: make WebSocket disconnects recoverable and actionable #217

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions TarkovMonitor/Diagnostics/DiagnosticModels.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,8 @@ public sealed record DiagnosticContext(
string Stage,
string DisplayMessage,
string? Endpoint = null,
string Outcome = "failure");
string Outcome = "failure",
string? IncidentId = null);

public sealed class DiagnosticSnapshot
{
Expand All @@ -27,6 +28,7 @@ public sealed class DiagnosticSnapshot
public string Outcome { get; init; } = "failure";
public string DisplayMessage { get; init; } = "";
public string? Endpoint { get; init; }
public string? IncidentId { get; init; }
public DateTime TimestampUtc { get; init; }
public long? DurationMilliseconds { get; init; }
public string ApplicationVersion { get; init; } = "";
Expand Down Expand Up @@ -174,7 +176,10 @@ public DiagnosticSnapshot Capture(DiagnosticContext context, Exception? exceptio
var hResult = exceptionDetails.Count == 0 ? "" : exceptionDetails[0].HResult;
var sanitizedCode = DiagnosticRedactor.Sanitize(context.Code, 128);
var sanitizedOperation = DiagnosticRedactor.Sanitize(context.Operation, 256);
var diagnosticKey = string.Join("|", sanitizedCode, sanitizedOperation, exceptionType, hResult);
var sanitizedIncidentId = DiagnosticRedactor.Sanitize(context.IncidentId, 128);
var diagnosticKey = string.IsNullOrWhiteSpace(sanitizedIncidentId)
? string.Join("|", sanitizedCode, sanitizedOperation, exceptionType, hResult)
: $"incident:{sanitizedIncidentId}";

int occurrenceCount;
lock (gate)
Expand All @@ -194,6 +199,7 @@ public DiagnosticSnapshot Capture(DiagnosticContext context, Exception? exceptio
Outcome = DiagnosticRedactor.Sanitize(context.Outcome, 64),
DisplayMessage = DiagnosticRedactor.Sanitize(context.DisplayMessage, 512),
Endpoint = DiagnosticRedactor.SanitizeEndpoint(context.Endpoint),
IncidentId = sanitizedIncidentId,
TimestampUtc = timestamp,
DurationMilliseconds = durationMilliseconds is long measuredDuration
? Math.Max(0, measuredDuration)
Expand Down Expand Up @@ -324,6 +330,7 @@ private void Persist(DiagnosticSnapshot snapshot, IReadOnlyList<ExceptionDetail>
snapshot.Outcome,
snapshot.DisplayMessage,
snapshot.Endpoint,
snapshot.IncidentId,
snapshot.TimestampUtc,
snapshot.DurationMilliseconds,
snapshot.ApplicationVersion,
Expand Down
72 changes: 61 additions & 11 deletions TarkovMonitor/MainBlazorUI.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@ protected override CreateParams CreateParams
private CancellationTokenSource? tarkovDevDataRefreshCancellation;
private long tarkovDevDataRefreshGeneration;
private Profile? tarkovDevDataProfile;
private bool closing;

private readonly record struct TrackerSessionNoticeIdentity(
string AccountId,
Expand Down Expand Up @@ -244,7 +245,7 @@ public MainBlazorUI(bool holdUntilSplashCompletes = false, DiagnosticsService? d
UpdateCheck.NewVersion += UpdateCheck_NewVersion;
UpdateCheck.Error += UpdateCheck_Error;

SocketClient.ExceptionThrown += SocketClient_ExceptionThrown;
SocketClient.ConnectionInterrupted += SocketClient_ConnectionInterrupted;

blazorWebView1.WebView.CoreWebView2InitializationCompleted += WebView_CoreWebView2InitializationCompleted;

Expand Down Expand Up @@ -287,9 +288,10 @@ private void RecordException(
string service,
string stage,
string? endpoint = null,
long? durationMilliseconds = null)
long? durationMilliseconds = null,
string? incidentId = null)
{
messageLog.AddException(displayMessage, code, operation, exception, service, stage, endpoint, durationMilliseconds);
messageLog.AddException(displayMessage, code, operation, exception, service, stage, endpoint, durationMilliseconds, incidentId);
}

public void MarkUiReady()
Expand Down Expand Up @@ -628,9 +630,26 @@ private void Eft_GroupRaidSettings(object? sender, LogContentEventArgs<GroupRaid
groupManager.ClearGroup();
}

private void SocketClient_ExceptionThrown(object? sender, ExceptionEventArgs e)
private void SocketClient_ConnectionInterrupted(object? sender, SocketConnectionIncidentEventArgs e)
{
RecordException("Tarkov.dev connection failed; copy diagnostics for details.", "TM-SOCKET-001", e.Context, e.Exception, "WebSocket", "Background");
if (closing)
{
return;
}

// A recoverable background disconnect is retained as sanitized
// telemetry, not rendered as a frightening error card. A later
// send owns user-facing reporting if lazy recovery fails.
diagnostics.Capture(
new DiagnosticContext(
"TM-SOCKET-001",
e.Operation,
"WebSocket",
"Background",
"Tarkov.dev connection interrupted.",
e.Endpoint,
IncidentId: e.IncidentId),
e.Exception);
}

protected override void OnShown(EventArgs e)
Expand Down Expand Up @@ -691,6 +710,15 @@ private void StartStartupServices()
{
RecordException("Update checking could not start.", "TM-UPDATE-002", "CheckForNewVersion", ex, "UpdateCheck", "Startup");
}

}

protected override void OnFormClosed(FormClosedEventArgs e)
{
closing = true;
SocketClient.ConnectionInterrupted -= SocketClient_ConnectionInterrupted;
_ = SocketClient.StopAsync();
base.OnFormClosed(e);
}

private async void Eft_PlayerPosition(object? sender, PlayerPositionEventArgs e)
Expand All @@ -715,7 +743,7 @@ private async void Eft_PlayerPosition(object? sender, PlayerPositionEventArgs e)
}
catch (Exception ex)
{
RecordException("Player position could not be sent to Tarkov.dev.", "TM-SOCKET-002", "SendPlayerPosition", ex, "WebSocket", "PlayerPosition", durationMilliseconds: DiagnosticsService.ElapsedMilliseconds(startedUtc));
RecordException("Tarkov.dev is unavailable. No messages were resent; the connection will be retried when needed.", "TM-SOCKET-002", "SendPlayerPosition", ex, "WebSocket", "PlayerPosition", endpoint: SocketClient.GetEndpointForDiagnostics(), durationMilliseconds: DiagnosticsService.ElapsedMilliseconds(startedUtc), incidentId: SocketClient.GetIncidentId(ex));
}
}

Expand Down Expand Up @@ -799,7 +827,7 @@ private async void Eft_MapLoading(object? sender, EventArgs e)
}
}

private void Eft_MapLoading_NavigateToMap(object? sender, RaidInfoEventArgs e)
private async void Eft_MapLoading_NavigateToMap(object? sender, RaidInfoEventArgs e)
{
if (!Properties.Settings.Default.autoNavigateMap)
{
Expand All @@ -809,7 +837,29 @@ private void Eft_MapLoading_NavigateToMap(object? sender, RaidInfoEventArgs e)
{
return;
}
SocketClient.NavigateToMap(e.RaidInfo.Map);
await NavigateToMapWithDiagnostics(e.RaidInfo.Map);
}

private async Task NavigateToMapWithDiagnostics(TarkovDev.Map map)
{
var startedUtc = DateTime.UtcNow;
try
{
await SocketClient.NavigateToMap(map);
}
catch (Exception exception)
{
RecordException(
"Tarkov.dev is unavailable. No messages were resent; the connection will be retried when needed.",
"TM-SOCKET-002",
"NavigateToMap",
exception,
"WebSocket",
"MapNavigation",
endpoint: SocketClient.GetEndpointForDiagnostics(),
durationMilliseconds: DiagnosticsService.ElapsedMilliseconds(startedUtc),
incidentId: SocketClient.GetIncidentId(exception));
}
}

private void Eft_GroupUserLeave(object? sender, LogContentEventArgs<GroupMatchUserLeaveLogContent> e)
Expand Down Expand Up @@ -1419,8 +1469,8 @@ private async void Eft_RaidStart(object? sender, RaidInfoEventArgs e)
}
select.Placeholder = "Choose a map";
monMessage.Selects.Add(select);
MonitorMessageButton mapButton = new("Set map", Icons.Material.Filled.Map);
mapButton.OnClick += () => {
MonitorMessageButton mapButton = new("Set map", Icons.Material.Filled.Map);
mapButton.OnClick += async () => {
if (select.Selected == null)
{
return;
Expand All @@ -1436,7 +1486,7 @@ private async void Eft_RaidStart(object? sender, RaidInfoEventArgs e)
{
return;
}
SocketClient.NavigateToMap(e.RaidInfo.Map);
await NavigateToMapWithDiagnostics(e.RaidInfo.Map);
}
};
monMessage.Buttons.Add(mapButton);
Expand Down
38 changes: 33 additions & 5 deletions TarkovMonitor/MessageLog.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ internal class MessageLog
private readonly object gate = new();
private readonly Dictionary<string, (MonitorMessage Message, DateTime LastSeen)> recentDiagnostics = new(StringComparer.Ordinal);
private readonly List<MonitorMessage> messages = new();
private readonly Dictionary<string, (MonitorMessage Message, DateTime LastSeen)> incidentDiagnostics = new(StringComparer.Ordinal);

public event NewLogMessage newMessage = delegate { };

Expand Down Expand Up @@ -115,10 +116,11 @@ public DiagnosticSnapshot AddException(
string service,
string stage,
string? endpoint = null,
long? durationMilliseconds = null)
long? durationMilliseconds = null,
string? incidentId = null)
{
var snapshot = Diagnostics.Capture(
new DiagnosticContext(code, operation, service, stage, displayMessage, endpoint),
new DiagnosticContext(code, operation, service, stage, displayMessage, endpoint, IncidentId: incidentId),
exception,
durationMilliseconds);
AddDiagnostic(snapshot);
Expand All @@ -133,8 +135,17 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot)

lock (gate)
{
if (recentDiagnostics.TryGetValue(snapshot.DiagnosticKey, out var previous)
&& now - previous.LastSeen <= DeduplicationWindow)
var isIncidentDiagnostic = !string.IsNullOrWhiteSpace(snapshot.IncidentId);
var existing = isIncidentDiagnostic
? incidentDiagnostics.TryGetValue(snapshot.IncidentId!, out var incidentPrevious)
? incidentPrevious
: ((MonitorMessage Message, DateTime LastSeen)?)null
: recentDiagnostics.TryGetValue(snapshot.DiagnosticKey, out var recentPrevious)
? recentPrevious
: null;

if (existing is { } previous
&& (isIncidentDiagnostic || now - previous.LastSeen <= DeduplicationWindow))
{
var occurrenceCount = Math.Max(previous.Message.DiagnosticOccurrenceCount, snapshot.OccurrenceCount);
if (snapshot.OccurrenceCount >= previous.Message.DiagnosticOccurrenceCount)
Expand All @@ -144,7 +155,14 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot)
previous.Message.DiagnosticKey = snapshot.DiagnosticKey;
previous.Message.DiagnosticOccurrenceCount = occurrenceCount;
previous.Message.Message = $"{snapshot.DisplayMessage} (repeated {occurrenceCount} times)";
recentDiagnostics[snapshot.DiagnosticKey] = (previous.Message, now);
if (isIncidentDiagnostic)
{
incidentDiagnostics[snapshot.IncidentId!] = (previous.Message, now);
}
else
{
recentDiagnostics[snapshot.DiagnosticKey] = (previous.Message, now);
}
isRepeat = true;
messageToRaise = previous.Message;
}
Expand All @@ -157,6 +175,10 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot)
};
AddToBoundedList(message);
recentDiagnostics[snapshot.DiagnosticKey] = (message, now);
if (isIncidentDiagnostic)
{
incidentDiagnostics[snapshot.IncidentId!] = (message, now);
}
TrimRecentDiagnostics();
messageToRaise = message;
}
Expand Down Expand Up @@ -201,6 +223,12 @@ private void TrimRecentDiagnostics()
var oldest = recentDiagnostics.MinBy(entry => entry.Value.LastSeen);
recentDiagnostics.Remove(oldest.Key);
}

while (incidentDiagnostics.Count > MaxRecentDiagnostics)
{
var oldest = incidentDiagnostics.MinBy(entry => entry.Value.LastSeen);
incidentDiagnostics.Remove(oldest.Key);
}
}

private void RaiseMessageAdded(MonitorMessage message, bool isRepeat = false)
Expand Down
Loading
Loading