From 42510efec38a0c4f206bd2500eb548d2ef26707d Mon Sep 17 00:00:00 2001 From: Giribaldi_TTV Date: Thu, 13 Aug 2026 02:18:37 -0700 Subject: [PATCH] fix(socket): make WebSocket failures recoverable and actionable Serialize socket lifecycle and sends, add bounded reconnect backoff, and prevent replay after partial batch failure. Keep recoverable background disconnects telemetry-only while surfacing one incident-scoped diagnostic for required send failures. Assemble fragmented frames, isolate malformed or binary records, bound payloads, and cancel idle or shutdown handshakes promptly. --- TarkovMonitor/Diagnostics/DiagnosticModels.cs | 11 +- TarkovMonitor/MainBlazorUI.cs | 72 +- TarkovMonitor/MessageLog.cs | 38 +- TarkovMonitor/SocketClient.cs | 836 ++++++++++++++++-- 4 files changed, 841 insertions(+), 116 deletions(-) diff --git a/TarkovMonitor/Diagnostics/DiagnosticModels.cs b/TarkovMonitor/Diagnostics/DiagnosticModels.cs index bcd9931..e9c7622 100644 --- a/TarkovMonitor/Diagnostics/DiagnosticModels.cs +++ b/TarkovMonitor/Diagnostics/DiagnosticModels.cs @@ -15,7 +15,8 @@ public sealed record DiagnosticContext( string Stage, string DisplayMessage, string? Endpoint = null, - string Outcome = "failure"); + string Outcome = "failure", + string? IncidentId = null); public sealed class DiagnosticSnapshot { @@ -27,6 +28,7 @@ public sealed class DiagnosticSnapshot public string Outcome { get; init; } = "failure"; public string DisplayMessage { get; init; } = ""; public string? Endpoint { get; init; } + public string? IncidentId { get; init; } public DateTime TimestampUtc { get; init; } public long? DurationMilliseconds { get; init; } public string ApplicationVersion { get; init; } = ""; @@ -174,7 +176,10 @@ public DiagnosticSnapshot Capture(DiagnosticContext context, Exception? exceptio var hResult = exceptionDetails.Count == 0 ? "" : exceptionDetails[0].HResult; var sanitizedCode = DiagnosticRedactor.Sanitize(context.Code, 128); var sanitizedOperation = DiagnosticRedactor.Sanitize(context.Operation, 256); - var diagnosticKey = string.Join("|", sanitizedCode, sanitizedOperation, exceptionType, hResult); + var sanitizedIncidentId = DiagnosticRedactor.Sanitize(context.IncidentId, 128); + var diagnosticKey = string.IsNullOrWhiteSpace(sanitizedIncidentId) + ? string.Join("|", sanitizedCode, sanitizedOperation, exceptionType, hResult) + : $"incident:{sanitizedIncidentId}"; int occurrenceCount; lock (gate) @@ -194,6 +199,7 @@ public DiagnosticSnapshot Capture(DiagnosticContext context, Exception? exceptio Outcome = DiagnosticRedactor.Sanitize(context.Outcome, 64), DisplayMessage = DiagnosticRedactor.Sanitize(context.DisplayMessage, 512), Endpoint = DiagnosticRedactor.SanitizeEndpoint(context.Endpoint), + IncidentId = sanitizedIncidentId, TimestampUtc = timestamp, DurationMilliseconds = durationMilliseconds is long measuredDuration ? Math.Max(0, measuredDuration) @@ -324,6 +330,7 @@ private void Persist(DiagnosticSnapshot snapshot, IReadOnlyList snapshot.Outcome, snapshot.DisplayMessage, snapshot.Endpoint, + snapshot.IncidentId, snapshot.TimestampUtc, snapshot.DurationMilliseconds, snapshot.ApplicationVersion, diff --git a/TarkovMonitor/MainBlazorUI.cs b/TarkovMonitor/MainBlazorUI.cs index 27da6d4..bf4c5b7 100644 --- a/TarkovMonitor/MainBlazorUI.cs +++ b/TarkovMonitor/MainBlazorUI.cs @@ -86,6 +86,7 @@ protected override CreateParams CreateParams private CancellationTokenSource? tarkovDevDataRefreshCancellation; private long tarkovDevDataRefreshGeneration; private Profile? tarkovDevDataProfile; + private bool closing; private readonly record struct TrackerSessionNoticeIdentity( string AccountId, @@ -244,7 +245,7 @@ public MainBlazorUI(bool holdUntilSplashCompletes = false, DiagnosticsService? d UpdateCheck.NewVersion += UpdateCheck_NewVersion; UpdateCheck.Error += UpdateCheck_Error; - SocketClient.ExceptionThrown += SocketClient_ExceptionThrown; + SocketClient.ConnectionInterrupted += SocketClient_ConnectionInterrupted; blazorWebView1.WebView.CoreWebView2InitializationCompleted += WebView_CoreWebView2InitializationCompleted; @@ -287,9 +288,10 @@ private void RecordException( string service, string stage, string? endpoint = null, - long? durationMilliseconds = null) + long? durationMilliseconds = null, + string? incidentId = null) { - messageLog.AddException(displayMessage, code, operation, exception, service, stage, endpoint, durationMilliseconds); + messageLog.AddException(displayMessage, code, operation, exception, service, stage, endpoint, durationMilliseconds, incidentId); } public void MarkUiReady() @@ -628,9 +630,26 @@ private void Eft_GroupRaidSettings(object? sender, LogContentEventArgs e) @@ -1419,8 +1469,8 @@ private async void Eft_RaidStart(object? sender, RaidInfoEventArgs e) } select.Placeholder = "Choose a map"; monMessage.Selects.Add(select); - MonitorMessageButton mapButton = new("Set map", Icons.Material.Filled.Map); - mapButton.OnClick += () => { + MonitorMessageButton mapButton = new("Set map", Icons.Material.Filled.Map); + mapButton.OnClick += async () => { if (select.Selected == null) { return; @@ -1436,7 +1486,7 @@ private async void Eft_RaidStart(object? sender, RaidInfoEventArgs e) { return; } - SocketClient.NavigateToMap(e.RaidInfo.Map); + await NavigateToMapWithDiagnostics(e.RaidInfo.Map); } }; monMessage.Buttons.Add(mapButton); diff --git a/TarkovMonitor/MessageLog.cs b/TarkovMonitor/MessageLog.cs index 3fef216..74b74ad 100644 --- a/TarkovMonitor/MessageLog.cs +++ b/TarkovMonitor/MessageLog.cs @@ -28,6 +28,7 @@ internal class MessageLog private readonly object gate = new(); private readonly Dictionary recentDiagnostics = new(StringComparer.Ordinal); private readonly List messages = new(); + private readonly Dictionary incidentDiagnostics = new(StringComparer.Ordinal); public event NewLogMessage newMessage = delegate { }; @@ -115,10 +116,11 @@ public DiagnosticSnapshot AddException( string service, string stage, string? endpoint = null, - long? durationMilliseconds = null) + long? durationMilliseconds = null, + string? incidentId = null) { var snapshot = Diagnostics.Capture( - new DiagnosticContext(code, operation, service, stage, displayMessage, endpoint), + new DiagnosticContext(code, operation, service, stage, displayMessage, endpoint, IncidentId: incidentId), exception, durationMilliseconds); AddDiagnostic(snapshot); @@ -133,8 +135,17 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot) lock (gate) { - if (recentDiagnostics.TryGetValue(snapshot.DiagnosticKey, out var previous) - && now - previous.LastSeen <= DeduplicationWindow) + var isIncidentDiagnostic = !string.IsNullOrWhiteSpace(snapshot.IncidentId); + var existing = isIncidentDiagnostic + ? incidentDiagnostics.TryGetValue(snapshot.IncidentId!, out var incidentPrevious) + ? incidentPrevious + : ((MonitorMessage Message, DateTime LastSeen)?)null + : recentDiagnostics.TryGetValue(snapshot.DiagnosticKey, out var recentPrevious) + ? recentPrevious + : null; + + if (existing is { } previous + && (isIncidentDiagnostic || now - previous.LastSeen <= DeduplicationWindow)) { var occurrenceCount = Math.Max(previous.Message.DiagnosticOccurrenceCount, snapshot.OccurrenceCount); if (snapshot.OccurrenceCount >= previous.Message.DiagnosticOccurrenceCount) @@ -144,7 +155,14 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot) previous.Message.DiagnosticKey = snapshot.DiagnosticKey; previous.Message.DiagnosticOccurrenceCount = occurrenceCount; previous.Message.Message = $"{snapshot.DisplayMessage} (repeated {occurrenceCount} times)"; - recentDiagnostics[snapshot.DiagnosticKey] = (previous.Message, now); + if (isIncidentDiagnostic) + { + incidentDiagnostics[snapshot.IncidentId!] = (previous.Message, now); + } + else + { + recentDiagnostics[snapshot.DiagnosticKey] = (previous.Message, now); + } isRepeat = true; messageToRaise = previous.Message; } @@ -157,6 +175,10 @@ public void AddDiagnostic(DiagnosticSnapshot snapshot) }; AddToBoundedList(message); recentDiagnostics[snapshot.DiagnosticKey] = (message, now); + if (isIncidentDiagnostic) + { + incidentDiagnostics[snapshot.IncidentId!] = (message, now); + } TrimRecentDiagnostics(); messageToRaise = message; } @@ -201,6 +223,12 @@ private void TrimRecentDiagnostics() var oldest = recentDiagnostics.MinBy(entry => entry.Value.LastSeen); recentDiagnostics.Remove(oldest.Key); } + + while (incidentDiagnostics.Count > MaxRecentDiagnostics) + { + var oldest = incidentDiagnostics.MinBy(entry => entry.Value.LastSeen); + incidentDiagnostics.Remove(oldest.Key); + } } private void RaiseMessageAdded(MonitorMessage message, bool isRepeat = false) diff --git a/TarkovMonitor/SocketClient.cs b/TarkovMonitor/SocketClient.cs index e2d608d..de23f37 100644 --- a/TarkovMonitor/SocketClient.cs +++ b/TarkovMonitor/SocketClient.cs @@ -1,185 +1,758 @@ -using System.Net.WebSockets; +using System.Net.WebSockets; using System.Text; +using System.Text.Json; using System.Text.Json.Nodes; namespace TarkovMonitor { internal static class SocketClient { - public static event EventHandler? ExceptionThrown; - private static readonly string wsUrl = "wss://socket.tarkov.dev"; - //private static readonly string wsUrl = "ws://localhost:8080"; - private static ClientWebSocket socket; - private static CancellationTokenSource cancellationToken; - private static Task receiveTask; - private static System.Timers.Timer idleTimer = new() + // Background transport interruptions are telemetry only. The UI is + // notified when a user-required send cannot complete, not when an + // otherwise recoverable receive loop loses its peer. + public static event EventHandler? ConnectionInterrupted; + + private const string wsUrl = "wss://socket.tarkov.dev"; + private const int ReceiveBufferSize = 4096; + private const int MaxMessageBytes = 64 * 1024; + private const int MaxReconnectBackoffSeconds = 60; + private static readonly TimeSpan[] ReconnectBackoff = + { + TimeSpan.Zero, + TimeSpan.FromSeconds(2), + TimeSpan.FromSeconds(5), + TimeSpan.FromSeconds(15), + TimeSpan.FromSeconds(30), + TimeSpan.FromSeconds(MaxReconnectBackoffSeconds), + }; + + // The production endpoint remains private so runtime configuration + // cannot redirect credentials or telemetry to an arbitrary host. + // The ignored sandbox replaces it by reflection in a separate process. + private static Uri socketEndpoint = new(wsUrl); + + private static readonly SemaphoreSlim lifecycleGate = new(1, 1); + private static readonly SemaphoreSlim sendBatchGate = new(1, 1); + private static readonly SemaphoreSlim socketSendGate = new(1, 1); + private static readonly object stateLock = new(); + private static readonly System.Timers.Timer idleTimer = new() { AutoReset = false, Interval = TimeSpan.FromMinutes(30).TotalMilliseconds, }; + private static ConnectionState? currentState; + private static SocketIncident? activeIncident; + private static int nextGeneration; + private static int connectFailureCount; + private static DateTimeOffset nextConnectAllowedUtc = DateTimeOffset.MinValue; + private static bool stopping; + private static CancellationTokenSource? pendingConnectCancellation; + static SocketClient() { - idleTimer.Elapsed += async (sender, e) => { - if (socket == null) + idleTimer.Elapsed += async (_, _) => await CloseIdleAsync(); + } + + public static async Task StartClient() + { + lock (stateLock) + { + stopping = false; + } + + await EnsureConnectedAsync(Properties.Settings.Default.remoteId ?? ""); + } + + public static Task VerifyClient() + { + return EnsureConnectedAsync(Properties.Settings.Default.remoteId ?? ""); + } + + public static async Task Send(List messages) + { + var remoteId = Properties.Settings.Default.remoteId; + if (string.IsNullOrWhiteSpace(remoteId) || messages.Count == 0) + { + return; + } + + await sendBatchGate.WaitAsync().ConfigureAwait(false); + ConnectionState? sendState = null; + try + { + sendState = await EnsureConnectedAsync(remoteId).ConfigureAwait(false); + + // Each item is sent once. There is deliberately no pending + // queue or replay after a partial batch failure: replaying an + // item already accepted by the server could duplicate data. + foreach (var message in messages) { - return; + message["sessionID"] = remoteId; + await SendSocketMessageAsync(sendState, message).ConfigureAwait(false); } - if (socket.State != WebSocketState.Open) + + ResetIdleTimer(); + MarkRecovered(); + } + catch (OperationCanceledException) when (IsStopping()) + { + // Application shutdown is an expected transport cancellation. + } + catch (Exception exception) + { + var incidentId = MarkSendFailure(sendState, exception); + throw new SocketSendException(incidentId, exception); + } + finally + { + sendBatchGate.Release(); + } + } + + public static Task Send(JsonObject message) + { + return Send(new List { message }); + } + + public static Task UpdatePlayerPosition(PlayerPositionEventArgs e) + { + if (e.RaidInfo.Map == null) + { + return Task.CompletedTask; + } + + return Send(GetPlayerPositionMessage(e)); + } + + public static Task NavigateToMap(TarkovDev.Map map) + { + return Send(GetNavigateToMapMessage(map)); + } + + public static async Task StopAsync() + { + Task? receiveTask; + ConnectionState? state; + CancellationTokenSource? connectCancellation; + + // Signal shutdown before waiting for the lifecycle gate. A + // ConnectAsync in progress owns that gate and must be cancelled + // by StopAsync rather than forcing form-close to wait on the + // platform's connection timeout. + lock (stateLock) + { + stopping = true; + connectCancellation = pendingConnectCancellation; + } + CancelPendingConnect(connectCancellation); + + await lifecycleGate.WaitAsync().ConfigureAwait(false); + try + { + lock (stateLock) { - return; + state = currentState; + connectCancellation = pendingConnectCancellation; + currentState = null; + if (state != null) + { + state.ExpectedClose = true; + } + receiveTask = state?.ReceiveTask; } + + idleTimer.Stop(); + CancelPendingConnect(connectCancellation); + DisposeState(state); + } + finally + { + lifecycleGate.Release(); + } + + if (receiveTask == null) + { + return; + } + + try + { + await Task.WhenAny(receiveTask, Task.Delay(TimeSpan.FromSeconds(1))).ConfigureAwait(false); + } + catch + { + // Shutdown must not create a second failure while the process + // is closing. + } + } + + public static string GetEndpointForDiagnostics() + { + var builder = new UriBuilder(socketEndpoint) + { + Query = "", + }; + return builder.Uri.ToString(); + } + + public static string? GetIncidentId(Exception exception) + { + return exception is SocketSendException sendException + ? sendException.IncidentId + : null; + } + + private static async Task EnsureConnectedAsync(string remoteId) + { + await lifecycleGate.WaitAsync().ConfigureAwait(false); + try + { + if (IsStopping()) + { + throw new OperationCanceledException("The socket client is stopping."); + } + + ConnectionState? oldState; + DateTimeOffset retryAt; + lock (stateLock) + { + if (currentState is { } openState + && IsOpen(openState.Socket) + && !openState.ExpectedClose) + { + return openState; + } + + oldState = currentState; + currentState = null; + if (oldState != null) + { + oldState.ExpectedClose = true; + } + + retryAt = nextConnectAllowedUtc; + } + + DisposeState(oldState); + + if (DateTimeOffset.UtcNow < retryAt) + { + throw new SocketReconnectThrottledException(retryAt); + } + + var client = new ClientWebSocket(); + var connectCancellation = new CancellationTokenSource(); + var endpoint = CreateSocketEndpoint(socketEndpoint, remoteId); + var clientIdentity = $"{System.Reflection.Assembly.GetExecutingAssembly().GetName().Name}/{System.Reflection.Assembly.GetExecutingAssembly().GetName().Version}"; + client.Options.SetRequestHeader("User-Agent", clientIdentity); + client.Options.SetRequestHeader("origin", clientIdentity); + + var stopRequestedBeforeConnect = false; + lock (stateLock) + { + if (stopping) + { + stopRequestedBeforeConnect = true; + } + else + { + pendingConnectCancellation = connectCancellation; + } + } + + if (stopRequestedBeforeConnect) + { + client.Dispose(); + connectCancellation.Dispose(); + throw new OperationCanceledException("The socket client is stopping."); + } + try { - await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Idle", CancellationToken.None); + await client.ConnectAsync(endpoint, connectCancellation.Token).ConfigureAwait(false); + } + catch (Exception) when (IsStopping()) + { + client.Dispose(); + throw; } - catch (Exception ex) + catch (Exception exception) { - ExceptionThrown?.Invoke(null, new(ex, "closing idle socket")); + client.Dispose(); + RegisterConnectFailure(exception); + throw; } finally { - socket.Dispose(); - socket = null; + lock (stateLock) + { + if (ReferenceEquals(pendingConnectCancellation, connectCancellation)) + { + pendingConnectCancellation = null; + } + } + + connectCancellation.Dispose(); } - }; + + ConnectionState state; + lock (stateLock) + { + state = new ConnectionState(client, new CancellationTokenSource(), ++nextGeneration); + if (stopping) + { + state.ExpectedClose = true; + } + else + { + currentState = state; + nextConnectAllowedUtc = DateTimeOffset.MinValue; + } + } + + if (state.ExpectedClose) + { + DisposeState(state); + throw new OperationCanceledException("The socket client is stopping."); + } + + state.ReceiveTask = ReceiveLoopAsync(state); + ResetIdleTimer(); + return state; + } + finally + { + lifecycleGate.Release(); + } } - private static Task SendSocketMessage(JsonNode payload) + private static async Task ReceiveLoopAsync(ConnectionState state) { - byte[] byteBuffer = Encoding.UTF8.GetBytes(payload.ToJsonString()); - return socket.SendAsync(new ArraySegment(byteBuffer), WebSocketMessageType.Text, true, cancellationToken.Token); + var buffer = new byte[ReceiveBufferSize]; + using var payload = new MemoryStream(); + + try + { + while (IsCurrentOpenState(state)) + { + var result = await state.Socket.ReceiveAsync(buffer, state.Cancellation.Token).ConfigureAwait(false); + + if (result.MessageType == WebSocketMessageType.Close) + { + await CompleteRemoteCloseAsync(state).ConfigureAwait(false); + MarkConnectionBroken( + state, + new WebSocketException("The remote WebSocket closed the connection."), + immediateReconnect: true, + operation: "receiving socket data", + notifyBackground: true); + break; + } + + var discardMessage = result.MessageType != WebSocketMessageType.Text + || !TryAppendPayload(payload, buffer, result.Count); + while (!result.EndOfMessage) + { + result = await state.Socket.ReceiveAsync(buffer, state.Cancellation.Token).ConfigureAwait(false); + if (result.MessageType != WebSocketMessageType.Text + || !TryAppendPayload(payload, buffer, result.Count)) + { + discardMessage = true; + } + } + + if (!discardMessage + && !await ProcessServerMessageAsync(state, payload).ConfigureAwait(false)) + { + break; + } + + payload.SetLength(0); + payload.Position = 0; + } + } + catch (OperationCanceledException) when (state.Cancellation.IsCancellationRequested || IsStopping()) + { + // Explicit stop, idle close, or replacement of an old socket. + } + catch (WebSocketException exception) when (state.ExpectedClose || IsStopping()) + { + // A close handshake can race disposal. It is still expected. + } + catch (Exception exception) + { + MarkConnectionBroken( + state, + exception, + immediateReconnect: true, + operation: "receiving socket data", + notifyBackground: true); + } + finally + { + CleanupCompletedState(state); + } } - public static async Task StartClient() + private static bool TryAppendPayload(MemoryStream payload, byte[] buffer, int count) + { + if (count < 0 || payload.Length + count > MaxMessageBytes) + { + return false; + } + + payload.Write(buffer, 0, count); + return true; + } + + private static async Task ProcessServerMessageAsync(ConnectionState state, MemoryStream payload) { - if (cancellationToken != null && !cancellationToken.IsCancellationRequested) + if (payload.Length == 0) { - cancellationToken.Cancel(); + return true; } - if (receiveTask != null) + + try { + var message = JsonNode.Parse(payload.GetBuffer().AsSpan(0, checked((int)payload.Length))); + if (message?["type"]?.ToString() != "ping") + { + return true; + } + try { - await receiveTask; + await SendSocketMessageAsync(state, new JsonObject { ["type"] = "pong" }).ConfigureAwait(false); + } + catch (Exception exception) + { + // A failed pong is a transport incident, not an unhandled + // receive-loop exception. Stop this stale loop cleanly. + MarkConnectionBroken( + state, + exception, + immediateReconnect: true, + operation: "sending socket pong", + notifyBackground: true); + return false; } - catch { } } - cancellationToken = new(); - var remoteid = Properties.Settings.Default.remoteId; - socket = new(); - socket.Options.SetRequestHeader("User-Agent", $"{System.Reflection.Assembly.GetExecutingAssembly().GetName().Name}/{System.Reflection.Assembly.GetExecutingAssembly().GetName().Version}"); - socket.Options.SetRequestHeader("origin", $"{System.Reflection.Assembly.GetExecutingAssembly().GetName().Name}/{System.Reflection.Assembly.GetExecutingAssembly().GetName().Version}"); - await socket.ConnectAsync(new Uri(wsUrl + $"?sessionid={remoteid}-tm"), new()); - idleTimer.Stop(); - idleTimer.Start(); + catch (JsonException) + { + // One malformed server record is isolated to that record. It + // must not tear down the transport or freeze the UI. + } - receiveTask = Task.Run(async () => + return true; + } + + private static async Task SendSocketMessageAsync(ConnectionState state, JsonNode payload) + { + var bytes = Encoding.UTF8.GetBytes(payload.ToJsonString()); + await socketSendGate.WaitAsync(state.Cancellation.Token).ConfigureAwait(false); + try { - try + if (!IsCurrentOpenState(state)) { - byte[] buffer = new byte[1024]; - while (socket != null && socket.State == WebSocketState.Open) - { - var result = await socket.ReceiveAsync(new ArraySegment(buffer), cancellationToken.Token); + throw new WebSocketException("The Tarkov.dev socket is no longer open."); + } - if (result.MessageType == WebSocketMessageType.Close) - { - if (socket.State == WebSocketState.Open) - { - await socket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Closing", CancellationToken.None); - } - break; - } + await state.Socket.SendAsync(bytes, WebSocketMessageType.Text, true, state.Cancellation.Token).ConfigureAwait(false); + } + finally + { + socketSendGate.Release(); + } + } - JsonNode? message = JsonNode.Parse(Encoding.UTF8.GetString(buffer, 0, result.Count)); - if (message == null) - { - return; - } - if (message["type"]?.ToString() == "ping") - { - await SendSocketMessage(new JsonObject - { - ["type"] = "pong" - }); - } + private static async Task CompleteRemoteCloseAsync(ConnectionState state) + { + try + { + if (state.Socket.State is WebSocketState.Open or WebSocketState.CloseReceived) + { + // A peer that already closed (or a local idle-close where + // the peer is not reading) must not hold the lifecycle + // gate indefinitely waiting for a close handshake. + using var closeTimeout = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + await state.Socket.CloseAsync( + WebSocketCloseStatus.NormalClosure, + "Closing", + closeTimeout.Token).ConfigureAwait(false); + } + } + catch + { + // The peer already closed or the OS released the connection. + } + } + + private static async Task CloseIdleAsync() + { + await lifecycleGate.WaitAsync().ConfigureAwait(false); + try + { + ConnectionState? state; + lock (stateLock) + { + state = currentState; + currentState = null; + if (state != null) + { + state.ExpectedClose = true; } } - catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + + if (state == null) { + return; } - catch (Exception ex) + + await CompleteRemoteCloseAsync(state).ConfigureAwait(false); + DisposeState(state); + } + catch + { + // Idle cleanup is intentionally silent. The next required + // send will establish a fresh connection if necessary. + } + finally + { + lifecycleGate.Release(); + } + } + + private static string MarkSendFailure(ConnectionState? state, Exception exception) + { + if (state != null) + { + return MarkConnectionBroken( + state, + exception, + immediateReconnect: false, + operation: "sending socket data", + notifyBackground: false); + } + + return EnsureIncident(exception, "connecting socket", notifyBackground: false); + } + + private static string MarkConnectionBroken( + ConnectionState? state, + Exception exception, + bool immediateReconnect, + string operation, + bool notifyBackground) + { + var incidentId = EnsureIncident(exception, operation, notifyBackground); + var ownsState = false; + + lock (stateLock) + { + if (state != null && ReferenceEquals(currentState, state)) { - ExceptionThrown?.Invoke(null, new(ex, "receiving socket data")); + currentState = null; + ownsState = true; + if (immediateReconnect) + { + nextConnectAllowedUtc = DateTimeOffset.UtcNow; + } + else + { + connectFailureCount = Math.Min(connectFailureCount + 1, ReconnectBackoff.Length - 1); + nextConnectAllowedUtc = DateTimeOffset.UtcNow + GetReconnectBackoff(connectFailureCount); + } } - }, cancellationToken.Token); + } + + if (ownsState) + { + DisposeState(state); + } + + return incidentId; } - public static async Task VerifyClient() + private static string RegisterConnectFailure(Exception exception) { - if (socket != null) + var incidentId = EnsureIncident(exception, "connecting socket", notifyBackground: false); + lock (stateLock) { - if (socket.State == WebSocketState.Open) + connectFailureCount = Math.Min(connectFailureCount + 1, ReconnectBackoff.Length - 1); + nextConnectAllowedUtc = DateTimeOffset.UtcNow + GetReconnectBackoff(connectFailureCount); + } + + return incidentId; + } + + private static string EnsureIncident(Exception exception, string operation, bool notifyBackground) + { + SocketConnectionIncidentEventArgs? notification = null; + string incidentId; + lock (stateLock) + { + if (activeIncident == null) { - return; + activeIncident = new SocketIncident(Guid.NewGuid().ToString("N"), DateTimeOffset.UtcNow); + if (notifyBackground && !stopping) + { + notification = new SocketConnectionIncidentEventArgs( + activeIncident.Id, + operation, + exception, + GetEndpointForDiagnostics()); + } } - socket.Dispose(); - socket = null; + + activeIncident.LastOperation = operation; + activeIncident.LastException = exception; + incidentId = activeIncident.Id; } - await StartClient(); - return; + + if (notification != null) + { + RaiseConnectionInterrupted(notification); + } + + return incidentId; } - public static async Task Send(List messages) + private static void RaiseConnectionInterrupted(SocketConnectionIncidentEventArgs args) { - var remoteid = Properties.Settings.Default.remoteId; - if (remoteid == null || remoteid == "") + foreach (EventHandler handler in ConnectionInterrupted?.GetInvocationList() + ?? Array.Empty()) { - return; + try + { + handler(null, args); + } + catch + { + // Transport telemetry must never become a new application + // failure or interrupt socket cleanup. + } + } + } + + private static TimeSpan GetReconnectBackoff(int failureCount) + { + var index = Math.Clamp(failureCount, 0, ReconnectBackoff.Length - 1); + return ReconnectBackoff[index]; + } + + private static void MarkRecovered() + { + lock (stateLock) + { + activeIncident = null; + connectFailureCount = 0; + nextConnectAllowedUtc = DateTimeOffset.MinValue; } - await VerifyClient(); - foreach (var message in messages) + } + + private static bool IsCurrentOpenState(ConnectionState state) + { + lock (stateLock) { - message["sessionID"] = remoteid; - await SendSocketMessage(message); + return currentState?.Generation == state.Generation + && ReferenceEquals(currentState, state) + && IsOpen(state.Socket) + && !state.ExpectedClose + && !stopping; } - idleTimer.Stop(); - idleTimer.Start(); } - public static Task Send(JsonObject message) + + private static bool IsOpen(ClientWebSocket client) { - return Send(new List { message }); + return client.State == WebSocketState.Open; } - public static async Task UpdatePlayerPosition(PlayerPositionEventArgs e) + private static bool IsStopping() { - if (e.RaidInfo.Map == null) + lock (stateLock) { - return; + return stopping; } - var payload = GetPlayerPositionMessage(e); + } + + private static void ResetIdleTimer() + { + idleTimer.Stop(); + if (!IsStopping()) + { + idleTimer.Start(); + } + } + + private static Uri CreateSocketEndpoint(Uri endpoint, string remoteId) + { + var builder = new UriBuilder(endpoint); + var queryParts = builder.Query + .TrimStart('?') + .Split('&', StringSplitOptions.RemoveEmptyEntries) + .Where(part => !part.StartsWith("sessionid=", StringComparison.OrdinalIgnoreCase)) + .ToList(); + queryParts.Add($"sessionid={Uri.EscapeDataString(remoteId + "-tm")}"); + builder.Query = string.Join('&', queryParts); + return builder.Uri; + } + + private static void CancelPendingConnect(CancellationTokenSource? connectCancellation) + { try { - await Send(payload); + connectCancellation?.Cancel(); } - catch (Exception ex) + catch (ObjectDisposedException) { - ExceptionThrown?.Invoke(payload, new(ex, "updating player position")); + // A completed connection may have disposed its cancellation + // source between the snapshot and shutdown request. } } - public static async Task NavigateToMap(TarkovDev.Map map) + private static void CleanupCompletedState(ConnectionState state) { - var payload = GetNavigateToMapMessage(map); + var wasCurrent = false; + lock (stateLock) + { + if (ReferenceEquals(currentState, state)) + { + currentState = null; + wasCurrent = true; + } + } + + if (wasCurrent) + { + idleTimer.Stop(); + } + + DisposeState(state); + } + + private static void DisposeState(ConnectionState? state) + { + if (state == null) + { + return; + } + try { - await Send(payload); + state.Cancellation.Cancel(); } - catch (Exception ex) + catch { - ExceptionThrown?.Invoke(payload, new(ex, $"navigating to map {map.name}")); } + + if (Interlocked.Exchange(ref state.Disposed, 1) != 0) + { + return; + } + + state.Socket.Dispose(); + state.Cancellation.Dispose(); } public static JsonObject GetPlayerPositionMessage(PlayerPositionEventArgs e) @@ -188,6 +761,7 @@ public static JsonObject GetPlayerPositionMessage(PlayerPositionEventArgs e) { throw new Exception("Map not found"); } + return new JsonObject { ["type"] = "command", @@ -218,5 +792,71 @@ public static JsonObject GetNavigateToMapMessage(TarkovDev.Map map) } }; } + + private sealed class ConnectionState + { + public ClientWebSocket Socket { get; } + public CancellationTokenSource Cancellation { get; } + public int Generation { get; } + public bool ExpectedClose { get; set; } + public Task? ReceiveTask { get; set; } + public int Disposed; + + public ConnectionState(ClientWebSocket socket, CancellationTokenSource cancellation, int generation) + { + Socket = socket; + Cancellation = cancellation; + Generation = generation; + } + } + + private sealed class SocketIncident + { + public SocketIncident(string id, DateTimeOffset startedUtc) + { + Id = id; + StartedUtc = startedUtc; + } + + public string Id { get; } + public DateTimeOffset StartedUtc { get; } + public string LastOperation { get; set; } = ""; + public Exception? LastException { get; set; } + } + + private sealed class SocketReconnectThrottledException : IOException + { + public SocketReconnectThrottledException(DateTimeOffset retryAt) + : base($"The Tarkov.dev connection is temporarily unavailable; retry after {retryAt:O}.") + { + } + } + + private sealed class SocketSendException : IOException + { + public SocketSendException(string incidentId, Exception innerException) + : base("The Tarkov.dev message could not be sent.", innerException) + { + IncidentId = incidentId; + } + + public string IncidentId { get; } + } + } + + internal sealed class SocketConnectionIncidentEventArgs : EventArgs + { + public SocketConnectionIncidentEventArgs(string incidentId, string operation, Exception exception, string endpoint) + { + IncidentId = incidentId; + Operation = operation; + Exception = exception; + Endpoint = endpoint; + } + + public string IncidentId { get; } + public string Operation { get; } + public Exception Exception { get; } + public string Endpoint { get; } } }