From 6ea35c78655e090fdf665ebea06ff1c17b0edbf8 Mon Sep 17 00:00:00 2001 From: Caleb Eden <58373773+calebeden@users.noreply.github.com> Date: Fri, 7 Aug 2026 14:40:40 -0700 Subject: [PATCH 1/2] Merge origin/main into calebeden-investigate-media-rendering Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> --- docs/ARCHITECTURE.md | 6 + .../GatewayClientFactory.cs | 3 +- .../GatewayConnectionManager.cs | 163 ++- .../ICredentialResolver.cs | 1 + .../InteractiveGatewayCredentialResolver.cs | 16 + .../AssistantMediaDirectiveParser.cs | 454 +++++++++ .../AssistantMediaResolution.cs | 25 + src/OpenClaw.Shared/Models.cs | 52 +- .../OpenClawGatewayClient.AssistantMedia.cs | 488 +++++++++ src/OpenClaw.Shared/OpenClawGatewayClient.cs | 300 +++++- src/OpenClaw.Shared/WebSocketClientBase.cs | 3 + .../Chat/ChatAssistantContentPresentation.cs | 104 ++ .../Chat/ChatAssistantImageDecodePolicy.cs | 33 + .../Chat/ChatAssistantMediaRenderer.cs | 343 +++++++ .../Chat/ChatAttachmentEchoCorrelation.cs | 46 + .../Chat/ChatAttachmentPresentation.cs | 36 + .../Chat/ChatConversationState.cs | 351 ++++++- .../Chat/ChatEntryMetadata.cs | 12 +- .../Chat/ChatHistoryReplayProjection.cs | 14 + .../Chat/ChatQueueState.cs | 97 +- src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs | 13 +- .../Chat/GatewayMediaMessageProjection.cs | 292 ++++++ .../Chat/IChatGatewayBridge.cs | 11 + .../Chat/OpenClawChatDataProvider.cs | 100 +- .../Chat/OpenClawChatTimeline.cs | 40 +- .../Chat/OpenClawReactorChatRoot.cs | 7 +- .../Chat/ReactorChatTimeline.cs | 117 ++- .../Strings/en-us/Resources.resw | 36 + .../Strings/fr-fr/Resources.resw | 12 + .../Strings/nl-nl/Resources.resw | 12 + .../Strings/zh-cn/Resources.resw | 12 + .../Strings/zh-tw/Resources.resw | 12 + .../GatewayConnectionManagerTests.cs | 232 ++++- .../AssistantMediaDirectiveParserTests.cs | 140 +++ ...penClawGatewayClientAssistantMediaTests.cs | 275 +++++ .../OpenClawGatewayClientTests.cs | 171 ++++ .../ChatAssistantContentPresentationTests.cs | 186 ++++ ...ChatAssistantMediaRendererContractTests.cs | 46 + .../GatewayMediaMessageProjectionTests.cs | 327 ++++++ .../OpenClaw.Tray.Tests.csproj | 5 + .../OpenClawChatDataProviderTests.cs | 954 +++++++++++++++++- 41 files changed, 5337 insertions(+), 210 deletions(-) create mode 100644 src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs create mode 100644 src/OpenClaw.Shared/AssistantMediaResolution.cs create mode 100644 src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs create mode 100644 src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs create mode 100644 tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs create mode 100644 tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs create mode 100644 tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs create mode 100644 tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs create mode 100644 tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0bc8d884e..97686b55a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -206,6 +206,12 @@ leading and trailing pipe. Columns, in order: | reactor-chat-timeline | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs | production chat message virtualization, row realization, and imperative scroll follow | ReactorChatTimeline through OpenClawReactorChatRoot and ReactorHostControl | OpenClawChatTimeline remains a legacy focused-test surface while its runtime route is migrated | the default chat route mounts one direct ReactorHostControl per XAML chat target; Reactor owns stable-key ItemsView and ItemContainer realization without a custom native list, collection reconciler, or scroll-layout mutation | review-only: user explicitly deferred new tests for this migration; required build and existing shared/tray suites still run | review-only | when Reactor timeline proof coverage replaces the legacy focused UI host coverage | | chat-tool-activity-renderer | authoritative | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | production standalone tool-call and grouped activity presentation, summaries, disclosures, and detail rendering | ChatToolActivityPresentation + ToolCallCardRenderer | ReactorChatTimeline projects rows and delegates realization only | consecutive invocation grouping preserves source chronology; stable group identity comes from session, generation, and first tool entry; selectable output remains capped at 240px | ChatToolActivityPresentationTests.Project_GroupsOnlyConsecutiveSpansOfAtLeastTwoTools | behavioral | - | | chat-history-replay-projection | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | array-valued history content ordering projection | ChatHistoryReplayProjection | provider applies projected text and tool parts to the reducer | interleaved text, calls, and results replay in source order without clearing active tool correlation | OpenClawChatDataProviderTests.LoadHistoryAsync_InterleavedContentParts_PreserveChronologyAndCorrelation | behavioral | - | +| assistant-media-protocol-projection | authoritative | src/OpenClaw.Shared/OpenClawGatewayClient.cs | structured assistant media content parsing and assistant-only legacy MEDIA directive redaction/projection | AssistantMediaDirectiveParser + ChatMediaContentInfo | gateway client preserves ordered typed media while tray presentation receives only safe filenames and metadata | user text never activates media directives; accepted local sources never enter visible assistant text or notifications; media-only messages survive live and history parsing | AssistantMediaDirectiveParserTests.Project_AssistantAbsolutePath_ProducesMediaWithoutExposingPath | behavioral | - | +| assistant-media-resolver | authoritative | src/OpenClaw.Shared/OpenClawGatewayClient.cs | authenticated structured artifact and legacy assistant-media byte retrieval | OpenClawGatewayClient.AssistantMedia | chat bridge exposes only lease-bound typed resolution results; renderer never receives credentials, tickets, or arbitrary URLs | accepts only current-connection results, matching media MIME families, managed ticket paths, and payloads within the 12 MiB image or 16 MiB playback caps | OpenClawGatewayClientAssistantMediaTests.ResolveLegacyMedia_UsesBearerMetadataAndSourceBoundTicket | behavioral | - | +| provider-assistant-media-parsing-closed | closed | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | parsing legacy MEDIA directives or structured Gateway media blocks | AssistantMediaDirectiveParser + OpenClawGatewayClient | provider owns message identity, streaming reconciliation, timeline application, and safe presentation metadata orchestration | provider consumes typed content parts and never reparses model text or exposes raw media sources | ChatAssistantContentPresentationTests.Project_UsesSafeFilenameWithoutExposingLegacySource | behavioral | when assistant message ingestion leaves OpenClawChatDataProvider | +| assistant-media-renderer | authoritative | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | assistant media card presentation, bounded image decode, retry, and row cancellation | ChatAssistantMediaRenderer | ReactorChatTimeline owns row placement and delegates media realization | at most four images render inline per message; unsupported or unresolved typed media remains visible as an accessible safe unavailable card; raw Gateway sources are never rendered | ChatAssistantContentPresentationTests.BuildRenderPlan_CapsImagesWithoutReorderingOtherMedia | behavioral | - | +| gateway-media-message-projection | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | gateway media-envelope parsing, safe filename/MIME normalization, attachment signatures, and provenance-safe attachment descriptors | GatewayMediaMessageProjection + ChatAttachmentPresentation | provider applies the projection to live, reset, backfill, and history ingress and owns stateful echo correlation | gateway text never becomes a private marker; gateway descriptors have no preview key; only local opaque preview keys can access image bytes | GatewayMediaMessageProjectionTests.ValidEnvelope_ProjectsSafeDescriptorAndCleanProse | behavioral | - | +| provider-gateway-media-parsing-closed | closed | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | private gateway media-envelope parsing or descriptor construction | GatewayMediaMessageProjection | provider retains stateful pending-echo queues, reset gates, sidecar matching, and reducer application | all user ingress paths call the focused projection and do not independently parse gateway media text | review-only | review-only | when user-message ingestion leaves OpenClawChatDataProvider | | reactor-tool-rendering-closed | closed | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | per-tool and grouped activity summary/detail rendering implementation | ToolCallCardRenderer | row projection, virtualization, hover state, assistant runs, and renderer delegation only | ReactorChatTimeline contains no tool detail renderer and delegates both standalone and grouped tool rows | ChatTimelinePresentationTests.ReactorTimeline_DelegatesToolAndActivityRenderingToFocusedOwner | source-shape | when ReactorChatTimeline is replaced as the production virtualization owner | | functional-chat-default-mount | closed | src/OpenClaw.Tray.WinUI/Chat/FunctionalChatHostExtensions.cs | mounting the FunctionalUI chat tree as the default ChatPage or ChatWindow surface | ReactorChatHostExtensions and OpenClawReactorChatRoot | legacy FunctionalUI chat files may remain for focused compatibility coverage only | ChatPage and ChatWindow mount the Reactor root directly into their existing ChatHost Borders; no FunctionalUI component mounts or nests Reactor on the default path | review-only: user explicitly deferred new tests for this migration; required build and existing shared/tray suites still run | review-only | when legacy FunctionalUI chat surfaces are removed | | settings-store | authoritative | settings and permission UI surfaces | direct SettingsManager mutation and blanket self-write suppression | ISettingsStore | non-permission legacy surfaces may read SettingsManager until migrated; direct saves publish origin null | every save publishes one versioned event; only the matching writer ignores its own origin while all other active consumers refresh | SettingsSharedStateContractTests.TwoActiveSettingsPageViewModels_IgnoreOnlyOwnWrites_InBothDirections | behavioral | when every settings surface reads and writes through ISettingsStore | diff --git a/src/OpenClaw.Connection/GatewayClientFactory.cs b/src/OpenClaw.Connection/GatewayClientFactory.cs index 983884f44..f87739ceb 100644 --- a/src/OpenClaw.Connection/GatewayClientFactory.cs +++ b/src/OpenClaw.Connection/GatewayClientFactory.cs @@ -21,7 +21,8 @@ public IGatewayClientLifecycle Create( tokenIsBootstrapToken: credential.IsBootstrapToken, bootstrapPairAsNode: false, identityPath: identityPath, - ignoreStoredDeviceToken: credential.IsBootstrapToken); + ignoreStoredDeviceToken: credential.IsBootstrapToken, + assistantMediaAuthToken: credential.InteractiveHttpToken); return new GatewayClientLifecycleAdapter(client); } diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 533b227d2..7c9cd2065 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -526,7 +526,52 @@ await StopOwnedTunnelAfterFailedConnectionAsync( IGatewayClientLifecycle lifecycle; try { - lifecycle = _clientFactory.Create(connectUrl, credential, perGatewayIdentityDir, diagLogger); + var httpCredential = + InteractiveGatewayCredentialResolver.ResolveForAssistantMediaHttpSurface( + record, + credential); + var interactiveHttpToken = string.Empty; + if (httpCredential is not null) + { + var httpAuthorization = await AuthorizeCredentialForEndpointAsync( + record, + httpCredential, + _operationCts!.Token) + .ConfigureAwait(false); + if (_disposed || + Interlocked.Read(ref _generation) != gen || + _operationCts?.IsCancellationRequested != false) + { + return; + } + if (httpAuthorization.Allowed) + { + interactiveHttpToken = httpCredential.Token; + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was withheld", + httpAuthorization.Detail); + } + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was unavailable"); + } + + var clientCredential = credential with + { + InteractiveHttpToken = interactiveHttpToken, + }; + lifecycle = _clientFactory.Create( + connectUrl, + clientCredential, + perGatewayIdentityDir, + diagLogger); } catch (DeviceIdentityLoadException ex) { @@ -557,6 +602,11 @@ await StopOwnedTunnelAfterFailedConnectionAsync( async Task AuthorizeLiveCredentialHandoffAsync( CancellationToken cancellationToken) { + // Clear the assistant-media HTTP credential up front so any + // in-flight media request never outlives this handoff attempt; + // it is only restored below once the fresh credential (or its + // interactive HTTP fallback) is re-authorized for this generation. + lifecycle.DataClient.SetAssistantMediaAuthToken(null); var authorization = await AuthorizeCredentialHandoffAsync( record, credential, @@ -578,6 +628,58 @@ await RecordOperatorCredentialHandoffFailureAsync( record.Id) .ConfigureAwait(false); } + + if (authorization.Allowed && + IsCurrentGatewayAttempt(gen, record.Id) && + IsAutomaticReconnectAllowed(record.Id)) + { + var currentRecord = _registry.GetById(record.Id); + if (currentRecord is not null) + { + GatewayCredential? currentHttpFallback = null; + if (string.IsNullOrWhiteSpace(currentRecord.SharedGatewayToken)) + { + currentHttpFallback = _credentialResolver.ResolveOperator( + currentRecord, + perGatewayIdentityDir); + } + var reconnectHttpCredential = + InteractiveGatewayCredentialResolver.ResolveForAssistantMediaHttpSurface( + currentRecord, + currentHttpFallback); + if (reconnectHttpCredential is null) + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was unavailable during reconnect"); + } + else + { + var httpAuthorization = await AuthorizeCredentialForEndpointAsync( + currentRecord, + reconnectHttpCredential, + cancellationToken) + .ConfigureAwait(false); + if (IsCurrentGatewayAttempt(gen, record.Id) && + IsAutomaticReconnectAllowed(record.Id)) + { + if (httpAuthorization.Allowed) + { + lifecycle.DataClient.SetAssistantMediaAuthToken( + reconnectHttpCredential.Token); + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was withheld during reconnect", + httpAuthorization.Detail); + } + } + } + } + } + return authorization; } @@ -2549,23 +2651,66 @@ private async Task HandleDeviceTokenReceivedAsync( identityPath, token, CancellationToken.None).ConfigureAwait(false); - if (result.Outcome != DeviceTokenHandlingOutcome.IdentityLoadFailure) + if (result.Outcome != DeviceTokenHandlingOutcome.IdentityLoadFailure && + result.Outcome != DeviceTokenHandlingOutcome.Stored) + { return; + } + var gatewayRecordId = attempt.GatewayRecordId ?? string.Empty; await _transitionSemaphore.WaitAsync().ConfigureAwait(false); try { - if (!IsCurrentGatewayAttempt( - attempt.LifecycleGeneration, - attempt.GatewayRecordId ?? string.Empty)) + if (!IsCurrentGatewayAttempt(attempt.LifecycleGeneration, gatewayRecordId)) { return; } - _stateMachine.TryTransition( - ConnectionTrigger.WebSocketError, - DeviceIdentityLoadException.RecoveryMessage); - EmitStateChanged(); + if (result.Outcome == DeviceTokenHandlingOutcome.IdentityLoadFailure) + { + _stateMachine.TryTransition( + ConnectionTrigger.WebSocketError, + DeviceIdentityLoadException.RecoveryMessage); + EmitStateChanged(); + return; + } + + // Stored: refresh the assistant-media HTTP credential bound to the + // fresh operator device token when this connection relies on a + // device token rather than a shared gateway token. + if (!string.Equals(token.Role, "operator", StringComparison.OrdinalIgnoreCase)) + return; + + var currentRecord = _registry.GetById(gatewayRecordId); + if (currentRecord is null || + !string.IsNullOrWhiteSpace(currentRecord.SharedGatewayToken)) + { + return; + } + + var deviceCredential = new GatewayCredential( + token.Token, + IsBootstrapToken: false, + CredentialResolver.SourceDeviceToken); + var authorization = await AuthorizeCredentialForEndpointAsync( + currentRecord, + deviceCredential, + _operationCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + if (!IsCurrentGatewayAttempt(attempt.LifecycleGeneration, gatewayRecordId)) + return; + if (authorization.Allowed) + { + _activeLifecycle?.DataClient.SetAssistantMediaAuthToken(token.Token); + } + else + { + _activeLifecycle?.DataClient.SetAssistantMediaAuthToken(null); + _diagnostics.Record( + "credentials", + "Interactive HTTP device credential was withheld after token refresh", + authorization.Detail); + } } finally { diff --git a/src/OpenClaw.Connection/ICredentialResolver.cs b/src/OpenClaw.Connection/ICredentialResolver.cs index a2c045792..92fc9ef29 100644 --- a/src/OpenClaw.Connection/ICredentialResolver.cs +++ b/src/OpenClaw.Connection/ICredentialResolver.cs @@ -11,6 +11,7 @@ public sealed record GatewayCredential(string Token, bool IsBootstrapToken, stri GatewayCredentialResolutionStatus.Resolved; public bool FallbackUsed { get; init; } public string? ResolutionDetail { get; init; } + public string? InteractiveHttpToken { get; init; } } /// diff --git a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs index fd0058153..7f588f421 100644 --- a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs +++ b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs @@ -10,6 +10,22 @@ namespace OpenClaw.Connection; /// public static class InteractiveGatewayCredentialResolver { + internal static GatewayCredential? ResolveForAssistantMediaHttpSurface( + GatewayRecord record, + GatewayCredential? fallback) => + !string.IsNullOrWhiteSpace(record.SharedGatewayToken) + ? new GatewayCredential( + record.SharedGatewayToken!, + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken) + : fallback is + { + IsBootstrapToken: false, + Source: CredentialResolver.SourceDeviceToken, + } + ? fallback + : null; + public static bool TryResolve( GatewayRegistry? registry, string settingsDirectory, diff --git a/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs b/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs new file mode 100644 index 000000000..2be240799 --- /dev/null +++ b/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs @@ -0,0 +1,454 @@ +using System.Net; +using System.Text; +using System.Text.RegularExpressions; + +namespace OpenClaw.Shared; + +internal sealed record AssistantMediaDirectiveProjection( + string Text, + IReadOnlyList ContentParts, + bool HasDirective); + +/// +/// Projects assistant-only legacy MEDIA directives into typed transport +/// references. This is deliberately shared by history and live parsing so raw +/// Gateway paths cannot leak through notification or presentation text. +/// +internal static class AssistantMediaDirectiveParser +{ + private const int MaxSourceLength = 4096; + private const int MaxFileNameLength = 255; + internal const int MaxMediaReferences = 16; + + private static readonly Regex s_hasFileExtension = + new(@"\.\w{1,10}$", RegexOptions.CultureInvariant); + private static readonly Regex s_traversalSegment = + new(@"(?:^|[/\\])\.\.(?:[/\\]|$)", RegexOptions.CultureInvariant); + private static readonly Regex s_windowsDrive = + new(@"^[a-zA-Z]:[\\/]", RegexOptions.CultureInvariant); + private static readonly Regex s_scheme = + new(@"^[a-zA-Z][a-zA-Z0-9+.-]*:", RegexOptions.CultureInvariant); + + public static AssistantMediaDirectiveProjection Project(string? role, string? raw) + { + var text = raw ?? string.Empty; + if (!string.Equals(role, "assistant", StringComparison.OrdinalIgnoreCase) + || text.IndexOf("MEDIA:", StringComparison.OrdinalIgnoreCase) < 0) + { + return new(text, TextOnly(text), HasDirective: false); + } + + var trimmedRaw = text.TrimEnd(); + if (string.IsNullOrWhiteSpace(trimmedRaw)) + return new(string.Empty, Array.Empty(), HasDirective: false); + + var keptLines = new List(); + var parts = new List(); + var inFence = false; + var fenceCharacter = '\0'; + var fenceLength = 0; + var foundDirective = false; + var mediaReferenceCount = 0; + + foreach (var line in trimmedRaw.Split('\n')) + { + var lineWithoutCarriageReturn = line.EndsWith('\r') ? line[..^1] : line; + if (TryReadFence(lineWithoutCarriageReturn, out var currentFence, out var currentLength)) + { + if (!inFence) + { + inFence = true; + fenceCharacter = currentFence; + fenceLength = currentLength; + } + else if (currentFence == fenceCharacter && currentLength >= fenceLength) + { + inFence = false; + } + + KeepTextLine(lineWithoutCarriageReturn, keptLines, parts); + continue; + } + + var trimmedStart = lineWithoutCarriageReturn.TrimStart(); + if (inFence || !trimmedStart.StartsWith("MEDIA:", StringComparison.OrdinalIgnoreCase)) + { + KeepTextLine(lineWithoutCarriageReturn, keptLines, parts); + continue; + } + + foundDirective = true; + var payload = trimmedStart["MEDIA:".Length..]; + var projection = ParsePayload( + payload, + Math.Max(0, MaxMediaReferences - mediaReferenceCount)); + if (projection.Media.Count > 0) + { + foreach (var media in projection.Media) + { + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }); + } + mediaReferenceCount += projection.Media.Count; + } + + if (!string.IsNullOrWhiteSpace(projection.ResidualText)) + { + var residual = projection.ResidualText.Trim(); + keptLines.Add(residual); + AppendTextPart(parts, residual); + } + else if (!projection.StripLine) + { + keptLines.Add(lineWithoutCarriageReturn); + AppendTextPart(parts, lineWithoutCarriageReturn); + } + } + + var visibleText = string.Join('\n', keptLines); + visibleText = Regex.Replace(visibleText, @"^(?:[ \t]*\n)+", string.Empty).TrimEnd(); + return new(visibleText, parts, foundDirective); + } + + private static IReadOnlyList TextOnly(string text) => + string.IsNullOrEmpty(text) + ? Array.Empty() + : new[] + { + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = text, + }, + }; + + private static void KeepTextLine( + string line, + List keptLines, + List parts) + { + keptLines.Add(line); + AppendTextPart(parts, line); + } + + private static void AppendTextPart(List parts, string text) + { + if (parts.Count > 0 && parts[^1].Kind == ChatMessageContentPartKind.Text) + { + parts[^1].Text = $"{parts[^1].Text}\n{(text.Trim().Length > 0 ? text : string.Empty)}"; + return; + } + + if (text.Trim().Length == 0) + return; + + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = text, + }); + } + + private static bool TryReadFence(string line, out char fenceCharacter, out int fenceLength) + { + fenceCharacter = '\0'; + fenceLength = 0; + var index = 0; + while (index < line.Length && index < 3 && line[index] == ' ') + index++; + if (index >= line.Length || line[index] is not ('`' or '~')) + return false; + + fenceCharacter = line[index]; + while (index + fenceLength < line.Length && line[index + fenceLength] == fenceCharacter) + fenceLength++; + return fenceLength >= 3; + } + + private static PayloadProjection ParsePayload(string rawPayload, int remainingReferences) + { + var unwrapped = UnwrapQuoted(rawPayload); + var payload = unwrapped ?? rawPayload; + var candidates = unwrapped is not null + ? new[] { unwrapped } + : SplitUnquotedParts(rawPayload); + var media = new List(); + var invalidParts = new List(); + + foreach (var part in candidates) + { + var candidate = NormalizeSource(CleanCandidate(part)); + if (IsValidSource(candidate, allowSpaces: unwrapped is not null || part.Any(char.IsWhiteSpace))) + { + if (media.Count < remainingReferences) + media.Add(CreateLegacyMedia(candidate)); + } + else if (!part.Any(char.IsWhiteSpace) || !HasTraversalOrUnsupportedHomePrefix(candidate)) + { + invalidParts.Add(part); + } + } + + var payloadValue = payload.Trim(); + var looksLocal = LooksLikeLocalPath(payloadValue) + || rawPayload.TrimStart().StartsWith("file://", StringComparison.OrdinalIgnoreCase); + + if (media.Count == 0 && payloadValue.Any(char.IsWhiteSpace)) + { + var fallback = NormalizeSource(CleanCandidate(payloadValue)); + if (IsValidSource(fallback, allowSpaces: true, allowBareFileName: true)) + { + if (remainingReferences > 0) + media.Add(CreateLegacyMedia(fallback)); + invalidParts.Clear(); + } + } + + if (media.Count == 0) + { + var fallback = NormalizeSource(CleanCandidate(payloadValue)); + if (IsValidSource(fallback, allowSpaces: true, allowBareFileName: true)) + { + if (remainingReferences > 0) + media.Add(CreateLegacyMedia(fallback)); + invalidParts.Clear(); + } + } + + if (media.Count > 0) + return new(media, CleanLineText(string.Join(' ', invalidParts)), StripLine: true); + + if (looksLocal) + { + if (remainingReferences > 0) + { + media.Add(new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Unknown, + Source = ChatMediaContentSource.Unavailable, + }); + } + return new(media, string.Empty, StripLine: true); + } + + return new(media, string.Empty, StripLine: false); + } + + private static IReadOnlyList SplitUnquotedParts(string payload) + { + var matches = Regex.Matches(payload, @"\S+"); + var parts = new List(matches.Count); + var previousEnd = 0; + foreach (Match match in matches) + { + var candidate = NormalizeSource(CleanCandidate(match.Value)); + var previous = parts.Count > 0 ? parts[^1] : null; + var previousCandidate = previous is null + ? string.Empty + : NormalizeSource(CleanCandidate(previous)); + if (previous is not null + && BeginsRootedSource(previousCandidate) + && !BeginsIndependentSource(candidate) + && (!s_hasFileExtension.IsMatch(previousCandidate) || !IsValidSource(candidate))) + { + parts[^1] = $"{previous}{payload[previousEnd..match.Index]}{match.Value}"; + } + else + { + parts.Add(match.Value); + } + + previousEnd = match.Index + match.Length; + } + + return parts; + } + + private static string? UnwrapQuoted(string value) + { + var trimmed = value.Trim(); + if (trimmed.Length < 2 || trimmed[0] != trimmed[^1] || trimmed[0] is not ('"' or '\'' or '`')) + return null; + return trimmed[1..^1].Trim(); + } + + private static string NormalizeSource(string source) => + source.StartsWith("file://", StringComparison.OrdinalIgnoreCase) + ? source["file://".Length..] + : source; + + private static string CleanCandidate(string raw) + { + var stripped = raw.TrimStart('`', '"', '\'', '[', '{', '(') + .TrimEnd('`', '"', '\'', '\\', '}', ')', ']', ','); + var quoteAfterExtension = Regex.Match( + stripped, + @"^(.*\.\w{1,10})\\?""(?=[\]},:]|$).*", + RegexOptions.Singleline | RegexOptions.CultureInvariant); + return quoteAfterExtension.Success ? quoteAfterExtension.Groups[1].Value : stripped; + } + + private static bool IsValidSource( + string candidate, + bool allowSpaces = false, + bool allowBareFileName = false) + { + if (string.IsNullOrEmpty(candidate) + || candidate.Length > MaxSourceLength + || (!allowSpaces && candidate.Any(char.IsWhiteSpace))) + { + return false; + } + + if (Uri.TryCreate(candidate, UriKind.Absolute, out var uri) + && uri.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + return string.IsNullOrEmpty(uri.UserInfo) && IsAllowedRemoteHost(uri.Host); + } + + if (IsLikelyLocalPath(candidate)) + return true; + + return allowBareFileName + && !s_scheme.IsMatch(candidate) + && s_hasFileExtension.IsMatch(candidate) + && !HasTraversalOrUnsupportedHomePrefix(candidate); + } + + private static bool IsAllowedRemoteHost(string host) + { + var normalized = host.Trim().Trim('[', ']').TrimEnd('.').ToLowerInvariant(); + if (string.IsNullOrEmpty(normalized) + || !normalized.Contains('.') + || normalized is "localhost" or "localhost.localdomain" or "metadata.google.internal" + || normalized.EndsWith(".localhost", StringComparison.Ordinal) + || normalized.EndsWith(".local", StringComparison.Ordinal) + || normalized.EndsWith(".internal", StringComparison.Ordinal)) + { + return false; + } + + if (!IPAddress.TryParse(normalized, out var address)) + return true; + + if (IPAddress.IsLoopback(address) || address.IsIPv6LinkLocal || address.IsIPv6SiteLocal) + return false; + if (address.IsIPv4MappedToIPv6) + address = address.MapToIPv4(); + var bytes = address.GetAddressBytes(); + return address.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork + || !(bytes[0] == 10 + || bytes[0] == 127 + || (bytes[0] == 169 && bytes[1] == 254) + || (bytes[0] == 172 && bytes[1] is >= 16 and <= 31) + || (bytes[0] == 192 && bytes[1] == 168) + || bytes[0] >= 224); + } + + private static bool IsLikelyLocalPath(string candidate) => + !HasTraversalOrUnsupportedHomePrefix(candidate) + && (candidate.StartsWith('/') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("~/", StringComparison.Ordinal) + || candidate.StartsWith("~\\", StringComparison.Ordinal) + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal) + || (!s_scheme.IsMatch(candidate) + && (candidate.Contains('/') || candidate.Contains('\\')))); + + private static bool LooksLikeLocalPath(string candidate) => + candidate.StartsWith('/') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("../", StringComparison.Ordinal) + || candidate.StartsWith('~') + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal) + || (!s_scheme.IsMatch(candidate) + && (candidate.Contains('/') || candidate.Contains('\\'))); + + private static bool HasTraversalOrUnsupportedHomePrefix(string candidate) => + candidate.StartsWith("../", StringComparison.Ordinal) + || candidate == ".." + || (candidate.StartsWith('~') + && !candidate.StartsWith("~/", StringComparison.Ordinal) + && !candidate.StartsWith("~\\", StringComparison.Ordinal)) + || s_traversalSegment.IsMatch(candidate); + + private static bool BeginsRootedSource(string candidate) => + candidate.StartsWith('/') + || candidate.StartsWith('~') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("../", StringComparison.Ordinal) + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal); + + private static bool BeginsIndependentSource(string candidate) => + BeginsRootedSource(candidate) || s_scheme.IsMatch(candidate); + + private static ChatMediaContentInfo CreateLegacyMedia(string source) + { + var fileName = SafeFileName(source); + return new ChatMediaContentInfo + { + Kind = ClassifyByFileName(fileName), + Source = ChatMediaContentSource.LegacyDirective, + FileName = fileName, + GatewaySource = source, + }; + } + + internal static string? SafeFileName(string source) + { + var path = source; + if (Uri.TryCreate(source, UriKind.Absolute, out var uri) && uri.Scheme == Uri.UriSchemeHttps) + path = Uri.UnescapeDataString(uri.AbsolutePath); + path = path.Replace('\\', '/'); + var leaf = path[(path.LastIndexOf('/') + 1)..]; + try + { + leaf = Uri.UnescapeDataString(leaf); + } + catch (UriFormatException) + { + return null; + } + + var builder = new StringBuilder(Math.Min(leaf.Length, MaxFileNameLength)); + foreach (var character in leaf) + { + if (!char.IsControl(character) && character is not '\r' and not '\n') + builder.Append(character); + if (builder.Length == MaxFileNameLength) + break; + } + + var result = builder.ToString().Trim(); + return result.Length == 0 ? null : result; + } + + private static ChatMediaContentKind ClassifyByFileName(string? fileName) + { + var extension = Path.GetExtension(fileName ?? string.Empty).ToLowerInvariant(); + return extension switch + { + ".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".heic" or ".avif" + => ChatMediaContentKind.Image, + ".mp3" or ".wav" or ".m4a" or ".aac" or ".ogg" or ".flac" + => ChatMediaContentKind.Audio, + ".mp4" or ".mov" or ".m4v" or ".webm" or ".avi" + => ChatMediaContentKind.Video, + _ => ChatMediaContentKind.File, + }; + } + + private static string CleanLineText(string text) => + Regex.Replace(text, @"[ \t]{2,}", " ").Trim(); + + private sealed record PayloadProjection( + IReadOnlyList Media, + string ResidualText, + bool StripLine); +} diff --git a/src/OpenClaw.Shared/AssistantMediaResolution.cs b/src/OpenClaw.Shared/AssistantMediaResolution.cs new file mode 100644 index 000000000..88296ad0f --- /dev/null +++ b/src/OpenClaw.Shared/AssistantMediaResolution.cs @@ -0,0 +1,25 @@ +namespace OpenClaw.Shared; + +public enum AssistantMediaResolutionStatus +{ + Ready, + Preparing, + Unavailable, +} + +public sealed record AssistantMediaResolutionResult( + AssistantMediaResolutionStatus Status, + byte[]? Data = null, + string? MimeType = null) +{ + public static AssistantMediaResolutionResult Preparing { get; } = + new(AssistantMediaResolutionStatus.Preparing); + + public static AssistantMediaResolutionResult Unavailable { get; } = + new(AssistantMediaResolutionStatus.Unavailable); +} + +internal readonly record struct GatewayConnectionLease( + Guid ClientId, + long Generation, + Uri HttpBaseUri); diff --git a/src/OpenClaw.Shared/Models.cs b/src/OpenClaw.Shared/Models.cs index e006151aa..b5b65a1eb 100644 --- a/src/OpenClaw.Shared/Models.cs +++ b/src/OpenClaw.Shared/Models.cs @@ -1809,7 +1809,7 @@ public static bool IsSilentAssistantDirective(string? role, string? text) => Array.Empty(); /// - /// Ordered text and tool blocks retained from array-valued message content. + /// Ordered text, tool, and media blocks retained from message content. /// Flat and remain populated for /// compatibility with consumers that do not need block-level chronology. /// @@ -1893,6 +1893,7 @@ public enum ChatMessageContentPartKind { Text, Tool, + Media, } public class ChatMessageContentPartInfo @@ -1900,6 +1901,55 @@ public class ChatMessageContentPartInfo public ChatMessageContentPartKind Kind { get; set; } public string? Text { get; set; } public ChatToolContentInfo? Tool { get; set; } + public ChatMediaContentInfo? Media { get; set; } +} + +public enum ChatMediaContentKind +{ + Image, + Audio, + Video, + File, + Unknown, +} + +public enum ChatMediaPlaybackMode +{ + Native, + Transcode, +} + +public enum ChatMediaContentSource +{ + Structured, + LegacyDirective, + Unavailable, +} + +/// +/// Typed assistant media metadata retained from the Gateway protocol. Legacy +/// Gateway sources are transient transport references and must never be +/// displayed, logged, persisted, or passed to local file APIs. +/// +public sealed class ChatMediaContentInfo +{ + public ChatMediaContentKind Kind { get; set; } + public ChatMediaContentSource Source { get; set; } + public string? Type { get; set; } + public string? MimeType { get; set; } + public string? FileName { get; set; } + public string? ArtifactId { get; set; } + public string? AgentId { get; set; } + public string? Url { get; set; } + public string? OpenUrl { get; set; } + public string? Alt { get; set; } + public int? Width { get; set; } + public int? Height { get; set; } + public long? SizeBytes { get; set; } + public double? DurationSeconds { get; set; } + public ChatMediaPlaybackMode? Playback { get; set; } + + internal string? GatewaySource { get; set; } } /// diff --git a/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs b/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs new file mode 100644 index 000000000..0f7425bde --- /dev/null +++ b/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs @@ -0,0 +1,488 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; + +namespace OpenClaw.Shared; + +public partial class OpenClawGatewayClient +{ + internal const int MaximumAssistantImageBytes = 12 * 1024 * 1024; + internal const int MaximumAssistantPlaybackBytes = 16 * 1024 * 1024; + private const int MaximumAssistantMediaMetadataBytes = 64 * 1024; + private const string ManagedMediaPathPrefix = "/api/chat/media/outgoing/"; + private readonly Guid _mediaClientId = Guid.NewGuid(); + private readonly HttpClient _assistantMediaHttpClient; + + public async Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(sessionKey) + || media.Kind is ChatMediaContentKind.File or ChatMediaContentKind.Unknown + || !TryCaptureMediaLease(out var lease)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + CancellationToken); + try + { + var result = media.Source switch + { + ChatMediaContentSource.Structured => + await ResolveStructuredMediaAsync( + lease, + sessionKey, + media, + linkedCancellation.Token).ConfigureAwait(false), + ChatMediaContentSource.LegacyDirective => + await ResolveLegacyMediaAsync( + lease, + media, + linkedCancellation.Token).ConfigureAwait(false), + _ => AssistantMediaResolutionResult.Unavailable, + }; + return IsCurrentMediaLease(lease) + ? result + : AssistantMediaResolutionResult.Unavailable; + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || CancellationToken.IsCancellationRequested) + { + return AssistantMediaResolutionResult.Unavailable; + } + catch (Exception ex) + { + _logger.Warn($"Assistant media resolution failed ({ex.GetType().Name})."); + return AssistantMediaResolutionResult.Unavailable; + } + } + + private async Task ResolveStructuredMediaAsync( + GatewayConnectionLease lease, + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(media.ArtifactId)) + return AssistantMediaResolutionResult.Unavailable; + + var parameters = string.IsNullOrWhiteSpace(media.AgentId) + ? new { sessionKey, artifactId = media.ArtifactId } + : (object)new { sessionKey, artifactId = media.ArtifactId, agentId = media.AgentId }; + var payload = await SendWizardRequestAsync( + "artifacts.download", + parameters, + timeoutMs: 20000) + .WaitAsync(cancellationToken) + .ConfigureAwait(false); + + if (!IsCurrentMediaLease(lease) + || !TryReadArtifactMetadata(payload, media.Kind, out var mimeType, out var sizeBytes)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var maximumBytes = MaximumBytes(media.Kind); + if (sizeBytes is > 0 && sizeBytes > maximumBytes) + return AssistantMediaResolutionResult.Unavailable; + + if (payload.TryGetProperty("data", out var dataElement) + && dataElement.ValueKind == JsonValueKind.String) + { + if (!payload.TryGetProperty("encoding", out var encoding) + || encoding.ValueKind != JsonValueKind.String + || !string.Equals(encoding.GetString(), "base64", StringComparison.Ordinal)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + return TryDecodeBoundedBase64(dataElement.GetString(), maximumBytes, out var bytes) + ? new AssistantMediaResolutionResult( + AssistantMediaResolutionStatus.Ready, + bytes, + mimeType) + : AssistantMediaResolutionResult.Unavailable; + } + + if (!payload.TryGetProperty("url", out var urlElement) + || urlElement.ValueKind != JsonValueKind.String + || !TryResolveManagedMediaUri(lease.HttpBaseUri, urlElement.GetString(), out var mediaUri)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + return await DownloadMediaBytesAsync( + lease, + mediaUri, + media.Kind, + maximumBytes, + cancellationToken).ConfigureAwait(false); + } + + private async Task ResolveLegacyMediaAsync( + GatewayConnectionLease lease, + ChatMediaContentInfo media, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(media.GatewaySource)) + return AssistantMediaResolutionResult.Unavailable; + + var metadataUri = BuildLegacyMediaUri( + lease.HttpBaseUri, + media.GatewaySource, + mediaTicket: null, + metadata: true, + playback: false); + using var metadataRequest = CreateAuthenticatedMediaRequest(metadataUri, "application/json"); + using var metadataResponse = await _assistantMediaHttpClient.SendAsync( + metadataRequest, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + if (!metadataResponse.IsSuccessStatusCode) + return AssistantMediaResolutionResult.Unavailable; + + var metadataBytes = await ReadBoundedAsync( + metadataResponse.Content, + MaximumAssistantMediaMetadataBytes, + cancellationToken).ConfigureAwait(false); + if (metadataBytes is null || !IsCurrentMediaLease(lease)) + return AssistantMediaResolutionResult.Unavailable; + + using var metadataDocument = JsonDocument.Parse(metadataBytes); + var root = metadataDocument.RootElement; + if (!root.TryGetProperty("available", out var available) + || available.ValueKind != JsonValueKind.True + || !root.TryGetProperty("mediaTicket", out var ticketElement) + || ticketElement.ValueKind != JsonValueKind.String + || string.IsNullOrWhiteSpace(ticketElement.GetString())) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var maximumBytes = MaximumBytes(media.Kind); + if (root.TryGetProperty("sizeBytes", out var declaredSize) + && declaredSize.TryGetInt64(out var sizeBytes) + && sizeBytes > maximumBytes) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var playback = root.TryGetProperty("playback", out var playbackElement) + && playbackElement.ValueKind == JsonValueKind.String + && string.Equals(playbackElement.GetString(), "transcode", StringComparison.Ordinal); + var bytesUri = BuildLegacyMediaUri( + lease.HttpBaseUri, + media.GatewaySource, + ticketElement.GetString(), + metadata: false, + playback); + return await DownloadMediaBytesAsync( + lease, + bytesUri, + media.Kind, + maximumBytes, + cancellationToken).ConfigureAwait(false); + } + + private async Task DownloadMediaBytesAsync( + GatewayConnectionLease lease, + Uri uri, + ChatMediaContentKind kind, + int maximumBytes, + CancellationToken cancellationToken) + { + using var request = CreateAuthenticatedMediaRequest(uri, $"{MimePrefix(kind)}/*"); + using var response = await _assistantMediaHttpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.Accepted) + return AssistantMediaResolutionResult.Preparing; + if (!response.IsSuccessStatusCode + || response.Content.Headers.ContentLength is > 0 + && response.Content.Headers.ContentLength > maximumBytes) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var mimeType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); + if (!MatchesKind(mimeType, kind)) + return AssistantMediaResolutionResult.Unavailable; + + var bytes = await ReadBoundedAsync( + response.Content, + maximumBytes, + cancellationToken).ConfigureAwait(false); + return bytes is not null && IsCurrentMediaLease(lease) + ? new AssistantMediaResolutionResult( + AssistantMediaResolutionStatus.Ready, + bytes, + mimeType) + : AssistantMediaResolutionResult.Unavailable; + } + + private HttpRequestMessage CreateAuthenticatedMediaRequest(Uri uri, string accept) + { + var request = new HttpRequestMessage(HttpMethod.Get, uri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(accept)); + var authToken = Volatile.Read(ref _assistantMediaAuthToken); + if (!string.IsNullOrWhiteSpace(authToken)) + request.Headers.Authorization = new AuthenticationHeaderValue( + "Bearer", + authToken); + return request; + } + + private bool TryCaptureMediaLease(out GatewayConnectionLease lease) + { + lease = default; + if (!IsConnectedToGateway || !TryBuildMediaHttpBaseUri(_currentGatewayUrl, out var baseUri)) + return false; + lease = new GatewayConnectionLease(_mediaClientId, ConnectionGeneration, baseUri); + return true; + } + + private bool IsCurrentMediaLease(GatewayConnectionLease lease) => + lease.ClientId == _mediaClientId + && lease.Generation == ConnectionGeneration + && IsConnectedToGateway; + + internal static bool TryBuildMediaHttpBaseUri(string gatewayUrl, out Uri baseUri) + { + baseUri = null!; + var normalized = GatewayUrlHelper.NormalizeForWebSocket(gatewayUrl); + if (!Uri.TryCreate(normalized, UriKind.Absolute, out var gatewayUri)) + return false; + var builder = new UriBuilder(gatewayUri) + { + Scheme = gatewayUri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase) + ? Uri.UriSchemeHttps + : Uri.UriSchemeHttp, + UserName = string.Empty, + Password = string.Empty, + Query = string.Empty, + Fragment = string.Empty, + }; + baseUri = builder.Uri; + return true; + } + + internal static bool TryResolveManagedMediaUri( + Uri baseUri, + string? relativePath, + out Uri mediaUri) + { + mediaUri = null!; + if (string.IsNullOrWhiteSpace(relativePath) + || !relativePath.StartsWith(ManagedMediaPathPrefix, StringComparison.Ordinal) + || relativePath.StartsWith("//", StringComparison.Ordinal) + || relativePath.Contains('\\') + || HasTraversal(relativePath.Split('?', 2)[0])) + { + return false; + } + + if (!Uri.TryCreate(relativePath, UriKind.Relative, out var relative) + || !Uri.TryCreate(new Uri(baseUri.GetLeftPart(UriPartial.Authority)), relative, out var candidate) + || !string.Equals(candidate.Host, baseUri.Host, StringComparison.OrdinalIgnoreCase) + || candidate.Port != baseUri.Port + || !string.IsNullOrEmpty(candidate.Fragment) + || HasTraversal(candidate.AbsolutePath) + || !HasNonEmptyQueryValue(candidate.Query, "mediaTicket")) + { + return false; + } + + mediaUri = candidate; + return true; + } + + private static Uri BuildLegacyMediaUri( + Uri baseUri, + string source, + string? mediaTicket, + bool metadata, + bool playback) + { + var basePath = baseUri.AbsolutePath.TrimEnd('/'); + var route = $"{basePath}/__openclaw__/assistant-media"; + var query = new StringBuilder() + .Append("source=") + .Append(Uri.EscapeDataString(source)); + if (metadata) + query.Append("&meta=1"); + if (!string.IsNullOrWhiteSpace(mediaTicket)) + query.Append("&mediaTicket=").Append(Uri.EscapeDataString(mediaTicket)); + if (playback) + query.Append("&playback=1"); + return new UriBuilder(baseUri) { Path = route, Query = query.ToString() }.Uri; + } + + private static bool TryReadArtifactMetadata( + JsonElement payload, + ChatMediaContentKind expectedKind, + out string mimeType, + out long? sizeBytes) + { + mimeType = string.Empty; + sizeBytes = null; + if (!payload.TryGetProperty("artifact", out var artifact) + || artifact.ValueKind != JsonValueKind.Object + || !artifact.TryGetProperty("type", out var typeElement) + || typeElement.ValueKind != JsonValueKind.String + || !string.Equals( + typeElement.GetString(), + expectedKind.ToString(), + StringComparison.OrdinalIgnoreCase) + || !artifact.TryGetProperty("mimeType", out var mimeElement) + || mimeElement.ValueKind != JsonValueKind.String + || !MatchesKind(mimeElement.GetString(), expectedKind)) + { + return false; + } + + mimeType = mimeElement.GetString()!.Trim().ToLowerInvariant(); + if (artifact.TryGetProperty("sizeBytes", out var sizeElement) + && sizeElement.TryGetInt64(out var declaredSize)) + { + sizeBytes = declaredSize; + } + return true; + } + + private static async Task ReadBoundedAsync( + HttpContent content, + int maximumBytes, + CancellationToken cancellationToken) + { + await using var input = await content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var output = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + var buffer = new byte[32 * 1024]; + while (true) + { + var read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read == 0) + return output.ToArray(); + if (output.Length + read > maximumBytes) + return null; + output.Write(buffer, 0, read); + } + } + + internal static bool TryDecodeBoundedBase64(string? encoded, int maximumBytes, out byte[] bytes) + { + bytes = Array.Empty(); + if (string.IsNullOrEmpty(encoded) || maximumBytes <= 0) + return false; + + var maximumEncodedLength = ((maximumBytes + 2L) / 3L) * 4L; + var compactLength = 0; + foreach (var character in encoded) + { + if (char.IsWhiteSpace(character)) + continue; + compactLength++; + if (compactLength > maximumEncodedLength) + return false; + } + + if (compactLength == 0 || compactLength % 4 == 1) + return false; + + var paddedLength = compactLength + (4 - compactLength % 4) % 4; + if (paddedLength > maximumEncodedLength) + return false; + + var normalized = string.Create(paddedLength, encoded, static (destination, source) => + { + var index = 0; + foreach (var character in source) + { + if (char.IsWhiteSpace(character)) + continue; + destination[index++] = character switch + { + '-' => '+', + '_' => '/', + _ => character, + }; + } + destination[index..].Fill('='); + }); + + var padding = normalized[^1] == '=' + ? normalized.Length > 1 && normalized[^2] == '=' ? 2 : 1 + : 0; + var decodedLength = paddedLength / 4 * 3 - padding; + if (decodedLength > maximumBytes) + return false; + + var decoded = GC.AllocateUninitializedArray(decodedLength); + if (!Convert.TryFromBase64Chars(normalized, decoded, out var bytesWritten) + || bytesWritten != decodedLength) + { + return false; + } + + bytes = decoded; + return true; + } + + private static int MaximumBytes(ChatMediaContentKind kind) => + kind == ChatMediaContentKind.Image + ? MaximumAssistantImageBytes + : MaximumAssistantPlaybackBytes; + + private static string MimePrefix(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => "image", + ChatMediaContentKind.Audio => "audio", + ChatMediaContentKind.Video => "video", + _ => "application", + }; + + private static bool MatchesKind(string? mimeType, ChatMediaContentKind kind) => + !string.IsNullOrWhiteSpace(mimeType) + && mimeType.StartsWith($"{MimePrefix(kind)}/", StringComparison.OrdinalIgnoreCase); + + private static bool HasTraversal(string absolutePath) + { + string decoded; + try + { + decoded = Uri.UnescapeDataString(absolutePath); + } + catch (UriFormatException) + { + return true; + } + return decoded.Split('/', StringSplitOptions.RemoveEmptyEntries) + .Any(static segment => segment is "." or ".."); + } + + private static bool HasNonEmptyQueryValue(string query, string key) + { + foreach (var pair in query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var separator = pair.IndexOf('='); + if (separator <= 0) + continue; + if (string.Equals( + Uri.UnescapeDataString(pair[..separator]), + key, + StringComparison.Ordinal) + && separator + 1 < pair.Length) + { + return true; + } + + } + return false; + } + +} diff --git a/src/OpenClaw.Shared/OpenClawGatewayClient.cs b/src/OpenClaw.Shared/OpenClawGatewayClient.cs index b8e9f8450..3c33b66e6 100644 --- a/src/OpenClaw.Shared/OpenClawGatewayClient.cs +++ b/src/OpenClaw.Shared/OpenClawGatewayClient.cs @@ -70,6 +70,7 @@ public partial class OpenClawGatewayClient : WebSocketClientBase, IOperatorGatew private string? _operatorDeviceId; private string[] _grantedOperatorScopes = Array.Empty(); private string _connectAuthToken; + private string? _assistantMediaAuthToken; private bool _useV2Signature; // true after v3 signature rejected by gateway /// Set to true to skip v3 and use v2 signatures directly (for gateways that don't support v3). @@ -201,6 +202,7 @@ protected override void OnDisconnected() protected override void OnDisposing() { _pendingRequests.Drain(); + _assistantMediaHttpClient.Dispose(); } protected override void OnError(Exception ex) @@ -291,13 +293,27 @@ protected override void OnReconnectAuthorizationDenied( protected void RaiseConnectionFailure(GatewayErrorKind kind) => ConnectionFailure?.Invoke(this, kind); - public OpenClawGatewayClient(string gatewayUrl, string token, IOpenClawLogger? logger = null, bool tokenIsBootstrapToken = false, bool bootstrapPairAsNode = false, string? identityPath = null, bool ignoreStoredDeviceToken = false, bool persistHandshakeDeviceTokens = true) + public OpenClawGatewayClient( + string gatewayUrl, + string token, + IOpenClawLogger? logger = null, + bool tokenIsBootstrapToken = false, + bool bootstrapPairAsNode = false, + string? identityPath = null, + bool ignoreStoredDeviceToken = false, + bool persistHandshakeDeviceTokens = true, + string? assistantMediaAuthToken = null, + HttpMessageHandler? assistantMediaHandler = null) : base(gatewayUrl, token, logger) { _tokenIsBootstrapToken = tokenIsBootstrapToken; _bootstrapPairAsNode = bootstrapPairAsNode; _ignoreStoredDeviceToken = ignoreStoredDeviceToken; _persistHandshakeDeviceTokens = persistHandshakeDeviceTokens; + _assistantMediaHttpClient = assistantMediaHandler is null + ? new HttpClient(new HttpClientHandler { AllowAutoRedirect = false }) + : new HttpClient(assistantMediaHandler, disposeHandler: true); + SetAssistantMediaAuthToken(assistantMediaAuthToken); _currentGatewayUrl = gatewayUrl; var dataPath = identityPath ?? OpenClawAppIdentity.ResolveRoamingDataDirectory( Environment.GetEnvironmentVariable); @@ -309,6 +325,15 @@ public OpenClawGatewayClient(string gatewayUrl, string token, IOpenClawLogger? l _useV2Signature |= _tokenIsBootstrapToken && !HasUsableOperatorDeviceToken; } + /// + /// Updates the independently authorized credential used by assistant-media + /// HTTP requests. A null or blank value disables authenticated media HTTP. + /// + public void SetAssistantMediaAuthToken(string? token) => + Volatile.Write( + ref _assistantMediaAuthToken, + string.IsNullOrWhiteSpace(token) ? null : token); + public async Task DisconnectAsync() { if (IsConnected) @@ -622,10 +647,15 @@ private static ChatHistoryInfo ParseChatHistory(JsonElement payload, string sess stopReason = sr.GetString(); // Content can include text and structured tool call/result blocks. - string text = ExtractMessageText(m); + string text = ExtractMessageText(m, role); var toolContent = ExtractToolContent(m, role, text); var contentParts = ExtractOrderedMessageContent(m, role, toolContent); - if (string.IsNullOrEmpty(text) && toolContent.Count == 0) continue; + if (string.IsNullOrEmpty(text) + && toolContent.Count == 0 + && contentParts.All(static part => part.Kind != ChatMessageContentPartKind.Media)) + { + continue; + } if (string.IsNullOrEmpty(role)) continue; if (!string.IsNullOrEmpty(text) && toolContent.Count == 0 @@ -660,12 +690,14 @@ private static ChatHistoryInfo ParseChatHistory(JsonElement payload, string sess return info; } - private static string ExtractMessageText(JsonElement message) + private static string ExtractMessageText(JsonElement message, string role) { if (!message.TryGetProperty("content", out var content)) return string.Empty; if (content.ValueKind == JsonValueKind.String) - return content.GetString() ?? string.Empty; + { + return AssistantMediaDirectiveParser.Project(role, content.GetString()).Text; + } if (content.ValueKind == JsonValueKind.Array) { @@ -674,6 +706,7 @@ private static string ExtractMessageText(JsonElement message) { if (item.ValueKind == JsonValueKind.String) { + if (sb.Length > 0) sb.Append('\n'); sb.Append(item.GetString()); } else if (item.ValueKind == JsonValueKind.Object && @@ -684,7 +717,7 @@ private static string ExtractMessageText(JsonElement message) sb.Append(tx.GetString()); } } - return sb.ToString(); + return AssistantMediaDirectiveParser.Project(role, sb.ToString()).Text; } return string.Empty; @@ -795,34 +828,42 @@ private static IReadOnlyList ExtractOrderedMessageCo IReadOnlyList toolContent) { if (!message.TryGetProperty("content", out var content) - || content.ValueKind != JsonValueKind.Array) + || content.ValueKind is not (JsonValueKind.Array or JsonValueKind.String)) { return Array.Empty(); } + if (content.ValueKind == JsonValueKind.String) + { + return AssistantMediaDirectiveParser.Project(role, content.GetString()).ContentParts; + } + var parts = new List(); - var text = new StringBuilder(); + var textRun = new StringBuilder(); var toolIndex = 0; var structuredToolCount = 0; - void FlushText() + void AppendTextBlock(string? value) { - if (text.Length == 0) - return; + if (textRun.Length > 0) + textRun.Append('\n'); + textRun.Append(value); + } - parts.Add(new ChatMessageContentPartInfo - { - Kind = ChatMessageContentPartKind.Text, - Text = text.ToString(), - }); - text.Clear(); + void FlushTextRun() + { + if (textRun.Length == 0) + return; + parts.AddRange( + AssistantMediaDirectiveParser.Project(role, textRun.ToString()).ContentParts); + textRun.Clear(); } foreach (var item in content.EnumerateArray()) { if (item.ValueKind == JsonValueKind.String) { - text.Append(item.GetString()); + AppendTextBlock(item.GetString()); continue; } @@ -830,6 +871,7 @@ void FlushText() || !item.TryGetProperty("type", out var typeElement) || typeElement.ValueKind != JsonValueKind.String) { + FlushTextRun(); continue; } @@ -841,17 +883,25 @@ void FlushText() if (item.TryGetProperty("text", out var textElement) && textElement.ValueKind == JsonValueKind.String) { - if (text.Length > 0) - text.Append('\n'); - text.Append(textElement.GetString()); + AppendTextBlock(textElement.GetString()); } continue; } + FlushTextRun(); + if (TryParseStructuredMedia(item, normalizedType, out var media)) + { + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }); + continue; + } + if (normalizedType is not ("toolcall" or "tooluse" or "toolresult")) continue; - FlushText(); structuredToolCount++; if (toolIndex < toolContent.Count) { @@ -863,7 +913,7 @@ void FlushText() } } - FlushText(); + FlushTextRun(); var normalizedRole = role.Replace("_", string.Empty, StringComparison.Ordinal) .ToLowerInvariant(); @@ -892,6 +942,178 @@ void FlushText() return parts; } + private static bool TryParseStructuredMedia( + JsonElement item, + string normalizedType, + out ChatMediaContentInfo media) + { + media = null!; + var mimeType = ReadFirstString(item, "mimeType", "mime_type")?.Trim().ToLowerInvariant(); + var hasMediaShape = normalizedType is "image" or "audio" or "video" or "file" or "attachment" + || !string.IsNullOrWhiteSpace(mimeType) + || ReadFirstString(item, "artifactId", "artifact_id") is not null + || ReadFirstString(item, "url") is not null; + if (!hasMediaShape) + return false; + + var kind = normalizedType switch + { + "image" => ChatMediaContentKind.Image, + "audio" => ChatMediaContentKind.Audio, + "video" => ChatMediaContentKind.Video, + "file" or "attachment" => ClassifyMediaMimeType(mimeType, ChatMediaContentKind.File), + _ => ClassifyMediaMimeType(mimeType, ChatMediaContentKind.Unknown), + }; + var url = ReadFirstString(item, "url"); + var artifactId = ReadFirstString(item, "artifactId", "artifact_id") + ?? TryCreateManagedArtifactId(url, kind); + + media = new ChatMediaContentInfo + { + Kind = kind, + Source = ChatMediaContentSource.Structured, + Type = ReadFirstString(item, "type"), + MimeType = NormalizeMediaMimeType(mimeType), + FileName = NormalizeMediaDisplayText( + ReadFirstString(item, "fileName", "file_name"), + 255), + ArtifactId = NormalizeMediaDisplayText(artifactId, 512), + AgentId = NormalizeMediaDisplayText(ReadFirstString(item, "agentId", "agent_id"), 256), + Url = NormalizeMediaProtocolValue(url, 4096), + OpenUrl = NormalizeMediaProtocolValue(ReadFirstString(item, "openUrl", "open_url"), 4096), + Alt = NormalizeMediaDisplayText(ReadFirstString(item, "alt"), 1024), + Width = ReadPositiveInt(item, "width"), + Height = ReadPositiveInt(item, "height"), + SizeBytes = ReadPositiveLong(item, "sizeBytes", "size_bytes"), + DurationSeconds = ReadDurationSeconds(item), + Playback = ReadFirstString(item, "playback")?.ToLowerInvariant() switch + { + "native" => ChatMediaPlaybackMode.Native, + "transcode" => ChatMediaPlaybackMode.Transcode, + _ => null, + }, + }; + return true; + } + + private static ChatMediaContentKind ClassifyMediaMimeType( + string? mimeType, + ChatMediaContentKind fallback) + { + if (mimeType?.StartsWith("image/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Image; + if (mimeType?.StartsWith("audio/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Audio; + if (mimeType?.StartsWith("video/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Video; + return fallback; + } + + private static string? NormalizeMediaMimeType(string? value) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 128) + return null; + var normalized = value.Trim().ToLowerInvariant(); + return normalized.Contains('/') && normalized.All(static character => + character is >= 'a' and <= 'z' + or >= '0' and <= '9' + or '/' or '+' or '-' or '.' or '_') + ? normalized + : null; + } + + private static string? NormalizeMediaDisplayText(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + var builder = new StringBuilder(Math.Min(value.Length, maxLength)); + foreach (var character in value) + { + if (!char.IsControl(character) && character is not '\r' and not '\n') + builder.Append(character); + if (builder.Length == maxLength) + break; + } + var result = builder.ToString().Trim(); + return result.Length == 0 ? null : result; + } + + private static string? NormalizeMediaProtocolValue(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > maxLength) + return null; + return value.Trim(); + } + + private static int? ReadPositiveInt(JsonElement item, string name) + { + if (!item.TryGetProperty(name, out var value) + || value.ValueKind != JsonValueKind.Number + || !value.TryGetInt32(out var result) + || result <= 0) + { + return null; + } + return result; + } + + private static long? ReadPositiveLong(JsonElement item, params string[] names) + { + foreach (var name in names) + { + if (item.TryGetProperty(name, out var value) + && value.ValueKind == JsonValueKind.Number + && value.TryGetInt64(out var result) + && result > 0) + { + return result; + } + } + return null; + } + + private static double? ReadDurationSeconds(JsonElement item) + { + if (item.TryGetProperty("durationSeconds", out var seconds) + && seconds.ValueKind == JsonValueKind.Number + && seconds.TryGetDouble(out var secondsValue) + && secondsValue >= 0) + { + return secondsValue; + } + if (item.TryGetProperty("durationMs", out var milliseconds) + && milliseconds.ValueKind == JsonValueKind.Number + && milliseconds.TryGetDouble(out var millisecondsValue) + && millisecondsValue >= 0) + { + return millisecondsValue / 1000d; + } + return null; + } + + private static string? TryCreateManagedArtifactId(string? rawUrl, ChatMediaContentKind kind) + { + if (string.IsNullOrWhiteSpace(rawUrl) + || !Uri.TryCreate(rawUrl, UriKind.Relative, out var relative) + || relative.IsAbsoluteUri) + { + return null; + } + var path = rawUrl.Split('?', '#')[0]; + var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length != 7 + || !segments[..4].SequenceEqual(new[] { "api", "chat", "media", "outgoing" }) + || !string.Equals(segments[6], "full", StringComparison.Ordinal) + || !Guid.TryParse(segments[5], out var attachmentId)) + { + return null; + } + var prefix = kind is ChatMediaContentKind.Audio or ChatMediaContentKind.Video + ? "artifact_managed_media_" + : "artifact_managed_image_"; + return prefix + attachmentId.ToString("D").ToLowerInvariant(); + } + private static JsonElement? ReadFirstValue(JsonElement value, params string[] propertyNames) { foreach (var propertyName in propertyNames) @@ -3602,8 +3824,14 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) if (inTok is null && outTok is null && respTok is null && ctxPct is null) (inTok, outTok, respTok, ctxPct) = ExtractChatUsage(message); - var text = ExtractMessageText(message); - if (string.IsNullOrEmpty(text)) return; + var text = ExtractMessageText(message, role); + var toolContent = ExtractToolContent(message, role, text); + var contentParts = ExtractOrderedMessageContent(message, role, toolContent); + if (string.IsNullOrEmpty(text) + && contentParts.All(static part => part.Kind != ChatMessageContentPartKind.Media)) + { + return; + } if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; var messageOpenClawMetadata = ExtractOpenClawMetadata(message); @@ -3622,25 +3850,29 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) messageOpenClawMetadata.Seq ?? payloadOpenClawMetadata.Seq, messageOpenClawMetadata.Kind ?? payloadOpenClawMetadata.Kind, messageOpenClawMetadata.TokensBefore ?? payloadOpenClawMetadata.TokensBefore, - messageOpenClawMetadata.TokensAfter ?? payloadOpenClawMetadata.TokensAfter); + messageOpenClawMetadata.TokensAfter ?? payloadOpenClawMetadata.TokensAfter, + contentParts); if (role == "assistant" && string.Equals(state, "final", StringComparison.OrdinalIgnoreCase)) { // HIGH 4: log shape only — content previously // surfaced in the operator log. _logger.Info($"Assistant response: role={role} state={state} len={text.Length}"); - EmitChatNotification(text, sessionKey); + if (!string.IsNullOrWhiteSpace(text)) + EmitChatNotification(text, sessionKey); } } // Legacy format: payload.text + payload.role else if (payload.TryGetProperty("text", out var textProp)) { - var text = textProp.GetString() ?? ""; var role = payload.TryGetProperty("role", out var roleProp) ? roleProp.GetString() ?? "" : ""; var state = payload.TryGetProperty("state", out var stateProp) ? stateProp.GetString() : null; + var projection = AssistantMediaDirectiveParser.Project(role, textProp.GetString()); + var text = projection.Text; - if (!string.IsNullOrEmpty(text)) + if (!string.IsNullOrEmpty(text) + || projection.ContentParts.Any(static part => part.Kind == ChatMessageContentPartKind.Media)) { if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; @@ -3659,13 +3891,15 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) openClawMetadata.Seq, openClawMetadata.Kind, openClawMetadata.TokensBefore, - openClawMetadata.TokensAfter); + openClawMetadata.TokensAfter, + projection.ContentParts); if (role == "assistant") { // HIGH 4: log shape only. _logger.Info($"Assistant response (legacy): role={role} state={state} len={text.Length}"); - EmitChatNotification(text, sessionKey); + if (!string.IsNullOrWhiteSpace(text)) + EmitChatNotification(text, sessionKey); } } } @@ -3735,7 +3969,8 @@ private void EmitChatMessageReceived( int? openClawSeq = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? contentParts = null) { if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; @@ -3747,6 +3982,7 @@ private void EmitChatMessageReceived( SessionKey = sessionKey, Role = role, Text = text, + ContentParts = contentParts ?? Array.Empty(), State = state, Ts = tsMs, InputTokens = inputTokens, diff --git a/src/OpenClaw.Shared/WebSocketClientBase.cs b/src/OpenClaw.Shared/WebSocketClientBase.cs index 534151000..5dcddca06 100644 --- a/src/OpenClaw.Shared/WebSocketClientBase.cs +++ b/src/OpenClaw.Shared/WebSocketClientBase.cs @@ -123,6 +123,9 @@ public abstract class WebSocketClientBase : IDisposable /// Cancellation token tied to this client's lifetime. protected CancellationToken CancellationToken => _cts.Token; + /// Monotonic identity for the current transport connection. + protected long ConnectionGeneration => Interlocked.Read(ref _connectionGeneration); + /// Close status from the current connection's server-originated close frame. protected int? RemoteCloseStatusCode { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs new file mode 100644 index 000000000..996cf2cff --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs @@ -0,0 +1,104 @@ +using OpenClaw.Shared; + +namespace OpenClawTray.Chat; + +public sealed record ChatAssistantContentPresentation( + IReadOnlyList Media); + +public sealed record ChatAssistantMediaPresentation( + ChatMediaContentKind Kind, + string DisplayName, + string? MimeType, + string? Alt, + ChatMediaContentInfo Reference); + +internal sealed record ChatAssistantMediaRenderPlan( + IReadOnlyList Media, + int OmittedImages); + +/// +/// Converts protocol media into renderer-safe presentation metadata. Transport +/// references remain opaque and are never included in display strings. +/// +internal static class ChatAssistantContentProjector +{ + internal const int MaximumInlineImages = 4; + + public static ChatAssistantContentPresentation? Project( + IEnumerable? contentParts) + { + if (contentParts is null) + return null; + + var media = contentParts + .Where(static part => part.Kind == ChatMessageContentPartKind.Media) + .Select(static part => part.Media) + .Where(static item => item is not null) + .Select(static item => new ChatAssistantMediaPresentation( + item!.Kind, + SafeDisplayName(item), + item.MimeType, + item.Alt, + item)) + .ToArray(); + return media.Length == 0 ? null : new ChatAssistantContentPresentation(media); + } + + public static ChatAssistantMediaRenderPlan BuildRenderPlan( + IReadOnlyList media) + { + var renderedImages = 0; + var planned = new List(media.Count); + foreach (var item in media) + { + if (item.Kind == ChatMediaContentKind.Image + && ++renderedImages > MaximumInlineImages) + { + continue; + } + planned.Add(item); + } + return new ChatAssistantMediaRenderPlan( + planned, + Math.Max(0, renderedImages - MaximumInlineImages)); + } + + public static ChatAssistantContentPresentation MergeLiveUpdate( + ChatAssistantContentPresentation? existing, + ChatAssistantContentPresentation incoming) + { + if (existing is null || existing.Media.Count != incoming.Media.Count) + return incoming; + + var merged = incoming.Media.ToArray(); + for (var index = 0; index < merged.Length; index++) + { + var previous = existing.Media[index]; + var next = incoming.Media[index]; + if (previous.Kind == next.Kind + && previous.Reference.Source == ChatMediaContentSource.LegacyDirective + && next.Reference.Source == ChatMediaContentSource.Structured + && (string.IsNullOrWhiteSpace(next.Reference.FileName) + || string.Equals( + previous.Reference.FileName, + next.Reference.FileName, + StringComparison.OrdinalIgnoreCase))) + { + merged[index] = previous; + } + } + return new ChatAssistantContentPresentation(merged); + } + + private static string SafeDisplayName(ChatMediaContentInfo media) + { + if (!string.IsNullOrWhiteSpace(media.FileName)) + { + var normalized = media.FileName.Replace('\\', '/'); + var leaf = normalized[(normalized.LastIndexOf('/') + 1)..].Trim(); + if (leaf.Length > 0) + return leaf; + } + return string.Empty; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs new file mode 100644 index 000000000..67f9a9857 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs @@ -0,0 +1,33 @@ +namespace OpenClawTray.Chat; + +internal static class ChatAssistantImageDecodePolicy +{ + internal const uint MaximumSourceDimension = 16_384; + internal const ulong MaximumDecodedPixels = 32UL * 1024 * 1024; + internal const uint MaximumDecodeDimension = 2_048; + + public static bool TryGetDecodeSize( + uint sourceWidth, + uint sourceHeight, + out int decodeWidth, + out int decodeHeight) + { + decodeWidth = 0; + decodeHeight = 0; + if (sourceWidth == 0 + || sourceHeight == 0 + || sourceWidth > MaximumSourceDimension + || sourceHeight > MaximumSourceDimension + || (ulong)sourceWidth * sourceHeight > MaximumDecodedPixels) + { + return false; + } + + var scale = Math.Min( + 1d, + MaximumDecodeDimension / (double)Math.Max(sourceWidth, sourceHeight)); + decodeWidth = Math.Max(1, (int)Math.Round(sourceWidth * scale)); + decodeHeight = Math.Max(1, (int)Math.Round(sourceHeight * scale)); + return true; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs new file mode 100644 index 000000000..8f11ba84e --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs @@ -0,0 +1,343 @@ +using Microsoft.UI.Reactor; +using Microsoft.UI.Reactor.Core; +using Microsoft.UI.Reactor.Hooks; +using Microsoft.UI.Text; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Media; +using Microsoft.UI.Xaml.Media.Imaging; +using OpenClaw.Shared; +using OpenClawTray.Helpers; +using OpenClawTray.Services; +using Windows.Graphics.Imaging; +using Windows.Storage.Streams; +using static Microsoft.UI.Reactor.Factories; + +namespace OpenClawTray.Chat; + +internal static class ChatAssistantMediaRenderer +{ + public static Element Render( + ChatAssistantMediaPresentation media, + string? sessionKey, + Func>? + resolver) + { + if (media.Kind == ChatMediaContentKind.Image + && !string.IsNullOrWhiteSpace(sessionKey) + && resolver is not null) + { + return Component( + new(media, sessionKey, resolver)); + } + + return BuildUnavailableCard(media); + } + + internal static Element BuildUnavailableCard( + ChatAssistantMediaPresentation media, + bool preparing = false, + Action? retry = null) + { + var kind = KindLabel(media.Kind); + var displayName = DisplayName(media); + var statusText = preparing + ? LocalizedOrDefault("Chat_AssistantMedia_Preparing", $"Preparing {kind.ToLowerInvariant()}") + : LocalizedOrDefault("Chat_AssistantMedia_Unavailable", $"{kind} unavailable"); + var detail = string.IsNullOrWhiteSpace(media.MimeType) + ? statusText + : $"{statusText} · {media.MimeType}"; + var accessibleName = $"{displayName}. {statusText}"; + + var glyph = TextBlock(Glyph(media.Kind)) + .FontSize(18) + .FontWeight(FontWeights.Normal) + .Foreground(Theme.Ref("TextFillColorSecondaryBrush")) + .Set(text => text.FontFamily = FluentIconCatalog.SymbolThemeFontFamily) + .Center(); + var glyphBackground = Border(glyph) + .Size(36, 36) + .CornerRadius(8) + .Background(Theme.Ref("SubtleFillColorSecondaryBrush")); + var title = TextBlock(displayName) + .FontSize(13) + .FontWeight(FontWeights.SemiBold) + .Foreground(Theme.Ref("TextFillColorPrimaryBrush")) + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(320); + var status = TextBlock(detail) + .FontSize(11) + .FontWeight(FontWeights.Normal) + .Foreground(Theme.Ref("TextFillColorSecondaryBrush")) + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(320); + var content = HStack( + 10, + glyphBackground, + VStack(2, title, status).VAlign(VerticalAlignment.Center)); + Element body = retry is null + ? content + : HStack( + 10, + content, + Button( + LocalizedOrDefault("Chat_AssistantMedia_Retry", "Retry"), + retry) + .AutomationName( + LocalizedOrDefault("Chat_AssistantMedia_Retry", "Retry"))); + + return Border(body) + .Padding(10, 8) + .CornerRadius(10) + .Background(Theme.Ref("SubtleFillColorTertiaryBrush")) + .BorderBrush(Theme.Ref("ControlStrokeColorDefaultBrush")) + .BorderThickness(1) + .HAlign(HorizontalAlignment.Left) + .AutomationName(accessibleName); + } + + private static string KindLabel(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => LocalizedOrDefault("Chat_AssistantMedia_Image", "Image"), + ChatMediaContentKind.Audio => LocalizedOrDefault("Chat_AssistantMedia_Audio", "Audio"), + ChatMediaContentKind.Video => LocalizedOrDefault("Chat_AssistantMedia_Video", "Video"), + ChatMediaContentKind.File => LocalizedOrDefault("Chat_AssistantMedia_File", "File"), + _ => LocalizedOrDefault("Chat_AssistantMedia_Media", "Media"), + }; + + internal static string DisplayName(ChatAssistantMediaPresentation media) => + string.IsNullOrWhiteSpace(media.DisplayName) + ? KindLabel(media.Kind) + : media.DisplayName; + + private static string Glyph(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => "\uEB9F", + ChatMediaContentKind.Audio => "\uE8D6", + ChatMediaContentKind.Video => "\uE714", + ChatMediaContentKind.File => "\uE8A5", + _ => "\uE7C3", + }; + + internal static string LocalizedOrDefault(string key, string fallback) + { + var localized = LocalizationHelper.GetString(key); + return string.IsNullOrWhiteSpace(localized) || string.Equals(localized, key, StringComparison.Ordinal) + ? fallback + : localized; + } +} + +internal sealed record ChatAssistantImageCardProps( + ChatAssistantMediaPresentation Media, + string SessionKey, + Func> + Resolver); + +internal sealed class ChatAssistantImageCard : Component +{ + public override Element Render() + { + var props = Props; + var (state, setState) = UseState(null, threadSafe: true); + var (attempt, setAttempt) = UseState(0, threadSafe: true); + var (viewerOpen, setViewerOpen) = UseState(false, threadSafe: true); + + UseEffect((Func)(() => + { + var cancellation = new CancellationTokenSource(); + setViewerOpen(false); + _ = LoadAsync(cancellation.Token); + return () => + { + cancellation.Cancel(); + cancellation.Dispose(); + }; + + async Task LoadAsync(CancellationToken cancellationToken) + { + try + { + var resolved = await props.Resolver( + props.SessionKey, + props.Media.Reference, + cancellationToken); + BitmapImage? bitmap = null; + if (resolved.Status == AssistantMediaResolutionStatus.Ready + && resolved.Data is { Length: > 0 } bytes) + { + bitmap = await TryDecodeBitmapAsync(bytes, cancellationToken); + if (bitmap is null) + resolved = AssistantMediaResolutionResult.Unavailable; + } + if (!cancellationToken.IsCancellationRequested) + { + setState(new ChatAssistantImageLoadState( + resolved.Status, + bitmap, + props.SessionKey, + props.Media.Reference)); + } + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + Logger.Warn($"Assistant image load failed ({ex.GetType().Name})."); + setState(new ChatAssistantImageLoadState( + AssistantMediaResolutionStatus.Unavailable, + null, + props.SessionKey, + props.Media.Reference)); + } + } + }), props.SessionKey, props.Media.Reference, attempt); + + var currentState = state is not null + && string.Equals(state.SessionKey, props.SessionKey, StringComparison.Ordinal) + && ReferenceEquals(state.Reference, props.Media.Reference) + ? state + : null; + if (currentState is + { Status: AssistantMediaResolutionStatus.Ready, Bitmap: { } bitmap }) + { + var image = BuildImage(props.Media, bitmap, () => setViewerOpen(true)); + var viewer = ContentDialog( + props.Media.Alt ?? ChatAssistantMediaRenderer.DisplayName(props.Media), + BuildViewerImage(props.Media, bitmap), + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_Close", + "Close")) with + { + IsOpen = viewerOpen, + OnClosed = _ => setViewerOpen(false), + }; + return Grid( + [GridSize.Star()], + [GridSize.Star()], + image, + viewer); + } + + var preparing = currentState is null + || currentState.Status == AssistantMediaResolutionStatus.Preparing; + return ChatAssistantMediaRenderer.BuildUnavailableCard( + props.Media, + preparing, + currentState is null ? null : () => + { + setState(null); + setAttempt(attempt + 1); + }); + } + + private static Element BuildImage( + ChatAssistantMediaPresentation media, + BitmapImage bitmap, + Action openViewer) + { + const double maximumWidth = 480; + const double maximumHeight = 320; + var pixelWidth = bitmap.PixelWidth > 0 ? bitmap.PixelWidth : (int)maximumWidth; + var pixelHeight = bitmap.PixelHeight > 0 ? bitmap.PixelHeight : (int)maximumHeight; + var scale = Math.Min( + Math.Min(maximumWidth / pixelWidth, maximumHeight / pixelHeight), + 1.0); + var preview = Border(Empty()) + .Set(border => border.Background = new ImageBrush + { + ImageSource = bitmap, + Stretch = Stretch.Uniform, + }) + .Size(pixelWidth * scale, pixelHeight * scale) + .CornerRadius(10) + .AutomationName(media.Alt ?? ChatAssistantMediaRenderer.DisplayName(media)); + return Button(preview, openViewer) + .Padding(0) + .Background(Theme.Ref("SubtleFillColorTransparentBrush")) + .BorderThickness(0) + .AutomationName(string.Format( + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_OpenImage", + "Open image {0}"), + ChatAssistantMediaRenderer.DisplayName(media))); + } + + private static Element BuildViewerImage( + ChatAssistantMediaPresentation media, + BitmapImage bitmap) + { + const double maximumWidth = 1200; + const double maximumHeight = 800; + var pixelWidth = bitmap.PixelWidth > 0 ? bitmap.PixelWidth : (int)maximumWidth; + var pixelHeight = bitmap.PixelHeight > 0 ? bitmap.PixelHeight : (int)maximumHeight; + var scale = Math.Min( + Math.Min(maximumWidth / pixelWidth, maximumHeight / pixelHeight), + 1.0); + return ScrollViewer( + Border(Empty()) + .Set(border => border.Background = new ImageBrush + { + ImageSource = bitmap, + Stretch = Stretch.Uniform, + }) + .Size(pixelWidth * scale, pixelHeight * scale) + .AutomationName(media.Alt ?? ChatAssistantMediaRenderer.DisplayName(media))); + } + + private static async Task TryDecodeBitmapAsync( + byte[] bytes, + CancellationToken cancellationToken) + { + try + { + using var stream = new InMemoryRandomAccessStream(); + using (var writer = new DataWriter(stream)) + { + writer.WriteBytes(bytes); + await writer.StoreAsync().AsTask(cancellationToken); + writer.DetachStream(); + } + stream.Seek(0); + var decoder = await BitmapDecoder.CreateAsync(stream).AsTask(cancellationToken); + if (!ChatAssistantImageDecodePolicy.TryGetDecodeSize( + decoder.PixelWidth, + decoder.PixelHeight, + out var decodeWidth, + out var decodeHeight)) + { + return null; + } + stream.Seek(0); + var bitmap = new BitmapImage + { + DecodePixelType = DecodePixelType.Physical, + DecodePixelWidth = decodeWidth, + DecodePixelHeight = decodeHeight, + }; + await bitmap.SetSourceAsync(stream).AsTask(cancellationToken); + return bitmap; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return null; + } + catch (Exception ex) + { + Logger.Warn($"Assistant image decode failed ({ex.GetType().Name})."); + return null; + } + } +} + +internal sealed record ChatAssistantImageLoadState( + AssistantMediaResolutionStatus Status, + BitmapImage? Bitmap, + string SessionKey, + ChatMediaContentInfo Reference); diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs new file mode 100644 index 000000000..128453711 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs @@ -0,0 +1,46 @@ +namespace OpenClawTray.Chat; + +internal sealed record ChatPendingEchoCandidate( + string MessageId, + string Text, + string AttachmentCorrelationSignature); + +internal static class ChatAttachmentEchoCorrelation +{ + internal static string? SelectMatchingMessageId( + IReadOnlyList candidates, + GatewayMediaMessageProjectionResult incoming) => + SelectMatchingMessageId( + candidates, + incoming.ReconciliationText, + incoming.AttachmentCorrelationSignature, + incoming.HasMediaEnvelope); + + internal static string? SelectMatchingMessageId( + IReadOnlyList candidates, + string text, + string attachmentCorrelationSignature, + bool hasMediaEnvelope) + { + var matching = candidates.Where(candidate => + string.Equals(candidate.Text, text, StringComparison.Ordinal) && + (!hasMediaEnvelope || + string.Equals( + candidate.AttachmentCorrelationSignature, + attachmentCorrelationSignature, + StringComparison.Ordinal))) + .ToArray(); + + if (matching.Length == 0) + return null; + if (hasMediaEnvelope && matching.Length != 1) + return null; + if (!hasMediaEnvelope && + matching.Any(candidate => candidate.AttachmentCorrelationSignature.Length > 0)) + { + return null; + } + + return matching[0].MessageId; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs new file mode 100644 index 000000000..dad092884 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs @@ -0,0 +1,36 @@ +namespace OpenClawTray.Chat; + +public enum ChatAttachmentOrigin +{ + Local, + GatewayReference, +} + +/// +/// Safe, renderer-facing attachment metadata. Preview cache access is allowed +/// only for local attachments that carry an opaque cache key. +/// +public sealed record ChatAttachmentPresentation( + ChatAttachmentOrigin Origin, + string DisplayFileName, + string MimeType, + bool IsImage, + string? PreviewCacheKey = null) +{ + public bool CanAccessPreviewCache => + Origin == ChatAttachmentOrigin.Local && + !string.IsNullOrWhiteSpace(PreviewCacheKey); +} + +internal static class ChatAttachmentPreviewResolver +{ + internal static bool TryGetBytes( + ChatAttachmentPresentation attachment, + IReadOnlyDictionary previewCache, + out byte[] bytes) + { + bytes = Array.Empty(); + return attachment.CanAccessPreviewCache && + previewCache.TryGetValue(attachment.PreviewCacheKey!, out bytes!); + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs index d97a26640..de6aa5147 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs @@ -548,7 +548,10 @@ internal ChatQueuedAdmission AdmitMessage( string nonce, IReadOnlyList? attachments, DateTimeOffset createdAt, - ChatProjectionContext context) + ChatProjectionContext context, + string? timelineText = null, + IReadOnlyList? attachmentPresentations = null, + string attachmentCorrelationSignature = "") { lock (_gate) { @@ -566,7 +569,10 @@ internal ChatQueuedAdmission AdmitMessage( text, displayText, nonce, - attachments?.ToArray()); + attachments?.ToArray(), + TimelineText: timelineText ?? displayText, + AttachmentPresentations: attachmentPresentations, + AttachmentCorrelationSignature: attachmentCorrelationSignature); var sendDirectly = CanSendDirectlyLocked(threadId); ChatQueuedSendDispatch? dispatch; @@ -980,12 +986,13 @@ private ChatQueuedSendDispatch StartDirectSendLocked( var entryId = $"e{current.NextId}"; _timelines[threadId] = ChatTimelineReducer.AddLocalUser( current, - request.DisplayText, + request.EffectiveTimelineText, request.LocalNonce); GetOrCreateThreadMetaLocked(threadId)[entryId] = BuildLiveMetaLocked( threadId, isLocalQueuedSend: true, - localQueuedMessageId: request.Id); + localQueuedMessageId: request.Id, + attachments: request.AttachmentPresentations); var dispatch = _queue.StartDirect( request, _history.ResolveSessionId(threadId), @@ -1061,6 +1068,7 @@ private bool PromoteQueuedMessageLocked( string messageId, ChatEntryMetadata? confirmedMeta = null) { + var request = _queue.FindRequest(threadId, messageId); if (!_queue.TryTakeForPromotion(threadId, messageId, out var queued)) return false; @@ -1068,18 +1076,20 @@ private bool PromoteQueuedMessageLocked( var entryId = $"e{current.NextId}"; _timelines[threadId] = ChatTimelineReducer.AddLocalUser( current, - queued.Text, + request?.EffectiveTimelineText ?? queued.Text, queued.LocalNonce); var meta = confirmedMeta is not null && HasGatewayIdentity(confirmedMeta) ? confirmedMeta with { IsLocalQueuedSend = false, LocalQueuedMessageId = messageId, + Attachments = request?.AttachmentPresentations ?? confirmedMeta.Attachments, } : BuildLiveMetaLocked( threadId, isLocalQueuedSend: true, - localQueuedMessageId: messageId); + localQueuedMessageId: messageId, + attachments: request?.AttachmentPresentations); GetOrCreateThreadMetaLocked(threadId)[entryId] = meta; return true; } @@ -1147,7 +1157,9 @@ internal ChatEntryMetadata BuildLiveMetadata( string? localQueuedMessageId = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? attachments = null, + ChatAssistantContentPresentation? assistantContent = null) { lock (_gate) { @@ -1160,7 +1172,9 @@ internal ChatEntryMetadata BuildLiveMetadata( localQueuedMessageId, openClawKind, compactionTokensBefore, - compactionTokensAfter); + compactionTokensAfter, + attachments, + assistantContent); } } @@ -1296,11 +1310,14 @@ internal ChatResetTransition ResetThread( internal ChatIncomingMessageGate GateIncomingChatMessage( ChatMessageInfo message, - ChatProjectionContext context) + ChatProjectionContext context, + GatewayMediaMessageProjectionResult? projection = null) { var threadId = message.SessionKey!; var role = message.Role?.ToLowerInvariant() ?? string.Empty; - var text = message.Text ?? string.Empty; + var text = projection?.ReconciliationText ?? message.Text ?? string.Empty; + var attachmentCorrelationSignature = projection?.AttachmentCorrelationSignature ?? ""; + var hasMediaEnvelope = projection?.HasMediaEnvelope ?? false; lock (_gate) { _lifecycle.TryGetActiveRun( @@ -1311,7 +1328,11 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( role, text, message.Ts, - _queue.HasPendingLocalEchoText(threadId, text), + _queue.HasPendingLocalEchoText( + threadId, + text, + attachmentCorrelationSignature, + hasMediaEnvelope), activeRunId); var openedLifecycle = ApplyBufferedLifecycleOpenLocked( @@ -1362,15 +1383,21 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( internal ChatLocalEchoTransition ConsumeLocalEcho( ChatMessageInfo message, bool removeQueuedMessage, - ChatProjectionContext context) + ChatProjectionContext context, + GatewayMediaMessageProjectionResult? projection = null) { var threadId = message.SessionKey!; - var text = (message.Text ?? string.Empty).Trim(); + var text = projection?.ReconciliationText ?? + (message.Text ?? string.Empty).Trim(); + var attachmentCorrelationSignature = projection?.AttachmentCorrelationSignature ?? ""; + var hasMediaEnvelope = projection?.HasMediaEnvelope ?? false; lock (_gate) { if (!_queue.TryConsumeLocalEcho( threadId, text, + attachmentCorrelationSignature, + hasMediaEnvelope, out var queuedMessageId)) { return new(false, null); @@ -1397,25 +1424,184 @@ internal ChatLocalEchoTransition ConsumeLocalEcho( internal ChatLocalEchoTransition ReconcileExistingLocalQueuedUser( ChatMessageInfo message, string userText, - ChatProjectionContext context) + ChatProjectionContext context, + IReadOnlyList? attachments = null, + string attachmentCorrelationSignature = "", + bool hasMediaEnvelope = false) { + var threadId = message.SessionKey!; lock (_gate) { var metadata = BuildLiveMetaLocked( - message.SessionKey!, + threadId, message.Ts, message.OpenClawId, - message.OpenClawSeq); - var reconciled = TryReconcileExistingLocalQueuedUserEchoLocked( - message.SessionKey!, + message.OpenClawSeq, + attachments: attachments); + if (TryReconcileExistingLocalQueuedUserEchoLocked( + threadId, + userText, + attachmentCorrelationSignature, + hasMediaEnvelope, + metadata)) + { + return new(true, BuildSnapshotLocked(context)); + } + + var remoteSnapshot = ApplyProjectedRemoteUserMessageLocked( + threadId, userText, - metadata); - return new( - reconciled, - reconciled ? BuildSnapshotLocked(context) : null); + attachmentCorrelationSignature, + metadata, + context); + return new(false, remoteSnapshot); } } + // Applies an incoming user message from another client (not a local + // echo). Gateway retransmits of the same message (e.g. once with a + // partial media resolve, once final) are merged into the existing + // timeline row instead of appended as a duplicate, matched first by + // gateway identity and — for identity-less rows — by same trailing-entry + // text + attachment signature. + private ChatDataSnapshot? ApplyProjectedRemoteUserMessageLocked( + string threadId, + string projectedText, + string attachmentCorrelationSignature, + ChatEntryMetadata incomingMeta, + ChatProjectionContext context) + { + var timeline = GetOrCreateTimelineLocked(threadId); + var threadMeta = GetOrCreateThreadMetaLocked(threadId); + ChatTimelineItem? matched = null; + ChatEntryMetadata? existingMeta = null; + + if (HasGatewayIdentity(incomingMeta)) + { + for (var i = timeline.Entries.Count - 1; i >= 0; i--) + { + var candidate = timeline.Entries[i]; + if (candidate.Kind != ChatTimelineItemKind.User || + !threadMeta.TryGetValue(candidate.Id, out var candidateMeta) || + candidateMeta.IsLocalQueuedSend || + !HasMatchingGatewayIdentity(candidateMeta, incomingMeta)) + { + continue; + } + + matched = candidate; + existingMeta = candidateMeta; + break; + } + } + + // Identity-less history/live twins are only safe to correlate + // against the current trailing user row. Crossing an + // assistant/status boundary would collapse a legitimate later turn + // that repeats the same prose. + if (matched is null && + timeline.Entries.Count > 0 && + timeline.Entries[^1] is { Kind: ChatTimelineItemKind.User } latestUser && + string.Equals(latestUser.Text, projectedText, StringComparison.Ordinal) && + threadMeta.TryGetValue(latestUser.Id, out var latestMeta) && + !latestMeta.IsLocalQueuedSend && + !HasConflictingGatewayIdentity(latestMeta, incomingMeta) && + string.Equals( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + latestMeta.Attachments), + attachmentCorrelationSignature, + StringComparison.Ordinal)) + { + matched = latestUser; + existingMeta = latestMeta; + } + + if (matched is null || existingMeta is null) + { + ApplyEventLocked( + threadId, + new ChatUserMessageEvent(projectedText), + incomingMeta); + return BuildSnapshotLocked(context); + } + + var mergedMeta = MergeProjectedUserMetadata(existingMeta, incomingMeta); + if (mergedMeta == existingMeta) + return null; + + threadMeta[matched.Id] = mergedMeta; + return HasRendererVisibleUserMetadataChange(existingMeta, mergedMeta) + ? BuildSnapshotLocked(context) + : null; + } + + private static bool HasMatchingGatewayIdentity( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) => + (!string.IsNullOrEmpty(incoming.GatewayMessageId) && + string.Equals( + existing.GatewayMessageId, + incoming.GatewayMessageId, + StringComparison.Ordinal)) || + (incoming.OpenClawSeq is not null && existing.OpenClawSeq == incoming.OpenClawSeq); + + private static bool HasConflictingGatewayIdentity( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) => + (!string.IsNullOrEmpty(existing.GatewayMessageId) && + !string.IsNullOrEmpty(incoming.GatewayMessageId) && + !string.Equals( + existing.GatewayMessageId, + incoming.GatewayMessageId, + StringComparison.Ordinal)) || + (existing.OpenClawSeq is not null && + incoming.OpenClawSeq is not null && + existing.OpenClawSeq != incoming.OpenClawSeq); + + private static ChatEntryMetadata MergeProjectedUserMetadata( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) + { + var mergedAttachments = existing.Attachments is { Count: > 0 } + ? existing.Attachments + : incoming.Attachments; + return existing with + { + Timestamp = existing.Timestamp ?? incoming.Timestamp, + Model = existing.Model ?? incoming.Model, + GatewayMessageId = string.IsNullOrEmpty(existing.GatewayMessageId) + ? incoming.GatewayMessageId + : existing.GatewayMessageId, + OpenClawSeq = existing.OpenClawSeq ?? incoming.OpenClawSeq, + Attachments = mergedAttachments, + }; + } + + private static bool HasRendererVisibleUserMetadataChange( + ChatEntryMetadata existing, + ChatEntryMetadata merged) => + existing.Timestamp != merged.Timestamp || + !AttachmentPresentationsEqual(existing.Attachments, merged.Attachments); + + private static bool AttachmentPresentationsEqual( + IReadOnlyList? left, + IReadOnlyList? right) + { + var leftCount = left?.Count ?? 0; + var rightCount = right?.Count ?? 0; + if (leftCount != rightCount) + return false; + if (leftCount == 0) + return true; + + for (var i = 0; i < leftCount; i++) + { + if (left![i] != right![i]) + return false; + } + return true; + } + internal (ChatEntryMetadata Metadata, string? ActiveRunId) BuildMetadataWithRun( ChatMessageInfo message) { @@ -1434,16 +1620,29 @@ internal ChatLocalEchoTransition ReconcileExistingLocalQueuedUser( internal ChatAssistantPreparation PrepareAssistant( ChatMessageInfo message, string assistantText, - ChatProjectionContext context) + ChatProjectionContext context, + ChatAssistantContentPresentation? assistantContent = null) { var threadId = message.SessionKey!; lock (_gate) { - var disposition = ClassifyAssistantQueueFrameLocked( - threadId, - assistantText, - message.OpenClawId, - message.OpenClawSeq); + // A frame carrying only structured/legacy media directives (no + // plain text) has nothing for the identified-duplicate/self-echo + // classifier to compare against, and it never carries a gateway + // identity either (media-only frames are synthesized locally + // from ContentParts, not gateway-sequenced) — so it can't be a + // resend of an already-rendered turn. Render it directly rather + // than routing it through text-based classification. + var disposition = assistantText.Length == 0 && + assistantContent is not null && + string.IsNullOrEmpty(message.OpenClawId) && + message.OpenClawSeq is null + ? AssistantQueueFrameDisposition.Render + : ClassifyAssistantQueueFrameLocked( + threadId, + assistantText, + message.OpenClawId, + message.OpenClawSeq); ChatDataSnapshot? promotionSnapshot = null; if (disposition == AssistantQueueFrameDisposition.Render && _queue.IsLocallyInitiated(threadId) && @@ -1459,7 +1658,8 @@ internal ChatAssistantPreparation PrepareAssistant( threadId, message.Ts, message.OpenClawId, - message.OpenClawSeq); + message.OpenClawSeq, + assistantContent: assistantContent); var hasUsage = message.InputTokens is not null || message.OutputTokens is not null || message.ResponseTokens is not null || @@ -1868,6 +2068,7 @@ internal void CompleteRemoteBackfill(string threadId) bool openResetGate, ChatProjectionContext context) { + var projection = GatewayMediaMessageProjection.Project(message.Text); lock (_gate) { if (GetResetVersionLocked(threadId) != expectedResetGeneration || @@ -1881,7 +2082,7 @@ internal void CompleteRemoteBackfill(string threadId) { if (timeline.Entries[i].Kind != ChatTimelineItemKind.User) continue; - if (timeline.Entries[i].Text == message.Text) + if (timeline.Entries[i].Text == projection.ReconciliationText) return null; break; } @@ -1895,12 +2096,14 @@ internal void CompleteRemoteBackfill(string threadId) ApplyEventLocked( threadId, new ChatUserMessageEvent( - ChatContentFormatting.TruncateForChatEntry(message.Text)), + ChatContentFormatting.TruncateForChatEntry( + projection.ReconciliationText)), BuildLiveMetaLocked( threadId, message.Ts, message.OpenClawId, - message.OpenClawSeq)); + message.OpenClawSeq, + attachments: projection.Attachments)); return new( BuildSnapshotLocked(context), openedLifecycle, @@ -2079,6 +2282,8 @@ metadata.OutputTokens is { } output private bool TryReconcileExistingLocalQueuedUserEchoLocked( string threadId, string text, + string attachmentCorrelationSignature, + bool hasMediaEnvelope, ChatEntryMetadata confirmed) { if (!HasGatewayIdentity(confirmed) || @@ -2087,25 +2292,45 @@ private bool TryReconcileExistingLocalQueuedUserEchoLocked( { return false; } + + var candidates = new List(); + var echoCandidates = new List(); for (var i = timeline.Entries.Count - 1; i >= 0; i--) { var entry = timeline.Entries[i]; if (entry.Kind != ChatTimelineItemKind.User || - !string.Equals(entry.Text, text, StringComparison.Ordinal) || !metadata.TryGetValue(entry.Id, out var existing) || !existing.IsLocalQueuedSend || !IsFreshLocalQueuedPromotion(existing, confirmed)) { continue; } - metadata[entry.Id] = confirmed with - { - IsLocalQueuedSend = false, - LocalQueuedMessageId = existing.LocalQueuedMessageId, - }; - return true; - } - return false; + candidates.Add(entry); + echoCandidates.Add(new ChatPendingEchoCandidate( + entry.Id, + entry.Text, + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + existing.Attachments))); + } + + var matchedMessageId = ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + echoCandidates, + text, + attachmentCorrelationSignature, + hasMediaEnvelope); + if (matchedMessageId is null) + return false; + + var matched = candidates.First(candidate => + string.Equals(candidate.Id, matchedMessageId, StringComparison.Ordinal)); + var matchedMeta = metadata[matched.Id]; + metadata[matched.Id] = confirmed with + { + IsLocalQueuedSend = false, + LocalQueuedMessageId = matchedMeta.LocalQueuedMessageId, + Attachments = matchedMeta.Attachments, + }; + return true; } private static bool IsFreshLocalQueuedPromotion( @@ -2143,6 +2368,7 @@ private bool ReconcileQueuedMessageEchoLocked( { IsLocalQueuedSend = false, LocalQueuedMessageId = messageId, + Attachments = match.Value.Attachments, }; return true; } @@ -2317,6 +2543,41 @@ private void ApplyEventLocked( if (!beforeIds.Contains(entry.Id) && !threadMetadata.ContainsKey(entry.Id)) threadMetadata[entry.Id] = metadata; } + + // Streaming assistant frames reconcile into the SAME entry id + // (see ChatTimelineReducer.UpsertAssistant), so the new-entry-only + // assignment above never touches it again after creation. Assistant + // structured media content can still refine across frames (e.g. a + // legacy directive resolved to a structured reference on a later + // frame), so merge it into the already-existing reconciled entry's + // metadata explicitly. + if (metadata.AssistantContent is not null) + { + for (var i = next.Entries.Count - 1; i >= 0; i--) + { + var entry = next.Entries[i]; + if (entry.Kind == ChatTimelineItemKind.User) + break; + if (entry.Kind != ChatTimelineItemKind.Assistant) + continue; + + if (beforeIds.Contains(entry.Id) && + threadMetadata.TryGetValue(entry.Id, out var existingEntryMeta)) + { + var mergedContent = ChatAssistantContentProjector.MergeLiveUpdate( + existingEntryMeta.AssistantContent, + metadata.AssistantContent); + if (!ReferenceEquals(mergedContent, existingEntryMeta.AssistantContent)) + { + threadMetadata[entry.Id] = existingEntryMeta with + { + AssistantContent = mergedContent, + }; + } + } + break; + } + } } private Dictionary GetOrCreateThreadMetaLocked( @@ -2339,7 +2600,9 @@ private ChatEntryMetadata BuildLiveMetaLocked( string? localQueuedMessageId = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? attachments = null, + ChatAssistantContentPresentation? assistantContent = null) { var timestamp = tsMs is { } value && value > 0 ? DateTimeOffset.FromUnixTimeMilliseconds(value).ToLocalTime() @@ -2353,7 +2616,9 @@ private ChatEntryMetadata BuildLiveMetaLocked( CompactionTokensBefore: compactionTokensBefore, CompactionTokensAfter: compactionTokensAfter, IsLocalQueuedSend: isLocalQueuedSend, - LocalQueuedMessageId: localQueuedMessageId); + LocalQueuedMessageId: localQueuedMessageId, + Attachments: attachments, + AssistantContent: assistantContent); } private ChatOpenedLifecycleTransition? AddResetAcceptedRunIdLocked( diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs index 0e0862520..eeabe0fcb 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs @@ -60,6 +60,14 @@ namespace OpenClawTray.Chat; /// Stable client-side id for a local send. Used to attach a later gateway /// identity to the exact optimistic transcript row without text matching. /// +/// +/// Structured attachment presentation metadata. Gateway references never carry +/// preview cache keys. +/// +/// +/// Renderer-safe assistant media presentation. Transport references remain +/// opaque and are never encoded into timeline text. +/// public sealed record ChatEntryMetadata( DateTimeOffset? Timestamp, string? Model, @@ -75,4 +83,6 @@ public sealed record ChatEntryMetadata( long? CompactionTokensBefore = null, long? CompactionTokensAfter = null, bool IsLocalQueuedSend = false, - string? LocalQueuedMessageId = null); + string? LocalQueuedMessageId = null, + IReadOnlyList? Attachments = null, + ChatAssistantContentPresentation? AssistantContent = null); diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs index b42431b6d..dc8a7fdf6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs @@ -8,6 +8,7 @@ internal sealed record ChatHistoryReplayPart( ChatMessageInfo Message, string Text, IReadOnlyList ToolContent, + IReadOnlyList AssistantContentParts, bool IsFirstPart); internal static class ChatHistoryReplayProjection @@ -41,6 +42,7 @@ public static IEnumerable Project( message, message.Text ?? string.Empty, message.ToolContent, + Array.Empty(), IsFirstPart: true); continue; } @@ -54,6 +56,7 @@ public static IEnumerable Project( message, part.Text ?? string.Empty, Array.Empty(), + new[] { part }, isFirstPart); isFirstPart = false; } @@ -63,6 +66,17 @@ public static IEnumerable Project( message, string.Empty, new[] { tool }, + Array.Empty(), + isFirstPart); + isFirstPart = false; + } + else if (part.Kind == ChatMessageContentPartKind.Media && part.Media is not null) + { + yield return new ChatHistoryReplayPart( + message, + string.Empty, + Array.Empty(), + new[] { part }, isFirstPart); isFirstPart = false; } diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs index 33a9ef283..5e942fcec 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs @@ -7,7 +7,8 @@ namespace OpenClawTray.Chat; internal readonly record struct ChatLocalSentText( string Text, DateTimeOffset SentAt, - string QueuedMessageId); + string QueuedMessageId, + string AttachmentCorrelationSignature = ""); internal sealed record ChatQueueRetryResult( bool Requeued, @@ -112,7 +113,11 @@ internal ChatQueuedSendDispatch StartDirect( long resetLifecycleSequence, long lifecycleStartSequence) { - EnqueueLocalEcho(request.ThreadId, request.Text, request.Id); + EnqueueLocalEcho( + request.ThreadId, + request.EffectiveTimelineText, + request.AttachmentCorrelationSignature, + request.Id); _locallyInitiatedThreads.Add(request.ThreadId); _assistantFallbackPromotedThreads.Add(request.ThreadId); return new ChatQueuedSendDispatch( @@ -178,7 +183,11 @@ internal ChatQueuedSendDispatch StartDirect( }; if (request.LifecycleCommand is null) { - EnqueueLocalEcho(threadId, request.Text, request.Id); + EnqueueLocalEcho( + threadId, + request.EffectiveTimelineText, + request.AttachmentCorrelationSignature, + request.Id); _locallyInitiatedThreads.Add(threadId); } @@ -436,15 +445,35 @@ internal ChatLocalSentText[] SnapshotLocalEchoes(string threadId) => ? queue.ToArray() : []; - internal bool HasPendingLocalEchoText(string threadId, string text) => - !string.IsNullOrWhiteSpace(text) && - _localSentTexts.TryGetValue(threadId, out var queue) && - queue.Any(pending => - string.Equals(pending.Text, text.Trim(), StringComparison.Ordinal)); + internal bool HasPendingLocalEchoText( + string threadId, + string text, + string attachmentCorrelationSignature = "", + bool hasMediaEnvelope = false) + { + if (string.IsNullOrWhiteSpace(text) || + !_localSentTexts.TryGetValue(threadId, out var queue)) + { + return false; + } + var candidates = queue + .Select(pending => new ChatPendingEchoCandidate( + pending.QueuedMessageId, + pending.Text, + pending.AttachmentCorrelationSignature)) + .ToArray(); + return ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + candidates, + text.Trim(), + attachmentCorrelationSignature, + hasMediaEnvelope) is not null; + } internal bool TryConsumeLocalEcho( string threadId, string echoText, + string attachmentCorrelationSignature, + bool hasMediaEnvelope, out string queuedMessageId) { queuedMessageId = string.Empty; @@ -460,24 +489,50 @@ internal bool TryConsumeLocalEcho( return false; } - var retained = new Queue(); - var matched = false; - while (queue.Count > 0) + var pending = queue.ToArray(); + var candidates = pending + .Select(candidate => new ChatPendingEchoCandidate( + candidate.QueuedMessageId, + candidate.Text, + candidate.AttachmentCorrelationSignature)) + .ToArray(); + var matchedMessageId = ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + candidates, + echoText, + attachmentCorrelationSignature, + hasMediaEnvelope); + if (matchedMessageId is null) + return false; + + var retained = new Queue(pending.Length); + foreach (var candidate in pending) { - var candidate = queue.Dequeue(); - if (!matched && - string.Equals(candidate.Text, echoText, StringComparison.Ordinal)) + if (!string.Equals( + candidate.QueuedMessageId, + matchedMessageId, + StringComparison.Ordinal)) { - queuedMessageId = candidate.QueuedMessageId; - matched = true; - continue; + retained.Enqueue(candidate); } - retained.Enqueue(candidate); } + queuedMessageId = matchedMessageId; StoreLocalEchoQueue(threadId, retained); - return matched; + return true; } + // Plain-text overload retained for call sites (e.g. reset-gate dropped + // messages) that never carry a media envelope. + internal bool TryConsumeLocalEcho( + string threadId, + string echoText, + out string queuedMessageId) => + TryConsumeLocalEcho( + threadId, + echoText, + attachmentCorrelationSignature: "", + hasMediaEnvelope: false, + out queuedMessageId); + internal void RemovePendingLocalEcho(string threadId, string messageId) { if (!_localSentTexts.TryGetValue(threadId, out var queue)) @@ -522,6 +577,7 @@ internal void ClearThreadForReset(string threadId) private void EnqueueLocalEcho( string threadId, string text, + string attachmentCorrelationSignature, string messageId) { RemovePendingLocalEcho(threadId, messageId); @@ -533,7 +589,8 @@ private void EnqueueLocalEcho( queue.Enqueue(new ChatLocalSentText( text, DateTimeOffset.UtcNow, - messageId)); + messageId, + attachmentCorrelationSignature)); while (queue.Count > MaxLocalEchoes) queue.Dequeue(); } diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs index 7f141881a..ebc5b0dc6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs @@ -13,7 +13,18 @@ internal sealed record ChatQueuedSendRequest( IReadOnlyList? Attachments, int DeferredAdmissionRetryCount = 0, DateTimeOffset? DeferredAdmissionRetryAfter = null, - ChatLifecycleCommandKind? LifecycleCommand = null); + ChatLifecycleCommandKind? LifecycleCommand = null, + string? TimelineText = null, + IReadOnlyList? AttachmentPresentations = null, + string AttachmentCorrelationSignature = "") +{ + // The final timeline entry text never carries the cosmetic attachment + // chip lines that DisplayText uses for the queued-message preview list + // — structured Attachments render those instead. Falls back to + // DisplayText for legacy callers (e.g. lifecycle commands) that never + // set TimelineText explicitly. + public string EffectiveTimelineText => TimelineText ?? DisplayText; +} internal sealed record ChatQueuedSendDispatch( ChatQueuedSendRequest Request, diff --git a/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs b/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs new file mode 100644 index 000000000..7af2056a8 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs @@ -0,0 +1,292 @@ +using System.Globalization; +using System.Text; +using OpenClaw.Shared; + +namespace OpenClawTray.Chat; + +public sealed record GatewayMediaMessageProjectionResult( + string ReconciliationText, + string ResidualText, + IReadOnlyList Attachments, + bool HasMediaEnvelope) +{ + public string AttachmentPresentationSignature => + GatewayMediaMessageProjection.BuildAttachmentPresentationSignature(Attachments); + + public string AttachmentCorrelationSignature => + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(Attachments); +} + +/// +/// Projects trusted gateway media envelopes into inert presentation metadata. +/// It never emits the tray's private zero-width attachment marker syntax. +/// +public static class GatewayMediaMessageProjection +{ + private const string EnvelopePrefix = "[media attached: "; + private const string MediaPrefix = "media://inbound/"; + private const int MaxDisplayFileNameLength = 160; + + public static GatewayMediaMessageProjectionResult Project(string? text) + { + var source = text ?? string.Empty; + var attachments = new List(); + var offset = 0; + + while (offset < source.Length) + { + var lineEnd = source.IndexOf('\n', offset); + var contentEnd = lineEnd >= 0 ? lineEnd : source.Length; + var line = source.AsSpan(offset, contentEnd - offset); + if (line.EndsWith("\r", StringComparison.Ordinal)) + line = line[..^1]; + + if (!TryParseEnvelopeLine(line, out var attachment)) + break; + + attachments.Add(attachment); + offset = lineEnd >= 0 ? lineEnd + 1 : source.Length; + } + + if (attachments.Count == 0) + { + return new GatewayMediaMessageProjectionResult( + source.Trim(), + source, + Array.Empty(), + HasMediaEnvelope: false); + } + + var residual = source[offset..]; + return new GatewayMediaMessageProjectionResult( + residual.Trim(), + residual, + attachments.ToArray(), + HasMediaEnvelope: true); + } + + public static IReadOnlyList CreateLocalPresentations( + IReadOnlyList? attachments, + Func previewKeyFactory) + { + if (attachments is null || attachments.Count == 0) + return Array.Empty(); + + var presentations = new List(attachments.Count); + foreach (var attachment in attachments) + { + var displayName = NormalizeDisplayFileName(attachment.FileName); + if (displayName.Length == 0) + displayName = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase) + ? "image" + : "attachment"; + + var mimeType = NormalizeMimeType(attachment.MimeType); + var isImage = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase) || + mimeType.StartsWith("image/", StringComparison.Ordinal); + presentations.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + displayName, + mimeType, + isImage, + isImage ? previewKeyFactory() : null)); + } + + return presentations.ToArray(); + } + + public static string BuildAttachmentPresentationSignature( + IEnumerable? attachments) + { + if (attachments is null) + return string.Empty; + + var items = attachments.ToArray(); + if (items.Length == 0) + return string.Empty; + + var builder = new StringBuilder(); + builder.Append(items.Length.ToString(CultureInfo.InvariantCulture)).Append('|'); + foreach (var attachment in items) + { + AppendSignaturePart(builder, NormalizeMimeType(attachment.MimeType)); + AppendSignaturePart(builder, NormalizeDisplayFileName(attachment.DisplayFileName)); + } + return builder.ToString(); + } + + public static string BuildAttachmentCorrelationSignature( + IEnumerable? attachments) + { + if (attachments is null) + return string.Empty; + + var items = attachments.ToArray(); + if (items.Length == 0) + return string.Empty; + + var builder = new StringBuilder(); + builder.Append(items.Length.ToString(CultureInfo.InvariantCulture)).Append('|'); + foreach (var attachment in items) + { + AppendSignaturePart(builder, NormalizeMimeType(attachment.MimeType)); + AppendSignaturePart(builder, attachment.IsImage ? "image" : "file"); + } + return builder.ToString(); + } + + public static string NormalizeDisplayFileName(string? value) + { + if (string.IsNullOrEmpty(value)) + return string.Empty; + + var normalizedSeparators = value.Replace('\\', '/'); + var leaf = normalizedSeparators[(normalizedSeparators.LastIndexOf('/') + 1)..]; + var builder = new StringBuilder(Math.Min(leaf.Length, MaxDisplayFileNameLength)); + var elements = StringInfo.GetTextElementEnumerator(leaf); + while (elements.MoveNext() && builder.Length < MaxDisplayFileNameLength) + { + var element = elements.GetTextElement(); + if (!IsPrintableSingleLine(element)) + continue; + if (builder.Length + element.Length > MaxDisplayFileNameLength) + break; + builder.Append(element); + } + + return builder.ToString().Trim(); + } + + public static string NormalizeMimeType(string? value) + { + var normalized = value?.Trim().ToLowerInvariant() ?? string.Empty; + return IsValidMimeType(normalized) ? normalized : "application/octet-stream"; + } + + /// + /// Normalizes text for echo-correlation comparisons: trims, strips the + /// tray's private attachment marker syntax, and caps length identically + /// to how the reducer bounds rendered entries so local/remote copies of + /// the same message always compare equal. + /// + public static string NormalizeEchoCorrelationText(string? text) => + ChatContentFormatting.TruncateForChatEntry( + ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(text?.Trim())); + + private static bool TryParseEnvelopeLine( + ReadOnlySpan line, + out ChatAttachmentPresentation attachment) + { + attachment = null!; + if (!line.StartsWith(EnvelopePrefix, StringComparison.Ordinal) || + !line.EndsWith(")]", StringComparison.Ordinal)) + { + return false; + } + + var body = line[EnvelopePrefix.Length..^1]; + var annotationStart = body.LastIndexOf(" (", StringComparison.Ordinal); + if (annotationStart <= 0) + return false; + + var mediaUri = body[..annotationStart].ToString(); + var mimeType = body[(annotationStart + 2)..^1].ToString(); + if (!mediaUri.StartsWith(MediaPrefix, StringComparison.Ordinal) || + !IsValidMimeType(mimeType) || + mediaUri.IndexOfAny(['?', '#', '\r', '\n', '\t', ' ']) >= 0) + { + return false; + } + + var encodedPath = mediaUri[MediaPrefix.Length..]; + var encodedLeaf = encodedPath[(encodedPath.LastIndexOf('/') + 1)..]; + if (encodedLeaf.Length == 0) + return false; + + string decodedLeaf; + try + { + decodedLeaf = Uri.UnescapeDataString(encodedLeaf); + } + catch (UriFormatException) + { + return false; + } + + var displayName = RemoveCanonicalStorageSuffix(NormalizeDisplayFileName(decodedLeaf)); + if (displayName.Length == 0) + return false; + + var normalizedMimeType = mimeType.ToLowerInvariant(); + attachment = new ChatAttachmentPresentation( + ChatAttachmentOrigin.GatewayReference, + displayName, + normalizedMimeType, + normalizedMimeType.StartsWith("image/", StringComparison.Ordinal), + PreviewCacheKey: null); + return true; + } + + private static bool IsValidMimeType(string value) + { + var slash = value.IndexOf('/'); + return slash > 0 && + slash == value.LastIndexOf('/') && + slash < value.Length - 1 && + IsMimeToken(value.AsSpan(0, slash)) && + IsMimeToken(value.AsSpan(slash + 1)); + } + + private static bool IsMimeToken(ReadOnlySpan value) + { + foreach (var ch in value) + { + if (ch is >= 'a' and <= 'z' or >= 'A' and <= 'Z' or >= '0' and <= '9') + continue; + if ("!#$%&'*+-.^_`|~".IndexOf(ch, StringComparison.Ordinal) >= 0) + continue; + return false; + } + return value.Length > 0; + } + + private static bool IsPrintableSingleLine(string value) + { + foreach (var ch in value) + { + var category = CharUnicodeInfo.GetUnicodeCategory(ch); + if (char.IsControl(ch) || + category is UnicodeCategory.Control or UnicodeCategory.Format or + UnicodeCategory.LineSeparator or UnicodeCategory.ParagraphSeparator) + { + return false; + } + } + return true; + } + + private static string RemoveCanonicalStorageSuffix(string fileName) + { + var extensionStart = fileName.LastIndexOf('.'); + var stem = extensionStart > 0 ? fileName[..extensionStart] : fileName; + var extension = extensionStart > 0 ? fileName[extensionStart..] : string.Empty; + if (stem.Length < 39) + return fileName; + + var suffix = stem[^39..]; + if (!suffix.StartsWith("---", StringComparison.Ordinal) || + !Guid.TryParseExact(suffix[3..], "D", out _)) + { + return fileName; + } + + var cleanedStem = stem[..^39]; + return cleanedStem.Length == 0 ? fileName : cleanedStem + extension; + } + + private static void AppendSignaturePart(StringBuilder builder, string value) => + builder.Append(value.Length.ToString(CultureInfo.InvariantCulture)) + .Append(':') + .Append(value) + .Append('|'); +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs b/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs index 5788ce32a..f17d338a6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs @@ -80,6 +80,11 @@ Task CompactSessionDetailedAsync(string sessionKey) => Task ClearSessionModelAsync(string sessionKey); Task PatchSessionThinkingLevelAsync(string sessionKey, string thinkingLevel); Task RequestChatHistoryAsync(string? sessionKey); + Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) => + Task.FromResult(AssistantMediaResolutionResult.Unavailable); Task SendChatAbortAsync(string runId, string? sessionKey = null); Task ResolveExecApprovalAsync(string approvalId, string decision); @@ -238,6 +243,12 @@ public Task RequestSessionsAsync() => public Task RequestChatHistoryAsync(string? sessionKey) => _client.RequestChatHistoryAsync(sessionKey); + public Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) => + _client.ResolveAssistantMediaAsync(sessionKey, media, cancellationToken); + public Task SendChatAbortAsync(string runId, string? sessionKey = null) => _client.SendChatAbortAsync(runId, sessionKey); public Task ResolveExecApprovalAsync(string approvalId, string decision) => diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs index 96a7ec835..0277bc72d 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs @@ -72,12 +72,8 @@ public sealed class OpenClawChatDataProvider : IChatDataProvider { internal const int MaxEntryTextBytes = 256 * 1024; /// - /// Process-wide cache mapping an attachment's filename to its raw image - /// bytes. Populated by for image - /// attachments so the timeline can render an actual thumbnail in the - /// user bubble (the display-text marker only carries the filename, not - /// the base64 content). Static so any timeline render after a re-mount - /// can still find the image. + /// Process-wide cache mapping an opaque local preview key to raw image + /// bytes. Gateway references never receive keys and cannot read this cache. /// public static readonly ConcurrentDictionary ImagePreviewCache = new(); @@ -198,18 +194,24 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation var trimmed = message.Trim(); var nonce = Guid.NewGuid().ToString("N"); + var attachmentPresentations = GatewayMediaMessageProjection.CreateLocalPresentations( + attachments, + static () => Guid.NewGuid().ToString("N")); + var attachmentCorrelationSignature = + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(attachmentPresentations); - // Cache image attachments by filename so the timeline can render an - // actual thumbnail preview (the display-text marker only carries the - // filename — see ImagePreviewCache notes). + // Cache image bytes only under collision-resistant keys carried by + // local structured descriptors. if (hasAttachments) { - foreach (var a in attachments!) + for (var i = 0; i < Math.Min(attachments!.Count, attachmentPresentations.Count); i++) { - if (a.Type == "image" && !string.IsNullOrEmpty(a.FileName) && !string.IsNullOrEmpty(a.Content)) + var source = attachments[i]; + var presentation = attachmentPresentations[i]; + if (presentation.CanAccessPreviewCache && !string.IsNullOrEmpty(source.Content)) { - try { ImagePreviewCache[a.FileName] = Convert.FromBase64String(a.Content); } - catch (Exception ex) { Logger.Debug($"ChatDataProvider: image attachment base64 decode failed for '{a.FileName}': {ex.Message}"); } + try { ImagePreviewCache[presentation.PreviewCacheKey!] = Convert.FromBase64String(source.Content); } + catch (Exception ex) { Logger.Debug($"ChatDataProvider: image attachment base64 decode failed for '{presentation.DisplayFileName}': {ex.Message}"); } } } } @@ -219,7 +221,7 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation // blank even if the typed message was empty. Uses a unique prefix // ("\u200B📎 " / "\u200B🖼️ ") with a zero-width space to prevent // false positives from normal user text. - var safeUserText = ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(trimmed); + var safeUserText = GatewayMediaMessageProjection.NormalizeEchoCorrelationText(trimmed); var displayText = safeUserText; if (hasAttachments) { @@ -236,7 +238,10 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation nonce, attachments, DateTimeOffset.UtcNow, - ProjectionContext()); + ProjectionContext(), + timelineText: safeUserText, + attachmentPresentations: attachmentPresentations, + attachmentCorrelationSignature: attachmentCorrelationSignature); _telemetry.StartLocalTurn( admission.MessageId, threadId, @@ -1028,6 +1033,12 @@ public ValueTask DisposeAsync() public IReadOnlyDictionary GetEntryMetadata(string threadId) => _state.GetEntryMetadata(threadId); + internal Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken) => + _bridge.ResolveAssistantMediaAsync(sessionKey, media, cancellationToken); + // ── Event handlers ── private void OnStatusChanged(object? sender, ConnectionStatus status) @@ -1183,7 +1194,10 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var threadId = message.SessionKey; var role = message.Role?.ToLowerInvariant() ?? string.Empty; var rawText = message.Text ?? string.Empty; - var gate = _state.GateIncomingChatMessage(message, ProjectionContext()); + var projection = role == "user" + ? GatewayMediaMessageProjection.Project(rawText) + : null; + var gate = _state.GateIncomingChatMessage(message, ProjectionContext(), projection); HandleOpenedLifecycle( threadId, gate.OpenedLifecycle, @@ -1230,9 +1244,12 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var echo = _state.ConsumeLocalEcho( message, removeQueuedMessage: true, - ProjectionContext()); + ProjectionContext(), + projection); if (echo.Consumed) + { return; + } ApplyEventAndPublish( threadId, new ChatStatusEvent(rawText.Trim(), ChatTone.Dim), @@ -1256,35 +1273,30 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var localEcho = _state.ConsumeLocalEcho( message, removeQueuedMessage: false, - ProjectionContext()); + ProjectionContext(), + projection); if (localEcho.Consumed) { if (localEcho.Snapshot is not null) Publish(localEcho.Snapshot); return; } - if (!string.IsNullOrEmpty(message.Text)) + if (!string.IsNullOrEmpty(message.Text) || + (projection?.Attachments.Count ?? 0) > 0) { var userText = ChatContentFormatting.TruncateForChatEntry( - ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(message.Text)); + projection is { HasMediaEnvelope: true } + ? projection.ReconciliationText + : ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(message.Text)); var reconciled = _state.ReconcileExistingLocalQueuedUser( message, userText, - ProjectionContext()); - if (reconciled.Consumed) - { - if (reconciled.Snapshot is not null) - Publish(reconciled.Snapshot); - return; - } - ApplyEventAndPublish( - threadId, - new ChatUserMessageEvent(userText), - _state.BuildLiveMetadata( - threadId, - message.Ts, - message.OpenClawId, - message.OpenClawSeq)); + ProjectionContext(), + projection?.Attachments, + projection?.AttachmentCorrelationSignature ?? "", + projection?.HasMediaEnvelope ?? false); + if (reconciled.Snapshot is not null) + Publish(reconciled.Snapshot); } return; } @@ -1302,9 +1314,12 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) return; } + var assistantContent = role == "assistant" + ? ChatAssistantContentProjector.Project(message.ContentParts) + : null; if (role != "assistant" || ChatMessageInfo.IsSilentAssistantDirective(role, message.Text) || - string.IsNullOrEmpty(message.Text)) + (string.IsNullOrEmpty(message.Text) && assistantContent is null)) { return; } @@ -1314,13 +1329,17 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var preparation = _state.PrepareAssistant( message, assistantText, - ProjectionContext()); + ProjectionContext(), + assistantContent); if (preparation.PromotionSnapshot is not null) Publish(preparation.PromotionSnapshot); if (preparation.Disposition != AssistantQueueFrameDisposition.Render) return; if (!message.IsFinal && _state.IsLateNonFinalAssistantFrame(threadId)) + { + Logger.Warn($"[ChatProvider] Dropping late non-final assistant frame after completed turn for threadId='{threadId}' len={traceText.Length}"); return; + } _telemetry.ObserveInboundOutput( threadId, @@ -1700,6 +1719,13 @@ private async Task FetchRemoteUserMessageAsync(string threadId, bool openResetGa } } +#if OPENCLAW_TRAY_TESTS + internal Task FetchRemoteUserMessageForTestsAsync( + string threadId, + bool openResetGateOnSuccess) => + FetchRemoteUserMessageAsync(threadId, openResetGateOnSuccess); +#endif + private async Task PersistAbortedMessageIdsAsync( string threadId, long resetGeneration) diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs index 8894c2c66..8e27ae0ee 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs @@ -55,7 +55,9 @@ public record OpenClawChatTimelineProps( Func? OnReadAloud = null, Action? OnStopSpeaking = null, int ScrollToBottomToken = 0, - Action? OnPermissionResponse = null); + Action? OnPermissionResponse = null, + Func>? + ResolveAssistantMediaAsync = null); /// /// OpenClaw-skinned variant of from the vendored @@ -1179,22 +1181,38 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var text = entry.Text ?? ""; var lines = text.Split('\n'); var messageLines = new List(); - var attachmentNames = new List<(string Icon, string Name, bool IsImage)>(); + var attachmentNames = new List(); foreach (var line in lines) { var trimLine = line.Trim(); if (trimLine.StartsWith("\u200B🖼️ ")) - attachmentNames.Add(("🖼️", trimLine.Substring(4).Trim(), true)); + attachmentNames.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + trimLine.Substring(4).Trim(), + "application/octet-stream", + IsImage: true)); else if (trimLine.StartsWith("\u200B📎 ")) - attachmentNames.Add(("📎", trimLine.Substring(3).Trim(), false)); + attachmentNames.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + trimLine.Substring(3).Trim(), + "application/octet-stream", + IsImage: false)); else messageLines.Add(line); } + if (MetaFor(entry.Id)?.Attachments is { Count: > 0 } structuredAttachments) + attachmentNames = structuredAttachments.ToList(); + var messageText = string.Join('\n', messageLines).Trim(); var hasMessage = !string.IsNullOrEmpty(messageText); var hasAttachments = attachmentNames.Count > 0; + var safeUserText = string.Join( + '\n', + (hasMessage ? new[] { messageText } : Array.Empty()) + .Concat(attachmentNames.Select(attachment => + $"{attachment.DisplayFileName} ({attachment.MimeType})"))); // Build attachment elements. Images become real thumbnail previews // by pulling the original bytes from OpenClawChatDataProvider's @@ -1206,9 +1224,14 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var attachmentElements = new List(); if (hasAttachments) { - foreach (var (_, name, isImage) in attachmentNames) + foreach (var attachment in attachmentNames) { - if (isImage && OpenClawChatDataProvider.ImagePreviewCache.TryGetValue(name, out var bytes)) + var name = attachment.DisplayFileName; + var isImage = attachment.IsImage; + if (isImage && ChatAttachmentPreviewResolver.TryGetBytes( + attachment, + OpenClawChatDataProvider.ImagePreviewCache, + out var bytes)) { var bmp = TryDecodeBitmap(bytes); if (bmp is not null) @@ -1359,7 +1382,7 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var timeStr = FormatTime(entryMeta?.Timestamp); var rightInset = showUserAvatar ? (36 + bubbleSideMargin) : 0; rightInset += (int)bubblePadding.Right; - footer = BuildUserFooter(userSender, timeStr, chatStampFg, entry.Id, entry.Text ?? "") + footer = BuildUserFooter(userSender, timeStr, chatStampFg, entry.Id, safeUserText) .Margin(0, 2, rightInset, 0); } @@ -1370,7 +1393,8 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst VStack(2, bubbleRow, footer) .HAlign(HorizontalAlignment.Stretch) ).Background(new SolidColorBrush(Colors.Transparent)) - .Margin(gutter, topMargin, 20, bottomMargin), + .Margin(gutter, topMargin, 20, bottomMargin) + .AutomationName(safeUserText), entry.Id); } diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs index 196ab3b61..86517bfee 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs @@ -233,6 +233,10 @@ public override Element Render() : isEmptyConversation ? ReactorChatTimelineMode.Empty : ReactorChatTimelineMode.Timeline; + Func>? + mediaResolver = props.Provider is OpenClawChatDataProvider dataProvider + ? dataProvider.ResolveAssistantMediaAsync + : null; var timelineProps = new OpenClawChatTimelineProps( effectiveThread?.Id, @@ -255,7 +259,8 @@ public override Element Render() scrollToBottomToken, effectiveThread is { } permissionThread ? (requestId, action) => OnPermission(permissionThread.Id, requestId, action) - : null); + : null, + mediaResolver); void SelectThread(string threadId) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs index 9b2d17ae4..784678396 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs @@ -384,7 +384,12 @@ private static Element BuildUser( bool isHovered, Action setEntryHovered) { - var (messageText, attachments) = ParseAttachments(entry.Text); + var (messageText, legacyAttachments) = ParseAttachments(entry.Text); + var attachments = row.Props.Timeline.EntryMetadata?.TryGetValue(entry.Id, out var metadata) == true && + metadata.Attachments is { Count: > 0 } structuredAttachments + ? structuredAttachments + : legacyAttachments; + var accessibleText = BuildAccessibleUserText(messageText, attachments); var content = attachments.Select(BuildAttachment).ToList(); if (messageText.Length > 0) { @@ -412,12 +417,12 @@ private static Element BuildUser( row.Props.Timeline.ShowToolCalls ? UserMetadata(row, entry, isHovered) : Empty(), - CopyAction(entry.Text, isHovered, setEntryHovered, entry.Id)) + CopyAction(accessibleText, isHovered, setEntryHovered, entry.Id)) .Margin(16, 2, 4, 0) .HAlign(HorizontalAlignment.Right)) .Margin(72, 4, 20, 4) .HAlign(HorizontalAlignment.Stretch) - .AutomationName(entry.Text ?? string.Empty); + .AutomationName(accessibleText); } private static Element BuildAssistant( @@ -428,9 +433,35 @@ private static Element BuildAssistant( Func toggleSpeechAsync, Action setEntryHovered) { - var message = BuildSafeMarkdown(entry.Text); + var content = new List(); + ChatEntryMetadata? metadata = null; + if (!string.IsNullOrWhiteSpace(entry.Text)) + content.Add(BuildSafeMarkdown(entry.Text)); + if (row.Props.Timeline.EntryMetadata?.TryGetValue(entry.Id, out var resolvedMetadata) == true) + metadata = resolvedMetadata; + if (metadata?.AssistantContent is { Media.Count: > 0 } assistantContent) + { + var renderPlan = ChatAssistantContentProjector.BuildRenderPlan(assistantContent.Media); + content.AddRange(renderPlan.Media.Select(media => + ChatAssistantMediaRenderer.Render( + media, + row.Props.Timeline.SessionId, + row.Props.Timeline.ResolveAssistantMediaAsync))); + if (renderPlan.OmittedImages > 0) + { + content.Add(TextBlock(string.Format( + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_ImagesOmitted", + "{0} more images not shown"), + renderPlan.OmittedImages)) + .FontSize(11) + .Foreground(Theme.SecondaryText)); + } + } + if (content.Count == 0) + content.Add(BuildSafeMarkdown(string.Empty)); - var bubble = Border(message) + var bubble = Border(VStack(8, content.ToArray())) .Background(BrushFor( "SubtleFillColorSecondaryBrush", Color.FromArgb(0x24, 0x80, 0x80, 0x80))) @@ -464,7 +495,25 @@ private static Element BuildAssistant( .Grid(column: 1)) .Margin(20, row.IsAssistantRunStart ? 6 : 1, 72, row.IsAssistantRunEnd ? 6 : 1) .HAlign(HorizontalAlignment.Stretch) - .AutomationName(entry.Text ?? string.Empty); + .AutomationName(BuildAccessibleAssistantText(entry.Text, metadata?.AssistantContent)); + } + + private static string BuildAccessibleAssistantText( + string? text, + ChatAssistantContentPresentation? content) + { + var lines = new List(); + if (!string.IsNullOrWhiteSpace(text)) + lines.Add(text); + if (content is not null) + { + var attachmentLabel = ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_MediaAttachment", + "Media attachment"); + lines.AddRange(content.Media.Select(media => + $"{ChatAssistantMediaRenderer.DisplayName(media)}. {attachmentLabel}")); + } + return string.Join('\n', lines); } private static Element BuildAssistantAvatarSlot(ReactorTimelineRow row) @@ -607,12 +656,12 @@ private static Element CompactIconAction( }); } - private static (string Message, IReadOnlyList Attachments) ParseAttachments(string? text) + private static (string Message, IReadOnlyList Attachments) ParseAttachments(string? text) { const string imagePrefix = "\u200B🖼️ "; const string filePrefix = "\u200B📎 "; var messageLines = new List(); - var attachments = new List(); + var attachments = new List(); foreach (var line in (text ?? string.Empty).Split('\n')) { @@ -621,13 +670,21 @@ private static (string Message, IReadOnlyList Attachments { var name = trimmed[imagePrefix.Length..].Trim(); if (name.Length > 0) - attachments.Add(new ChatAttachmentPreview(name, true)); + attachments.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + name, + "application/octet-stream", + IsImage: true)); } else if (trimmed.StartsWith(filePrefix, StringComparison.Ordinal)) { var name = trimmed[filePrefix.Length..].Trim(); if (name.Length > 0) - attachments.Add(new ChatAttachmentPreview(name, false)); + attachments.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + name, + "application/octet-stream", + IsImage: false)); } else { @@ -638,10 +695,25 @@ private static (string Message, IReadOnlyList Attachments return (string.Join('\n', messageLines).Trim(), attachments); } - private static Element BuildAttachment(ChatAttachmentPreview attachment) + private static string BuildAccessibleUserText( + string message, + IReadOnlyList attachments) + { + var lines = new List(); + if (message.Length > 0) + lines.Add(message); + lines.AddRange(attachments.Select(attachment => + $"{attachment.DisplayFileName} ({attachment.MimeType})")); + return string.Join('\n', lines); + } + + private static Element BuildAttachment(ChatAttachmentPresentation attachment) { if (attachment.IsImage - && OpenClawChatDataProvider.ImagePreviewCache.TryGetValue(attachment.Name, out var bytes) + && ChatAttachmentPreviewResolver.TryGetBytes( + attachment, + OpenClawChatDataProvider.ImagePreviewCache, + out var bytes) && TryDecodeAttachmentBitmap(bytes) is { } bitmap) { const double maxWidth = 280; @@ -658,7 +730,7 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .Size(pixelWidth * scale, pixelHeight * scale) .CornerRadius(8) .HAlign(HorizontalAlignment.Right) - .AutomationName(attachment.Name); + .AutomationName(attachment.DisplayFileName); } var glyph = Text( @@ -673,7 +745,7 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .CornerRadius(6) .Background(Theme.Ref("SubtleFillColorSecondaryBrush")); var name = Text( - attachment.Name, + attachment.DisplayFileName, 13, FontWeights.Normal, "TextOnAccentFillColorPrimaryBrush") @@ -685,13 +757,25 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .MaxWidth(240) .VAlign(VerticalAlignment.Center); - return Border(HStack(8, glyphBackground, name)) + var mimeType = Text( + attachment.MimeType, + 11, + FontWeights.Normal, + "TextOnAccentFillColorSecondaryBrush") + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(240); + + return Border(HStack(8, glyphBackground, VStack(1, name, mimeType))) .Padding(8, 6, 12, 6) .CornerRadius(6) .BorderThickness(1) .BorderBrush(Theme.Ref("ControlStrokeColorDefaultBrush")) .Background(Theme.Ref("SubtleFillColorSecondaryBrush")) - .AutomationName(attachment.Name); + .AutomationName($"{attachment.DisplayFileName}, {attachment.MimeType}"); } private static BitmapImage? TryDecodeAttachmentBitmap(byte[] bytes) @@ -1026,7 +1110,6 @@ private static string StripMarkdownForSpeech(string text) private static readonly System.Runtime.CompilerServices.ConditionalWeakTable s_attachmentBitmaps = new(); - private sealed record ChatAttachmentPreview(string Name, bool IsImage); } internal sealed record ReactorTimelineRow( diff --git a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw index ae0e8bb49..55b7a8a74 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw @@ -7003,4 +7003,40 @@ Make sure the gateway is running. Gateway version not supported + + Preparing media + + + Media unavailable + + + Retry + + + Image + + + Audio + + + Video + + + File + + + Media + + + {0} more images not shown + + + Media attachment + + + Open image {0} + + + Close + diff --git a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw index b1cac9614..b605cb141 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw @@ -6963,4 +6963,16 @@ Le binaire wxc-exec est introuvable. {1} S'il s'agit d'une build développeur, c Version de Gateway non prise en charge + Préparation du média + Média indisponible + Réessayer + Illustration + Son + Vidéo + Fichier + Média + {0} images supplémentaires non affichées + Pièce jointe multimédia + Ouvrir l’image {0} + Fermer diff --git a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw index 9b7cb92ab..d4f3d9a7d 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw @@ -6964,4 +6964,16 @@ Het binaire bestand wxc-exec is niet gevonden. {1} Als dit een ontwikkelaarsbuil Gateway-versie wordt niet ondersteund + Media voorbereiden + Media niet beschikbaar + Opnieuw proberen + Afbeelding + Geluid + Videofragment + Bestand + Mediabestand + {0} extra afbeeldingen niet weergegeven + Mediabijlage + Afbeelding {0} openen + Sluiten diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw index 2664efb31..2836babbb 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw @@ -6963,4 +6963,16 @@ 不支持此 Gateway 版本 + 正在准备媒体 + 媒体不可用 + 重试 + 图像 + 音频 + 视频 + 文件 + 媒体 + 还有 {0} 张图像未显示 + 媒体附件 + 打开图像 {0} + 关闭 diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw index 936750db7..a49abc4d9 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw @@ -6963,4 +6963,16 @@ 不支援此 Gateway 版本 + 正在準備媒體 + 媒體無法使用 + 重試 + 圖片 + 音訊 + 影片 + 檔案 + 媒體 + 還有 {0} 張圖片未顯示 + 媒體附件 + 開啟圖片 {0} + 關閉 diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 93d1eb23e..19d30a26c 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -88,6 +88,196 @@ public async Task ConnectAsync_WithCredential_TransitionsToConnecting() Assert.Equal("test", _manager.CurrentSnapshot.OperatorCredentialSource); } + [Fact] + public async Task ConnectAsync_PrefersSharedTokenForInteractiveHttpSurfaces() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-1", + Url = "wss://test", + SharedGatewayToken = "shared-http-token", + }); + _registry.SetActive("gw-1"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + + var created = Assert.Single(_factory.CreatedCredentials); + Assert.Equal("paired-device-token", created.Token); + Assert.Equal("shared-http-token", created.InteractiveHttpToken); + } + + [Fact] + public async Task ConnectAsync_BootstrapOnlyDisablesAssistantMediaHttpAuth() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-1", + Url = "wss://test", + BootstrapToken = "bootstrap-token", + }); + _registry.SetActive("gw-1"); + _resolver.OperatorCredential = new GatewayCredential( + "bootstrap-token", + true, + CredentialResolver.SourceBootstrapToken); + + await _manager.ConnectAsync("gw-1"); + + var created = Assert.Single(_factory.CreatedCredentials); + Assert.Equal("bootstrap-token", created.Token); + Assert.Equal(string.Empty, created.InteractiveHttpToken); + Assert.Null(Assert.Single(_factory.CreatedClients).AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RefreshesHttpTokenAcrossProvenanceChanges() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-http-token", + }); + _registry.SetActive("gw-local"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + var provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789); + using var manager = new GatewayConnectionManager( + _resolver, + _factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult(provenance)); + + await manager.ConnectAsync("gw-local"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("shared-http-token", client.AssistantMediaAuthToken); + + provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + ProcessId: 42, + ProcessName: "unknown"); + var deniedHttp = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(deniedHttp.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + + provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789); + var restoredHttp = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(restoredHttp.Allowed); + Assert.Equal("shared-http-token", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RefreshesFallbackDeviceCredential() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-1", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-1", client.AssistantMediaAuthToken); + + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-2", + false, + CredentialResolver.SourceDeviceToken); + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(reconnect.Allowed); + Assert.Equal("paired-device-token-2", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RejectsSameUrlTrustConfigurationChange() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.AddOrUpdate(_registry.GetById("gw-1")! with + { + IsLocal = true, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.False(reconnect.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_AllowsRuntimeV2SignatureUpgrade() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = true, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(reconnect.Allowed); + Assert.Equal("paired-device-token", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task OperatorDeviceTokenReceived_RefreshesAssistantMediaAuthWithoutReconnect() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-1", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-1", client.AssistantMediaAuthToken); + + client.SimulateDeviceTokenReceived( + "paired-device-token-2", + "operator", + ["operator.read"]); + await WaitUntilAsync( + () => client.AssistantMediaAuthToken == "paired-device-token-2"); + + Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-2", client.AssistantMediaAuthToken); + } + [Fact] public async Task ConnectAndReconnect_EmitCompletedOperatorSpans() { @@ -4968,7 +5158,10 @@ public IGatewayClientLifecycle Create(string gatewayUrl, GatewayCredential crede if (CreateException != null) throw CreateException; - var mock = new MockLifecycle(gatewayUrl, identityPath); + var mock = new MockLifecycle( + gatewayUrl, + identityPath, + credential.InteractiveHttpToken); CreatedClients.Add(mock); CreatedCredentials.Add(credential); CreatedIdentityPaths.Add(identityPath); @@ -5017,12 +5210,19 @@ internal sealed class MockLifecycle : IGatewayClientLifecycle { private readonly MockGatewayClient _client; - public MockLifecycle(string url, string identityPath) + public MockLifecycle( + string url, + string identityPath, + string? assistantMediaAuthToken = null) { - _client = new MockGatewayClient(url, identityPath); + _client = new MockGatewayClient( + url, + identityPath, + assistantMediaAuthToken); } public OpenClawGatewayClient DataClient => _client; + public string? AssistantMediaAuthToken => _client.AssistantMediaAuthToken; public bool IsDisposed { get; private set; } public event EventHandler? StatusChanged; public event EventHandler? AuthenticationFailed; @@ -5066,8 +5266,30 @@ private sealed class MockGatewayClient : OpenClawGatewayClient { private bool _isConnected = true; - public MockGatewayClient(string url, string identityPath) - : base(url, "mock-token", NullLogger.Instance, identityPath: identityPath) { } + public MockGatewayClient( + string url, + string identityPath, + string? assistantMediaAuthToken = null) + : base( + url, + "mock-token", + NullLogger.Instance, + identityPath: identityPath, + assistantMediaAuthToken: assistantMediaAuthToken) { } + + public string? AssistantMediaAuthToken + { + get + { + var fieldInfo = typeof(OpenClawGatewayClient).GetField( + "_assistantMediaAuthToken", + System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "Assistant media auth token field was not found."); + return fieldInfo.GetValue(this) as string; + } + } public override bool IsConnectedToGateway => _isConnected; diff --git a/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs b/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs new file mode 100644 index 000000000..8f4ab3aca --- /dev/null +++ b/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs @@ -0,0 +1,140 @@ +using OpenClaw.Shared; + +namespace OpenClaw.Shared.Tests; + +public sealed class AssistantMediaDirectiveParserTests +{ + [Fact] + public void Project_AssistantAbsolutePath_ProducesMediaWithoutExposingPath() + { + const string raw = + "Here is the image.\nMEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal("Here is the image.", projection.Text); + Assert.DoesNotContain("/home/openclaw", projection.Text, StringComparison.Ordinal); + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentKind.Image, media.Kind); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Source); + Assert.Equal("banner.png", media.FileName); + } + + [Fact] + public void Project_UserDirective_RemainsInertText() + { + const string raw = "MEDIA:/home/openclaw/private.png"; + + var projection = AssistantMediaDirectiveParser.Project("user", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void Project_FencedDirective_RemainsInertText() + { + const string raw = "```text\nMEDIA:/home/openclaw/example.png\n```"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void Project_MidLineDirective_RemainsInertText() + { + const string raw = "Example: MEDIA:/home/openclaw/example.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Theory] + [InlineData("MEDIA:../../.env")] + [InlineData("MEDIA:~someone/private.png")] + [InlineData("MEDIA:file:///home/openclaw/../private.png")] + public void Project_InvalidPathLikeDirective_RedactsSource(string raw) + { + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(string.Empty, projection.Text); + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentSource.Unavailable, media.Source); + Assert.Null(media.GatewaySource); + } + + [Fact] + public void Project_QuotedPathWithSpaces_ProducesSingleMediaReference() + { + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + "MEDIA:\"/home/openclaw/My Images/banner light.png\""); + + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal("banner light.png", media.FileName); + } + + [Fact] + public void Project_MultipleIndependentSources_PreservesOrder() + { + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + "MEDIA:/tmp/one.png /tmp/two.mp4"); + + var media = projection.ContentParts + .Where(part => part.Kind == ChatMessageContentPartKind.Media) + .Select(part => part.Media) + .ToArray(); + Assert.Collection( + media, + item => Assert.Equal("one.png", item?.FileName), + item => Assert.Equal("two.mp4", item?.FileName)); + } + + [Fact] + public void Project_TooManySources_CapsMediaReferencesAndRedactsRemainder() + { + var sources = Enumerable.Range(1, AssistantMediaDirectiveParser.MaxMediaReferences + 5) + .Select(index => $"/tmp/image-{index}.png"); + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + $"MEDIA:{string.Join(' ', sources)}"); + + Assert.Equal(string.Empty, projection.Text); + Assert.Equal( + AssistantMediaDirectiveParser.MaxMediaReferences, + projection.ContentParts.Count(part => part.Kind == ChatMessageContentPartKind.Media)); + } + + [Fact] + public void Project_Ipv4MappedPrivateHttpsSource_RemainsInert() + { + const string text = "MEDIA:https://[::ffff:192.168.1.1]/private.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", text); + + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + Assert.Equal(text, projection.Text); + } +} diff --git a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs new file mode 100644 index 000000000..829041c83 --- /dev/null +++ b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs @@ -0,0 +1,275 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using OpenClaw.TestSupport; + +namespace OpenClaw.Shared.Tests; + +public sealed class OpenClawGatewayClientAssistantMediaTests +{ + [Fact] + public async Task ResolveStructuredMedia_InlineBase64_ReturnsBoundedTypedBytes() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "test-token", + identityPath: identity.Path); + await client.ConnectAsync(); + + var resolution = client.ResolveAssistantMediaAsync( + "main", + new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + }); + var request = await server.ReceiveTextAsync().WaitAsync(TimeSpan.FromSeconds(2)); + using var requestDocument = JsonDocument.Parse(request); + Assert.Equal("artifacts.download", requestDocument.RootElement.GetProperty("method").GetString()); + Assert.Equal( + "main", + requestDocument.RootElement.GetProperty("params").GetProperty("sessionKey").GetString()); + + await server.SendTextAsync(JsonSerializer.Serialize(new + { + type = "res", + id = requestDocument.RootElement.GetProperty("id").GetString(), + ok = true, + payload = new + { + artifact = new + { + id = "artifact-1", + type = "image", + title = "banner.png", + mimeType = "image/png", + sizeBytes = 4, + download = new { mode = "bytes" }, + }, + encoding = "base64", + data = Convert.ToBase64String([1, 2, 3, 4]), + }, + })); + + var result = await resolution.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.Equal("image/png", result.MimeType); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, result.Data); + } + + [Fact] + public async Task ResolveLegacyMedia_UsesBearerMetadataAndSourceBoundTicket() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaAuthToken: "shared-http-token", + assistantMediaHandler: handler); + await client.ConnectAsync(); + + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }; + var result = await client.ResolveAssistantMediaAsync( + "main", + media); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, result.Data); + Assert.Equal(2, handler.Requests.Count); + Assert.All( + handler.Requests, + request => Assert.Equal("shared-http-token", request.AuthorizationParameter)); + Assert.Contains("meta=1", handler.Requests[0].Uri.Query, StringComparison.Ordinal); + Assert.Contains("mediaTicket=ticket-1", handler.Requests[1].Uri.Query, StringComparison.Ordinal); + Assert.Contains( + "source=%2Fhome%2Fopenclaw%2Fprivate%2Fbanner.png", + handler.Requests[1].Uri.Query, + StringComparison.Ordinal); + } + + [Fact] + public async Task ResolveLegacyMedia_WithoutExplicitHttpCredential_DoesNotUseWebSocketToken() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaHandler: handler); + await client.ConnectAsync(); + + var result = await client.ResolveAssistantMediaAsync( + "main", + new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.All(handler.Requests, request => Assert.Null(request.AuthorizationParameter)); + } + + [Fact] + public async Task ResolveLegacyMedia_UsesUpdatedHttpCredential() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png"), + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-2"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaAuthToken: "shared-token-1", + assistantMediaHandler: handler); + await client.ConnectAsync(); + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }; + + await client.ResolveAssistantMediaAsync("main", media); + client.SetAssistantMediaAuthToken("shared-token-2"); + await client.ResolveAssistantMediaAsync("main", media); + + Assert.All( + handler.Requests.Take(2), + request => Assert.Equal("shared-token-1", request.AuthorizationParameter)); + Assert.All( + handler.Requests.Skip(2), + request => Assert.Equal("shared-token-2", request.AuthorizationParameter)); + } + + [Fact] + public void TryDecodeBoundedBase64_RejectsDecodedSizeOverflow() + { + const int maximumBytes = 4; + var encoded = string.Concat(new string(' ', 1024), "AQIDBAUG"); + + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + encoded, + maximumBytes, + out var bytes); + + Assert.False(decoded); + Assert.Empty(bytes); + } + + [Fact] + public void TryDecodeBoundedBase64_RejectsOversizedCompactInputBeforeNormalization() + { + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + " AQIDBAUGB ", + maximumBytes: 4, + out var bytes); + + Assert.False(decoded); + Assert.Empty(bytes); + } + + [Fact] + public void TryDecodeBoundedBase64_DecodesWhitespaceAndUnpaddedBase64Url() + { + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + " AQI-\n_w ", + maximumBytes: 5, + out var bytes); + + Assert.True(decoded); + Assert.Equal(new byte[] { 1, 2, 62, 255 }, bytes); + } + + [Theory] + [InlineData("/api/chat/media/outgoing/item?mediaTicket=ticket", true)] + [InlineData("/api/chat/media/outgoing/item", false)] + [InlineData("/api/chat/media/outgoing/../secret?mediaTicket=ticket", false)] + [InlineData("//other.example/api/chat/media/outgoing/item?mediaTicket=ticket", false)] + [InlineData("https://other.example/api/chat/media/outgoing/item?mediaTicket=ticket", false)] + public void TryResolveManagedMediaUri_EnforcesGatewayRelativeTicketPath( + string path, + bool expected) + { + var baseUri = new Uri("https://gateway.example/base"); + + var resolved = OpenClawGatewayClient.TryResolveManagedMediaUri(baseUri, path, out var uri); + + Assert.Equal(expected, resolved); + if (expected) + Assert.Equal("gateway.example", uri.Host); + } + + private static HttpResponseMessage JsonResponse(string json) => + new(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + + private static HttpResponseMessage BytesResponse(byte[] data, string mimeType) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(data), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue(mimeType); + return response; + } + + private sealed class SequentialMediaHandler(params HttpResponseMessage[] responses) + : HttpMessageHandler + { + private int _nextResponse; + + public List Requests { get; } = new(); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(new CapturedRequest( + request.RequestUri!, + request.Headers.Authorization?.ToString(), + request.Headers.Authorization?.Parameter)); + var index = Interlocked.Increment(ref _nextResponse) - 1; + return Task.FromResult(responses[index]); + } + } + + private sealed record CapturedRequest( + Uri Uri, + string? Authorization, + string? AuthorizationParameter); +} diff --git a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs index b469d332e..0c3b0f45d 100644 --- a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs +++ b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs @@ -2006,6 +2006,177 @@ public void ParseChatHistoryPayload_InterleavedBlocks_PreserveSourceOrder() }); } + [Fact] + public void ParseChatHistoryPayload_StructuredMedia_PreservesTypedFieldsAndOrder() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + { "type": "text", "text": "Created it." }, + { + "type": "image", + "mimeType": "image/png", + "fileName": "banner.png", + "artifactId": "artifact_managed_image_123", + "alt": "OpenClaw banner", + "width": 1200, + "height": 774, + "sizeBytes": 12345 + }, + { "type": "text", "text": "Finished." } + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal("Created it.\nFinished.", message.Text); + Assert.Collection( + message.ContentParts, + part => + { + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Equal("Created it.", part.Text); + }, + part => + { + Assert.Equal(ChatMessageContentPartKind.Media, part.Kind); + Assert.Equal(ChatMediaContentKind.Image, part.Media?.Kind); + Assert.Equal("image/png", part.Media?.MimeType); + Assert.Equal("banner.png", part.Media?.FileName); + Assert.Equal("artifact_managed_image_123", part.Media?.ArtifactId); + Assert.Equal(1200, part.Media?.Width); + Assert.Equal(774, part.Media?.Height); + }, + part => + { + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Equal("Finished.", part.Text); + }); + } + + [Fact] + public void ParseChatHistoryPayload_LegacyMediaOnly_PreservesMessageAndRedactsPath() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png", + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal(string.Empty, message.Text); + var media = Assert.Single(message.ContentParts).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Source); + Assert.Equal("banner.png", media.FileName); + } + + [Fact] + public void ParseChatHistoryPayload_StringArray_RedactsLegacyMediaPath() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + "Created it.", + "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png" + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal("Created it.", message.Text); + Assert.DoesNotContain("/home/openclaw", message.Text, StringComparison.Ordinal); + Assert.Single( + message.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + Assert.Equal( + "Created it.", + Assert.Single( + message.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Text).Text); + } + + [Fact] + public void ParseChatHistoryPayload_SplitFence_KeepsMediaDirectiveAsTextOnly() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + "```", + "MEDIA:/home/openclaw/private.png\n```" + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + var part = Assert.Single(message.ContentParts); + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Contains("MEDIA:/home/openclaw/private.png", part.Text, StringComparison.Ordinal); + Assert.DoesNotContain( + message.ContentParts, + contentPart => contentPart.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void ChatEvent_LegacyMediaOnly_RaisesTypedMessageWithoutRawPath() + { + var helper = new GatewayClientTestHelper(); + ChatMessageInfo? received = null; + helper.Client.ChatMessageReceived += (_, message) => received = message; + + helper.ProcessRawMessage(""" + { + "type": "event", + "event": "chat", + "payload": { + "sessionKey": "main", + "state": "final", + "message": { + "role": "assistant", + "content": "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png" + } + } + } + """); + + Assert.NotNull(received); + Assert.Equal(string.Empty, received.Text); + var media = Assert.Single(received.ContentParts).Media; + Assert.Equal("banner.png", media?.FileName); + } + [Fact] public void ParseChatHistoryPayload_OpenClawMetadata_PreservesMessageIdentity() { diff --git a/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs b/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs new file mode 100644 index 000000000..dadf86513 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs @@ -0,0 +1,186 @@ +using OpenClaw.Shared; +using OpenClawTray.Chat; + +namespace OpenClaw.Tray.Tests; + +public sealed class ChatAssistantContentPresentationTests +{ + [Theory] + [InlineData(1200u, 774u, 1200, 774)] + [InlineData(4096u, 1024u, 2048, 512)] + public void ImageDecodePolicy_BoundsDecodeSize( + uint width, + uint height, + int expectedWidth, + int expectedHeight) + { + Assert.True(ChatAssistantImageDecodePolicy.TryGetDecodeSize( + width, + height, + out var decodeWidth, + out var decodeHeight)); + Assert.Equal(expectedWidth, decodeWidth); + Assert.Equal(expectedHeight, decodeHeight); + } + + [Theory] + [InlineData(1u, 20_000u)] + [InlineData(16_384u, 16_384u)] + public void ImageDecodePolicy_RejectsUnsafeDimensions(uint width, uint height) + { + Assert.False(ChatAssistantImageDecodePolicy.TryGetDecodeSize( + width, + height, + out _, + out _)); + } + + [Fact] + public void Project_StructuredPathLikeFileName_UsesLeafName() + { + var presentation = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + FileName = "/home/openclaw/private/banner.png", + }, + }, + ]); + + Assert.Equal("banner.png", Assert.Single(presentation!.Media).DisplayName); + } + + [Fact] + public void Project_UsesSafeFilenameWithoutExposingLegacySource() + { + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }; + + var presentation = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }, + ]); + + var item = Assert.Single(presentation!.Media); + Assert.Equal("banner.png", item.DisplayName); + Assert.DoesNotContain("/", item.DisplayName, StringComparison.Ordinal); + } + + [Fact] + public void HistoryProjection_MediaPart_RemainsChronological() + { + var message = new ChatMessageInfo + { + Role = "assistant", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = "Before", + }, + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + FileName = "banner.png", + }, + }, + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = "After", + }, + ], + }; + + var parts = ChatHistoryReplayProjection.Project([message]).ToArray(); + + Assert.Collection( + parts, + part => Assert.Equal("Before", part.Text), + part => Assert.Equal( + ChatMessageContentPartKind.Media, + Assert.Single(part.AssistantContentParts).Kind), + part => Assert.Equal("After", part.Text)); + } + + [Fact] + public void BuildRenderPlan_CapsImagesWithoutReorderingOtherMedia() + { + var media = Enumerable.Range(1, 5) + .Select(index => Presentation(ChatMediaContentKind.Image, $"image-{index}.png")) + .Append(Presentation(ChatMediaContentKind.Audio, "audio.mp3")) + .ToArray(); + + var plan = ChatAssistantContentProjector.BuildRenderPlan(media); + + Assert.Equal(1, plan.OmittedImages); + Assert.Equal( + new[] { "image-1.png", "image-2.png", "image-3.png", "image-4.png", "audio.mp3" }, + plan.Media.Select(item => item.DisplayName)); + } + + [Fact] + public void MergeLiveUpdate_DoesNotReplaceLegacyReferenceWithStructuredReference() + { + var legacy = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ])!; + var structured = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-unavailable", + }, + }, + ])!; + + var merged = ChatAssistantContentProjector.MergeLiveUpdate(legacy, structured); + + Assert.Equal( + ChatMediaContentSource.LegacyDirective, + Assert.Single(merged.Media).Reference.Source); + } + + private static ChatAssistantMediaPresentation Presentation( + ChatMediaContentKind kind, + string name) => + new( + kind, + name, + null, + null, + new ChatMediaContentInfo { Kind = kind, FileName = name }); +} diff --git a/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs new file mode 100644 index 000000000..818b2bb5e --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs @@ -0,0 +1,46 @@ +namespace OpenClaw.Tray.Tests; + +public sealed class ChatAssistantMediaRendererContractTests +{ + [Fact] + public void ImageViewer_UsesRegisteredContentDialogFactory() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + + Assert.Contains("var viewer = ContentDialog(", source, StringComparison.Ordinal); + Assert.DoesNotContain("new ContentDialogElement(", source, StringComparison.Ordinal); + } + + [Fact] + public void ImageLoader_DoesNotRenderStateFromPreviousReference() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + + Assert.Contains( + "ReferenceEquals(state.Reference, props.Media.Reference)", + source, + StringComparison.Ordinal); + Assert.Contains( + "props.SessionKey, props.Media.Reference, attempt", + source, + StringComparison.Ordinal); + Assert.Contains( + "AssistantMediaResolutionStatus Status", + source, + StringComparison.Ordinal); + Assert.DoesNotContain( + "AssistantMediaResolutionResult Result", + source, + StringComparison.Ordinal); + } +} diff --git a/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs new file mode 100644 index 000000000..1811447c0 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs @@ -0,0 +1,327 @@ +using OpenClaw.Shared; +using OpenClawTray.Chat; + +namespace OpenClaw.Tray.Tests; + +public class GatewayMediaMessageProjectionTests +{ + private const string ObservedLocalFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png"; + private const string ObservedGatewayFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png"; + + [Fact] + public void ValidEnvelope_ProjectsSafeDescriptorAndCleanProse() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/pasted%20image---7f122605-290a-467c-a5df-8a744c093004.png (image/PNG)]\r\nDescribe this"); + + Assert.True(projection.HasMediaEnvelope); + Assert.Equal("Describe this", projection.ReconciliationText); + Assert.Equal("Describe this", projection.ResidualText); + var attachment = Assert.Single(projection.Attachments); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("pasted image.png", attachment.DisplayFileName); + Assert.Equal("image/png", attachment.MimeType); + Assert.True(attachment.IsImage); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public void MultipleEnvelopeLines_PreserveOrderAndFileClassification() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/photo.jpg (image/jpeg)]\n" + + "[media attached: media://inbound/notes.txt (text/plain)]\nhello"); + + Assert.Collection( + projection.Attachments, + image => + { + Assert.Equal("photo.jpg", image.DisplayFileName); + Assert.True(image.IsImage); + }, + file => + { + Assert.Equal("notes.txt", file.DisplayFileName); + Assert.False(file.IsImage); + }); + Assert.NotEmpty(projection.AttachmentPresentationSignature); + Assert.NotEmpty(projection.AttachmentCorrelationSignature); + } + + [Fact] + public void EncodedPathAndControls_AreReducedToPrintableBoundedLeaf() + { + var longName = new string('a', 220); + var projection = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/folder%2F..%2F{longName}%0Aname.txt (text/plain)]"); + + var attachment = Assert.Single(projection.Attachments); + Assert.DoesNotContain("/", attachment.DisplayFileName); + Assert.DoesNotContain("\n", attachment.DisplayFileName); + Assert.True(attachment.DisplayFileName.Length <= 160); + } + + [Theory] + [InlineData("[media attached: https://inbound/file.png (image/png)]")] + [InlineData("[media attached: media://outbound/file.png (image/png)]")] + [InlineData("[media attached: media://inbound/file.png (not a mime)]")] + [InlineData("[media attached: media://inbound/file.png image/png)]")] + [InlineData("[media attached: media://inbound/ (image/png)]")] + public void MalformedOrUnsupportedLookalike_RemainsOrdinaryProse(string text) + { + var projection = GatewayMediaMessageProjection.Project(text); + + Assert.False(projection.HasMediaEnvelope); + Assert.Empty(projection.Attachments); + Assert.Equal(text, projection.ResidualText); + } + + [Fact] + public void EmbeddedEnvelope_RemainsOrdinaryProse() + { + const string text = "User-authored text\n[media attached: media://inbound/file.png (image/png)]"; + + var projection = GatewayMediaMessageProjection.Project(text); + + Assert.False(projection.HasMediaEnvelope); + Assert.Equal(text, projection.ResidualText); + } + + [Fact] + public void GatewayProjection_NeverEmitsPrivateMarkersOrPreviewKeys() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/file.png (image/png)]"); + + Assert.False(projection.ResidualText.Contains('\u200B')); + Assert.All(projection.Attachments, attachment => + { + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.False(attachment.CanAccessPreviewCache); + }); + } + + [Fact] + public void LocalPresentations_UseOpaquePreviewKeysAndOriginalSafeNames() + { + var presentations = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = @"C:\fake\original.png", + }, + ], + () => "opaque-key"); + + var presentation = Assert.Single(presentations); + Assert.Equal(ChatAttachmentOrigin.Local, presentation.Origin); + Assert.Equal("original.png", presentation.DisplayFileName); + Assert.Equal("opaque-key", presentation.PreviewCacheKey); + Assert.True(presentation.CanAccessPreviewCache); + } + + [Fact] + public void LocalPresentations_PreserveCanonicalLookingNamesAndSourceOrder() + { + const string canonicalLookingName = + "report---7f122605-290a-467c-a5df-8a744c093004.png"; + var keys = new Queue(["first-key", "second-key"]); + var presentations = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "", + }, + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = canonicalLookingName, + }, + ], + () => keys.Dequeue()); + + Assert.Collection( + presentations, + first => + { + Assert.Equal("image", first.DisplayFileName); + Assert.Equal("first-key", first.PreviewCacheKey); + }, + second => + { + Assert.Equal(canonicalLookingName, second.DisplayFileName); + Assert.Equal("second-key", second.PreviewCacheKey); + }); + } + + [Fact] + public void ObservedGatewayRewrite_ChangesPresentationButPreservesCorrelationSignature() + { + var local = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = ObservedLocalFileName, + }, + ], + () => "local-preview"); + var gateway = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/{ObservedGatewayFileName} (image/png)]\nidentical caption"); + + Assert.NotEqual( + GatewayMediaMessageProjection.BuildAttachmentPresentationSignature(local), + gateway.AttachmentPresentationSignature); + Assert.Equal( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(local), + gateway.AttachmentCorrelationSignature); + } + + [Fact] + public void RemoteSameFilename_CannotResolveLocalPreviewBytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var cache = new Dictionary + { + ["opaque-local-key"] = bytes, + ["same.png"] = new byte[] { 9 }, + }; + var local = new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + "same.png", + "image/png", + IsImage: true, + PreviewCacheKey: "opaque-local-key"); + var remote = new ChatAttachmentPresentation( + ChatAttachmentOrigin.GatewayReference, + "same.png", + "image/png", + IsImage: true); + + Assert.True(ChatAttachmentPreviewResolver.TryGetBytes(local, cache, out var resolved)); + Assert.Same(bytes, resolved); + Assert.False(ChatAttachmentPreviewResolver.TryGetBytes(remote, cache, out _)); + } + + [Fact] + public void AttachmentOnlyEcho_AmbiguousMatchingCandidatesAreNotConsumed() + { + var incoming = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/same.png (image/png)]"); + var candidates = new[] + { + new ChatPendingEchoCandidate("one", "", incoming.AttachmentCorrelationSignature), + new ChatPendingEchoCandidate("two", "", incoming.AttachmentCorrelationSignature), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void PlainTextEcho_SingleMediaCandidateIsNotConsumed() + { + var media = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/same.png (image/png)]\nsame caption"); + var plain = GatewayMediaMessageProjection.Project("same caption"); + var candidates = new[] + { + new ChatPendingEchoCandidate("media", "same caption", media.AttachmentCorrelationSignature), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, plain)); + } + + [Fact] + public void MediaEcho_RequiresMatchingAttachmentCorrelationSignature() + { + var incoming = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/right.png (image/png)]\ncaption"); + var wrong = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/wrong.pdf (application/pdf)]\ncaption"); + var candidates = new[] + { + new ChatPendingEchoCandidate("wrong", "caption", wrong.AttachmentCorrelationSignature), + new ChatPendingEchoCandidate("right", "caption", incoming.AttachmentCorrelationSignature), + }; + + Assert.Equal( + "right", + ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void CaptionedMediaEcho_WithTwoSameMimeCandidatesIsAmbiguousAfterFilenameRewrite() + { + var incoming = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/{ObservedGatewayFileName} (image/png)]\nsame caption"); + var firstLocal = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = ObservedLocalFileName, + }, + ], + () => "first-preview"); + var secondLocal = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "another-local-name.png", + }, + ], + () => "second-preview"); + var candidates = new[] + { + new ChatPendingEchoCandidate( + "first", + "same caption", + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(firstLocal)), + new ChatPendingEchoCandidate( + "second", + "same caption", + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(secondLocal)), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void NoEnvelopeEcho_MixedPlainAndAttachmentCandidatesAreAmbiguous() + { + var incoming = GatewayMediaMessageProjection.Project("same prose"); + var candidates = new[] + { + new ChatPendingEchoCandidate("media", "same prose", "1|9:image/png|8:file.png|"), + new ChatPendingEchoCandidate("plain", "same prose", string.Empty), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void NoEnvelopeEcho_AllPlainCandidatesPreserveFifo() + { + var incoming = GatewayMediaMessageProjection.Project("same prose"); + var candidates = new[] + { + new ChatPendingEchoCandidate("first", "same prose", string.Empty), + new ChatPendingEchoCandidate("second", "same prose", string.Empty), + }; + + Assert.Equal( + "first", + ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } +} diff --git a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj index cacaa3045..27f309181 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj +++ b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj @@ -38,6 +38,11 @@ + + + + + diff --git a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs index 72dad8108..1483c1749 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs +++ b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs @@ -2826,6 +2826,88 @@ public async Task ChatMessageReceived_UserEcho_DoesNotReconcileStaleIdenticalCon !afterMeta[user.Id].IsLocalQueuedSend); } + [Fact] + public async Task ChatMessageReceived_NoEnvelopeEchoDoesNotChooseBetweenPlainAndMediaPromotions() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-media", Status = "started" }); + await provider.SendMessageAsync("main", "same", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "same.png", + Content = Convert.ToBase64String([1]), + }, + ]); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "run-media")); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"end"}""", runId: "run-media")); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-plain", Status = "started" }); + await provider.SendMessageAsync("main", "same"); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "run-plain")); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "same", + State = "final", + OpenClawId = "remote-same", + OpenClawSeq = 99, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "same") + .ToArray(); + Assert.Equal(3, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Equal(2, users.Count(user => metadata[user.Id].IsLocalQueuedSend)); + Assert.Single(users, user => metadata[user.Id].GatewayMessageId == "remote-same"); + } + + [Fact] + public async Task ChatMessageReceived_NoEnvelopeEchoDoesNotConsumeSingleMediaPromotion() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-media", Status = "started" }); + await provider.SendMessageAsync("main", "same", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "same.png", + Content = Convert.ToBase64String([1]), + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "same", + State = "final", + OpenClawId = "remote-same", + OpenClawSeq = 99, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "same") + .ToArray(); + Assert.Equal(2, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Contains(users, user => metadata[user.Id].IsLocalQueuedSend); + Assert.Contains(users, user => metadata[user.Id].GatewayMessageId == "remote-same"); + } + [Fact] public async Task SendMessageAsync_WhenGatewayThrows_DoesNotSuppressFutureRemoteUserEcho() { @@ -3560,6 +3642,63 @@ public async Task SessionResetCompletion_TimestamplessRemoteUserCanOpenGateViaHi e.Kind == ChatTimelineItemKind.Assistant && e.Text == "remote response"); } + [Fact] + public async Task SessionResetCompletion_DuplicateBackfillMergesAndOpensBufferedLifecycleGate() + { + var remoteTimestamp = DateTimeOffset.UtcNow.AddSeconds(5).ToUnixTimeMilliseconds(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/reset.pdf (application/pdf)]\nremote after reset", + Ts = remoteTimestamp, + OpenClawId = "reset-user", + OpenClawSeq = 41, + }, + ], + }); + await provider.LoadAsync(); + bridge.RaiseSessionCommandCompleted(new SessionCommandResult + { + Method = "sessions.reset", + Ok = true, + Key = "main", + }); + snapshots.Clear(); + + // Insert the projected row without reset proof, then buffer the + // lifecycle start before the reset-aware duplicate backfill arrives. + await provider.FetchRemoteUserMessageForTestsAsync("main", openResetGateOnSuccess: false); + var pendingStart = MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "reset-remote-run"); + pendingStart.Ts = remoteTimestamp + 1; + bridge.RaiseAgent(pendingStart); + snapshots.Clear(); + + await provider.FetchRemoteUserMessageForTestsAsync("main", openResetGateOnSuccess: true); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "reset gate opened", + State = "final", + Ts = remoteTimestamp + 2, + }); + + var latest = snapshots[^1].Timelines["main"]; + var user = Assert.Single(latest.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("remote after reset", user.Text); + Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Contains(latest.Entries, entry => + entry.Kind == ChatTimelineItemKind.Assistant && + entry.Text == "reset gate opened"); + } + [Fact] public async Task SessionResetCompletion_LocalSendDoesNotReopenGateForStaleChatFrames() { @@ -10430,6 +10569,217 @@ public async Task OnChatMessageReceived_LiveUserPlain_ShownAsRemoteUser() Assert.Equal("hello there", entry.Text); } + [Fact] + public async Task OnChatMessageReceived_MediaOnlyAssistant_CreatesSafePresentationEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + State = "final", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries); + Assert.Equal(ChatTimelineItemKind.Assistant, entry.Kind); + Assert.Equal(string.Empty, entry.Text); + var metadata = provider.GetEntryMetadata("main"); + var media = Assert.Single(metadata[entry.Id].AssistantContent!.Media); + Assert.Equal("banner.png", media.DisplayName); + } + + [Fact] + public async Task OnChatMessageReceived_FinalMedia_MergesIntoStreamingAssistantEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "Rendering", + State = "delta", + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "Rendering complete", + State = "final", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + FileName = "banner.png", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + Assert.Equal("Rendering complete", entry.Text); + Assert.False(entry.IsStreaming); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal("artifact-1", media.Reference.ArtifactId); + } + + [Fact] + public async Task OnChatMessageReceived_StructuredFollowUp_PreservesLegacyMediaReference() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-unavailable", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Reference.Source); + Assert.Equal("banner.png", media.DisplayName); + } + + [Fact] + public async Task OnChatMessageReceived_IdentifiedMediaOnlyRetransmit_DoesNotDuplicateEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + var message = new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + State = "final", + OpenClawId = "assistant-media-1", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }; + + bridge.RaiseChat(message); + bridge.RaiseChat(message); + + var timeline = snapshots[^1].Timelines["main"]; + Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + } + + [Fact] + public async Task LoadHistoryAsync_StructuredMedia_CreatesSafePresentationEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + FileName = "banner.png", + }, + }, + ], + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + await provider.LoadHistoryAsync("main"); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal("banner.png", media.DisplayName); + Assert.Equal("artifact-1", media.Reference.ArtifactId); + } + // ── chat rubber-duck MEDIUM 4: per-message size cap ── [Fact] @@ -10568,44 +10918,396 @@ public async Task SendMessageAsync_ClearsPendingAbortCounts() } [Fact] - public async Task SendMessageAsync_WithAttachment_SendsThroughInterface() + public async Task ChatMessageReceived_LocalMediaEcho_ReconcilesOneCleanRowAndPreservesLocalPreview() { + var sendGate = new TaskCompletionSource(); var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; await provider.LoadAsync(); - var attachment = new ChatAttachment - { - Type = "file", - MimeType = "text/plain", - FileName = "test.txt", - Content = Convert.ToBase64String(new byte[] { 72, 101, 108, 108, 111 }), - SizeBytes = 5 - }; - - await provider.SendMessageAsync("main", "Check this", default, new[] { attachment }); - - Assert.Contains(bridge.SentMessages, m => m == "Check this"); - var sentAttachment = Assert.Single(bridge.SentAttachments); - Assert.NotNull(sentAttachment); - Assert.Same(attachment, sentAttachment![0]); - Assert.Contains("test.txt", snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User).Text); + _ = provider.SendMessageAsync("main", "Describe this", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1, 2, 3]), + SizeBytes = 3, + }, + ]); + snapshots.Clear(); bridge.RaiseChat(new ChatMessageInfo { SessionKey = "main", Role = "user", - Text = "Check this", - State = "final" + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nDescribe this", + State = "final", + OpenClawId = "gateway-media", + OpenClawSeq = 12, }); - // The display text in the timeline should include the attachment indicator - var timeline = snapshots[^1].Timelines["main"]; - var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); - Assert.Contains("test.txt", userEntry.Text); + var timeline = Assert.Single(snapshots).Timelines["main"]; + var user = Assert.Single(timeline.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("Describe this", user.Text); + Assert.DoesNotContain("media://", user.Text); + var metadata = provider.GetEntryMetadata("main")[user.Id]; + Assert.Equal("gateway-media", metadata.GatewayMessageId); + var attachment = Assert.Single(metadata.Attachments!); + Assert.Equal(ChatAttachmentOrigin.Local, attachment.Origin); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + attachment.DisplayFileName); + Assert.True(attachment.CanAccessPreviewCache); + Assert.True(OpenClawChatDataProvider.ImagePreviewCache.ContainsKey(attachment.PreviewCacheKey!)); + + sendGate.SetResult(); } [Fact] - public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMarkers() + public async Task ChatMessageReceived_AttachmentOnlyMediaEcho_ReconcilesWithoutEmptyTextMatching() + { + var sendGate = new TaskCompletionSource(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; + await provider.LoadAsync(); + + _ = provider.SendMessageAsync("main", "", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1]), + SizeBytes = 1, + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]", + State = "final", + OpenClawId = "gateway-only", + }); + + var user = Assert.Single(Assert.Single(snapshots).Timelines["main"].Entries); + Assert.Equal(string.Empty, user.Text); + var metadata = provider.GetEntryMetadata("main")[user.Id]; + Assert.Equal("gateway-only", metadata.GatewayMessageId); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Assert.Single(metadata.Attachments!).DisplayFileName); + sendGate.SetResult(); + } + + [Fact] + public async Task SessionResetCompletion_MediaEchoUsesCorrelationSignatureAcrossResetGate() + { + var sendGate = new TaskCompletionSource(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; + await provider.LoadAsync(); + + _ = provider.SendMessageAsync("main", "before reset", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1]), + }, + ]); + bridge.RaiseSessionCommandCompleted(new SessionCommandResult + { + Method = "sessions.reset", + Ok = true, + Key = "main", + }); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nbefore reset", + Ts = DateTimeOffset.UtcNow.AddSeconds(1).ToUnixTimeMilliseconds(), + }); + + Assert.Empty((await provider.LoadAsync()).Timelines["main"].Entries); + sendGate.SetResult(); + + await provider.SendMessageAsync("main", "after reset", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "after.png", + Content = Convert.ToBase64String([2]), + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/after---7f122605-290a-467c-a5df-8a744c093004.png (image/png)]\nafter reset", + Ts = DateTimeOffset.UtcNow.AddSeconds(2).ToUnixTimeMilliseconds(), + OpenClawId = "after-reset-media", + }); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal("after reset", user.Text); + Assert.Equal("after-reset-media", provider.GetEntryMetadata("main")[user.Id].GatewayMessageId); + } + + [Fact] + public async Task RemoteLifecycleBackfill_MediaEnvelopeBecomesCleanStructuredUserRow() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/remote.pdf (application/pdf)]\nReview it", + Ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "remote-media-run")); + for (var i = 0; i < 50 && snapshots.Count == 0; i++) + await Task.Delay(10); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("Review it", user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("remote.pdf", attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task RemoteLifecycleBackfill_WhenLiveFrameArrivesLater_MergesOneUserRow() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "race", + Ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + OpenClawId = "race-user", + OpenClawSeq = 73, + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "race-run")); + for (var i = 0; i < 50; i++) + { + if (snapshots.Count > 0 && + snapshots[^1].Timelines["main"].Entries.Any(entry => + entry.Kind == ChatTimelineItemKind.User && entry.Text == "race")) + { + break; + } + await Task.Delay(10); + } + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/race.png (image/png)]\nrace", + State = "final", + OpenClawId = "race-user", + OpenClawSeq = 73, + }); + + var latest = snapshots[^1].Timelines["main"]; + var user = Assert.Single(latest.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("race", user.Text); + Assert.Equal( + "race.png", + Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task ChatMessageReceived_RepeatedSameTextWithDistinctGatewayIdentityKeepsBothTurns() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "repeat", + State = "final", + OpenClawId = "repeat-1", + OpenClawSeq = 1, + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "repeat", + State = "final", + OpenClawId = "repeat-2", + OpenClawSeq = 2, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "repeat") + .ToArray(); + Assert.Equal(2, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Equal( + new[] { "repeat-1", "repeat-2" }, + users.Select(user => metadata[user.Id].GatewayMessageId).ToArray()); + } + + [Fact] + public async Task LoadHistoryAsync_GatewayMediaUsesLocalSidecarPrecedenceWithoutRestoringBytes() + { + using var tempDir = new TempDirectory(); + var toolPath = Path.Combine(tempDir.DirectoryPath, "tool-metadata.json"); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sentTs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var (_, sender, _, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + await sender.LoadAsync(); + await sender.SendMessageAsync("main", "history caption", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1, 2]), + }, + ]); + + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nhistory caption", + Ts = sentTs, + }, + ], + }); + + await provider.LoadHistoryAsync("main"); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal("history caption", user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.Local, attachment.Origin); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task LoadHistoryAsync_RemoteMediaEnvelopeFallsBackToGatewayReference() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/report.pdf (application/pdf)]", + Ts = 123, + }, + ], + }); + + await provider.LoadHistoryAsync("main"); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal(string.Empty, user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("report.pdf", attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task SendMessageAsync_WithAttachment_SendsThroughInterface() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + var attachment = new ChatAttachment + { + Type = "file", + MimeType = "text/plain", + FileName = "test.txt", + Content = Convert.ToBase64String(new byte[] { 72, 101, 108, 108, 111 }), + SizeBytes = 5 + }; + + await provider.SendMessageAsync("main", "Check this", default, new[] { attachment }); + + Assert.Contains(bridge.SentMessages, m => m == "Check this"); + var sentAttachment = Assert.Single(bridge.SentAttachments); + Assert.NotNull(sentAttachment); + Assert.Same(attachment, sentAttachment![0]); + var optimistic = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("Check this", optimistic.Text); + Assert.Equal( + "test.txt", + Assert.Single(provider.GetEntryMetadata("main")[optimistic.Id].Attachments!).DisplayFileName); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "Check this", + State = "final" + }); + + var timeline = snapshots[^1].Timelines["main"]; + var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("Check this", userEntry.Text); + } + + [Fact] + public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMarkers() { var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); await provider.LoadAsync(); @@ -10635,9 +11337,9 @@ public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMar sentAttachments!, a => Assert.Same(fileAttachment, a), a => Assert.Same(imageAttachment, a)); - Assert.Equal( - "See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", - snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User).Text); + var optimistic = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("See both", optimistic.Text); + Assert.Equal(2, provider.GetEntryMetadata("main")[optimistic.Id].Attachments!.Count); bridge.RaiseChat(new ChatMessageInfo { @@ -10649,7 +11351,50 @@ public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMar var timeline = snapshots[^1].Timelines["main"]; var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", userEntry.Text); + Assert.Equal("See both", userEntry.Text); + } + + [Fact] + public async Task SendMessageAsync_OversizedMediaCaptionUsesSameBoundedEchoText() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + var caption = new string('x', OpenClawChatDataProvider.MaxEntryTextBytes + 50_000); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-large", Status = "started" }); + await provider.SendMessageAsync("main", caption, default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "large.png", + Content = Convert.ToBase64String([1]), + SizeBytes = 1, + }, + ]); + + var optimistic = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.True( + System.Text.Encoding.UTF8.GetByteCount(optimistic.Text) <= + OpenClawChatDataProvider.MaxEntryTextBytes); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = $"[media attached: media://inbound/large.png (image/png)]\n{caption}", + State = "final", + OpenClawId = "large-echo", + OpenClawSeq = 1, + }); + + var user = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("large-echo", provider.GetEntryMetadata("main")[user.Id].GatewayMessageId); } [Fact] @@ -10681,14 +11426,138 @@ await provider1.SendMessageAsync("main", "Check this", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "Check this", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/test.txt (text/plain)]\nCheck this", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("Check this\n\u200B📎 test.txt", userEntry.Text); + Assert.Equal("Check this", userEntry.Text); + Assert.Equal( + "test.txt", + Assert.Single(provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task AttachmentMetadata_IdentitylessRewrittenLiveEchoDoesNotDuplicateRehydratedHistory() + { + const string localFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png"; + const string gatewayFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png"; + using var tempDir = new TempDirectory(); + var toolPath = Path.Combine(tempDir.DirectoryPath, "tool-metadata.json"); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sentTs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + var (_, provider1, _, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + await provider1.LoadAsync(); + await provider1.SendMessageAsync("main", "Check this", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = localFileName, + Content = Convert.ToBase64String([1, 2, 3]), + SizeBytes = 3, + }, + ]); + + var gatewayText = + $"[media attached: media://inbound/{gatewayFileName} (image/png)]\nCheck this"; + var (bridge2, provider2, snapshots, _) = CreateProvider( + new[] { MainSession() }, + toolPath, + attachmentPath); + bridge2.HistoryBehavior = key => Task.FromResult(new ChatHistoryInfo + { + SessionKey = key ?? "", + SessionId = "session-1", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = gatewayText, + State = "final", + Ts = sentTs, + }, + ], + }); + + await provider2.LoadHistoryAsync("main"); + bridge2.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = gatewayText, + State = "final", + Ts = sentTs, + }); + + var user = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal( + localFileName, + Assert.Single(provider2.GetEntryMetadata("main")[user.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task AttachmentMetadata_ConcurrentSavesPreserveEveryCompletedSend() + { + const int sendCount = 16; + using var tempDir = new TempDirectory(); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sessions = Enumerable.Range(0, sendCount) + .Select(index => new SessionInfo + { + Key = $"thread-{index}", + DisplayName = $"Thread {index}", + Status = "active", + IsMain = index == 0, + }) + .ToArray(); + var (bridge, provider, _, _) = CreateProvider( + sessions, + attachmentMetaCachePath: attachmentPath); + await provider.LoadAsync(); + + var allStarted = new CountdownEvent(sendCount); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bridge.SendBehavior = async (_, _, _) => + { + allStarted.Signal(); + await release.Task; + }; + + var sends = sessions.Select((session, index) => + provider.SendMessageAsync(session.Key, $"caption-{index}", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = $"file-{index}.png", + Content = Convert.ToBase64String([(byte)index]), + SizeBytes = 1, + }, + ])).ToArray(); + Assert.True(allStarted.Wait(TimeSpan.FromSeconds(5))); + release.SetResult(); + await Task.WhenAll(sends); + + var persisted = await File.ReadAllTextAsync(attachmentPath); + for (var index = 0; index < sendCount; index++) + Assert.Contains($"file-{index}.png", persisted, StringComparison.Ordinal); } [Fact] @@ -10728,14 +11597,22 @@ await provider1.SendMessageAsync("main", "See both", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "See both", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/notes.txt (text/plain)]\n" + + "[media attached: media://inbound/diagram.png (image/png)]\nSee both", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", userEntry.Text); + Assert.Equal("See both", userEntry.Text); + Assert.Equal(2, provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!.Count); } [Fact] @@ -10767,14 +11644,23 @@ await provider1.SendMessageAsync("main", "", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/screenshot.png (image/png)]", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("\u200B🖼️ screenshot.png", userEntry.Text); + Assert.Equal(string.Empty, userEntry.Text); + Assert.Equal( + "screenshot.png", + Assert.Single(provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!).DisplayFileName); } [Fact] From bf763987a45c0b5859226537c7524c72b650d77c Mon Sep 17 00:00:00 2001 From: Karen Lai <7976322+karkarl@users.noreply.github.com> Date: Tue, 18 Aug 2026 14:49:23 -0700 Subject: [PATCH 2/2] fix(chat): harden media replay and previews Bound local preview memory, enforce guarded local image decoding, preserve media metadata across history and reset reconciliation, and keep runtime V2 credential upgrades without allowing downgrades. Co-authored-by: Copilot App <223556219+Copilot@users.noreply.github.com> Copilot-Session: 4f74b3dc-62de-4bf8-abee-aae3915b2f89 --- .../GatewayConnectionManager.cs | 2 +- .../Chat/ChatAssistantMediaRenderer.cs | 43 ++++++ .../Chat/ChatAttachmentPresentation.cs | 127 ++++++++++++++++++ .../Chat/ChatConversationState.cs | 46 +++---- .../Chat/ChatHistoryLoader.cs | 56 ++++++-- .../Chat/ChatMetadataStore.cs | 67 ++++++--- .../Chat/ChatQueueState.cs | 11 +- .../Chat/OpenClawChatDataProvider.cs | 39 ++++-- .../Chat/OpenClawChatTimeline.cs | 14 +- .../Chat/ReactorChatTimeline.cs | 14 +- .../GatewayConnectionManagerTests.cs | 27 ++++ ...ChatAssistantMediaRendererContractTests.cs | 38 ++++++ .../GatewayMediaMessageProjectionTests.cs | 36 +++++ .../OpenClawChatDataProviderTests.cs | 2 +- 14 files changed, 436 insertions(+), 86 deletions(-) diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 7c9cd2065..c53823e54 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -2565,7 +2565,7 @@ private static bool IsSameCredentialHandoffRecord( expected.BootstrapToken, StringComparison.Ordinal) && current.IsLocal == expected.IsLocal && - current.RequiresV2Signature == expected.RequiresV2Signature && + (current.RequiresV2Signature || !expected.RequiresV2Signature) && string.Equals( current.SetupManagedDistroName, expected.SetupManagedDistroName, diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs index 8f11ba84e..4cc361ce2 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs @@ -341,3 +341,46 @@ internal sealed record ChatAssistantImageLoadState( BitmapImage? Bitmap, string SessionKey, ChatMediaContentInfo Reference); + +internal static class ChatAttachmentBitmapDecoder +{ + internal static BitmapImage? TryDecode(byte[] bytes) + { + try + { + using var stream = new InMemoryRandomAccessStream(); + using (var writer = new DataWriter(stream)) + { + writer.WriteBytes(bytes); + writer.StoreAsync().AsTask().GetAwaiter().GetResult(); + writer.DetachStream(); + } + + stream.Seek(0); + var decoder = BitmapDecoder.CreateAsync(stream).AsTask().GetAwaiter().GetResult(); + if (!ChatAssistantImageDecodePolicy.TryGetDecodeSize( + decoder.PixelWidth, + decoder.PixelHeight, + out var decodeWidth, + out var decodeHeight)) + { + return null; + } + + stream.Seek(0); + var bitmap = new BitmapImage + { + DecodePixelType = DecodePixelType.Physical, + DecodePixelWidth = decodeWidth, + DecodePixelHeight = decodeHeight, + }; + bitmap.SetSource(stream); + return bitmap; + } + catch (Exception ex) + { + Logger.Warn($"Attachment image decode failed ({ex.GetType().Name})."); + return null; + } + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs index dad092884..57f3bf754 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs @@ -1,3 +1,5 @@ +using OpenClaw.Shared; + namespace OpenClawTray.Chat; public enum ChatAttachmentOrigin @@ -24,6 +26,15 @@ public sealed record ChatAttachmentPresentation( internal static class ChatAttachmentPreviewResolver { + internal static bool TryGetBytes( + ChatAttachmentPresentation attachment, + out byte[] bytes) + { + bytes = Array.Empty(); + return attachment.CanAccessPreviewCache && + ChatImagePreviewCache.TryGet(attachment.PreviewCacheKey!, out bytes); + } + internal static bool TryGetBytes( ChatAttachmentPresentation attachment, IReadOnlyDictionary previewCache, @@ -34,3 +45,119 @@ internal static bool TryGetBytes( previewCache.TryGetValue(attachment.PreviewCacheKey!, out bytes!); } } + +internal static class ChatImagePreviewCache +{ + internal const int MaximumEntries = 32; + internal const long MaximumTotalBytes = 64L * 1024 * 1024; + internal const int MaximumPreviewBytes = (int)ChatAttachment.MaxSizeBytes; + + private static readonly object s_gate = new(); + private static readonly Dictionary s_entries = new(StringComparer.Ordinal); + private static readonly Queue s_insertionOrder = new(); + private static long s_totalBytes; + + internal static int Count + { + get + { + lock (s_gate) + return s_entries.Count; + } + } + + internal static long TotalBytes + { + get + { + lock (s_gate) + return s_totalBytes; + } + } + + internal static bool TryStoreBase64(string key, string encoded) + { + if (string.IsNullOrWhiteSpace(key) || + !TryDecodeBoundedBase64(encoded, MaximumPreviewBytes, out var bytes)) + { + return false; + } + + lock (s_gate) + { + if (s_entries.TryGetValue(key, out var previous)) + { + s_totalBytes -= previous.Length; + } + else + { + s_insertionOrder.Enqueue(key); + } + + s_entries[key] = bytes; + s_totalBytes += bytes.Length; + TrimLocked(); + return s_entries.ContainsKey(key); + } + } + + internal static bool TryGet(string key, out byte[] bytes) + { + lock (s_gate) + return s_entries.TryGetValue(key, out bytes!); + } + + internal static bool Contains(string key) + { + lock (s_gate) + return s_entries.ContainsKey(key); + } + + internal static bool TryDecodeBoundedBase64( + string? encoded, + int maximumBytes, + out byte[] bytes) + { + bytes = Array.Empty(); + if (string.IsNullOrEmpty(encoded) || maximumBytes <= 0) + return false; + + var maximumEncodedLength = ((maximumBytes + 2L) / 3L) * 4L; + if (encoded.Length > maximumEncodedLength) + return false; + + try + { + bytes = Convert.FromBase64String(encoded); + if (bytes.Length <= maximumBytes) + return true; + } + catch (FormatException) + { + } + + bytes = Array.Empty(); + return false; + } + + internal static void Clear() + { + lock (s_gate) + { + s_entries.Clear(); + s_insertionOrder.Clear(); + s_totalBytes = 0; + } + } + + private static void TrimLocked() + { + while ((s_entries.Count > MaximumEntries || + s_totalBytes > MaximumTotalBytes) && + s_insertionOrder.TryDequeue(out var oldestKey)) + { + if (s_entries.Remove(oldestKey, out var removed)) + s_totalBytes -= removed.Length; + } + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs index de6aa5147..f8aaa620d 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs @@ -1347,6 +1347,8 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( _queue.TryConsumeLocalEcho( threadId, resetGate.ConsumeEchoText, + attachmentCorrelationSignature, + hasMediaEnvelope, out var queuedMessageId)) { var confirmed = BuildLiveMetaLocked( @@ -2064,11 +2066,12 @@ internal void CompleteRemoteBackfill(string threadId) internal ChatRemoteUserBackfillTransition? ApplyRemoteUserBackfill( string threadId, ChatMessageInfo message, + GatewayMediaMessageProjectionResult projection, + IReadOnlyList attachments, long expectedResetGeneration, bool openResetGate, ChatProjectionContext context) { - var projection = GatewayMediaMessageProjection.Project(message.Text); lock (_gate) { if (GetResetVersionLocked(threadId) != expectedResetGeneration || @@ -2076,36 +2079,33 @@ internal void CompleteRemoteBackfill(string threadId) { return null; } - if (_timelines.TryGetValue(threadId, out var timeline)) - { - for (var i = timeline.Entries.Count - 1; i >= 0; i--) - { - if (timeline.Entries[i].Kind != ChatTimelineItemKind.User) - continue; - if (timeline.Entries[i].Text == projection.ReconciliationText) - return null; - break; - } - } var openedLifecycle = openResetGate ? ApplyBufferedLifecycleOpenLocked( threadId, _reset.RecordRemoteUser(threadId), allowRemoteTurn: true) : null; - ApplyEventLocked( + var metadata = BuildLiveMetaLocked( threadId, - new ChatUserMessageEvent( - ChatContentFormatting.TruncateForChatEntry( - projection.ReconciliationText)), - BuildLiveMetaLocked( - threadId, - message.Ts, - message.OpenClawId, - message.OpenClawSeq, - attachments: projection.Attachments)); + message.Ts, + message.OpenClawId, + message.OpenClawSeq, + attachments: attachments); + var snapshot = ApplyProjectedRemoteUserMessageLocked( + threadId, + ChatContentFormatting.TruncateForChatEntry( + projection.HasMediaEnvelope + ? projection.ReconciliationText + : ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines( + message.Text)), + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + attachments), + metadata, + context); + if (snapshot is null && openedLifecycle is null) + return null; return new( - BuildSnapshotLocked(context), + snapshot ?? BuildSnapshotLocked(context), openedLifecycle, CurrentRuntimeGenerationLocked(threadId)); } diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs index 33f754336..8fbec307b 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs @@ -487,6 +487,18 @@ ChatTimelineState Apply( if (!before.Contains(entry.Id) && !metadata.ContainsKey(entry.Id)) metadata[entry.Id] = entryMetadata; } + if (entryMetadata.AssistantContent is not null && + next.ActiveAssistantId is { } assistantId && + metadata.TryGetValue(assistantId, out var existingMetadata)) + { + metadata[assistantId] = existingMetadata with + { + AssistantContent = + ChatAssistantContentProjector.MergeLiveUpdate( + existingMetadata.AssistantContent, + entryMetadata.AssistantContent), + }; + } } return next; } @@ -504,6 +516,10 @@ ChatTimelineState Apply( } var role = message.Role?.ToLowerInvariant() ?? string.Empty; + var rawText = replayPart.Text; + var userProjection = role == "user" + ? GatewayMediaMessageProjection.Project(rawText) + : null; var entryMetadata = new ChatEntryMetadata( message.Ts > 0 ? DateTimeOffset.FromUnixTimeMilliseconds(message.Ts).ToLocalTime() @@ -517,17 +533,37 @@ ChatTimelineState Apply( OpenClawSeq: message.OpenClawSeq, OpenClawKind: message.OpenClawKind, CompactionTokensBefore: message.CompactionTokensBefore, - CompactionTokensAfter: message.CompactionTokensAfter); + CompactionTokensAfter: message.CompactionTokensAfter, + AssistantContent: role == "assistant" + ? ChatAssistantContentProjector.Project( + replayPart.AssistantContentParts) + : null); var text = ChatContentFormatting.TruncateForChatEntry( ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines( - replayPart.Text)); - if (role == "user") - text = ChatMetadataStore.RehydrateAttachmentMarkers( - attachmentMatcher, - text, + userProjection?.HasMediaEnvelope == true + ? userProjection.ReconciliationText + : rawText)); + if (userProjection is not null) + { + var cachedAttachment = attachmentMatcher.TryMatch( + userProjection.ReconciliationText, + userProjection.AttachmentCorrelationSignature, message.Ts); + var attachmentPresentations = cachedAttachment is not null + ? ChatMetadataStore.CreatePersistedLocalPresentations( + cachedAttachment.Attachments) + : userProjection.Attachments; + entryMetadata = entryMetadata with + { + Attachments = attachmentPresentations, + }; + } var hasStructuredToolContent = replayPart.ToolContent.Count > 0; + var hasUserAttachments = + entryMetadata.Attachments is { Count: > 0 }; + var hasAssistantMedia = + entryMetadata.AssistantContent is { Media.Count: > 0 }; if (role == "user" && _persistence.IsMessageAborted( @@ -564,12 +600,16 @@ ChatTimelineState Apply( } if (string.IsNullOrEmpty(text) && - !hasStructuredToolContent) + !hasStructuredToolContent && + !hasUserAttachments && + !hasAssistantMedia) { continue; } - if (!string.IsNullOrEmpty(text)) + if (!string.IsNullOrEmpty(text) || + (role == "user" && hasUserAttachments) || + (role == "assistant" && hasAssistantMedia)) { switch (role) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs index 4b41a5a17..de5597231 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs @@ -20,8 +20,14 @@ public AttachmentMetaMatcher(List entrie _used = new bool[entries.Count]; } - public ChatMetadataStore.CachedAttachmentMeta? TryMatch(string text, long historyTsMs) + public ChatMetadataStore.CachedAttachmentMeta? TryMatch( + string text, + string attachmentCorrelationSignature, + long historyTsMs) { + if (string.IsNullOrEmpty(attachmentCorrelationSignature)) + return null; + for (var i = 0; i < _entries.Count; i++) { if (_used[i]) @@ -30,6 +36,15 @@ public AttachmentMetaMatcher(List entrie var entry = _entries[i]; if (!string.Equals(entry.Text, text, StringComparison.Ordinal)) continue; + if (!string.Equals( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + ChatMetadataStore.CreatePersistedLocalPresentations( + entry.Attachments)), + attachmentCorrelationSignature, + StringComparison.Ordinal)) + { + continue; + } if (historyTsMs > 0 && entry.Ts > 0 && Math.Abs(historyTsMs - entry.Ts) > MatchWindow.TotalMilliseconds) @@ -78,6 +93,7 @@ internal sealed class CachedAttachmentMeta internal sealed class CachedAttachmentItem { public string FileName { get; set; } = ""; + public string MimeType { get; set; } = "application/octet-stream"; public bool IsImage { get; set; } } @@ -272,10 +288,21 @@ internal void CacheAttachments( var items = attachments .Where(attachment => !string.IsNullOrWhiteSpace(attachment.FileName)) - .Select(attachment => new CachedAttachmentItem + .Select(attachment => { - FileName = NormalizeCachedDisplayText(attachment.FileName), - IsImage = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase), + var mimeType = + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType); + return new CachedAttachmentItem + { + FileName = NormalizeCachedDisplayText(attachment.FileName), + MimeType = mimeType, + IsImage = string.Equals( + attachment.Type, + "image", + StringComparison.OrdinalIgnoreCase) || + mimeType.StartsWith("image/", StringComparison.Ordinal), + }; }) .ToList(); if (items.Count == 0) @@ -675,6 +702,8 @@ private static void AtomicWrite(string path, string json, string cacheName) Attachments = entry.Attachments.Select(attachment => new CachedAttachmentItem { FileName = NormalizeCachedDisplayText(attachment.FileName), + MimeType = GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType), IsImage = attachment.IsImage, }).ToList(), }; @@ -714,7 +743,12 @@ internal static Dictionary> LoadAttachmentMet { entry.Text = NormalizeCachedDisplayText(entry.Text); foreach (var attachment in entry.Attachments) + { attachment.FileName = NormalizeCachedDisplayText(attachment.FileName); + attachment.MimeType = + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType); + } } return cache; } @@ -760,20 +794,17 @@ internal static string BuildAttachmentMarkerLines(IEnumerable + CreatePersistedLocalPresentations( + IEnumerable attachments) => + attachments.Select(attachment => new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + GatewayMediaMessageProjection.NormalizeDisplayFileName( + attachment.FileName), + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType), + attachment.IsImage, + PreviewCacheKey: null)).ToArray(); internal static string NormalizeCachedDisplayText(string? value) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs index 5e942fcec..0f71c37eb 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs @@ -451,7 +451,10 @@ internal bool HasPendingLocalEchoText( string attachmentCorrelationSignature = "", bool hasMediaEnvelope = false) { - if (string.IsNullOrWhiteSpace(text) || + var normalizedText = + GatewayMediaMessageProjection.NormalizeEchoCorrelationText(text); + if ((normalizedText.Length == 0 && + string.IsNullOrEmpty(attachmentCorrelationSignature)) || !_localSentTexts.TryGetValue(threadId, out var queue)) { return false; @@ -464,7 +467,7 @@ internal bool HasPendingLocalEchoText( .ToArray(); return ChatAttachmentEchoCorrelation.SelectMatchingMessageId( candidates, - text.Trim(), + normalizedText, attachmentCorrelationSignature, hasMediaEnvelope) is not null; } @@ -479,6 +482,8 @@ internal bool TryConsumeLocalEcho( queuedMessageId = string.Empty; if (!_localSentTexts.TryGetValue(threadId, out var queue)) return false; + var normalizedEchoText = + GatewayMediaMessageProjection.NormalizeEchoCorrelationText(echoText); var now = DateTimeOffset.Now; while (queue.Count > 0 && now - queue.Peek().SentAt > LocalEchoWindow) @@ -498,7 +503,7 @@ internal bool TryConsumeLocalEcho( .ToArray(); var matchedMessageId = ChatAttachmentEchoCorrelation.SelectMatchingMessageId( candidates, - echoText, + normalizedEchoText, attachmentCorrelationSignature, hasMediaEnvelope); if (matchedMessageId is null) diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs index 0277bc72d..e0a726952 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Collections.Concurrent; using System.Globalization; using System.IO; using System.Linq; @@ -71,12 +70,6 @@ internal static class LocalizationHelper public sealed class OpenClawChatDataProvider : IChatDataProvider { internal const int MaxEntryTextBytes = 256 * 1024; - /// - /// Process-wide cache mapping an opaque local preview key to raw image - /// bytes. Gateway references never receive keys and cannot read this cache. - /// - public static readonly ConcurrentDictionary ImagePreviewCache = new(); - private readonly IChatGatewayBridge _bridge; private readonly ChatTelemetryTracker _telemetry = new(); private readonly ChatMetadataStore _metadataStore; @@ -210,8 +203,13 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation var presentation = attachmentPresentations[i]; if (presentation.CanAccessPreviewCache && !string.IsNullOrEmpty(source.Content)) { - try { ImagePreviewCache[presentation.PreviewCacheKey!] = Convert.FromBase64String(source.Content); } - catch (Exception ex) { Logger.Debug($"ChatDataProvider: image attachment base64 decode failed for '{presentation.DisplayFileName}': {ex.Message}"); } + if (!ChatImagePreviewCache.TryStoreBase64( + presentation.PreviewCacheKey!, + source.Content)) + { + Logger.Debug( + $"ChatDataProvider: image attachment preview rejected for '{presentation.DisplayFileName}'"); + } } } } @@ -1686,11 +1684,32 @@ private async Task FetchRemoteUserMessageAsync(string threadId, bool openResetGa break; } } - if (lastUser is null || string.IsNullOrEmpty(lastUser.Text)) return; + if (lastUser is null) return; + var projection = GatewayMediaMessageProjection.Project(lastUser.Text); + if (projection.ReconciliationText.Length == 0 && + projection.Attachments.Count == 0) + { + return; + } + var cachedAttachment = _metadataStore + .CreateAttachmentMatcher( + history.SessionId, + threadId, + requestResetVersion) + .TryMatch( + projection.ReconciliationText, + projection.AttachmentCorrelationSignature, + lastUser.Ts); + var projectedAttachments = cachedAttachment is not null + ? ChatMetadataStore.CreatePersistedLocalPresentations( + cachedAttachment.Attachments) + : projection.Attachments; var transition = _state.ApplyRemoteUserBackfill( threadId, lastUser, + projection, + projectedAttachments, requestResetVersion, openResetGateOnSuccess, ProjectionContext()); diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs index 8e27ae0ee..03f68595c 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs @@ -369,16 +369,9 @@ static double ClampOffset(double offset, double max) => return existing; try { - var stream = new global::Windows.Storage.Streams.InMemoryRandomAccessStream(); - using (var writer = new global::Windows.Storage.Streams.DataWriter(stream)) - { - writer.WriteBytes(bytes); - writer.StoreAsync().AsTask().GetAwaiter().GetResult(); - writer.DetachStream(); - } - stream.Seek(0); - var bmp = new Microsoft.UI.Xaml.Media.Imaging.BitmapImage(); - bmp.SetSource(stream); + var bmp = ChatAttachmentBitmapDecoder.TryDecode(bytes); + if (bmp is null) + return null; _bitmapCache.Add(bytes, bmp); return bmp; } @@ -1230,7 +1223,6 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var isImage = attachment.IsImage; if (isImage && ChatAttachmentPreviewResolver.TryGetBytes( attachment, - OpenClawChatDataProvider.ImagePreviewCache, out var bytes)) { var bmp = TryDecodeBitmap(bytes); diff --git a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs index 784678396..25de71d33 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs @@ -712,7 +712,6 @@ private static Element BuildAttachment(ChatAttachmentPresentation attachment) if (attachment.IsImage && ChatAttachmentPreviewResolver.TryGetBytes( attachment, - OpenClawChatDataProvider.ImagePreviewCache, out var bytes) && TryDecodeAttachmentBitmap(bytes) is { } bitmap) { @@ -785,16 +784,9 @@ private static Element BuildAttachment(ChatAttachmentPresentation attachment) try { - var stream = new global::Windows.Storage.Streams.InMemoryRandomAccessStream(); - using (var writer = new global::Windows.Storage.Streams.DataWriter(stream)) - { - writer.WriteBytes(bytes); - writer.StoreAsync().AsTask().GetAwaiter().GetResult(); - writer.DetachStream(); - } - stream.Seek(0); - var bitmap = new BitmapImage(); - bitmap.SetSource(stream); + var bitmap = ChatAttachmentBitmapDecoder.TryDecode(bytes); + if (bitmap is null) + return null; s_attachmentBitmaps.Add(bytes, bitmap); return bitmap; } diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 19d30a26c..8ea435be0 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -254,6 +254,33 @@ public async Task ReconnectAuthorization_AllowsRuntimeV2SignatureUpgrade() Assert.Equal("paired-device-token", client.AssistantMediaAuthToken); } + [Fact] + public async Task ReconnectAuthorization_RejectsRuntimeV2SignatureDowngrade() + { + SetupGateway("gw-1", "wss://test"); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = true, + }); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = false, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.False(reconnect.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + } + [Fact] public async Task OperatorDeviceTokenReceived_RefreshesAssistantMediaAuthWithoutReconnect() { diff --git a/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs index 818b2bb5e..89985e896 100644 --- a/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs +++ b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs @@ -43,4 +43,42 @@ public void ImageLoader_DoesNotRenderStateFromPreviousReference() source, StringComparison.Ordinal); } + + [Theory] + [InlineData("ReactorChatTimeline.cs")] + [InlineData("OpenClawChatTimeline.cs")] + public void LocalAttachmentRenderer_UsesBoundedSharedDecoder(string fileName) + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + fileName)); + + Assert.Contains( + "ChatAttachmentBitmapDecoder.TryDecode(bytes)", + source, + StringComparison.Ordinal); + Assert.DoesNotContain("bitmap.SetSource(stream)", source, StringComparison.Ordinal); + Assert.DoesNotContain("bmp.SetSource(stream)", source, StringComparison.Ordinal); + } + + [Fact] + public void SharedAttachmentDecoder_EnforcesPixelPolicyBeforeSetSource() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + var policy = source.IndexOf( + "ChatAssistantImageDecodePolicy.TryGetDecodeSize(", + StringComparison.Ordinal); + var setSource = source.LastIndexOf("bitmap.SetSource(stream)", StringComparison.Ordinal); + + Assert.True(policy >= 0); + Assert.True(setSource > policy); + } } diff --git a/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs index 1811447c0..02b0878a6 100644 --- a/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs +++ b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs @@ -211,6 +211,42 @@ public void RemoteSameFilename_CannotResolveLocalPreviewBytes() Assert.False(ChatAttachmentPreviewResolver.TryGetBytes(remote, cache, out _)); } + [Fact] + public void PreviewCache_RejectsDecodedOverflowBeforeStorage() + { + Assert.False(ChatImagePreviewCache.TryDecodeBoundedBase64( + Convert.ToBase64String([1, 2, 3, 4, 5]), + maximumBytes: 4, + out var bytes)); + Assert.Empty(bytes); + } + + [Fact] + public void PreviewCache_EvictsOldestEntriesWithinCountBound() + { + ChatImagePreviewCache.Clear(); + try + { + for (var index = 0; index <= ChatImagePreviewCache.MaximumEntries; index++) + { + Assert.True(ChatImagePreviewCache.TryStoreBase64( + $"preview-{index}", + Convert.ToBase64String([(byte)index]))); + } + + Assert.Equal(ChatImagePreviewCache.MaximumEntries, ChatImagePreviewCache.Count); + Assert.False(ChatImagePreviewCache.Contains("preview-0")); + Assert.True(ChatImagePreviewCache.Contains( + $"preview-{ChatImagePreviewCache.MaximumEntries}")); + Assert.True(ChatImagePreviewCache.TotalBytes <= + ChatImagePreviewCache.MaximumTotalBytes); + } + finally + { + ChatImagePreviewCache.Clear(); + } + } + [Fact] public void AttachmentOnlyEcho_AmbiguousMatchingCandidatesAreNotConsumed() { diff --git a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs index 1483c1749..90666ab59 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs +++ b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs @@ -10960,7 +10960,7 @@ public async Task ChatMessageReceived_LocalMediaEcho_ReconcilesOneCleanRowAndPre "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", attachment.DisplayFileName); Assert.True(attachment.CanAccessPreviewCache); - Assert.True(OpenClawChatDataProvider.ImagePreviewCache.ContainsKey(attachment.PreviewCacheKey!)); + Assert.True(ChatImagePreviewCache.Contains(attachment.PreviewCacheKey!)); sendGate.SetResult(); }