diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 0bc8d884e..97686b55a 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -206,6 +206,12 @@ leading and trailing pipe. Columns, in order: | reactor-chat-timeline | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs | production chat message virtualization, row realization, and imperative scroll follow | ReactorChatTimeline through OpenClawReactorChatRoot and ReactorHostControl | OpenClawChatTimeline remains a legacy focused-test surface while its runtime route is migrated | the default chat route mounts one direct ReactorHostControl per XAML chat target; Reactor owns stable-key ItemsView and ItemContainer realization without a custom native list, collection reconciler, or scroll-layout mutation | review-only: user explicitly deferred new tests for this migration; required build and existing shared/tray suites still run | review-only | when Reactor timeline proof coverage replaces the legacy focused UI host coverage | | chat-tool-activity-renderer | authoritative | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | production standalone tool-call and grouped activity presentation, summaries, disclosures, and detail rendering | ChatToolActivityPresentation + ToolCallCardRenderer | ReactorChatTimeline projects rows and delegates realization only | consecutive invocation grouping preserves source chronology; stable group identity comes from session, generation, and first tool entry; selectable output remains capped at 240px | ChatToolActivityPresentationTests.Project_GroupsOnlyConsecutiveSpansOfAtLeastTwoTools | behavioral | - | | chat-history-replay-projection | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | array-valued history content ordering projection | ChatHistoryReplayProjection | provider applies projected text and tool parts to the reducer | interleaved text, calls, and results replay in source order without clearing active tool correlation | OpenClawChatDataProviderTests.LoadHistoryAsync_InterleavedContentParts_PreserveChronologyAndCorrelation | behavioral | - | +| assistant-media-protocol-projection | authoritative | src/OpenClaw.Shared/OpenClawGatewayClient.cs | structured assistant media content parsing and assistant-only legacy MEDIA directive redaction/projection | AssistantMediaDirectiveParser + ChatMediaContentInfo | gateway client preserves ordered typed media while tray presentation receives only safe filenames and metadata | user text never activates media directives; accepted local sources never enter visible assistant text or notifications; media-only messages survive live and history parsing | AssistantMediaDirectiveParserTests.Project_AssistantAbsolutePath_ProducesMediaWithoutExposingPath | behavioral | - | +| assistant-media-resolver | authoritative | src/OpenClaw.Shared/OpenClawGatewayClient.cs | authenticated structured artifact and legacy assistant-media byte retrieval | OpenClawGatewayClient.AssistantMedia | chat bridge exposes only lease-bound typed resolution results; renderer never receives credentials, tickets, or arbitrary URLs | accepts only current-connection results, matching media MIME families, managed ticket paths, and payloads within the 12 MiB image or 16 MiB playback caps | OpenClawGatewayClientAssistantMediaTests.ResolveLegacyMedia_UsesBearerMetadataAndSourceBoundTicket | behavioral | - | +| provider-assistant-media-parsing-closed | closed | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | parsing legacy MEDIA directives or structured Gateway media blocks | AssistantMediaDirectiveParser + OpenClawGatewayClient | provider owns message identity, streaming reconciliation, timeline application, and safe presentation metadata orchestration | provider consumes typed content parts and never reparses model text or exposes raw media sources | ChatAssistantContentPresentationTests.Project_UsesSafeFilenameWithoutExposingLegacySource | behavioral | when assistant message ingestion leaves OpenClawChatDataProvider | +| assistant-media-renderer | authoritative | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | assistant media card presentation, bounded image decode, retry, and row cancellation | ChatAssistantMediaRenderer | ReactorChatTimeline owns row placement and delegates media realization | at most four images render inline per message; unsupported or unresolved typed media remains visible as an accessible safe unavailable card; raw Gateway sources are never rendered | ChatAssistantContentPresentationTests.BuildRenderPlan_CapsImagesWithoutReorderingOtherMedia | behavioral | - | +| gateway-media-message-projection | authoritative | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | gateway media-envelope parsing, safe filename/MIME normalization, attachment signatures, and provenance-safe attachment descriptors | GatewayMediaMessageProjection + ChatAttachmentPresentation | provider applies the projection to live, reset, backfill, and history ingress and owns stateful echo correlation | gateway text never becomes a private marker; gateway descriptors have no preview key; only local opaque preview keys can access image bytes | GatewayMediaMessageProjectionTests.ValidEnvelope_ProjectsSafeDescriptorAndCleanProse | behavioral | - | +| provider-gateway-media-parsing-closed | closed | src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs | private gateway media-envelope parsing or descriptor construction | GatewayMediaMessageProjection | provider retains stateful pending-echo queues, reset gates, sidecar matching, and reducer application | all user ingress paths call the focused projection and do not independently parse gateway media text | review-only | review-only | when user-message ingestion leaves OpenClawChatDataProvider | | reactor-tool-rendering-closed | closed | src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs | per-tool and grouped activity summary/detail rendering implementation | ToolCallCardRenderer | row projection, virtualization, hover state, assistant runs, and renderer delegation only | ReactorChatTimeline contains no tool detail renderer and delegates both standalone and grouped tool rows | ChatTimelinePresentationTests.ReactorTimeline_DelegatesToolAndActivityRenderingToFocusedOwner | source-shape | when ReactorChatTimeline is replaced as the production virtualization owner | | functional-chat-default-mount | closed | src/OpenClaw.Tray.WinUI/Chat/FunctionalChatHostExtensions.cs | mounting the FunctionalUI chat tree as the default ChatPage or ChatWindow surface | ReactorChatHostExtensions and OpenClawReactorChatRoot | legacy FunctionalUI chat files may remain for focused compatibility coverage only | ChatPage and ChatWindow mount the Reactor root directly into their existing ChatHost Borders; no FunctionalUI component mounts or nests Reactor on the default path | review-only: user explicitly deferred new tests for this migration; required build and existing shared/tray suites still run | review-only | when legacy FunctionalUI chat surfaces are removed | | settings-store | authoritative | settings and permission UI surfaces | direct SettingsManager mutation and blanket self-write suppression | ISettingsStore | non-permission legacy surfaces may read SettingsManager until migrated; direct saves publish origin null | every save publishes one versioned event; only the matching writer ignores its own origin while all other active consumers refresh | SettingsSharedStateContractTests.TwoActiveSettingsPageViewModels_IgnoreOnlyOwnWrites_InBothDirections | behavioral | when every settings surface reads and writes through ISettingsStore | diff --git a/src/OpenClaw.Connection/GatewayClientFactory.cs b/src/OpenClaw.Connection/GatewayClientFactory.cs index 983884f44..f87739ceb 100644 --- a/src/OpenClaw.Connection/GatewayClientFactory.cs +++ b/src/OpenClaw.Connection/GatewayClientFactory.cs @@ -21,7 +21,8 @@ public IGatewayClientLifecycle Create( tokenIsBootstrapToken: credential.IsBootstrapToken, bootstrapPairAsNode: false, identityPath: identityPath, - ignoreStoredDeviceToken: credential.IsBootstrapToken); + ignoreStoredDeviceToken: credential.IsBootstrapToken, + assistantMediaAuthToken: credential.InteractiveHttpToken); return new GatewayClientLifecycleAdapter(client); } diff --git a/src/OpenClaw.Connection/GatewayConnectionManager.cs b/src/OpenClaw.Connection/GatewayConnectionManager.cs index 533b227d2..c53823e54 100644 --- a/src/OpenClaw.Connection/GatewayConnectionManager.cs +++ b/src/OpenClaw.Connection/GatewayConnectionManager.cs @@ -526,7 +526,52 @@ await StopOwnedTunnelAfterFailedConnectionAsync( IGatewayClientLifecycle lifecycle; try { - lifecycle = _clientFactory.Create(connectUrl, credential, perGatewayIdentityDir, diagLogger); + var httpCredential = + InteractiveGatewayCredentialResolver.ResolveForAssistantMediaHttpSurface( + record, + credential); + var interactiveHttpToken = string.Empty; + if (httpCredential is not null) + { + var httpAuthorization = await AuthorizeCredentialForEndpointAsync( + record, + httpCredential, + _operationCts!.Token) + .ConfigureAwait(false); + if (_disposed || + Interlocked.Read(ref _generation) != gen || + _operationCts?.IsCancellationRequested != false) + { + return; + } + if (httpAuthorization.Allowed) + { + interactiveHttpToken = httpCredential.Token; + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was withheld", + httpAuthorization.Detail); + } + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was unavailable"); + } + + var clientCredential = credential with + { + InteractiveHttpToken = interactiveHttpToken, + }; + lifecycle = _clientFactory.Create( + connectUrl, + clientCredential, + perGatewayIdentityDir, + diagLogger); } catch (DeviceIdentityLoadException ex) { @@ -557,6 +602,11 @@ await StopOwnedTunnelAfterFailedConnectionAsync( async Task AuthorizeLiveCredentialHandoffAsync( CancellationToken cancellationToken) { + // Clear the assistant-media HTTP credential up front so any + // in-flight media request never outlives this handoff attempt; + // it is only restored below once the fresh credential (or its + // interactive HTTP fallback) is re-authorized for this generation. + lifecycle.DataClient.SetAssistantMediaAuthToken(null); var authorization = await AuthorizeCredentialHandoffAsync( record, credential, @@ -578,6 +628,58 @@ await RecordOperatorCredentialHandoffFailureAsync( record.Id) .ConfigureAwait(false); } + + if (authorization.Allowed && + IsCurrentGatewayAttempt(gen, record.Id) && + IsAutomaticReconnectAllowed(record.Id)) + { + var currentRecord = _registry.GetById(record.Id); + if (currentRecord is not null) + { + GatewayCredential? currentHttpFallback = null; + if (string.IsNullOrWhiteSpace(currentRecord.SharedGatewayToken)) + { + currentHttpFallback = _credentialResolver.ResolveOperator( + currentRecord, + perGatewayIdentityDir); + } + var reconnectHttpCredential = + InteractiveGatewayCredentialResolver.ResolveForAssistantMediaHttpSurface( + currentRecord, + currentHttpFallback); + if (reconnectHttpCredential is null) + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was unavailable during reconnect"); + } + else + { + var httpAuthorization = await AuthorizeCredentialForEndpointAsync( + currentRecord, + reconnectHttpCredential, + cancellationToken) + .ConfigureAwait(false); + if (IsCurrentGatewayAttempt(gen, record.Id) && + IsAutomaticReconnectAllowed(record.Id)) + { + if (httpAuthorization.Allowed) + { + lifecycle.DataClient.SetAssistantMediaAuthToken( + reconnectHttpCredential.Token); + } + else + { + _diagnostics.Record( + "credentials", + "Interactive HTTP credential was withheld during reconnect", + httpAuthorization.Detail); + } + } + } + } + } + return authorization; } @@ -2463,7 +2565,7 @@ private static bool IsSameCredentialHandoffRecord( expected.BootstrapToken, StringComparison.Ordinal) && current.IsLocal == expected.IsLocal && - current.RequiresV2Signature == expected.RequiresV2Signature && + (current.RequiresV2Signature || !expected.RequiresV2Signature) && string.Equals( current.SetupManagedDistroName, expected.SetupManagedDistroName, @@ -2549,23 +2651,66 @@ private async Task HandleDeviceTokenReceivedAsync( identityPath, token, CancellationToken.None).ConfigureAwait(false); - if (result.Outcome != DeviceTokenHandlingOutcome.IdentityLoadFailure) + if (result.Outcome != DeviceTokenHandlingOutcome.IdentityLoadFailure && + result.Outcome != DeviceTokenHandlingOutcome.Stored) + { return; + } + var gatewayRecordId = attempt.GatewayRecordId ?? string.Empty; await _transitionSemaphore.WaitAsync().ConfigureAwait(false); try { - if (!IsCurrentGatewayAttempt( - attempt.LifecycleGeneration, - attempt.GatewayRecordId ?? string.Empty)) + if (!IsCurrentGatewayAttempt(attempt.LifecycleGeneration, gatewayRecordId)) { return; } - _stateMachine.TryTransition( - ConnectionTrigger.WebSocketError, - DeviceIdentityLoadException.RecoveryMessage); - EmitStateChanged(); + if (result.Outcome == DeviceTokenHandlingOutcome.IdentityLoadFailure) + { + _stateMachine.TryTransition( + ConnectionTrigger.WebSocketError, + DeviceIdentityLoadException.RecoveryMessage); + EmitStateChanged(); + return; + } + + // Stored: refresh the assistant-media HTTP credential bound to the + // fresh operator device token when this connection relies on a + // device token rather than a shared gateway token. + if (!string.Equals(token.Role, "operator", StringComparison.OrdinalIgnoreCase)) + return; + + var currentRecord = _registry.GetById(gatewayRecordId); + if (currentRecord is null || + !string.IsNullOrWhiteSpace(currentRecord.SharedGatewayToken)) + { + return; + } + + var deviceCredential = new GatewayCredential( + token.Token, + IsBootstrapToken: false, + CredentialResolver.SourceDeviceToken); + var authorization = await AuthorizeCredentialForEndpointAsync( + currentRecord, + deviceCredential, + _operationCts?.Token ?? CancellationToken.None) + .ConfigureAwait(false); + if (!IsCurrentGatewayAttempt(attempt.LifecycleGeneration, gatewayRecordId)) + return; + if (authorization.Allowed) + { + _activeLifecycle?.DataClient.SetAssistantMediaAuthToken(token.Token); + } + else + { + _activeLifecycle?.DataClient.SetAssistantMediaAuthToken(null); + _diagnostics.Record( + "credentials", + "Interactive HTTP device credential was withheld after token refresh", + authorization.Detail); + } } finally { diff --git a/src/OpenClaw.Connection/ICredentialResolver.cs b/src/OpenClaw.Connection/ICredentialResolver.cs index a2c045792..92fc9ef29 100644 --- a/src/OpenClaw.Connection/ICredentialResolver.cs +++ b/src/OpenClaw.Connection/ICredentialResolver.cs @@ -11,6 +11,7 @@ public sealed record GatewayCredential(string Token, bool IsBootstrapToken, stri GatewayCredentialResolutionStatus.Resolved; public bool FallbackUsed { get; init; } public string? ResolutionDetail { get; init; } + public string? InteractiveHttpToken { get; init; } } /// diff --git a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs index fd0058153..7f588f421 100644 --- a/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs +++ b/src/OpenClaw.Connection/InteractiveGatewayCredentialResolver.cs @@ -10,6 +10,22 @@ namespace OpenClaw.Connection; /// public static class InteractiveGatewayCredentialResolver { + internal static GatewayCredential? ResolveForAssistantMediaHttpSurface( + GatewayRecord record, + GatewayCredential? fallback) => + !string.IsNullOrWhiteSpace(record.SharedGatewayToken) + ? new GatewayCredential( + record.SharedGatewayToken!, + IsBootstrapToken: false, + CredentialResolver.SourceSharedGatewayToken) + : fallback is + { + IsBootstrapToken: false, + Source: CredentialResolver.SourceDeviceToken, + } + ? fallback + : null; + public static bool TryResolve( GatewayRegistry? registry, string settingsDirectory, diff --git a/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs b/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs new file mode 100644 index 000000000..2be240799 --- /dev/null +++ b/src/OpenClaw.Shared/AssistantMediaDirectiveParser.cs @@ -0,0 +1,454 @@ +using System.Net; +using System.Text; +using System.Text.RegularExpressions; + +namespace OpenClaw.Shared; + +internal sealed record AssistantMediaDirectiveProjection( + string Text, + IReadOnlyList ContentParts, + bool HasDirective); + +/// +/// Projects assistant-only legacy MEDIA directives into typed transport +/// references. This is deliberately shared by history and live parsing so raw +/// Gateway paths cannot leak through notification or presentation text. +/// +internal static class AssistantMediaDirectiveParser +{ + private const int MaxSourceLength = 4096; + private const int MaxFileNameLength = 255; + internal const int MaxMediaReferences = 16; + + private static readonly Regex s_hasFileExtension = + new(@"\.\w{1,10}$", RegexOptions.CultureInvariant); + private static readonly Regex s_traversalSegment = + new(@"(?:^|[/\\])\.\.(?:[/\\]|$)", RegexOptions.CultureInvariant); + private static readonly Regex s_windowsDrive = + new(@"^[a-zA-Z]:[\\/]", RegexOptions.CultureInvariant); + private static readonly Regex s_scheme = + new(@"^[a-zA-Z][a-zA-Z0-9+.-]*:", RegexOptions.CultureInvariant); + + public static AssistantMediaDirectiveProjection Project(string? role, string? raw) + { + var text = raw ?? string.Empty; + if (!string.Equals(role, "assistant", StringComparison.OrdinalIgnoreCase) + || text.IndexOf("MEDIA:", StringComparison.OrdinalIgnoreCase) < 0) + { + return new(text, TextOnly(text), HasDirective: false); + } + + var trimmedRaw = text.TrimEnd(); + if (string.IsNullOrWhiteSpace(trimmedRaw)) + return new(string.Empty, Array.Empty(), HasDirective: false); + + var keptLines = new List(); + var parts = new List(); + var inFence = false; + var fenceCharacter = '\0'; + var fenceLength = 0; + var foundDirective = false; + var mediaReferenceCount = 0; + + foreach (var line in trimmedRaw.Split('\n')) + { + var lineWithoutCarriageReturn = line.EndsWith('\r') ? line[..^1] : line; + if (TryReadFence(lineWithoutCarriageReturn, out var currentFence, out var currentLength)) + { + if (!inFence) + { + inFence = true; + fenceCharacter = currentFence; + fenceLength = currentLength; + } + else if (currentFence == fenceCharacter && currentLength >= fenceLength) + { + inFence = false; + } + + KeepTextLine(lineWithoutCarriageReturn, keptLines, parts); + continue; + } + + var trimmedStart = lineWithoutCarriageReturn.TrimStart(); + if (inFence || !trimmedStart.StartsWith("MEDIA:", StringComparison.OrdinalIgnoreCase)) + { + KeepTextLine(lineWithoutCarriageReturn, keptLines, parts); + continue; + } + + foundDirective = true; + var payload = trimmedStart["MEDIA:".Length..]; + var projection = ParsePayload( + payload, + Math.Max(0, MaxMediaReferences - mediaReferenceCount)); + if (projection.Media.Count > 0) + { + foreach (var media in projection.Media) + { + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }); + } + mediaReferenceCount += projection.Media.Count; + } + + if (!string.IsNullOrWhiteSpace(projection.ResidualText)) + { + var residual = projection.ResidualText.Trim(); + keptLines.Add(residual); + AppendTextPart(parts, residual); + } + else if (!projection.StripLine) + { + keptLines.Add(lineWithoutCarriageReturn); + AppendTextPart(parts, lineWithoutCarriageReturn); + } + } + + var visibleText = string.Join('\n', keptLines); + visibleText = Regex.Replace(visibleText, @"^(?:[ \t]*\n)+", string.Empty).TrimEnd(); + return new(visibleText, parts, foundDirective); + } + + private static IReadOnlyList TextOnly(string text) => + string.IsNullOrEmpty(text) + ? Array.Empty() + : new[] + { + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = text, + }, + }; + + private static void KeepTextLine( + string line, + List keptLines, + List parts) + { + keptLines.Add(line); + AppendTextPart(parts, line); + } + + private static void AppendTextPart(List parts, string text) + { + if (parts.Count > 0 && parts[^1].Kind == ChatMessageContentPartKind.Text) + { + parts[^1].Text = $"{parts[^1].Text}\n{(text.Trim().Length > 0 ? text : string.Empty)}"; + return; + } + + if (text.Trim().Length == 0) + return; + + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = text, + }); + } + + private static bool TryReadFence(string line, out char fenceCharacter, out int fenceLength) + { + fenceCharacter = '\0'; + fenceLength = 0; + var index = 0; + while (index < line.Length && index < 3 && line[index] == ' ') + index++; + if (index >= line.Length || line[index] is not ('`' or '~')) + return false; + + fenceCharacter = line[index]; + while (index + fenceLength < line.Length && line[index + fenceLength] == fenceCharacter) + fenceLength++; + return fenceLength >= 3; + } + + private static PayloadProjection ParsePayload(string rawPayload, int remainingReferences) + { + var unwrapped = UnwrapQuoted(rawPayload); + var payload = unwrapped ?? rawPayload; + var candidates = unwrapped is not null + ? new[] { unwrapped } + : SplitUnquotedParts(rawPayload); + var media = new List(); + var invalidParts = new List(); + + foreach (var part in candidates) + { + var candidate = NormalizeSource(CleanCandidate(part)); + if (IsValidSource(candidate, allowSpaces: unwrapped is not null || part.Any(char.IsWhiteSpace))) + { + if (media.Count < remainingReferences) + media.Add(CreateLegacyMedia(candidate)); + } + else if (!part.Any(char.IsWhiteSpace) || !HasTraversalOrUnsupportedHomePrefix(candidate)) + { + invalidParts.Add(part); + } + } + + var payloadValue = payload.Trim(); + var looksLocal = LooksLikeLocalPath(payloadValue) + || rawPayload.TrimStart().StartsWith("file://", StringComparison.OrdinalIgnoreCase); + + if (media.Count == 0 && payloadValue.Any(char.IsWhiteSpace)) + { + var fallback = NormalizeSource(CleanCandidate(payloadValue)); + if (IsValidSource(fallback, allowSpaces: true, allowBareFileName: true)) + { + if (remainingReferences > 0) + media.Add(CreateLegacyMedia(fallback)); + invalidParts.Clear(); + } + } + + if (media.Count == 0) + { + var fallback = NormalizeSource(CleanCandidate(payloadValue)); + if (IsValidSource(fallback, allowSpaces: true, allowBareFileName: true)) + { + if (remainingReferences > 0) + media.Add(CreateLegacyMedia(fallback)); + invalidParts.Clear(); + } + } + + if (media.Count > 0) + return new(media, CleanLineText(string.Join(' ', invalidParts)), StripLine: true); + + if (looksLocal) + { + if (remainingReferences > 0) + { + media.Add(new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Unknown, + Source = ChatMediaContentSource.Unavailable, + }); + } + return new(media, string.Empty, StripLine: true); + } + + return new(media, string.Empty, StripLine: false); + } + + private static IReadOnlyList SplitUnquotedParts(string payload) + { + var matches = Regex.Matches(payload, @"\S+"); + var parts = new List(matches.Count); + var previousEnd = 0; + foreach (Match match in matches) + { + var candidate = NormalizeSource(CleanCandidate(match.Value)); + var previous = parts.Count > 0 ? parts[^1] : null; + var previousCandidate = previous is null + ? string.Empty + : NormalizeSource(CleanCandidate(previous)); + if (previous is not null + && BeginsRootedSource(previousCandidate) + && !BeginsIndependentSource(candidate) + && (!s_hasFileExtension.IsMatch(previousCandidate) || !IsValidSource(candidate))) + { + parts[^1] = $"{previous}{payload[previousEnd..match.Index]}{match.Value}"; + } + else + { + parts.Add(match.Value); + } + + previousEnd = match.Index + match.Length; + } + + return parts; + } + + private static string? UnwrapQuoted(string value) + { + var trimmed = value.Trim(); + if (trimmed.Length < 2 || trimmed[0] != trimmed[^1] || trimmed[0] is not ('"' or '\'' or '`')) + return null; + return trimmed[1..^1].Trim(); + } + + private static string NormalizeSource(string source) => + source.StartsWith("file://", StringComparison.OrdinalIgnoreCase) + ? source["file://".Length..] + : source; + + private static string CleanCandidate(string raw) + { + var stripped = raw.TrimStart('`', '"', '\'', '[', '{', '(') + .TrimEnd('`', '"', '\'', '\\', '}', ')', ']', ','); + var quoteAfterExtension = Regex.Match( + stripped, + @"^(.*\.\w{1,10})\\?""(?=[\]},:]|$).*", + RegexOptions.Singleline | RegexOptions.CultureInvariant); + return quoteAfterExtension.Success ? quoteAfterExtension.Groups[1].Value : stripped; + } + + private static bool IsValidSource( + string candidate, + bool allowSpaces = false, + bool allowBareFileName = false) + { + if (string.IsNullOrEmpty(candidate) + || candidate.Length > MaxSourceLength + || (!allowSpaces && candidate.Any(char.IsWhiteSpace))) + { + return false; + } + + if (Uri.TryCreate(candidate, UriKind.Absolute, out var uri) + && uri.Scheme.Equals(Uri.UriSchemeHttps, StringComparison.OrdinalIgnoreCase)) + { + return string.IsNullOrEmpty(uri.UserInfo) && IsAllowedRemoteHost(uri.Host); + } + + if (IsLikelyLocalPath(candidate)) + return true; + + return allowBareFileName + && !s_scheme.IsMatch(candidate) + && s_hasFileExtension.IsMatch(candidate) + && !HasTraversalOrUnsupportedHomePrefix(candidate); + } + + private static bool IsAllowedRemoteHost(string host) + { + var normalized = host.Trim().Trim('[', ']').TrimEnd('.').ToLowerInvariant(); + if (string.IsNullOrEmpty(normalized) + || !normalized.Contains('.') + || normalized is "localhost" or "localhost.localdomain" or "metadata.google.internal" + || normalized.EndsWith(".localhost", StringComparison.Ordinal) + || normalized.EndsWith(".local", StringComparison.Ordinal) + || normalized.EndsWith(".internal", StringComparison.Ordinal)) + { + return false; + } + + if (!IPAddress.TryParse(normalized, out var address)) + return true; + + if (IPAddress.IsLoopback(address) || address.IsIPv6LinkLocal || address.IsIPv6SiteLocal) + return false; + if (address.IsIPv4MappedToIPv6) + address = address.MapToIPv4(); + var bytes = address.GetAddressBytes(); + return address.AddressFamily != System.Net.Sockets.AddressFamily.InterNetwork + || !(bytes[0] == 10 + || bytes[0] == 127 + || (bytes[0] == 169 && bytes[1] == 254) + || (bytes[0] == 172 && bytes[1] is >= 16 and <= 31) + || (bytes[0] == 192 && bytes[1] == 168) + || bytes[0] >= 224); + } + + private static bool IsLikelyLocalPath(string candidate) => + !HasTraversalOrUnsupportedHomePrefix(candidate) + && (candidate.StartsWith('/') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("~/", StringComparison.Ordinal) + || candidate.StartsWith("~\\", StringComparison.Ordinal) + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal) + || (!s_scheme.IsMatch(candidate) + && (candidate.Contains('/') || candidate.Contains('\\')))); + + private static bool LooksLikeLocalPath(string candidate) => + candidate.StartsWith('/') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("../", StringComparison.Ordinal) + || candidate.StartsWith('~') + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal) + || (!s_scheme.IsMatch(candidate) + && (candidate.Contains('/') || candidate.Contains('\\'))); + + private static bool HasTraversalOrUnsupportedHomePrefix(string candidate) => + candidate.StartsWith("../", StringComparison.Ordinal) + || candidate == ".." + || (candidate.StartsWith('~') + && !candidate.StartsWith("~/", StringComparison.Ordinal) + && !candidate.StartsWith("~\\", StringComparison.Ordinal)) + || s_traversalSegment.IsMatch(candidate); + + private static bool BeginsRootedSource(string candidate) => + candidate.StartsWith('/') + || candidate.StartsWith('~') + || candidate.StartsWith("./", StringComparison.Ordinal) + || candidate.StartsWith("../", StringComparison.Ordinal) + || s_windowsDrive.IsMatch(candidate) + || candidate.StartsWith(@"\\", StringComparison.Ordinal); + + private static bool BeginsIndependentSource(string candidate) => + BeginsRootedSource(candidate) || s_scheme.IsMatch(candidate); + + private static ChatMediaContentInfo CreateLegacyMedia(string source) + { + var fileName = SafeFileName(source); + return new ChatMediaContentInfo + { + Kind = ClassifyByFileName(fileName), + Source = ChatMediaContentSource.LegacyDirective, + FileName = fileName, + GatewaySource = source, + }; + } + + internal static string? SafeFileName(string source) + { + var path = source; + if (Uri.TryCreate(source, UriKind.Absolute, out var uri) && uri.Scheme == Uri.UriSchemeHttps) + path = Uri.UnescapeDataString(uri.AbsolutePath); + path = path.Replace('\\', '/'); + var leaf = path[(path.LastIndexOf('/') + 1)..]; + try + { + leaf = Uri.UnescapeDataString(leaf); + } + catch (UriFormatException) + { + return null; + } + + var builder = new StringBuilder(Math.Min(leaf.Length, MaxFileNameLength)); + foreach (var character in leaf) + { + if (!char.IsControl(character) && character is not '\r' and not '\n') + builder.Append(character); + if (builder.Length == MaxFileNameLength) + break; + } + + var result = builder.ToString().Trim(); + return result.Length == 0 ? null : result; + } + + private static ChatMediaContentKind ClassifyByFileName(string? fileName) + { + var extension = Path.GetExtension(fileName ?? string.Empty).ToLowerInvariant(); + return extension switch + { + ".png" or ".jpg" or ".jpeg" or ".gif" or ".webp" or ".bmp" or ".heic" or ".avif" + => ChatMediaContentKind.Image, + ".mp3" or ".wav" or ".m4a" or ".aac" or ".ogg" or ".flac" + => ChatMediaContentKind.Audio, + ".mp4" or ".mov" or ".m4v" or ".webm" or ".avi" + => ChatMediaContentKind.Video, + _ => ChatMediaContentKind.File, + }; + } + + private static string CleanLineText(string text) => + Regex.Replace(text, @"[ \t]{2,}", " ").Trim(); + + private sealed record PayloadProjection( + IReadOnlyList Media, + string ResidualText, + bool StripLine); +} diff --git a/src/OpenClaw.Shared/AssistantMediaResolution.cs b/src/OpenClaw.Shared/AssistantMediaResolution.cs new file mode 100644 index 000000000..88296ad0f --- /dev/null +++ b/src/OpenClaw.Shared/AssistantMediaResolution.cs @@ -0,0 +1,25 @@ +namespace OpenClaw.Shared; + +public enum AssistantMediaResolutionStatus +{ + Ready, + Preparing, + Unavailable, +} + +public sealed record AssistantMediaResolutionResult( + AssistantMediaResolutionStatus Status, + byte[]? Data = null, + string? MimeType = null) +{ + public static AssistantMediaResolutionResult Preparing { get; } = + new(AssistantMediaResolutionStatus.Preparing); + + public static AssistantMediaResolutionResult Unavailable { get; } = + new(AssistantMediaResolutionStatus.Unavailable); +} + +internal readonly record struct GatewayConnectionLease( + Guid ClientId, + long Generation, + Uri HttpBaseUri); diff --git a/src/OpenClaw.Shared/Models.cs b/src/OpenClaw.Shared/Models.cs index e006151aa..b5b65a1eb 100644 --- a/src/OpenClaw.Shared/Models.cs +++ b/src/OpenClaw.Shared/Models.cs @@ -1809,7 +1809,7 @@ public static bool IsSilentAssistantDirective(string? role, string? text) => Array.Empty(); /// - /// Ordered text and tool blocks retained from array-valued message content. + /// Ordered text, tool, and media blocks retained from message content. /// Flat and remain populated for /// compatibility with consumers that do not need block-level chronology. /// @@ -1893,6 +1893,7 @@ public enum ChatMessageContentPartKind { Text, Tool, + Media, } public class ChatMessageContentPartInfo @@ -1900,6 +1901,55 @@ public class ChatMessageContentPartInfo public ChatMessageContentPartKind Kind { get; set; } public string? Text { get; set; } public ChatToolContentInfo? Tool { get; set; } + public ChatMediaContentInfo? Media { get; set; } +} + +public enum ChatMediaContentKind +{ + Image, + Audio, + Video, + File, + Unknown, +} + +public enum ChatMediaPlaybackMode +{ + Native, + Transcode, +} + +public enum ChatMediaContentSource +{ + Structured, + LegacyDirective, + Unavailable, +} + +/// +/// Typed assistant media metadata retained from the Gateway protocol. Legacy +/// Gateway sources are transient transport references and must never be +/// displayed, logged, persisted, or passed to local file APIs. +/// +public sealed class ChatMediaContentInfo +{ + public ChatMediaContentKind Kind { get; set; } + public ChatMediaContentSource Source { get; set; } + public string? Type { get; set; } + public string? MimeType { get; set; } + public string? FileName { get; set; } + public string? ArtifactId { get; set; } + public string? AgentId { get; set; } + public string? Url { get; set; } + public string? OpenUrl { get; set; } + public string? Alt { get; set; } + public int? Width { get; set; } + public int? Height { get; set; } + public long? SizeBytes { get; set; } + public double? DurationSeconds { get; set; } + public ChatMediaPlaybackMode? Playback { get; set; } + + internal string? GatewaySource { get; set; } } /// diff --git a/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs b/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs new file mode 100644 index 000000000..0f7425bde --- /dev/null +++ b/src/OpenClaw.Shared/OpenClawGatewayClient.AssistantMedia.cs @@ -0,0 +1,488 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; + +namespace OpenClaw.Shared; + +public partial class OpenClawGatewayClient +{ + internal const int MaximumAssistantImageBytes = 12 * 1024 * 1024; + internal const int MaximumAssistantPlaybackBytes = 16 * 1024 * 1024; + private const int MaximumAssistantMediaMetadataBytes = 64 * 1024; + private const string ManagedMediaPathPrefix = "/api/chat/media/outgoing/"; + private readonly Guid _mediaClientId = Guid.NewGuid(); + private readonly HttpClient _assistantMediaHttpClient; + + public async Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) + { + if (string.IsNullOrWhiteSpace(sessionKey) + || media.Kind is ChatMediaContentKind.File or ChatMediaContentKind.Unknown + || !TryCaptureMediaLease(out var lease)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + using var linkedCancellation = CancellationTokenSource.CreateLinkedTokenSource( + cancellationToken, + CancellationToken); + try + { + var result = media.Source switch + { + ChatMediaContentSource.Structured => + await ResolveStructuredMediaAsync( + lease, + sessionKey, + media, + linkedCancellation.Token).ConfigureAwait(false), + ChatMediaContentSource.LegacyDirective => + await ResolveLegacyMediaAsync( + lease, + media, + linkedCancellation.Token).ConfigureAwait(false), + _ => AssistantMediaResolutionResult.Unavailable, + }; + return IsCurrentMediaLease(lease) + ? result + : AssistantMediaResolutionResult.Unavailable; + } + catch (OperationCanceledException) when ( + cancellationToken.IsCancellationRequested || CancellationToken.IsCancellationRequested) + { + return AssistantMediaResolutionResult.Unavailable; + } + catch (Exception ex) + { + _logger.Warn($"Assistant media resolution failed ({ex.GetType().Name})."); + return AssistantMediaResolutionResult.Unavailable; + } + } + + private async Task ResolveStructuredMediaAsync( + GatewayConnectionLease lease, + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(media.ArtifactId)) + return AssistantMediaResolutionResult.Unavailable; + + var parameters = string.IsNullOrWhiteSpace(media.AgentId) + ? new { sessionKey, artifactId = media.ArtifactId } + : (object)new { sessionKey, artifactId = media.ArtifactId, agentId = media.AgentId }; + var payload = await SendWizardRequestAsync( + "artifacts.download", + parameters, + timeoutMs: 20000) + .WaitAsync(cancellationToken) + .ConfigureAwait(false); + + if (!IsCurrentMediaLease(lease) + || !TryReadArtifactMetadata(payload, media.Kind, out var mimeType, out var sizeBytes)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var maximumBytes = MaximumBytes(media.Kind); + if (sizeBytes is > 0 && sizeBytes > maximumBytes) + return AssistantMediaResolutionResult.Unavailable; + + if (payload.TryGetProperty("data", out var dataElement) + && dataElement.ValueKind == JsonValueKind.String) + { + if (!payload.TryGetProperty("encoding", out var encoding) + || encoding.ValueKind != JsonValueKind.String + || !string.Equals(encoding.GetString(), "base64", StringComparison.Ordinal)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + return TryDecodeBoundedBase64(dataElement.GetString(), maximumBytes, out var bytes) + ? new AssistantMediaResolutionResult( + AssistantMediaResolutionStatus.Ready, + bytes, + mimeType) + : AssistantMediaResolutionResult.Unavailable; + } + + if (!payload.TryGetProperty("url", out var urlElement) + || urlElement.ValueKind != JsonValueKind.String + || !TryResolveManagedMediaUri(lease.HttpBaseUri, urlElement.GetString(), out var mediaUri)) + { + return AssistantMediaResolutionResult.Unavailable; + } + + return await DownloadMediaBytesAsync( + lease, + mediaUri, + media.Kind, + maximumBytes, + cancellationToken).ConfigureAwait(false); + } + + private async Task ResolveLegacyMediaAsync( + GatewayConnectionLease lease, + ChatMediaContentInfo media, + CancellationToken cancellationToken) + { + if (string.IsNullOrWhiteSpace(media.GatewaySource)) + return AssistantMediaResolutionResult.Unavailable; + + var metadataUri = BuildLegacyMediaUri( + lease.HttpBaseUri, + media.GatewaySource, + mediaTicket: null, + metadata: true, + playback: false); + using var metadataRequest = CreateAuthenticatedMediaRequest(metadataUri, "application/json"); + using var metadataResponse = await _assistantMediaHttpClient.SendAsync( + metadataRequest, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + if (!metadataResponse.IsSuccessStatusCode) + return AssistantMediaResolutionResult.Unavailable; + + var metadataBytes = await ReadBoundedAsync( + metadataResponse.Content, + MaximumAssistantMediaMetadataBytes, + cancellationToken).ConfigureAwait(false); + if (metadataBytes is null || !IsCurrentMediaLease(lease)) + return AssistantMediaResolutionResult.Unavailable; + + using var metadataDocument = JsonDocument.Parse(metadataBytes); + var root = metadataDocument.RootElement; + if (!root.TryGetProperty("available", out var available) + || available.ValueKind != JsonValueKind.True + || !root.TryGetProperty("mediaTicket", out var ticketElement) + || ticketElement.ValueKind != JsonValueKind.String + || string.IsNullOrWhiteSpace(ticketElement.GetString())) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var maximumBytes = MaximumBytes(media.Kind); + if (root.TryGetProperty("sizeBytes", out var declaredSize) + && declaredSize.TryGetInt64(out var sizeBytes) + && sizeBytes > maximumBytes) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var playback = root.TryGetProperty("playback", out var playbackElement) + && playbackElement.ValueKind == JsonValueKind.String + && string.Equals(playbackElement.GetString(), "transcode", StringComparison.Ordinal); + var bytesUri = BuildLegacyMediaUri( + lease.HttpBaseUri, + media.GatewaySource, + ticketElement.GetString(), + metadata: false, + playback); + return await DownloadMediaBytesAsync( + lease, + bytesUri, + media.Kind, + maximumBytes, + cancellationToken).ConfigureAwait(false); + } + + private async Task DownloadMediaBytesAsync( + GatewayConnectionLease lease, + Uri uri, + ChatMediaContentKind kind, + int maximumBytes, + CancellationToken cancellationToken) + { + using var request = CreateAuthenticatedMediaRequest(uri, $"{MimePrefix(kind)}/*"); + using var response = await _assistantMediaHttpClient.SendAsync( + request, + HttpCompletionOption.ResponseHeadersRead, + cancellationToken).ConfigureAwait(false); + if (response.StatusCode == HttpStatusCode.Accepted) + return AssistantMediaResolutionResult.Preparing; + if (!response.IsSuccessStatusCode + || response.Content.Headers.ContentLength is > 0 + && response.Content.Headers.ContentLength > maximumBytes) + { + return AssistantMediaResolutionResult.Unavailable; + } + + var mimeType = response.Content.Headers.ContentType?.MediaType?.ToLowerInvariant(); + if (!MatchesKind(mimeType, kind)) + return AssistantMediaResolutionResult.Unavailable; + + var bytes = await ReadBoundedAsync( + response.Content, + maximumBytes, + cancellationToken).ConfigureAwait(false); + return bytes is not null && IsCurrentMediaLease(lease) + ? new AssistantMediaResolutionResult( + AssistantMediaResolutionStatus.Ready, + bytes, + mimeType) + : AssistantMediaResolutionResult.Unavailable; + } + + private HttpRequestMessage CreateAuthenticatedMediaRequest(Uri uri, string accept) + { + var request = new HttpRequestMessage(HttpMethod.Get, uri); + request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue(accept)); + var authToken = Volatile.Read(ref _assistantMediaAuthToken); + if (!string.IsNullOrWhiteSpace(authToken)) + request.Headers.Authorization = new AuthenticationHeaderValue( + "Bearer", + authToken); + return request; + } + + private bool TryCaptureMediaLease(out GatewayConnectionLease lease) + { + lease = default; + if (!IsConnectedToGateway || !TryBuildMediaHttpBaseUri(_currentGatewayUrl, out var baseUri)) + return false; + lease = new GatewayConnectionLease(_mediaClientId, ConnectionGeneration, baseUri); + return true; + } + + private bool IsCurrentMediaLease(GatewayConnectionLease lease) => + lease.ClientId == _mediaClientId + && lease.Generation == ConnectionGeneration + && IsConnectedToGateway; + + internal static bool TryBuildMediaHttpBaseUri(string gatewayUrl, out Uri baseUri) + { + baseUri = null!; + var normalized = GatewayUrlHelper.NormalizeForWebSocket(gatewayUrl); + if (!Uri.TryCreate(normalized, UriKind.Absolute, out var gatewayUri)) + return false; + var builder = new UriBuilder(gatewayUri) + { + Scheme = gatewayUri.Scheme.Equals("wss", StringComparison.OrdinalIgnoreCase) + ? Uri.UriSchemeHttps + : Uri.UriSchemeHttp, + UserName = string.Empty, + Password = string.Empty, + Query = string.Empty, + Fragment = string.Empty, + }; + baseUri = builder.Uri; + return true; + } + + internal static bool TryResolveManagedMediaUri( + Uri baseUri, + string? relativePath, + out Uri mediaUri) + { + mediaUri = null!; + if (string.IsNullOrWhiteSpace(relativePath) + || !relativePath.StartsWith(ManagedMediaPathPrefix, StringComparison.Ordinal) + || relativePath.StartsWith("//", StringComparison.Ordinal) + || relativePath.Contains('\\') + || HasTraversal(relativePath.Split('?', 2)[0])) + { + return false; + } + + if (!Uri.TryCreate(relativePath, UriKind.Relative, out var relative) + || !Uri.TryCreate(new Uri(baseUri.GetLeftPart(UriPartial.Authority)), relative, out var candidate) + || !string.Equals(candidate.Host, baseUri.Host, StringComparison.OrdinalIgnoreCase) + || candidate.Port != baseUri.Port + || !string.IsNullOrEmpty(candidate.Fragment) + || HasTraversal(candidate.AbsolutePath) + || !HasNonEmptyQueryValue(candidate.Query, "mediaTicket")) + { + return false; + } + + mediaUri = candidate; + return true; + } + + private static Uri BuildLegacyMediaUri( + Uri baseUri, + string source, + string? mediaTicket, + bool metadata, + bool playback) + { + var basePath = baseUri.AbsolutePath.TrimEnd('/'); + var route = $"{basePath}/__openclaw__/assistant-media"; + var query = new StringBuilder() + .Append("source=") + .Append(Uri.EscapeDataString(source)); + if (metadata) + query.Append("&meta=1"); + if (!string.IsNullOrWhiteSpace(mediaTicket)) + query.Append("&mediaTicket=").Append(Uri.EscapeDataString(mediaTicket)); + if (playback) + query.Append("&playback=1"); + return new UriBuilder(baseUri) { Path = route, Query = query.ToString() }.Uri; + } + + private static bool TryReadArtifactMetadata( + JsonElement payload, + ChatMediaContentKind expectedKind, + out string mimeType, + out long? sizeBytes) + { + mimeType = string.Empty; + sizeBytes = null; + if (!payload.TryGetProperty("artifact", out var artifact) + || artifact.ValueKind != JsonValueKind.Object + || !artifact.TryGetProperty("type", out var typeElement) + || typeElement.ValueKind != JsonValueKind.String + || !string.Equals( + typeElement.GetString(), + expectedKind.ToString(), + StringComparison.OrdinalIgnoreCase) + || !artifact.TryGetProperty("mimeType", out var mimeElement) + || mimeElement.ValueKind != JsonValueKind.String + || !MatchesKind(mimeElement.GetString(), expectedKind)) + { + return false; + } + + mimeType = mimeElement.GetString()!.Trim().ToLowerInvariant(); + if (artifact.TryGetProperty("sizeBytes", out var sizeElement) + && sizeElement.TryGetInt64(out var declaredSize)) + { + sizeBytes = declaredSize; + } + return true; + } + + private static async Task ReadBoundedAsync( + HttpContent content, + int maximumBytes, + CancellationToken cancellationToken) + { + await using var input = await content.ReadAsStreamAsync(cancellationToken).ConfigureAwait(false); + using var output = new MemoryStream(Math.Min(maximumBytes, 64 * 1024)); + var buffer = new byte[32 * 1024]; + while (true) + { + var read = await input.ReadAsync(buffer, cancellationToken).ConfigureAwait(false); + if (read == 0) + return output.ToArray(); + if (output.Length + read > maximumBytes) + return null; + output.Write(buffer, 0, read); + } + } + + internal static bool TryDecodeBoundedBase64(string? encoded, int maximumBytes, out byte[] bytes) + { + bytes = Array.Empty(); + if (string.IsNullOrEmpty(encoded) || maximumBytes <= 0) + return false; + + var maximumEncodedLength = ((maximumBytes + 2L) / 3L) * 4L; + var compactLength = 0; + foreach (var character in encoded) + { + if (char.IsWhiteSpace(character)) + continue; + compactLength++; + if (compactLength > maximumEncodedLength) + return false; + } + + if (compactLength == 0 || compactLength % 4 == 1) + return false; + + var paddedLength = compactLength + (4 - compactLength % 4) % 4; + if (paddedLength > maximumEncodedLength) + return false; + + var normalized = string.Create(paddedLength, encoded, static (destination, source) => + { + var index = 0; + foreach (var character in source) + { + if (char.IsWhiteSpace(character)) + continue; + destination[index++] = character switch + { + '-' => '+', + '_' => '/', + _ => character, + }; + } + destination[index..].Fill('='); + }); + + var padding = normalized[^1] == '=' + ? normalized.Length > 1 && normalized[^2] == '=' ? 2 : 1 + : 0; + var decodedLength = paddedLength / 4 * 3 - padding; + if (decodedLength > maximumBytes) + return false; + + var decoded = GC.AllocateUninitializedArray(decodedLength); + if (!Convert.TryFromBase64Chars(normalized, decoded, out var bytesWritten) + || bytesWritten != decodedLength) + { + return false; + } + + bytes = decoded; + return true; + } + + private static int MaximumBytes(ChatMediaContentKind kind) => + kind == ChatMediaContentKind.Image + ? MaximumAssistantImageBytes + : MaximumAssistantPlaybackBytes; + + private static string MimePrefix(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => "image", + ChatMediaContentKind.Audio => "audio", + ChatMediaContentKind.Video => "video", + _ => "application", + }; + + private static bool MatchesKind(string? mimeType, ChatMediaContentKind kind) => + !string.IsNullOrWhiteSpace(mimeType) + && mimeType.StartsWith($"{MimePrefix(kind)}/", StringComparison.OrdinalIgnoreCase); + + private static bool HasTraversal(string absolutePath) + { + string decoded; + try + { + decoded = Uri.UnescapeDataString(absolutePath); + } + catch (UriFormatException) + { + return true; + } + return decoded.Split('/', StringSplitOptions.RemoveEmptyEntries) + .Any(static segment => segment is "." or ".."); + } + + private static bool HasNonEmptyQueryValue(string query, string key) + { + foreach (var pair in query.TrimStart('?').Split('&', StringSplitOptions.RemoveEmptyEntries)) + { + var separator = pair.IndexOf('='); + if (separator <= 0) + continue; + if (string.Equals( + Uri.UnescapeDataString(pair[..separator]), + key, + StringComparison.Ordinal) + && separator + 1 < pair.Length) + { + return true; + } + + } + return false; + } + +} diff --git a/src/OpenClaw.Shared/OpenClawGatewayClient.cs b/src/OpenClaw.Shared/OpenClawGatewayClient.cs index b8e9f8450..3c33b66e6 100644 --- a/src/OpenClaw.Shared/OpenClawGatewayClient.cs +++ b/src/OpenClaw.Shared/OpenClawGatewayClient.cs @@ -70,6 +70,7 @@ public partial class OpenClawGatewayClient : WebSocketClientBase, IOperatorGatew private string? _operatorDeviceId; private string[] _grantedOperatorScopes = Array.Empty(); private string _connectAuthToken; + private string? _assistantMediaAuthToken; private bool _useV2Signature; // true after v3 signature rejected by gateway /// Set to true to skip v3 and use v2 signatures directly (for gateways that don't support v3). @@ -201,6 +202,7 @@ protected override void OnDisconnected() protected override void OnDisposing() { _pendingRequests.Drain(); + _assistantMediaHttpClient.Dispose(); } protected override void OnError(Exception ex) @@ -291,13 +293,27 @@ protected override void OnReconnectAuthorizationDenied( protected void RaiseConnectionFailure(GatewayErrorKind kind) => ConnectionFailure?.Invoke(this, kind); - public OpenClawGatewayClient(string gatewayUrl, string token, IOpenClawLogger? logger = null, bool tokenIsBootstrapToken = false, bool bootstrapPairAsNode = false, string? identityPath = null, bool ignoreStoredDeviceToken = false, bool persistHandshakeDeviceTokens = true) + public OpenClawGatewayClient( + string gatewayUrl, + string token, + IOpenClawLogger? logger = null, + bool tokenIsBootstrapToken = false, + bool bootstrapPairAsNode = false, + string? identityPath = null, + bool ignoreStoredDeviceToken = false, + bool persistHandshakeDeviceTokens = true, + string? assistantMediaAuthToken = null, + HttpMessageHandler? assistantMediaHandler = null) : base(gatewayUrl, token, logger) { _tokenIsBootstrapToken = tokenIsBootstrapToken; _bootstrapPairAsNode = bootstrapPairAsNode; _ignoreStoredDeviceToken = ignoreStoredDeviceToken; _persistHandshakeDeviceTokens = persistHandshakeDeviceTokens; + _assistantMediaHttpClient = assistantMediaHandler is null + ? new HttpClient(new HttpClientHandler { AllowAutoRedirect = false }) + : new HttpClient(assistantMediaHandler, disposeHandler: true); + SetAssistantMediaAuthToken(assistantMediaAuthToken); _currentGatewayUrl = gatewayUrl; var dataPath = identityPath ?? OpenClawAppIdentity.ResolveRoamingDataDirectory( Environment.GetEnvironmentVariable); @@ -309,6 +325,15 @@ public OpenClawGatewayClient(string gatewayUrl, string token, IOpenClawLogger? l _useV2Signature |= _tokenIsBootstrapToken && !HasUsableOperatorDeviceToken; } + /// + /// Updates the independently authorized credential used by assistant-media + /// HTTP requests. A null or blank value disables authenticated media HTTP. + /// + public void SetAssistantMediaAuthToken(string? token) => + Volatile.Write( + ref _assistantMediaAuthToken, + string.IsNullOrWhiteSpace(token) ? null : token); + public async Task DisconnectAsync() { if (IsConnected) @@ -622,10 +647,15 @@ private static ChatHistoryInfo ParseChatHistory(JsonElement payload, string sess stopReason = sr.GetString(); // Content can include text and structured tool call/result blocks. - string text = ExtractMessageText(m); + string text = ExtractMessageText(m, role); var toolContent = ExtractToolContent(m, role, text); var contentParts = ExtractOrderedMessageContent(m, role, toolContent); - if (string.IsNullOrEmpty(text) && toolContent.Count == 0) continue; + if (string.IsNullOrEmpty(text) + && toolContent.Count == 0 + && contentParts.All(static part => part.Kind != ChatMessageContentPartKind.Media)) + { + continue; + } if (string.IsNullOrEmpty(role)) continue; if (!string.IsNullOrEmpty(text) && toolContent.Count == 0 @@ -660,12 +690,14 @@ private static ChatHistoryInfo ParseChatHistory(JsonElement payload, string sess return info; } - private static string ExtractMessageText(JsonElement message) + private static string ExtractMessageText(JsonElement message, string role) { if (!message.TryGetProperty("content", out var content)) return string.Empty; if (content.ValueKind == JsonValueKind.String) - return content.GetString() ?? string.Empty; + { + return AssistantMediaDirectiveParser.Project(role, content.GetString()).Text; + } if (content.ValueKind == JsonValueKind.Array) { @@ -674,6 +706,7 @@ private static string ExtractMessageText(JsonElement message) { if (item.ValueKind == JsonValueKind.String) { + if (sb.Length > 0) sb.Append('\n'); sb.Append(item.GetString()); } else if (item.ValueKind == JsonValueKind.Object && @@ -684,7 +717,7 @@ private static string ExtractMessageText(JsonElement message) sb.Append(tx.GetString()); } } - return sb.ToString(); + return AssistantMediaDirectiveParser.Project(role, sb.ToString()).Text; } return string.Empty; @@ -795,34 +828,42 @@ private static IReadOnlyList ExtractOrderedMessageCo IReadOnlyList toolContent) { if (!message.TryGetProperty("content", out var content) - || content.ValueKind != JsonValueKind.Array) + || content.ValueKind is not (JsonValueKind.Array or JsonValueKind.String)) { return Array.Empty(); } + if (content.ValueKind == JsonValueKind.String) + { + return AssistantMediaDirectiveParser.Project(role, content.GetString()).ContentParts; + } + var parts = new List(); - var text = new StringBuilder(); + var textRun = new StringBuilder(); var toolIndex = 0; var structuredToolCount = 0; - void FlushText() + void AppendTextBlock(string? value) { - if (text.Length == 0) - return; + if (textRun.Length > 0) + textRun.Append('\n'); + textRun.Append(value); + } - parts.Add(new ChatMessageContentPartInfo - { - Kind = ChatMessageContentPartKind.Text, - Text = text.ToString(), - }); - text.Clear(); + void FlushTextRun() + { + if (textRun.Length == 0) + return; + parts.AddRange( + AssistantMediaDirectiveParser.Project(role, textRun.ToString()).ContentParts); + textRun.Clear(); } foreach (var item in content.EnumerateArray()) { if (item.ValueKind == JsonValueKind.String) { - text.Append(item.GetString()); + AppendTextBlock(item.GetString()); continue; } @@ -830,6 +871,7 @@ void FlushText() || !item.TryGetProperty("type", out var typeElement) || typeElement.ValueKind != JsonValueKind.String) { + FlushTextRun(); continue; } @@ -841,17 +883,25 @@ void FlushText() if (item.TryGetProperty("text", out var textElement) && textElement.ValueKind == JsonValueKind.String) { - if (text.Length > 0) - text.Append('\n'); - text.Append(textElement.GetString()); + AppendTextBlock(textElement.GetString()); } continue; } + FlushTextRun(); + if (TryParseStructuredMedia(item, normalizedType, out var media)) + { + parts.Add(new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }); + continue; + } + if (normalizedType is not ("toolcall" or "tooluse" or "toolresult")) continue; - FlushText(); structuredToolCount++; if (toolIndex < toolContent.Count) { @@ -863,7 +913,7 @@ void FlushText() } } - FlushText(); + FlushTextRun(); var normalizedRole = role.Replace("_", string.Empty, StringComparison.Ordinal) .ToLowerInvariant(); @@ -892,6 +942,178 @@ void FlushText() return parts; } + private static bool TryParseStructuredMedia( + JsonElement item, + string normalizedType, + out ChatMediaContentInfo media) + { + media = null!; + var mimeType = ReadFirstString(item, "mimeType", "mime_type")?.Trim().ToLowerInvariant(); + var hasMediaShape = normalizedType is "image" or "audio" or "video" or "file" or "attachment" + || !string.IsNullOrWhiteSpace(mimeType) + || ReadFirstString(item, "artifactId", "artifact_id") is not null + || ReadFirstString(item, "url") is not null; + if (!hasMediaShape) + return false; + + var kind = normalizedType switch + { + "image" => ChatMediaContentKind.Image, + "audio" => ChatMediaContentKind.Audio, + "video" => ChatMediaContentKind.Video, + "file" or "attachment" => ClassifyMediaMimeType(mimeType, ChatMediaContentKind.File), + _ => ClassifyMediaMimeType(mimeType, ChatMediaContentKind.Unknown), + }; + var url = ReadFirstString(item, "url"); + var artifactId = ReadFirstString(item, "artifactId", "artifact_id") + ?? TryCreateManagedArtifactId(url, kind); + + media = new ChatMediaContentInfo + { + Kind = kind, + Source = ChatMediaContentSource.Structured, + Type = ReadFirstString(item, "type"), + MimeType = NormalizeMediaMimeType(mimeType), + FileName = NormalizeMediaDisplayText( + ReadFirstString(item, "fileName", "file_name"), + 255), + ArtifactId = NormalizeMediaDisplayText(artifactId, 512), + AgentId = NormalizeMediaDisplayText(ReadFirstString(item, "agentId", "agent_id"), 256), + Url = NormalizeMediaProtocolValue(url, 4096), + OpenUrl = NormalizeMediaProtocolValue(ReadFirstString(item, "openUrl", "open_url"), 4096), + Alt = NormalizeMediaDisplayText(ReadFirstString(item, "alt"), 1024), + Width = ReadPositiveInt(item, "width"), + Height = ReadPositiveInt(item, "height"), + SizeBytes = ReadPositiveLong(item, "sizeBytes", "size_bytes"), + DurationSeconds = ReadDurationSeconds(item), + Playback = ReadFirstString(item, "playback")?.ToLowerInvariant() switch + { + "native" => ChatMediaPlaybackMode.Native, + "transcode" => ChatMediaPlaybackMode.Transcode, + _ => null, + }, + }; + return true; + } + + private static ChatMediaContentKind ClassifyMediaMimeType( + string? mimeType, + ChatMediaContentKind fallback) + { + if (mimeType?.StartsWith("image/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Image; + if (mimeType?.StartsWith("audio/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Audio; + if (mimeType?.StartsWith("video/", StringComparison.OrdinalIgnoreCase) == true) + return ChatMediaContentKind.Video; + return fallback; + } + + private static string? NormalizeMediaMimeType(string? value) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > 128) + return null; + var normalized = value.Trim().ToLowerInvariant(); + return normalized.Contains('/') && normalized.All(static character => + character is >= 'a' and <= 'z' + or >= '0' and <= '9' + or '/' or '+' or '-' or '.' or '_') + ? normalized + : null; + } + + private static string? NormalizeMediaDisplayText(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value)) + return null; + var builder = new StringBuilder(Math.Min(value.Length, maxLength)); + foreach (var character in value) + { + if (!char.IsControl(character) && character is not '\r' and not '\n') + builder.Append(character); + if (builder.Length == maxLength) + break; + } + var result = builder.ToString().Trim(); + return result.Length == 0 ? null : result; + } + + private static string? NormalizeMediaProtocolValue(string? value, int maxLength) + { + if (string.IsNullOrWhiteSpace(value) || value.Length > maxLength) + return null; + return value.Trim(); + } + + private static int? ReadPositiveInt(JsonElement item, string name) + { + if (!item.TryGetProperty(name, out var value) + || value.ValueKind != JsonValueKind.Number + || !value.TryGetInt32(out var result) + || result <= 0) + { + return null; + } + return result; + } + + private static long? ReadPositiveLong(JsonElement item, params string[] names) + { + foreach (var name in names) + { + if (item.TryGetProperty(name, out var value) + && value.ValueKind == JsonValueKind.Number + && value.TryGetInt64(out var result) + && result > 0) + { + return result; + } + } + return null; + } + + private static double? ReadDurationSeconds(JsonElement item) + { + if (item.TryGetProperty("durationSeconds", out var seconds) + && seconds.ValueKind == JsonValueKind.Number + && seconds.TryGetDouble(out var secondsValue) + && secondsValue >= 0) + { + return secondsValue; + } + if (item.TryGetProperty("durationMs", out var milliseconds) + && milliseconds.ValueKind == JsonValueKind.Number + && milliseconds.TryGetDouble(out var millisecondsValue) + && millisecondsValue >= 0) + { + return millisecondsValue / 1000d; + } + return null; + } + + private static string? TryCreateManagedArtifactId(string? rawUrl, ChatMediaContentKind kind) + { + if (string.IsNullOrWhiteSpace(rawUrl) + || !Uri.TryCreate(rawUrl, UriKind.Relative, out var relative) + || relative.IsAbsoluteUri) + { + return null; + } + var path = rawUrl.Split('?', '#')[0]; + var segments = path.Split('/', StringSplitOptions.RemoveEmptyEntries); + if (segments.Length != 7 + || !segments[..4].SequenceEqual(new[] { "api", "chat", "media", "outgoing" }) + || !string.Equals(segments[6], "full", StringComparison.Ordinal) + || !Guid.TryParse(segments[5], out var attachmentId)) + { + return null; + } + var prefix = kind is ChatMediaContentKind.Audio or ChatMediaContentKind.Video + ? "artifact_managed_media_" + : "artifact_managed_image_"; + return prefix + attachmentId.ToString("D").ToLowerInvariant(); + } + private static JsonElement? ReadFirstValue(JsonElement value, params string[] propertyNames) { foreach (var propertyName in propertyNames) @@ -3602,8 +3824,14 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) if (inTok is null && outTok is null && respTok is null && ctxPct is null) (inTok, outTok, respTok, ctxPct) = ExtractChatUsage(message); - var text = ExtractMessageText(message); - if (string.IsNullOrEmpty(text)) return; + var text = ExtractMessageText(message, role); + var toolContent = ExtractToolContent(message, role, text); + var contentParts = ExtractOrderedMessageContent(message, role, toolContent); + if (string.IsNullOrEmpty(text) + && contentParts.All(static part => part.Kind != ChatMessageContentPartKind.Media)) + { + return; + } if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; var messageOpenClawMetadata = ExtractOpenClawMetadata(message); @@ -3622,25 +3850,29 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) messageOpenClawMetadata.Seq ?? payloadOpenClawMetadata.Seq, messageOpenClawMetadata.Kind ?? payloadOpenClawMetadata.Kind, messageOpenClawMetadata.TokensBefore ?? payloadOpenClawMetadata.TokensBefore, - messageOpenClawMetadata.TokensAfter ?? payloadOpenClawMetadata.TokensAfter); + messageOpenClawMetadata.TokensAfter ?? payloadOpenClawMetadata.TokensAfter, + contentParts); if (role == "assistant" && string.Equals(state, "final", StringComparison.OrdinalIgnoreCase)) { // HIGH 4: log shape only — content previously // surfaced in the operator log. _logger.Info($"Assistant response: role={role} state={state} len={text.Length}"); - EmitChatNotification(text, sessionKey); + if (!string.IsNullOrWhiteSpace(text)) + EmitChatNotification(text, sessionKey); } } // Legacy format: payload.text + payload.role else if (payload.TryGetProperty("text", out var textProp)) { - var text = textProp.GetString() ?? ""; var role = payload.TryGetProperty("role", out var roleProp) ? roleProp.GetString() ?? "" : ""; var state = payload.TryGetProperty("state", out var stateProp) ? stateProp.GetString() : null; + var projection = AssistantMediaDirectiveParser.Project(role, textProp.GetString()); + var text = projection.Text; - if (!string.IsNullOrEmpty(text)) + if (!string.IsNullOrEmpty(text) + || projection.ContentParts.Any(static part => part.Kind == ChatMessageContentPartKind.Media)) { if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; @@ -3659,13 +3891,15 @@ private void HandleChatEvent(JsonElement root, int rawMessageLength) openClawMetadata.Seq, openClawMetadata.Kind, openClawMetadata.TokensBefore, - openClawMetadata.TokensAfter); + openClawMetadata.TokensAfter, + projection.ContentParts); if (role == "assistant") { // HIGH 4: log shape only. _logger.Info($"Assistant response (legacy): role={role} state={state} len={text.Length}"); - EmitChatNotification(text, sessionKey); + if (!string.IsNullOrWhiteSpace(text)) + EmitChatNotification(text, sessionKey); } } } @@ -3735,7 +3969,8 @@ private void EmitChatMessageReceived( int? openClawSeq = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? contentParts = null) { if (ChatMessageInfo.IsSilentAssistantDirective(role, text)) return; @@ -3747,6 +3982,7 @@ private void EmitChatMessageReceived( SessionKey = sessionKey, Role = role, Text = text, + ContentParts = contentParts ?? Array.Empty(), State = state, Ts = tsMs, InputTokens = inputTokens, diff --git a/src/OpenClaw.Shared/WebSocketClientBase.cs b/src/OpenClaw.Shared/WebSocketClientBase.cs index 534151000..5dcddca06 100644 --- a/src/OpenClaw.Shared/WebSocketClientBase.cs +++ b/src/OpenClaw.Shared/WebSocketClientBase.cs @@ -123,6 +123,9 @@ public abstract class WebSocketClientBase : IDisposable /// Cancellation token tied to this client's lifetime. protected CancellationToken CancellationToken => _cts.Token; + /// Monotonic identity for the current transport connection. + protected long ConnectionGeneration => Interlocked.Read(ref _connectionGeneration); + /// Close status from the current connection's server-originated close frame. protected int? RemoteCloseStatusCode { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs new file mode 100644 index 000000000..996cf2cff --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantContentPresentation.cs @@ -0,0 +1,104 @@ +using OpenClaw.Shared; + +namespace OpenClawTray.Chat; + +public sealed record ChatAssistantContentPresentation( + IReadOnlyList Media); + +public sealed record ChatAssistantMediaPresentation( + ChatMediaContentKind Kind, + string DisplayName, + string? MimeType, + string? Alt, + ChatMediaContentInfo Reference); + +internal sealed record ChatAssistantMediaRenderPlan( + IReadOnlyList Media, + int OmittedImages); + +/// +/// Converts protocol media into renderer-safe presentation metadata. Transport +/// references remain opaque and are never included in display strings. +/// +internal static class ChatAssistantContentProjector +{ + internal const int MaximumInlineImages = 4; + + public static ChatAssistantContentPresentation? Project( + IEnumerable? contentParts) + { + if (contentParts is null) + return null; + + var media = contentParts + .Where(static part => part.Kind == ChatMessageContentPartKind.Media) + .Select(static part => part.Media) + .Where(static item => item is not null) + .Select(static item => new ChatAssistantMediaPresentation( + item!.Kind, + SafeDisplayName(item), + item.MimeType, + item.Alt, + item)) + .ToArray(); + return media.Length == 0 ? null : new ChatAssistantContentPresentation(media); + } + + public static ChatAssistantMediaRenderPlan BuildRenderPlan( + IReadOnlyList media) + { + var renderedImages = 0; + var planned = new List(media.Count); + foreach (var item in media) + { + if (item.Kind == ChatMediaContentKind.Image + && ++renderedImages > MaximumInlineImages) + { + continue; + } + planned.Add(item); + } + return new ChatAssistantMediaRenderPlan( + planned, + Math.Max(0, renderedImages - MaximumInlineImages)); + } + + public static ChatAssistantContentPresentation MergeLiveUpdate( + ChatAssistantContentPresentation? existing, + ChatAssistantContentPresentation incoming) + { + if (existing is null || existing.Media.Count != incoming.Media.Count) + return incoming; + + var merged = incoming.Media.ToArray(); + for (var index = 0; index < merged.Length; index++) + { + var previous = existing.Media[index]; + var next = incoming.Media[index]; + if (previous.Kind == next.Kind + && previous.Reference.Source == ChatMediaContentSource.LegacyDirective + && next.Reference.Source == ChatMediaContentSource.Structured + && (string.IsNullOrWhiteSpace(next.Reference.FileName) + || string.Equals( + previous.Reference.FileName, + next.Reference.FileName, + StringComparison.OrdinalIgnoreCase))) + { + merged[index] = previous; + } + } + return new ChatAssistantContentPresentation(merged); + } + + private static string SafeDisplayName(ChatMediaContentInfo media) + { + if (!string.IsNullOrWhiteSpace(media.FileName)) + { + var normalized = media.FileName.Replace('\\', '/'); + var leaf = normalized[(normalized.LastIndexOf('/') + 1)..].Trim(); + if (leaf.Length > 0) + return leaf; + } + return string.Empty; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs new file mode 100644 index 000000000..67f9a9857 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantImageDecodePolicy.cs @@ -0,0 +1,33 @@ +namespace OpenClawTray.Chat; + +internal static class ChatAssistantImageDecodePolicy +{ + internal const uint MaximumSourceDimension = 16_384; + internal const ulong MaximumDecodedPixels = 32UL * 1024 * 1024; + internal const uint MaximumDecodeDimension = 2_048; + + public static bool TryGetDecodeSize( + uint sourceWidth, + uint sourceHeight, + out int decodeWidth, + out int decodeHeight) + { + decodeWidth = 0; + decodeHeight = 0; + if (sourceWidth == 0 + || sourceHeight == 0 + || sourceWidth > MaximumSourceDimension + || sourceHeight > MaximumSourceDimension + || (ulong)sourceWidth * sourceHeight > MaximumDecodedPixels) + { + return false; + } + + var scale = Math.Min( + 1d, + MaximumDecodeDimension / (double)Math.Max(sourceWidth, sourceHeight)); + decodeWidth = Math.Max(1, (int)Math.Round(sourceWidth * scale)); + decodeHeight = Math.Max(1, (int)Math.Round(sourceHeight * scale)); + return true; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs new file mode 100644 index 000000000..4cc361ce2 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAssistantMediaRenderer.cs @@ -0,0 +1,386 @@ +using Microsoft.UI.Reactor; +using Microsoft.UI.Reactor.Core; +using Microsoft.UI.Reactor.Hooks; +using Microsoft.UI.Text; +using Microsoft.UI.Xaml; +using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Media; +using Microsoft.UI.Xaml.Media.Imaging; +using OpenClaw.Shared; +using OpenClawTray.Helpers; +using OpenClawTray.Services; +using Windows.Graphics.Imaging; +using Windows.Storage.Streams; +using static Microsoft.UI.Reactor.Factories; + +namespace OpenClawTray.Chat; + +internal static class ChatAssistantMediaRenderer +{ + public static Element Render( + ChatAssistantMediaPresentation media, + string? sessionKey, + Func>? + resolver) + { + if (media.Kind == ChatMediaContentKind.Image + && !string.IsNullOrWhiteSpace(sessionKey) + && resolver is not null) + { + return Component( + new(media, sessionKey, resolver)); + } + + return BuildUnavailableCard(media); + } + + internal static Element BuildUnavailableCard( + ChatAssistantMediaPresentation media, + bool preparing = false, + Action? retry = null) + { + var kind = KindLabel(media.Kind); + var displayName = DisplayName(media); + var statusText = preparing + ? LocalizedOrDefault("Chat_AssistantMedia_Preparing", $"Preparing {kind.ToLowerInvariant()}") + : LocalizedOrDefault("Chat_AssistantMedia_Unavailable", $"{kind} unavailable"); + var detail = string.IsNullOrWhiteSpace(media.MimeType) + ? statusText + : $"{statusText} · {media.MimeType}"; + var accessibleName = $"{displayName}. {statusText}"; + + var glyph = TextBlock(Glyph(media.Kind)) + .FontSize(18) + .FontWeight(FontWeights.Normal) + .Foreground(Theme.Ref("TextFillColorSecondaryBrush")) + .Set(text => text.FontFamily = FluentIconCatalog.SymbolThemeFontFamily) + .Center(); + var glyphBackground = Border(glyph) + .Size(36, 36) + .CornerRadius(8) + .Background(Theme.Ref("SubtleFillColorSecondaryBrush")); + var title = TextBlock(displayName) + .FontSize(13) + .FontWeight(FontWeights.SemiBold) + .Foreground(Theme.Ref("TextFillColorPrimaryBrush")) + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(320); + var status = TextBlock(detail) + .FontSize(11) + .FontWeight(FontWeights.Normal) + .Foreground(Theme.Ref("TextFillColorSecondaryBrush")) + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(320); + var content = HStack( + 10, + glyphBackground, + VStack(2, title, status).VAlign(VerticalAlignment.Center)); + Element body = retry is null + ? content + : HStack( + 10, + content, + Button( + LocalizedOrDefault("Chat_AssistantMedia_Retry", "Retry"), + retry) + .AutomationName( + LocalizedOrDefault("Chat_AssistantMedia_Retry", "Retry"))); + + return Border(body) + .Padding(10, 8) + .CornerRadius(10) + .Background(Theme.Ref("SubtleFillColorTertiaryBrush")) + .BorderBrush(Theme.Ref("ControlStrokeColorDefaultBrush")) + .BorderThickness(1) + .HAlign(HorizontalAlignment.Left) + .AutomationName(accessibleName); + } + + private static string KindLabel(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => LocalizedOrDefault("Chat_AssistantMedia_Image", "Image"), + ChatMediaContentKind.Audio => LocalizedOrDefault("Chat_AssistantMedia_Audio", "Audio"), + ChatMediaContentKind.Video => LocalizedOrDefault("Chat_AssistantMedia_Video", "Video"), + ChatMediaContentKind.File => LocalizedOrDefault("Chat_AssistantMedia_File", "File"), + _ => LocalizedOrDefault("Chat_AssistantMedia_Media", "Media"), + }; + + internal static string DisplayName(ChatAssistantMediaPresentation media) => + string.IsNullOrWhiteSpace(media.DisplayName) + ? KindLabel(media.Kind) + : media.DisplayName; + + private static string Glyph(ChatMediaContentKind kind) => kind switch + { + ChatMediaContentKind.Image => "\uEB9F", + ChatMediaContentKind.Audio => "\uE8D6", + ChatMediaContentKind.Video => "\uE714", + ChatMediaContentKind.File => "\uE8A5", + _ => "\uE7C3", + }; + + internal static string LocalizedOrDefault(string key, string fallback) + { + var localized = LocalizationHelper.GetString(key); + return string.IsNullOrWhiteSpace(localized) || string.Equals(localized, key, StringComparison.Ordinal) + ? fallback + : localized; + } +} + +internal sealed record ChatAssistantImageCardProps( + ChatAssistantMediaPresentation Media, + string SessionKey, + Func> + Resolver); + +internal sealed class ChatAssistantImageCard : Component +{ + public override Element Render() + { + var props = Props; + var (state, setState) = UseState(null, threadSafe: true); + var (attempt, setAttempt) = UseState(0, threadSafe: true); + var (viewerOpen, setViewerOpen) = UseState(false, threadSafe: true); + + UseEffect((Func)(() => + { + var cancellation = new CancellationTokenSource(); + setViewerOpen(false); + _ = LoadAsync(cancellation.Token); + return () => + { + cancellation.Cancel(); + cancellation.Dispose(); + }; + + async Task LoadAsync(CancellationToken cancellationToken) + { + try + { + var resolved = await props.Resolver( + props.SessionKey, + props.Media.Reference, + cancellationToken); + BitmapImage? bitmap = null; + if (resolved.Status == AssistantMediaResolutionStatus.Ready + && resolved.Data is { Length: > 0 } bytes) + { + bitmap = await TryDecodeBitmapAsync(bytes, cancellationToken); + if (bitmap is null) + resolved = AssistantMediaResolutionResult.Unavailable; + } + if (!cancellationToken.IsCancellationRequested) + { + setState(new ChatAssistantImageLoadState( + resolved.Status, + bitmap, + props.SessionKey, + props.Media.Reference)); + } + } + catch (Exception ex) when (!cancellationToken.IsCancellationRequested) + { + Logger.Warn($"Assistant image load failed ({ex.GetType().Name})."); + setState(new ChatAssistantImageLoadState( + AssistantMediaResolutionStatus.Unavailable, + null, + props.SessionKey, + props.Media.Reference)); + } + } + }), props.SessionKey, props.Media.Reference, attempt); + + var currentState = state is not null + && string.Equals(state.SessionKey, props.SessionKey, StringComparison.Ordinal) + && ReferenceEquals(state.Reference, props.Media.Reference) + ? state + : null; + if (currentState is + { Status: AssistantMediaResolutionStatus.Ready, Bitmap: { } bitmap }) + { + var image = BuildImage(props.Media, bitmap, () => setViewerOpen(true)); + var viewer = ContentDialog( + props.Media.Alt ?? ChatAssistantMediaRenderer.DisplayName(props.Media), + BuildViewerImage(props.Media, bitmap), + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_Close", + "Close")) with + { + IsOpen = viewerOpen, + OnClosed = _ => setViewerOpen(false), + }; + return Grid( + [GridSize.Star()], + [GridSize.Star()], + image, + viewer); + } + + var preparing = currentState is null + || currentState.Status == AssistantMediaResolutionStatus.Preparing; + return ChatAssistantMediaRenderer.BuildUnavailableCard( + props.Media, + preparing, + currentState is null ? null : () => + { + setState(null); + setAttempt(attempt + 1); + }); + } + + private static Element BuildImage( + ChatAssistantMediaPresentation media, + BitmapImage bitmap, + Action openViewer) + { + const double maximumWidth = 480; + const double maximumHeight = 320; + var pixelWidth = bitmap.PixelWidth > 0 ? bitmap.PixelWidth : (int)maximumWidth; + var pixelHeight = bitmap.PixelHeight > 0 ? bitmap.PixelHeight : (int)maximumHeight; + var scale = Math.Min( + Math.Min(maximumWidth / pixelWidth, maximumHeight / pixelHeight), + 1.0); + var preview = Border(Empty()) + .Set(border => border.Background = new ImageBrush + { + ImageSource = bitmap, + Stretch = Stretch.Uniform, + }) + .Size(pixelWidth * scale, pixelHeight * scale) + .CornerRadius(10) + .AutomationName(media.Alt ?? ChatAssistantMediaRenderer.DisplayName(media)); + return Button(preview, openViewer) + .Padding(0) + .Background(Theme.Ref("SubtleFillColorTransparentBrush")) + .BorderThickness(0) + .AutomationName(string.Format( + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_OpenImage", + "Open image {0}"), + ChatAssistantMediaRenderer.DisplayName(media))); + } + + private static Element BuildViewerImage( + ChatAssistantMediaPresentation media, + BitmapImage bitmap) + { + const double maximumWidth = 1200; + const double maximumHeight = 800; + var pixelWidth = bitmap.PixelWidth > 0 ? bitmap.PixelWidth : (int)maximumWidth; + var pixelHeight = bitmap.PixelHeight > 0 ? bitmap.PixelHeight : (int)maximumHeight; + var scale = Math.Min( + Math.Min(maximumWidth / pixelWidth, maximumHeight / pixelHeight), + 1.0); + return ScrollViewer( + Border(Empty()) + .Set(border => border.Background = new ImageBrush + { + ImageSource = bitmap, + Stretch = Stretch.Uniform, + }) + .Size(pixelWidth * scale, pixelHeight * scale) + .AutomationName(media.Alt ?? ChatAssistantMediaRenderer.DisplayName(media))); + } + + private static async Task TryDecodeBitmapAsync( + byte[] bytes, + CancellationToken cancellationToken) + { + try + { + using var stream = new InMemoryRandomAccessStream(); + using (var writer = new DataWriter(stream)) + { + writer.WriteBytes(bytes); + await writer.StoreAsync().AsTask(cancellationToken); + writer.DetachStream(); + } + stream.Seek(0); + var decoder = await BitmapDecoder.CreateAsync(stream).AsTask(cancellationToken); + if (!ChatAssistantImageDecodePolicy.TryGetDecodeSize( + decoder.PixelWidth, + decoder.PixelHeight, + out var decodeWidth, + out var decodeHeight)) + { + return null; + } + stream.Seek(0); + var bitmap = new BitmapImage + { + DecodePixelType = DecodePixelType.Physical, + DecodePixelWidth = decodeWidth, + DecodePixelHeight = decodeHeight, + }; + await bitmap.SetSourceAsync(stream).AsTask(cancellationToken); + return bitmap; + } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + return null; + } + catch (Exception ex) + { + Logger.Warn($"Assistant image decode failed ({ex.GetType().Name})."); + return null; + } + } +} + +internal sealed record ChatAssistantImageLoadState( + AssistantMediaResolutionStatus Status, + BitmapImage? Bitmap, + string SessionKey, + ChatMediaContentInfo Reference); + +internal static class ChatAttachmentBitmapDecoder +{ + internal static BitmapImage? TryDecode(byte[] bytes) + { + try + { + using var stream = new InMemoryRandomAccessStream(); + using (var writer = new DataWriter(stream)) + { + writer.WriteBytes(bytes); + writer.StoreAsync().AsTask().GetAwaiter().GetResult(); + writer.DetachStream(); + } + + stream.Seek(0); + var decoder = BitmapDecoder.CreateAsync(stream).AsTask().GetAwaiter().GetResult(); + if (!ChatAssistantImageDecodePolicy.TryGetDecodeSize( + decoder.PixelWidth, + decoder.PixelHeight, + out var decodeWidth, + out var decodeHeight)) + { + return null; + } + + stream.Seek(0); + var bitmap = new BitmapImage + { + DecodePixelType = DecodePixelType.Physical, + DecodePixelWidth = decodeWidth, + DecodePixelHeight = decodeHeight, + }; + bitmap.SetSource(stream); + return bitmap; + } + catch (Exception ex) + { + Logger.Warn($"Attachment image decode failed ({ex.GetType().Name})."); + return null; + } + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs new file mode 100644 index 000000000..128453711 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentEchoCorrelation.cs @@ -0,0 +1,46 @@ +namespace OpenClawTray.Chat; + +internal sealed record ChatPendingEchoCandidate( + string MessageId, + string Text, + string AttachmentCorrelationSignature); + +internal static class ChatAttachmentEchoCorrelation +{ + internal static string? SelectMatchingMessageId( + IReadOnlyList candidates, + GatewayMediaMessageProjectionResult incoming) => + SelectMatchingMessageId( + candidates, + incoming.ReconciliationText, + incoming.AttachmentCorrelationSignature, + incoming.HasMediaEnvelope); + + internal static string? SelectMatchingMessageId( + IReadOnlyList candidates, + string text, + string attachmentCorrelationSignature, + bool hasMediaEnvelope) + { + var matching = candidates.Where(candidate => + string.Equals(candidate.Text, text, StringComparison.Ordinal) && + (!hasMediaEnvelope || + string.Equals( + candidate.AttachmentCorrelationSignature, + attachmentCorrelationSignature, + StringComparison.Ordinal))) + .ToArray(); + + if (matching.Length == 0) + return null; + if (hasMediaEnvelope && matching.Length != 1) + return null; + if (!hasMediaEnvelope && + matching.Any(candidate => candidate.AttachmentCorrelationSignature.Length > 0)) + { + return null; + } + + return matching[0].MessageId; + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs new file mode 100644 index 000000000..57f3bf754 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatAttachmentPresentation.cs @@ -0,0 +1,163 @@ +using OpenClaw.Shared; + +namespace OpenClawTray.Chat; + +public enum ChatAttachmentOrigin +{ + Local, + GatewayReference, +} + +/// +/// Safe, renderer-facing attachment metadata. Preview cache access is allowed +/// only for local attachments that carry an opaque cache key. +/// +public sealed record ChatAttachmentPresentation( + ChatAttachmentOrigin Origin, + string DisplayFileName, + string MimeType, + bool IsImage, + string? PreviewCacheKey = null) +{ + public bool CanAccessPreviewCache => + Origin == ChatAttachmentOrigin.Local && + !string.IsNullOrWhiteSpace(PreviewCacheKey); +} + +internal static class ChatAttachmentPreviewResolver +{ + internal static bool TryGetBytes( + ChatAttachmentPresentation attachment, + out byte[] bytes) + { + bytes = Array.Empty(); + return attachment.CanAccessPreviewCache && + ChatImagePreviewCache.TryGet(attachment.PreviewCacheKey!, out bytes); + } + + internal static bool TryGetBytes( + ChatAttachmentPresentation attachment, + IReadOnlyDictionary previewCache, + out byte[] bytes) + { + bytes = Array.Empty(); + return attachment.CanAccessPreviewCache && + previewCache.TryGetValue(attachment.PreviewCacheKey!, out bytes!); + } +} + +internal static class ChatImagePreviewCache +{ + internal const int MaximumEntries = 32; + internal const long MaximumTotalBytes = 64L * 1024 * 1024; + internal const int MaximumPreviewBytes = (int)ChatAttachment.MaxSizeBytes; + + private static readonly object s_gate = new(); + private static readonly Dictionary s_entries = new(StringComparer.Ordinal); + private static readonly Queue s_insertionOrder = new(); + private static long s_totalBytes; + + internal static int Count + { + get + { + lock (s_gate) + return s_entries.Count; + } + } + + internal static long TotalBytes + { + get + { + lock (s_gate) + return s_totalBytes; + } + } + + internal static bool TryStoreBase64(string key, string encoded) + { + if (string.IsNullOrWhiteSpace(key) || + !TryDecodeBoundedBase64(encoded, MaximumPreviewBytes, out var bytes)) + { + return false; + } + + lock (s_gate) + { + if (s_entries.TryGetValue(key, out var previous)) + { + s_totalBytes -= previous.Length; + } + else + { + s_insertionOrder.Enqueue(key); + } + + s_entries[key] = bytes; + s_totalBytes += bytes.Length; + TrimLocked(); + return s_entries.ContainsKey(key); + } + } + + internal static bool TryGet(string key, out byte[] bytes) + { + lock (s_gate) + return s_entries.TryGetValue(key, out bytes!); + } + + internal static bool Contains(string key) + { + lock (s_gate) + return s_entries.ContainsKey(key); + } + + internal static bool TryDecodeBoundedBase64( + string? encoded, + int maximumBytes, + out byte[] bytes) + { + bytes = Array.Empty(); + if (string.IsNullOrEmpty(encoded) || maximumBytes <= 0) + return false; + + var maximumEncodedLength = ((maximumBytes + 2L) / 3L) * 4L; + if (encoded.Length > maximumEncodedLength) + return false; + + try + { + bytes = Convert.FromBase64String(encoded); + if (bytes.Length <= maximumBytes) + return true; + } + catch (FormatException) + { + } + + bytes = Array.Empty(); + return false; + } + + internal static void Clear() + { + lock (s_gate) + { + s_entries.Clear(); + s_insertionOrder.Clear(); + s_totalBytes = 0; + } + } + + private static void TrimLocked() + { + while ((s_entries.Count > MaximumEntries || + s_totalBytes > MaximumTotalBytes) && + s_insertionOrder.TryDequeue(out var oldestKey)) + { + if (s_entries.Remove(oldestKey, out var removed)) + s_totalBytes -= removed.Length; + } + } +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs index d97a26640..f8aaa620d 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatConversationState.cs @@ -548,7 +548,10 @@ internal ChatQueuedAdmission AdmitMessage( string nonce, IReadOnlyList? attachments, DateTimeOffset createdAt, - ChatProjectionContext context) + ChatProjectionContext context, + string? timelineText = null, + IReadOnlyList? attachmentPresentations = null, + string attachmentCorrelationSignature = "") { lock (_gate) { @@ -566,7 +569,10 @@ internal ChatQueuedAdmission AdmitMessage( text, displayText, nonce, - attachments?.ToArray()); + attachments?.ToArray(), + TimelineText: timelineText ?? displayText, + AttachmentPresentations: attachmentPresentations, + AttachmentCorrelationSignature: attachmentCorrelationSignature); var sendDirectly = CanSendDirectlyLocked(threadId); ChatQueuedSendDispatch? dispatch; @@ -980,12 +986,13 @@ private ChatQueuedSendDispatch StartDirectSendLocked( var entryId = $"e{current.NextId}"; _timelines[threadId] = ChatTimelineReducer.AddLocalUser( current, - request.DisplayText, + request.EffectiveTimelineText, request.LocalNonce); GetOrCreateThreadMetaLocked(threadId)[entryId] = BuildLiveMetaLocked( threadId, isLocalQueuedSend: true, - localQueuedMessageId: request.Id); + localQueuedMessageId: request.Id, + attachments: request.AttachmentPresentations); var dispatch = _queue.StartDirect( request, _history.ResolveSessionId(threadId), @@ -1061,6 +1068,7 @@ private bool PromoteQueuedMessageLocked( string messageId, ChatEntryMetadata? confirmedMeta = null) { + var request = _queue.FindRequest(threadId, messageId); if (!_queue.TryTakeForPromotion(threadId, messageId, out var queued)) return false; @@ -1068,18 +1076,20 @@ private bool PromoteQueuedMessageLocked( var entryId = $"e{current.NextId}"; _timelines[threadId] = ChatTimelineReducer.AddLocalUser( current, - queued.Text, + request?.EffectiveTimelineText ?? queued.Text, queued.LocalNonce); var meta = confirmedMeta is not null && HasGatewayIdentity(confirmedMeta) ? confirmedMeta with { IsLocalQueuedSend = false, LocalQueuedMessageId = messageId, + Attachments = request?.AttachmentPresentations ?? confirmedMeta.Attachments, } : BuildLiveMetaLocked( threadId, isLocalQueuedSend: true, - localQueuedMessageId: messageId); + localQueuedMessageId: messageId, + attachments: request?.AttachmentPresentations); GetOrCreateThreadMetaLocked(threadId)[entryId] = meta; return true; } @@ -1147,7 +1157,9 @@ internal ChatEntryMetadata BuildLiveMetadata( string? localQueuedMessageId = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? attachments = null, + ChatAssistantContentPresentation? assistantContent = null) { lock (_gate) { @@ -1160,7 +1172,9 @@ internal ChatEntryMetadata BuildLiveMetadata( localQueuedMessageId, openClawKind, compactionTokensBefore, - compactionTokensAfter); + compactionTokensAfter, + attachments, + assistantContent); } } @@ -1296,11 +1310,14 @@ internal ChatResetTransition ResetThread( internal ChatIncomingMessageGate GateIncomingChatMessage( ChatMessageInfo message, - ChatProjectionContext context) + ChatProjectionContext context, + GatewayMediaMessageProjectionResult? projection = null) { var threadId = message.SessionKey!; var role = message.Role?.ToLowerInvariant() ?? string.Empty; - var text = message.Text ?? string.Empty; + var text = projection?.ReconciliationText ?? message.Text ?? string.Empty; + var attachmentCorrelationSignature = projection?.AttachmentCorrelationSignature ?? ""; + var hasMediaEnvelope = projection?.HasMediaEnvelope ?? false; lock (_gate) { _lifecycle.TryGetActiveRun( @@ -1311,7 +1328,11 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( role, text, message.Ts, - _queue.HasPendingLocalEchoText(threadId, text), + _queue.HasPendingLocalEchoText( + threadId, + text, + attachmentCorrelationSignature, + hasMediaEnvelope), activeRunId); var openedLifecycle = ApplyBufferedLifecycleOpenLocked( @@ -1326,6 +1347,8 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( _queue.TryConsumeLocalEcho( threadId, resetGate.ConsumeEchoText, + attachmentCorrelationSignature, + hasMediaEnvelope, out var queuedMessageId)) { var confirmed = BuildLiveMetaLocked( @@ -1362,15 +1385,21 @@ internal ChatIncomingMessageGate GateIncomingChatMessage( internal ChatLocalEchoTransition ConsumeLocalEcho( ChatMessageInfo message, bool removeQueuedMessage, - ChatProjectionContext context) + ChatProjectionContext context, + GatewayMediaMessageProjectionResult? projection = null) { var threadId = message.SessionKey!; - var text = (message.Text ?? string.Empty).Trim(); + var text = projection?.ReconciliationText ?? + (message.Text ?? string.Empty).Trim(); + var attachmentCorrelationSignature = projection?.AttachmentCorrelationSignature ?? ""; + var hasMediaEnvelope = projection?.HasMediaEnvelope ?? false; lock (_gate) { if (!_queue.TryConsumeLocalEcho( threadId, text, + attachmentCorrelationSignature, + hasMediaEnvelope, out var queuedMessageId)) { return new(false, null); @@ -1397,23 +1426,182 @@ internal ChatLocalEchoTransition ConsumeLocalEcho( internal ChatLocalEchoTransition ReconcileExistingLocalQueuedUser( ChatMessageInfo message, string userText, - ChatProjectionContext context) + ChatProjectionContext context, + IReadOnlyList? attachments = null, + string attachmentCorrelationSignature = "", + bool hasMediaEnvelope = false) { + var threadId = message.SessionKey!; lock (_gate) { var metadata = BuildLiveMetaLocked( - message.SessionKey!, + threadId, message.Ts, message.OpenClawId, - message.OpenClawSeq); - var reconciled = TryReconcileExistingLocalQueuedUserEchoLocked( - message.SessionKey!, + message.OpenClawSeq, + attachments: attachments); + if (TryReconcileExistingLocalQueuedUserEchoLocked( + threadId, + userText, + attachmentCorrelationSignature, + hasMediaEnvelope, + metadata)) + { + return new(true, BuildSnapshotLocked(context)); + } + + var remoteSnapshot = ApplyProjectedRemoteUserMessageLocked( + threadId, userText, - metadata); - return new( - reconciled, - reconciled ? BuildSnapshotLocked(context) : null); + attachmentCorrelationSignature, + metadata, + context); + return new(false, remoteSnapshot); + } + } + + // Applies an incoming user message from another client (not a local + // echo). Gateway retransmits of the same message (e.g. once with a + // partial media resolve, once final) are merged into the existing + // timeline row instead of appended as a duplicate, matched first by + // gateway identity and — for identity-less rows — by same trailing-entry + // text + attachment signature. + private ChatDataSnapshot? ApplyProjectedRemoteUserMessageLocked( + string threadId, + string projectedText, + string attachmentCorrelationSignature, + ChatEntryMetadata incomingMeta, + ChatProjectionContext context) + { + var timeline = GetOrCreateTimelineLocked(threadId); + var threadMeta = GetOrCreateThreadMetaLocked(threadId); + ChatTimelineItem? matched = null; + ChatEntryMetadata? existingMeta = null; + + if (HasGatewayIdentity(incomingMeta)) + { + for (var i = timeline.Entries.Count - 1; i >= 0; i--) + { + var candidate = timeline.Entries[i]; + if (candidate.Kind != ChatTimelineItemKind.User || + !threadMeta.TryGetValue(candidate.Id, out var candidateMeta) || + candidateMeta.IsLocalQueuedSend || + !HasMatchingGatewayIdentity(candidateMeta, incomingMeta)) + { + continue; + } + + matched = candidate; + existingMeta = candidateMeta; + break; + } + } + + // Identity-less history/live twins are only safe to correlate + // against the current trailing user row. Crossing an + // assistant/status boundary would collapse a legitimate later turn + // that repeats the same prose. + if (matched is null && + timeline.Entries.Count > 0 && + timeline.Entries[^1] is { Kind: ChatTimelineItemKind.User } latestUser && + string.Equals(latestUser.Text, projectedText, StringComparison.Ordinal) && + threadMeta.TryGetValue(latestUser.Id, out var latestMeta) && + !latestMeta.IsLocalQueuedSend && + !HasConflictingGatewayIdentity(latestMeta, incomingMeta) && + string.Equals( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + latestMeta.Attachments), + attachmentCorrelationSignature, + StringComparison.Ordinal)) + { + matched = latestUser; + existingMeta = latestMeta; + } + + if (matched is null || existingMeta is null) + { + ApplyEventLocked( + threadId, + new ChatUserMessageEvent(projectedText), + incomingMeta); + return BuildSnapshotLocked(context); + } + + var mergedMeta = MergeProjectedUserMetadata(existingMeta, incomingMeta); + if (mergedMeta == existingMeta) + return null; + + threadMeta[matched.Id] = mergedMeta; + return HasRendererVisibleUserMetadataChange(existingMeta, mergedMeta) + ? BuildSnapshotLocked(context) + : null; + } + + private static bool HasMatchingGatewayIdentity( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) => + (!string.IsNullOrEmpty(incoming.GatewayMessageId) && + string.Equals( + existing.GatewayMessageId, + incoming.GatewayMessageId, + StringComparison.Ordinal)) || + (incoming.OpenClawSeq is not null && existing.OpenClawSeq == incoming.OpenClawSeq); + + private static bool HasConflictingGatewayIdentity( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) => + (!string.IsNullOrEmpty(existing.GatewayMessageId) && + !string.IsNullOrEmpty(incoming.GatewayMessageId) && + !string.Equals( + existing.GatewayMessageId, + incoming.GatewayMessageId, + StringComparison.Ordinal)) || + (existing.OpenClawSeq is not null && + incoming.OpenClawSeq is not null && + existing.OpenClawSeq != incoming.OpenClawSeq); + + private static ChatEntryMetadata MergeProjectedUserMetadata( + ChatEntryMetadata existing, + ChatEntryMetadata incoming) + { + var mergedAttachments = existing.Attachments is { Count: > 0 } + ? existing.Attachments + : incoming.Attachments; + return existing with + { + Timestamp = existing.Timestamp ?? incoming.Timestamp, + Model = existing.Model ?? incoming.Model, + GatewayMessageId = string.IsNullOrEmpty(existing.GatewayMessageId) + ? incoming.GatewayMessageId + : existing.GatewayMessageId, + OpenClawSeq = existing.OpenClawSeq ?? incoming.OpenClawSeq, + Attachments = mergedAttachments, + }; + } + + private static bool HasRendererVisibleUserMetadataChange( + ChatEntryMetadata existing, + ChatEntryMetadata merged) => + existing.Timestamp != merged.Timestamp || + !AttachmentPresentationsEqual(existing.Attachments, merged.Attachments); + + private static bool AttachmentPresentationsEqual( + IReadOnlyList? left, + IReadOnlyList? right) + { + var leftCount = left?.Count ?? 0; + var rightCount = right?.Count ?? 0; + if (leftCount != rightCount) + return false; + if (leftCount == 0) + return true; + + for (var i = 0; i < leftCount; i++) + { + if (left![i] != right![i]) + return false; } + return true; } internal (ChatEntryMetadata Metadata, string? ActiveRunId) BuildMetadataWithRun( @@ -1434,16 +1622,29 @@ internal ChatLocalEchoTransition ReconcileExistingLocalQueuedUser( internal ChatAssistantPreparation PrepareAssistant( ChatMessageInfo message, string assistantText, - ChatProjectionContext context) + ChatProjectionContext context, + ChatAssistantContentPresentation? assistantContent = null) { var threadId = message.SessionKey!; lock (_gate) { - var disposition = ClassifyAssistantQueueFrameLocked( - threadId, - assistantText, - message.OpenClawId, - message.OpenClawSeq); + // A frame carrying only structured/legacy media directives (no + // plain text) has nothing for the identified-duplicate/self-echo + // classifier to compare against, and it never carries a gateway + // identity either (media-only frames are synthesized locally + // from ContentParts, not gateway-sequenced) — so it can't be a + // resend of an already-rendered turn. Render it directly rather + // than routing it through text-based classification. + var disposition = assistantText.Length == 0 && + assistantContent is not null && + string.IsNullOrEmpty(message.OpenClawId) && + message.OpenClawSeq is null + ? AssistantQueueFrameDisposition.Render + : ClassifyAssistantQueueFrameLocked( + threadId, + assistantText, + message.OpenClawId, + message.OpenClawSeq); ChatDataSnapshot? promotionSnapshot = null; if (disposition == AssistantQueueFrameDisposition.Render && _queue.IsLocallyInitiated(threadId) && @@ -1459,7 +1660,8 @@ internal ChatAssistantPreparation PrepareAssistant( threadId, message.Ts, message.OpenClawId, - message.OpenClawSeq); + message.OpenClawSeq, + assistantContent: assistantContent); var hasUsage = message.InputTokens is not null || message.OutputTokens is not null || message.ResponseTokens is not null || @@ -1864,6 +2066,8 @@ internal void CompleteRemoteBackfill(string threadId) internal ChatRemoteUserBackfillTransition? ApplyRemoteUserBackfill( string threadId, ChatMessageInfo message, + GatewayMediaMessageProjectionResult projection, + IReadOnlyList attachments, long expectedResetGeneration, bool openResetGate, ChatProjectionContext context) @@ -1875,34 +2079,33 @@ internal void CompleteRemoteBackfill(string threadId) { return null; } - if (_timelines.TryGetValue(threadId, out var timeline)) - { - for (var i = timeline.Entries.Count - 1; i >= 0; i--) - { - if (timeline.Entries[i].Kind != ChatTimelineItemKind.User) - continue; - if (timeline.Entries[i].Text == message.Text) - return null; - break; - } - } var openedLifecycle = openResetGate ? ApplyBufferedLifecycleOpenLocked( threadId, _reset.RecordRemoteUser(threadId), allowRemoteTurn: true) : null; - ApplyEventLocked( + var metadata = BuildLiveMetaLocked( threadId, - new ChatUserMessageEvent( - ChatContentFormatting.TruncateForChatEntry(message.Text)), - BuildLiveMetaLocked( - threadId, - message.Ts, - message.OpenClawId, - message.OpenClawSeq)); + message.Ts, + message.OpenClawId, + message.OpenClawSeq, + attachments: attachments); + var snapshot = ApplyProjectedRemoteUserMessageLocked( + threadId, + ChatContentFormatting.TruncateForChatEntry( + projection.HasMediaEnvelope + ? projection.ReconciliationText + : ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines( + message.Text)), + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + attachments), + metadata, + context); + if (snapshot is null && openedLifecycle is null) + return null; return new( - BuildSnapshotLocked(context), + snapshot ?? BuildSnapshotLocked(context), openedLifecycle, CurrentRuntimeGenerationLocked(threadId)); } @@ -2079,6 +2282,8 @@ metadata.OutputTokens is { } output private bool TryReconcileExistingLocalQueuedUserEchoLocked( string threadId, string text, + string attachmentCorrelationSignature, + bool hasMediaEnvelope, ChatEntryMetadata confirmed) { if (!HasGatewayIdentity(confirmed) || @@ -2087,25 +2292,45 @@ private bool TryReconcileExistingLocalQueuedUserEchoLocked( { return false; } + + var candidates = new List(); + var echoCandidates = new List(); for (var i = timeline.Entries.Count - 1; i >= 0; i--) { var entry = timeline.Entries[i]; if (entry.Kind != ChatTimelineItemKind.User || - !string.Equals(entry.Text, text, StringComparison.Ordinal) || !metadata.TryGetValue(entry.Id, out var existing) || !existing.IsLocalQueuedSend || !IsFreshLocalQueuedPromotion(existing, confirmed)) { continue; } - metadata[entry.Id] = confirmed with - { - IsLocalQueuedSend = false, - LocalQueuedMessageId = existing.LocalQueuedMessageId, - }; - return true; - } - return false; + candidates.Add(entry); + echoCandidates.Add(new ChatPendingEchoCandidate( + entry.Id, + entry.Text, + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + existing.Attachments))); + } + + var matchedMessageId = ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + echoCandidates, + text, + attachmentCorrelationSignature, + hasMediaEnvelope); + if (matchedMessageId is null) + return false; + + var matched = candidates.First(candidate => + string.Equals(candidate.Id, matchedMessageId, StringComparison.Ordinal)); + var matchedMeta = metadata[matched.Id]; + metadata[matched.Id] = confirmed with + { + IsLocalQueuedSend = false, + LocalQueuedMessageId = matchedMeta.LocalQueuedMessageId, + Attachments = matchedMeta.Attachments, + }; + return true; } private static bool IsFreshLocalQueuedPromotion( @@ -2143,6 +2368,7 @@ private bool ReconcileQueuedMessageEchoLocked( { IsLocalQueuedSend = false, LocalQueuedMessageId = messageId, + Attachments = match.Value.Attachments, }; return true; } @@ -2317,6 +2543,41 @@ private void ApplyEventLocked( if (!beforeIds.Contains(entry.Id) && !threadMetadata.ContainsKey(entry.Id)) threadMetadata[entry.Id] = metadata; } + + // Streaming assistant frames reconcile into the SAME entry id + // (see ChatTimelineReducer.UpsertAssistant), so the new-entry-only + // assignment above never touches it again after creation. Assistant + // structured media content can still refine across frames (e.g. a + // legacy directive resolved to a structured reference on a later + // frame), so merge it into the already-existing reconciled entry's + // metadata explicitly. + if (metadata.AssistantContent is not null) + { + for (var i = next.Entries.Count - 1; i >= 0; i--) + { + var entry = next.Entries[i]; + if (entry.Kind == ChatTimelineItemKind.User) + break; + if (entry.Kind != ChatTimelineItemKind.Assistant) + continue; + + if (beforeIds.Contains(entry.Id) && + threadMetadata.TryGetValue(entry.Id, out var existingEntryMeta)) + { + var mergedContent = ChatAssistantContentProjector.MergeLiveUpdate( + existingEntryMeta.AssistantContent, + metadata.AssistantContent); + if (!ReferenceEquals(mergedContent, existingEntryMeta.AssistantContent)) + { + threadMetadata[entry.Id] = existingEntryMeta with + { + AssistantContent = mergedContent, + }; + } + } + break; + } + } } private Dictionary GetOrCreateThreadMetaLocked( @@ -2339,7 +2600,9 @@ private ChatEntryMetadata BuildLiveMetaLocked( string? localQueuedMessageId = null, string? openClawKind = null, long? compactionTokensBefore = null, - long? compactionTokensAfter = null) + long? compactionTokensAfter = null, + IReadOnlyList? attachments = null, + ChatAssistantContentPresentation? assistantContent = null) { var timestamp = tsMs is { } value && value > 0 ? DateTimeOffset.FromUnixTimeMilliseconds(value).ToLocalTime() @@ -2353,7 +2616,9 @@ private ChatEntryMetadata BuildLiveMetaLocked( CompactionTokensBefore: compactionTokensBefore, CompactionTokensAfter: compactionTokensAfter, IsLocalQueuedSend: isLocalQueuedSend, - LocalQueuedMessageId: localQueuedMessageId); + LocalQueuedMessageId: localQueuedMessageId, + Attachments: attachments, + AssistantContent: assistantContent); } private ChatOpenedLifecycleTransition? AddResetAcceptedRunIdLocked( diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs index 0e0862520..eeabe0fcb 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatEntryMetadata.cs @@ -60,6 +60,14 @@ namespace OpenClawTray.Chat; /// Stable client-side id for a local send. Used to attach a later gateway /// identity to the exact optimistic transcript row without text matching. /// +/// +/// Structured attachment presentation metadata. Gateway references never carry +/// preview cache keys. +/// +/// +/// Renderer-safe assistant media presentation. Transport references remain +/// opaque and are never encoded into timeline text. +/// public sealed record ChatEntryMetadata( DateTimeOffset? Timestamp, string? Model, @@ -75,4 +83,6 @@ public sealed record ChatEntryMetadata( long? CompactionTokensBefore = null, long? CompactionTokensAfter = null, bool IsLocalQueuedSend = false, - string? LocalQueuedMessageId = null); + string? LocalQueuedMessageId = null, + IReadOnlyList? Attachments = null, + ChatAssistantContentPresentation? AssistantContent = null); diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs index 33f754336..8fbec307b 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryLoader.cs @@ -487,6 +487,18 @@ ChatTimelineState Apply( if (!before.Contains(entry.Id) && !metadata.ContainsKey(entry.Id)) metadata[entry.Id] = entryMetadata; } + if (entryMetadata.AssistantContent is not null && + next.ActiveAssistantId is { } assistantId && + metadata.TryGetValue(assistantId, out var existingMetadata)) + { + metadata[assistantId] = existingMetadata with + { + AssistantContent = + ChatAssistantContentProjector.MergeLiveUpdate( + existingMetadata.AssistantContent, + entryMetadata.AssistantContent), + }; + } } return next; } @@ -504,6 +516,10 @@ ChatTimelineState Apply( } var role = message.Role?.ToLowerInvariant() ?? string.Empty; + var rawText = replayPart.Text; + var userProjection = role == "user" + ? GatewayMediaMessageProjection.Project(rawText) + : null; var entryMetadata = new ChatEntryMetadata( message.Ts > 0 ? DateTimeOffset.FromUnixTimeMilliseconds(message.Ts).ToLocalTime() @@ -517,17 +533,37 @@ ChatTimelineState Apply( OpenClawSeq: message.OpenClawSeq, OpenClawKind: message.OpenClawKind, CompactionTokensBefore: message.CompactionTokensBefore, - CompactionTokensAfter: message.CompactionTokensAfter); + CompactionTokensAfter: message.CompactionTokensAfter, + AssistantContent: role == "assistant" + ? ChatAssistantContentProjector.Project( + replayPart.AssistantContentParts) + : null); var text = ChatContentFormatting.TruncateForChatEntry( ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines( - replayPart.Text)); - if (role == "user") - text = ChatMetadataStore.RehydrateAttachmentMarkers( - attachmentMatcher, - text, + userProjection?.HasMediaEnvelope == true + ? userProjection.ReconciliationText + : rawText)); + if (userProjection is not null) + { + var cachedAttachment = attachmentMatcher.TryMatch( + userProjection.ReconciliationText, + userProjection.AttachmentCorrelationSignature, message.Ts); + var attachmentPresentations = cachedAttachment is not null + ? ChatMetadataStore.CreatePersistedLocalPresentations( + cachedAttachment.Attachments) + : userProjection.Attachments; + entryMetadata = entryMetadata with + { + Attachments = attachmentPresentations, + }; + } var hasStructuredToolContent = replayPart.ToolContent.Count > 0; + var hasUserAttachments = + entryMetadata.Attachments is { Count: > 0 }; + var hasAssistantMedia = + entryMetadata.AssistantContent is { Media.Count: > 0 }; if (role == "user" && _persistence.IsMessageAborted( @@ -564,12 +600,16 @@ ChatTimelineState Apply( } if (string.IsNullOrEmpty(text) && - !hasStructuredToolContent) + !hasStructuredToolContent && + !hasUserAttachments && + !hasAssistantMedia) { continue; } - if (!string.IsNullOrEmpty(text)) + if (!string.IsNullOrEmpty(text) || + (role == "user" && hasUserAttachments) || + (role == "assistant" && hasAssistantMedia)) { switch (role) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs index b42431b6d..dc8a7fdf6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatHistoryReplayProjection.cs @@ -8,6 +8,7 @@ internal sealed record ChatHistoryReplayPart( ChatMessageInfo Message, string Text, IReadOnlyList ToolContent, + IReadOnlyList AssistantContentParts, bool IsFirstPart); internal static class ChatHistoryReplayProjection @@ -41,6 +42,7 @@ public static IEnumerable Project( message, message.Text ?? string.Empty, message.ToolContent, + Array.Empty(), IsFirstPart: true); continue; } @@ -54,6 +56,7 @@ public static IEnumerable Project( message, part.Text ?? string.Empty, Array.Empty(), + new[] { part }, isFirstPart); isFirstPart = false; } @@ -63,6 +66,17 @@ public static IEnumerable Project( message, string.Empty, new[] { tool }, + Array.Empty(), + isFirstPart); + isFirstPart = false; + } + else if (part.Kind == ChatMessageContentPartKind.Media && part.Media is not null) + { + yield return new ChatHistoryReplayPart( + message, + string.Empty, + Array.Empty(), + new[] { part }, isFirstPart); isFirstPart = false; } diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs index 4b41a5a17..de5597231 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatMetadataStore.cs @@ -20,8 +20,14 @@ public AttachmentMetaMatcher(List entrie _used = new bool[entries.Count]; } - public ChatMetadataStore.CachedAttachmentMeta? TryMatch(string text, long historyTsMs) + public ChatMetadataStore.CachedAttachmentMeta? TryMatch( + string text, + string attachmentCorrelationSignature, + long historyTsMs) { + if (string.IsNullOrEmpty(attachmentCorrelationSignature)) + return null; + for (var i = 0; i < _entries.Count; i++) { if (_used[i]) @@ -30,6 +36,15 @@ public AttachmentMetaMatcher(List entrie var entry = _entries[i]; if (!string.Equals(entry.Text, text, StringComparison.Ordinal)) continue; + if (!string.Equals( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature( + ChatMetadataStore.CreatePersistedLocalPresentations( + entry.Attachments)), + attachmentCorrelationSignature, + StringComparison.Ordinal)) + { + continue; + } if (historyTsMs > 0 && entry.Ts > 0 && Math.Abs(historyTsMs - entry.Ts) > MatchWindow.TotalMilliseconds) @@ -78,6 +93,7 @@ internal sealed class CachedAttachmentMeta internal sealed class CachedAttachmentItem { public string FileName { get; set; } = ""; + public string MimeType { get; set; } = "application/octet-stream"; public bool IsImage { get; set; } } @@ -272,10 +288,21 @@ internal void CacheAttachments( var items = attachments .Where(attachment => !string.IsNullOrWhiteSpace(attachment.FileName)) - .Select(attachment => new CachedAttachmentItem + .Select(attachment => { - FileName = NormalizeCachedDisplayText(attachment.FileName), - IsImage = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase), + var mimeType = + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType); + return new CachedAttachmentItem + { + FileName = NormalizeCachedDisplayText(attachment.FileName), + MimeType = mimeType, + IsImage = string.Equals( + attachment.Type, + "image", + StringComparison.OrdinalIgnoreCase) || + mimeType.StartsWith("image/", StringComparison.Ordinal), + }; }) .ToList(); if (items.Count == 0) @@ -675,6 +702,8 @@ private static void AtomicWrite(string path, string json, string cacheName) Attachments = entry.Attachments.Select(attachment => new CachedAttachmentItem { FileName = NormalizeCachedDisplayText(attachment.FileName), + MimeType = GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType), IsImage = attachment.IsImage, }).ToList(), }; @@ -714,7 +743,12 @@ internal static Dictionary> LoadAttachmentMet { entry.Text = NormalizeCachedDisplayText(entry.Text); foreach (var attachment in entry.Attachments) + { attachment.FileName = NormalizeCachedDisplayText(attachment.FileName); + attachment.MimeType = + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType); + } } return cache; } @@ -760,20 +794,17 @@ internal static string BuildAttachmentMarkerLines(IEnumerable + CreatePersistedLocalPresentations( + IEnumerable attachments) => + attachments.Select(attachment => new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + GatewayMediaMessageProjection.NormalizeDisplayFileName( + attachment.FileName), + GatewayMediaMessageProjection.NormalizeMimeType( + attachment.MimeType), + attachment.IsImage, + PreviewCacheKey: null)).ToArray(); internal static string NormalizeCachedDisplayText(string? value) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs index 33a9ef283..0f71c37eb 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatQueueState.cs @@ -7,7 +7,8 @@ namespace OpenClawTray.Chat; internal readonly record struct ChatLocalSentText( string Text, DateTimeOffset SentAt, - string QueuedMessageId); + string QueuedMessageId, + string AttachmentCorrelationSignature = ""); internal sealed record ChatQueueRetryResult( bool Requeued, @@ -112,7 +113,11 @@ internal ChatQueuedSendDispatch StartDirect( long resetLifecycleSequence, long lifecycleStartSequence) { - EnqueueLocalEcho(request.ThreadId, request.Text, request.Id); + EnqueueLocalEcho( + request.ThreadId, + request.EffectiveTimelineText, + request.AttachmentCorrelationSignature, + request.Id); _locallyInitiatedThreads.Add(request.ThreadId); _assistantFallbackPromotedThreads.Add(request.ThreadId); return new ChatQueuedSendDispatch( @@ -178,7 +183,11 @@ internal ChatQueuedSendDispatch StartDirect( }; if (request.LifecycleCommand is null) { - EnqueueLocalEcho(threadId, request.Text, request.Id); + EnqueueLocalEcho( + threadId, + request.EffectiveTimelineText, + request.AttachmentCorrelationSignature, + request.Id); _locallyInitiatedThreads.Add(threadId); } @@ -436,20 +445,45 @@ internal ChatLocalSentText[] SnapshotLocalEchoes(string threadId) => ? queue.ToArray() : []; - internal bool HasPendingLocalEchoText(string threadId, string text) => - !string.IsNullOrWhiteSpace(text) && - _localSentTexts.TryGetValue(threadId, out var queue) && - queue.Any(pending => - string.Equals(pending.Text, text.Trim(), StringComparison.Ordinal)); + internal bool HasPendingLocalEchoText( + string threadId, + string text, + string attachmentCorrelationSignature = "", + bool hasMediaEnvelope = false) + { + var normalizedText = + GatewayMediaMessageProjection.NormalizeEchoCorrelationText(text); + if ((normalizedText.Length == 0 && + string.IsNullOrEmpty(attachmentCorrelationSignature)) || + !_localSentTexts.TryGetValue(threadId, out var queue)) + { + return false; + } + var candidates = queue + .Select(pending => new ChatPendingEchoCandidate( + pending.QueuedMessageId, + pending.Text, + pending.AttachmentCorrelationSignature)) + .ToArray(); + return ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + candidates, + normalizedText, + attachmentCorrelationSignature, + hasMediaEnvelope) is not null; + } internal bool TryConsumeLocalEcho( string threadId, string echoText, + string attachmentCorrelationSignature, + bool hasMediaEnvelope, out string queuedMessageId) { queuedMessageId = string.Empty; if (!_localSentTexts.TryGetValue(threadId, out var queue)) return false; + var normalizedEchoText = + GatewayMediaMessageProjection.NormalizeEchoCorrelationText(echoText); var now = DateTimeOffset.Now; while (queue.Count > 0 && now - queue.Peek().SentAt > LocalEchoWindow) @@ -460,24 +494,50 @@ internal bool TryConsumeLocalEcho( return false; } - var retained = new Queue(); - var matched = false; - while (queue.Count > 0) + var pending = queue.ToArray(); + var candidates = pending + .Select(candidate => new ChatPendingEchoCandidate( + candidate.QueuedMessageId, + candidate.Text, + candidate.AttachmentCorrelationSignature)) + .ToArray(); + var matchedMessageId = ChatAttachmentEchoCorrelation.SelectMatchingMessageId( + candidates, + normalizedEchoText, + attachmentCorrelationSignature, + hasMediaEnvelope); + if (matchedMessageId is null) + return false; + + var retained = new Queue(pending.Length); + foreach (var candidate in pending) { - var candidate = queue.Dequeue(); - if (!matched && - string.Equals(candidate.Text, echoText, StringComparison.Ordinal)) + if (!string.Equals( + candidate.QueuedMessageId, + matchedMessageId, + StringComparison.Ordinal)) { - queuedMessageId = candidate.QueuedMessageId; - matched = true; - continue; + retained.Enqueue(candidate); } - retained.Enqueue(candidate); } + queuedMessageId = matchedMessageId; StoreLocalEchoQueue(threadId, retained); - return matched; + return true; } + // Plain-text overload retained for call sites (e.g. reset-gate dropped + // messages) that never carry a media envelope. + internal bool TryConsumeLocalEcho( + string threadId, + string echoText, + out string queuedMessageId) => + TryConsumeLocalEcho( + threadId, + echoText, + attachmentCorrelationSignature: "", + hasMediaEnvelope: false, + out queuedMessageId); + internal void RemovePendingLocalEcho(string threadId, string messageId) { if (!_localSentTexts.TryGetValue(threadId, out var queue)) @@ -522,6 +582,7 @@ internal void ClearThreadForReset(string threadId) private void EnqueueLocalEcho( string threadId, string text, + string attachmentCorrelationSignature, string messageId) { RemovePendingLocalEcho(threadId, messageId); @@ -533,7 +594,8 @@ private void EnqueueLocalEcho( queue.Enqueue(new ChatLocalSentText( text, DateTimeOffset.UtcNow, - messageId)); + messageId, + attachmentCorrelationSignature)); while (queue.Count > MaxLocalEchoes) queue.Dequeue(); } diff --git a/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs b/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs index 7f141881a..ebc5b0dc6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ChatSendQueue.cs @@ -13,7 +13,18 @@ internal sealed record ChatQueuedSendRequest( IReadOnlyList? Attachments, int DeferredAdmissionRetryCount = 0, DateTimeOffset? DeferredAdmissionRetryAfter = null, - ChatLifecycleCommandKind? LifecycleCommand = null); + ChatLifecycleCommandKind? LifecycleCommand = null, + string? TimelineText = null, + IReadOnlyList? AttachmentPresentations = null, + string AttachmentCorrelationSignature = "") +{ + // The final timeline entry text never carries the cosmetic attachment + // chip lines that DisplayText uses for the queued-message preview list + // — structured Attachments render those instead. Falls back to + // DisplayText for legacy callers (e.g. lifecycle commands) that never + // set TimelineText explicitly. + public string EffectiveTimelineText => TimelineText ?? DisplayText; +} internal sealed record ChatQueuedSendDispatch( ChatQueuedSendRequest Request, diff --git a/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs b/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs new file mode 100644 index 000000000..7af2056a8 --- /dev/null +++ b/src/OpenClaw.Tray.WinUI/Chat/GatewayMediaMessageProjection.cs @@ -0,0 +1,292 @@ +using System.Globalization; +using System.Text; +using OpenClaw.Shared; + +namespace OpenClawTray.Chat; + +public sealed record GatewayMediaMessageProjectionResult( + string ReconciliationText, + string ResidualText, + IReadOnlyList Attachments, + bool HasMediaEnvelope) +{ + public string AttachmentPresentationSignature => + GatewayMediaMessageProjection.BuildAttachmentPresentationSignature(Attachments); + + public string AttachmentCorrelationSignature => + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(Attachments); +} + +/// +/// Projects trusted gateway media envelopes into inert presentation metadata. +/// It never emits the tray's private zero-width attachment marker syntax. +/// +public static class GatewayMediaMessageProjection +{ + private const string EnvelopePrefix = "[media attached: "; + private const string MediaPrefix = "media://inbound/"; + private const int MaxDisplayFileNameLength = 160; + + public static GatewayMediaMessageProjectionResult Project(string? text) + { + var source = text ?? string.Empty; + var attachments = new List(); + var offset = 0; + + while (offset < source.Length) + { + var lineEnd = source.IndexOf('\n', offset); + var contentEnd = lineEnd >= 0 ? lineEnd : source.Length; + var line = source.AsSpan(offset, contentEnd - offset); + if (line.EndsWith("\r", StringComparison.Ordinal)) + line = line[..^1]; + + if (!TryParseEnvelopeLine(line, out var attachment)) + break; + + attachments.Add(attachment); + offset = lineEnd >= 0 ? lineEnd + 1 : source.Length; + } + + if (attachments.Count == 0) + { + return new GatewayMediaMessageProjectionResult( + source.Trim(), + source, + Array.Empty(), + HasMediaEnvelope: false); + } + + var residual = source[offset..]; + return new GatewayMediaMessageProjectionResult( + residual.Trim(), + residual, + attachments.ToArray(), + HasMediaEnvelope: true); + } + + public static IReadOnlyList CreateLocalPresentations( + IReadOnlyList? attachments, + Func previewKeyFactory) + { + if (attachments is null || attachments.Count == 0) + return Array.Empty(); + + var presentations = new List(attachments.Count); + foreach (var attachment in attachments) + { + var displayName = NormalizeDisplayFileName(attachment.FileName); + if (displayName.Length == 0) + displayName = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase) + ? "image" + : "attachment"; + + var mimeType = NormalizeMimeType(attachment.MimeType); + var isImage = string.Equals(attachment.Type, "image", StringComparison.OrdinalIgnoreCase) || + mimeType.StartsWith("image/", StringComparison.Ordinal); + presentations.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + displayName, + mimeType, + isImage, + isImage ? previewKeyFactory() : null)); + } + + return presentations.ToArray(); + } + + public static string BuildAttachmentPresentationSignature( + IEnumerable? attachments) + { + if (attachments is null) + return string.Empty; + + var items = attachments.ToArray(); + if (items.Length == 0) + return string.Empty; + + var builder = new StringBuilder(); + builder.Append(items.Length.ToString(CultureInfo.InvariantCulture)).Append('|'); + foreach (var attachment in items) + { + AppendSignaturePart(builder, NormalizeMimeType(attachment.MimeType)); + AppendSignaturePart(builder, NormalizeDisplayFileName(attachment.DisplayFileName)); + } + return builder.ToString(); + } + + public static string BuildAttachmentCorrelationSignature( + IEnumerable? attachments) + { + if (attachments is null) + return string.Empty; + + var items = attachments.ToArray(); + if (items.Length == 0) + return string.Empty; + + var builder = new StringBuilder(); + builder.Append(items.Length.ToString(CultureInfo.InvariantCulture)).Append('|'); + foreach (var attachment in items) + { + AppendSignaturePart(builder, NormalizeMimeType(attachment.MimeType)); + AppendSignaturePart(builder, attachment.IsImage ? "image" : "file"); + } + return builder.ToString(); + } + + public static string NormalizeDisplayFileName(string? value) + { + if (string.IsNullOrEmpty(value)) + return string.Empty; + + var normalizedSeparators = value.Replace('\\', '/'); + var leaf = normalizedSeparators[(normalizedSeparators.LastIndexOf('/') + 1)..]; + var builder = new StringBuilder(Math.Min(leaf.Length, MaxDisplayFileNameLength)); + var elements = StringInfo.GetTextElementEnumerator(leaf); + while (elements.MoveNext() && builder.Length < MaxDisplayFileNameLength) + { + var element = elements.GetTextElement(); + if (!IsPrintableSingleLine(element)) + continue; + if (builder.Length + element.Length > MaxDisplayFileNameLength) + break; + builder.Append(element); + } + + return builder.ToString().Trim(); + } + + public static string NormalizeMimeType(string? value) + { + var normalized = value?.Trim().ToLowerInvariant() ?? string.Empty; + return IsValidMimeType(normalized) ? normalized : "application/octet-stream"; + } + + /// + /// Normalizes text for echo-correlation comparisons: trims, strips the + /// tray's private attachment marker syntax, and caps length identically + /// to how the reducer bounds rendered entries so local/remote copies of + /// the same message always compare equal. + /// + public static string NormalizeEchoCorrelationText(string? text) => + ChatContentFormatting.TruncateForChatEntry( + ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(text?.Trim())); + + private static bool TryParseEnvelopeLine( + ReadOnlySpan line, + out ChatAttachmentPresentation attachment) + { + attachment = null!; + if (!line.StartsWith(EnvelopePrefix, StringComparison.Ordinal) || + !line.EndsWith(")]", StringComparison.Ordinal)) + { + return false; + } + + var body = line[EnvelopePrefix.Length..^1]; + var annotationStart = body.LastIndexOf(" (", StringComparison.Ordinal); + if (annotationStart <= 0) + return false; + + var mediaUri = body[..annotationStart].ToString(); + var mimeType = body[(annotationStart + 2)..^1].ToString(); + if (!mediaUri.StartsWith(MediaPrefix, StringComparison.Ordinal) || + !IsValidMimeType(mimeType) || + mediaUri.IndexOfAny(['?', '#', '\r', '\n', '\t', ' ']) >= 0) + { + return false; + } + + var encodedPath = mediaUri[MediaPrefix.Length..]; + var encodedLeaf = encodedPath[(encodedPath.LastIndexOf('/') + 1)..]; + if (encodedLeaf.Length == 0) + return false; + + string decodedLeaf; + try + { + decodedLeaf = Uri.UnescapeDataString(encodedLeaf); + } + catch (UriFormatException) + { + return false; + } + + var displayName = RemoveCanonicalStorageSuffix(NormalizeDisplayFileName(decodedLeaf)); + if (displayName.Length == 0) + return false; + + var normalizedMimeType = mimeType.ToLowerInvariant(); + attachment = new ChatAttachmentPresentation( + ChatAttachmentOrigin.GatewayReference, + displayName, + normalizedMimeType, + normalizedMimeType.StartsWith("image/", StringComparison.Ordinal), + PreviewCacheKey: null); + return true; + } + + private static bool IsValidMimeType(string value) + { + var slash = value.IndexOf('/'); + return slash > 0 && + slash == value.LastIndexOf('/') && + slash < value.Length - 1 && + IsMimeToken(value.AsSpan(0, slash)) && + IsMimeToken(value.AsSpan(slash + 1)); + } + + private static bool IsMimeToken(ReadOnlySpan value) + { + foreach (var ch in value) + { + if (ch is >= 'a' and <= 'z' or >= 'A' and <= 'Z' or >= '0' and <= '9') + continue; + if ("!#$%&'*+-.^_`|~".IndexOf(ch, StringComparison.Ordinal) >= 0) + continue; + return false; + } + return value.Length > 0; + } + + private static bool IsPrintableSingleLine(string value) + { + foreach (var ch in value) + { + var category = CharUnicodeInfo.GetUnicodeCategory(ch); + if (char.IsControl(ch) || + category is UnicodeCategory.Control or UnicodeCategory.Format or + UnicodeCategory.LineSeparator or UnicodeCategory.ParagraphSeparator) + { + return false; + } + } + return true; + } + + private static string RemoveCanonicalStorageSuffix(string fileName) + { + var extensionStart = fileName.LastIndexOf('.'); + var stem = extensionStart > 0 ? fileName[..extensionStart] : fileName; + var extension = extensionStart > 0 ? fileName[extensionStart..] : string.Empty; + if (stem.Length < 39) + return fileName; + + var suffix = stem[^39..]; + if (!suffix.StartsWith("---", StringComparison.Ordinal) || + !Guid.TryParseExact(suffix[3..], "D", out _)) + { + return fileName; + } + + var cleanedStem = stem[..^39]; + return cleanedStem.Length == 0 ? fileName : cleanedStem + extension; + } + + private static void AppendSignaturePart(StringBuilder builder, string value) => + builder.Append(value.Length.ToString(CultureInfo.InvariantCulture)) + .Append(':') + .Append(value) + .Append('|'); +} diff --git a/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs b/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs index 5788ce32a..f17d338a6 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/IChatGatewayBridge.cs @@ -80,6 +80,11 @@ Task CompactSessionDetailedAsync(string sessionKey) => Task ClearSessionModelAsync(string sessionKey); Task PatchSessionThinkingLevelAsync(string sessionKey, string thinkingLevel); Task RequestChatHistoryAsync(string? sessionKey); + Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) => + Task.FromResult(AssistantMediaResolutionResult.Unavailable); Task SendChatAbortAsync(string runId, string? sessionKey = null); Task ResolveExecApprovalAsync(string approvalId, string decision); @@ -238,6 +243,12 @@ public Task RequestSessionsAsync() => public Task RequestChatHistoryAsync(string? sessionKey) => _client.RequestChatHistoryAsync(sessionKey); + public Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken = default) => + _client.ResolveAssistantMediaAsync(sessionKey, media, cancellationToken); + public Task SendChatAbortAsync(string runId, string? sessionKey = null) => _client.SendChatAbortAsync(runId, sessionKey); public Task ResolveExecApprovalAsync(string approvalId, string decision) => diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs index 96a7ec835..e0a726952 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatDataProvider.cs @@ -1,5 +1,4 @@ using System.Buffers; -using System.Collections.Concurrent; using System.Globalization; using System.IO; using System.Linq; @@ -71,16 +70,6 @@ internal static class LocalizationHelper public sealed class OpenClawChatDataProvider : IChatDataProvider { internal const int MaxEntryTextBytes = 256 * 1024; - /// - /// Process-wide cache mapping an attachment's filename to its raw image - /// bytes. Populated by for image - /// attachments so the timeline can render an actual thumbnail in the - /// user bubble (the display-text marker only carries the filename, not - /// the base64 content). Static so any timeline render after a re-mount - /// can still find the image. - /// - public static readonly ConcurrentDictionary ImagePreviewCache = new(); - private readonly IChatGatewayBridge _bridge; private readonly ChatTelemetryTracker _telemetry = new(); private readonly ChatMetadataStore _metadataStore; @@ -198,18 +187,29 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation var trimmed = message.Trim(); var nonce = Guid.NewGuid().ToString("N"); + var attachmentPresentations = GatewayMediaMessageProjection.CreateLocalPresentations( + attachments, + static () => Guid.NewGuid().ToString("N")); + var attachmentCorrelationSignature = + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(attachmentPresentations); - // Cache image attachments by filename so the timeline can render an - // actual thumbnail preview (the display-text marker only carries the - // filename — see ImagePreviewCache notes). + // Cache image bytes only under collision-resistant keys carried by + // local structured descriptors. if (hasAttachments) { - foreach (var a in attachments!) + for (var i = 0; i < Math.Min(attachments!.Count, attachmentPresentations.Count); i++) { - if (a.Type == "image" && !string.IsNullOrEmpty(a.FileName) && !string.IsNullOrEmpty(a.Content)) + var source = attachments[i]; + var presentation = attachmentPresentations[i]; + if (presentation.CanAccessPreviewCache && !string.IsNullOrEmpty(source.Content)) { - try { ImagePreviewCache[a.FileName] = Convert.FromBase64String(a.Content); } - catch (Exception ex) { Logger.Debug($"ChatDataProvider: image attachment base64 decode failed for '{a.FileName}': {ex.Message}"); } + if (!ChatImagePreviewCache.TryStoreBase64( + presentation.PreviewCacheKey!, + source.Content)) + { + Logger.Debug( + $"ChatDataProvider: image attachment preview rejected for '{presentation.DisplayFileName}'"); + } } } } @@ -219,7 +219,7 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation // blank even if the typed message was empty. Uses a unique prefix // ("\u200B📎 " / "\u200B🖼️ ") with a zero-width space to prevent // false positives from normal user text. - var safeUserText = ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(trimmed); + var safeUserText = GatewayMediaMessageProjection.NormalizeEchoCorrelationText(trimmed); var displayText = safeUserText; if (hasAttachments) { @@ -236,7 +236,10 @@ public async Task SendMessageAsync(string threadId, string message, Cancellation nonce, attachments, DateTimeOffset.UtcNow, - ProjectionContext()); + ProjectionContext(), + timelineText: safeUserText, + attachmentPresentations: attachmentPresentations, + attachmentCorrelationSignature: attachmentCorrelationSignature); _telemetry.StartLocalTurn( admission.MessageId, threadId, @@ -1028,6 +1031,12 @@ public ValueTask DisposeAsync() public IReadOnlyDictionary GetEntryMetadata(string threadId) => _state.GetEntryMetadata(threadId); + internal Task ResolveAssistantMediaAsync( + string sessionKey, + ChatMediaContentInfo media, + CancellationToken cancellationToken) => + _bridge.ResolveAssistantMediaAsync(sessionKey, media, cancellationToken); + // ── Event handlers ── private void OnStatusChanged(object? sender, ConnectionStatus status) @@ -1183,7 +1192,10 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var threadId = message.SessionKey; var role = message.Role?.ToLowerInvariant() ?? string.Empty; var rawText = message.Text ?? string.Empty; - var gate = _state.GateIncomingChatMessage(message, ProjectionContext()); + var projection = role == "user" + ? GatewayMediaMessageProjection.Project(rawText) + : null; + var gate = _state.GateIncomingChatMessage(message, ProjectionContext(), projection); HandleOpenedLifecycle( threadId, gate.OpenedLifecycle, @@ -1230,9 +1242,12 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var echo = _state.ConsumeLocalEcho( message, removeQueuedMessage: true, - ProjectionContext()); + ProjectionContext(), + projection); if (echo.Consumed) + { return; + } ApplyEventAndPublish( threadId, new ChatStatusEvent(rawText.Trim(), ChatTone.Dim), @@ -1256,35 +1271,30 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var localEcho = _state.ConsumeLocalEcho( message, removeQueuedMessage: false, - ProjectionContext()); + ProjectionContext(), + projection); if (localEcho.Consumed) { if (localEcho.Snapshot is not null) Publish(localEcho.Snapshot); return; } - if (!string.IsNullOrEmpty(message.Text)) + if (!string.IsNullOrEmpty(message.Text) || + (projection?.Attachments.Count ?? 0) > 0) { var userText = ChatContentFormatting.TruncateForChatEntry( - ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(message.Text)); + projection is { HasMediaEnvelope: true } + ? projection.ReconciliationText + : ChatMetadataStore.EscapeUntrustedAttachmentMarkerLines(message.Text)); var reconciled = _state.ReconcileExistingLocalQueuedUser( message, userText, - ProjectionContext()); - if (reconciled.Consumed) - { - if (reconciled.Snapshot is not null) - Publish(reconciled.Snapshot); - return; - } - ApplyEventAndPublish( - threadId, - new ChatUserMessageEvent(userText), - _state.BuildLiveMetadata( - threadId, - message.Ts, - message.OpenClawId, - message.OpenClawSeq)); + ProjectionContext(), + projection?.Attachments, + projection?.AttachmentCorrelationSignature ?? "", + projection?.HasMediaEnvelope ?? false); + if (reconciled.Snapshot is not null) + Publish(reconciled.Snapshot); } return; } @@ -1302,9 +1312,12 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) return; } + var assistantContent = role == "assistant" + ? ChatAssistantContentProjector.Project(message.ContentParts) + : null; if (role != "assistant" || ChatMessageInfo.IsSilentAssistantDirective(role, message.Text) || - string.IsNullOrEmpty(message.Text)) + (string.IsNullOrEmpty(message.Text) && assistantContent is null)) { return; } @@ -1314,13 +1327,17 @@ private void OnChatMessageReceived(object? sender, ChatMessageInfo message) var preparation = _state.PrepareAssistant( message, assistantText, - ProjectionContext()); + ProjectionContext(), + assistantContent); if (preparation.PromotionSnapshot is not null) Publish(preparation.PromotionSnapshot); if (preparation.Disposition != AssistantQueueFrameDisposition.Render) return; if (!message.IsFinal && _state.IsLateNonFinalAssistantFrame(threadId)) + { + Logger.Warn($"[ChatProvider] Dropping late non-final assistant frame after completed turn for threadId='{threadId}' len={traceText.Length}"); return; + } _telemetry.ObserveInboundOutput( threadId, @@ -1667,11 +1684,32 @@ private async Task FetchRemoteUserMessageAsync(string threadId, bool openResetGa break; } } - if (lastUser is null || string.IsNullOrEmpty(lastUser.Text)) return; + if (lastUser is null) return; + var projection = GatewayMediaMessageProjection.Project(lastUser.Text); + if (projection.ReconciliationText.Length == 0 && + projection.Attachments.Count == 0) + { + return; + } + var cachedAttachment = _metadataStore + .CreateAttachmentMatcher( + history.SessionId, + threadId, + requestResetVersion) + .TryMatch( + projection.ReconciliationText, + projection.AttachmentCorrelationSignature, + lastUser.Ts); + var projectedAttachments = cachedAttachment is not null + ? ChatMetadataStore.CreatePersistedLocalPresentations( + cachedAttachment.Attachments) + : projection.Attachments; var transition = _state.ApplyRemoteUserBackfill( threadId, lastUser, + projection, + projectedAttachments, requestResetVersion, openResetGateOnSuccess, ProjectionContext()); @@ -1700,6 +1738,13 @@ private async Task FetchRemoteUserMessageAsync(string threadId, bool openResetGa } } +#if OPENCLAW_TRAY_TESTS + internal Task FetchRemoteUserMessageForTestsAsync( + string threadId, + bool openResetGateOnSuccess) => + FetchRemoteUserMessageAsync(threadId, openResetGateOnSuccess); +#endif + private async Task PersistAbortedMessageIdsAsync( string threadId, long resetGeneration) diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs index 8894c2c66..03f68595c 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawChatTimeline.cs @@ -55,7 +55,9 @@ public record OpenClawChatTimelineProps( Func? OnReadAloud = null, Action? OnStopSpeaking = null, int ScrollToBottomToken = 0, - Action? OnPermissionResponse = null); + Action? OnPermissionResponse = null, + Func>? + ResolveAssistantMediaAsync = null); /// /// OpenClaw-skinned variant of from the vendored @@ -367,16 +369,9 @@ static double ClampOffset(double offset, double max) => return existing; try { - var stream = new global::Windows.Storage.Streams.InMemoryRandomAccessStream(); - using (var writer = new global::Windows.Storage.Streams.DataWriter(stream)) - { - writer.WriteBytes(bytes); - writer.StoreAsync().AsTask().GetAwaiter().GetResult(); - writer.DetachStream(); - } - stream.Seek(0); - var bmp = new Microsoft.UI.Xaml.Media.Imaging.BitmapImage(); - bmp.SetSource(stream); + var bmp = ChatAttachmentBitmapDecoder.TryDecode(bytes); + if (bmp is null) + return null; _bitmapCache.Add(bytes, bmp); return bmp; } @@ -1179,22 +1174,38 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var text = entry.Text ?? ""; var lines = text.Split('\n'); var messageLines = new List(); - var attachmentNames = new List<(string Icon, string Name, bool IsImage)>(); + var attachmentNames = new List(); foreach (var line in lines) { var trimLine = line.Trim(); if (trimLine.StartsWith("\u200B🖼️ ")) - attachmentNames.Add(("🖼️", trimLine.Substring(4).Trim(), true)); + attachmentNames.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + trimLine.Substring(4).Trim(), + "application/octet-stream", + IsImage: true)); else if (trimLine.StartsWith("\u200B📎 ")) - attachmentNames.Add(("📎", trimLine.Substring(3).Trim(), false)); + attachmentNames.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + trimLine.Substring(3).Trim(), + "application/octet-stream", + IsImage: false)); else messageLines.Add(line); } + if (MetaFor(entry.Id)?.Attachments is { Count: > 0 } structuredAttachments) + attachmentNames = structuredAttachments.ToList(); + var messageText = string.Join('\n', messageLines).Trim(); var hasMessage = !string.IsNullOrEmpty(messageText); var hasAttachments = attachmentNames.Count > 0; + var safeUserText = string.Join( + '\n', + (hasMessage ? new[] { messageText } : Array.Empty()) + .Concat(attachmentNames.Select(attachment => + $"{attachment.DisplayFileName} ({attachment.MimeType})"))); // Build attachment elements. Images become real thumbnail previews // by pulling the original bytes from OpenClawChatDataProvider's @@ -1206,9 +1217,13 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var attachmentElements = new List(); if (hasAttachments) { - foreach (var (_, name, isImage) in attachmentNames) + foreach (var attachment in attachmentNames) { - if (isImage && OpenClawChatDataProvider.ImagePreviewCache.TryGetValue(name, out var bytes)) + var name = attachment.DisplayFileName; + var isImage = attachment.IsImage; + if (isImage && ChatAttachmentPreviewResolver.TryGetBytes( + attachment, + out var bytes)) { var bmp = TryDecodeBitmap(bytes); if (bmp is not null) @@ -1359,7 +1374,7 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst var timeStr = FormatTime(entryMeta?.Timestamp); var rightInset = showUserAvatar ? (36 + bubbleSideMargin) : 0; rightInset += (int)bubblePadding.Right; - footer = BuildUserFooter(userSender, timeStr, chatStampFg, entry.Id, entry.Text ?? "") + footer = BuildUserFooter(userSender, timeStr, chatStampFg, entry.Id, safeUserText) .Margin(0, 2, rightInset, 0); } @@ -1370,7 +1385,8 @@ Element RenderUserEntry(ChatTimelineItem entry, bool startsBurst, bool endsBurst VStack(2, bubbleRow, footer) .HAlign(HorizontalAlignment.Stretch) ).Background(new SolidColorBrush(Colors.Transparent)) - .Margin(gutter, topMargin, 20, bottomMargin), + .Margin(gutter, topMargin, 20, bottomMargin) + .AutomationName(safeUserText), entry.Id); } diff --git a/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs b/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs index 196ab3b61..86517bfee 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/OpenClawReactorChatRoot.cs @@ -233,6 +233,10 @@ public override Element Render() : isEmptyConversation ? ReactorChatTimelineMode.Empty : ReactorChatTimelineMode.Timeline; + Func>? + mediaResolver = props.Provider is OpenClawChatDataProvider dataProvider + ? dataProvider.ResolveAssistantMediaAsync + : null; var timelineProps = new OpenClawChatTimelineProps( effectiveThread?.Id, @@ -255,7 +259,8 @@ public override Element Render() scrollToBottomToken, effectiveThread is { } permissionThread ? (requestId, action) => OnPermission(permissionThread.Id, requestId, action) - : null); + : null, + mediaResolver); void SelectThread(string threadId) { diff --git a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs index 9b2d17ae4..25de71d33 100644 --- a/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs +++ b/src/OpenClaw.Tray.WinUI/Chat/ReactorChatTimeline.cs @@ -384,7 +384,12 @@ private static Element BuildUser( bool isHovered, Action setEntryHovered) { - var (messageText, attachments) = ParseAttachments(entry.Text); + var (messageText, legacyAttachments) = ParseAttachments(entry.Text); + var attachments = row.Props.Timeline.EntryMetadata?.TryGetValue(entry.Id, out var metadata) == true && + metadata.Attachments is { Count: > 0 } structuredAttachments + ? structuredAttachments + : legacyAttachments; + var accessibleText = BuildAccessibleUserText(messageText, attachments); var content = attachments.Select(BuildAttachment).ToList(); if (messageText.Length > 0) { @@ -412,12 +417,12 @@ private static Element BuildUser( row.Props.Timeline.ShowToolCalls ? UserMetadata(row, entry, isHovered) : Empty(), - CopyAction(entry.Text, isHovered, setEntryHovered, entry.Id)) + CopyAction(accessibleText, isHovered, setEntryHovered, entry.Id)) .Margin(16, 2, 4, 0) .HAlign(HorizontalAlignment.Right)) .Margin(72, 4, 20, 4) .HAlign(HorizontalAlignment.Stretch) - .AutomationName(entry.Text ?? string.Empty); + .AutomationName(accessibleText); } private static Element BuildAssistant( @@ -428,9 +433,35 @@ private static Element BuildAssistant( Func toggleSpeechAsync, Action setEntryHovered) { - var message = BuildSafeMarkdown(entry.Text); + var content = new List(); + ChatEntryMetadata? metadata = null; + if (!string.IsNullOrWhiteSpace(entry.Text)) + content.Add(BuildSafeMarkdown(entry.Text)); + if (row.Props.Timeline.EntryMetadata?.TryGetValue(entry.Id, out var resolvedMetadata) == true) + metadata = resolvedMetadata; + if (metadata?.AssistantContent is { Media.Count: > 0 } assistantContent) + { + var renderPlan = ChatAssistantContentProjector.BuildRenderPlan(assistantContent.Media); + content.AddRange(renderPlan.Media.Select(media => + ChatAssistantMediaRenderer.Render( + media, + row.Props.Timeline.SessionId, + row.Props.Timeline.ResolveAssistantMediaAsync))); + if (renderPlan.OmittedImages > 0) + { + content.Add(TextBlock(string.Format( + ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_ImagesOmitted", + "{0} more images not shown"), + renderPlan.OmittedImages)) + .FontSize(11) + .Foreground(Theme.SecondaryText)); + } + } + if (content.Count == 0) + content.Add(BuildSafeMarkdown(string.Empty)); - var bubble = Border(message) + var bubble = Border(VStack(8, content.ToArray())) .Background(BrushFor( "SubtleFillColorSecondaryBrush", Color.FromArgb(0x24, 0x80, 0x80, 0x80))) @@ -464,7 +495,25 @@ private static Element BuildAssistant( .Grid(column: 1)) .Margin(20, row.IsAssistantRunStart ? 6 : 1, 72, row.IsAssistantRunEnd ? 6 : 1) .HAlign(HorizontalAlignment.Stretch) - .AutomationName(entry.Text ?? string.Empty); + .AutomationName(BuildAccessibleAssistantText(entry.Text, metadata?.AssistantContent)); + } + + private static string BuildAccessibleAssistantText( + string? text, + ChatAssistantContentPresentation? content) + { + var lines = new List(); + if (!string.IsNullOrWhiteSpace(text)) + lines.Add(text); + if (content is not null) + { + var attachmentLabel = ChatAssistantMediaRenderer.LocalizedOrDefault( + "Chat_AssistantMedia_MediaAttachment", + "Media attachment"); + lines.AddRange(content.Media.Select(media => + $"{ChatAssistantMediaRenderer.DisplayName(media)}. {attachmentLabel}")); + } + return string.Join('\n', lines); } private static Element BuildAssistantAvatarSlot(ReactorTimelineRow row) @@ -607,12 +656,12 @@ private static Element CompactIconAction( }); } - private static (string Message, IReadOnlyList Attachments) ParseAttachments(string? text) + private static (string Message, IReadOnlyList Attachments) ParseAttachments(string? text) { const string imagePrefix = "\u200B🖼️ "; const string filePrefix = "\u200B📎 "; var messageLines = new List(); - var attachments = new List(); + var attachments = new List(); foreach (var line in (text ?? string.Empty).Split('\n')) { @@ -621,13 +670,21 @@ private static (string Message, IReadOnlyList Attachments { var name = trimmed[imagePrefix.Length..].Trim(); if (name.Length > 0) - attachments.Add(new ChatAttachmentPreview(name, true)); + attachments.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + name, + "application/octet-stream", + IsImage: true)); } else if (trimmed.StartsWith(filePrefix, StringComparison.Ordinal)) { var name = trimmed[filePrefix.Length..].Trim(); if (name.Length > 0) - attachments.Add(new ChatAttachmentPreview(name, false)); + attachments.Add(new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + name, + "application/octet-stream", + IsImage: false)); } else { @@ -638,10 +695,24 @@ private static (string Message, IReadOnlyList Attachments return (string.Join('\n', messageLines).Trim(), attachments); } - private static Element BuildAttachment(ChatAttachmentPreview attachment) + private static string BuildAccessibleUserText( + string message, + IReadOnlyList attachments) + { + var lines = new List(); + if (message.Length > 0) + lines.Add(message); + lines.AddRange(attachments.Select(attachment => + $"{attachment.DisplayFileName} ({attachment.MimeType})")); + return string.Join('\n', lines); + } + + private static Element BuildAttachment(ChatAttachmentPresentation attachment) { if (attachment.IsImage - && OpenClawChatDataProvider.ImagePreviewCache.TryGetValue(attachment.Name, out var bytes) + && ChatAttachmentPreviewResolver.TryGetBytes( + attachment, + out var bytes) && TryDecodeAttachmentBitmap(bytes) is { } bitmap) { const double maxWidth = 280; @@ -658,7 +729,7 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .Size(pixelWidth * scale, pixelHeight * scale) .CornerRadius(8) .HAlign(HorizontalAlignment.Right) - .AutomationName(attachment.Name); + .AutomationName(attachment.DisplayFileName); } var glyph = Text( @@ -673,7 +744,7 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .CornerRadius(6) .Background(Theme.Ref("SubtleFillColorSecondaryBrush")); var name = Text( - attachment.Name, + attachment.DisplayFileName, 13, FontWeights.Normal, "TextOnAccentFillColorPrimaryBrush") @@ -685,13 +756,25 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) .MaxWidth(240) .VAlign(VerticalAlignment.Center); - return Border(HStack(8, glyphBackground, name)) + var mimeType = Text( + attachment.MimeType, + 11, + FontWeights.Normal, + "TextOnAccentFillColorSecondaryBrush") + .Set(text => + { + text.TextWrapping = TextWrapping.NoWrap; + text.TextTrimming = TextTrimming.CharacterEllipsis; + }) + .MaxWidth(240); + + return Border(HStack(8, glyphBackground, VStack(1, name, mimeType))) .Padding(8, 6, 12, 6) .CornerRadius(6) .BorderThickness(1) .BorderBrush(Theme.Ref("ControlStrokeColorDefaultBrush")) .Background(Theme.Ref("SubtleFillColorSecondaryBrush")) - .AutomationName(attachment.Name); + .AutomationName($"{attachment.DisplayFileName}, {attachment.MimeType}"); } private static BitmapImage? TryDecodeAttachmentBitmap(byte[] bytes) @@ -701,16 +784,9 @@ private static Element BuildAttachment(ChatAttachmentPreview attachment) try { - var stream = new global::Windows.Storage.Streams.InMemoryRandomAccessStream(); - using (var writer = new global::Windows.Storage.Streams.DataWriter(stream)) - { - writer.WriteBytes(bytes); - writer.StoreAsync().AsTask().GetAwaiter().GetResult(); - writer.DetachStream(); - } - stream.Seek(0); - var bitmap = new BitmapImage(); - bitmap.SetSource(stream); + var bitmap = ChatAttachmentBitmapDecoder.TryDecode(bytes); + if (bitmap is null) + return null; s_attachmentBitmaps.Add(bytes, bitmap); return bitmap; } @@ -1026,7 +1102,6 @@ private static string StripMarkdownForSpeech(string text) private static readonly System.Runtime.CompilerServices.ConditionalWeakTable s_attachmentBitmaps = new(); - private sealed record ChatAttachmentPreview(string Name, bool IsImage); } internal sealed record ReactorTimelineRow( diff --git a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw index ae0e8bb49..55b7a8a74 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/en-us/Resources.resw @@ -7003,4 +7003,40 @@ Make sure the gateway is running. Gateway version not supported + + Preparing media + + + Media unavailable + + + Retry + + + Image + + + Audio + + + Video + + + File + + + Media + + + {0} more images not shown + + + Media attachment + + + Open image {0} + + + Close + diff --git a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw index b1cac9614..b605cb141 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/fr-fr/Resources.resw @@ -6963,4 +6963,16 @@ Le binaire wxc-exec est introuvable. {1} S'il s'agit d'une build développeur, c Version de Gateway non prise en charge + Préparation du média + Média indisponible + Réessayer + Illustration + Son + Vidéo + Fichier + Média + {0} images supplémentaires non affichées + Pièce jointe multimédia + Ouvrir l’image {0} + Fermer diff --git a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw index 9b7cb92ab..d4f3d9a7d 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/nl-nl/Resources.resw @@ -6964,4 +6964,16 @@ Het binaire bestand wxc-exec is niet gevonden. {1} Als dit een ontwikkelaarsbuil Gateway-versie wordt niet ondersteund + Media voorbereiden + Media niet beschikbaar + Opnieuw proberen + Afbeelding + Geluid + Videofragment + Bestand + Mediabestand + {0} extra afbeeldingen niet weergegeven + Mediabijlage + Afbeelding {0} openen + Sluiten diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw index 2664efb31..2836babbb 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-cn/Resources.resw @@ -6963,4 +6963,16 @@ 不支持此 Gateway 版本 + 正在准备媒体 + 媒体不可用 + 重试 + 图像 + 音频 + 视频 + 文件 + 媒体 + 还有 {0} 张图像未显示 + 媒体附件 + 打开图像 {0} + 关闭 diff --git a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw index 936750db7..a49abc4d9 100644 --- a/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw +++ b/src/OpenClaw.Tray.WinUI/Strings/zh-tw/Resources.resw @@ -6963,4 +6963,16 @@ 不支援此 Gateway 版本 + 正在準備媒體 + 媒體無法使用 + 重試 + 圖片 + 音訊 + 影片 + 檔案 + 媒體 + 還有 {0} 張圖片未顯示 + 媒體附件 + 開啟圖片 {0} + 關閉 diff --git a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs index 93d1eb23e..8ea435be0 100644 --- a/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs +++ b/tests/OpenClaw.Connection.Tests/GatewayConnectionManagerTests.cs @@ -88,6 +88,223 @@ public async Task ConnectAsync_WithCredential_TransitionsToConnecting() Assert.Equal("test", _manager.CurrentSnapshot.OperatorCredentialSource); } + [Fact] + public async Task ConnectAsync_PrefersSharedTokenForInteractiveHttpSurfaces() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-1", + Url = "wss://test", + SharedGatewayToken = "shared-http-token", + }); + _registry.SetActive("gw-1"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + + var created = Assert.Single(_factory.CreatedCredentials); + Assert.Equal("paired-device-token", created.Token); + Assert.Equal("shared-http-token", created.InteractiveHttpToken); + } + + [Fact] + public async Task ConnectAsync_BootstrapOnlyDisablesAssistantMediaHttpAuth() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-1", + Url = "wss://test", + BootstrapToken = "bootstrap-token", + }); + _registry.SetActive("gw-1"); + _resolver.OperatorCredential = new GatewayCredential( + "bootstrap-token", + true, + CredentialResolver.SourceBootstrapToken); + + await _manager.ConnectAsync("gw-1"); + + var created = Assert.Single(_factory.CreatedCredentials); + Assert.Equal("bootstrap-token", created.Token); + Assert.Equal(string.Empty, created.InteractiveHttpToken); + Assert.Null(Assert.Single(_factory.CreatedClients).AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RefreshesHttpTokenAcrossProvenanceChanges() + { + _registry.AddOrUpdate(new GatewayRecord + { + Id = "gw-local", + Url = "ws://localhost:18789", + IsLocal = true, + SetupManagedDistroName = "OpenClawGateway", + SharedGatewayToken = "shared-http-token", + }); + _registry.SetActive("gw-local"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + var provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789); + using var manager = new GatewayConnectionManager( + _resolver, + _factory, + _registry, + NullLogger.Instance, + endpointProvenanceProbe: (_, _) => Task.FromResult(provenance)); + + await manager.ConnectAsync("gw-local"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("shared-http-token", client.AssistantMediaAuthToken); + + provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.UnknownListener, + 18789, + ProcessId: 42, + ProcessName: "unknown"); + var deniedHttp = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(deniedHttp.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + + provenance = new GatewayEndpointProvenance( + GatewayEndpointProvenanceKind.ExpectedManagedGateway, + 18789); + var restoredHttp = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(restoredHttp.Allowed); + Assert.Equal("shared-http-token", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RefreshesFallbackDeviceCredential() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-1", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-1", client.AssistantMediaAuthToken); + + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-2", + false, + CredentialResolver.SourceDeviceToken); + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(reconnect.Allowed); + Assert.Equal("paired-device-token-2", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RejectsSameUrlTrustConfigurationChange() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.AddOrUpdate(_registry.GetById("gw-1")! with + { + IsLocal = true, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.False(reconnect.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_AllowsRuntimeV2SignatureUpgrade() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = true, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.True(reconnect.Allowed); + Assert.Equal("paired-device-token", client.AssistantMediaAuthToken); + } + + [Fact] + public async Task ReconnectAuthorization_RejectsRuntimeV2SignatureDowngrade() + { + SetupGateway("gw-1", "wss://test"); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = true, + }); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + _registry.Update("gw-1", record => record with + { + RequiresV2Signature = false, + }); + + var reconnect = await client.DataClient.ReconnectAuthorizationAsync!( + CancellationToken.None); + + Assert.False(reconnect.Allowed); + Assert.Null(client.AssistantMediaAuthToken); + } + + [Fact] + public async Task OperatorDeviceTokenReceived_RefreshesAssistantMediaAuthWithoutReconnect() + { + SetupGateway("gw-1", "wss://test"); + _resolver.OperatorCredential = new GatewayCredential( + "paired-device-token-1", + false, + CredentialResolver.SourceDeviceToken); + + await _manager.ConnectAsync("gw-1"); + var client = Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-1", client.AssistantMediaAuthToken); + + client.SimulateDeviceTokenReceived( + "paired-device-token-2", + "operator", + ["operator.read"]); + await WaitUntilAsync( + () => client.AssistantMediaAuthToken == "paired-device-token-2"); + + Assert.Single(_factory.CreatedClients); + Assert.Equal("paired-device-token-2", client.AssistantMediaAuthToken); + } + [Fact] public async Task ConnectAndReconnect_EmitCompletedOperatorSpans() { @@ -4968,7 +5185,10 @@ public IGatewayClientLifecycle Create(string gatewayUrl, GatewayCredential crede if (CreateException != null) throw CreateException; - var mock = new MockLifecycle(gatewayUrl, identityPath); + var mock = new MockLifecycle( + gatewayUrl, + identityPath, + credential.InteractiveHttpToken); CreatedClients.Add(mock); CreatedCredentials.Add(credential); CreatedIdentityPaths.Add(identityPath); @@ -5017,12 +5237,19 @@ internal sealed class MockLifecycle : IGatewayClientLifecycle { private readonly MockGatewayClient _client; - public MockLifecycle(string url, string identityPath) + public MockLifecycle( + string url, + string identityPath, + string? assistantMediaAuthToken = null) { - _client = new MockGatewayClient(url, identityPath); + _client = new MockGatewayClient( + url, + identityPath, + assistantMediaAuthToken); } public OpenClawGatewayClient DataClient => _client; + public string? AssistantMediaAuthToken => _client.AssistantMediaAuthToken; public bool IsDisposed { get; private set; } public event EventHandler? StatusChanged; public event EventHandler? AuthenticationFailed; @@ -5066,8 +5293,30 @@ private sealed class MockGatewayClient : OpenClawGatewayClient { private bool _isConnected = true; - public MockGatewayClient(string url, string identityPath) - : base(url, "mock-token", NullLogger.Instance, identityPath: identityPath) { } + public MockGatewayClient( + string url, + string identityPath, + string? assistantMediaAuthToken = null) + : base( + url, + "mock-token", + NullLogger.Instance, + identityPath: identityPath, + assistantMediaAuthToken: assistantMediaAuthToken) { } + + public string? AssistantMediaAuthToken + { + get + { + var fieldInfo = typeof(OpenClawGatewayClient).GetField( + "_assistantMediaAuthToken", + System.Reflection.BindingFlags.Instance | + System.Reflection.BindingFlags.NonPublic) + ?? throw new InvalidOperationException( + "Assistant media auth token field was not found."); + return fieldInfo.GetValue(this) as string; + } + } public override bool IsConnectedToGateway => _isConnected; diff --git a/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs b/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs new file mode 100644 index 000000000..8f4ab3aca --- /dev/null +++ b/tests/OpenClaw.Shared.Tests/AssistantMediaDirectiveParserTests.cs @@ -0,0 +1,140 @@ +using OpenClaw.Shared; + +namespace OpenClaw.Shared.Tests; + +public sealed class AssistantMediaDirectiveParserTests +{ + [Fact] + public void Project_AssistantAbsolutePath_ProducesMediaWithoutExposingPath() + { + const string raw = + "Here is the image.\nMEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal("Here is the image.", projection.Text); + Assert.DoesNotContain("/home/openclaw", projection.Text, StringComparison.Ordinal); + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentKind.Image, media.Kind); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Source); + Assert.Equal("banner.png", media.FileName); + } + + [Fact] + public void Project_UserDirective_RemainsInertText() + { + const string raw = "MEDIA:/home/openclaw/private.png"; + + var projection = AssistantMediaDirectiveParser.Project("user", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void Project_FencedDirective_RemainsInertText() + { + const string raw = "```text\nMEDIA:/home/openclaw/example.png\n```"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void Project_MidLineDirective_RemainsInertText() + { + const string raw = "Example: MEDIA:/home/openclaw/example.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(raw, projection.Text); + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + } + + [Theory] + [InlineData("MEDIA:../../.env")] + [InlineData("MEDIA:~someone/private.png")] + [InlineData("MEDIA:file:///home/openclaw/../private.png")] + public void Project_InvalidPathLikeDirective_RedactsSource(string raw) + { + var projection = AssistantMediaDirectiveParser.Project("assistant", raw); + + Assert.Equal(string.Empty, projection.Text); + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentSource.Unavailable, media.Source); + Assert.Null(media.GatewaySource); + } + + [Fact] + public void Project_QuotedPathWithSpaces_ProducesSingleMediaReference() + { + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + "MEDIA:\"/home/openclaw/My Images/banner light.png\""); + + var media = Assert.Single( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media).Media; + Assert.NotNull(media); + Assert.Equal("banner light.png", media.FileName); + } + + [Fact] + public void Project_MultipleIndependentSources_PreservesOrder() + { + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + "MEDIA:/tmp/one.png /tmp/two.mp4"); + + var media = projection.ContentParts + .Where(part => part.Kind == ChatMessageContentPartKind.Media) + .Select(part => part.Media) + .ToArray(); + Assert.Collection( + media, + item => Assert.Equal("one.png", item?.FileName), + item => Assert.Equal("two.mp4", item?.FileName)); + } + + [Fact] + public void Project_TooManySources_CapsMediaReferencesAndRedactsRemainder() + { + var sources = Enumerable.Range(1, AssistantMediaDirectiveParser.MaxMediaReferences + 5) + .Select(index => $"/tmp/image-{index}.png"); + var projection = AssistantMediaDirectiveParser.Project( + "assistant", + $"MEDIA:{string.Join(' ', sources)}"); + + Assert.Equal(string.Empty, projection.Text); + Assert.Equal( + AssistantMediaDirectiveParser.MaxMediaReferences, + projection.ContentParts.Count(part => part.Kind == ChatMessageContentPartKind.Media)); + } + + [Fact] + public void Project_Ipv4MappedPrivateHttpsSource_RemainsInert() + { + const string text = "MEDIA:https://[::ffff:192.168.1.1]/private.png"; + + var projection = AssistantMediaDirectiveParser.Project("assistant", text); + + Assert.DoesNotContain( + projection.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + Assert.Equal(text, projection.Text); + } +} diff --git a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs new file mode 100644 index 000000000..829041c83 --- /dev/null +++ b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientAssistantMediaTests.cs @@ -0,0 +1,275 @@ +using System.Net; +using System.Net.Http.Headers; +using System.Text; +using System.Text.Json; +using OpenClaw.TestSupport; + +namespace OpenClaw.Shared.Tests; + +public sealed class OpenClawGatewayClientAssistantMediaTests +{ + [Fact] + public async Task ResolveStructuredMedia_InlineBase64_ReturnsBoundedTypedBytes() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "test-token", + identityPath: identity.Path); + await client.ConnectAsync(); + + var resolution = client.ResolveAssistantMediaAsync( + "main", + new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + }); + var request = await server.ReceiveTextAsync().WaitAsync(TimeSpan.FromSeconds(2)); + using var requestDocument = JsonDocument.Parse(request); + Assert.Equal("artifacts.download", requestDocument.RootElement.GetProperty("method").GetString()); + Assert.Equal( + "main", + requestDocument.RootElement.GetProperty("params").GetProperty("sessionKey").GetString()); + + await server.SendTextAsync(JsonSerializer.Serialize(new + { + type = "res", + id = requestDocument.RootElement.GetProperty("id").GetString(), + ok = true, + payload = new + { + artifact = new + { + id = "artifact-1", + type = "image", + title = "banner.png", + mimeType = "image/png", + sizeBytes = 4, + download = new { mode = "bytes" }, + }, + encoding = "base64", + data = Convert.ToBase64String([1, 2, 3, 4]), + }, + })); + + var result = await resolution.WaitAsync(TimeSpan.FromSeconds(2)); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.Equal("image/png", result.MimeType); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, result.Data); + } + + [Fact] + public async Task ResolveLegacyMedia_UsesBearerMetadataAndSourceBoundTicket() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaAuthToken: "shared-http-token", + assistantMediaHandler: handler); + await client.ConnectAsync(); + + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }; + var result = await client.ResolveAssistantMediaAsync( + "main", + media); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.Equal(new byte[] { 1, 2, 3, 4 }, result.Data); + Assert.Equal(2, handler.Requests.Count); + Assert.All( + handler.Requests, + request => Assert.Equal("shared-http-token", request.AuthorizationParameter)); + Assert.Contains("meta=1", handler.Requests[0].Uri.Query, StringComparison.Ordinal); + Assert.Contains("mediaTicket=ticket-1", handler.Requests[1].Uri.Query, StringComparison.Ordinal); + Assert.Contains( + "source=%2Fhome%2Fopenclaw%2Fprivate%2Fbanner.png", + handler.Requests[1].Uri.Query, + StringComparison.Ordinal); + } + + [Fact] + public async Task ResolveLegacyMedia_WithoutExplicitHttpCredential_DoesNotUseWebSocketToken() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaHandler: handler); + await client.ConnectAsync(); + + var result = await client.ResolveAssistantMediaAsync( + "main", + new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }); + + Assert.Equal(AssistantMediaResolutionStatus.Ready, result.Status); + Assert.All(handler.Requests, request => Assert.Null(request.AuthorizationParameter)); + } + + [Fact] + public async Task ResolveLegacyMedia_UsesUpdatedHttpCredential() + { + using var server = new LoopbackWebSocketServer(); + using var identity = new TempDirectory("assistant-media-"); + await server.StartAsync(); + var handler = new SequentialMediaHandler( + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-1"}"""), + BytesResponse([1, 2, 3, 4], "image/png"), + JsonResponse( + """{"available":true,"mimeType":"image/png","sizeBytes":4,"mediaTicket":"ticket-2"}"""), + BytesResponse([1, 2, 3, 4], "image/png")); + using var client = new OpenClawGatewayClient( + server.WebSocketUrl, + "paired-device-token", + identityPath: identity.Path, + assistantMediaAuthToken: "shared-token-1", + assistantMediaHandler: handler); + await client.ConnectAsync(); + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + GatewaySource = "/home/openclaw/private/banner.png", + }; + + await client.ResolveAssistantMediaAsync("main", media); + client.SetAssistantMediaAuthToken("shared-token-2"); + await client.ResolveAssistantMediaAsync("main", media); + + Assert.All( + handler.Requests.Take(2), + request => Assert.Equal("shared-token-1", request.AuthorizationParameter)); + Assert.All( + handler.Requests.Skip(2), + request => Assert.Equal("shared-token-2", request.AuthorizationParameter)); + } + + [Fact] + public void TryDecodeBoundedBase64_RejectsDecodedSizeOverflow() + { + const int maximumBytes = 4; + var encoded = string.Concat(new string(' ', 1024), "AQIDBAUG"); + + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + encoded, + maximumBytes, + out var bytes); + + Assert.False(decoded); + Assert.Empty(bytes); + } + + [Fact] + public void TryDecodeBoundedBase64_RejectsOversizedCompactInputBeforeNormalization() + { + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + " AQIDBAUGB ", + maximumBytes: 4, + out var bytes); + + Assert.False(decoded); + Assert.Empty(bytes); + } + + [Fact] + public void TryDecodeBoundedBase64_DecodesWhitespaceAndUnpaddedBase64Url() + { + var decoded = OpenClawGatewayClient.TryDecodeBoundedBase64( + " AQI-\n_w ", + maximumBytes: 5, + out var bytes); + + Assert.True(decoded); + Assert.Equal(new byte[] { 1, 2, 62, 255 }, bytes); + } + + [Theory] + [InlineData("/api/chat/media/outgoing/item?mediaTicket=ticket", true)] + [InlineData("/api/chat/media/outgoing/item", false)] + [InlineData("/api/chat/media/outgoing/../secret?mediaTicket=ticket", false)] + [InlineData("//other.example/api/chat/media/outgoing/item?mediaTicket=ticket", false)] + [InlineData("https://other.example/api/chat/media/outgoing/item?mediaTicket=ticket", false)] + public void TryResolveManagedMediaUri_EnforcesGatewayRelativeTicketPath( + string path, + bool expected) + { + var baseUri = new Uri("https://gateway.example/base"); + + var resolved = OpenClawGatewayClient.TryResolveManagedMediaUri(baseUri, path, out var uri); + + Assert.Equal(expected, resolved); + if (expected) + Assert.Equal("gateway.example", uri.Host); + } + + private static HttpResponseMessage JsonResponse(string json) => + new(HttpStatusCode.OK) + { + Content = new StringContent(json, Encoding.UTF8, "application/json"), + }; + + private static HttpResponseMessage BytesResponse(byte[] data, string mimeType) + { + var response = new HttpResponseMessage(HttpStatusCode.OK) + { + Content = new ByteArrayContent(data), + }; + response.Content.Headers.ContentType = new MediaTypeHeaderValue(mimeType); + return response; + } + + private sealed class SequentialMediaHandler(params HttpResponseMessage[] responses) + : HttpMessageHandler + { + private int _nextResponse; + + public List Requests { get; } = new(); + + protected override Task SendAsync( + HttpRequestMessage request, + CancellationToken cancellationToken) + { + Requests.Add(new CapturedRequest( + request.RequestUri!, + request.Headers.Authorization?.ToString(), + request.Headers.Authorization?.Parameter)); + var index = Interlocked.Increment(ref _nextResponse) - 1; + return Task.FromResult(responses[index]); + } + } + + private sealed record CapturedRequest( + Uri Uri, + string? Authorization, + string? AuthorizationParameter); +} diff --git a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs index b469d332e..0c3b0f45d 100644 --- a/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs +++ b/tests/OpenClaw.Shared.Tests/OpenClawGatewayClientTests.cs @@ -2006,6 +2006,177 @@ public void ParseChatHistoryPayload_InterleavedBlocks_PreserveSourceOrder() }); } + [Fact] + public void ParseChatHistoryPayload_StructuredMedia_PreservesTypedFieldsAndOrder() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + { "type": "text", "text": "Created it." }, + { + "type": "image", + "mimeType": "image/png", + "fileName": "banner.png", + "artifactId": "artifact_managed_image_123", + "alt": "OpenClaw banner", + "width": 1200, + "height": 774, + "sizeBytes": 12345 + }, + { "type": "text", "text": "Finished." } + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal("Created it.\nFinished.", message.Text); + Assert.Collection( + message.ContentParts, + part => + { + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Equal("Created it.", part.Text); + }, + part => + { + Assert.Equal(ChatMessageContentPartKind.Media, part.Kind); + Assert.Equal(ChatMediaContentKind.Image, part.Media?.Kind); + Assert.Equal("image/png", part.Media?.MimeType); + Assert.Equal("banner.png", part.Media?.FileName); + Assert.Equal("artifact_managed_image_123", part.Media?.ArtifactId); + Assert.Equal(1200, part.Media?.Width); + Assert.Equal(774, part.Media?.Height); + }, + part => + { + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Equal("Finished.", part.Text); + }); + } + + [Fact] + public void ParseChatHistoryPayload_LegacyMediaOnly_PreservesMessageAndRedactsPath() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png", + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal(string.Empty, message.Text); + var media = Assert.Single(message.ContentParts).Media; + Assert.NotNull(media); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Source); + Assert.Equal("banner.png", media.FileName); + } + + [Fact] + public void ParseChatHistoryPayload_StringArray_RedactsLegacyMediaPath() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + "Created it.", + "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png" + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + Assert.Equal("Created it.", message.Text); + Assert.DoesNotContain("/home/openclaw", message.Text, StringComparison.Ordinal); + Assert.Single( + message.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Media); + Assert.Equal( + "Created it.", + Assert.Single( + message.ContentParts, + part => part.Kind == ChatMessageContentPartKind.Text).Text); + } + + [Fact] + public void ParseChatHistoryPayload_SplitFence_KeepsMediaDirectiveAsTextOnly() + { + var helper = new GatewayClientTestHelper(); + + var history = helper.ParseChatHistoryPayload(""" + { + "messages": [ + { + "role": "assistant", + "content": [ + "```", + "MEDIA:/home/openclaw/private.png\n```" + ], + "timestamp": 1 + } + ] + } + """); + + var message = Assert.Single(history.Messages); + var part = Assert.Single(message.ContentParts); + Assert.Equal(ChatMessageContentPartKind.Text, part.Kind); + Assert.Contains("MEDIA:/home/openclaw/private.png", part.Text, StringComparison.Ordinal); + Assert.DoesNotContain( + message.ContentParts, + contentPart => contentPart.Kind == ChatMessageContentPartKind.Media); + } + + [Fact] + public void ChatEvent_LegacyMediaOnly_RaisesTypedMessageWithoutRawPath() + { + var helper = new GatewayClientTestHelper(); + ChatMessageInfo? received = null; + helper.Client.ChatMessageReceived += (_, message) => received = message; + + helper.ProcessRawMessage(""" + { + "type": "event", + "event": "chat", + "payload": { + "sessionKey": "main", + "state": "final", + "message": { + "role": "assistant", + "content": "MEDIA:/home/openclaw/.openclaw/workspace/downloads/banner.png" + } + } + } + """); + + Assert.NotNull(received); + Assert.Equal(string.Empty, received.Text); + var media = Assert.Single(received.ContentParts).Media; + Assert.Equal("banner.png", media?.FileName); + } + [Fact] public void ParseChatHistoryPayload_OpenClawMetadata_PreservesMessageIdentity() { diff --git a/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs b/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs new file mode 100644 index 000000000..dadf86513 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ChatAssistantContentPresentationTests.cs @@ -0,0 +1,186 @@ +using OpenClaw.Shared; +using OpenClawTray.Chat; + +namespace OpenClaw.Tray.Tests; + +public sealed class ChatAssistantContentPresentationTests +{ + [Theory] + [InlineData(1200u, 774u, 1200, 774)] + [InlineData(4096u, 1024u, 2048, 512)] + public void ImageDecodePolicy_BoundsDecodeSize( + uint width, + uint height, + int expectedWidth, + int expectedHeight) + { + Assert.True(ChatAssistantImageDecodePolicy.TryGetDecodeSize( + width, + height, + out var decodeWidth, + out var decodeHeight)); + Assert.Equal(expectedWidth, decodeWidth); + Assert.Equal(expectedHeight, decodeHeight); + } + + [Theory] + [InlineData(1u, 20_000u)] + [InlineData(16_384u, 16_384u)] + public void ImageDecodePolicy_RejectsUnsafeDimensions(uint width, uint height) + { + Assert.False(ChatAssistantImageDecodePolicy.TryGetDecodeSize( + width, + height, + out _, + out _)); + } + + [Fact] + public void Project_StructuredPathLikeFileName_UsesLeafName() + { + var presentation = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + FileName = "/home/openclaw/private/banner.png", + }, + }, + ]); + + Assert.Equal("banner.png", Assert.Single(presentation!.Media).DisplayName); + } + + [Fact] + public void Project_UsesSafeFilenameWithoutExposingLegacySource() + { + var media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }; + + var presentation = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = media, + }, + ]); + + var item = Assert.Single(presentation!.Media); + Assert.Equal("banner.png", item.DisplayName); + Assert.DoesNotContain("/", item.DisplayName, StringComparison.Ordinal); + } + + [Fact] + public void HistoryProjection_MediaPart_RemainsChronological() + { + var message = new ChatMessageInfo + { + Role = "assistant", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = "Before", + }, + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + FileName = "banner.png", + }, + }, + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Text, + Text = "After", + }, + ], + }; + + var parts = ChatHistoryReplayProjection.Project([message]).ToArray(); + + Assert.Collection( + parts, + part => Assert.Equal("Before", part.Text), + part => Assert.Equal( + ChatMessageContentPartKind.Media, + Assert.Single(part.AssistantContentParts).Kind), + part => Assert.Equal("After", part.Text)); + } + + [Fact] + public void BuildRenderPlan_CapsImagesWithoutReorderingOtherMedia() + { + var media = Enumerable.Range(1, 5) + .Select(index => Presentation(ChatMediaContentKind.Image, $"image-{index}.png")) + .Append(Presentation(ChatMediaContentKind.Audio, "audio.mp3")) + .ToArray(); + + var plan = ChatAssistantContentProjector.BuildRenderPlan(media); + + Assert.Equal(1, plan.OmittedImages); + Assert.Equal( + new[] { "image-1.png", "image-2.png", "image-3.png", "image-4.png", "audio.mp3" }, + plan.Media.Select(item => item.DisplayName)); + } + + [Fact] + public void MergeLiveUpdate_DoesNotReplaceLegacyReferenceWithStructuredReference() + { + var legacy = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ])!; + var structured = ChatAssistantContentProjector.Project( + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-unavailable", + }, + }, + ])!; + + var merged = ChatAssistantContentProjector.MergeLiveUpdate(legacy, structured); + + Assert.Equal( + ChatMediaContentSource.LegacyDirective, + Assert.Single(merged.Media).Reference.Source); + } + + private static ChatAssistantMediaPresentation Presentation( + ChatMediaContentKind kind, + string name) => + new( + kind, + name, + null, + null, + new ChatMediaContentInfo { Kind = kind, FileName = name }); +} diff --git a/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs new file mode 100644 index 000000000..89985e896 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/ChatAssistantMediaRendererContractTests.cs @@ -0,0 +1,84 @@ +namespace OpenClaw.Tray.Tests; + +public sealed class ChatAssistantMediaRendererContractTests +{ + [Fact] + public void ImageViewer_UsesRegisteredContentDialogFactory() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + + Assert.Contains("var viewer = ContentDialog(", source, StringComparison.Ordinal); + Assert.DoesNotContain("new ContentDialogElement(", source, StringComparison.Ordinal); + } + + [Fact] + public void ImageLoader_DoesNotRenderStateFromPreviousReference() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + + Assert.Contains( + "ReferenceEquals(state.Reference, props.Media.Reference)", + source, + StringComparison.Ordinal); + Assert.Contains( + "props.SessionKey, props.Media.Reference, attempt", + source, + StringComparison.Ordinal); + Assert.Contains( + "AssistantMediaResolutionStatus Status", + source, + StringComparison.Ordinal); + Assert.DoesNotContain( + "AssistantMediaResolutionResult Result", + source, + StringComparison.Ordinal); + } + + [Theory] + [InlineData("ReactorChatTimeline.cs")] + [InlineData("OpenClawChatTimeline.cs")] + public void LocalAttachmentRenderer_UsesBoundedSharedDecoder(string fileName) + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + fileName)); + + Assert.Contains( + "ChatAttachmentBitmapDecoder.TryDecode(bytes)", + source, + StringComparison.Ordinal); + Assert.DoesNotContain("bitmap.SetSource(stream)", source, StringComparison.Ordinal); + Assert.DoesNotContain("bmp.SetSource(stream)", source, StringComparison.Ordinal); + } + + [Fact] + public void SharedAttachmentDecoder_EnforcesPixelPolicyBeforeSetSource() + { + var source = File.ReadAllText(Path.Combine( + TestRepositoryPaths.GetRepositoryRoot(), + "src", + "OpenClaw.Tray.WinUI", + "Chat", + "ChatAssistantMediaRenderer.cs")); + var policy = source.IndexOf( + "ChatAssistantImageDecodePolicy.TryGetDecodeSize(", + StringComparison.Ordinal); + var setSource = source.LastIndexOf("bitmap.SetSource(stream)", StringComparison.Ordinal); + + Assert.True(policy >= 0); + Assert.True(setSource > policy); + } +} diff --git a/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs new file mode 100644 index 000000000..02b0878a6 --- /dev/null +++ b/tests/OpenClaw.Tray.Tests/GatewayMediaMessageProjectionTests.cs @@ -0,0 +1,363 @@ +using OpenClaw.Shared; +using OpenClawTray.Chat; + +namespace OpenClaw.Tray.Tests; + +public class GatewayMediaMessageProjectionTests +{ + private const string ObservedLocalFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png"; + private const string ObservedGatewayFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png"; + + [Fact] + public void ValidEnvelope_ProjectsSafeDescriptorAndCleanProse() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/pasted%20image---7f122605-290a-467c-a5df-8a744c093004.png (image/PNG)]\r\nDescribe this"); + + Assert.True(projection.HasMediaEnvelope); + Assert.Equal("Describe this", projection.ReconciliationText); + Assert.Equal("Describe this", projection.ResidualText); + var attachment = Assert.Single(projection.Attachments); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("pasted image.png", attachment.DisplayFileName); + Assert.Equal("image/png", attachment.MimeType); + Assert.True(attachment.IsImage); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public void MultipleEnvelopeLines_PreserveOrderAndFileClassification() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/photo.jpg (image/jpeg)]\n" + + "[media attached: media://inbound/notes.txt (text/plain)]\nhello"); + + Assert.Collection( + projection.Attachments, + image => + { + Assert.Equal("photo.jpg", image.DisplayFileName); + Assert.True(image.IsImage); + }, + file => + { + Assert.Equal("notes.txt", file.DisplayFileName); + Assert.False(file.IsImage); + }); + Assert.NotEmpty(projection.AttachmentPresentationSignature); + Assert.NotEmpty(projection.AttachmentCorrelationSignature); + } + + [Fact] + public void EncodedPathAndControls_AreReducedToPrintableBoundedLeaf() + { + var longName = new string('a', 220); + var projection = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/folder%2F..%2F{longName}%0Aname.txt (text/plain)]"); + + var attachment = Assert.Single(projection.Attachments); + Assert.DoesNotContain("/", attachment.DisplayFileName); + Assert.DoesNotContain("\n", attachment.DisplayFileName); + Assert.True(attachment.DisplayFileName.Length <= 160); + } + + [Theory] + [InlineData("[media attached: https://inbound/file.png (image/png)]")] + [InlineData("[media attached: media://outbound/file.png (image/png)]")] + [InlineData("[media attached: media://inbound/file.png (not a mime)]")] + [InlineData("[media attached: media://inbound/file.png image/png)]")] + [InlineData("[media attached: media://inbound/ (image/png)]")] + public void MalformedOrUnsupportedLookalike_RemainsOrdinaryProse(string text) + { + var projection = GatewayMediaMessageProjection.Project(text); + + Assert.False(projection.HasMediaEnvelope); + Assert.Empty(projection.Attachments); + Assert.Equal(text, projection.ResidualText); + } + + [Fact] + public void EmbeddedEnvelope_RemainsOrdinaryProse() + { + const string text = "User-authored text\n[media attached: media://inbound/file.png (image/png)]"; + + var projection = GatewayMediaMessageProjection.Project(text); + + Assert.False(projection.HasMediaEnvelope); + Assert.Equal(text, projection.ResidualText); + } + + [Fact] + public void GatewayProjection_NeverEmitsPrivateMarkersOrPreviewKeys() + { + var projection = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/file.png (image/png)]"); + + Assert.False(projection.ResidualText.Contains('\u200B')); + Assert.All(projection.Attachments, attachment => + { + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.False(attachment.CanAccessPreviewCache); + }); + } + + [Fact] + public void LocalPresentations_UseOpaquePreviewKeysAndOriginalSafeNames() + { + var presentations = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = @"C:\fake\original.png", + }, + ], + () => "opaque-key"); + + var presentation = Assert.Single(presentations); + Assert.Equal(ChatAttachmentOrigin.Local, presentation.Origin); + Assert.Equal("original.png", presentation.DisplayFileName); + Assert.Equal("opaque-key", presentation.PreviewCacheKey); + Assert.True(presentation.CanAccessPreviewCache); + } + + [Fact] + public void LocalPresentations_PreserveCanonicalLookingNamesAndSourceOrder() + { + const string canonicalLookingName = + "report---7f122605-290a-467c-a5df-8a744c093004.png"; + var keys = new Queue(["first-key", "second-key"]); + var presentations = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "", + }, + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = canonicalLookingName, + }, + ], + () => keys.Dequeue()); + + Assert.Collection( + presentations, + first => + { + Assert.Equal("image", first.DisplayFileName); + Assert.Equal("first-key", first.PreviewCacheKey); + }, + second => + { + Assert.Equal(canonicalLookingName, second.DisplayFileName); + Assert.Equal("second-key", second.PreviewCacheKey); + }); + } + + [Fact] + public void ObservedGatewayRewrite_ChangesPresentationButPreservesCorrelationSignature() + { + var local = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = ObservedLocalFileName, + }, + ], + () => "local-preview"); + var gateway = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/{ObservedGatewayFileName} (image/png)]\nidentical caption"); + + Assert.NotEqual( + GatewayMediaMessageProjection.BuildAttachmentPresentationSignature(local), + gateway.AttachmentPresentationSignature); + Assert.Equal( + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(local), + gateway.AttachmentCorrelationSignature); + } + + [Fact] + public void RemoteSameFilename_CannotResolveLocalPreviewBytes() + { + var bytes = new byte[] { 1, 2, 3 }; + var cache = new Dictionary + { + ["opaque-local-key"] = bytes, + ["same.png"] = new byte[] { 9 }, + }; + var local = new ChatAttachmentPresentation( + ChatAttachmentOrigin.Local, + "same.png", + "image/png", + IsImage: true, + PreviewCacheKey: "opaque-local-key"); + var remote = new ChatAttachmentPresentation( + ChatAttachmentOrigin.GatewayReference, + "same.png", + "image/png", + IsImage: true); + + Assert.True(ChatAttachmentPreviewResolver.TryGetBytes(local, cache, out var resolved)); + Assert.Same(bytes, resolved); + Assert.False(ChatAttachmentPreviewResolver.TryGetBytes(remote, cache, out _)); + } + + [Fact] + public void PreviewCache_RejectsDecodedOverflowBeforeStorage() + { + Assert.False(ChatImagePreviewCache.TryDecodeBoundedBase64( + Convert.ToBase64String([1, 2, 3, 4, 5]), + maximumBytes: 4, + out var bytes)); + Assert.Empty(bytes); + } + + [Fact] + public void PreviewCache_EvictsOldestEntriesWithinCountBound() + { + ChatImagePreviewCache.Clear(); + try + { + for (var index = 0; index <= ChatImagePreviewCache.MaximumEntries; index++) + { + Assert.True(ChatImagePreviewCache.TryStoreBase64( + $"preview-{index}", + Convert.ToBase64String([(byte)index]))); + } + + Assert.Equal(ChatImagePreviewCache.MaximumEntries, ChatImagePreviewCache.Count); + Assert.False(ChatImagePreviewCache.Contains("preview-0")); + Assert.True(ChatImagePreviewCache.Contains( + $"preview-{ChatImagePreviewCache.MaximumEntries}")); + Assert.True(ChatImagePreviewCache.TotalBytes <= + ChatImagePreviewCache.MaximumTotalBytes); + } + finally + { + ChatImagePreviewCache.Clear(); + } + } + + [Fact] + public void AttachmentOnlyEcho_AmbiguousMatchingCandidatesAreNotConsumed() + { + var incoming = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/same.png (image/png)]"); + var candidates = new[] + { + new ChatPendingEchoCandidate("one", "", incoming.AttachmentCorrelationSignature), + new ChatPendingEchoCandidate("two", "", incoming.AttachmentCorrelationSignature), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void PlainTextEcho_SingleMediaCandidateIsNotConsumed() + { + var media = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/same.png (image/png)]\nsame caption"); + var plain = GatewayMediaMessageProjection.Project("same caption"); + var candidates = new[] + { + new ChatPendingEchoCandidate("media", "same caption", media.AttachmentCorrelationSignature), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, plain)); + } + + [Fact] + public void MediaEcho_RequiresMatchingAttachmentCorrelationSignature() + { + var incoming = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/right.png (image/png)]\ncaption"); + var wrong = GatewayMediaMessageProjection.Project( + "[media attached: media://inbound/wrong.pdf (application/pdf)]\ncaption"); + var candidates = new[] + { + new ChatPendingEchoCandidate("wrong", "caption", wrong.AttachmentCorrelationSignature), + new ChatPendingEchoCandidate("right", "caption", incoming.AttachmentCorrelationSignature), + }; + + Assert.Equal( + "right", + ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void CaptionedMediaEcho_WithTwoSameMimeCandidatesIsAmbiguousAfterFilenameRewrite() + { + var incoming = GatewayMediaMessageProjection.Project( + $"[media attached: media://inbound/{ObservedGatewayFileName} (image/png)]\nsame caption"); + var firstLocal = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = ObservedLocalFileName, + }, + ], + () => "first-preview"); + var secondLocal = GatewayMediaMessageProjection.CreateLocalPresentations( + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "another-local-name.png", + }, + ], + () => "second-preview"); + var candidates = new[] + { + new ChatPendingEchoCandidate( + "first", + "same caption", + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(firstLocal)), + new ChatPendingEchoCandidate( + "second", + "same caption", + GatewayMediaMessageProjection.BuildAttachmentCorrelationSignature(secondLocal)), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void NoEnvelopeEcho_MixedPlainAndAttachmentCandidatesAreAmbiguous() + { + var incoming = GatewayMediaMessageProjection.Project("same prose"); + var candidates = new[] + { + new ChatPendingEchoCandidate("media", "same prose", "1|9:image/png|8:file.png|"), + new ChatPendingEchoCandidate("plain", "same prose", string.Empty), + }; + + Assert.Null(ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } + + [Fact] + public void NoEnvelopeEcho_AllPlainCandidatesPreserveFifo() + { + var incoming = GatewayMediaMessageProjection.Project("same prose"); + var candidates = new[] + { + new ChatPendingEchoCandidate("first", "same prose", string.Empty), + new ChatPendingEchoCandidate("second", "same prose", string.Empty), + }; + + Assert.Equal( + "first", + ChatAttachmentEchoCorrelation.SelectMatchingMessageId(candidates, incoming)); + } +} diff --git a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj index cacaa3045..27f309181 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj +++ b/tests/OpenClaw.Tray.Tests/OpenClaw.Tray.Tests.csproj @@ -38,6 +38,11 @@ + + + + + diff --git a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs index 72dad8108..90666ab59 100644 --- a/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs +++ b/tests/OpenClaw.Tray.Tests/OpenClawChatDataProviderTests.cs @@ -2826,6 +2826,88 @@ public async Task ChatMessageReceived_UserEcho_DoesNotReconcileStaleIdenticalCon !afterMeta[user.Id].IsLocalQueuedSend); } + [Fact] + public async Task ChatMessageReceived_NoEnvelopeEchoDoesNotChooseBetweenPlainAndMediaPromotions() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-media", Status = "started" }); + await provider.SendMessageAsync("main", "same", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "same.png", + Content = Convert.ToBase64String([1]), + }, + ]); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "run-media")); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"end"}""", runId: "run-media")); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-plain", Status = "started" }); + await provider.SendMessageAsync("main", "same"); + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "run-plain")); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "same", + State = "final", + OpenClawId = "remote-same", + OpenClawSeq = 99, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "same") + .ToArray(); + Assert.Equal(3, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Equal(2, users.Count(user => metadata[user.Id].IsLocalQueuedSend)); + Assert.Single(users, user => metadata[user.Id].GatewayMessageId == "remote-same"); + } + + [Fact] + public async Task ChatMessageReceived_NoEnvelopeEchoDoesNotConsumeSingleMediaPromotion() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-media", Status = "started" }); + await provider.SendMessageAsync("main", "same", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "same.png", + Content = Convert.ToBase64String([1]), + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "same", + State = "final", + OpenClawId = "remote-same", + OpenClawSeq = 99, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "same") + .ToArray(); + Assert.Equal(2, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Contains(users, user => metadata[user.Id].IsLocalQueuedSend); + Assert.Contains(users, user => metadata[user.Id].GatewayMessageId == "remote-same"); + } + [Fact] public async Task SendMessageAsync_WhenGatewayThrows_DoesNotSuppressFutureRemoteUserEcho() { @@ -3560,6 +3642,63 @@ public async Task SessionResetCompletion_TimestamplessRemoteUserCanOpenGateViaHi e.Kind == ChatTimelineItemKind.Assistant && e.Text == "remote response"); } + [Fact] + public async Task SessionResetCompletion_DuplicateBackfillMergesAndOpensBufferedLifecycleGate() + { + var remoteTimestamp = DateTimeOffset.UtcNow.AddSeconds(5).ToUnixTimeMilliseconds(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/reset.pdf (application/pdf)]\nremote after reset", + Ts = remoteTimestamp, + OpenClawId = "reset-user", + OpenClawSeq = 41, + }, + ], + }); + await provider.LoadAsync(); + bridge.RaiseSessionCommandCompleted(new SessionCommandResult + { + Method = "sessions.reset", + Ok = true, + Key = "main", + }); + snapshots.Clear(); + + // Insert the projected row without reset proof, then buffer the + // lifecycle start before the reset-aware duplicate backfill arrives. + await provider.FetchRemoteUserMessageForTestsAsync("main", openResetGateOnSuccess: false); + var pendingStart = MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "reset-remote-run"); + pendingStart.Ts = remoteTimestamp + 1; + bridge.RaiseAgent(pendingStart); + snapshots.Clear(); + + await provider.FetchRemoteUserMessageForTestsAsync("main", openResetGateOnSuccess: true); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "reset gate opened", + State = "final", + Ts = remoteTimestamp + 2, + }); + + var latest = snapshots[^1].Timelines["main"]; + var user = Assert.Single(latest.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("remote after reset", user.Text); + Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Contains(latest.Entries, entry => + entry.Kind == ChatTimelineItemKind.Assistant && + entry.Text == "reset gate opened"); + } + [Fact] public async Task SessionResetCompletion_LocalSendDoesNotReopenGateForStaleChatFrames() { @@ -10430,6 +10569,217 @@ public async Task OnChatMessageReceived_LiveUserPlain_ShownAsRemoteUser() Assert.Equal("hello there", entry.Text); } + [Fact] + public async Task OnChatMessageReceived_MediaOnlyAssistant_CreatesSafePresentationEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + State = "final", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries); + Assert.Equal(ChatTimelineItemKind.Assistant, entry.Kind); + Assert.Equal(string.Empty, entry.Text); + var metadata = provider.GetEntryMetadata("main"); + var media = Assert.Single(metadata[entry.Id].AssistantContent!.Media); + Assert.Equal("banner.png", media.DisplayName); + } + + [Fact] + public async Task OnChatMessageReceived_FinalMedia_MergesIntoStreamingAssistantEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "Rendering", + State = "delta", + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = "Rendering complete", + State = "final", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + FileName = "banner.png", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + Assert.Equal("Rendering complete", entry.Text); + Assert.False(entry.IsStreaming); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal("artifact-1", media.Reference.ArtifactId); + } + + [Fact] + public async Task OnChatMessageReceived_StructuredFollowUp_PreservesLegacyMediaReference() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-unavailable", + }, + }, + ], + }); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal(ChatMediaContentSource.LegacyDirective, media.Reference.Source); + Assert.Equal("banner.png", media.DisplayName); + } + + [Fact] + public async Task OnChatMessageReceived_IdentifiedMediaOnlyRetransmit_DoesNotDuplicateEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + var message = new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + Text = string.Empty, + State = "final", + OpenClawId = "assistant-media-1", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.LegacyDirective, + FileName = "banner.png", + }, + }, + ], + }; + + bridge.RaiseChat(message); + bridge.RaiseChat(message); + + var timeline = snapshots[^1].Timelines["main"]; + Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + } + + [Fact] + public async Task LoadHistoryAsync_StructuredMedia_CreatesSafePresentationEntry() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "assistant", + ContentParts = + [ + new ChatMessageContentPartInfo + { + Kind = ChatMessageContentPartKind.Media, + Media = new ChatMediaContentInfo + { + Kind = ChatMediaContentKind.Image, + Source = ChatMediaContentSource.Structured, + ArtifactId = "artifact-1", + FileName = "banner.png", + }, + }, + ], + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + await provider.LoadHistoryAsync("main"); + + var timeline = snapshots[^1].Timelines["main"]; + var entry = Assert.Single(timeline.Entries, item => item.Kind == ChatTimelineItemKind.Assistant); + var media = Assert.Single(provider.GetEntryMetadata("main")[entry.Id].AssistantContent!.Media); + Assert.Equal("banner.png", media.DisplayName); + Assert.Equal("artifact-1", media.Reference.ArtifactId); + } + // ── chat rubber-duck MEDIUM 4: per-message size cap ── [Fact] @@ -10568,44 +10918,396 @@ public async Task SendMessageAsync_ClearsPendingAbortCounts() } [Fact] - public async Task SendMessageAsync_WithAttachment_SendsThroughInterface() + public async Task ChatMessageReceived_LocalMediaEcho_ReconcilesOneCleanRowAndPreservesLocalPreview() { + var sendGate = new TaskCompletionSource(); var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; await provider.LoadAsync(); - var attachment = new ChatAttachment - { - Type = "file", - MimeType = "text/plain", - FileName = "test.txt", - Content = Convert.ToBase64String(new byte[] { 72, 101, 108, 108, 111 }), - SizeBytes = 5 - }; - - await provider.SendMessageAsync("main", "Check this", default, new[] { attachment }); - - Assert.Contains(bridge.SentMessages, m => m == "Check this"); - var sentAttachment = Assert.Single(bridge.SentAttachments); - Assert.NotNull(sentAttachment); - Assert.Same(attachment, sentAttachment![0]); - Assert.Contains("test.txt", snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User).Text); + _ = provider.SendMessageAsync("main", "Describe this", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1, 2, 3]), + SizeBytes = 3, + }, + ]); + snapshots.Clear(); bridge.RaiseChat(new ChatMessageInfo { SessionKey = "main", Role = "user", - Text = "Check this", - State = "final" + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nDescribe this", + State = "final", + OpenClawId = "gateway-media", + OpenClawSeq = 12, }); - // The display text in the timeline should include the attachment indicator - var timeline = snapshots[^1].Timelines["main"]; - var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); - Assert.Contains("test.txt", userEntry.Text); + var timeline = Assert.Single(snapshots).Timelines["main"]; + var user = Assert.Single(timeline.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("Describe this", user.Text); + Assert.DoesNotContain("media://", user.Text); + var metadata = provider.GetEntryMetadata("main")[user.Id]; + Assert.Equal("gateway-media", metadata.GatewayMessageId); + var attachment = Assert.Single(metadata.Attachments!); + Assert.Equal(ChatAttachmentOrigin.Local, attachment.Origin); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + attachment.DisplayFileName); + Assert.True(attachment.CanAccessPreviewCache); + Assert.True(ChatImagePreviewCache.Contains(attachment.PreviewCacheKey!)); + + sendGate.SetResult(); } [Fact] - public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMarkers() + public async Task ChatMessageReceived_AttachmentOnlyMediaEcho_ReconcilesWithoutEmptyTextMatching() + { + var sendGate = new TaskCompletionSource(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; + await provider.LoadAsync(); + + _ = provider.SendMessageAsync("main", "", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1]), + SizeBytes = 1, + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]", + State = "final", + OpenClawId = "gateway-only", + }); + + var user = Assert.Single(Assert.Single(snapshots).Timelines["main"].Entries); + Assert.Equal(string.Empty, user.Text); + var metadata = provider.GetEntryMetadata("main")[user.Id]; + Assert.Equal("gateway-only", metadata.GatewayMessageId); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Assert.Single(metadata.Attachments!).DisplayFileName); + sendGate.SetResult(); + } + + [Fact] + public async Task SessionResetCompletion_MediaEchoUsesCorrelationSignatureAcrossResetGate() + { + var sendGate = new TaskCompletionSource(); + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.SendBehavior = (_, _, _) => sendGate.Task; + await provider.LoadAsync(); + + _ = provider.SendMessageAsync("main", "before reset", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1]), + }, + ]); + bridge.RaiseSessionCommandCompleted(new SessionCommandResult + { + Method = "sessions.reset", + Ok = true, + Key = "main", + }); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nbefore reset", + Ts = DateTimeOffset.UtcNow.AddSeconds(1).ToUnixTimeMilliseconds(), + }); + + Assert.Empty((await provider.LoadAsync()).Timelines["main"].Entries); + sendGate.SetResult(); + + await provider.SendMessageAsync("main", "after reset", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "after.png", + Content = Convert.ToBase64String([2]), + }, + ]); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/after---7f122605-290a-467c-a5df-8a744c093004.png (image/png)]\nafter reset", + Ts = DateTimeOffset.UtcNow.AddSeconds(2).ToUnixTimeMilliseconds(), + OpenClawId = "after-reset-media", + }); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal("after reset", user.Text); + Assert.Equal("after-reset-media", provider.GetEntryMetadata("main")[user.Id].GatewayMessageId); + } + + [Fact] + public async Task RemoteLifecycleBackfill_MediaEnvelopeBecomesCleanStructuredUserRow() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/remote.pdf (application/pdf)]\nReview it", + Ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "remote-media-run")); + for (var i = 0; i < 50 && snapshots.Count == 0; i++) + await Task.Delay(10); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("Review it", user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("remote.pdf", attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task RemoteLifecycleBackfill_WhenLiveFrameArrivesLater_MergesOneUserRow() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "race", + Ts = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), + OpenClawId = "race-user", + OpenClawSeq = 73, + }, + ], + }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseAgent(MakeAgentEvent("lifecycle", """{"phase":"start"}""", runId: "race-run")); + for (var i = 0; i < 50; i++) + { + if (snapshots.Count > 0 && + snapshots[^1].Timelines["main"].Entries.Any(entry => + entry.Kind == ChatTimelineItemKind.User && entry.Text == "race")) + { + break; + } + await Task.Delay(10); + } + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "[media attached: media://inbound/race.png (image/png)]\nrace", + State = "final", + OpenClawId = "race-user", + OpenClawSeq = 73, + }); + + var latest = snapshots[^1].Timelines["main"]; + var user = Assert.Single(latest.Entries, entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("race", user.Text); + Assert.Equal( + "race.png", + Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task ChatMessageReceived_RepeatedSameTextWithDistinctGatewayIdentityKeepsBothTurns() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + snapshots.Clear(); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "repeat", + State = "final", + OpenClawId = "repeat-1", + OpenClawSeq = 1, + }); + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "repeat", + State = "final", + OpenClawId = "repeat-2", + OpenClawSeq = 2, + }); + + var users = snapshots[^1].Timelines["main"].Entries + .Where(entry => entry.Kind == ChatTimelineItemKind.User && entry.Text == "repeat") + .ToArray(); + Assert.Equal(2, users.Length); + var metadata = provider.GetEntryMetadata("main"); + Assert.Equal( + new[] { "repeat-1", "repeat-2" }, + users.Select(user => metadata[user.Id].GatewayMessageId).ToArray()); + } + + [Fact] + public async Task LoadHistoryAsync_GatewayMediaUsesLocalSidecarPrecedenceWithoutRestoringBytes() + { + using var tempDir = new TempDirectory(); + var toolPath = Path.Combine(tempDir.DirectoryPath, "tool-metadata.json"); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sentTs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + var (_, sender, _, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + await sender.LoadAsync(); + await sender.SendMessageAsync("main", "history caption", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + Content = Convert.ToBase64String([1, 2]), + }, + ]); + + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png (image/png)]\nhistory caption", + Ts = sentTs, + }, + ], + }); + + await provider.LoadHistoryAsync("main"); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal("history caption", user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.Local, attachment.Origin); + Assert.Equal( + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png", + attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task LoadHistoryAsync_RemoteMediaEnvelopeFallsBackToGatewayReference() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + bridge.HistoryBehavior = _ => Task.FromResult(new ChatHistoryInfo + { + SessionKey = "main", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/report.pdf (application/pdf)]", + Ts = 123, + }, + ], + }); + + await provider.LoadHistoryAsync("main"); + + var user = Assert.Single(snapshots[^1].Timelines["main"].Entries); + Assert.Equal(string.Empty, user.Text); + var attachment = Assert.Single(provider.GetEntryMetadata("main")[user.Id].Attachments!); + Assert.Equal(ChatAttachmentOrigin.GatewayReference, attachment.Origin); + Assert.Equal("report.pdf", attachment.DisplayFileName); + Assert.Null(attachment.PreviewCacheKey); + } + + [Fact] + public async Task SendMessageAsync_WithAttachment_SendsThroughInterface() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + + var attachment = new ChatAttachment + { + Type = "file", + MimeType = "text/plain", + FileName = "test.txt", + Content = Convert.ToBase64String(new byte[] { 72, 101, 108, 108, 111 }), + SizeBytes = 5 + }; + + await provider.SendMessageAsync("main", "Check this", default, new[] { attachment }); + + Assert.Contains(bridge.SentMessages, m => m == "Check this"); + var sentAttachment = Assert.Single(bridge.SentAttachments); + Assert.NotNull(sentAttachment); + Assert.Same(attachment, sentAttachment![0]); + var optimistic = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("Check this", optimistic.Text); + Assert.Equal( + "test.txt", + Assert.Single(provider.GetEntryMetadata("main")[optimistic.Id].Attachments!).DisplayFileName); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = "Check this", + State = "final" + }); + + var timeline = snapshots[^1].Timelines["main"]; + var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("Check this", userEntry.Text); + } + + [Fact] + public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMarkers() { var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); await provider.LoadAsync(); @@ -10635,9 +11337,9 @@ public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMar sentAttachments!, a => Assert.Same(fileAttachment, a), a => Assert.Same(imageAttachment, a)); - Assert.Equal( - "See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", - snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User).Text); + var optimistic = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); + Assert.Equal("See both", optimistic.Text); + Assert.Equal(2, provider.GetEntryMetadata("main")[optimistic.Id].Attachments!.Count); bridge.RaiseChat(new ChatMessageInfo { @@ -10649,7 +11351,50 @@ public async Task SendMessageAsync_WithMultipleAttachments_SendsAndRendersAllMar var timeline = snapshots[^1].Timelines["main"]; var userEntry = timeline.Entries.Last(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", userEntry.Text); + Assert.Equal("See both", userEntry.Text); + } + + [Fact] + public async Task SendMessageAsync_OversizedMediaCaptionUsesSameBoundedEchoText() + { + var (bridge, provider, snapshots, _) = CreateProvider(new[] { MainSession() }); + await provider.LoadAsync(); + var caption = new string('x', OpenClawChatDataProvider.MaxEntryTextBytes + 50_000); + + bridge.SendResults.Enqueue(new ChatSendResult { RunId = "run-large", Status = "started" }); + await provider.SendMessageAsync("main", caption, default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = "large.png", + Content = Convert.ToBase64String([1]), + SizeBytes = 1, + }, + ]); + + var optimistic = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.True( + System.Text.Encoding.UTF8.GetByteCount(optimistic.Text) <= + OpenClawChatDataProvider.MaxEntryTextBytes); + + bridge.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = $"[media attached: media://inbound/large.png (image/png)]\n{caption}", + State = "final", + OpenClawId = "large-echo", + OpenClawSeq = 1, + }); + + var user = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal("large-echo", provider.GetEntryMetadata("main")[user.Id].GatewayMessageId); } [Fact] @@ -10681,14 +11426,138 @@ await provider1.SendMessageAsync("main", "Check this", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "Check this", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/test.txt (text/plain)]\nCheck this", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("Check this\n\u200B📎 test.txt", userEntry.Text); + Assert.Equal("Check this", userEntry.Text); + Assert.Equal( + "test.txt", + Assert.Single(provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task AttachmentMetadata_IdentitylessRewrittenLiveEchoDoesNotDuplicateRehydratedHistory() + { + const string localFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe-c9b9131e8b5a-clipboard.png"; + const string gatewayFileName = + "f4f160f1-07b9-4eb5-8de4-2b12c403d0fe-0d950ec0-98f0-4398-a7fe.png"; + using var tempDir = new TempDirectory(); + var toolPath = Path.Combine(tempDir.DirectoryPath, "tool-metadata.json"); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sentTs = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(); + + var (_, provider1, _, _) = CreateProvider(new[] { MainSession() }, toolPath, attachmentPath); + await provider1.LoadAsync(); + await provider1.SendMessageAsync("main", "Check this", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = localFileName, + Content = Convert.ToBase64String([1, 2, 3]), + SizeBytes = 3, + }, + ]); + + var gatewayText = + $"[media attached: media://inbound/{gatewayFileName} (image/png)]\nCheck this"; + var (bridge2, provider2, snapshots, _) = CreateProvider( + new[] { MainSession() }, + toolPath, + attachmentPath); + bridge2.HistoryBehavior = key => Task.FromResult(new ChatHistoryInfo + { + SessionKey = key ?? "", + SessionId = "session-1", + Messages = + [ + new ChatMessageInfo + { + Role = "user", + Text = gatewayText, + State = "final", + Ts = sentTs, + }, + ], + }); + + await provider2.LoadHistoryAsync("main"); + bridge2.RaiseChat(new ChatMessageInfo + { + SessionKey = "main", + Role = "user", + Text = gatewayText, + State = "final", + Ts = sentTs, + }); + + var user = Assert.Single( + snapshots[^1].Timelines["main"].Entries, + entry => entry.Kind == ChatTimelineItemKind.User); + Assert.Equal( + localFileName, + Assert.Single(provider2.GetEntryMetadata("main")[user.Id].Attachments!).DisplayFileName); + } + + [Fact] + public async Task AttachmentMetadata_ConcurrentSavesPreserveEveryCompletedSend() + { + const int sendCount = 16; + using var tempDir = new TempDirectory(); + var attachmentPath = Path.Combine(tempDir.DirectoryPath, "attachment-metadata.json"); + var sessions = Enumerable.Range(0, sendCount) + .Select(index => new SessionInfo + { + Key = $"thread-{index}", + DisplayName = $"Thread {index}", + Status = "active", + IsMain = index == 0, + }) + .ToArray(); + var (bridge, provider, _, _) = CreateProvider( + sessions, + attachmentMetaCachePath: attachmentPath); + await provider.LoadAsync(); + + var allStarted = new CountdownEvent(sendCount); + var release = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + bridge.SendBehavior = async (_, _, _) => + { + allStarted.Signal(); + await release.Task; + }; + + var sends = sessions.Select((session, index) => + provider.SendMessageAsync(session.Key, $"caption-{index}", default, + [ + new ChatAttachment + { + Type = "image", + MimeType = "image/png", + FileName = $"file-{index}.png", + Content = Convert.ToBase64String([(byte)index]), + SizeBytes = 1, + }, + ])).ToArray(); + Assert.True(allStarted.Wait(TimeSpan.FromSeconds(5))); + release.SetResult(); + await Task.WhenAll(sends); + + var persisted = await File.ReadAllTextAsync(attachmentPath); + for (var index = 0; index < sendCount; index++) + Assert.Contains($"file-{index}.png", persisted, StringComparison.Ordinal); } [Fact] @@ -10728,14 +11597,22 @@ await provider1.SendMessageAsync("main", "See both", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "See both", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/notes.txt (text/plain)]\n" + + "[media attached: media://inbound/diagram.png (image/png)]\nSee both", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("See both\n\u200B📎 notes.txt\n\u200B🖼️ diagram.png", userEntry.Text); + Assert.Equal("See both", userEntry.Text); + Assert.Equal(2, provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!.Count); } [Fact] @@ -10767,14 +11644,23 @@ await provider1.SendMessageAsync("main", "", default, new[] SessionId = "session-1", Messages = new[] { - new ChatMessageInfo { Role = "user", Text = "", State = "final", Ts = sentTs } + new ChatMessageInfo + { + Role = "user", + Text = "[media attached: media://inbound/screenshot.png (image/png)]", + State = "final", + Ts = sentTs, + } } }); await provider2.LoadHistoryAsync("main"); var userEntry = snapshots[^1].Timelines["main"].Entries.Single(e => e.Kind == ChatTimelineItemKind.User); - Assert.Equal("\u200B🖼️ screenshot.png", userEntry.Text); + Assert.Equal(string.Empty, userEntry.Text); + Assert.Equal( + "screenshot.png", + Assert.Single(provider2.GetEntryMetadata("main")[userEntry.Id].Attachments!).DisplayFileName); } [Fact]