From a9994528acbd71a2b079c6c10ec5807643e80db0 Mon Sep 17 00:00:00 2001 From: Saphi Date: Thu, 25 Jun 2026 07:12:46 -0300 Subject: [PATCH 1/3] feat: Multi-toggle for what App Context sends (Off / App summary / Screenshot LLM) + per-app "send screenshot" allow-list Modified files: - Sources/AppContextSource.swift (new) - Sources/AppContextScopeStore.swift (new) - Sources/AppContextService.swift - Sources/SettingsView.swift --- Sources/AppContextScopeStore.swift | 195 +++++++++++++++ Sources/AppContextService.swift | 35 +++ Sources/AppContextSource.swift | 301 +++++++++++++++++++++++ Sources/SettingsView.swift | 367 +++++++++++++++++------------ 4 files changed, 744 insertions(+), 154 deletions(-) create mode 100644 Sources/AppContextScopeStore.swift create mode 100644 Sources/AppContextSource.swift diff --git a/Sources/AppContextScopeStore.swift b/Sources/AppContextScopeStore.swift new file mode 100644 index 00000000..5c8c5f4d --- /dev/null +++ b/Sources/AppContextScopeStore.swift @@ -0,0 +1,195 @@ +import SwiftUI +import AppKit +import UniformTypeIdentifiers + +/// Observable store for the per-app screenshot allowlist, persisted to `AppContextSource.appsKey`. +/// Uses `ObservableObject` (not `@Observable`) because the deployment target is macOS 13. +@MainActor +final class AppContextAllowlist: ObservableObject { + /// The allowlisted bundle identifiers (persisted). + @Published private(set) var bundleIDs: [String] + + /// Loads the persisted allowlist (empty by default). + init() { + bundleIDs = UserDefaults.standard.array(forKey: AppContextSource.appsKey) as? [String] ?? [] + } + + /// Adds bundle ids, skipping blanks and duplicates; persists only if something changed. + func add(_ ids: [String]) { + let existing = Set(bundleIDs) + let fresh = ids.filter { !$0.isEmpty && !existing.contains($0) } + guard !fresh.isEmpty else { return } + bundleIDs.append(contentsOf: fresh) + persist() + } + + /// Removes a bundle id and persists. + func remove(_ id: String) { + bundleIDs.removeAll { $0 == id } + persist() + } + + /// Writes the allowlist back to `UserDefaults`. + private func persist() { + UserDefaults.standard.set(bundleIDs, forKey: AppContextSource.appsKey) + } + + /// Opens the "Add app…" picker and adds the bundle ids of any chosen apps. + func presentPicker() { + // AppKit open panel rooted at /Applications, limited to .app bundles. + let panel = NSOpenPanel() + panel.directoryURL = URL(fileURLWithPath: "/Applications") + panel.allowedContentTypes = [.application] + panel.allowsMultipleSelection = true + panel.canChooseDirectories = false + panel.canChooseFiles = true + guard panel.runModal() == .OK else { return } + add(panel.urls.compactMap { Bundle(url: $0)?.bundleIdentifier }) + } +} + +/// Resolves and caches an app's display name + icon from its bundle id (for the chips). +@MainActor +enum AppContextAppInfo { + private static var cache: [String: (name: String, icon: NSImage?)] = [:] + + /// Returns the app's display name (falls back to the bundle id) and icon. Resolved once per id. + static func info(forBundleID id: String) -> (name: String, icon: NSImage?) { + if let hit = cache[id] { return hit } + // Resolve the app's URL once, then its Finder name and icon (AppKit). + let url = NSWorkspace.shared.urlForApplication(withBundleIdentifier: id) + let name = url.map { FileManager.default.displayName(atPath: $0.path) } ?? id + let icon = url.map { NSWorkspace.shared.icon(forFile: $0.path) } + let result = (name, icon) + cache[id] = result + return result + } +} + +/// The screenshot scope multiselector: two radio cards plus the per-app chips and "Add app…" picker. +/// Binds the scope to `AppContextSource.scopeKey` via `@AppStorage` and observes the allowlist store. +struct ScreenshotScopeSection: View { + /// Persisted scope (`"all"` | `"specific"`), defaulting to `all`. + @AppStorage(AppContextSource.scopeKey) private var scopeRaw: String = AppContextSource.Scope.all.rawValue + /// The allowlist store, owned by this view. + @StateObject private var allowlist = AppContextAllowlist() + /// Reflects the parent `.disabled()` (true only in Screenshot mode) — used to grey out chip icons. + @Environment(\.isEnabled) private var isEnabled + + /// Current scope from the persisted value. + private var scope: AppContextSource.Scope { AppContextSource.Scope(rawValue: scopeRaw) ?? .all } + + /// Header + the two scope options, plus the app chips when `specific`. + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Text("Screenshot scope") + .font(.caption.weight(.semibold)) + scopeOption(.all, title: "Send screenshot to all apps", subtitle: "Every app takes a screenshot.") + scopeOption(.specific, title: "Only specific apps", subtitle: "Other apps use App summary instead.") + if scope == .specific { + specificAppsEditor + } + } + } + + /// One selectable scope row in the app's radio-row style (checkmark + blue accent when selected). + private func scopeOption(_ option: AppContextSource.Scope, title: String, subtitle: String) -> some View { + let selected = scope == option + return Button { + scopeRaw = option.rawValue + } label: { + HStack(alignment: .top, spacing: 10) { + Image(systemName: selected ? "checkmark.circle.fill" : "circle") + .foregroundStyle(selected ? .blue : .secondary) + VStack(alignment: .leading, spacing: 2) { + Text(title) + .foregroundStyle(.primary) + Text(subtitle) + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer(minLength: 0) + } + .padding(12) + .frame(maxWidth: .infinity, alignment: .leading) + .background(selected ? Color.blue.opacity(0.1) : Color(nsColor: .controlBackgroundColor)) + .cornerRadius(8) + .overlay( + RoundedRectangle(cornerRadius: 8) + .stroke(selected ? Color.blue : Color.clear, lineWidth: 1.5) + ) + .contentShape(Rectangle()) + } + .buttonStyle(.plain) + // Read the row as one element to VoiceOver, with its selected state. + .accessibilityElement(children: .ignore) + .accessibilityLabel("\(title). \(subtitle)") + .accessibilityAddTraits(selected ? [.isButton, .isSelected] : .isButton) + } + + /// The app chips (or an empty-state line) plus the dashed "Add app…" button. + private var specificAppsEditor: some View { + VStack(alignment: .leading, spacing: 8) { + if allowlist.bundleIDs.isEmpty { + Text("No apps added yet.") + .font(.caption) + .foregroundStyle(.secondary) + } + FlowLayout(spacing: 6) { + ForEach(allowlist.bundleIDs, id: \.self) { id in + chip(for: id) + } + addButton + } + } + } + + /// A removable app chip: icon + name + ✕. + private func chip(for id: String) -> some View { + let info = AppContextAppInfo.info(forBundleID: id) + return HStack(spacing: 6) { + if let icon = info.icon { + Image(nsImage: icon) + .resizable() + .frame(width: 16, height: 16) + // Desaturate when the screenshot block is disabled, so the icon dims like the text. + .grayscale(isEnabled ? 0 : 1) + } + Text(info.name) + .font(.caption) + Button { + allowlist.remove(id) + } label: { + Image(systemName: "xmark") + .font(.caption2) + } + .buttonStyle(.plain) + .foregroundStyle(.secondary) + .accessibilityLabel("Remove \(info.name)") + } + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(RoundedRectangle(cornerRadius: 6).fill(Color(nsColor: .controlBackgroundColor))) + .overlay(RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.25), lineWidth: 1)) + } + + /// The dashed "Add app…" button that opens the picker. + private var addButton: some View { + Button { + allowlist.presentPicker() + } label: { + HStack(spacing: 4) { + Image(systemName: "plus") + Text("Add app…") + } + .font(.caption) + .padding(.horizontal, 10) + .padding(.vertical, 5) + .overlay( + RoundedRectangle(cornerRadius: 6).stroke(Color.secondary.opacity(0.5), style: StrokeStyle(lineWidth: 1, dash: [4])) + ) + } + .buttonStyle(.plain) + .accessibilityLabel("Add app") + } +} diff --git a/Sources/AppContextService.swift b/Sources/AppContextService.swift index d82fdbe8..e2ba1dd5 100644 --- a/Sources/AppContextService.swift +++ b/Sources/AppContextService.swift @@ -45,6 +45,9 @@ Return only two sentences, no labels, no markdown, no extra commentary. private let maxScreenshotDataURILength = 500_000 private let screenshotCompressionPrimary = 0.5 private let screenshotMaxDimension: CGFloat + /// Forces the vision path on this instance only — set by the "Test Vision Prompt" button on its own + /// throwaway service so the test works in any mode/scope. Real dictations use a separate instance. + var forceScreenshotForTest = false private var contextRequestTimeoutSeconds: TimeInterval { let override = UserDefaults.standard.double(forKey: "context_request_timeout_seconds") return override > 0 ? override : 20 @@ -176,6 +179,11 @@ Return only two sentences, no labels, no markdown, no extra commentary. screenshotDataURL: String?, contextSystemPrompt: String ) async -> (activity: String, prompt: String)? { + // Skip vision/LLM inference unless this app uses Screenshot mode. + guard forceScreenshotForTest || AppContextSource.effectiveSource(forBundleID: bundleIdentifier) == .screenshot else { + return nil + } + let attempts: [(model: String, screenshotDataURL: String?)] = if let screenshotDataURL { [ @@ -318,6 +326,26 @@ Selected text: \(selectedText ?? "None") windowTitle: String?, screenshotAvailable: Bool ) -> String { + // Off → empty, App summary → metadata line, Screenshot → original error text below. + let source: AppContextSource.EffectiveSource = + forceScreenshotForTest ? .screenshot : AppContextSource.effectiveSource(forBundleID: bundleIdentifier) + switch source { + case .off: + return "" + case .metadata: + // One AX read: host / web-capable / address-bar focus. + let web = AppContextSource.webContext() + return AppContextSource.metadataSummary( + appName: appName, + pageTitle: windowTitle, + webHost: web.host, + isWebApp: web.isWebApp, + addressBarFocused: web.addressBarFocused + ) + case .screenshot: + break + } + let activeApp = appName ?? "the active application" if screenshotAvailable { return "Could not reliably infer a two-sentence summary for \(activeApp) from the screenshot and metadata." @@ -423,6 +451,13 @@ Selected text: \(selectedText ?? "None") appElement: AXUIElement, focusedWindowTitle: String? ) -> (dataURL: String?, mimeType: String?, error: String?) { + // Derive the bundle id from the pid; skip capture unless this app uses Screenshot mode. + let bundleID = NSRunningApplication(processIdentifier: processIdentifier)?.bundleIdentifier + guard forceScreenshotForTest || AppContextSource.effectiveSource(forBundleID: bundleID) == .screenshot else { + // Intentional skip, not a failure — no error so it isn't logged/shown as one. + return (nil, nil, nil) + } + if !CGPreflightScreenCaptureAccess() { return ( nil, diff --git a/Sources/AppContextSource.swift b/Sources/AppContextSource.swift new file mode 100644 index 00000000..b2a1d387 --- /dev/null +++ b/Sources/AppContextSource.swift @@ -0,0 +1,301 @@ +import Foundation +import AppKit +import ApplicationServices + +/// Model for the "App Context" feature: the three UserDefaults settings (mode, screenshot scope, +/// allowlist), the effective-source decision, the "App summary" line, and the browser-URL AX reads. +/// All static / value-typed so it can be read from any thread (leaves read UserDefaults; the UI writes +/// via `@AppStorage`). Defaults reproduce upstream (Screenshot + all apps), so a fresh install is unchanged. +enum AppContextSource { + + // MARK: - UserDefaults keys + + /// Context source mode: `"off" | "metadata" | "screenshot"`. + static let modeKey = "appContextMode" + /// Per-app screenshot scope: `"all" | "specific"`. + static let scopeKey = "appContextScreenshotScope" + /// Screenshot allowlist: an array of bundle identifiers. + static let appsKey = "appContextScreenshotApps" + + // MARK: - Settings enums + + /// The chosen context source. `screenshot` is the upstream default. + enum Mode: String, CaseIterable { + /// No context is sent. + case off + /// A one-line on-device text summary; no image, no vision model. + case metadata + /// Capture the window and ask the vision model (upstream behavior). + case screenshot + } + + /// Whether Screenshot mode applies to every app or only an allowlisted subset. + enum Scope: String, CaseIterable { + /// Every app takes a screenshot. + case all + /// Only allowlisted apps; the rest fall back to `metadata`. + case specific + } + + /// The resolved source for one `collectContext()` call. + enum EffectiveSource { + /// Capture + vision model. + case screenshot + /// One-line metadata summary; no image, no LLM. + case metadata + /// No context. + case off + } + + // MARK: - Settings reads (default = upstream behavior) + + /// Current mode; defaults to `.screenshot`. + static var currentMode: Mode { + Mode(rawValue: UserDefaults.standard.string(forKey: modeKey) ?? "") ?? .screenshot + } + + /// Current scope; defaults to `.all`. + static var currentScope: Scope { + Scope(rawValue: UserDefaults.standard.string(forKey: scopeKey) ?? "") ?? .all + } + + /// Current allowlist (bundle ids); empty by default. `UserDefaults` stores `[String]` natively. + static var allowlist: [String] { + UserDefaults.standard.array(forKey: appsKey) as? [String] ?? [] + } + + // MARK: - Effective source + + /// Resolves the source for one call from the frontmost app's bundle id. + /// In Screenshot+specific, apps outside the allowlist (or a `nil` id) fall back to `.metadata`. + static func effectiveSource(forBundleID id: String?) -> EffectiveSource { + switch currentMode { + case .off: + return .off + case .metadata: + return .metadata + case .screenshot: + if currentScope == .all { return .screenshot } + if let id, allowlist.contains(id) { return .screenshot } + return .metadata + } + } + + // MARK: - Deterministic metadata summary (App summary mode) + + /// Builds the "App summary" one-line activity string (offline, no LLM, pure — testable without AX). + /// Forms: `on in ` (collapsed to `in ` when the host matches the app name), + /// `in the address bar`, `on "" in <app>` (web app with no readable URL), else `in <app>`. + /// - Parameters: + /// - appName: Frontmost app's localized name. + /// - pageTitle: Focused window/page title (cleaned here). + /// - webHost: Current page host (e.g. `github.com`), or `nil`. + /// - isWebApp: Whether the app renders web content. + /// - addressBarFocused: Whether the address/search bar is focused. + static func metadataSummary( + appName: String?, + pageTitle: String?, + webHost: String?, + isWebApp: Bool, + addressBarFocused: Bool + ) -> String { + let app = (appName?.isEmpty == false) ? appName! : "the active app" + let title = cleanedPageTitle(pageTitle, appName: appName, host: webHost) + + if let webHost, !webHost.isEmpty { + // Drop "in <app>" when the host already names the app (e.g. the ChatGPT web app). + let base = hostMatchesApp(webHost, appName: appName) + ? "User is dictating in \(app)" + : "User is dictating on \(webHost) in \(app)" + if let title { return "\(base) (\"\(title)\")" } + return base + } + + if addressBarFocused { + return "User is dictating in the \(app) address bar" + } + + // Web app whose URL the browser doesn't expose (e.g. Firefox/Zen) — use the page title. + if isWebApp, let title { + return "User is dictating on \"\(title)\" in \(app)" + } + + return "User is dictating in \(app)" + } + + // MARK: - Metadata summary helpers (pure) + + /// Lowercased letters/digits only, for loose name-vs-host comparison. + private static func normalizedToken(_ value: String) -> String { + value.lowercased().unicodeScalars + .filter { CharacterSet.alphanumerics.contains($0) } + .map(String.init) + .joined() + } + + /// The host's brand label (the second-to-last DNS label: `github.com`→`github`). + /// Note: not public-suffix aware, so multi-part TLDs are approximate (`bbc.co.uk`→`co`); only used + /// for the optional cosmetic collapse below, so a miss just keeps the full `on <host>` form. + private static func hostBrand(_ host: String) -> String { + let labels = host.split(separator: ".").map(String.init) + guard labels.count >= 2 else { return host } + return labels[labels.count - 2] + } + + /// True when the host's brand equals the app name (a web app named after its site, e.g. ChatGPT). + /// Exact match, so a real browser ("Google Chrome" on google.com) is not collapsed. + private static func hostMatchesApp(_ host: String, appName: String?) -> Bool { + guard let appName, !appName.isEmpty else { return false } + return normalizedToken(hostBrand(host)) == normalizedToken(appName) + } + + /// Cleans the page title: strips a trailing " — <app>" suffix browsers add, removes embedded quotes, + /// and returns `nil` if it's empty or just repeats the app name or host. + private static func cleanedPageTitle(_ pageTitle: String?, appName: String?, host: String?) -> String? { + guard var title = pageTitle?.trimmingCharacters(in: .whitespacesAndNewlines), !title.isEmpty else { + return nil + } + + // Strip the trailing " — <BrowserName>" / " - <BrowserName>" browsers append. + if let appName, !appName.isEmpty { + for separator in [" — ", " – ", " - "] { + let suffix = separator + appName + if title.count > suffix.count, title.lowercased().hasSuffix(suffix.lowercased()) { + title = String(title.dropLast(suffix.count)).trimmingCharacters(in: .whitespacesAndNewlines) + break + } + } + } + + // Remove embedded double quotes so the quoted summary stays balanced. + title = title.replacingOccurrences(of: "\"", with: "").trimmingCharacters(in: .whitespacesAndNewlines) + + guard !title.isEmpty else { return nil } + if let appName, normalizedToken(title) == normalizedToken(appName) { return nil } + if let host, normalizedToken(title) == normalizedToken(host) { return nil } + return title + } + + // MARK: - Browser URL & focus (Accessibility) + // + // Reads the frontmost app's page host and focus via AX. Detection is structural (finds an AXWebArea, + // no browser allowlist), so it covers Safari, Chromium browsers (Chrome, Edge, Brave, Arc, Opera, + // Vivaldi, newer ones), PWAs, and Gecko (Firefox/Zen) where AX is exposed. Requires the Accessibility + // permission the app already uses. + + /// Max AX nodes visited when searching for the web area — bounds cost on large non-web trees. + private static let traversalBudget = 250 + /// Max parent hops when checking if the focused element is inside the web content. + private static let maxParentHops = 12 + + /// Reads one dictation's web context: page host (if readable), whether the app renders web content, + /// and whether the address/search bar is focused. Reads `frontmostApplication` independently (the + /// caller can't pass an element down without touching the byte-identical `collectContext`); the gap + /// is synchronous, so the app is effectively stable. + static func webContext() -> (host: String?, isWebApp: Bool, addressBarFocused: Bool) { + guard AXIsProcessTrusted() else { return (nil, false, false) } + guard let app = NSWorkspace.shared.frontmostApplication else { return (nil, false, false) } + let appElement = AXUIElementCreateApplication(app.processIdentifier) + + // Opt Chromium/Electron apps into exposing their AX tree (harmless for apps that already do). + AXUIElementSetAttributeValue(appElement, "AXManualAccessibility" as CFString, kCFBooleanTrue) + + // Safari exposes the page URL on the focused window's AXDocument. + var urlString = copyElement(appElement, "AXFocusedWindow").flatMap { copyURLString($0, "AXDocument") } + + // Find the web area once; reused for the Chromium/PWA URL and the web-capable flag. + let webArea = findWebArea(appElement) + if urlString == nil, let webArea { + urlString = copyURLString(webArea, "AXURL") + } + + let host = hostFromURLString(urlString) + let isWebApp = (webArea != nil) || (host != nil) + let addressBarFocused = isWebApp && focusedIsChromeTextField(appElement) + return (host, isWebApp, addressBarFocused) + } + + /// Clean host from a URL string: http(s) only, lowercased, leading `www.` stripped; else `nil`. + /// (IDN hosts come back as punycode `xn--…` — left as-is; rare for these users.) + private static func hostFromURLString(_ raw: String?) -> String? { + guard let raw, let url = URL(string: raw), + let scheme = url.scheme?.lowercased(), scheme == "http" || scheme == "https", + var host = url.host?.lowercased(), !host.isEmpty + else { return nil } + if host.hasPrefix("www.") { host = String(host.dropFirst(4)) } + return host + } + + /// True when the focused element is the address/search bar: a text-input role that is browser chrome + /// (not inside the web page). + private static func focusedIsChromeTextField(_ appElement: AXUIElement) -> Bool { + guard let focused = copyElement(appElement, "AXFocusedUIElement"), + let role = copyString(focused, "AXRole") else { return false } + let textInputRoles: Set<String> = ["AXTextField", "AXComboBox", "AXSearchField"] + guard textInputRoles.contains(role) else { return false } + return !isInsideWebArea(focused) + } + + /// Walks up the parent chain (capped) to see if the element sits inside an `AXWebArea`. + private static func isInsideWebArea(_ element: AXUIElement) -> Bool { + var current = element + var hops = 0 + while hops < maxParentHops { + guard let parent = copyElement(current, "AXParent") else { return false } + if copyString(parent, "AXRole") == "AXWebArea" { return true } + current = parent + hops += 1 + } + return false + } + + /// Breadth-first search for the first `AXWebArea`, bounded by `traversalBudget` visited nodes. + /// BFS reaches the shallow web area before a wide toolbar subtree can exhaust the budget. + private static func findWebArea(_ root: AXUIElement) -> AXUIElement? { + var queue = [root] + var head = 0 + var visited = 0 + while head < queue.count, visited < traversalBudget { + let element = queue[head] + head += 1 + visited += 1 + if copyString(element, "AXRole") == "AXWebArea" { return element } + if let children = copyChildren(element) { queue.append(contentsOf: children) } + } + return nil + } + + // MARK: - AX attribute helpers + + /// Copies an element-typed AX attribute (e.g. the focused window). + private static func copyElement(_ element: AXUIElement, _ attribute: String) -> AXUIElement? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let value, CFGetTypeID(value) == AXUIElementGetTypeID() else { return nil } + return (value as! AXUIElement) + } + + /// Copies a string-typed AX attribute. + private static func copyString(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success else { return nil } + return value as? String + } + + /// Copies a URL- or string-typed AX attribute as a URL string (value may be a CFURL or a String). + private static func copyURLString(_ element: AXUIElement, _ attribute: String) -> String? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, attribute as CFString, &value) == .success, + let value else { return nil } + if CFGetTypeID(value) == CFURLGetTypeID() { return ((value as! CFURL) as URL).absoluteString } + return value as? String + } + + /// Copies the element's AX children (`as?` bridges the CFArray and yields nil on any non-array). + private static func copyChildren(_ element: AXUIElement) -> [AXUIElement]? { + var value: CFTypeRef? + guard AXUIElementCopyAttributeValue(element, "AXChildren" as CFString, &value) == .success, + let children = value as? [AXUIElement] else { return nil } + return children + } +} diff --git a/Sources/SettingsView.swift b/Sources/SettingsView.swift index 348a8e6a..8c8ec7fd 100644 --- a/Sources/SettingsView.swift +++ b/Sources/SettingsView.swift @@ -1523,6 +1523,7 @@ struct PromptsSettingsView: View { @State private var contextTestOutput: String? = nil @State private var contextTestError: String? = nil @State private var contextTestPrompt: String? = nil + @AppStorage(AppContextSource.modeKey) private var appContextMode: String = AppContextSource.Mode.screenshot.rawValue var body: some View { ScrollView { @@ -1533,7 +1534,7 @@ struct PromptsSettingsView: View { SettingsCard("Instruction Guard", icon: "shield.lefthalf.filled") { instructionGuardSection } - SettingsCard("Context Prompt", icon: "eye.fill") { + SettingsCard("App Context", icon: "doc.text.viewfinder") { contextPromptSection } } @@ -1793,6 +1794,18 @@ struct PromptsSettingsView: View { // MARK: Context Prompt + /// The per-mode description shown under the Context source segmented control. + private var contextSourceDescription: String { + switch AppContextSource.Mode(rawValue: appContextMode) ?? .screenshot { + case .off: + return "No context is sent — the cleanup model sees only your transcript." + case .metadata: + return "Sends one line of text about your app and page — instant, no added latency. E.g. \"User is dictating on github.com (Pull requests) in Safari\", or \"in the Chrome address bar\"." + case .screenshot: + return "Captures the window and asks the vision model — catches on-screen names and terms (e.g. in email). Uses vision quota; depending on your connection (ping, upload speed) it can add up to ~3 s per transcription." + } + } + private var contextPromptSection: some View { let isCustom = !appState.customContextPrompt.isEmpty let hasNewerDefault = isCustom @@ -1800,201 +1813,245 @@ struct PromptsSettingsView: View { && appState.customContextPromptLastModified < AppContextService.defaultContextPromptDate return VStack(alignment: .leading, spacing: 10) { - Text("Controls how \(AppName.displayName) infers your current activity from app metadata and screenshots.") + Text("Controls how \(AppName.displayName) infers your current activity and what it sends to the AI.") .font(.caption) .foregroundStyle(.secondary) - if hasNewerDefault { - HStack(spacing: 8) { - Image(systemName: "arrow.triangle.2.circlepath") - .foregroundStyle(.blue) - Text("A newer default prompt is available.") + // MARK: Context source (always active) + VStack(alignment: .leading, spacing: 8) { + HStack(alignment: .firstTextBaseline) { + Text("Context source") .font(.caption.weight(.semibold)) Spacer() - Button("View Default") { - showDefaultContextPrompt.toggle() - } - .font(.caption) - Button("Switch to Default") { - customContextPromptInput = AppContextService.defaultContextPrompt - appState.customContextPrompt = "" - appState.customContextPromptLastModified = "" + Picker("", selection: $appContextMode) { + Text("Off").tag(AppContextSource.Mode.off.rawValue) + Text("App summary").tag(AppContextSource.Mode.metadata.rawValue) + Text("Screenshot").tag(AppContextSource.Mode.screenshot.rawValue) } - .font(.caption) + .pickerStyle(.segmented) + .labelsHidden() + .fixedSize() + .accessibilityLabel("Context source") } - .padding(10) - .background(Color.blue.opacity(0.1)) - .cornerRadius(6) + + Text(contextSourceDescription) + .font(.caption) + .foregroundStyle(.secondary) } - if showDefaultContextPrompt { - VStack(alignment: .leading, spacing: 6) { - HStack { - Text("Default Context Prompt") + Divider() + + // MARK: Screenshot-only block — obscured + disabled unless in Screenshot mode + VStack(alignment: .leading, spacing: 10) { + // a) Vision Prompt + VStack(alignment: .leading, spacing: 10) { + VStack(alignment: .leading, spacing: 2) { + Text("Vision Prompt") .font(.caption.weight(.semibold)) - Spacer() - Button("Hide") { - showDefaultContextPrompt = false + Text("Instruction sent to the vision model in Screenshot mode.") + .font(.caption) + .foregroundStyle(.secondary) + } + + if hasNewerDefault { + HStack(spacing: 8) { + Image(systemName: "arrow.triangle.2.circlepath") + .foregroundStyle(.blue) + Text("A newer default prompt is available.") + .font(.caption.weight(.semibold)) + Spacer() + Button("View Default") { + showDefaultContextPrompt.toggle() + } + .font(.caption) + Button("Switch to Default") { + customContextPromptInput = AppContextService.defaultContextPrompt + appState.customContextPrompt = "" + appState.customContextPromptLastModified = "" + } + .font(.caption) } - .font(.caption) + .padding(10) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) } - Text(AppContextService.defaultContextPrompt) - .font(.system(.caption, design: .monospaced)) - .foregroundStyle(.secondary) - .textSelection(.enabled) - } - .padding(10) - .background(Color(nsColor: .controlBackgroundColor)) - .cornerRadius(6) - } - TextEditor(text: $customContextPromptInput) - .font(.system(.body, design: .monospaced)) - .frame(minHeight: 120, maxHeight: 200) - .overlay( - RoundedRectangle(cornerRadius: 6) - .stroke(Color.secondary.opacity(0.3), lineWidth: 1) - ) - .onChange(of: customContextPromptInput) { newValue in - let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) - let defaultTrimmed = AppContextService.defaultContextPrompt.trimmingCharacters(in: .whitespacesAndNewlines) - if trimmed == defaultTrimmed || trimmed.isEmpty { - if !appState.customContextPrompt.isEmpty { - appState.customContextPrompt = "" - appState.customContextPromptLastModified = "" + if showDefaultContextPrompt { + VStack(alignment: .leading, spacing: 6) { + HStack { + Text("Default Vision Prompt") + .font(.caption.weight(.semibold)) + Spacer() + Button("Hide") { + showDefaultContextPrompt = false + } + .font(.caption) + } + Text(AppContextService.defaultContextPrompt) + .font(.system(.caption, design: .monospaced)) + .foregroundStyle(.secondary) + .textSelection(.enabled) } - } else { - appState.customContextPrompt = trimmed - let today = iso8601DayFormatter.string(from: Date()) - if appState.customContextPromptLastModified != today { - appState.customContextPromptLastModified = today + .padding(10) + .background(Color(nsColor: .controlBackgroundColor)) + .cornerRadius(6) + } + + TextEditor(text: $customContextPromptInput) + .font(.system(.body, design: .monospaced)) + .frame(minHeight: 120, maxHeight: 200) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.secondary.opacity(0.3), lineWidth: 1) + ) + .onChange(of: customContextPromptInput) { newValue in + let trimmed = newValue.trimmingCharacters(in: .whitespacesAndNewlines) + let defaultTrimmed = AppContextService.defaultContextPrompt.trimmingCharacters(in: .whitespacesAndNewlines) + if trimmed == defaultTrimmed || trimmed.isEmpty { + if !appState.customContextPrompt.isEmpty { + appState.customContextPrompt = "" + appState.customContextPromptLastModified = "" + } + } else { + appState.customContextPrompt = trimmed + let today = iso8601DayFormatter.string(from: Date()) + if appState.customContextPromptLastModified != today { + appState.customContextPromptLastModified = today + } + } + } + + HStack { + if isCustom { + Label("Using custom prompt", systemImage: "pencil") + .font(.caption) + .foregroundStyle(.blue) + } else { + Label("Using default", systemImage: "checkmark.circle") + .font(.caption) + .foregroundStyle(.secondary) + } + Spacer() + if isCustom { + Button("Reset to Default") { + customContextPromptInput = AppContextService.defaultContextPrompt + appState.customContextPrompt = "" + appState.customContextPromptLastModified = "" + } + .font(.caption) } } } - HStack { - if isCustom { - Label("Using custom prompt", systemImage: "pencil") - .font(.caption) - .foregroundStyle(.blue) - } else { - Label("Using default", systemImage: "checkmark.circle") + // b) Test Vision Prompt + VStack(alignment: .leading, spacing: 8) { + Text("Test Vision Prompt") + .font(.caption.weight(.semibold)) + Text("Runs the vision prompt on a fresh screenshot of the frontmost app.") .font(.caption) .foregroundStyle(.secondary) - } - Spacer() - if isCustom { - Button("Reset to Default") { - customContextPromptInput = AppContextService.defaultContextPrompt - appState.customContextPrompt = "" - appState.customContextPromptLastModified = "" - } - .font(.caption) - } - } - Divider() - - VStack(alignment: .leading, spacing: 8) { - Text("Screenshot Resolution") - .font(.caption.weight(.semibold)) - - Text("Controls the maximum image dimension sent for context inference.") - .font(.caption) - .foregroundStyle(.secondary) - - Picker("", selection: $appState.contextScreenshotMaxDimension) { - ForEach(AppState.contextScreenshotDimensionOptions, id: \.self) { dimension in - Text("\(dimension) px").tag(dimension) + Button { + runContextPromptTest() + } label: { + HStack(spacing: 6) { + if contextTestRunning { + ProgressView() + .controlSize(.small) + Text("Running...") + } else { + Image(systemName: "play.fill") + Text("Test Vision Prompt") + } + } } - } - .pickerStyle(.segmented) - .labelsHidden() - .accessibilityLabel("Screenshot Resolution") + .disabled(contextTestRunning || appState.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) - HStack { - if appState.contextScreenshotMaxDimension == AppState.defaultContextScreenshotMaxDimension { - Label("Using default", systemImage: "checkmark.circle") + if appState.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + Label("API key required to test", systemImage: "exclamationmark.triangle") .font(.caption) - .foregroundStyle(.secondary) - } else { - Label("Using custom value", systemImage: "pencil") + .foregroundStyle(.orange) + } + + if let error = contextTestError { + Label(error, systemImage: "xmark.circle.fill") .font(.caption) - .foregroundStyle(.blue) + .foregroundStyle(.red) } - Spacer() - if appState.contextScreenshotMaxDimension != AppState.defaultContextScreenshotMaxDimension { - Button("Reset to Default") { - appState.contextScreenshotMaxDimension = AppState.defaultContextScreenshotMaxDimension + + if let output = contextTestOutput { + VStack(alignment: .leading, spacing: 4) { + Text("Result:") + .font(.caption.weight(.semibold)) + Text(output.isEmpty ? "(empty — no output)" : output) + .font(.system(.caption, design: .monospaced)) + .textSelection(.enabled) + .padding(8) + .frame(maxWidth: .infinity, alignment: .leading) + .background(Color.green.opacity(0.08)) + .cornerRadius(6) + } + } + + if let prompt = contextTestPrompt { + DisclosureGroup("Full prompt sent") { + Text(prompt) + .font(.system(.caption2, design: .monospaced)) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) } .font(.caption) + .foregroundStyle(.secondary) } } - } - Divider() + Divider() - // Test section - VStack(alignment: .leading, spacing: 8) { - Text("Test Context Prompt") - .font(.caption.weight(.semibold)) - Text("Captures a screenshot and metadata from the frontmost app, then runs the context prompt to infer activity.") - .font(.caption) - .foregroundStyle(.secondary) + // c) Screenshot scope — the per-app multiselector + ScreenshotScopeSection() - Button { - runContextPromptTest() - } label: { - HStack(spacing: 6) { - if contextTestRunning { - ProgressView() - .controlSize(.small) - Text("Running...") - } else { - Image(systemName: "play.fill") - Text("Test Context Prompt") - } - } - } - .disabled(contextTestRunning || appState.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + Divider() - if appState.apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { - Label("API key required to test", systemImage: "exclamationmark.triangle") - .font(.caption) - .foregroundStyle(.orange) - } + // d) Screenshot Resolution + VStack(alignment: .leading, spacing: 8) { + Text("Screenshot Resolution") + .font(.caption.weight(.semibold)) - if let error = contextTestError { - Label(error, systemImage: "xmark.circle.fill") + Text("Maximum image dimension sent for context inference.") .font(.caption) - .foregroundStyle(.red) - } + .foregroundStyle(.secondary) - if let output = contextTestOutput { - VStack(alignment: .leading, spacing: 4) { - Text("Result:") - .font(.caption.weight(.semibold)) - Text(output.isEmpty ? "(empty — no output)" : output) - .font(.system(.caption, design: .monospaced)) - .textSelection(.enabled) - .padding(8) - .frame(maxWidth: .infinity, alignment: .leading) - .background(Color.green.opacity(0.08)) - .cornerRadius(6) + Picker("", selection: $appState.contextScreenshotMaxDimension) { + ForEach(AppState.contextScreenshotDimensionOptions, id: \.self) { dimension in + Text("\(dimension) px").tag(dimension) + } } - } + .pickerStyle(.segmented) + .labelsHidden() + .accessibilityLabel("Screenshot Resolution") - if let prompt = contextTestPrompt { - DisclosureGroup("Full prompt sent") { - Text(prompt) - .font(.system(.caption2, design: .monospaced)) - .textSelection(.enabled) - .frame(maxWidth: .infinity, alignment: .leading) + HStack { + if appState.contextScreenshotMaxDimension == AppState.defaultContextScreenshotMaxDimension { + Label("Using default", systemImage: "checkmark.circle") + .font(.caption) + .foregroundStyle(.secondary) + } else { + Label("Using custom value", systemImage: "pencil") + .font(.caption) + .foregroundStyle(.blue) + } + Spacer() + if appState.contextScreenshotMaxDimension != AppState.defaultContextScreenshotMaxDimension { + Button("Reset to Default") { + appState.contextScreenshotMaxDimension = AppState.defaultContextScreenshotMaxDimension + } + .font(.caption) + } } - .font(.caption) - .foregroundStyle(.secondary) } } + .opacity(appContextMode == AppContextSource.Mode.screenshot.rawValue ? 1 : 0.5) + .disabled(appContextMode != AppContextSource.Mode.screenshot.rawValue) } } @@ -2005,6 +2062,8 @@ struct PromptsSettingsView: View { contextTestPrompt = nil let service = appState.makeAppContextService() + // Force the vision path on this throwaway instance, regardless of the saved mode/scope. + service.forceScreenshotForTest = true Task { let context = await service.collectContext() From 3306672287a78eb9fd0688f3b3d6e4510889ae72 Mon Sep 17 00:00:00 2001 From: Saphi <saphi@airSaphi.local> Date: Thu, 25 Jun 2026 07:51:35 -0300 Subject: [PATCH 2/3] =?UTF-8?q?fix(app-context):=20address=20CodeRabbit=20?= =?UTF-8?q?=E2=80=94=20frontmost-app=20consistency=20guard=20+=20address-b?= =?UTF-8?q?ar=20precedence?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - webContext(expectedBundleID:) bails when the frontmost app changed since collectContext captured it, so the App-summary line never mixes one app's page host with another app's name/title (CodeRabbit Major #244). Kept collectContext byte-identical: the leaf passes the already-captured bundleIdentifier instead of threading the AXUIElement (which would edit collectContext and break cross-feature coexistence). - metadataSummary checks addressBarFocused before the page host, so typing in the omnibox reads 'in the <app> address bar' even when a page is loaded (CodeRabbit Minor #244). Files: Sources/AppContextSource.swift, Sources/AppContextService.swift. --- Sources/AppContextService.swift | 4 ++-- Sources/AppContextSource.swift | 20 ++++++++++++-------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/Sources/AppContextService.swift b/Sources/AppContextService.swift index e2ba1dd5..e879d74e 100644 --- a/Sources/AppContextService.swift +++ b/Sources/AppContextService.swift @@ -333,8 +333,8 @@ Selected text: \(selectedText ?? "None") case .off: return "" case .metadata: - // One AX read: host / web-capable / address-bar focus. - let web = AppContextSource.webContext() + // One AX read: host / web-capable / address-bar focus (skips if the frontmost app changed). + let web = AppContextSource.webContext(expectedBundleID: bundleIdentifier) return AppContextSource.metadataSummary( appName: appName, pageTitle: windowTitle, diff --git a/Sources/AppContextSource.swift b/Sources/AppContextSource.swift index b2a1d387..2b07eafb 100644 --- a/Sources/AppContextSource.swift +++ b/Sources/AppContextSource.swift @@ -102,6 +102,11 @@ enum AppContextSource { let app = (appName?.isEmpty == false) ? appName! : "the active app" let title = cleanedPageTitle(pageTitle, appName: appName, host: webHost) + // Address bar focused → you're typing a URL/search, not into the page; check before the host. + if addressBarFocused { + return "User is dictating in the \(app) address bar" + } + if let webHost, !webHost.isEmpty { // Drop "in <app>" when the host already names the app (e.g. the ChatGPT web app). let base = hostMatchesApp(webHost, appName: appName) @@ -111,10 +116,6 @@ enum AppContextSource { return base } - if addressBarFocused { - return "User is dictating in the \(app) address bar" - } - // Web app whose URL the browser doesn't expose (e.g. Firefox/Zen) — use the page title. if isWebApp, let title { return "User is dictating on \"\(title)\" in \(app)" @@ -189,12 +190,15 @@ enum AppContextSource { private static let maxParentHops = 12 /// Reads one dictation's web context: page host (if readable), whether the app renders web content, - /// and whether the address/search bar is focused. Reads `frontmostApplication` independently (the - /// caller can't pass an element down without touching the byte-identical `collectContext`); the gap - /// is synchronous, so the app is effectively stable. - static func webContext() -> (host: String?, isWebApp: Bool, addressBarFocused: Bool) { + /// and whether the address/search bar is focused. Reads `frontmostApplication` here (the caller can't + /// pass the element down without editing the byte-identical `collectContext`), but bails when the + /// frontmost app no longer matches `expectedBundleID`, so it never mixes another app's page into a + /// summary captured for a different app. + static func webContext(expectedBundleID: String?) -> (host: String?, isWebApp: Bool, addressBarFocused: Bool) { guard AXIsProcessTrusted() else { return (nil, false, false) } guard let app = NSWorkspace.shared.frontmostApplication else { return (nil, false, false) } + // Focus changed since collectContext captured the app → don't read a different app's page. + if let expectedBundleID, app.bundleIdentifier != expectedBundleID { return (nil, false, false) } let appElement = AXUIElementCreateApplication(app.processIdentifier) // Opt Chromium/Electron apps into exposing their AX tree (harmless for apps that already do). From af7b93db9977a556401197844adc5b7794c39491 Mon Sep 17 00:00:00 2001 From: Saphi <saphi@MacBook-Pro.local> Date: Sun, 19 Jul 2026 04:55:32 -0300 Subject: [PATCH 3/3] build: include AppContextSource.swift in the make test compile unit The upstream test target hardcodes its source list; this branch's AppContextService.swift now references AppContextSource, so the test runner failed to compile. Declared as a separate prerequisite line (prerequisites accumulate in make) and switched the recipe to $^ so the compile list always mirrors the declared prerequisites. Verified: make test builds and passes. Modified files: - Makefile --- Makefile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 00fda5c5..47e7aff2 100644 --- a/Makefile +++ b/Makefile @@ -69,6 +69,9 @@ endif @codesign --force --options runtime --sign "$(CODESIGN_IDENTITY)" --entitlements FreeFlow.entitlements "$(APP_BUNDLE)" @echo "Built $(APP_BUNDLE)" +# AppContextService now depends on this source, so the test build needs it too. +$(TEST_RUNNER): Sources/AppContextSource.swift + test: $(TEST_RUNNER) @$(TEST_RUNNER) @@ -79,7 +82,7 @@ $(TEST_RUNNER): Sources/AppContextService.swift Sources/LLMAPITransport.swift So -o "$(TEST_RUNNER)" \ -sdk $(shell xcrun --show-sdk-path) \ -target $(ARCH)-apple-macosx13.0 \ - Sources/AppContextService.swift Sources/LLMAPITransport.swift Sources/ModelConfiguration.swift Tests/AppContextServiceTests.swift + $^ icon: $(ICON_ICNS)