Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
164 changes: 161 additions & 3 deletions Sources/Fluid/ContentView.swift
Original file line number Diff line number Diff line change
Expand Up @@ -151,6 +151,7 @@ struct ContentView: View {
@State private var previousSidebarItem: SidebarItem? = nil // Track previous for mode transitions
@State private var playgroundUsed: Bool = SettingsStore.shared.playgroundUsed
@State private var recordingAppInfo: (name: String, bundleId: String, windowTitle: String)? = nil
@State private var recordingAutoLearnElement: AXUIElement? = nil

// Command Mode State
// @State private var showCommandMode: Bool = false
Expand Down Expand Up @@ -1391,6 +1392,14 @@ struct ContentView: View {
let focusedPID = TypingService.captureSystemFocusedPID()
?? NSWorkspace.shared.frontmostApplication?.processIdentifier
NotchContentState.shared.recordingTargetPID = focusedPID
if SettingsStore.shared.autoLearnCustomDictionaryEnabled {
self.recordingAutoLearnElement = AutoLearnDictionaryService.shared.captureFocusedElement(
preferredPID: focusedPID,
allowsSystemFocusedPIDMismatch: true
)
} else {
self.recordingAutoLearnElement = nil
}

let info = self.getCurrentAppInfo()
self.recordingAppInfo = info
Expand Down Expand Up @@ -2015,10 +2024,74 @@ struct ContentView: View {
if typingTarget.shouldRestoreOriginalFocus {
await self.restoreFocusToRecordingTarget()
}
self.asr.typeTextToActiveField(
let shouldCaptureAutoLearn = SettingsStore.shared.autoLearnCustomDictionaryEnabled
let preInsertionMonitoringElement = shouldCaptureAutoLearn
? AutoLearnDictionaryService.shared.captureFocusedElement(
preferredPID: typingTarget.pid,
allowsSystemFocusedPIDMismatch: true
)
: nil
let recordingMonitoringElement = shouldCaptureAutoLearn ? self.recordingAutoLearnElement : nil
let textInsertionMode = SettingsStore.shared.textInsertionMode
let didAcceptTyping = self.asr.typeTextToActiveField(
finalText,
preferredTargetPID: typingTarget.pid
)
) {
guard SettingsStore.shared.autoLearnCustomDictionaryEnabled else {
self.recordingAutoLearnElement = nil
return
}

// Now that typing is physically complete, we can start monitoring.
// Prefer target-PID matches so focus restore failures do not attach to FluidVoice or another app.
// Some editors expose the focused AX text element from a helper process, so trusted
// pre-insertion captures can still be valid even when their PID differs from the app PID.
let monitoringElement = self.autoLearnMonitoringElement(
targetPID: typingTarget.pid,
candidates: [
AutoLearnMonitoringCandidate(
label: "pre-insertion",
element: preInsertionMonitoringElement,
allowsHelperProcessFallback: true
),
AutoLearnMonitoringCandidate(
label: "recording",
element: recordingMonitoringElement,
allowsHelperProcessFallback: true
),
AutoLearnMonitoringCandidate(
label: "completion",
element: AutoLearnDictionaryService.shared.captureFocusedElement(preferredPID: typingTarget.pid),
allowsHelperProcessFallback: false
),
]
)
self.recordingAutoLearnElement = nil
guard let monitoringElement else {
DebugLogger.shared.debug(
"Auto-learn AX monitoring skipped: no AX element matched typing target; starting event fallback.",
source: "ContentView"
)
AutoLearnDictionaryService.shared.beginEventFallbackMonitoring(
pastedText: finalText,
targetPID: typingTarget.pid
)
return
}
AutoLearnDictionaryService.shared.beginMonitoring(
pastedText: finalText,
element: monitoringElement,
targetPID: typingTarget.pid
)
}
if shouldCaptureAutoLearn, textInsertionMode == .reliablePaste, didAcceptTyping {
// Reliable Paste completion may wait for slow paste targets. Prime the AX-blind fallback
// after typing is accepted so quick corrections in prompt boxes are still observed.
AutoLearnDictionaryService.shared.beginEventFallbackMonitoring(
pastedText: finalText,
targetPID: typingTarget.pid
)
}
didTypeExternally = true
}

Expand All @@ -2042,7 +2115,6 @@ struct ContentView: View {
aiModel: modelInfo.model,
aiProvider: modelInfo.provider
)

NotchOverlayManager.shared.hide()
} else if shouldPersistOutputs,
SettingsStore.shared.copyTranscriptionToClipboard == false,
Expand Down Expand Up @@ -2074,6 +2146,92 @@ struct ContentView: View {
return .normal
}

private struct AutoLearnMonitoringCandidate {
let label: String
let element: AXUIElement?
let allowsHelperProcessFallback: Bool
}

private func autoLearnMonitoringElement(
targetPID: pid_t?,
candidates: [AutoLearnMonitoringCandidate]
) -> AXUIElement? {
let resolvedCandidates = candidates.compactMap { candidate -> (candidate: AutoLearnMonitoringCandidate, pid: pid_t?)? in
guard let element = candidate.element else { return nil }
return (candidate, AutoLearnDictionaryService.shared.pid(for: element))
}

self.logAutoLearnCandidatePIDs(targetPID: targetPID, candidates: resolvedCandidates)

guard let targetPID, targetPID > 0 else {
return resolvedCandidates.first?.candidate.element
}

if let matchedCandidate = resolvedCandidates.first(where: { $0.pid == targetPID }) {
return matchedCandidate.candidate.element
}

let selfPID = ProcessInfo.processInfo.processIdentifier
let selfBundleID = Bundle.main.bundleIdentifier
let targetBundleID = self.autoLearnTargetBundleID(targetPID: targetPID)
for resolvedCandidate in resolvedCandidates where resolvedCandidate.candidate.allowsHelperProcessFallback {
guard let candidatePID = resolvedCandidate.pid,
candidatePID != selfPID
else {
continue
}

let candidateBundleID = NSRunningApplication(processIdentifier: candidatePID)?.bundleIdentifier
guard candidateBundleID != selfBundleID else { continue }
guard let candidateBundleID,
let targetBundleID,
self.isAutoLearnSameAppFamily(candidateBundleID, targetBundleID)
else {
continue
}

DebugLogger.shared.debug(
"Auto-learn monitoring using helper-process AX element: " +
"source=\(resolvedCandidate.candidate.label), targetPID=\(targetPID), " +
"elementPID=\(candidatePID), elementBundle=\(candidateBundleID)",
source: "ContentView"
)
return resolvedCandidate.candidate.element
}

return nil
}

private func autoLearnTargetBundleID(targetPID: pid_t) -> String? {
NSRunningApplication(processIdentifier: targetPID)?.bundleIdentifier
}

private func isAutoLearnSameAppFamily(_ lhs: String, _ rhs: String) -> Bool {
let lhs = lhs.lowercased()
let rhs = rhs.lowercased()
guard !lhs.isEmpty, !rhs.isEmpty else { return false }
return lhs == rhs ||
lhs.hasPrefix("\(rhs).") ||
rhs.hasPrefix("\(lhs).")
}

private func logAutoLearnCandidatePIDs(
targetPID: pid_t?,
candidates: [(candidate: AutoLearnMonitoringCandidate, pid: pid_t?)]
) {
let summary = candidates.map { resolvedCandidate -> String in
let pid = resolvedCandidate.pid.map(String.init) ?? "nil"
let bundleID = resolvedCandidate.pid
.flatMap { NSRunningApplication(processIdentifier: $0)?.bundleIdentifier }
?? "unknown"
return "\(resolvedCandidate.candidate.label):pid=\(pid),bundle=\(bundleID)"
}.joined(separator: "; ")
DebugLogger.shared.debug(
"Auto-learn monitoring candidates: targetPID=\(targetPID.map(String.init) ?? "nil"); \(summary)",
source: "ContentView"
)
}

private func reprocessLastDictationFromHistory() {
guard let last = TranscriptionHistoryStore.shared.entries.first else {
DebugLogger.shared.info("Actions: Reprocess requested but history is empty", source: "ContentView")
Expand Down
Loading