diff --git a/Sources/StorageScope/Stores/ScanStore.swift b/Sources/StorageScope/Stores/ScanStore.swift index 41f5309..a171621 100644 --- a/Sources/StorageScope/Stores/ScanStore.swift +++ b/Sources/StorageScope/Stores/ScanStore.swift @@ -970,7 +970,10 @@ func setSelectedView(_ view: SmartView) { // this the progress-consumer sibling would block on `for await` // forever and the task group would hang waiting for it. defer { progressContinuation.finish() } - let scanner = FileSystemScanner(hashCache: hashCache) + let scanner = FileSystemScanner( + hashCache: hashCache, + incrementalRescans: true + ) let scan = try scanner.scan( root: url, options: options, diff --git a/Sources/StorageScopeBenchmark/main.swift b/Sources/StorageScopeBenchmark/main.swift index 55851b6..fdb73ec 100644 --- a/Sources/StorageScopeBenchmark/main.swift +++ b/Sources/StorageScopeBenchmark/main.swift @@ -14,12 +14,13 @@ struct BenchmarkArguments { /// Fraction in [0, 1] of `items` that become content-identical duplicates. Stored /// as the raw user-supplied string so we can validate before parsing (e.g. 0.05 = 5%). var duplicateRatio: Double = 0 + var repeatCount = 1 } func usage() -> String { """ Usage: - StorageScopeBenchmark [--show-full-path] + StorageScopeBenchmark [--show-full-path] [--repeat ] StorageScopeBenchmark --synthetic [--keep-fixture] [--show-full-path] StorageScopeBenchmark --synthetic --items [--depth ] [--duplicates <0..1>] [--keep-fixture] @@ -31,6 +32,7 @@ func usage() -> String { Examples: swift run StorageScopeBenchmark --synthetic --items 100000 --depth 8 --duplicates 0.2 swift run StorageScopeBenchmark --synthetic --items 500000 --depth 12 --keep-fixture + STORAGESCOPE_INCREMENTAL_RESCAN=1 swift run -c release StorageScopeBenchmark --repeat 3 """ } @@ -73,6 +75,12 @@ func parseArguments(_ rawArguments: [String]) throws -> BenchmarkArguments { throw BenchmarkError.invalidArgument("--duplicates requires a fraction in [0,1], got \(raw)") } arguments.duplicateRatio = parsed + case "--repeat": + let raw = try consumeNextValue(flag: value) + guard let parsed = Int(raw), parsed > 0 else { + throw BenchmarkError.invalidArgument("--repeat requires a positive integer, got \(raw)") + } + arguments.repeatCount = parsed case "-h", "--help": print(usage()) exit(0) @@ -140,15 +148,21 @@ do { cleanup?() } - let report = try ScanBenchmarkRunner().run( - rootURL: rootURL, - showFullPath: arguments.showFullPath - ) - print(report.text) + let runner = ScanBenchmarkRunner() + for run in 1...arguments.repeatCount { + if arguments.repeatCount > 1 { + print("Run \(run)/\(arguments.repeatCount)") + } + let report = try runner.run( + rootURL: rootURL, + showFullPath: arguments.showFullPath + ) + print(report.text) + } } catch { let message = (error as? LocalizedError)?.errorDescription ?? "\(error)" FileHandle.standardError.write( Data("StorageScopeBenchmark failed: \(message)\n\n\(usage())\n".utf8) ) exit(2) -} \ No newline at end of file +} diff --git a/Sources/StorageScopeCore/Models/StorageScan.swift b/Sources/StorageScopeCore/Models/StorageScan.swift index 8a8715c..7fcbf85 100644 --- a/Sources/StorageScopeCore/Models/StorageScan.swift +++ b/Sources/StorageScopeCore/Models/StorageScan.swift @@ -2,6 +2,12 @@ import Foundation import os.signpost public struct StorageScan: Sendable { + public enum RescanKind: String, Sendable { + case full + case incrementalUnchanged + case incrementalChanged + } + /// os_signpost surface for Instruments. Shares the scan subsystem/category so the /// `lookup_build` span shows up alongside FileSystemScanner and ScanStore spans (#81-pattern). private static let log = OSLog(subsystem: "com.rasputinkaiser.StorageScope", category: "scan") @@ -33,6 +39,9 @@ public struct StorageScan: Sendable { public let enumerateDuration: TimeInterval public let cleanupCandidates: [CleanupCandidate] public let isPartial: Bool + public let rescanKind: RescanKind + public let incrementalDirtySubtreeCount: Int + public let incrementalFallbackReason: String? private let itemLookupByID: [String: StorageItem] public init( @@ -61,7 +70,10 @@ public struct StorageScan: Sendable { duplicateVerificationBytesRead: Int64 = 0, enumerateDuration: TimeInterval = 0, cleanupCandidates: [CleanupCandidate], - isPartial: Bool = false + isPartial: Bool = false, + rescanKind: RescanKind = .full, + incrementalDirtySubtreeCount: Int = 0, + incrementalFallbackReason: String? = nil ) { self.rootURL = rootURL self.startedAt = startedAt @@ -89,6 +101,9 @@ public struct StorageScan: Sendable { self.enumerateDuration = enumerateDuration self.cleanupCandidates = cleanupCandidates self.isPartial = isPartial + self.rescanKind = rescanKind + self.incrementalDirtySubtreeCount = incrementalDirtySubtreeCount + self.incrementalFallbackReason = incrementalFallbackReason self.itemLookupByID = Self.buildItemLookup( retainedItems: retainedItems, largestFiles: largestFiles, @@ -104,6 +119,40 @@ public struct StorageScan: Sendable { itemLookupByID[id] } + func reusedIncrementally(startedAt: Date, finishedAt: Date) -> StorageScan { + StorageScan( + rootURL: rootURL, + startedAt: startedAt, + finishedAt: finishedAt, + rootItem: rootItem, + retainedItems: retainedItems, + scannedItemCount: scannedItemCount, + inaccessibleItemCount: inaccessibleItemCount, + totalBytes: totalBytes, + largestFiles: largestFiles, + largestFolders: largestFolders, + oldLargeFiles: oldLargeFiles, + typeBreakdown: typeBreakdown, + categoryBreakdown: categoryBreakdown, + duplicateSizeGroups: duplicateSizeGroups, + verifiedDuplicateGroups: verifiedDuplicateGroups, + duplicateCandidateItemLimit: duplicateCandidateItemLimit, + duplicateCandidateItemsRetained: duplicateCandidateItemsRetained, + duplicateCandidateItemsConsidered: duplicateCandidateItemsConsidered, + duplicateCandidateEvictionCount: duplicateCandidateEvictionCount, + duplicateCandidateLimitReached: duplicateCandidateLimitReached, + snapshotBuildCount: 0, + duplicateVerificationDuration: 0, + duplicateVerificationBytesRead: 0, + enumerateDuration: finishedAt.timeIntervalSince(startedAt), + cleanupCandidates: cleanupCandidates, + isPartial: false, + rescanKind: .incrementalUnchanged, + incrementalDirtySubtreeCount: 0, + incrementalFallbackReason: nil + ) + } + private static func buildItemLookup( retainedItems: [StorageItem], largestFiles: [StorageItem], diff --git a/Sources/StorageScopeCore/Services/FileSystemScanner.swift b/Sources/StorageScopeCore/Services/FileSystemScanner.swift index 86fc8c2..cc842ac 100644 --- a/Sources/StorageScopeCore/Services/FileSystemScanner.swift +++ b/Sources/StorageScopeCore/Services/FileSystemScanner.swift @@ -115,6 +115,9 @@ public final class FileSystemScanner { private let fileManager: FileManager private let hashCache: DuplicateHashCache? private let walkerMode: FileSystemScannerWalkerMode + private let incrementalPersistence: IncrementalScanPersistence? + private let incrementalChangeSource: IncrementalChangeSource? + private let incrementalTrustButVerify: Bool private let resourceKeys: Set = [ .isDirectoryKey, .isRegularFileKey, @@ -128,22 +131,40 @@ public final class FileSystemScanner { .contentModificationDateKey ] - public convenience init(fileManager: FileManager = .default, hashCache: DuplicateHashCache? = nil) { + public convenience init( + fileManager: FileManager = .default, + hashCache: DuplicateHashCache? = nil, + incrementalRescans: Bool = false + ) { + let environment = ProcessInfo.processInfo.environment + let incrementalEnabled = environment["STORAGESCOPE_LEGACY_WALKER"] != "1" && + (incrementalRescans || environment["STORAGESCOPE_INCREMENTAL_RESCAN"] == "1") self.init( fileManager: fileManager, hashCache: hashCache, - walkerMode: Self.configuredWalkerMode() + walkerMode: Self.configuredWalkerMode(), + incrementalPersistence: incrementalEnabled + ? IncrementalScanPersistence(baseURL: IncrementalScanPersistence.defaultBaseURL(fileManager: fileManager), fileManager: fileManager) + : nil, + incrementalChangeSource: incrementalEnabled ? SystemFSEventsChangeSource() : nil, + incrementalTrustButVerify: incrementalEnabled && environment["STORAGESCOPE_INCREMENTAL_TRUST_VERIFY"] != "0" ) } init( fileManager: FileManager = .default, hashCache: DuplicateHashCache? = nil, - walkerMode: FileSystemScannerWalkerMode + walkerMode: FileSystemScannerWalkerMode, + incrementalPersistence: IncrementalScanPersistence? = nil, + incrementalChangeSource: IncrementalChangeSource? = nil, + incrementalTrustButVerify: Bool = false ) { self.fileManager = fileManager self.hashCache = hashCache - self.walkerMode = walkerMode + self.walkerMode = incrementalPersistence == nil ? walkerMode : .fixedWorker + self.incrementalPersistence = incrementalPersistence + self.incrementalChangeSource = incrementalChangeSource + self.incrementalTrustButVerify = incrementalTrustButVerify } private static func configuredWalkerMode() -> FileSystemScannerWalkerMode { @@ -175,8 +196,23 @@ public final class FileSystemScanner { "root=%{public}@", rootURL.path) let startedAt = Date() + try cancellation?.check() + if let reused = try reuseUnchangedScanIfAvailable( + rootURL: rootURL, + options: options, + startedAt: startedAt, + cancellation: cancellation, + progress: progress + ) { + os_signpost(.end, log: Self.log, name: "scan", signpostID: Self.signpostID, + "items=%d verified=%d", reused.scannedItemCount, reused.verifiedDuplicateGroups.count) + return reused + } let accumulator = ScanAccumulator(options: options, progress: progress, onSnapshot: onSnapshot, rootURL: rootURL, startedAt: startedAt) let rootItem: StorageItem + let rescanKind: StorageScan.RescanKind + let incrementalDirtySubtreeCount: Int + let incrementalFallbackReason: String? switch walkerMode { case .legacy: rootItem = try scanItem( @@ -186,13 +222,20 @@ public final class FileSystemScanner { accumulator: accumulator, depth: 0 ) + rescanKind = .full + incrementalDirtySubtreeCount = 0 + incrementalFallbackReason = nil case .fixedWorker: - rootItem = try scanWithFixedWorker( + let fixedWorkerResult = try scanWithFixedWorker( at: rootURL, options: options, cancellation: cancellation, accumulator: accumulator ) + rootItem = fixedWorkerResult.rootItem + rescanKind = fixedWorkerResult.rescanKind + incrementalDirtySubtreeCount = fixedWorkerResult.dirtySubtreeCount + incrementalFallbackReason = fixedWorkerResult.fallbackReason } let retainedItems = rootItem.flattened() let duplicateSizeGroups = accumulator.duplicateSizeGroups @@ -211,7 +254,7 @@ public final class FileSystemScanner { os_signpost(.end, log: Self.log, name: "scan", signpostID: Self.signpostID, "items=%d verified=%d", accumulator.scannedItemCount, verifiedDuplicateGroups.count) - return StorageScan( + let finalScan = StorageScan( rootURL: rootURL, startedAt: startedAt, finishedAt: finishedAt, @@ -241,10 +284,108 @@ public final class FileSystemScanner { verifiedDuplicateGroups: verifiedDuplicateGroups, limit: options.maxRankedResults ), - isPartial: false + isPartial: false, + rescanKind: rescanKind, + incrementalDirtySubtreeCount: incrementalDirtySubtreeCount, + incrementalFallbackReason: incrementalFallbackReason + ) + storeIncrementalScanInMemory(finalScan, options: options) + return finalScan + } + + private func reuseUnchangedScanIfAvailable( + rootURL: URL, + options: ScanOptions, + startedAt: Date, + cancellation: ScanCancellation?, + progress: ProgressHandler? + ) throws -> StorageScan? { + guard walkerMode == .fixedWorker, + let incrementalPersistence, + let incrementalChangeSource, + let volumeIdentity = IncrementalVolumeIdentity.read(from: rootURL) else { + return nil + } + try cancellation?.check() + let fingerprint = IncrementalScanOptionsFingerprint(options) + guard let entry = IncrementalScanMemoryCache.shared.entry( + rootURL: rootURL, + volumeIdentity: volumeIdentity, + options: fingerprint, + cacheIdentity: incrementalPersistence.cacheIdentity(rootURL: rootURL) + ) else { + return nil + } + let checkpoint = incrementalChangeSource.currentEventID() + guard entry.checkpoint <= checkpoint else { return nil } + let changes: IncrementalChangeSet + if incrementalChangeSource is SystemFSEventsChangeSource, + let liveChanges = LiveFSEventsMonitorRegistry.shared.flushAndDrainIfReady(rootURL: rootURL) { + changes = liveChanges + } else if entry.checkpoint == checkpoint { + changes = IncrementalChangeSet(paths: [], requiresFullScan: false, reason: nil) + } else { + changes = incrementalChangeSource.changes(rootURL: rootURL, since: entry.checkpoint) + } + guard !changes.requiresFullScan, changes.paths.isEmpty else { + return nil + } + + try cancellation?.check() + try incrementalPersistence.saveCheckpoint(checkpoint, rootURL: rootURL) + let finishedAt = Date() + let reused = entry.scan.reusedIncrementally(startedAt: startedAt, finishedAt: finishedAt) + if let cacheIdentity = incrementalPersistence.cacheIdentity(rootURL: rootURL) { + IncrementalScanMemoryCache.shared.store( + scan: reused, + checkpoint: checkpoint, + volumeIdentity: volumeIdentity, + options: fingerprint, + cacheIdentity: cacheIdentity + ) + } + progress?(ScanProgress( + scannedItemCount: reused.scannedItemCount, + totalBytes: reused.totalBytes, + currentPath: rootURL.path, + phase: .enumerating + )) + schedulePersistedTrustVerification(rootURL: rootURL, options: options) + return reused + } + + private func storeIncrementalScanInMemory(_ scan: StorageScan, options: ScanOptions) { + guard let incrementalPersistence, + let volumeIdentity = IncrementalVolumeIdentity.read(from: scan.rootURL), + let checkpoint = incrementalPersistence.loadCheckpoint(rootURL: scan.rootURL), + let cacheIdentity = incrementalPersistence.cacheIdentity(rootURL: scan.rootURL) else { + return + } + IncrementalScanMemoryCache.shared.store( + scan: scan, + checkpoint: checkpoint, + volumeIdentity: volumeIdentity, + options: IncrementalScanOptionsFingerprint(options), + cacheIdentity: cacheIdentity ) } + private func schedulePersistedTrustVerification(rootURL: URL, options: ScanOptions) { + guard incrementalTrustButVerify, + let incrementalPersistence else { return } + DispatchQueue.global(qos: .utility).async { + guard let expected = try? incrementalPersistence.load(rootURL: rootURL) else { + incrementalPersistence.invalidate(rootURL: rootURL) + return + } + self.scheduleIncrementalTrustVerification( + expected: expected, + rootURL: rootURL, + options: options + ) + } + } + /// Hashes every item in `group` and returns verified duplicate groups (matching SHA-256, /// count > 1). Use after a scan to verify a same-size candidate group that fell outside /// the auto-verification budget. Hits the persisted `hashCache` when items are unchanged, @@ -495,8 +636,13 @@ public final class FileSystemScanner { options: ScanOptions, cancellation: ScanCancellation?, accumulator: ScanAccumulator - ) throws -> StorageItem { - let prepared: (rootSummary: FixedWorkerItemSummary?, summaryStore: FixedWorkerSummaryStore?) + ) throws -> ( + rootItem: StorageItem, + rescanKind: StorageScan.RescanKind, + dirtySubtreeCount: Int, + fallbackReason: String? + ) { + let prepared: FixedWorkerPreparedSummary do { prepared = try prepareFixedWorkerSummary( at: rootURL, @@ -511,18 +657,19 @@ public final class FileSystemScanner { // record set is available. If a worker-only failure escapes the // filesystem error handling below, retry the root through the legacy // implementation rather than returning a partial tree. - return try scanItem( + let rootItem = try scanItem( at: rootURL, options: options, cancellation: cancellation, accumulator: accumulator, depth: 0 ) + return (rootItem, .full, 0, "fixed-worker-failed") } guard let rootSummary = prepared.rootSummary, let summaryStore = prepared.summaryStore else { - return StorageItem( + let rootItem = StorageItem( url: rootURL, kind: .other, byteSize: 0, @@ -533,9 +680,15 @@ public final class FileSystemScanner { isReadable: false, fileExtension: nil ) + return (rootItem, prepared.rescanKind, prepared.dirtySubtreeCount, prepared.fallbackReason) } - return summaryStore.makeItem(summary: rootSummary, urlOverride: rootURL) + return ( + summaryStore.makeItem(summary: rootSummary, urlOverride: rootURL), + prepared.rescanKind, + prepared.dirtySubtreeCount, + prepared.fallbackReason + ) } private func prepareFixedWorkerSummary( @@ -543,22 +696,21 @@ public final class FileSystemScanner { options: ScanOptions, cancellation: ScanCancellation?, accumulator: ScanAccumulator - ) throws -> ( - rootSummary: FixedWorkerItemSummary?, - summaryStore: FixedWorkerSummaryStore? - ) { - let walker = FixedWorkerDirectoryWalker( - fileManager: fileManager, - resourceKeys: resourceKeys, + ) throws -> FixedWorkerPreparedSummary { + let preparedWalk = try prepareFixedWorkerWalk( + at: rootURL, options: options, - cancellation: cancellation, - shouldExclude: { url in - options.excludeEnabled && Self.isExcluded(url, options: options) - } + cancellation: cancellation ) - let result = try walker.walk(root: rootURL) + let result = preparedWalk.result guard let rootMetadata = result.rootMetadata else { - return (nil, nil) + return FixedWorkerPreparedSummary( + rootSummary: nil, + summaryStore: nil, + rescanKind: preparedWalk.rescanKind, + dirtySubtreeCount: preparedWalk.dirtySubtreeCount, + fallbackReason: preparedWalk.fallbackReason + ) } let (rootSummary, summaryStore) = try buildFixedWorkerSummaryStore( @@ -569,7 +721,277 @@ public final class FileSystemScanner { accumulator: accumulator, rootURL: rootURL ) - return (rootSummary, summaryStore) + return FixedWorkerPreparedSummary( + rootSummary: rootSummary, + summaryStore: summaryStore, + rescanKind: preparedWalk.rescanKind, + dirtySubtreeCount: preparedWalk.dirtySubtreeCount, + fallbackReason: preparedWalk.fallbackReason + ) + } + + private func prepareFixedWorkerWalk( + at rootURL: URL, + options: ScanOptions, + cancellation: ScanCancellation? + ) throws -> FixedWorkerPreparedWalk { + let walker = makeFixedWorkerWalker(rootOptions: options, cancellation: cancellation) + guard let incrementalPersistence, + let incrementalChangeSource else { + return FixedWorkerPreparedWalk( + result: try walker.walk(root: rootURL), + rescanKind: .full, + dirtySubtreeCount: 0, + fallbackReason: nil + ) + } + + let checkpointBeforeWalk = incrementalChangeSource.currentEventID() + if incrementalChangeSource is SystemFSEventsChangeSource { + LiveFSEventsMonitorRegistry.shared.ensure( + rootURL: rootURL, + since: checkpointBeforeWalk + ) + } + let standardizedRootPath = rootURL.standardizedFileURL.path + let fingerprint = IncrementalScanOptionsFingerprint(options) + let volumeIdentity = IncrementalVolumeIdentity.read(from: rootURL) + var fallbackReason: String? + + do { + var tree = try incrementalPersistence.load(rootURL: rootURL) + guard tree.schemaVersion == PersistedWalkTree.schemaVersion else { + throw IncrementalScanFallback.schemaIncompatible + } + guard tree.standardizedRootPath == standardizedRootPath else { + throw IncrementalScanFallback.rootChanged + } + guard let volumeIdentity, tree.volumeIdentity == volumeIdentity else { + throw IncrementalScanFallback.volumeChanged + } + guard tree.options == fingerprint else { + throw IncrementalScanFallback.optionsChanged + } + guard tree.checkpoint <= checkpointBeforeWalk else { + throw IncrementalScanFallback.eventHistoryUnavailable + } + let changes = tree.checkpoint == checkpointBeforeWalk + ? IncrementalChangeSet(paths: [], requiresFullScan: false, reason: nil) + : incrementalChangeSource.changes(rootURL: rootURL, since: tree.checkpoint) + guard !changes.requiresFullScan else { + fallbackReason = changes.reason ?? IncrementalScanFallback.eventHistoryUnavailable.rawValue + throw IncrementalScanFallback.eventHistoryUnavailable + } + let dirtySubtrees = try incrementalDirtySubtrees( + eventPaths: changes.paths, + rootURL: rootURL, + fileManager: fileManager + ) + guard dirtySubtrees.count <= 128 else { + throw IncrementalScanFallback.tooManyDirtySubtrees + } + + if dirtySubtrees.isEmpty { + tree.checkpoint = checkpointBeforeWalk + try incrementalPersistence.saveCheckpoint(checkpointBeforeWalk, rootURL: rootURL) + let result = try tree.makeWalkResult() + scheduleIncrementalTrustVerification( + expected: tree, + rootURL: rootURL, + options: options + ) + return FixedWorkerPreparedWalk( + result: result, + rescanKind: .incrementalUnchanged, + dirtySubtreeCount: 0, + fallbackReason: nil + ) + } + + for relativePath in dirtySubtrees { + try cancellation?.check() + let subtreeURL = relativePath.isEmpty + ? rootURL + : rootURL.appendingPathComponent(relativePath, isDirectory: true) + var isDirectory: ObjCBool = false + if fileManager.fileExists(atPath: subtreeURL.path, isDirectory: &isDirectory), isDirectory.boolValue { + let subtreeResult = try walker.walk(root: subtreeURL) + let replacement = try PersistedWalkTree( + result: subtreeResult, + rootURL: subtreeURL, + volumeIdentity: volumeIdentity, + options: fingerprint, + checkpoint: checkpointBeforeWalk + ) + try tree.replaceSubtree(at: relativePath, with: replacement) + } else { + try tree.replaceSubtree(at: relativePath, with: nil) + } + } + tree.checkpoint = checkpointBeforeWalk + try incrementalPersistence.save(tree, rootURL: rootURL) + if incrementalChangeSource is SystemFSEventsChangeSource { + LiveFSEventsMonitorRegistry.shared.restart( + rootURL: rootURL, + since: checkpointBeforeWalk + ) + } + let result = try tree.makeWalkResult() + scheduleIncrementalTrustVerification( + expected: tree, + rootURL: rootURL, + options: options + ) + return FixedWorkerPreparedWalk( + result: result, + rescanKind: .incrementalChanged, + dirtySubtreeCount: dirtySubtrees.count, + fallbackReason: nil + ) + } catch let fallback as IncrementalScanFallback { + fallbackReason = fallbackReason ?? fallback.rawValue + } catch FileSystemScannerError.cancelled { + throw FileSystemScannerError.cancelled + } catch { + fallbackReason = IncrementalScanFallback.persistenceCorrupt.rawValue + } + + if incrementalChangeSource is SystemFSEventsChangeSource { + LiveFSEventsMonitorRegistry.shared.restart( + rootURL: rootURL, + since: checkpointBeforeWalk + ) + } + let fullResult = try walker.walk(root: rootURL) + if let volumeIdentity { + do { + let tree = try PersistedWalkTree( + result: fullResult, + rootURL: rootURL, + volumeIdentity: volumeIdentity, + options: fingerprint, + checkpoint: checkpointBeforeWalk + ) + try incrementalPersistence.save(tree, rootURL: rootURL) + } catch { + fallbackReason = fallbackReason ?? "persistence-write-failed" + } + } else { + fallbackReason = fallbackReason ?? IncrementalScanFallback.volumeChanged.rawValue + } + return FixedWorkerPreparedWalk( + result: fullResult, + rescanKind: .full, + dirtySubtreeCount: 0, + fallbackReason: fallbackReason + ) + } + + private func makeFixedWorkerWalker( + rootOptions options: ScanOptions, + cancellation: ScanCancellation? + ) -> FixedWorkerDirectoryWalker { + FixedWorkerDirectoryWalker( + fileManager: fileManager, + resourceKeys: resourceKeys, + options: options, + cancellation: cancellation, + shouldExclude: { url in + options.excludeEnabled && Self.isExcluded(url, options: options) + } + ) + } + + private func incrementalDirtySubtrees( + eventPaths: [String], + rootURL: URL, + fileManager: FileManager + ) throws -> [String] { + let rootPath = IncrementalPathIdentity.canonicalPath(rootURL.path, fileManager: fileManager) + var candidates: Set = [] + for rawPath in eventPaths { + let path = IncrementalPathIdentity.canonicalPath(rawPath, fileManager: fileManager) + guard path == rootPath || path.hasPrefix(rootPath + "/") else { + continue + } + if path == rootPath { + candidates.insert("") + continue + } + var isDirectory: ObjCBool = false + let exists = fileManager.fileExists(atPath: path, isDirectory: &isDirectory) + let dirtyPath = exists && isDirectory.boolValue + ? path + : URL(fileURLWithPath: path).deletingLastPathComponent().path + if dirtyPath == rootPath { + candidates.insert("") + } else if dirtyPath.hasPrefix(rootPath + "/") { + candidates.insert(String(dirtyPath.dropFirst(rootPath.count + 1))) + } else { + throw IncrementalScanFallback.stateInconsistent + } + } + + let ordered = candidates.sorted { lhs, rhs in + let lhsDepth = lhs.isEmpty ? 0 : lhs.split(separator: "/").count + let rhsDepth = rhs.isEmpty ? 0 : rhs.split(separator: "/").count + return lhsDepth == rhsDepth ? lhs < rhs : lhsDepth < rhsDepth + } + var coalesced: [String] = [] + for candidate in ordered { + if coalesced.contains(where: { ancestor in + ancestor.isEmpty || candidate == ancestor || candidate.hasPrefix(ancestor + "/") + }) { + continue + } + coalesced.append(candidate) + } + return coalesced + } + + private func scheduleIncrementalTrustVerification( + expected: PersistedWalkTree, + rootURL: URL, + options: ScanOptions + ) { + guard incrementalTrustButVerify, + let volumeIdentity = IncrementalVolumeIdentity.read(from: rootURL), + let incrementalPersistence, + IncrementalTrustVerificationRegistry.shared.begin(rootURL: rootURL) else { + return + } + let fileManager = self.fileManager + let resourceKeys = self.resourceKeys + DispatchQueue.global(qos: .utility).async { + defer { IncrementalTrustVerificationRegistry.shared.finish(rootURL: rootURL) } + let walker = FixedWorkerDirectoryWalker( + fileManager: fileManager, + resourceKeys: resourceKeys, + options: options, + cancellation: nil, + shouldExclude: { url in + options.excludeEnabled && Self.isExcluded(url, options: options) + } + ) + do { + let fullResult = try walker.walk(root: rootURL) + let fullTree = try PersistedWalkTree( + result: fullResult, + rootURL: rootURL, + volumeIdentity: volumeIdentity, + options: IncrementalScanOptionsFingerprint(options), + checkpoint: expected.checkpoint + ) + if fullTree.comparisonSignature != expected.comparisonSignature { + incrementalPersistence.invalidate(rootURL: rootURL) + os_log("incremental trust verification diverged; next scan must use full fallback", + log: Self.log, type: .fault) + } + } catch { + os_log("incremental trust verification failed: %{public}@", + log: Self.log, type: .error, String(describing: error)) + } + } } private func buildFixedWorkerSummaryStore( @@ -1113,7 +1535,7 @@ private extension StorageItem.Kind { } } -private struct FixedWorkerResourceIdentifier: Hashable, Sendable { +struct FixedWorkerResourceIdentifier: Hashable, Sendable { let low: UInt64 let high: UInt64 @@ -1132,7 +1554,7 @@ private struct FixedWorkerMetadataSeed: Sendable { let hardLinkCount: UInt16 } -private struct FixedWorkerWalkRecord: Sendable { +struct FixedWorkerWalkRecord: Sendable { let id: Int let parentID: Int let name: String @@ -1156,17 +1578,32 @@ private struct FixedWorkerDirectoryJob: Sendable { let url: URL } -private struct FixedWorkerDirectoryRecord: Sendable { +struct FixedWorkerDirectoryRecord: Sendable { let metadata: FixedWorkerWalkRecord let children: [FixedWorkerWalkRecord] let isInaccessible: Bool } -private struct FixedWorkerWalkResult: Sendable { +struct FixedWorkerWalkResult: Sendable { let rootMetadata: FixedWorkerWalkRecord? let directoryRecords: [FixedWorkerDirectoryRecord] } +private struct FixedWorkerPreparedWalk: Sendable { + let result: FixedWorkerWalkResult + let rescanKind: StorageScan.RescanKind + let dirtySubtreeCount: Int + let fallbackReason: String? +} + +private struct FixedWorkerPreparedSummary { + let rootSummary: FixedWorkerItemSummary? + let summaryStore: FixedWorkerSummaryStore? + let rescanKind: StorageScan.RescanKind + let dirtySubtreeCount: Int + let fallbackReason: String? +} + private struct FixedWorkerCompactChild: Sendable { let id: Int let displaySize: Int64 diff --git a/Sources/StorageScopeCore/Services/IncrementalScanPersistence.swift b/Sources/StorageScopeCore/Services/IncrementalScanPersistence.swift new file mode 100644 index 0000000..b497227 --- /dev/null +++ b/Sources/StorageScopeCore/Services/IncrementalScanPersistence.swift @@ -0,0 +1,821 @@ +import CoreServices +import CryptoKit +import Darwin +import Foundation + +struct IncrementalChangeSet: Sendable { + let paths: [String] + let requiresFullScan: Bool + let reason: String? +} + +protocol IncrementalChangeSource: Sendable { + func currentEventID() -> UInt64 + func changes(rootURL: URL, since eventID: UInt64) -> IncrementalChangeSet +} + +final class SystemFSEventsChangeSource: IncrementalChangeSource, @unchecked Sendable { + private final class Collector { + let lock = NSLock() + var paths: [String] = [] + var requiresFullScan = false + var reason: String? + + func append(paths newPaths: [String], flags: UnsafePointer, count: Int) { + lock.lock() + defer { lock.unlock() } + for index in 0.. UInt64 { + FSEventsGetCurrentEventId() + } + + func changes(rootURL: URL, since eventID: UInt64) -> IncrementalChangeSet { + let collector = Collector() + var context = FSEventStreamContext( + version: 0, + info: Unmanaged.passUnretained(collector).toOpaque(), + retain: nil, + release: nil, + copyDescription: nil + ) + let callback: FSEventStreamCallback = { _, info, count, rawPaths, flags, _ in + guard let info else { return } + let collector = Unmanaged.fromOpaque(info).takeUnretainedValue() + let paths = unsafeBitCast(rawPaths, to: CFArray.self) as? [String] ?? [] + collector.append(paths: paths, flags: flags, count: count) + } + let createFlags = FSEventStreamCreateFlags( + kFSEventStreamCreateFlagUseCFTypes | + kFSEventStreamCreateFlagWatchRoot | + kFSEventStreamCreateFlagFileEvents | + kFSEventStreamCreateFlagNoDefer + ) + guard let stream = FSEventStreamCreate( + nil, + callback, + &context, + [rootURL.path] as CFArray, + eventID, + 0, + createFlags + ) else { + return IncrementalChangeSet(paths: [], requiresFullScan: true, reason: "event-stream-unavailable") + } + + let queue = DispatchQueue(label: "com.rasputinkaiser.StorageScope.incremental-fsevents") + FSEventStreamSetDispatchQueue(stream, queue) + guard FSEventStreamStart(stream) else { + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + return IncrementalChangeSet(paths: [], requiresFullScan: true, reason: "event-stream-start-failed") + } + FSEventStreamFlushSync(stream) + FSEventStreamStop(stream) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + + collector.lock.lock() + defer { collector.lock.unlock() } + return IncrementalChangeSet( + paths: collector.paths, + requiresFullScan: collector.requiresFullScan, + reason: collector.reason + ) + } +} + +final class LiveFSEventsMonitor: @unchecked Sendable { + private final class State { + let lock = NSLock() + var paths: [String] = [] + var requiresFullScan = false + var reason: String? + // Every monitor in the registry starts from an event ID captured immediately + // before construction, so there is no older range to replay before it is safe + // to inspect newly delivered events. + var historyReady = true + + func append(paths newPaths: [String], flags: UnsafePointer, count: Int) { + lock.lock() + defer { lock.unlock() } + for index in 0.. IncrementalChangeSet? { + lock.lock() + defer { lock.unlock() } + guard historyReady else { return nil } + let result = IncrementalChangeSet( + paths: paths, + requiresFullScan: requiresFullScan, + reason: reason + ) + paths.removeAll(keepingCapacity: true) + requiresFullScan = false + reason = nil + return result + } + } + + private let state: State + private let stream: FSEventStreamRef + + init?(rootURL: URL, since eventID: UInt64) { + let state = State() + self.state = state + var context = FSEventStreamContext( + version: 0, + info: Unmanaged.passUnretained(state).toOpaque(), + retain: nil, + release: nil, + copyDescription: nil + ) + let callback: FSEventStreamCallback = { _, info, count, rawPaths, flags, _ in + guard let info else { return } + let state = Unmanaged.fromOpaque(info).takeUnretainedValue() + let paths = unsafeBitCast(rawPaths, to: CFArray.self) as? [String] ?? [] + state.append(paths: paths, flags: flags, count: count) + } + let createFlags = FSEventStreamCreateFlags( + kFSEventStreamCreateFlagUseCFTypes | + kFSEventStreamCreateFlagWatchRoot | + kFSEventStreamCreateFlagFileEvents | + kFSEventStreamCreateFlagNoDefer + ) + guard let stream = FSEventStreamCreate( + nil, + callback, + &context, + [rootURL.path] as CFArray, + eventID, + 0.05, + createFlags + ) else { return nil } + self.stream = stream + FSEventStreamSetDispatchQueue(stream, DispatchQueue( + label: "com.rasputinkaiser.StorageScope.incremental-live-fsevents" + )) + guard FSEventStreamStart(stream) else { + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + return nil + } + } + + deinit { + FSEventStreamStop(stream) + FSEventStreamInvalidate(stream) + FSEventStreamRelease(stream) + } + + func flushAndDrainIfReady() -> IncrementalChangeSet? { + Thread.sleep(forTimeInterval: 0.06) + FSEventStreamFlushSync(stream) + return state.drainIfReady() + } +} + +final class LiveFSEventsMonitorRegistry: @unchecked Sendable { + static let shared = LiveFSEventsMonitorRegistry() + private let lock = NSLock() + private var monitors: [String: LiveFSEventsMonitor] = [:] + private var recency: [String] = [] + + func ensure(rootURL: URL, since eventID: UInt64) { + let key = IncrementalPathIdentity.canonicalPath(rootURL.path) + lock.lock() + if monitors[key] != nil { + lock.unlock() + return + } + lock.unlock() + restart(rootURL: rootURL, since: eventID) + } + + func restart(rootURL: URL, since eventID: UInt64) { + let key = IncrementalPathIdentity.canonicalPath(rootURL.path) + guard let monitor = LiveFSEventsMonitor(rootURL: rootURL, since: eventID) else { return } + lock.lock() + monitors[key] = monitor + recency.removeAll { $0 == key } + recency.append(key) + while recency.count > 4 { + monitors.removeValue(forKey: recency.removeFirst()) + } + lock.unlock() + } + + func flushAndDrainIfReady(rootURL: URL) -> IncrementalChangeSet? { + let key = IncrementalPathIdentity.canonicalPath(rootURL.path) + lock.lock() + let monitor = monitors[key] + lock.unlock() + return monitor?.flushAndDrainIfReady() + } +} + +struct IncrementalScanOptionsFingerprint: Codable, Equatable, Hashable, Sendable { + let includeHidden: Bool + let oldFileAgeDays: Int + let largeFileThreshold: Int64 + let duplicateCandidateThreshold: Int64 + let duplicateVerificationByteLimit: Int64 + let maxDuplicateVerificationFiles: Int + let maxDuplicateCandidateItems: Int + let maxRankedResults: Int + let maxChildrenPerDirectory: Int + let maxRetainedItems: Int + let excludeEnabled: Bool + let excludedPathComponents: [String] + let excludedAbsolutePrefixes: [String] + + init(_ options: ScanOptions) { + includeHidden = options.includeHidden + oldFileAgeDays = options.oldFileAgeDays + largeFileThreshold = options.largeFileThreshold + duplicateCandidateThreshold = options.duplicateCandidateThreshold + duplicateVerificationByteLimit = options.duplicateVerificationByteLimit + maxDuplicateVerificationFiles = options.maxDuplicateVerificationFiles + maxDuplicateCandidateItems = options.maxDuplicateCandidateItems + maxRankedResults = options.maxRankedResults + maxChildrenPerDirectory = options.maxChildrenPerDirectory + maxRetainedItems = options.maxRetainedItems + excludeEnabled = options.excludeEnabled + excludedPathComponents = options.excludedPathComponents.sorted() + excludedAbsolutePrefixes = options.excludedAbsolutePrefixes.sorted() + } +} + +struct PersistedWalkNode: Codable, Sendable { + let id: Int + let parentID: Int + let name: String + let kind: String + let byteSize: Int64 + let allocatedSize: Int64 + let modifiedAt: Date? + let isReadable: Bool + let volumeIdentifierLow: UInt64 + let volumeIdentifierHigh: UInt64 + let fileResourceIdentifierLow: UInt64 + let fileResourceIdentifierHigh: UInt64 + let hardLinkCount: UInt16 + let isInaccessible: Bool +} + +struct PersistedWalkTree: Codable, Sendable { + static let schemaVersion = 2 + + let schemaVersion: Int + let standardizedRootPath: String + let volumeIdentity: String + let options: IncrementalScanOptionsFingerprint + var checkpoint: UInt64 + var nodes: [PersistedWalkNode] + + init( + standardizedRootPath: String, + volumeIdentity: String, + options: IncrementalScanOptionsFingerprint, + checkpoint: UInt64, + nodes: [PersistedWalkNode] + ) { + schemaVersion = Self.schemaVersion + self.standardizedRootPath = standardizedRootPath + self.volumeIdentity = volumeIdentity + self.options = options + self.checkpoint = checkpoint + self.nodes = nodes + } + + init( + result: FixedWorkerWalkResult, + rootURL: URL, + volumeIdentity: String, + options: IncrementalScanOptionsFingerprint, + checkpoint: UInt64 + ) throws { + guard let rootMetadata = result.rootMetadata else { + throw IncrementalScanFallback.stateInconsistent + } + var inaccessibleIDs: Set = [] + var metadataByID: [Int: FixedWorkerWalkRecord] = [rootMetadata.id: rootMetadata] + for record in result.directoryRecords { + if record.isInaccessible { + inaccessibleIDs.insert(record.metadata.id) + } + for child in record.children { + guard metadataByID.updateValue(child, forKey: child.id) == nil else { + throw IncrementalScanFallback.stateInconsistent + } + } + } + let sortedMetadata = metadataByID.values.sorted { $0.id < $1.id } + guard sortedMetadata.first?.id == 0, + sortedMetadata.indices.allSatisfy({ sortedMetadata[$0].id == $0 }) else { + throw IncrementalScanFallback.stateInconsistent + } + var persisted: [PersistedWalkNode] = [] + persisted.reserveCapacity(sortedMetadata.count) + for metadata in sortedMetadata { + guard metadata.id == 0 || metadata.parentID < metadata.id else { + throw IncrementalScanFallback.stateInconsistent + } + persisted.append(PersistedWalkNode( + id: metadata.id, + parentID: metadata.parentID, + name: metadata.name, + kind: metadata.kind.rawValue, + byteSize: metadata.byteSize, + allocatedSize: metadata.allocatedSize, + modifiedAt: metadata.modifiedAt, + isReadable: metadata.isReadable, + volumeIdentifierLow: metadata.volumeIdentifier.low, + volumeIdentifierHigh: metadata.volumeIdentifier.high, + fileResourceIdentifierLow: metadata.fileResourceIdentifier.low, + fileResourceIdentifierHigh: metadata.fileResourceIdentifier.high, + hardLinkCount: metadata.hardLinkCount, + isInaccessible: inaccessibleIDs.contains(metadata.id) + )) + } + let expectedNodeCount = result.directoryRecords.reduce(1) { count, record in + count + record.children.count + } + guard persisted.count == expectedNodeCount else { + throw IncrementalScanFallback.stateInconsistent + } + + self.init( + standardizedRootPath: rootURL.standardizedFileURL.path, + volumeIdentity: volumeIdentity, + options: options, + checkpoint: checkpoint, + nodes: persisted + ) + } + + func makeWalkResult() throws -> FixedWorkerWalkResult { + guard let rootNode = nodes.first, + rootNode.id == 0, + rootNode.parentID == 0, + nodes.indices.allSatisfy({ nodes[$0].id == $0 }) else { + throw IncrementalScanFallback.stateInconsistent + } + func metadata(for node: PersistedWalkNode) throws -> FixedWorkerWalkRecord { + guard let kind = StorageItem.Kind(rawValue: node.kind), + node.id == 0 || (node.parentID >= 0 && node.parentID < node.id) else { + throw IncrementalScanFallback.stateInconsistent + } + return FixedWorkerWalkRecord( + id: node.id, + parentID: node.parentID, + name: node.name, + kind: kind, + byteSize: node.byteSize, + allocatedSize: node.allocatedSize, + modifiedAt: node.modifiedAt, + isReadable: node.isReadable, + volumeIdentifier: FixedWorkerResourceIdentifier( + low: node.volumeIdentifierLow, + high: node.volumeIdentifierHigh + ), + fileResourceIdentifier: FixedWorkerResourceIdentifier( + low: node.fileResourceIdentifierLow, + high: node.fileResourceIdentifierHigh + ), + hardLinkCount: node.hardLinkCount + ) + } + + var childrenByParent: [[FixedWorkerWalkRecord]] = Array(repeating: [], count: nodes.count) + for node in nodes.dropFirst() { + guard node.parentID >= 0, node.parentID < nodes.count else { + throw IncrementalScanFallback.stateInconsistent + } + childrenByParent[node.parentID].append(try metadata(for: node)) + } + var directoryRecords: [FixedWorkerDirectoryRecord] = [] + directoryRecords.reserveCapacity(nodes.count / 4) + for node in nodes { + guard let kind = StorageItem.Kind(rawValue: node.kind) else { + throw IncrementalScanFallback.stateInconsistent + } + let isDirectory = kind == .folder || kind == .package + if !isDirectory { + guard childrenByParent[node.id].isEmpty else { + throw IncrementalScanFallback.stateInconsistent + } + continue + } + directoryRecords.append(FixedWorkerDirectoryRecord( + metadata: try metadata(for: node), + children: childrenByParent[node.id].sorted { $0.name < $1.name }, + isInaccessible: node.isInaccessible + )) + } + return FixedWorkerWalkResult( + rootMetadata: try metadata(for: rootNode), + directoryRecords: directoryRecords + ) + } + + mutating func replaceSubtree( + at relativePath: String, + with replacement: PersistedWalkTree? + ) throws { + _ = try makeWalkResult() + if relativePath.isEmpty { + guard let replacement else { + throw IncrementalScanFallback.stateInconsistent + } + nodes = replacement.nodes + return + } + + var childIDs: [[Int]] = Array(repeating: [], count: nodes.count) + for node in nodes.dropFirst() { + childIDs[node.parentID].append(node.id) + } + let components = relativePath.split(separator: "/").map(String.init) + guard let targetName = components.last else { + throw IncrementalScanFallback.stateInconsistent + } + var parentID = 0 + for component in components.dropLast() { + guard let childID = childIDs[parentID].first(where: { nodes[$0].name == component }) else { + throw IncrementalScanFallback.stateInconsistent + } + parentID = childID + } + let targetID = childIDs[parentID].first(where: { nodes[$0].name == targetName }) + var removed: Set = [] + if let targetID { + var pending = [targetID] + while let id = pending.popLast() { + guard removed.insert(id).inserted else { continue } + pending.append(contentsOf: childIDs[id]) + } + } + + let survivors = nodes.filter { !removed.contains($0.id) } + var compacted: [PersistedWalkNode] = [] + compacted.reserveCapacity(survivors.count + (replacement?.nodes.count ?? 0)) + var newIDByOldID: [Int: Int] = [:] + for node in survivors { + let newID = compacted.count + let newParentID = node.id == 0 ? 0 : try Self.requiredID(newIDByOldID[node.parentID]) + newIDByOldID[node.id] = newID + compacted.append(node.reidentified(id: newID, parentID: newParentID)) + } + guard let compactedParentID = newIDByOldID[parentID] else { + throw IncrementalScanFallback.stateInconsistent + } + if let replacement { + var replacementIDMap: [Int: Int] = [:] + for node in replacement.nodes { + let newID = compacted.count + let newParentID = node.id == 0 + ? compactedParentID + : try Self.requiredID(replacementIDMap[node.parentID]) + replacementIDMap[node.id] = newID + compacted.append(node.reidentified(id: newID, parentID: newParentID)) + } + } + nodes = compacted + _ = try makeWalkResult() + } + + var comparisonSignature: [String] { + guard !nodes.isEmpty else { return [] } + var childIDs: [[Int]] = Array(repeating: [], count: nodes.count) + for node in nodes.dropFirst() where node.parentID < nodes.count { + childIDs[node.parentID].append(node.id) + } + var signatures: [String] = [] + var pending: [(Int, String)] = [(0, "")] + while let (id, path) = pending.popLast() { + let node = nodes[id] + let children = childIDs[id].sorted { nodes[$0].name < nodes[$1].name } + signatures.append([ + path, node.kind, String(node.byteSize), String(node.allocatedSize), + String(node.isReadable), String(node.isInaccessible), + node.modifiedAt.map { String($0.timeIntervalSinceReferenceDate.bitPattern) } ?? "nil", + String(node.volumeIdentifierLow), String(node.volumeIdentifierHigh), + String(node.fileResourceIdentifierLow), String(node.fileResourceIdentifierHigh), + String(node.hardLinkCount), + children.map { nodes[$0].name }.joined(separator: ",") + ].joined(separator: "|")) + for childID in children.reversed() { + let childPath = path.isEmpty ? nodes[childID].name : path + "/" + nodes[childID].name + pending.append((childID, childPath)) + } + } + return signatures.sorted() + } + + private static func requiredID(_ value: Int?) throws -> Int { + guard let value else { throw IncrementalScanFallback.stateInconsistent } + return value + } +} + +private extension PersistedWalkNode { + func reidentified(id: Int, parentID: Int) -> PersistedWalkNode { + PersistedWalkNode( + id: id, + parentID: parentID, + name: name, + kind: kind, + byteSize: byteSize, + allocatedSize: allocatedSize, + modifiedAt: modifiedAt, + isReadable: isReadable, + volumeIdentifierLow: volumeIdentifierLow, + volumeIdentifierHigh: volumeIdentifierHigh, + fileResourceIdentifierLow: fileResourceIdentifierLow, + fileResourceIdentifierHigh: fileResourceIdentifierHigh, + hardLinkCount: hardLinkCount, + isInaccessible: isInaccessible + ) + } +} + +enum IncrementalScanFallback: String, Error, Sendable { + case persistenceMissing = "persistence-missing" + case persistenceCorrupt = "persistence-corrupt" + case schemaIncompatible = "schema-incompatible" + case rootChanged = "root-changed" + case volumeChanged = "volume-changed" + case optionsChanged = "traversal-options-changed" + case eventHistoryUnavailable = "event-history-unavailable" + case stateInconsistent = "parent-child-state-inconsistent" + case tooManyDirtySubtrees = "too-many-dirty-subtrees" +} + +final class IncrementalScanPersistence: @unchecked Sendable { + struct CacheIdentity: Equatable, Sendable { + let size: UInt64 + let modifiedAt: Date + } + + private let baseURL: URL + private let fileManager: FileManager + private let encoder: PropertyListEncoder + private let decoder: PropertyListDecoder + + init(baseURL: URL, fileManager: FileManager = .default) { + self.baseURL = baseURL + self.fileManager = fileManager + encoder = PropertyListEncoder() + encoder.outputFormat = .binary + decoder = PropertyListDecoder() + } + + static func defaultBaseURL(fileManager: FileManager = .default) -> URL { + let cache = fileManager.urls(for: .cachesDirectory, in: .userDomainMask).first + ?? fileManager.temporaryDirectory + return cache.appendingPathComponent("StorageScope/IncrementalScans", isDirectory: true) + } + + func load(rootURL: URL) throws -> PersistedWalkTree { + let url = cacheURL(rootURL: rootURL) + guard fileManager.fileExists(atPath: url.path) else { + throw IncrementalScanFallback.persistenceMissing + } + do { + var tree = try decoder.decode(PersistedWalkTree.self, from: Data(contentsOf: url)) + guard tree.schemaVersion == PersistedWalkTree.schemaVersion else { + throw IncrementalScanFallback.schemaIncompatible + } + let checkpointURL = checkpointURL(rootURL: rootURL) + if fileManager.fileExists(atPath: checkpointURL.path) { + guard let rawCheckpoint = String(data: try Data(contentsOf: checkpointURL), encoding: .utf8), + let checkpoint = UInt64(rawCheckpoint) else { + throw IncrementalScanFallback.persistenceCorrupt + } + tree.checkpoint = checkpoint + } + return tree + } catch let fallback as IncrementalScanFallback { + throw fallback + } catch { + throw IncrementalScanFallback.persistenceCorrupt + } + } + + func save(_ tree: PersistedWalkTree, rootURL: URL) throws { + try fileManager.createDirectory(at: baseURL, withIntermediateDirectories: true) + try encoder.encode(tree).write(to: cacheURL(rootURL: rootURL), options: .atomic) + try saveCheckpoint(tree.checkpoint, rootURL: rootURL) + } + + func saveCheckpoint(_ checkpoint: UInt64, rootURL: URL) throws { + try fileManager.createDirectory(at: baseURL, withIntermediateDirectories: true) + try Data(String(checkpoint).utf8).write(to: checkpointURL(rootURL: rootURL), options: .atomic) + } + + func loadCheckpoint(rootURL: URL) -> UInt64? { + guard let data = try? Data(contentsOf: checkpointURL(rootURL: rootURL)), + let raw = String(data: data, encoding: .utf8) else { + return nil + } + return UInt64(raw) + } + + func cacheIdentity(rootURL: URL) -> CacheIdentity? { + guard let attributes = try? fileManager.attributesOfItem(atPath: cacheURL(rootURL: rootURL).path), + let size = attributes[.size] as? NSNumber, + let modifiedAt = attributes[.modificationDate] as? Date else { + return nil + } + return CacheIdentity(size: size.uint64Value, modifiedAt: modifiedAt) + } + + func cacheURL(rootURL: URL) -> URL { + let path = rootURL.standardizedFileURL.path + let digest = SHA256.hash(data: Data(path.utf8)).map { String(format: "%02x", $0) }.joined() + return baseURL.appendingPathComponent("\(digest).plist") + } + + func invalidate(rootURL: URL) { + try? fileManager.removeItem(at: cacheURL(rootURL: rootURL)) + try? fileManager.removeItem(at: checkpointURL(rootURL: rootURL)) + } + + private func checkpointURL(rootURL: URL) -> URL { + cacheURL(rootURL: rootURL).appendingPathExtension("checkpoint") + } +} + +final class IncrementalScanMemoryCache: @unchecked Sendable { + private struct Key: Hashable { + let rootPath: String + let volumeIdentity: String + let options: IncrementalScanOptionsFingerprint + } + + struct Entry: Sendable { + let scan: StorageScan + let checkpoint: UInt64 + let cacheIdentity: IncrementalScanPersistence.CacheIdentity + } + + static let shared = IncrementalScanMemoryCache() + private let lock = NSLock() + private var entries: [Key: Entry] = [:] + private var recency: [Key] = [] + + func entry( + rootURL: URL, + volumeIdentity: String, + options: IncrementalScanOptionsFingerprint, + cacheIdentity: IncrementalScanPersistence.CacheIdentity? + ) -> Entry? { + guard let cacheIdentity else { return nil } + let key = Self.key(rootURL: rootURL, volumeIdentity: volumeIdentity, options: options) + lock.lock() + defer { lock.unlock() } + guard let entry = entries[key], entry.cacheIdentity == cacheIdentity else { + entries.removeValue(forKey: key) + recency.removeAll { $0 == key } + return nil + } + recency.removeAll { $0 == key } + recency.append(key) + return entry + } + + func store( + scan: StorageScan, + checkpoint: UInt64, + volumeIdentity: String, + options: IncrementalScanOptionsFingerprint, + cacheIdentity: IncrementalScanPersistence.CacheIdentity + ) { + let key = Self.key(rootURL: scan.rootURL, volumeIdentity: volumeIdentity, options: options) + lock.lock() + entries[key] = Entry(scan: scan, checkpoint: checkpoint, cacheIdentity: cacheIdentity) + recency.removeAll { $0 == key } + recency.append(key) + while recency.count > 4 { + entries.removeValue(forKey: recency.removeFirst()) + } + lock.unlock() + } + + private static func key( + rootURL: URL, + volumeIdentity: String, + options: IncrementalScanOptionsFingerprint + ) -> Key { + Key( + rootPath: rootURL.standardizedFileURL.path, + volumeIdentity: volumeIdentity, + options: options + ) + } +} + +final class IncrementalTrustVerificationRegistry: @unchecked Sendable { + static let shared = IncrementalTrustVerificationRegistry() + private let lock = NSLock() + private var rootsInFlight: Set = [] + + func begin(rootURL: URL) -> Bool { + let key = IncrementalPathIdentity.canonicalPath(rootURL.path) + lock.lock() + defer { lock.unlock() } + return rootsInFlight.insert(key).inserted + } + + func finish(rootURL: URL) { + let key = IncrementalPathIdentity.canonicalPath(rootURL.path) + lock.lock() + rootsInFlight.remove(key) + lock.unlock() + } +} + +enum IncrementalVolumeIdentity { + static func read(from rootURL: URL) -> String? { + let keys: Set = [.volumeUUIDStringKey, .volumeURLKey] + guard let values = try? rootURL.resourceValues(forKeys: keys) else { return nil } + if let uuid = values.volumeUUIDString, !uuid.isEmpty { + return uuid + } + return values.volume?.standardizedFileURL.path + } +} + +enum IncrementalPathIdentity { + static func canonicalPath(_ path: String, fileManager: FileManager = .default) -> String { + var existingPath = URL(fileURLWithPath: path).standardizedFileURL.path + var missingComponents: [String] = [] + while !fileManager.fileExists(atPath: existingPath), existingPath != "/" { + let url = URL(fileURLWithPath: existingPath) + missingComponents.insert(url.lastPathComponent, at: 0) + existingPath = url.deletingLastPathComponent().path + } + + let resolvedBase: String + if let pointer = realpath(existingPath, nil) { + resolvedBase = String(cString: pointer) + free(pointer) + } else { + resolvedBase = existingPath + } + return missingComponents.reduce(resolvedBase) { partial, component in + URL(fileURLWithPath: partial).appendingPathComponent(component).path + } + } +} diff --git a/Sources/StorageScopeCore/Services/ScanBenchmark.swift b/Sources/StorageScopeCore/Services/ScanBenchmark.swift index 23a4091..5a8e1bd 100644 --- a/Sources/StorageScopeCore/Services/ScanBenchmark.swift +++ b/Sources/StorageScopeCore/Services/ScanBenchmark.swift @@ -23,6 +23,9 @@ public struct ScanBenchmarkReport: Hashable, Sendable { public let persistDuration: TimeInterval public let cleanupCandidateCount: Int public let peakMemoryBytes: UInt64? + public let rescanKind: StorageScan.RescanKind + public let incrementalDirtySubtreeCount: Int + public let incrementalFallbackReason: String? /// Sum of the three instrumented phases (enumerate + verify + persist). This typically /// under-shoots `duration` because bookkeeping between phases (cleanup candidate @@ -58,6 +61,9 @@ public struct ScanBenchmarkReport: Hashable, Sendable { self.persistDuration = persistDuration self.cleanupCandidateCount = scan.cleanupCandidates.count self.peakMemoryBytes = peakMemoryBytes + self.rescanKind = scan.rescanKind + self.incrementalDirtySubtreeCount = scan.incrementalDirtySubtreeCount + self.incrementalFallbackReason = scan.incrementalFallbackReason } public var text: String { @@ -84,6 +90,9 @@ public struct ScanBenchmarkReport: Hashable, Sendable { "Phase total (enum+verify+persist): \(Self.seconds(totalDuration))", "Cleanup candidates: \(cleanupCandidateCount.formatted())", "Peak memory: \(peakMemoryBytes.map(Self.bytes) ?? "unavailable")", + "Rescan mode: \(rescanKind.rawValue)", + "Incremental dirty subtrees: \(incrementalDirtySubtreeCount.formatted())", + "Incremental fallback: \(incrementalFallbackReason ?? "none")", "Results are local only." ].joined(separator: "\n") } diff --git a/StorageScope-Performance-Plan.md b/StorageScope-Performance-Plan.md index 84d86d2..86d3c26 100644 --- a/StorageScope-Performance-Plan.md +++ b/StorageScope-Performance-Plan.md @@ -111,6 +111,22 @@ Users re-scan the same folders repeatedly; today every re-scan pays full price. Effort: L (the one new-feature-sized item). Risk: medium — correctness depends on FSEvents semantics; the trust-but-verify rollout and the fallback rules bound it. +**Implemented 2026-07-13.** The app now uses the compact fixed-worker tree as a +binary persisted snapshot and maintains a bounded live FSEvents monitor per recent +root. Re-scans coalesce changed paths to ancestor subtrees, walk only those +directories, splice them into the parent-indexed tree, and rebuild the public scan +result. Missing/corrupt/incompatible state, volume or option changes, dropped or +overflowed event history, inconsistent parent IDs, and excessive dirty-subtree counts +all take a conservative full-walk fallback. A deduplicated background full walk +compares names, kinds, sizes, timestamps, readability, inaccessible state, filesystem +identities, and hard-link metadata; divergence invalidates the cache for the next scan. + +The final 100k kept-fixture release proof measured same-process unchanged re-scans at +0.10 s and 0.07 s with 126.9 MB peak runtime memory. Cold-process persisted-tree +reconstruction took 4.03 s and is not claimed as sub-second. Full commands, scope, +and proof boundaries are recorded in +`docs/perf-baselines/v0.7.1/phase3-incremental.md`. + ### 3.2 Perceived performance during first scan - **Converge big numbers first:** walk the frontier largest-known-first (size from parent enumeration) so top-level folder sizes and the Overview stabilize in the first seconds even if leaves take longer. @@ -282,8 +298,9 @@ three app runs. The opt-in proof is `ScanStoreAppPerformanceProofTests` and requ build plus `STORAGESCOPE_TESTING`, so ordinary release builds retain no debug counters or test drain hooks. Focused differential/fault/pause/cancel coverage passed 10 tests across 3 suites; the full rebased default suite passed 221 tests across 22 suites with the expensive proof skipped by -default. All Phase 2 exit criteria are now satisfied. Phase 3 starts with tiered verification, -hard-link reclaimability, and perceived-progress streaming. +default. All Phase 2 exit criteria are now satisfied. Phase 3 incremental re-scan is +implemented with trust-but-verify; tiered verification, hard-link reclaimability, and +perceived-progress streaming remain follow-up tracks. --- @@ -322,8 +339,8 @@ hard-link reclaimability, and perceived-progress streaming. | 0 | v0.7.2 | §6.1–2 hygiene + counters + re-baseline | Trustworthy baseline incl. profile committed | | 1 | v0.7.3 | §1 eviction fix + lock decomposition + heaps | `__psynch_mutexwait` < 5 % of samples; enumerate ≥ 5× vs Phase-0 baseline | | 2 | v0.8.0 | §2.1 worker-pool walker (flagged), §2.2.1–2 per-item costs, §2.3 app snapshot fix | Enumerate < 10 s; pause/cancel green on both walkers; in-app scan within 15 % of benchmark scan | -| 3 | v0.8.1 | §3.3 tiered hashing; §3.2 perceived-perf streaming | Verification bytes −90 % on duplicates fixture; hard links excluded; Overview stable < 3 s into a 100 k scan | -| 4 | v0.9.0 | §4 walk records; §2.2.3 `getattrlistbulk` (flagged); §3.1 incremental re-scan (trust-but-verify) | RSS < 150 MB @ 100 k; unchanged-tree re-scan < 1 s; bulk backend ≥ 2× with identical trees | +| 3 | v0.8.1 | §3.1 incremental re-scan (trust-but-verify) | Same-process unchanged-tree re-scan < 1 s; RSS < 150 MB @ 100 k; mutation and fallback parity green | +| 4 | v0.9.0 | §4 remaining memory work; §2.2.3 `getattrlistbulk` (flagged); §3.2–3 tiered verification and perceived-progress follow-ups | Verification bytes −90 % on duplicates fixture; hard links excluded; bulk backend ≥ 2× with identical trees | | 5 | v0.9.x | §5 `@Observable` + search keys | SwiftUI update count during live scan −80 %; no dropped frames typing in filter @ 25 k items | **Decision rule** (replacing PLAN §6.3's debug-based thresholds): each phase re-runs fixtures in release, 3 runs, medians, warm *and* cold. A change ships if its target counter improves ≥ 10 % without regressing any other counter > 5 %; otherwise it stays flagged and the hypothesis table is updated with the measured result. Any claim of "X is the bottleneck" must cite a profile, not code reading — this document needed two corrections for exactly that reason. diff --git a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift index e910859..fa4c162 100644 --- a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift +++ b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift @@ -1122,8 +1122,8 @@ struct FileSystemScannerTests { contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeCore/Services/FileSystemScanner.swift"), encoding: .utf8 ) - let recordStart = try #require(source.range(of: "private struct FixedWorkerWalkRecord")) - let recordEnd = try #require(source[recordStart.upperBound...].range(of: "private struct FixedWorkerDirectoryJob")?.lowerBound) + let recordStart = try #require(source.range(of: "struct FixedWorkerWalkRecord")) + let recordEnd = try #require(source[recordStart.upperBound...].range(of: "struct FixedWorkerDirectoryJob")?.lowerBound) let recordSource = String(source[recordStart.lowerBound.. = [] + for index in 0..<1_000 { + let directoryIndex = index % 10 + let directory = fixture.root.appendingPathComponent("mutation-\(directoryIndex)", isDirectory: true) + touchedDirectories.insert(directory.path) + switch index % 4 { + case 0: + try Data(repeating: UInt8(index % 251), count: 64 + index).write( + to: directory.appendingPathComponent("created-\(index).bin") + ) + case 1: + try Data(repeating: UInt8(index % 251), count: 128 + index).write( + to: directory.appendingPathComponent(anchorNames[directoryIndex]) + ) + case 2: + let oldName = anchorNames[directoryIndex] + let newName = oldName == "anchor-a.bin" ? "anchor-b.bin" : "anchor-a.bin" + try FileManager.default.moveItem( + at: directory.appendingPathComponent(oldName), + to: directory.appendingPathComponent(newName) + ) + anchorNames[directoryIndex] = newName + default: + let createdIndex = index - 3 + let createdDirectory = fixture.root.appendingPathComponent( + "mutation-\(createdIndex % 10)", + isDirectory: true + ) + touchedDirectories.insert(createdDirectory.path) + try FileManager.default.removeItem( + at: createdDirectory.appendingPathComponent("created-\(createdIndex).bin") + ) + } + + if (index + 1).isMultiple(of: 25) { + source.set( + eventID: UInt64(1_001 + index), + changes: IncrementalChangeSet( + paths: touchedDirectories.sorted(), + requiresFullScan: false, + reason: nil + ) + ) + let incremental = try scanner.scan(root: fixture.root, options: fixture.options) + let full = try fixture.fullScanner().scan(root: fixture.root, options: fixture.options) + #expect(incremental.rescanKind == .incrementalChanged) + #expect(ScannerProofHarness.signature(of: incremental, root: fixture.root) == + ScannerProofHarness.signature(of: full, root: fixture.root)) + touchedDirectories.removeAll(keepingCapacity: true) + } + } + } + + @Test("event overflow falls back to a full scan") + func eventOverflowFallsBack() throws { + let fixture = try IncrementalFixture() + defer { fixture.tearDown() } + try fixture.seed() + + let source = TestIncrementalChangeSource(eventID: 300) + let scanner = fixture.scanner(changeSource: source) + _ = try scanner.scan(root: fixture.root, options: fixture.options) + source.set( + eventID: 301, + changes: IncrementalChangeSet(paths: [fixture.root.path], requiresFullScan: true, reason: "event-log-overflow") + ) + + let scan = try scanner.scan(root: fixture.root, options: fixture.options) + #expect(scan.rescanKind == .full) + #expect(scan.incrementalFallbackReason == "event-log-overflow") + } + + @Test("changed traversal options fall back to a full scan") + func traversalOptionsChangeFallsBack() throws { + let fixture = try IncrementalFixture() + defer { fixture.tearDown() } + try fixture.seed() + + let source = TestIncrementalChangeSource(eventID: 400) + let scanner = fixture.scanner(changeSource: source) + _ = try scanner.scan(root: fixture.root, options: fixture.options) + var changedOptions = fixture.options + changedOptions.includeHidden.toggle() + + let scan = try scanner.scan(root: fixture.root, options: changedOptions) + #expect(scan.rescanKind == .full) + #expect(scan.incrementalFallbackReason == IncrementalScanFallback.optionsChanged.rawValue) + } + + @Test("corrupt persistence falls back to a full scan and regenerates") + func corruptPersistenceFallsBack() throws { + let fixture = try IncrementalFixture() + defer { fixture.tearDown() } + try fixture.seed() + + let source = TestIncrementalChangeSource(eventID: 500) + let scanner = fixture.scanner(changeSource: source) + _ = try scanner.scan(root: fixture.root, options: fixture.options) + try Data("not-json".utf8).write(to: fixture.persistence.cacheURL(rootURL: fixture.root), options: .atomic) + + let recovered = try scanner.scan(root: fixture.root, options: fixture.options) + let next = try scanner.scan(root: fixture.root, options: fixture.options) + #expect(recovered.rescanKind == .full) + #expect(recovered.incrementalFallbackReason == IncrementalScanFallback.persistenceCorrupt.rawValue) + #expect(next.rescanKind == .incrementalUnchanged) + } + + @Test("incompatible schema and volume identity fall back instead of using stale state", arguments: [ + ("schemaVersion", IncrementalScanFallback.schemaIncompatible.rawValue), + ("volumeIdentity", IncrementalScanFallback.volumeChanged.rawValue) + ]) + func incompatibleMetadataFallsBack(field: String, expectedReason: String) throws { + let fixture = try IncrementalFixture() + defer { fixture.tearDown() } + try fixture.seed() + + let source = TestIncrementalChangeSource(eventID: 600) + let scanner = fixture.scanner(changeSource: source) + _ = try scanner.scan(root: fixture.root, options: fixture.options) + let cacheURL = fixture.persistence.cacheURL(rootURL: fixture.root) + var plist = try #require(PropertyListSerialization.propertyList( + from: Data(contentsOf: cacheURL), + options: [], + format: nil + ) as? [String: Any]) + plist[field] = field == "schemaVersion" ? 999 : "different-volume" + try PropertyListSerialization.data( + fromPropertyList: plist, + format: .binary, + options: 0 + ).write(to: cacheURL, options: .atomic) + + let scan = try scanner.scan(root: fixture.root, options: fixture.options) + #expect(scan.rescanKind == .full) + #expect(scan.incrementalFallbackReason == expectedReason) + } + + @Test("inconsistent parent child persistence falls back to a full scan") + func inconsistentStateFallsBack() throws { + let fixture = try IncrementalFixture() + defer { fixture.tearDown() } + try fixture.seed() + + let source = TestIncrementalChangeSource(eventID: 700) + let scanner = fixture.scanner(changeSource: source) + _ = try scanner.scan(root: fixture.root, options: fixture.options) + let cacheURL = fixture.persistence.cacheURL(rootURL: fixture.root) + var plist = try #require(PropertyListSerialization.propertyList( + from: Data(contentsOf: cacheURL), + options: [], + format: nil + ) as? [String: Any]) + var nodes = try #require(plist["nodes"] as? [[String: Any]]) + var child = try #require(nodes.dropFirst().first) + child["parentID"] = 999 + nodes[1] = child + plist["nodes"] = nodes + try PropertyListSerialization.data( + fromPropertyList: plist, + format: .binary, + options: 0 + ).write(to: cacheURL, options: .atomic) + + let scan = try scanner.scan(root: fixture.root, options: fixture.options) + #expect(scan.rescanKind == .full) + #expect(scan.incrementalFallbackReason == IncrementalScanFallback.stateInconsistent.rawValue) + } +} + +private final class TestIncrementalChangeSource: IncrementalChangeSource, @unchecked Sendable { + private let lock = NSLock() + private var eventID: UInt64 + private var result = IncrementalChangeSet(paths: [], requiresFullScan: false, reason: nil) + + init(eventID: UInt64) { + self.eventID = eventID + } + + func set(eventID: UInt64, changes: IncrementalChangeSet) { + lock.lock() + self.eventID = eventID + result = changes + lock.unlock() + } + + func currentEventID() -> UInt64 { + lock.lock() + defer { lock.unlock() } + return eventID + } + + func changes(rootURL: URL, since eventID: UInt64) -> IncrementalChangeSet { + lock.lock() + defer { lock.unlock() } + return result + } +} + +private final class IncrementalFixture { + let root: URL + let cache: URL + let persistence: IncrementalScanPersistence + let options = ScanOptions( + oldFileAgeDays: 180, + largeFileThreshold: 1, + duplicateCandidateThreshold: 1, + duplicateVerificationByteLimit: 0, + maxDuplicateVerificationFiles: 0, + maxDuplicateCandidateItems: 10_000, + maxRankedResults: 10_000, + maxChildrenPerDirectory: 10_000, + maxRetainedItems: 100_000 + ) + + init() throws { + let base = FileManager.default.temporaryDirectory + .appendingPathComponent("StorageScopeIncrementalTests-\(UUID().uuidString)", isDirectory: true) + root = base.appendingPathComponent("root", isDirectory: true) + cache = base.appendingPathComponent("cache", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + persistence = IncrementalScanPersistence(baseURL: cache) + } + + func seed() throws { + for name in ["changed", "unchanged"] { + let directory = root.appendingPathComponent(name, isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + for index in 0..<8 { + try Data(repeating: UInt8(index), count: 256 + index).write( + to: directory.appendingPathComponent("file-\(index).bin") + ) + } + } + } + + func seedMutationCampaign() throws { + for index in 0..<10 { + let directory = root.appendingPathComponent("mutation-\(index)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + try Data(repeating: UInt8(index), count: 128).write( + to: directory.appendingPathComponent("anchor-a.bin") + ) + } + } + + func scanner(changeSource: IncrementalChangeSource) -> FileSystemScanner { + FileSystemScanner( + fileManager: FileManager(), + walkerMode: .fixedWorker, + incrementalPersistence: persistence, + incrementalChangeSource: changeSource, + incrementalTrustButVerify: false + ) + } + + func fullScanner() -> FileSystemScanner { + FileSystemScanner(fileManager: FileManager(), walkerMode: .fixedWorker) + } + + func tearDown() { + try? FileManager.default.removeItem(at: root.deletingLastPathComponent()) + } +} diff --git a/docs/perf-baselines/v0.7.1/phase3-incremental.md b/docs/perf-baselines/v0.7.1/phase3-incremental.md new file mode 100644 index 0000000..8938c1f --- /dev/null +++ b/docs/perf-baselines/v0.7.1/phase3-incremental.md @@ -0,0 +1,46 @@ +# Phase 3 incremental re-scan proof + +Captured 2026-07-13 on an Apple M1 with Swift 6.3.2. The release binary scanned a +kept local fixture containing 100,000 empty files and 101 directories. The fixture +was already persisted before this process started, so run 1 measures cold-process +tree reconstruction and runs 2-3 measure unchanged re-scans in the same process. + +```sh +swift build -c release --disable-index-store --jobs 1 +/usr/bin/time -l env STORAGESCOPE_INCREMENTAL_RESCAN=1 \ + .build/release/StorageScopeBenchmark \ + /tmp/storagescope-incremental-100k-20260713 --repeat 3 +``` + +| Run | Mode | Duration | Dirty subtrees | Fallback | +|---|---|---:|---:|---| +| 1 | `incrementalUnchanged` | 4.03 s | 0 | none | +| 2 | `incrementalUnchanged` | 0.10 s | 0 | none | +| 3 | `incrementalUnchanged` | 0.07 s | 0 | none | + +The process reported 126.9 MB peak memory (`133087232` maximum resident bytes from +`time -l`). Runs 2-3 clear the `<1 s` unchanged re-scan target and the process stays +below the `<150 MB` 100k RSS gate. This does not prove a sub-second cold-process +reload; the persisted reconstruction in run 1 took 4.03 s. + +The app enables incremental scans explicitly. A live FSEvents monitor starts from +the pre-walk checkpoint, changed directory subtrees are coalesced and spliced into +a compact parent-indexed binary property-list tree, and the previous scan is reused +in memory when no event arrived. The scanner falls back to a full walk for missing, +corrupt, incompatible, or inconsistent persistence; changed root, volume, or scan +options; unavailable/dropped/overflowed event history; and excessive dirty subtrees. +The legacy-walker environment override disables the incremental backend. + +Trust-but-verify remains enabled in the app: an incremental result is followed by a +deduplicated background full walk. A metadata/tree divergence invalidates persistence +so the next scan takes the conservative full-walk fallback. + +Correctness proof: + +- 11 focused incremental tests cover real one-shot and live FSEvents delivery, + immediate-mutation races, unchanged reuse, cancellation, subtree splicing, + 1,000 generated create/delete/rename/resize mutations, event overflow, option, + schema and volume changes, corrupt persistence, and inconsistent parent IDs. +- The final post-tightening full package run passed 232 tests across 23 suites. + +Results are local only. The fixture contains generated empty files, not user data.