From a2d6a8fad6b0f54318432b504b385c42d2939af5 Mon Sep 17 00:00:00 2001 From: RasputinKaiser <178525839+RasputinKaiser@users.noreply.github.com> Date: Sun, 12 Jul 2026 18:58:03 -0400 Subject: [PATCH] Complete scanner performance Phase 2 --- Sources/StorageScope/Stores/ScanStore.swift | 51 +- .../Stores/SearchRecentsStore.swift | 4 +- .../StorageScopeCore/Models/StorageScan.swift | 9 + .../Services/FileSystemScanner.swift | 2142 +++++++++++++++-- .../Services/ScanBenchmark.swift | 25 + StorageScope-Performance-Plan.md | 346 +++ .../FileSystemScannerTests.swift | 162 ++ .../ScannerProofHarness.swift | 572 +++++ .../ScannerProofHarnessTests.swift | 264 ++ .../ScanStoreAppPerformanceProofTests.swift | 116 + script/benchmark_scan.sh | 17 +- 11 files changed, 3457 insertions(+), 251 deletions(-) create mode 100644 StorageScope-Performance-Plan.md create mode 100644 Tests/StorageScopeCoreTests/ScannerProofHarness.swift create mode 100644 Tests/StorageScopeCoreTests/ScannerProofHarnessTests.swift create mode 100644 Tests/StorageScopeTests/ScanStoreAppPerformanceProofTests.swift diff --git a/Sources/StorageScope/Stores/ScanStore.swift b/Sources/StorageScope/Stores/ScanStore.swift index 8394152..41f5309 100644 --- a/Sources/StorageScope/Stores/ScanStore.swift +++ b/Sources/StorageScope/Stores/ScanStore.swift @@ -233,8 +233,8 @@ func setSelectedView(_ view: SmartView) { private var cachedVerifiedDuplicateGroupsKey: DerivedCacheKey? private var cachedVerifiedDuplicateGroups: [VerifiedDuplicateGroup] = [] - /// Test observability for `items(for:)` cache. DEBUG-only so production pays nothing. - #if DEBUG + /// Test observability for `items(for:)` cache. Excluded from normal release builds. + #if DEBUG || STORAGESCOPE_TESTING private(set) var itemsCacheHitCount: Int = 0 private(set) var itemsCacheMissCount: Int = 0 @@ -933,18 +933,7 @@ func setSelectedView(_ view: SmartView) { progress = ScanProgress(scannedItemCount: 0, totalBytes: 0, currentPath: url.path) let scanID = UUID() - let thresholds = ScanOptionPolicy.interactiveScanThresholds() - let (excludedComponents, excludedPrefixes) = Self.splitExcludedPaths(filters.excludedPaths) - let options = ScanOptions( - includeHidden: filters.includeHiddenFiles, - oldFileAgeDays: filters.oldFileAgeDays, - largeFileThreshold: thresholds.largeFileThreshold, - duplicateCandidateThreshold: Int64(filters.duplicateCandidateThresholdMB) * 1_000_000, - maxRankedResults: Self.rankedResultsCap, - excludeEnabled: filters.excludeFoldersEnabled, - excludedPathComponents: excludedComponents, - excludedAbsolutePrefixes: excludedPrefixes - ) + let options = makeScannerOptions() let optionsSnapshot = currentScanOptions let scanCancellation = ScanCancellation() activeScanID = scanID @@ -967,11 +956,10 @@ func setSelectedView(_ view: SmartView) { // Task.cancel() into the scanner's ScanCancellation handle AND finishes the // stream so the consumer exits cleanly. let (progressStream, progressContinuation) = AsyncStream.makeStream() - // The scanner's `progress` and `onSnapshot` callbacks fire at the same - // throttled cadence (`onSnapshot` is invoked from inside `emitProgressLocked` - // right after the progress callback) but as two separate calls. Pair the - // latest snapshot with the next progress tick on this side rather than adding - // a second stream, so the consumer only has one `for await` loop to drain. + // Snapshots are emitted before their paired progress tick, but at a slower + // cadence than progress. Pair a newly staged snapshot with the next tick on + // this side rather than adding a second stream, so the consumer only has one + // `for await` loop to drain; intervening progress-only ticks carry nil. let pendingSnapshotLock = NSLock() var pendingSnapshot: StorageScan? let result = try await withTaskCancellationHandler { @@ -1043,10 +1031,11 @@ func setSelectedView(_ view: SmartView) { scan = result let cacheToPersist = hashCache - let pathsScanned = Set(result.retainedItems.map { $0.url.standardizedFileURL.path }) + let retainedItems = result.retainedItems let persistLog = Self.log let persistSignpostID = Self.signpostID Task.detached(priority: .utility) { [weak self] in + let pathsScanned = Set(retainedItems.map { $0.url.standardizedFileURL.path }) os_signpost(.begin, log: persistLog, name: "persist", signpostID: persistSignpostID, "entries=%d", cacheToPersist.entryCount) _ = cacheToPersist.purgeStale(except: pathsScanned) @@ -1126,6 +1115,24 @@ func setSelectedView(_ view: SmartView) { } } + /// Builds the exact scanner configuration used by the interactive app path. + /// Internal so the opt-in release performance proof can compare ScanStore against + /// a callback-free FileSystemScanner without drifting to different thresholds. + func makeScannerOptions() -> ScanOptions { + let thresholds = ScanOptionPolicy.interactiveScanThresholds() + let (excludedComponents, excludedPrefixes) = Self.splitExcludedPaths(filters.excludedPaths) + return ScanOptions( + includeHidden: filters.includeHiddenFiles, + oldFileAgeDays: filters.oldFileAgeDays, + largeFileThreshold: thresholds.largeFileThreshold, + duplicateCandidateThreshold: Int64(filters.duplicateCandidateThresholdMB) * 1_000_000, + maxRankedResults: Self.rankedResultsCap, + excludeEnabled: filters.excludeFoldersEnabled, + excludedPathComponents: excludedComponents, + excludedAbsolutePrefixes: excludedPrefixes + ) + } + func items(for view: SmartView) -> [StorageItem] { guard let scan else { return [] @@ -1147,7 +1154,7 @@ func setSelectedView(_ view: SmartView) { ignoredCleanupCandidateIDs: cleanupRelevant ? ignoredCleanupCandidateIDs : [] ) if cachedItemsKey == key { - #if DEBUG + #if DEBUG || STORAGESCOPE_TESTING itemsCacheHitCount &+= 1 #endif return cachedItems @@ -1158,7 +1165,7 @@ func setSelectedView(_ view: SmartView) { defer { os_signpost(.end, log: Self.log, name: "items_rebuild", signpostID: Self.signpostID) } - #if DEBUG + #if DEBUG || STORAGESCOPE_TESTING itemsCacheMissCount &+= 1 #endif diff --git a/Sources/StorageScope/Stores/SearchRecentsStore.swift b/Sources/StorageScope/Stores/SearchRecentsStore.swift index 12dc6e1..341ab02 100644 --- a/Sources/StorageScope/Stores/SearchRecentsStore.swift +++ b/Sources/StorageScope/Stores/SearchRecentsStore.swift @@ -61,7 +61,7 @@ final class SearchRecentsStore: ObservableObject { let task = Task.detached(priority: .utility) { [writer = Self.writer] in await writer.write(snapshot, generation: generation) } - #if DEBUG + #if DEBUG || STORAGESCOPE_TESTING Self.pendingPersistTasks.append(task) #endif } @@ -81,7 +81,7 @@ final class SearchRecentsStore: ObservableObject { } } - #if DEBUG + #if DEBUG || STORAGESCOPE_TESTING /// Drains detached `persist()` tasks from earlier tests so a test that /// mutates `UserDefaults` directly can assert deterministically. Production /// callers should never need this: the actor-serialized writer is already diff --git a/Sources/StorageScopeCore/Models/StorageScan.swift b/Sources/StorageScopeCore/Models/StorageScan.swift index 499927d..8a8715c 100644 --- a/Sources/StorageScopeCore/Models/StorageScan.swift +++ b/Sources/StorageScopeCore/Models/StorageScan.swift @@ -25,8 +25,11 @@ public struct StorageScan: Sendable { public let duplicateCandidateItemLimit: Int public let duplicateCandidateItemsRetained: Int public let duplicateCandidateItemsConsidered: Int + public let duplicateCandidateEvictionCount: Int public let duplicateCandidateLimitReached: Bool + public let snapshotBuildCount: Int public let duplicateVerificationDuration: TimeInterval + public let duplicateVerificationBytesRead: Int64 public let enumerateDuration: TimeInterval public let cleanupCandidates: [CleanupCandidate] public let isPartial: Bool @@ -51,8 +54,11 @@ public struct StorageScan: Sendable { duplicateCandidateItemLimit: Int = 0, duplicateCandidateItemsRetained: Int = 0, duplicateCandidateItemsConsidered: Int = 0, + duplicateCandidateEvictionCount: Int = 0, duplicateCandidateLimitReached: Bool = false, + snapshotBuildCount: Int = 0, duplicateVerificationDuration: TimeInterval = 0, + duplicateVerificationBytesRead: Int64 = 0, enumerateDuration: TimeInterval = 0, cleanupCandidates: [CleanupCandidate], isPartial: Bool = false @@ -75,8 +81,11 @@ public struct StorageScan: Sendable { self.duplicateCandidateItemLimit = duplicateCandidateItemLimit self.duplicateCandidateItemsRetained = duplicateCandidateItemsRetained self.duplicateCandidateItemsConsidered = duplicateCandidateItemsConsidered + self.duplicateCandidateEvictionCount = duplicateCandidateEvictionCount self.duplicateCandidateLimitReached = duplicateCandidateLimitReached + self.snapshotBuildCount = snapshotBuildCount self.duplicateVerificationDuration = duplicateVerificationDuration + self.duplicateVerificationBytesRead = duplicateVerificationBytesRead self.enumerateDuration = enumerateDuration self.cleanupCandidates = cleanupCandidates self.isPartial = isPartial diff --git a/Sources/StorageScopeCore/Services/FileSystemScanner.swift b/Sources/StorageScopeCore/Services/FileSystemScanner.swift index 82351f9..86fc8c2 100644 --- a/Sources/StorageScopeCore/Services/FileSystemScanner.swift +++ b/Sources/StorageScopeCore/Services/FileSystemScanner.swift @@ -89,6 +89,11 @@ public final class ScanCancellation: @unchecked Sendable { } } +enum FileSystemScannerWalkerMode: Sendable { + case legacy + case fixedWorker +} + public final class FileSystemScanner { public typealias ProgressHandler = (ScanProgress) -> Void @@ -100,6 +105,7 @@ public final class FileSystemScanner { let cores = ProcessInfo.processInfo.processorCount return min(6, max(2, cores / 2)) }() + private static let duplicatePrefixByteCount = 64 * 1_024 /// os_signpost surface for Instruments. Subsystem mirrors the bundle identifier prefix; /// category ties Scanner-only work together so it can be filtered from app-side spans. @@ -108,6 +114,7 @@ public final class FileSystemScanner { private let fileManager: FileManager private let hashCache: DuplicateHashCache? + private let walkerMode: FileSystemScannerWalkerMode private let resourceKeys: Set = [ .isDirectoryKey, .isRegularFileKey, @@ -121,9 +128,35 @@ public final class FileSystemScanner { .contentModificationDateKey ] - public init(fileManager: FileManager = .default, hashCache: DuplicateHashCache? = nil) { + public convenience init(fileManager: FileManager = .default, hashCache: DuplicateHashCache? = nil) { + self.init( + fileManager: fileManager, + hashCache: hashCache, + walkerMode: Self.configuredWalkerMode() + ) + } + + init( + fileManager: FileManager = .default, + hashCache: DuplicateHashCache? = nil, + walkerMode: FileSystemScannerWalkerMode + ) { self.fileManager = fileManager self.hashCache = hashCache + self.walkerMode = walkerMode + } + + private static func configuredWalkerMode() -> FileSystemScannerWalkerMode { + let environment = ProcessInfo.processInfo.environment + if environment["STORAGESCOPE_LEGACY_WALKER"] == "1" { + return .legacy + } + if environment["STORAGESCOPE_EXPERIMENTAL_WORKER_WALKER"] == "1" { + return .fixedWorker + } + // Keep the existing implementation as the default until the experimental + // path clears the differential, stability, and release-performance gates. + return .legacy } public func scan( @@ -143,13 +176,24 @@ public final class FileSystemScanner { let startedAt = Date() let accumulator = ScanAccumulator(options: options, progress: progress, onSnapshot: onSnapshot, rootURL: rootURL, startedAt: startedAt) - let rootItem = try scanItem( - at: rootURL, - options: options, - cancellation: cancellation, - accumulator: accumulator, - depth: 0 - ) + let rootItem: StorageItem + switch walkerMode { + case .legacy: + rootItem = try scanItem( + at: rootURL, + options: options, + cancellation: cancellation, + accumulator: accumulator, + depth: 0 + ) + case .fixedWorker: + rootItem = try scanWithFixedWorker( + at: rootURL, + options: options, + cancellation: cancellation, + accumulator: accumulator + ) + } let retainedItems = rootItem.flattened() let duplicateSizeGroups = accumulator.duplicateSizeGroups let enumerateDuration = Date().timeIntervalSince(startedAt) @@ -161,6 +205,7 @@ public final class FileSystemScanner { cancellation: cancellation ) let duplicateVerificationDuration = Date().timeIntervalSince(duplicateVerificationStartedAt) + let duplicateVerificationBytesRead = accumulator.duplicateVerificationBytesRead let finishedAt = Date() os_signpost(.end, log: Self.log, name: "scan", signpostID: Self.signpostID, @@ -185,8 +230,11 @@ public final class FileSystemScanner { duplicateCandidateItemLimit: accumulator.duplicateCandidateItemLimit, duplicateCandidateItemsRetained: accumulator.duplicateCandidateItemsRetained, duplicateCandidateItemsConsidered: accumulator.duplicateCandidateItemsConsidered, + duplicateCandidateEvictionCount: accumulator.duplicateCandidateEvictionCount, duplicateCandidateLimitReached: accumulator.duplicateCandidateLimitReached, + snapshotBuildCount: accumulator.snapshotBuildCount, duplicateVerificationDuration: duplicateVerificationDuration, + duplicateVerificationBytesRead: duplicateVerificationBytesRead, enumerateDuration: enumerateDuration, cleanupCandidates: accumulator.cleanupCandidates( rootID: rootItem.id, @@ -223,36 +271,13 @@ public final class FileSystemScanner { let ioSemaphore = DispatchSemaphore(value: Self.hashConcurrency) let cacheLock = NSLock() - var hashedItems: [HashedStorageItem] = [] - hashedItems.reserveCapacity(group.items.count) - - for item in group.items { - // Cooperative cancellation: re-check before opening I/O for the next file so a - // cancelled scan doesn't keep burning file descriptors or queuing reads behind - // the ioSemaphore. Cancellation aborts the whole on-demand verify because the - // caller asked us to stop. - try cancellation?.check() - - if let hashed = try hashedFormItem( - item, - ioSemaphore: ioSemaphore, - cacheLock: cacheLock, - cancellation: cancellation - ) { - hashedItems.append(hashed) - } - } - - let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) - return groupedByHash.compactMap { checksum, hashedItems in - let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } - return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: group.byteSize, items: items) : nil - }.sorted { lhs, rhs in - if lhs.reclaimableBytes == rhs.reclaimableBytes { - return lhs.byteSize > rhs.byteSize - } - return lhs.reclaimableBytes > rhs.reclaimableBytes - } + return try verifiedDuplicateGroups( + in: group, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: nil, + cancellation: cancellation + ) } private func scanItem( @@ -293,11 +318,8 @@ public final class FileSystemScanner { ) } - accumulator.recordVisit(path: url.path) - if values?.isSymbolicLink == true { let size = Int64(values?.fileSize ?? 0) - accumulator.recordBytes(size) let item = StorageItem( url: url, kind: .alias, @@ -309,7 +331,7 @@ public final class FileSystemScanner { isReadable: values?.isReadable ?? true, fileExtension: url.pathExtension.nonEmptyLowercased ) - accumulator.recordItem(item) + accumulator.recordScannedItem(item, countedBytes: size, path: url.path) return item } @@ -319,7 +341,6 @@ public final class FileSystemScanner { guard isDirectory else { let logicalSize = Int64(values?.fileSize ?? 0) let allocatedSize = Int64(values?.totalFileAllocatedSize ?? values?.fileAllocatedSize ?? values?.fileSize ?? 0) - accumulator.recordBytes(max(logicalSize, allocatedSize)) let item = StorageItem( url: url, @@ -332,10 +353,12 @@ public final class FileSystemScanner { isReadable: values?.isReadable ?? true, fileExtension: url.pathExtension.nonEmptyLowercased ) - accumulator.recordItem(item) + accumulator.recordScannedItem(item, countedBytes: max(logicalSize, allocatedSize), path: url.path) return item } + accumulator.recordVisit(path: url.path) + do { let directoryOptions: FileManager.DirectoryEnumerationOptions = options.includeHidden ? [] : [.skipsHiddenFiles] let childURLs = try fileManager.contentsOfDirectory( @@ -467,6 +490,225 @@ public final class FileSystemScanner { } } + private func scanWithFixedWorker( + at rootURL: URL, + options: ScanOptions, + cancellation: ScanCancellation?, + accumulator: ScanAccumulator + ) throws -> StorageItem { + let prepared: (rootSummary: FixedWorkerItemSummary?, summaryStore: FixedWorkerSummaryStore?) + do { + prepared = try prepareFixedWorkerSummary( + at: rootURL, + options: options, + cancellation: cancellation, + accumulator: accumulator + ) + } catch FileSystemScannerError.cancelled { + throw FileSystemScannerError.cancelled + } catch { + // The experimental path owns no user-visible state until its complete + // record set is available. If a worker-only failure escapes the + // filesystem error handling below, retry the root through the legacy + // implementation rather than returning a partial tree. + return try scanItem( + at: rootURL, + options: options, + cancellation: cancellation, + accumulator: accumulator, + depth: 0 + ) + } + + guard let rootSummary = prepared.rootSummary, + let summaryStore = prepared.summaryStore else { + return StorageItem( + url: rootURL, + kind: .other, + byteSize: 0, + allocatedSize: 0, + modifiedAt: nil, + immediateChildCount: 0, + descendantCount: 0, + isReadable: false, + fileExtension: nil + ) + } + + return summaryStore.makeItem(summary: rootSummary, urlOverride: rootURL) + } + + private func prepareFixedWorkerSummary( + at rootURL: URL, + options: ScanOptions, + cancellation: ScanCancellation?, + accumulator: ScanAccumulator + ) throws -> ( + rootSummary: FixedWorkerItemSummary?, + summaryStore: FixedWorkerSummaryStore? + ) { + let walker = FixedWorkerDirectoryWalker( + fileManager: fileManager, + resourceKeys: resourceKeys, + options: options, + cancellation: cancellation, + shouldExclude: { url in + options.excludeEnabled && Self.isExcluded(url, options: options) + } + ) + let result = try walker.walk(root: rootURL) + guard let rootMetadata = result.rootMetadata else { + return (nil, nil) + } + + let (rootSummary, summaryStore) = try buildFixedWorkerSummaryStore( + rootMetadata: rootMetadata, + directoryRecords: result.directoryRecords, + options: options, + cancellation: cancellation, + accumulator: accumulator, + rootURL: rootURL + ) + return (rootSummary, summaryStore) + } + + private func buildFixedWorkerSummaryStore( + rootMetadata: FixedWorkerWalkRecord, + directoryRecords: [FixedWorkerDirectoryRecord], + options: ScanOptions, + cancellation: ScanCancellation?, + accumulator: ScanAccumulator, + rootURL: URL + ) throws -> (FixedWorkerItemSummary, FixedWorkerSummaryStore) { + var recordsByID: [Int: FixedWorkerDirectoryRecord] = [:] + recordsByID.reserveCapacity(directoryRecords.count) + for record in directoryRecords { + recordsByID[record.metadata.id] = record + } + + let summaryStore = FixedWorkerSummaryStore(rootURL: rootURL) + accumulator.configureFixedWorkerStore(summaryStore) + let rootSummary = try buildFixedWorkerSummary( + rootMetadata, + url: rootURL, + recordsByID: recordsByID, + options: options, + cancellation: cancellation, + accumulator: accumulator, + depth: 0 + ) + return (rootSummary, summaryStore) + } + + private func buildFixedWorkerSummary( + _ metadata: FixedWorkerWalkRecord, + url: URL, + recordsByID: [Int: FixedWorkerDirectoryRecord], + options: ScanOptions, + cancellation: ScanCancellation?, + accumulator: ScanAccumulator, + depth: Int + ) throws -> FixedWorkerItemSummary { + try cancellation?.check() + + guard metadata.isDirectory else { + let summary = FixedWorkerItemSummary( + metadata: metadata, + isInaccessible: false, + logicalSize: metadata.byteSize, + allocatedSize: metadata.allocatedSize, + immediateChildCount: 0, + descendantCount: 0, + retainedChildren: [], + retainedTreeCount: 1 + ) + accumulator.recordFixedWorkerScannedItem(summary, url: url) + return summary + } + + accumulator.recordVisit(path: url.path) + guard let record = recordsByID[metadata.id] else { + accumulator.recordInaccessible(path: url.path) + let summary = FixedWorkerItemSummary( + metadata: metadata, + isInaccessible: true, + logicalSize: 0, + allocatedSize: 0, + immediateChildCount: 0, + descendantCount: 0, + retainedChildren: [], + retainedTreeCount: 1 + ) + accumulator.recordFixedWorkerItem(summary, url: url) + return summary + } + + if record.isInaccessible { + accumulator.recordInaccessible(path: url.path) + let summary = FixedWorkerItemSummary( + metadata: metadata, + isInaccessible: true, + logicalSize: 0, + allocatedSize: 0, + immediateChildCount: 0, + descendantCount: 0, + retainedChildren: [], + retainedTreeCount: 1 + ) + accumulator.recordFixedWorkerItem(summary, url: url) + return summary + } + + var directorySummary = FixedWorkerDirectorySummaryBuilder( + retainedCandidateLimit: options.maxChildrenPerDirectory + ) + for childMetadata in record.children { + try cancellation?.check() + let childURL = url.appendingPathComponent(childMetadata.name, isDirectory: childMetadata.isDirectory) + let childSummary = try buildFixedWorkerSummary( + childMetadata, + url: childURL, + recordsByID: recordsByID, + options: options, + cancellation: cancellation, + accumulator: accumulator, + depth: depth + 1 + ) + directorySummary.record(childSummary) + } + + let retainedSelections = accumulator.retainedFixedWorkerChildren( + from: directorySummary.retainedCandidates, + depth: depth + ) + let retainedChildren: [FixedWorkerRetainedChild] = retainedSelections.compactMap { selection in + guard let childSummary = directorySummary.summary(for: selection.id) else { + return nil + } + return FixedWorkerRetainedChild( + id: selection.id, + includesDescendants: selection.includesDescendants, + retainedTreeCount: selection.retainedTreeCount, + summary: childSummary + ) + } + let retainedTreeCount = 1 + retainedChildren.reduce(0) { partialResult, child in + partialResult + child.retainedTreeCount + } + let summary = FixedWorkerItemSummary( + metadata: metadata, + isInaccessible: false, + logicalSize: directorySummary.logicalSize, + allocatedSize: directorySummary.allocatedSize, + immediateChildCount: directorySummary.immediateChildCount, + descendantCount: directorySummary.descendantCount, + retainedChildren: retainedChildren, + retainedTreeCount: retainedTreeCount + ) + accumulator.recordFixedWorkerItem(summary, url: url) + return summary + } + /// True when `url` matches an exclusion rule: its last path component names an /// excluded folder name (e.g. `node_modules`, `.git`), or its standardized path starts /// with an excluded absolute prefix (tilde-expanded, e.g. `~/Library/Caches`). @@ -545,32 +787,13 @@ public final class FileSystemScanner { try cancellation?.check() let sizeGroup = verificationGroups[groupIndex] - var hashedItems: [HashedStorageItem] = [] - hashedItems.reserveCapacity(sizeGroup.items.count) - - for item in sizeGroup.items { - // Per-item cancellation probe keeps an outlier slow hash from orphaning - // the rest of a group's work after the user cancels. - if cancellation?.isCancelled ?? false { - markCancelled() - return - } - - if let hashed = try hashedFormItem( - item, - ioSemaphore: ioSemaphore, - cacheLock: cacheLock, - cancellation: cancellation - ) { - hashedItems.append(hashed) - } - } - - let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) - let verifiedForSize = groupedByHash.compactMap { checksum, hashedItems -> VerifiedDuplicateGroup? in - let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } - return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: sizeGroup.byteSize, items: items) : nil - } + let verifiedForSize = try verifiedDuplicateGroups( + in: sizeGroup, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: accumulator.recordDuplicateVerificationBytes, + cancellation: cancellation + ) guard !verifiedForSize.isEmpty else { return @@ -600,6 +823,103 @@ public final class FileSystemScanner { } } + private func verifiedDuplicateGroups( + in sizeGroup: DuplicateSizeGroup, + ioSemaphore: DispatchSemaphore, + cacheLock: NSLock, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> [VerifiedDuplicateGroup] { + var cachedFullHashesByPath: [String: String] = [:] + cachedFullHashesByPath.reserveCapacity(sizeGroup.items.count) + + for item in sizeGroup.items { + try cancellation?.check() + let cacheKey = DuplicateHashCache.LookupKey(item: item) + if let cached = hashCache?.checksum(for: cacheKey) { + cachedFullHashesByPath[cacheKey.path] = cached + } + } + + if cachedFullHashesByPath.count == sizeGroup.items.count { + let hashedItems = sizeGroup.items.compactMap { item -> HashedStorageItem? in + let path = DuplicateHashCache.LookupKey(item: item).path + guard let checksum = cachedFullHashesByPath[path] else { return nil } + return HashedStorageItem(checksum: checksum, item: item) + } + return verifiedDuplicateGroups(from: hashedItems, byteSize: sizeGroup.byteSize) + } + + var prefixHashedItems: [PrefixHashedStorageItem] = [] + prefixHashedItems.reserveCapacity(sizeGroup.items.count) + + for item in sizeGroup.items { + if cancellation?.isCancelled ?? false { + throw FileSystemScannerError.cancelled + } + + if let prefixed = try prefixHashedFormItem( + item, + ioSemaphore: ioSemaphore, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) { + prefixHashedItems.append(prefixed) + } + } + + let groupedByPrefix = Dictionary(grouping: prefixHashedItems, by: \.prefixChecksum) + var fullyHashedItems: [HashedStorageItem] = [] + + for prefixGroup in groupedByPrefix.values where prefixGroup.count > 1 { + for prefixed in prefixGroup { + let cacheKey = DuplicateHashCache.LookupKey(item: prefixed.item) + if let cached = cachedFullHashesByPath[cacheKey.path] { + fullyHashedItems.append(HashedStorageItem(checksum: cached, item: prefixed.item)) + continue + } + + if prefixed.isCompleteFile { + if let hashCache { + cacheLock.lock() + hashCache.record(cacheKey, checksum: prefixed.prefixChecksum) + cacheLock.unlock() + } + fullyHashedItems.append(HashedStorageItem(checksum: prefixed.prefixChecksum, item: prefixed.item)) + continue + } + + if let hashed = try hashedFormItem( + prefixed.item, + ioSemaphore: ioSemaphore, + cacheLock: cacheLock, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) { + fullyHashedItems.append(hashed) + } + } + } + + return verifiedDuplicateGroups(from: fullyHashedItems, byteSize: sizeGroup.byteSize) + } + + private func verifiedDuplicateGroups( + from hashedItems: [HashedStorageItem], + byteSize: Int64 + ) -> [VerifiedDuplicateGroup] { + let groupedByHash = Dictionary(grouping: hashedItems, by: \.checksum) + return groupedByHash.compactMap { checksum, hashedItems -> VerifiedDuplicateGroup? in + let items = hashedItems.map(\.item).sorted { $0.url.path < $1.url.path } + return items.count > 1 ? VerifiedDuplicateGroup(checksum: checksum, byteSize: byteSize, items: items) : nil + }.sorted { lhs, rhs in + if lhs.reclaimableBytes == rhs.reclaimableBytes { + return lhs.byteSize > rhs.byteSize + } + return lhs.reclaimableBytes > rhs.reclaimableBytes + } + } + private func duplicateGroupsWithinVerificationBudget( _ sizeGroups: [DuplicateSizeGroup], options: ScanOptions @@ -639,7 +959,11 @@ public final class FileSystemScanner { return plannedGroups } - private func sha256Checksum(for url: URL, cancellation: ScanCancellation?) throws -> String { + private func sha256Checksum( + for url: URL, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> String { // Probe cancellation before opening so a cancelled batch doesn't keep paying the // cost of `FileHandle(forReadingFrom:)` for files that no one will ever read. try cancellation?.check() @@ -666,12 +990,80 @@ public final class FileSystemScanner { if data.isEmpty { break } + recordBytesRead?(data.count) hasher.update(data: data) } return hasher.finalize().hexEncodedString() } + private func prefixChecksum( + for url: URL, + maxBytes: Int, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> (checksum: String, bytesRead: Int) { + try cancellation?.check() + + let handle = try FileHandle(forReadingFrom: url) + defer { + do { + try handle.close() + } catch { + os_signpost(.event, log: Self.log, name: "handle_close_failed", + "path=%{public}@ reason=%{public}@", url.path, "\(error)") + } + } + + var remaining = max(0, maxBytes) + var bytesRead = 0 + var hasher = SHA256() + while remaining > 0 { + try cancellation?.check() + let readSize = min(remaining, 1_048_576) + let data = try handle.read(upToCount: readSize) ?? Data() + if data.isEmpty { + break + } + bytesRead += data.count + remaining -= data.count + recordBytesRead?(data.count) + hasher.update(data: data) + } + + return (hasher.finalize().hexEncodedString(), bytesRead) + } + + private func prefixHashedFormItem( + _ item: StorageItem, + ioSemaphore: DispatchSemaphore, + recordBytesRead: ((Int) -> Void)?, + cancellation: ScanCancellation? + ) throws -> PrefixHashedStorageItem? { + ioSemaphore.wait() + defer { ioSemaphore.signal() } + + do { + let result = try prefixChecksum( + for: item.url, + maxBytes: Self.duplicatePrefixByteCount, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) + return PrefixHashedStorageItem( + prefixChecksum: result.checksum, + bytesRead: result.bytesRead, + item: item + ) + } catch FileSystemScannerError.cancelled { + throw FileSystemScannerError.cancelled + } catch { + os_signpost(.event, log: Self.log, name: "hash_prefix_skip", + "path=%{public}@ reason=%{public}@", item.url.path, "\(error)") + return nil + } + } + /// Hashes one item, consulting the persisted `hashCache` fast-path before falling back /// to a full `sha256Checksum` read. I/O is throttled through `ioSemaphore`; cache writes /// are serialised through `cacheLock`. Returns `nil` for per-file read failures (logged @@ -681,6 +1073,7 @@ public final class FileSystemScanner { _ item: StorageItem, ioSemaphore: DispatchSemaphore, cacheLock: NSLock, + recordBytesRead: ((Int) -> Void)?, cancellation: ScanCancellation? ) throws -> HashedStorageItem? { let cacheKey = DuplicateHashCache.LookupKey(item: item) @@ -693,7 +1086,11 @@ public final class FileSystemScanner { defer { ioSemaphore.signal() } do { - let checksum = try sha256Checksum(for: item.url, cancellation: cancellation) + let checksum = try sha256Checksum( + for: item.url, + recordBytesRead: recordBytesRead, + cancellation: cancellation + ) if let hashCache { cacheLock.lock() defer { cacheLock.unlock() } @@ -710,26 +1107,736 @@ public final class FileSystemScanner { } } -private struct HashedStorageItem { - let checksum: String - let item: StorageItem +private extension StorageItem.Kind { + var isDirectoryKind: Bool { + self == .folder || self == .package + } } -private struct DirectoryScanSummary { - private let retainedCandidateLimit: Int - private var retainedCandidateItems: [StorageItem] = [] +private struct FixedWorkerResourceIdentifier: Hashable, Sendable { + let low: UInt64 + let high: UInt64 - var logicalSize: Int64 = 0 - var allocatedSize: Int64 = 0 - var immediateChildCount = 0 - var descendantCount = 0 + static let zero = FixedWorkerResourceIdentifier(low: 0, high: 0) +} - init(retainedCandidateLimit: Int) { - self.retainedCandidateLimit = max(0, retainedCandidateLimit) +private struct FixedWorkerMetadataSeed: Sendable { + let name: String + let kind: StorageItem.Kind + let byteSize: Int64 + let allocatedSize: Int64 + let modifiedAt: Date? + let isReadable: Bool + let volumeIdentifier: FixedWorkerResourceIdentifier + let fileResourceIdentifier: FixedWorkerResourceIdentifier + let hardLinkCount: UInt16 +} + +private struct FixedWorkerWalkRecord: Sendable { + let id: Int + let parentID: Int + let name: String + let kind: StorageItem.Kind + let byteSize: Int64 + let allocatedSize: Int64 + let modifiedAt: Date? + let isReadable: Bool + let volumeIdentifier: FixedWorkerResourceIdentifier + let fileResourceIdentifier: FixedWorkerResourceIdentifier + /// Reserved for the hard-link reclaimability policy; zero means not enriched yet. + let hardLinkCount: UInt16 + + var isDirectory: Bool { + kind.isDirectoryKind } +} - mutating func record(_ child: StorageItem) { - logicalSize += child.byteSize +private struct FixedWorkerDirectoryJob: Sendable { + let metadata: FixedWorkerWalkRecord + let url: URL +} + +private struct FixedWorkerDirectoryRecord: Sendable { + let metadata: FixedWorkerWalkRecord + let children: [FixedWorkerWalkRecord] + let isInaccessible: Bool +} + +private struct FixedWorkerWalkResult: Sendable { + let rootMetadata: FixedWorkerWalkRecord? + let directoryRecords: [FixedWorkerDirectoryRecord] +} + +private struct FixedWorkerCompactChild: Sendable { + let id: Int + let displaySize: Int64 + let name: String + let retainedTreeCount: Int +} + +private struct FixedWorkerRetainedSelection: Sendable { + let id: Int + let includesDescendants: Bool + let retainedTreeCount: Int +} + +private struct FixedWorkerRetainedChild: Sendable { + let id: Int + let includesDescendants: Bool + let retainedTreeCount: Int + let summary: FixedWorkerItemSummary +} + +private struct FixedWorkerItemSummary: Sendable { + let metadata: FixedWorkerWalkRecord + let isInaccessible: Bool + let logicalSize: Int64 + let allocatedSize: Int64 + let immediateChildCount: Int + let descendantCount: Int + let retainedChildren: [FixedWorkerRetainedChild] + let retainedTreeCount: Int + + var kind: StorageItem.Kind { + isInaccessible ? .inaccessible : metadata.kind + } + + var displaySize: Int64 { + max(logicalSize, allocatedSize) + } + + var fileExtension: String? { + switch kind { + case .folder, .inaccessible: + return nil + case .package, .file, .alias, .other: + return (metadata.name as NSString).pathExtension.nonEmptyLowercased + } + } +} + +private struct FixedWorkerRankedReference: Sendable { + let summary: FixedWorkerItemSummary + let displaySize: Int64 + let url: URL + let path: String +} + +private struct FixedWorkerCleanupCandidateReference: Sendable { + let summary: FixedWorkerItemSummary + let url: URL + let kind: CleanupCandidate.Kind + let reason: String + let reclaimableBytes: Int64 + let confidence: CleanupCandidate.Confidence +} + +private final class FixedWorkerSummaryStore { + let rootURL: URL + + init(rootURL: URL) { + self.rootURL = rootURL + } + + func makeItem(summary: FixedWorkerItemSummary, urlOverride: URL? = nil) -> StorageItem { + makeItem(summary: summary, urlOverride: urlOverride, includeDescendants: true) + } + + private func makeItem( + summary: FixedWorkerItemSummary, + urlOverride: URL?, + includeDescendants: Bool + ) -> StorageItem { + let url = urlOverride ?? rootURL + let children: [StorageItem] + if includeDescendants { + children = summary.retainedChildren.map { child in + makeItem( + summary: child.summary, + urlOverride: url.appendingPathComponent( + child.summary.metadata.name, + isDirectory: child.summary.metadata.isDirectory + ), + includeDescendants: child.includesDescendants + ) + } + } else { + children = [] + } + return StorageItem( + url: url, + name: summary.metadata.name, + kind: summary.kind, + byteSize: summary.logicalSize, + allocatedSize: summary.allocatedSize, + modifiedAt: summary.metadata.modifiedAt, + immediateChildCount: summary.immediateChildCount, + descendantCount: summary.descendantCount, + children: children, + isReadable: summary.isInaccessible ? false : summary.metadata.isReadable, + fileExtension: summary.fileExtension + ) + } +} + +private struct FixedWorkerDirectorySummaryBuilder { + private let retainedCandidateLimit: Int + private var retainedCandidateItems: [FixedWorkerItemSummary] = [] + + var logicalSize: Int64 = 0 + var allocatedSize: Int64 = 0 + var immediateChildCount = 0 + var descendantCount = 0 + + init(retainedCandidateLimit: Int) { + self.retainedCandidateLimit = max(0, retainedCandidateLimit) + } + + mutating func record(_ child: FixedWorkerItemSummary) { + guard child.kind != .other || child.displaySize > 0 else { + return + } + + logicalSize += child.logicalSize + allocatedSize += child.allocatedSize + immediateChildCount += 1 + descendantCount += 1 + child.descendantCount + + guard retainedCandidateLimit > 0 else { + return + } + + retainedCandidateItems.append(child) + if retainedCandidateItems.count > retainedCandidateLimit * 4 { + retainedCandidateItems = sortedRetainedCandidates(from: retainedCandidateItems) + } + } + + var retainedCandidates: [FixedWorkerCompactChild] { + sortedRetainedCandidates(from: retainedCandidateItems).map { + FixedWorkerCompactChild( + id: $0.metadata.id, + displaySize: $0.displaySize, + name: $0.metadata.name, + retainedTreeCount: $0.retainedTreeCount + ) + } + } + + func summary(for id: Int) -> FixedWorkerItemSummary? { + retainedCandidateItems.first { $0.metadata.id == id } + } + + private func sortedRetainedCandidates(from items: [FixedWorkerItemSummary]) -> [FixedWorkerItemSummary] { + Array( + items.sorted { lhs, rhs in + if lhs.displaySize == rhs.displaySize { + return lhs.metadata.name < rhs.metadata.name + } + return lhs.displaySize > rhs.displaySize + } + .prefix(retainedCandidateLimit) + ) + } +} + +private enum FixedWorkerWalkerError: Error { + case workerFailed +} + +private final class FixedWorkerFailure: @unchecked Sendable { + private let lock = NSLock() + private var recorded = false + + func record() { + lock.lock() + recorded = true + lock.unlock() + } + + var hasFailure: Bool { + lock.lock() + defer { lock.unlock() } + return recorded + } +} + +private final class FixedWorkerRecordIDSource: @unchecked Sendable { + private let lock = NSLock() + private var nextID = 1 + + func allocate(count: Int) -> [Int] { + guard count > 0 else { return [] } + lock.lock() + defer { lock.unlock() } + let ids = Array(nextID..<(nextID + count)) + nextID += count + return ids + } +} + +private final class FixedWorkerDirectoryFrontier: @unchecked Sendable { + private let condition = NSCondition() + private let capacity: Int + private var pending: [FixedWorkerDirectoryJob] + private var activeWorkerSlots = 0 + private var finished = false + private var cancelled = false + + init(root: FixedWorkerDirectoryJob, capacity: Int) { + self.capacity = max(1, capacity) + self.pending = [root] + } + + func pop(cancellation: ScanCancellation?) -> FixedWorkerDirectoryJob? { + condition.lock() + while pending.isEmpty && !finished && !cancelled { + if cancellation?.isCancelled ?? false { + cancelled = true + condition.broadcast() + break + } + condition.wait() + } + + guard !cancelled, !finished, !pending.isEmpty else { + condition.unlock() + return nil + } + + activeWorkerSlots += 1 + let job = pending.removeLast() + condition.unlock() + return job + } + + func tryEnqueue(_ job: FixedWorkerDirectoryJob) -> Bool { + condition.lock() + defer { condition.unlock() } + guard !cancelled, !finished, pending.count < capacity else { + return false + } + pending.append(job) + condition.broadcast() + return true + } + + func complete(preserveWorkerSlot: Bool) { + condition.lock() + if !preserveWorkerSlot { + activeWorkerSlots = max(0, activeWorkerSlots - 1) + } + if activeWorkerSlots == 0 && pending.isEmpty { + finished = true + } + condition.broadcast() + condition.unlock() + } + + func cancel() { + condition.lock() + cancelled = true + condition.broadcast() + condition.unlock() + } +} + +private final class FixedWorkerRecordChannel: @unchecked Sendable { + private let condition = NSCondition() + private let capacity: Int + private var buffer: [FixedWorkerDirectoryRecord] = [] + private var producersFinished = false + private var cancelled = false + + init(capacity: Int) { + self.capacity = max(1, capacity) + buffer.reserveCapacity(self.capacity) + } + + func send(_ record: FixedWorkerDirectoryRecord, cancellation: ScanCancellation?) -> Bool { + condition.lock() + while buffer.count >= capacity && !producersFinished && !cancelled { + if cancellation?.isCancelled ?? false { + cancelled = true + condition.broadcast() + break + } + condition.wait() + } + + guard !cancelled, !producersFinished, !(cancellation?.isCancelled ?? false) else { + condition.unlock() + return false + } + + buffer.append(record) + condition.signal() + condition.unlock() + return true + } + + func receive(cancellation: ScanCancellation?) -> FixedWorkerDirectoryRecord? { + condition.lock() + while buffer.isEmpty && !producersFinished && !cancelled { + if cancellation?.isCancelled ?? false { + cancelled = true + condition.broadcast() + break + } + condition.wait() + } + + guard !cancelled, !buffer.isEmpty else { + condition.unlock() + return nil + } + + let record = buffer.removeLast() + condition.signal() + condition.unlock() + return record + } + + func finishProducers() { + condition.lock() + producersFinished = true + condition.broadcast() + condition.unlock() + } + + func cancel() { + condition.lock() + cancelled = true + condition.broadcast() + condition.unlock() + } +} + +private final class FixedWorkerRecordCollector: @unchecked Sendable { + private let queue = DispatchQueue(label: "com.rasputinkaiser.StorageScope.fixed-worker-records") + private var records: [FixedWorkerDirectoryRecord] = [] + + func append(_ record: FixedWorkerDirectoryRecord) { + queue.sync { + records.append(record) + } + } + + func allRecords() -> [FixedWorkerDirectoryRecord] { + queue.sync { + records + } + } +} + +private final class FixedWorkerDirectoryWalker: @unchecked Sendable { + private let fileManager: FileManager + private let resourceKeys: Set + private let options: ScanOptions + private let cancellation: ScanCancellation? + private let shouldExclude: (URL) -> Bool + + init( + fileManager: FileManager, + resourceKeys: Set, + options: ScanOptions, + cancellation: ScanCancellation?, + shouldExclude: @escaping (URL) -> Bool + ) { + self.fileManager = fileManager + self.resourceKeys = resourceKeys.union([ + .fileResourceIdentifierKey, + .volumeIdentifierKey + ]) + self.options = options + self.cancellation = cancellation + self.shouldExclude = shouldExclude + } + + func walk(root: URL) throws -> FixedWorkerWalkResult { + guard let rootSeed = metadataSeed(for: root) else { + return FixedWorkerWalkResult(rootMetadata: nil, directoryRecords: []) + } + + let rootMetadata = makeRecord(rootSeed, id: 0, parentID: 0) + guard rootMetadata.isDirectory else { + return FixedWorkerWalkResult(rootMetadata: rootMetadata, directoryRecords: []) + } + + let workerCount = configuredWorkerCount() + let recordIDSource = FixedWorkerRecordIDSource() + let frontier = FixedWorkerDirectoryFrontier( + root: FixedWorkerDirectoryJob(metadata: rootMetadata, url: root), + capacity: workerCount * 8 + ) + let channel = FixedWorkerRecordChannel(capacity: workerCount * 2) + let collector = FixedWorkerRecordCollector() + let failure = FixedWorkerFailure() + let consumerGroup = DispatchGroup() + + consumerGroup.enter() + DispatchQueue.global(qos: .userInitiated).async { + while let record = channel.receive(cancellation: self.cancellation) { + collector.append(record) + } + consumerGroup.leave() + } + + DispatchQueue.concurrentPerform(iterations: workerCount) { _ in + var localJobs: [FixedWorkerDirectoryJob] = [] + + while true { + self.cancellation?.waitIfPaused() + if self.cancellation?.isCancelled ?? false { + frontier.cancel() + channel.cancel() + return + } + + let job: FixedWorkerDirectoryJob + if let localJob = localJobs.popLast() { + job = localJob + } else { + guard let nextJob = frontier.pop(cancellation: self.cancellation) else { + return + } + job = nextJob + } + + do { + let record = try autoreleasepool { + try self.enumerate(job, recordIDSource: recordIDSource) + } + guard channel.send(record, cancellation: self.cancellation) else { + frontier.cancel() + channel.cancel() + frontier.complete(preserveWorkerSlot: false) + return + } + + for child in record.children where child.isDirectory { + let childJob = FixedWorkerDirectoryJob( + metadata: child, + url: job.url.appendingPathComponent(child.name, isDirectory: true) + ) + if !frontier.tryEnqueue(childJob) { + localJobs.append(childJob) + } + } + frontier.complete(preserveWorkerSlot: !localJobs.isEmpty) + } catch FileSystemScannerError.cancelled { + frontier.cancel() + channel.cancel() + frontier.complete(preserveWorkerSlot: false) + return + } catch { + // Filesystem failures are converted into inaccessible records. + // Keep this guard for future worker-only failures. + failure.record() + frontier.cancel() + channel.cancel() + frontier.complete(preserveWorkerSlot: false) + return + } + } + } + + channel.finishProducers() + consumerGroup.wait() + try cancellation?.check() + if failure.hasFailure { + throw FixedWorkerWalkerError.workerFailed + } + + return FixedWorkerWalkResult( + rootMetadata: rootMetadata, + directoryRecords: collector.allRecords() + ) + } + + private func enumerate( + _ job: FixedWorkerDirectoryJob, + recordIDSource: FixedWorkerRecordIDSource + ) throws -> FixedWorkerDirectoryRecord { + try cancellation?.check() + + let childURLs: [URL] + do { + let directoryOptions: FileManager.DirectoryEnumerationOptions = options.includeHidden ? [] : [.skipsHiddenFiles] + childURLs = try fileManager.contentsOfDirectory( + at: job.url, + includingPropertiesForKeys: Array(resourceKeys), + options: directoryOptions + ) + } catch FileSystemScannerError.cancelled { + throw FileSystemScannerError.cancelled + } catch { + return FixedWorkerDirectoryRecord( + metadata: job.metadata, + children: [], + isInaccessible: true + ) + } + + var childSeeds: [FixedWorkerMetadataSeed] = [] + childSeeds.reserveCapacity(childURLs.count) + for childURL in childURLs.sorted(by: { $0.lastPathComponent < $1.lastPathComponent }) { + try cancellation?.check() + if shouldExclude(childURL) { + continue + } + guard let seed = metadataSeed(for: childURL) else { + continue + } + childSeeds.append(seed) + } + + let childIDs = recordIDSource.allocate(count: childSeeds.count) + let children = zip(childIDs, childSeeds).map { id, seed in + makeRecord(seed, id: id, parentID: job.metadata.id) + } + + return FixedWorkerDirectoryRecord( + metadata: job.metadata, + children: children, + isInaccessible: false + ) + } + + private func metadataSeed(for url: URL) -> FixedWorkerMetadataSeed? { + do { + let values = try url.resourceValues(forKeys: resourceKeys) + if values.isHidden == true && !options.includeHidden { + return nil + } + + let isSymbolicLink = values.isSymbolicLink == true + let isDirectory = !isSymbolicLink && values.isDirectory == true + let kind: StorageItem.Kind + if isSymbolicLink { + kind = .alias + } else if isDirectory { + kind = values.isPackage == true ? .package : .folder + } else { + kind = values.isRegularFile == true ? .file : .other + } + + let logicalSize = Int64(values.fileSize ?? 0) + let allocatedSize = Int64( + values.totalFileAllocatedSize ?? + values.fileAllocatedSize ?? + values.fileSize ?? + 0 + ) + return FixedWorkerMetadataSeed( + name: url.lastPathComponent, + kind: kind, + byteSize: logicalSize, + allocatedSize: allocatedSize, + modifiedAt: values.contentModificationDate, + isReadable: values.isReadable ?? true, + volumeIdentifier: resourceIdentifierValue(values.volumeIdentifier), + fileResourceIdentifier: resourceIdentifierValue(values.fileResourceIdentifier), + hardLinkCount: 0 + ) + } catch { + return FixedWorkerMetadataSeed( + name: url.lastPathComponent, + kind: .other, + byteSize: 0, + allocatedSize: 0, + modifiedAt: nil, + isReadable: true, + volumeIdentifier: .zero, + fileResourceIdentifier: .zero, + hardLinkCount: 0 + ) + } + } + + private func makeRecord( + _ seed: FixedWorkerMetadataSeed, + id: Int, + parentID: Int + ) -> FixedWorkerWalkRecord { + FixedWorkerWalkRecord( + id: id, + parentID: parentID, + name: seed.name, + kind: seed.kind, + byteSize: seed.byteSize, + allocatedSize: seed.allocatedSize, + modifiedAt: seed.modifiedAt, + isReadable: seed.isReadable, + volumeIdentifier: seed.volumeIdentifier, + fileResourceIdentifier: seed.fileResourceIdentifier, + hardLinkCount: seed.hardLinkCount + ) + } + + private func resourceIdentifierValue(_ value: Any?) -> FixedWorkerResourceIdentifier { + if let number = value as? NSNumber { + return FixedWorkerResourceIdentifier(low: number.uint64Value, high: 0) + } + guard let data = value as? NSData, + data.length >= MemoryLayout.size else { + return .zero + } + + var low = UInt64.zero + data.getBytes(&low, length: MemoryLayout.size) + guard data.length >= MemoryLayout.size * 2 else { + return FixedWorkerResourceIdentifier(low: low, high: 0) + } + + var high = UInt64.zero + data.getBytes(&high, range: NSRange( + location: MemoryLayout.size, + length: MemoryLayout.size + )) + return FixedWorkerResourceIdentifier(low: low, high: high) + } + + private func configuredWorkerCount() -> Int { + if let raw = ProcessInfo.processInfo.environment["STORAGESCOPE_WORKER_COUNT"], + let value = Int(raw), + (1...16).contains(value) { + return value + } + return min(8, max(4, ProcessInfo.processInfo.activeProcessorCount)) + } +} + +private struct HashedStorageItem { + let checksum: String + let item: StorageItem +} + +private struct PrefixHashedStorageItem { + let prefixChecksum: String + let bytesRead: Int + let item: StorageItem + + var isCompleteFile: Bool { + Int64(bytesRead) >= item.byteSize + } +} + +private struct DirectoryScanSummary { + private let retainedCandidateLimit: Int + private var retainedCandidateItems: [StorageItem] = [] + + var logicalSize: Int64 = 0 + var allocatedSize: Int64 = 0 + var immediateChildCount = 0 + var descendantCount = 0 + + init(retainedCandidateLimit: Int) { + self.retainedCandidateLimit = max(0, retainedCandidateLimit) + } + + mutating func record(_ child: StorageItem) { + logicalSize += child.byteSize allocatedSize += child.allocatedSize immediateChildCount += 1 descendantCount += 1 + child.descendantCount @@ -752,7 +1859,7 @@ private struct DirectoryScanSummary { Array( items.sorted { lhs, rhs in if lhs.displaySize == rhs.displaySize { - return lhs.name.localizedStandardCompare(rhs.name) == .orderedAscending + return lhs.name < rhs.name } return lhs.displaySize > rhs.displaySize } @@ -761,6 +1868,98 @@ private struct DirectoryScanSummary { } } +private struct FixedWorkerDuplicateCandidateReference: Sendable { + let summary: FixedWorkerItemSummary + let byteSize: Int64 + let url: URL +} + +private struct FixedWorkerDuplicateCandidateRetention { + private let limit: Int + private var candidatesBySize: [Int64: [FixedWorkerDuplicateCandidateReference]] = [:] + private var smallestRetainedSize: Int64? + + private(set) var retainedCount = 0 + private(set) var consideredCount = 0 + private(set) var evictionCount = 0 + private(set) var limitReached = false + + init(limit: Int) { + self.limit = max(0, limit) + } + + mutating func record(_ summary: FixedWorkerItemSummary, url: URL) { + consideredCount += 1 + guard limit > 0 else { + limitReached = true + return + } + + if retainedCount >= limit, + let smallestRetainedSize, + summary.metadata.byteSize < smallestRetainedSize { + limitReached = true + return + } + + candidatesBySize[summary.metadata.byteSize, default: []].append( + FixedWorkerDuplicateCandidateReference( + summary: summary, + byteSize: summary.metadata.byteSize, + url: url + ) + ) + retainedCount += 1 + if smallestRetainedSize.map({ summary.metadata.byteSize < $0 }) ?? true { + smallestRetainedSize = summary.metadata.byteSize + } + + guard retainedCount > limit else { + return + } + + limitReached = true + evictSmallestRetained() + } + + func sizeGroups(using summaryStore: FixedWorkerSummaryStore) -> [DuplicateSizeGroup] { + candidatesBySize + .compactMap { byteSize, references in + guard references.count > 1 else { return nil } + let items = references + .map { summaryStore.makeItem(summary: $0.summary, urlOverride: $0.url) } + .sorted { $0.url.path < $1.url.path } + return DuplicateSizeGroup(byteSize: byteSize, items: items) + } + .sorted { lhs, rhs in + if lhs.totalBytes == rhs.totalBytes { + return lhs.byteSize > rhs.byteSize + } + return lhs.totalBytes > rhs.totalBytes + } + } + + private mutating func evictSmallestRetained() { + guard let smallestByteSize = smallestRetainedSize ?? candidatesBySize.keys.min(), + var items = candidatesBySize[smallestByteSize], + !items.isEmpty else { + smallestRetainedSize = candidatesBySize.keys.min() + return + } + + items.removeLast() + retainedCount -= 1 + evictionCount += 1 + + if items.isEmpty { + candidatesBySize.removeValue(forKey: smallestByteSize) + smallestRetainedSize = candidatesBySize.keys.min() + } else { + candidatesBySize[smallestByteSize] = items + } + } +} + private extension CleanupCandidate.Confidence { var sortRank: Int { switch self { @@ -772,71 +1971,360 @@ private extension CleanupCandidate.Confidence { return 2 } } -} +} + +struct DuplicateCandidateRetention { + private let limit: Int + private var candidatesBySize: [Int64: [StorageItem]] = [:] + private var smallestRetainedSize: Int64? + + private(set) var retainedCount = 0 + private(set) var consideredCount = 0 + private(set) var evictionCount = 0 + private(set) var limitReached = false + + init(limit: Int) { + self.limit = max(0, limit) + } + + var sizeGroups: [DuplicateSizeGroup] { + candidatesBySize + .compactMap { byteSize, items in + items.count > 1 ? DuplicateSizeGroup(byteSize: byteSize, items: items.sorted { $0.url.path < $1.url.path }) : nil + } + .sorted { lhs, rhs in + if lhs.totalBytes == rhs.totalBytes { + return lhs.byteSize > rhs.byteSize + } + return lhs.totalBytes > rhs.totalBytes + } + } + + mutating func record(_ item: StorageItem) { + consideredCount += 1 + guard limit > 0 else { + limitReached = true + return + } + + if retainedCount >= limit, + let smallestRetainedSize, + item.byteSize < smallestRetainedSize { + limitReached = true + return + } + + candidatesBySize[item.byteSize, default: []].append(item) + retainedCount += 1 + if smallestRetainedSize.map({ item.byteSize < $0 }) ?? true { + smallestRetainedSize = item.byteSize + } + + guard retainedCount > limit else { + return + } + + limitReached = true + evictSmallestRetained() + } + + private mutating func evictSmallestRetained() { + guard let smallestByteSize = smallestRetainedSize ?? candidatesBySize.keys.min(), + var items = candidatesBySize[smallestByteSize], + !items.isEmpty else { + smallestRetainedSize = candidatesBySize.keys.min() + return + } + + items.removeLast() + retainedCount -= 1 + evictionCount += 1 + + if items.isEmpty { + candidatesBySize.removeValue(forKey: smallestByteSize) + smallestRetainedSize = candidatesBySize.keys.min() + } else { + candidatesBySize[smallestByteSize] = items + } + } +} + +private final class ScanAccumulator { + /// Shares the FileSystemScanner signpost subsystem/category so cleanup-candidate + /// build shows up alongside the scan/enumerate/verify spans when profiling. + private static let log = OSLog(subsystem: "com.rasputinkaiser.StorageScope", category: "scan") + private static let signpostID = OSSignpostID(log: log) + + private struct FileTypeAccumulator { + var category: FileTypeStat.Category = .other + var fileCount = 0 + var totalBytes: Int64 = 0 + } + + private let options: ScanOptions + private let progress: FileSystemScanner.ProgressHandler? + private let onSnapshot: ((StorageScan) -> Void)? + private let rootURL: URL + private let startedAt: Date + private var lastProgressDate = Date.distantPast + private var lastSnapshotDate = Date.distantPast + private let oldFileCutoff: Date + private var retainedItemCount: Int + private var largestFileItems: [StorageItem] = [] + private var largestFolderItems: [StorageItem] = [] + private var oldLargeFileItems: [StorageItem] = [] + private var fileTypeStats: [String: FileTypeAccumulator] = [:] + private var duplicateCandidateRetention: DuplicateCandidateRetention + private var cleanupCandidatesByID: [String: CleanupCandidate] = [:] + private var fixedWorkerSummaryStore: FixedWorkerSummaryStore? + private var fixedWorkerLargestFileItems: [FixedWorkerRankedReference] = [] + private var fixedWorkerLargestFolderItems: [FixedWorkerRankedReference] = [] + private var fixedWorkerOldLargeFileItems: [FixedWorkerRankedReference] = [] + private var fixedWorkerCleanupCandidatesByID: [Int: FixedWorkerCleanupCandidateReference] = [:] + private var fixedWorkerDuplicateCandidateRetention: FixedWorkerDuplicateCandidateRetention? + + var scannedItemCount = 0 + var inaccessibleItemCount = 0 + var totalBytes: Int64 = 0 + private(set) var duplicateVerificationBytesRead: Int64 = 0 + private(set) var snapshotBuildCount = 0 + + /// Guards every mutable field above. Held briefly during directory enumeration's + /// record*() calls; the user `progress` and `onSnapshot` closures are both invoked + /// under this lock, so neither must re-enter the accumulator or block for long — every + /// concurrentPerform worker thread recording an item serializes on whichever thread is + /// currently inside one of these callbacks. Contention is bounded today because both + /// callbacks only do cheap NSLock bookkeeping on the ScanStore side; a callback that + /// becomes non-trivial (e.g. blocking on a busy MainActor) would serialize the whole + /// scan on this lock. + private let lock = NSLock() + + init( + options: ScanOptions, + progress: FileSystemScanner.ProgressHandler?, + onSnapshot: ((StorageScan) -> Void)? = nil, + rootURL: URL = URL(fileURLWithPath: "/"), + startedAt: Date = Date() + ) { + self.options = options + self.progress = progress + self.onSnapshot = onSnapshot + self.rootURL = rootURL + self.startedAt = startedAt + self.retainedItemCount = 1 + self.duplicateCandidateRetention = DuplicateCandidateRetention(limit: options.maxDuplicateCandidateItems) + self.oldFileCutoff = Calendar.current.date( + byAdding: .day, + value: -options.oldFileAgeDays, + to: Date() + ) ?? Date() + } + + func configureFixedWorkerStore(_ summaryStore: FixedWorkerSummaryStore) { + fixedWorkerSummaryStore = summaryStore + fixedWorkerDuplicateCandidateRetention = FixedWorkerDuplicateCandidateRetention( + limit: options.maxDuplicateCandidateItems + ) + } + + func recordFixedWorkerScannedItem( + _ summary: FixedWorkerItemSummary, + url: URL + ) { + // Fixed-worker reconstruction runs after the worker walk has completed, so this + // accumulator is owned by the reconstruction thread until scan() publishes its + // final StorageScan. Avoid taking the legacy per-item lock in that serial phase. + scannedItemCount += 1 + totalBytes += max(summary.logicalSize, summary.allocatedSize) + recordFixedWorkerItemLocked(summary, url: url) + emitProgressLocked(path: url.path) + } + + func recordFixedWorkerItem( + _ summary: FixedWorkerItemSummary, + url: URL + ) { + recordFixedWorkerItemLocked(summary, url: url) + } + + private func recordFixedWorkerItemLocked( + _ summary: FixedWorkerItemSummary, + url: URL + ) { + switch summary.kind { + case .file: + insertFixedWorkerRankedReference( + for: summary, + url: url, + into: &fixedWorkerLargestFileItems + ) + + if let modifiedAt = summary.metadata.modifiedAt, + summary.displaySize >= options.largeFileThreshold, + modifiedAt <= oldFileCutoff { + insertFixedWorkerRankedReference( + for: summary, + url: url, + into: &fixedWorkerOldLargeFileItems + ) + } + + let typeLabel: String + if let fileExtension = summary.fileExtension, !fileExtension.isEmpty { + typeLabel = ".\(fileExtension)" + } else { + typeLabel = "No Extension" + } + + var typeStat = fileTypeStats[typeLabel] ?? FileTypeAccumulator() + typeStat.category = FileTypeCategoryClassifier.category(forExtension: summary.fileExtension) + typeStat.fileCount += 1 + typeStat.totalBytes += summary.displaySize + fileTypeStats[typeLabel] = typeStat + + if summary.metadata.byteSize >= options.duplicateCandidateThreshold { + fixedWorkerDuplicateCandidateRetention?.record(summary, url: url) + } + case .folder, .package: + insertFixedWorkerRankedReference( + for: summary, + url: url, + into: &fixedWorkerLargestFolderItems + ) + case .alias, .inaccessible, .other: + break + } + + if let candidate = fixedWorkerCleanupCandidate(for: summary, url: url) { + insertFixedWorkerCleanupCandidateLocked(candidate) + } + } + + private func insertFixedWorkerRankedReference( + for summary: FixedWorkerItemSummary, + url: URL, + into references: inout [FixedWorkerRankedReference] + ) { + let limit = max(0, options.maxRankedResults) + guard limit > 0 else { + references.removeAll(keepingCapacity: false) + return + } + + if references.count < limit { + references.append( + FixedWorkerRankedReference( + summary: summary, + displaySize: summary.displaySize, + url: url, + path: url.path + ) + ) + siftUpFixedWorkerRankedReferences(&references) + return + } + + let worst = references[0] + if summary.displaySize < worst.displaySize { + return + } + + let path: String + if summary.displaySize == worst.displaySize { + path = url.path + guard path < worst.path else { + return + } + } else { + path = url.path + } + + let reference = FixedWorkerRankedReference( + summary: summary, + displaySize: summary.displaySize, + url: url, + path: path + ) + references[0] = reference + siftDownFixedWorkerRankedReferences(&references) + } -private final class ScanAccumulator { - /// Shares the FileSystemScanner signpost subsystem/category so cleanup-candidate - /// build shows up alongside the scan/enumerate/verify spans when profiling. - private static let log = OSLog(subsystem: "com.rasputinkaiser.StorageScope", category: "scan") - private static let signpostID = OSSignpostID(log: log) + private func fixedWorkerReferenceIsBetter( + _ lhs: FixedWorkerRankedReference, + _ rhs: FixedWorkerRankedReference + ) -> Bool { + if lhs.displaySize == rhs.displaySize { + return lhs.path < rhs.path + } + return lhs.displaySize > rhs.displaySize + } - private struct FileTypeAccumulator { - var category: FileTypeStat.Category = .other - var fileCount = 0 - var totalBytes: Int64 = 0 + private func fixedWorkerReferenceIsWorse( + _ lhs: FixedWorkerRankedReference, + _ rhs: FixedWorkerRankedReference + ) -> Bool { + if lhs.displaySize == rhs.displaySize { + return lhs.path > rhs.path + } + return lhs.displaySize < rhs.displaySize } - private let options: ScanOptions - private let progress: FileSystemScanner.ProgressHandler? - private let onSnapshot: ((StorageScan) -> Void)? - private let rootURL: URL - private let startedAt: Date - private var lastProgressDate = Date.distantPast - private let oldFileCutoff: Date - private var retainedItemCount: Int - private var largestFileItems: [StorageItem] = [] - private var largestFolderItems: [StorageItem] = [] - private var oldLargeFileItems: [StorageItem] = [] - private var fileTypeStats: [String: FileTypeAccumulator] = [:] - private var duplicateCandidatesBySize: [Int64: [StorageItem]] = [:] - private var duplicateCandidateItemCount = 0 - private var duplicateCandidateConsideredCount = 0 - private var smallestDuplicateCandidateByteSize: Int64? - private(set) var duplicateCandidateLimitReached = false - private var cleanupCandidatesByID: [String: CleanupCandidate] = [:] + private func siftUpFixedWorkerRankedReferences( + _ references: inout [FixedWorkerRankedReference] + ) { + var childIndex = references.index(before: references.endIndex) + while childIndex > references.startIndex { + let parentIndex = (childIndex - references.startIndex - 1) / 2 + references.startIndex + guard fixedWorkerReferenceIsWorse(references[childIndex], references[parentIndex]) else { + break + } + references.swapAt(childIndex, parentIndex) + childIndex = parentIndex + } + } - var scannedItemCount = 0 - var inaccessibleItemCount = 0 - var totalBytes: Int64 = 0 + private func siftDownFixedWorkerRankedReferences( + _ references: inout [FixedWorkerRankedReference] + ) { + var parentIndex = references.startIndex + while true { + let relativeParentIndex = parentIndex - references.startIndex + let leftIndex = relativeParentIndex * 2 + 1 + references.startIndex + guard leftIndex < references.endIndex else { + return + } - /// Guards every mutable field above. Held briefly during directory enumeration's - /// record*() calls; the user `progress` and `onSnapshot` closures are both invoked - /// under this lock, so neither must re-enter the accumulator or block for long — every - /// concurrentPerform worker thread recording an item serializes on whichever thread is - /// currently inside one of these callbacks. Contention is bounded today because both - /// callbacks only do cheap NSLock bookkeeping on the ScanStore side; a callback that - /// becomes non-trivial (e.g. blocking on a busy MainActor) would serialize the whole - /// scan on this lock. - private let lock = NSLock() + var worstIndex = parentIndex + if fixedWorkerReferenceIsWorse(references[leftIndex], references[worstIndex]) { + worstIndex = leftIndex + } - init( - options: ScanOptions, - progress: FileSystemScanner.ProgressHandler?, - onSnapshot: ((StorageScan) -> Void)? = nil, - rootURL: URL = URL(fileURLWithPath: "/"), - startedAt: Date = Date() + let rightIndex = leftIndex + 1 + if rightIndex < references.endIndex, + fixedWorkerReferenceIsWorse(references[rightIndex], references[worstIndex]) { + worstIndex = rightIndex + } + + guard worstIndex != parentIndex else { + return + } + references.swapAt(parentIndex, worstIndex) + parentIndex = worstIndex + } + } + + private func insertFixedWorkerCleanupCandidateLocked( + _ candidate: FixedWorkerCleanupCandidateReference ) { - self.options = options - self.progress = progress - self.onSnapshot = onSnapshot - self.rootURL = rootURL - self.startedAt = startedAt - self.retainedItemCount = 1 - self.oldFileCutoff = Calendar.current.date( - byAdding: .day, - value: -options.oldFileAgeDays, - to: Date() - ) ?? Date() + let id = candidate.summary.metadata.id + if let existing = fixedWorkerCleanupCandidatesByID[id] { + if candidate.confidence.sortRank < existing.confidence.sortRank || + candidate.reclaimableBytes > existing.reclaimableBytes { + fixedWorkerCleanupCandidatesByID[id] = candidate + } + } else { + fixedWorkerCleanupCandidatesByID[id] = candidate + } } func recordVisit(path: String) { @@ -846,12 +2334,6 @@ private final class ScanAccumulator { emitProgressLocked(path: path) } - func recordBytes(_ bytes: Int64) { - lock.lock() - defer { lock.unlock() } - totalBytes += bytes - } - func recordInaccessible(path: String) { lock.lock() defer { lock.unlock() } @@ -859,15 +2341,35 @@ private final class ScanAccumulator { emitProgressLocked(path: path) } + func recordDuplicateVerificationBytes(_ bytes: Int) { + guard bytes > 0 else { return } + lock.lock() + duplicateVerificationBytesRead += Int64(bytes) + lock.unlock() + } + func recordPhase(path: String, phase: ScanPhase = .enumerating) { lock.lock() defer { lock.unlock() } emitProgressLocked(path: path, force: true, phase: phase) } + func recordScannedItem(_ item: StorageItem, countedBytes: Int64, path: String) { + lock.lock() + defer { lock.unlock() } + scannedItemCount += 1 + totalBytes += countedBytes + recordItemLocked(item) + emitProgressLocked(path: path) + } + func recordItem(_ item: StorageItem) { lock.lock() defer { lock.unlock() } + recordItemLocked(item) + } + + private func recordItemLocked(_ item: StorageItem) { switch item.kind { case .file: recordFileLocked(item) @@ -932,18 +2434,77 @@ private final class ScanAccumulator { return retained } + func retainedFixedWorkerChildren( + from children: [FixedWorkerCompactChild], + depth: Int + ) -> [FixedWorkerRetainedSelection] { + let perDirectoryLimit = max(0, options.maxChildrenPerDirectory) + guard perDirectoryLimit > 0 else { + return [] + } + + let retainedLimit = max(1, options.maxRetainedItems) + let isGuaranteedDepth = depth == 0 + + guard isGuaranteedDepth || retainedItemCount < retainedLimit else { + return [] + } + + var retained: [FixedWorkerRetainedSelection] = [] + retained.reserveCapacity(min(children.count, perDirectoryLimit)) + + for child in children.prefix(perDirectoryLimit) { + guard isGuaranteedDepth || retainedItemCount < retainedLimit else { + break + } + + let fullCount = child.retainedTreeCount + if retainedItemCount + fullCount <= retainedLimit { + retained.append( + FixedWorkerRetainedSelection( + id: child.id, + includesDescendants: true, + retainedTreeCount: fullCount + ) + ) + retainedItemCount += fullCount + } else { + retained.append( + FixedWorkerRetainedSelection( + id: child.id, + includesDescendants: false, + retainedTreeCount: 1 + ) + ) + retainedItemCount += 1 + } + } + + return retained + } + var largestFiles: [StorageItem] { - sortedRankedItems(largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + if let summaryStore = fixedWorkerSummaryStore { + return materializedFixedWorkerRankedItems(fixedWorkerLargestFileItems, summaryStore: summaryStore) + } + return sortedRankedItems(largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } } func largestFolders(excluding rootID: String) -> [StorageItem] { - sortedRankedItems(largestFolderItems.filter { $0.id != rootID }, limit: options.maxRankedResults) { + if let summaryStore = fixedWorkerSummaryStore { + return materializedFixedWorkerRankedItems(fixedWorkerLargestFolderItems, summaryStore: summaryStore) + .filter { $0.id != rootID } + } + return sortedRankedItems(largestFolderItems.filter { $0.id != rootID }, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } } var oldLargeFiles: [StorageItem] { - sortedRankedItems(oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } + if let summaryStore = fixedWorkerSummaryStore { + return materializedFixedWorkerRankedItems(fixedWorkerOldLargeFileItems, summaryStore: summaryStore) + } + return sortedRankedItems(oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize } } var typeBreakdown: [FileTypeStat] { @@ -986,16 +2547,11 @@ private final class ScanAccumulator { } var duplicateSizeGroups: [DuplicateSizeGroup] { - duplicateCandidatesBySize - .compactMap { byteSize, items in - items.count > 1 ? DuplicateSizeGroup(byteSize: byteSize, items: items.sorted { $0.url.path < $1.url.path }) : nil - } - .sorted { lhs, rhs in - if lhs.totalBytes == rhs.totalBytes { - return lhs.byteSize > rhs.byteSize - } - return lhs.totalBytes > rhs.totalBytes - } + if let summaryStore = fixedWorkerSummaryStore, + let retention = fixedWorkerDuplicateCandidateRetention { + return retention.sizeGroups(using: summaryStore) + } + return duplicateCandidateRetention.sizeGroups } var duplicateCandidateItemLimit: Int { @@ -1003,11 +2559,45 @@ private final class ScanAccumulator { } var duplicateCandidateItemsRetained: Int { - duplicateCandidateItemCount + if let retention = fixedWorkerDuplicateCandidateRetention { + return retention.retainedCount + } + return duplicateCandidateRetention.retainedCount } var duplicateCandidateItemsConsidered: Int { - duplicateCandidateConsideredCount + if let retention = fixedWorkerDuplicateCandidateRetention { + return retention.consideredCount + } + return duplicateCandidateRetention.consideredCount + } + + var duplicateCandidateEvictionCount: Int { + if let retention = fixedWorkerDuplicateCandidateRetention { + return retention.evictionCount + } + return duplicateCandidateRetention.evictionCount + } + + var duplicateCandidateLimitReached: Bool { + if let retention = fixedWorkerDuplicateCandidateRetention { + return retention.limitReached + } + return duplicateCandidateRetention.limitReached + } + + private func materializedFixedWorkerRankedItems( + _ references: [FixedWorkerRankedReference], + summaryStore: FixedWorkerSummaryStore + ) -> [StorageItem] { + references + .sorted { lhs, rhs in + if lhs.displaySize == rhs.displaySize { + return lhs.path < rhs.path + } + return lhs.displaySize > rhs.displaySize + } + .map { summaryStore.makeItem(summary: $0.summary, urlOverride: $0.url) } } func cleanupCandidates( @@ -1023,8 +2613,27 @@ private final class ScanAccumulator { } let duplicateItemIDs = Set(verifiedDuplicateGroups.flatMap { group in group.items.dropFirst().map(\.id) }) - var candidatesByID = cleanupCandidatesByID.filter { id, _ in - id != rootID && !duplicateItemIDs.contains(id) + var candidatesByID: [String: CleanupCandidate] + if let summaryStore = fixedWorkerSummaryStore { + var fixedCandidatesByID: [String: CleanupCandidate] = [:] + for candidate in fixedWorkerCleanupCandidatesByID.values { + let item = summaryStore.makeItem(summary: candidate.summary, urlOverride: candidate.url) + guard item.id != rootID, !duplicateItemIDs.contains(item.id) else { + continue + } + fixedCandidatesByID[item.id] = CleanupCandidate( + kind: candidate.kind, + item: item, + reason: candidate.reason, + reclaimableBytes: candidate.reclaimableBytes, + confidence: candidate.confidence + ) + } + candidatesByID = fixedCandidatesByID + } else { + candidatesByID = cleanupCandidatesByID.filter { id, _ in + id != rootID && !duplicateItemIDs.contains(id) + } } for group in verifiedDuplicateGroups { @@ -1058,7 +2667,7 @@ private final class ScanAccumulator { private func emitProgressLocked(path: String, force: Bool = false, phase: ScanPhase = .enumerating) { let now = Date() - guard force || scannedItemCount == 1 || scannedItemCount.isMultiple(of: 25) || now.timeIntervalSince(lastProgressDate) > 0.35 else { + guard force || scannedItemCount == 1 || now.timeIntervalSince(lastProgressDate) > 0.35 else { return } @@ -1068,7 +2677,10 @@ private final class ScanAccumulator { // ScanTick. Firing progress first would pair each tick with the *previous* tick's // snapshot — a one-tick lag that leaves cancel-preserved partial results stale by // a full throttle interval. - if let onSnapshot { + let shouldBuildSnapshot = force || scannedItemCount == 1 || now.timeIntervalSince(lastSnapshotDate) > 1.0 + if let onSnapshot, shouldBuildSnapshot { + lastSnapshotDate = now + snapshotBuildCount += 1 onSnapshot(snapshotLocked()) } progress?(ScanProgress(scannedItemCount: scannedItemCount, totalBytes: totalBytes, currentPath: path, phase: phase)) @@ -1091,6 +2703,36 @@ private final class ScanAccumulator { descendantCount: scannedItemCount, isReadable: true ) + let snapshotLargestFiles: [StorageItem] + let snapshotLargestFolders: [StorageItem] + let snapshotOldLargeFiles: [StorageItem] + if let summaryStore = fixedWorkerSummaryStore { + snapshotLargestFiles = materializedFixedWorkerRankedItems( + fixedWorkerLargestFileItems, + summaryStore: summaryStore + ) + snapshotLargestFolders = materializedFixedWorkerRankedItems( + fixedWorkerLargestFolderItems, + summaryStore: summaryStore + ) + snapshotOldLargeFiles = materializedFixedWorkerRankedItems( + fixedWorkerOldLargeFileItems, + summaryStore: summaryStore + ) + } else { + snapshotLargestFiles = sortedRankedItems( + largestFileItems, + limit: options.maxRankedResults + ) { $0.displaySize > $1.displaySize } + snapshotLargestFolders = sortedRankedItems( + largestFolderItems, + limit: options.maxRankedResults + ) { $0.displaySize > $1.displaySize } + snapshotOldLargeFiles = sortedRankedItems( + oldLargeFileItems, + limit: options.maxRankedResults + ) { $0.displaySize > $1.displaySize } + } return StorageScan( rootURL: rootURL, startedAt: startedAt, @@ -1100,17 +2742,19 @@ private final class ScanAccumulator { scannedItemCount: scannedItemCount, inaccessibleItemCount: inaccessibleItemCount, totalBytes: totalBytes, - largestFiles: sortedRankedItems(largestFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, - largestFolders: sortedRankedItems(largestFolderItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, - oldLargeFiles: sortedRankedItems(oldLargeFileItems, limit: options.maxRankedResults) { $0.displaySize > $1.displaySize }, + largestFiles: snapshotLargestFiles, + largestFolders: snapshotLargestFolders, + oldLargeFiles: snapshotOldLargeFiles, typeBreakdown: typeBreakdown, categoryBreakdown: categoryBreakdown, duplicateSizeGroups: [], verifiedDuplicateGroups: [], duplicateCandidateItemLimit: duplicateCandidateItemLimit, - duplicateCandidateItemsRetained: duplicateCandidateItemCount, - duplicateCandidateItemsConsidered: duplicateCandidateConsideredCount, + duplicateCandidateItemsRetained: duplicateCandidateItemsRetained, + duplicateCandidateItemsConsidered: duplicateCandidateItemsConsidered, + duplicateCandidateEvictionCount: duplicateCandidateEvictionCount, duplicateCandidateLimitReached: duplicateCandidateLimitReached, + snapshotBuildCount: snapshotBuildCount, duplicateVerificationDuration: 0, enumerateDuration: Date().timeIntervalSince(startedAt), cleanupCandidates: [], @@ -1151,58 +2795,7 @@ private final class ScanAccumulator { fileTypeStats[typeLabel] = typeStat if item.byteSize >= options.duplicateCandidateThreshold { - recordDuplicateCandidateLocked(item) - } - } - - private func recordDuplicateCandidateLocked(_ item: StorageItem) { - duplicateCandidateConsideredCount += 1 - let candidateLimit = max(0, options.maxDuplicateCandidateItems) - guard candidateLimit > 0 else { - duplicateCandidateLimitReached = true - return - } - - if duplicateCandidateItemCount >= candidateLimit, - let smallestDuplicateCandidateByteSize, - item.byteSize < smallestDuplicateCandidateByteSize { - duplicateCandidateLimitReached = true - return - } - - duplicateCandidatesBySize[item.byteSize, default: []].append(item) - duplicateCandidateItemCount += 1 - if smallestDuplicateCandidateByteSize.map({ item.byteSize < $0 }) ?? true { - smallestDuplicateCandidateByteSize = item.byteSize - } - - guard duplicateCandidateItemCount > candidateLimit else { - return - } - - duplicateCandidateLimitReached = true - removeSmallestDuplicateCandidateLocked() - } - - private func removeSmallestDuplicateCandidateLocked() { - guard let smallestByteSize = smallestDuplicateCandidateByteSize ?? duplicateCandidatesBySize.keys.min(), - var items = duplicateCandidatesBySize[smallestByteSize], - !items.isEmpty else { - smallestDuplicateCandidateByteSize = duplicateCandidatesBySize.keys.min() - return - } - - let removalIndex = items.indices.max { lhs, rhs in - items[lhs].url.path.localizedStandardCompare(items[rhs].url.path) == .orderedAscending - } ?? items.startIndex - items.remove(at: removalIndex) - duplicateCandidateItemCount -= 1 - - if items.isEmpty { - duplicateCandidatesBySize.removeValue(forKey: smallestByteSize) - smallestDuplicateCandidateByteSize = duplicateCandidatesBySize.keys.min() - } else { - duplicateCandidatesBySize[smallestByteSize] = items + duplicateCandidateRetention.record(item) } } @@ -1290,6 +2883,103 @@ private final class ScanAccumulator { return nil } + private func fixedWorkerCleanupCandidate( + for summary: FixedWorkerItemSummary, + url: URL + ) -> FixedWorkerCleanupCandidateReference? { + let lowercasedName = summary.metadata.name.lowercased() + let path = url.path.lowercased() + let fileExtension = summary.fileExtension ?? "" + + if summary.kind == .folder || summary.kind == .package { + if ["cache", "caches", ".cache"].contains(lowercasedName) { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .cacheFolder, + reason: "Cache folder. Review before deleting if an app is currently using it.", + reclaimableBytes: summary.displaySize, + confidence: .medium + ) + } + + if ["deriveddata", "node_modules", ".build", "build", "dist", "target", ".gradle"].contains(lowercasedName) || + path.contains("/deriveddata/") { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .buildArtifact, + reason: "Build or dependency artifact. Usually rebuildable, but project-specific.", + reclaimableBytes: summary.displaySize, + confidence: .medium + ) + } + } + + guard summary.kind == .file else { + return nil + } + + if ["dmg", "iso", "toast", "sparsebundle", "sparseimage"].contains(fileExtension) { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .diskImage, + reason: "Disk image. Often disposable after installation or extraction.", + reclaimableBytes: summary.displaySize, + confidence: .medium + ) + } + + if ["pkg", "mpkg", "xip", "ipsw"].contains(fileExtension) { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .installer, + reason: "Installer package. Usually removable after the install is complete.", + reclaimableBytes: summary.displaySize, + confidence: .medium + ) + } + + if ["zip", "zipx", "rar", "7z", "tar", "tgz", "gz", "bz", "bz2", "tbz", "tbz2", "xz", "txz", "zst", "tzst", "lz4"].contains(fileExtension) { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .archive, + reason: "Archive file. Review whether the extracted copy already exists.", + reclaimableBytes: summary.displaySize, + confidence: .review + ) + } + + if ["tmp", "temp", "bak", "old"].contains(fileExtension) || lowercasedName.hasSuffix("~") { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .temporary, + reason: "Temporary or backup-looking file.", + reclaimableBytes: summary.displaySize, + confidence: .review + ) + } + + if let modifiedAt = summary.metadata.modifiedAt, + summary.displaySize >= 1_000_000_000, + modifiedAt <= oldFileCutoff { + return FixedWorkerCleanupCandidateReference( + summary: summary, + url: url, + kind: .oldLargeFile, + reason: "Large file not modified recently.", + reclaimableBytes: summary.displaySize, + confidence: .review + ) + } + + return nil + } + private func insertCandidateLocked(_ candidate: CleanupCandidate, into candidatesByID: inout [String: CleanupCandidate]) { if let existing = candidatesByID[candidate.item.id] { if candidate.confidence.sortRank < existing.confidence.sortRank || diff --git a/Sources/StorageScopeCore/Services/ScanBenchmark.swift b/Sources/StorageScopeCore/Services/ScanBenchmark.swift index ac1e75c..23a4091 100644 --- a/Sources/StorageScopeCore/Services/ScanBenchmark.swift +++ b/Sources/StorageScopeCore/Services/ScanBenchmark.swift @@ -3,6 +3,7 @@ import os public struct ScanBenchmarkReport: Hashable, Sendable { public let scopeLabel: String + public let buildConfiguration: String public let duration: TimeInterval public let scannedItemCount: Int public let inaccessibleItemCount: Int @@ -11,9 +12,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { public let largestFolderCount: Int public let duplicateCandidateItemsConsidered: Int public let duplicateCandidateItemsRetained: Int + public let duplicateCandidateEvictionCount: Int public let duplicateCandidateLimitReached: Bool + public let snapshotBuildCount: Int public let verifiedDuplicateGroupCount: Int public let duplicateVerificationDuration: TimeInterval + public let duplicateVerificationBytesRead: Int64 public let enumerateDuration: TimeInterval public let verifyDuration: TimeInterval public let persistDuration: TimeInterval @@ -34,6 +38,7 @@ public struct ScanBenchmarkReport: Hashable, Sendable { peakMemoryBytes: UInt64? = Self.currentPeakResidentMemoryBytes() ) { self.scopeLabel = scopeLabel + self.buildConfiguration = Self.currentBuildConfiguration() self.duration = scan.finishedAt.timeIntervalSince(scan.startedAt) self.scannedItemCount = scan.scannedItemCount self.inaccessibleItemCount = scan.inaccessibleItemCount @@ -42,9 +47,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { self.largestFolderCount = scan.largestFolders.count self.duplicateCandidateItemsConsidered = scan.duplicateCandidateItemsConsidered self.duplicateCandidateItemsRetained = scan.duplicateCandidateItemsRetained + self.duplicateCandidateEvictionCount = scan.duplicateCandidateEvictionCount self.duplicateCandidateLimitReached = scan.duplicateCandidateLimitReached + self.snapshotBuildCount = scan.snapshotBuildCount self.verifiedDuplicateGroupCount = scan.verifiedDuplicateGroups.count self.duplicateVerificationDuration = scan.duplicateVerificationDuration + self.duplicateVerificationBytesRead = scan.duplicateVerificationBytesRead self.enumerateDuration = scan.enumerateDuration self.verifyDuration = scan.duplicateVerificationDuration self.persistDuration = persistDuration @@ -56,6 +64,7 @@ public struct ScanBenchmarkReport: Hashable, Sendable { [ "StorageScope benchmark", "Scope: \(scopeLabel)", + "Build configuration: \(buildConfiguration)", "Duration: \(Self.seconds(duration))", "Items scanned: \(scannedItemCount.formatted())", "Inaccessible: \(inaccessibleItemCount.formatted())", @@ -63,9 +72,12 @@ public struct ScanBenchmarkReport: Hashable, Sendable { "Largest files retained: \(largestFileCount.formatted())", "Largest folders retained: \(largestFolderCount.formatted())", "Duplicate candidates: \(duplicateCandidateItemsRetained.formatted()) retained / \(duplicateCandidateItemsConsidered.formatted()) considered", + "Duplicate evictions: \(duplicateCandidateEvictionCount.formatted())", "Duplicate cap reached: \(duplicateCandidateLimitReached ? "yes" : "no")", + "Snapshots built: \(snapshotBuildCount.formatted())", "Verified duplicate groups: \(verifiedDuplicateGroupCount.formatted())", "Duplicate verification: \(Self.seconds(duplicateVerificationDuration))", + "Duplicate verification bytes read: \(Self.bytes(duplicateVerificationBytesRead))", "Enumerate duration: \(Self.seconds(enumerateDuration))", "Verify duration: \(Self.seconds(verifyDuration))", "Persist duration: \(Self.seconds(persistDuration))", @@ -96,6 +108,19 @@ public struct ScanBenchmarkReport: Hashable, Sendable { ByteCountFormatter.string(fromByteCount: Int64(clamping: value), countStyle: .memory) } + public static func currentBuildConfiguration() -> String { + if let override = ProcessInfo.processInfo.environment["STORAGESCOPE_BENCHMARK_CONFIGURATION"], + !override.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + return override + } + + #if DEBUG + return "debug" + #else + return "release" + #endif + } + public static func currentPeakResidentMemoryBytes() -> UInt64? { var usage = rusage() guard getrusage(RUSAGE_SELF, &usage) == 0 else { diff --git a/StorageScope-Performance-Plan.md b/StorageScope-Performance-Plan.md new file mode 100644 index 0000000..84d86d2 --- /dev/null +++ b/StorageScope-Performance-Plan.md @@ -0,0 +1,346 @@ +# StorageScope Performance Plan v2.1 — measured, not guessed + +**Scope:** StorageScope v0.7.1 (`main` @ 9cb1f36, 2026-07-02) +**Relationship to `PLAN.md`:** Supersedes and extends P-1..P-28. This revision replaces v2's code-reading hypotheses with **profile evidence captured on this machine** (Apple silicon, Swift 6.3.2, release build, 10 k synthetic fixture, `sample(1)` at 1 ms for 3 s mid-scan). Two of v2's claims were wrong and are corrected below. + +**Headline goal:** 100 k-item scan enumerates in **< 10 s** (release, SSD), re-scans of an unchanged tree in **< 1 s**, duplicate verification reads **≥ 90 % fewer bytes**, peak RSS **< 150 MB** @ 100 k items, 60 fps UI during a live scan. + +--- + +## 0. What the profiler actually says + +Measured runs, same machine, same 10 k / depth-5 / 10 %-duplicates fixture: + +| Configuration | Enumerate | Rate | +|---|---:|---:| +| Debug (`-Onone`) | 5.25 s | ~1,970 items/s | +| Release (`-O`) | 4.91 s | ~2,110 items/s | + +**Correction 1:** debug vs release is only ~7 %. The committed baselines being debug builds (`script/benchmark_scan.sh:7` lacks `-c release`; `100k.txt` starts with "Building for debugging...") is a hygiene bug worth fixing, but it is **not** why scans are slow. A workload this insensitive to optimizer level is not CPU-bound in Swift code — it is blocked. + +`sample(1)` during the release scan, top-of-stack across ~18,000 thread-samples on 12 threads: + +| Frame | Samples | Meaning | +|---|---:|---| +| `__psynch_mutexwait` | 14,697 (~80 %) | Threads parked waiting on a mutex | +| `CFStringCompareWithOptionsAndLocale` | 814 | `localizedStandardCompare` (ICU collation) | +| `__workq_kernreturn` / `__ulock_wait` | 1,140 | GCD pool idle/starved under nested `concurrentPerform` | +| `CanonicalFileURLStringToFileSystemRepresentation` | 345 | URL path canonicalization (`standardizedFileURL`, `.path`) | +| `stat` | 28 | Actual filesystem I/O | + +Per-thread call trees attribute the mutex waits: each of ~11 worker threads spent ~1.2 s of the 3 s window blocked in **`ScanAccumulator.recordItem` → `-[NSLock lock]` → `__psynch_mutexwait`**. + +**The measured story:** the parallel directory walk is effectively serialized through `ScanAccumulator`'s single `NSLock`, and the thread *holding* the lock spends its time doing ICU string comparisons in the duplicate-candidate eviction path. Filesystem I/O (`stat`, 28 samples) is nearly free — the hardware is idle while the code argues over a mutex. This is why 105 k items took 185 s: eviction runs once per candidate insert after the 5,000 cap (100,000 considered ⇒ ~95,000 evictions, each a linear ICU-compare scan under the global lock), so cost grows superlinearly with item count. + +**Correction 2:** v2 blamed the every-25-items snapshot build (`emitProgressLocked`'s `isMultiple(of: 25)` branch) for the baseline. The benchmark installs **no** progress/onSnapshot callbacks (`ScanBenchmark.swift:127`), so snapshots contribute *zero* to the committed baselines. The snapshot problem is still real **in the app** — `ScanStore` installs both callbacks, so real users pay it on top of everything the benchmark measures — but it is an app-only cost and is re-scoped accordingly (§2.3). + +--- + +## 1. Phase 1 — kill the lock convoy (the measured 80 %) + +### 1.1 Fix the eviction path — highest priority in the plan + +`ScanAccumulator.removeSmallestDuplicateCandidateLocked` (`FileSystemScanner.swift:1187`): + +```swift +let removalIndex = items.indices.max { lhs, rhs in + items[lhs].url.path.localizedStandardCompare(items[rhs].url.path) == .orderedAscending +} +``` + +Once `maxDuplicateCandidateItems` (5,000) is reached, **every further candidate insert** triggers: a linear scan of the smallest-size bucket with full ICU collation per pair (`CFStringCompareWithOptionsAndLocale` — the #1 working frame in the profile), plus `duplicateCandidatesBySize.keys.min()` when a bucket drains — all while holding the global lock every other worker needs to record anything. + +Changes: + +1. **Reject-before-lock:** keep an atomic `smallestRetainedSize`; workers whose `item.byteSize < smallestRetainedSize` (the common case post-cap) skip the insert without touching the lock. +2. Replace `[Int64: [StorageItem]]` + linear-min-scan with a **min-heap keyed by `byteSize`** — O(log n) insert/evict, zero string comparisons on the hot path. +3. Determinism moves to one final sort of the retained set at scan end (existing determinism tests are the safety net). +4. Audit-and-ban `localizedStandardCompare` from all hot paths: it also runs in `DirectoryScanSummary.sortedRetainedCandidates` (`:762`, re-sorts up to 800 items *per directory*) and in ranked-list tie-breaks. Use `displaySize` + byte-wise UTF-8 compare during the scan; apply localized ordering only at display time. + +### 1.2 Decompose the accumulator lock + +Even with cheap critical sections, one `NSLock` across 11 workers for every `recordVisit` + `recordBytes` + `recordItem` (that's **2–3 separate lock acquisitions per file**, `:844-882`) leaves a convoy. In order of preference: + +1. **Per-worker sharded accumulators:** each walker thread owns a private accumulator (counters, ranked heaps, type stats, candidate heap); merge shards on emission ticks and at scan end. Lock contention drops to ~zero; merging 8 heaps of ≤ 500 items is microseconds. +2. Failing that, minimum viable: collapse the per-item triple acquisition into a single `record(visit:bytes:item:)` call, and swap `NSLock` for `os_unfair_lock`/`OSAllocatedUnfairLock` (no syscall on uncontended acquire, better convoy behavior). +3. Replace append-then-resort ranked lists (`trimRankedItems` sorts at 4× limit) with **bounded min-heaps** of `maxRankedResults` — O(log 500) insert, no comparator closures over Strings. + +**Exit criterion (Phase 1):** on the re-captured release 100 k baseline, `__psynch_mutexwait` < 5 % of samples and enumerate improves ≥ 5× (the profile says ≥ 80 % of thread-time is recoverable, and the superlinear eviction term disappears entirely). Effort: S–M. Risk: low — internal to `ScanAccumulator`; determinism + cancellation tests must pass. + +--- + +## 2. Phase 2 — structural fixes the profile also implicates + +### 2.1 Flatten the nested `concurrentPerform` walk + +`scanItem` fans out via `DispatchQueue.concurrentPerform` **per directory, recursively** (`:355`). Every non-leaf directory blocks a GCD worker while its children run; the 1,140 `__workq_kernreturn`/`__ulock_wait` samples are that machinery idling. The code's own `waitIfPaused` comment (`:78-84`) admits the pool-starvation risk on pause. + +Replace with a **fixed worker pool** (≈ `activeProcessorCount`, capped ~8 on SSD, 2–4 on network/spinning volumes) pulling directories from a shared frontier deque: pop directory → enumerate children → record files → push subdirectories. Directory sizes finalize via a pending-children counter bubbling up (no call-stack dependency). One cancellation check per pop; pause parks workers on one condition variable without occupying the GCD pool. This also deletes the `[StorageItem?]` placeholder array and the per-child `OSSignpostID(log:object: index as NSNumber)` allocation — an NSNumber + signpost begin/end **per file** even with no profiler attached; re-scope signposts to per-directory. + +Keep the old walker behind `STORAGESCOPE_LEGACY_WALKER` for one release; gate on tree-equality differential tests plus pause/cancel fuzzing. Effort: L. Risk: medium — the one big refactor here, but it also fixes the documented pause-starvation bug, so it pays for itself even if Phase 1 alone hits the wall-time target. + +### 2.2 Cut per-item Foundation costs (345 samples of URL canonicalization) + +1. Don't re-fetch `url.resourceValues(forKeys:)` in `scanItem` (`:274`) — `contentsOfDirectory(includingPropertiesForKeys:)` already prefetched those keys; fall back to a stat only on cache miss. +2. Build `StorageItem.id` as parent-path + `/` + name instead of `url.standardizedFileURL.path` per item (`StorageItem.swift:41`) — that call is the `CanonicalFileURLStringToFileSystemRepresentation` hot frame. Intern lowercase extensions (they repeat massively). +3. **Stretch, behind a flag:** a `getattrlistbulk(2)` enumeration backend — one syscall returns name/type/sizes/mtime for hundreds of entries, zero NSURL objects. Runtime capability probe per volume; FileManager fallback per subtree; differential tree-equality tests. This is the difference between "fast" and "instant" once §1's lock work stops hiding the syscall layer. Only meaningful *after* Phase 1 — today it would speed up the 20 % the process isn't blocked. + +### 2.3 Fix the app-only snapshot cost + +In the app (not the benchmark), `emitProgressLocked` (`:1059`) builds a full `StorageScan` snapshot **every 25 items regardless of elapsed time** — three ranked sorts + both breakdown rebuilds + `StorageScan.init` — under the accumulator lock, with workers blocked behind it. + +1. Make the throttle time-only (~0.35 s progress, ~1 s snapshots); delete the `isMultiple(of: 25)` branch. +2. Copy raw state under the lock (cheap COW), build the derived `StorageScan` **outside** it. +3. Add an in-app instrumented measurement (signpost around snapshot build) to the re-baselined numbers so "benchmark-fast but app-slow" can't recur — that divergence is exactly what let this hide until now. + +--- + +## 3. Phase 3 — do less work, not just faster work + +Two tracks v2 under-weighted; both matter more to perceived speed than raw enumeration: + +### 3.1 Incremental re-scan (biggest user-facing win in this document) + +Users re-scan the same folders repeatedly; today every re-scan pays full price. Design: + +1. Persist the completed scan tree (compact walk records, §4) keyed by root + volume ID. +2. On re-scan, start an **FSEvents** listener rooted at the scan root from the moment a scan completes; record dirtied subtree paths (coalesced by FSEvents for free) between scans. +3. Re-scan = re-walk dirty subtrees only, splice into the persisted tree, re-derive accumulator stats from merged records. Unchanged-tree re-scan touches ~nothing: **< 1 s target.** +4. Fall back to a full walk when the event log overflowed (`kFSEventStreamEventFlagMustScanSubDirs`), the volume ID changed, or persistence is missing/corrupt. Hash cache already persists (`DuplicateHashCache`), so verification is incremental for free. +5. Ship read-only-trust-but-verify first: incremental result + background full walk comparing item counts, log divergence, before making incremental the default. + +Effort: L (the one new-feature-sized item). Risk: medium — correctness depends on FSEvents semantics; the trust-but-verify rollout and the fallback rules bound it. + +### 3.2 Perceived performance during first scan + +- **Converge big numbers first:** walk the frontier largest-known-first (size from parent enumeration) so top-level folder sizes and the Overview stabilize in the first seconds even if leaves take longer. +- Stream a real (partial) retained tree in snapshots instead of the current placeholder root (`snapshotLocked` sends an empty tree, `:1083`), so Folder Tree/Storage Map populate during the scan rather than at the end. + +### 3.3 Duplicate verification: tiered hashing + +1. **Tier 0 — metadata:** detect hard links (`st_nlink`/`st_ino`) and exclude them — they reclaim nothing and are reported as duplicates today. Investigate APFS clone detection (clones also reclaim ~nothing on delete); no clean public API, so at minimum caveat clone-suspect results in UI. +2. **Tier 1 — 64 KB prefix hash** per candidate; most same-size non-duplicates diverge in the first block and drop out for the cost of one small read. +3. **Tier 2 — full hash** only for still-colliding groups. Cache both tiers in `DuplicateHashCache` under separate key namespaces. +4. Mechanics: reuse a read buffer instead of a fresh 1 MB `Data` per chunk (`:663`), raise chunks to 4–8 MB, keep digests as 32 raw bytes end-to-end (hex only at display; subsumes PLAN P-7/P-23), batch cache writes per group instead of per file (`cacheLock` per record, `:697`). + +Expected: ≥ 90 % fewer verification bytes on media/VM-heavy sets; the 20 GB budget then verifies far more real duplicates per scan. Add a "verification bytes read" counter to `ScanBenchmarkReport` so this is measured, not asserted. + +--- + +### 3.4 Proof harness (implemented 2026-07-11) + +Before changing the walker, `Tests/StorageScopeCoreTests/ScannerProofHarness.swift` now +provides a reusable differential surface. It canonicalizes relative paths, sibling order, +tree metadata, derived rankings, duplicate groups, cleanup candidates, and scan counters +while intentionally ignoring UUID roots, timestamps, and durations. + +The fixture set covers: + +1. deep and wide directory shapes; +2. a mutation sequence of creates, deletes, renames, and resizes; +3. a hard-link pair with an inode identity check; +4. deterministic permission-loss injection; and +5. deterministic volume-loss injection. + +`ScannerProofHarness.compare` accepts separate baseline and candidate runners, so the +fixed-worker walker can be introduced behind a flag and compared state-for-state before +it becomes the default. The current tests run both sides through the existing scanner to +prove the harness itself, and deliberately preserve the current hard-link semantics until +the compact-record phase changes that policy. + +Focused proof: + +```sh +swift test --scratch-path /tmp/storagescope-worker-focused-serial --disable-index-store --jobs 1 --filter ScannerProofHarnessTests +``` + +Latest result: 7 differential/stability tests in 1 suite passed; the compact-record +contract extends the latest focused run to 8 tests across 2 suites. + +### 3.5 Fixed-worker walker (implemented opt-in 2026-07-11) + +`FileSystemScanner` now has an experimental fixed-worker backend behind +`STORAGESCOPE_EXPERIMENTAL_WORKER_WALKER=1`. It uses a bounded directory frontier, +bounded record channel, fixed worker count, deterministic post-walk reconstruction, and +the same public `StorageScan` accumulation rules as the legacy walker. The legacy path +remains the default; `STORAGESCOPE_LEGACY_WALKER=1` is an explicit override. If the +experimental backend encounters an unexpected worker-only failure, it retries the root +through the legacy path before publishing a scan. + +The differential harness now compares legacy and fixed-worker output across deep/wide, +mutation-heavy, hard-link, permission-loss, and volume-loss fixtures. It also covers +pause/resume completion and cancellation while paused. The focused suite passed 7 tests. + +Release measurements on the same synthetic depth-5 fixture show why promotion is not +yet justified: + +| Fixture | Legacy median / RSS | Fixed-worker median / RSS | Result | +|---|---:|---:|---| +| 10 k items, 3 runs | 0.57 s / 36 MB | 0.99 s / 49.6 MB | worker slower and higher RSS | +| 100 k items, 1 run | 17.82 s / 84.8 MB | 14.66 s / 249 MB | wall time improves, memory gate fails | + +The fixed-worker path therefore stays opt-in. The next implementation step is the compact +metadata-record phase: remove URL-rich transient retention, carry parent and volume/file- +resource identity explicitly, and re-run the same differential and release gates before +changing the default. + +### 3.6 Compact walk records (implemented opt-in 2026-07-11) + +The fixed-worker backend now uses `FixedWorkerWalkRecord` for the retained walk state: +name, parent ID, scalar sizes/timestamps/readability, compact kind metadata, and fixed-width +opaque volume/file-resource identifiers from `URLResourceValues`. URLs remain only on +directory jobs and are materialized when the final public tree is rebuilt. Per-directory +autorelease pools bound Foundation-object retention during enumeration. The hard-link +reclaimability policy is intentionally unchanged in this slice; `hardLinkCount` is reserved +for the follow-up identity policy rather than being inferred from an opaque resource identifier. + +The proof surface now includes a source contract that rejects URL storage inside the compact +record and requires parent/resource identity fields. The differential hard-link fixture still +passes with the existing public semantics. + +Kept-fixture release measurements isolate scan cost from fixture-generation memory: + +| Fixture | Legacy control | Compact fixed-worker | Result | +|---|---:|---:|---| +| 10 k items, 3-run median | 0.58 s / 29.7 MB | 0.44 s / 37.3 MB | wall time improves, RSS +26% | +| 100 k items, 1 run | 8.67 s / 31.3 MB | 7.48 s / 54.5 MB | wall time improves, RSS +74% | + +The compact representation removes the earlier URL-rich worker spike, but it still fails the +memory promotion gate. The 100 k profile is no longer mutex-dominated; `__workq_kernreturn` +leads, followed by `StorageItem` copy/destroy activity in `makeFixedWorkerItem`. The next +step is lazy rich-item materialization for retained/ranked/duplicate candidates, plus a +separate hard-link reclaimability policy, before considering default promotion. + +### 3.7 Lazy rich-item materialization (implemented opt-in 2026-07-12) + +The fixed-worker reconstruction pass now returns compact `FixedWorkerItemSummary` values and +retains rich summary payloads only for the bounded retained tree and bounded ranked, duplicate, +and cleanup references. `FixedWorkerSummaryStore` is a stateless materializer rather than a +global ID-to-summary map, so non-retained entries are released after aggregation. Ranked, +duplicate, and cleanup references carry their source URL explicitly; this preserves parity for +results that are not present in the retained tree. + +The hard-link reclaimability policy remains deliberately deferred. `hardLinkCount` stays a +reserved zero field and opaque resource identifiers are not interpreted as link counts. + +Latest proof and release measurements for the post-materialization implementation: + +| Fixture | Fixed-worker release result | Gate | +|---|---:|---| +| 10 k items, 3 runs | 1.14 s median / 40.5 MB RSS (1.04, 1.14, 1.37 s; 39.8, 40.5, 41.7 MB) | memory healthy; worker remains slower than the historical legacy control | +| 100 k items, 1 run | 12.83 s / 58.2 MB RSS | RSS clears `<150 MB`; wall time misses `<10 s` | + +The worker therefore remains opt-in behind `STORAGESCOPE_EXPERIMENTAL_WORKER_WALKER=1` and +the legacy walker remains the default. The 100 k three-run recheck was not promoted to evidence +because SwiftPM hit its known “input file was modified during the build” race before the scan +started. That handoff led to the ranked-reference heap pass below rather than reintroducing a +global summary map. + +### 3.8 Bounded ranked-reference heap (implemented opt-in 2026-07-12) + +A fresh release sample localized the remaining reconstruction cost to +`buildFixedWorkerSummary` → `insertFixedWorkerRankedReference`: every scanned candidate could +scan the bounded ranked array, recompute its minimum, and repeatedly materialize `URL.path` for +tie-breaking. The ranked collections now use a worst-first bounded heap. Path keys are captured +only for references that enter the bounded set, and final display ordering remains deterministic +through one read-time sort. + +The change is covered by a source contract for the heap operations and by the existing +differential, hard-link, cleanup, fault, pause, and cancellation tests. + +Latest kept-fixture release measurements: + +| Fixture | Fixed-worker result | Gate | +|---|---:|---| +| 10 k items, 3 runs | 0.40 s median / 39.3 MB RSS (0.42, 0.39, 0.40 s; 39.3, 37.9, 39.9 MB) | clears the small-fixture wall gate | +| 100 k items, 3 runs | 6.34 s median / 56.6 MB RSS (6.25, 6.40, 6.34 s; 53.2, 59.5, 56.6 MB) | clears `<10 s` and `<150 MB` | + +The benchmark wall-time gate is green. The worker remains opt-in for the planned one-release +fallback window; hard-link reclaimability is a Phase 3 policy, not a Phase 2 promotion gate. +The next profiled cost is copy-on-write eviction in +`FixedWorkerDuplicateCandidateRetention.evictSmallestRetained()`; it is not changed in this +slice. + +### 3.9 Phase 2 app-path gate — complete (2026-07-12) + +The first release comparison failed at 1.278× app/callback-free overhead: 2.938 s app median +versus 2.299 s callback-free. Two app-only costs were then removed without changing scan +semantics: snapshot builds were separated from the 0.35 s progress cadence and throttled to +approximately 1 s, and retained-path canonicalization for hash-cache persistence moved off the +main actor into the existing detached persistence task. + +The same-process, same-fixture, alternating three-run release proof then passed: + +| Path | 100 k median | Relative | +|---|---:|---:| +| Callback-free `FileSystemScanner` | 2.541 s | 1.000× | +| Interactive `ScanStore` | 2.561 s | 1.008× | +| Scanner duration reported through app path | 2.348 s | 0.924× | + +App overhead is 0.8%, clearing the ≤15% exit gate. Snapshot counts were 3, 2, and 2 across the +three app runs. The opt-in proof is `ScanStoreAppPerformanceProofTests` and requires a release +build plus `STORAGESCOPE_TESTING`, so ordinary release builds retain no debug counters or test +drain hooks. Focused differential/fault/pause/cancel coverage passed 10 tests across 3 suites; +the full rebased default suite passed 221 tests across 22 suites with the expensive proof skipped by +default. All Phase 2 exit criteria are now satisfied. Phase 3 starts with tiered verification, +hard-link reclaimability, and perceived-progress streaming. + +--- + +## 4. Phase 4 — memory + +271.8 MB peak @ 100 k ≈ 2.7 KB/item even though only 25 k items are retained: every visited item transiently materializes URL + id String + name String + extension + NSURL bridging garbage. + +1. Split the model: a fixed-size **walk record** (sizes, mtime, flags, interned-extension index, parent index) for enumeration/stats, with rich `StorageItem`s built only for the ≤ 25 k retained items at finalization. This same compact record is the persistence format for §3.1. +2. `ContiguousArray` + capacity reservation for children buffers; per-worker scratch reuse in the §2.1 pool. +3. Target < 150 MB @ 100 k, sublinear to 500 k; verify with mid-scan `mach_task_basic_info` sampling (PLAN P-25). + +--- + +## 5. Phase 5 — UI and main actor + +1. **`@Observable` migration** (target is already macOS 14+): `ScanStore` (2,170 lines) currently invalidates every observing view on any change. Observation-framework per-property tracking largely obsoletes PLAN's invasive P-13 `SelectionStore` extraction and P-15 `Equatable` rows in one move. Sequence store-by-store (`FilterStore` → small stores → `ScanStore` last), comparing SwiftUI update counts in Instruments at each step. +2. **Search:** `matchesTerm` (`StorageItem.swift:102`) runs `localizedCaseInsensitiveContains` on name *and* path plus a `pathComponents` split, per item per keystroke, over up to 25 k items. Precompute lowercase name/path keys once per scan, use plain `contains`, debounce the query ~150 ms. +3. Keep the still-valid cheap PLAN items and fold them in here: P-8/P-9 (memoization), P-11 (async `mountedVolumes`), P-14 (GeometryReader fills), P-16/P-17. + +--- + +## 6. Measurement hygiene (demoted from v2's #1, still required) + +1. `script/benchmark_scan.sh` → `swift run -c release ...` (with `--debug` escape hatch); benchmark report header prints build configuration; CI rejects debug-mode baseline files. +2. Re-capture 10 k / 100 k baselines in release into `docs/perf-baselines/v0.7.1/`, 3 runs, median + spread — **plus a `sample(1)`/xctrace capture committed alongside**, since top-of-stack profiles (not wall-time alone) are what caught the real bottleneck. +3. **Cold-cache variant:** the synthetic benchmark scans files it just wrote — everything is warm in the unified buffer cache, which understates I/O and overstates CPU/lock effects relative to real first scans. Add a purge step (`purge`, or re-mount fixture on a disk image) and record both warm and cold numbers. +4. New counters in `ScanBenchmarkReport`: mutex-wait share (from an instrumented build), evictions performed, snapshot builds (app-instrumented), verification bytes read, cache hit/miss, worker utilization. +5. CI perf gate on the 10 k fixture: items/second vs. committed reference with ±35 % band (GitHub runners are noisy) plus noise-immune ratio gates ("evictions ≤ considered − retained", "snapshot builds per 1,000 items ≤ 2"); fail only on two consecutive regressions. + +--- + +## 7. Sequenced rollout + +| Phase | Release | Contents | Exit criterion (release build, 100 k fixture) | +|---|---|---|---| +| 0 | v0.7.2 | §6.1–2 hygiene + counters + re-baseline | Trustworthy baseline incl. profile committed | +| 1 | v0.7.3 | §1 eviction fix + lock decomposition + heaps | `__psynch_mutexwait` < 5 % of samples; enumerate ≥ 5× vs Phase-0 baseline | +| 2 | v0.8.0 | §2.1 worker-pool walker (flagged), §2.2.1–2 per-item costs, §2.3 app snapshot fix | Enumerate < 10 s; pause/cancel green on both walkers; in-app scan within 15 % of benchmark scan | +| 3 | v0.8.1 | §3.3 tiered hashing; §3.2 perceived-perf streaming | Verification bytes −90 % on duplicates fixture; hard links excluded; Overview stable < 3 s into a 100 k scan | +| 4 | v0.9.0 | §4 walk records; §2.2.3 `getattrlistbulk` (flagged); §3.1 incremental re-scan (trust-but-verify) | RSS < 150 MB @ 100 k; unchanged-tree re-scan < 1 s; bulk backend ≥ 2× with identical trees | +| 5 | v0.9.x | §5 `@Observable` + search keys | SwiftUI update count during live scan −80 %; no dropped frames typing in filter @ 25 k items | + +**Decision rule** (replacing PLAN §6.3's debug-based thresholds): each phase re-runs fixtures in release, 3 runs, medians, warm *and* cold. A change ships if its target counter improves ≥ 10 % without regressing any other counter > 5 %; otherwise it stays flagged and the hypothesis table is updated with the measured result. Any claim of "X is the bottleneck" must cite a profile, not code reading — this document needed two corrections for exactly that reason. + +--- + +## 8. Risk register + +- **§2.1 walker refactor:** concurrency contract change (pause/cancel/determinism). Mitigation: legacy flag one release, differential tree-equality tests, pause/resume/cancel fuzz (randomized timing, 1,000 CI iterations). +- **§3.1 incremental re-scan:** correctness rests on FSEvents delivery. Mitigation: trust-but-verify rollout, conservative fallback triggers, full-walk always available. +- **`getattrlistbulk`:** per-filesystem quirks (SMB/NFS/FUSE). Mitigation: per-volume probe, per-subtree fallback, non-default until v1.0. +- **`@Observable`:** silent update-timing changes. Mitigation: store-by-store migration with render-count snapshots. +- **Determinism:** current stable orderings lean on ICU compares in hot paths; the plan moves all determinism to final read-time sorts. `parallelEnumerationIsDeterministicAcrossRuns` and friends are the safety net and must not be weakened. +- **CI noise:** wide bands, ratio gates, two-strike policy (§6.5). + +--- + +## 9. Expected end state + +The profile shows the hardware is nearly idle during today's scans (28 `stat` samples vs 14,697 mutex-wait samples). The syscall floor for 100 k stat-equivalents via bulk enumeration is well under 2 s on Apple silicon + SSD; tiered hashing keeps verification to a few hundred MB of reads against a 2.5 GB/s device. Phases 0–2 remove the self-inflicted serialization between that hardware and the user; Phases 3–4 stop repeating work across scans. A first scan in single-digit seconds and a re-scan in under one second are engineering outcomes of this sequence, not aspirations. diff --git a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift index 6f0b2b0..e910859 100644 --- a/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift +++ b/Tests/StorageScopeCoreTests/FileSystemScannerTests.swift @@ -124,6 +124,46 @@ struct FileSystemScannerTests { ).isEmpty) } + @Test("duplicate verification records bytes read") + func duplicateVerificationRecordsBytesRead() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + try Data("same-content".utf8).write(to: temporaryRoot.appendingPathComponent("copy-a.txt")) + try Data("same-content".utf8).write(to: temporaryRoot.appendingPathComponent("copy-b.txt")) + + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(duplicateCandidateThreshold: 1) + ) + + #expect(scan.verifiedDuplicateGroups.count == 1) + #expect(scan.duplicateVerificationBytesRead == 24) + } + + @Test("duplicate verification avoids full reads for prefix-mismatched same-size files") + func duplicateVerificationSkipsFullHashForPrefixMismatches() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + let fileSize = 1_048_576 + for index in 0..<4 { + var data = Data(repeating: UInt8(index), count: fileSize) + data[0] = UInt8(index + 1) + try data.write(to: temporaryRoot.appendingPathComponent("unique-\(index).bin")) + } + + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(duplicateCandidateThreshold: 1) + ) + + #expect(scan.duplicateSizeGroups.count == 1) + #expect(scan.verifiedDuplicateGroups.isEmpty) + #expect(scan.duplicateVerificationBytesRead == Int64(4 * 64 * 1_024)) + #expect(scan.duplicateVerificationBytesRead <= scan.duplicateSizeGroups[0].totalBytes / 10) + } + @Test("caps automatic duplicate content verification") func duplicateVerificationHonorsByteBudget() throws { let temporaryRoot = try makeTemporaryRoot() @@ -663,6 +703,7 @@ struct FileSystemScannerTests { #expect(scan.duplicateCandidateItemLimit == 12) #expect(scan.duplicateCandidateItemsRetained == 12) #expect(scan.duplicateCandidateItemsConsidered == 14) + #expect(scan.duplicateCandidateEvictionCount == 2) #expect(scan.duplicateCandidateLimitReached) } @@ -697,6 +738,25 @@ struct FileSystemScannerTests { #expect(scan.duplicateCandidateLimitReached) } + @Test("duplicate candidate cap rejects smaller candidates without eviction churn") + func duplicateCandidateCapRejectsSmallerCandidatesBeforeEviction() throws { + var candidates = DuplicateCandidateRetention(limit: 12) + for index in 0..<12 { + candidates.record(fileItem(named: String(format: "large-%03d.bin", index), bytes: 8_192)) + } + + for index in 0..<100 { + candidates.record(fileItem(named: String(format: "small-%03d.bin", index), bytes: 2_048)) + } + + #expect(candidates.consideredCount == 112) + #expect(candidates.retainedCount == 12) + #expect(candidates.evictionCount == 0) + #expect(candidates.limitReached) + #expect(candidates.sizeGroups.map(\.byteSize) == [8_192]) + #expect(candidates.sizeGroups.first?.items.count == 12) + } + @Test("duplicate candidates include files pruned from retained tree") func duplicateCandidatesSurviveRetainedTreePruning() throws { let temporaryRoot = try makeTemporaryRoot() @@ -975,9 +1035,12 @@ struct FileSystemScannerTests { #expect(text.contains("StorageScope benchmark")) #expect(text.contains("Scope: \(temporaryRoot.lastPathComponent)")) + #expect(text.contains("Build configuration:")) #expect(text.contains("Items scanned:")) #expect(text.contains("Duplicate candidates:")) + #expect(text.contains("Duplicate evictions:")) #expect(text.contains("Duplicate verification:")) + #expect(text.contains("Snapshots built:")) #expect(text.contains("Results are local only.")) #expect(!text.contains(temporaryRoot.deletingLastPathComponent().path)) #expect(!text.localizedCaseInsensitiveContains("http://")) @@ -1030,6 +1093,68 @@ struct FileSystemScannerTests { #expect(process.terminationStatus == 0) } + @Test("benchmark script runs release builds by default") + func benchmarkScriptRunsReleaseBuildsByDefault() throws { + let repoRoot = try repositoryRoot() + let script = try String(contentsOf: repoRoot.appendingPathComponent("script/benchmark_scan.sh"), encoding: .utf8) + + #expect(script.contains("CONFIGURATION=\"release\"")) + #expect(script.contains("swift run -c \"$CONFIGURATION\" StorageScopeBenchmark")) + #expect(script.contains("--debug")) + } + + @Test("scanner records item bytes in a single accumulator call") + func scannerRecordsItemBytesInSingleAccumulatorCall() throws { + let repoRoot = try repositoryRoot() + let source = try String( + contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeCore/Services/FileSystemScanner.swift"), + encoding: .utf8 + ) + + #expect(!source.contains("func recordBytes")) + #expect(!source.contains("accumulator.recordBytes(")) + } + + @Test("fixed-worker records keep compact parent and identity metadata") + func fixedWorkerRecordsKeepCompactMetadata() throws { + let repoRoot = try repositoryRoot() + let source = try String( + contentsOf: repoRoot.appendingPathComponent("Sources/StorageScopeCore/Services/FileSystemScanner.swift"), + encoding: .utf8 + ) + let recordStart = try #require(source.range(of: "private struct FixedWorkerWalkRecord")) + let recordEnd = try #require(source[recordStart.upperBound...].range(of: "private struct FixedWorkerDirectoryJob")?.lowerBound) + let recordSource = String(source[recordStart.lowerBound..= 0) #expect(report.verifyDuration >= 0) #expect(report.verifyDuration == report.duplicateVerificationDuration) + #expect(report.duplicateVerificationBytesRead >= 0) // No cache wired up: persistDuration is a no-op plus two Date() readouts, which // can register sub-microsecond timing noise instead of an exact 0. #expect(abs(report.persistDuration) < 0.001) @@ -1050,6 +1176,7 @@ struct FileSystemScannerTests { let text = report.text #expect(text.contains("Enumerate duration:")) #expect(text.contains("Verify duration:")) + #expect(text.contains("Duplicate verification bytes read:")) #expect(text.contains("Persist duration:")) #expect(text.contains("Phase total (enum+verify+persist):")) } @@ -1533,6 +1660,26 @@ struct FileSystemScannerTests { #expect(nonEmptySnapshot?.isPartial == true) } + @Test("snapshot cadence is time throttled, not every 25 items") + func snapshotCadenceIsTimeThrottled() throws { + let temporaryRoot = try makeTemporaryRoot() + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + + for index in 0..<100 { + try writeFile(named: "file-\(index).bin", bytes: 1_000 + index, in: temporaryRoot) + } + + var snapshotCount = 0 + let scan = try FileSystemScanner().scan( + root: temporaryRoot, + options: ScanOptions(largeFileThreshold: 1, duplicateCandidateThreshold: 10_000), + onSnapshot: { _ in snapshotCount += 1 } + ) + + #expect(snapshotCount <= 2) + #expect(scan.snapshotBuildCount == snapshotCount) + } + @Test("pause blocks scan progress and resume continues to completion") func pauseBlocksProgressAndResumeCompletes() throws { let temporaryRoot = try makeTemporaryRoot() @@ -1723,6 +1870,21 @@ struct FileSystemScannerTests { ) } + private func fileItem(named name: String, bytes: Int64) -> StorageItem { + StorageItem( + url: URL(fileURLWithPath: "/tmp/\(name)"), + name: name, + kind: .file, + byteSize: bytes, + allocatedSize: bytes, + modifiedAt: nil, + immediateChildCount: 0, + descendantCount: 0, + isReadable: true, + fileExtension: URL(fileURLWithPath: name).pathExtension + ) + } + private func storageScan( rootURL: URL, scannedItemCount: Int, diff --git a/Tests/StorageScopeCoreTests/ScannerProofHarness.swift b/Tests/StorageScopeCoreTests/ScannerProofHarness.swift new file mode 100644 index 0000000..467e7d0 --- /dev/null +++ b/Tests/StorageScopeCoreTests/ScannerProofHarness.swift @@ -0,0 +1,572 @@ +import Foundation +@testable import StorageScopeCore + +/// Test-only fixture and comparison infrastructure for scanner architecture changes. +/// +/// The harness deliberately compares semantic scan output rather than absolute paths, +/// timestamps, or wall-clock timings. A future walker can be passed as `candidate` and +/// compared with the current implementation without changing production APIs. +struct ScannerProofHarness { + typealias Runner = (_ root: URL, _ fileManager: FileManager) throws -> StorageScan + + static let proofOptions = ScanOptions( + oldFileAgeDays: 180, + largeFileThreshold: 1, + duplicateCandidateThreshold: 1, + duplicateVerificationByteLimit: 512 * 1_024 * 1_024, + maxDuplicateVerificationFiles: 10_000, + maxDuplicateCandidateItems: 10_000, + maxRankedResults: 10_000, + maxChildrenPerDirectory: 10_000, + maxRetainedItems: 100_000 + ) + + static let legacyRunner: Runner = { root, fileManager in + try FileSystemScanner( + fileManager: fileManager, + walkerMode: .legacy + ).scan(root: root, options: proofOptions) + } + + static let fixedWorkerRunner: Runner = { root, fileManager in + try FileSystemScanner( + fileManager: fileManager, + walkerMode: .fixedWorker + ).scan(root: root, options: proofOptions) + } + + static let currentRunner: Runner = { root, fileManager in + try FileSystemScanner(fileManager: fileManager).scan(root: root, options: proofOptions) + } + + static func scan(_ fixture: ScannerProofFixture, runner: Runner = currentRunner) throws -> StorageScan { + try runner(fixture.root, fixture.makeFileManager()) + } + + /// Compare two scanner implementations against the same fixture state. + /// + /// Each runner gets a fresh file manager so fault injection is identical and no + /// implementation can accidentally share mutable enumeration state with the other. + @discardableResult + static func compare( + _ fixture: ScannerProofFixture, + baseline: Runner, + candidate: Runner, + context: String + ) throws -> ScanSignature { + let baselineScan = try baseline(fixture.root, fixture.makeFileManager()) + let candidateScan = try candidate(fixture.root, fixture.makeFileManager()) + let baselineSignature = signature(of: baselineScan, root: fixture.root) + let candidateSignature = signature(of: candidateScan, root: fixture.root) + + if let difference = firstDifference(baselineSignature, candidateSignature) { + throw ScannerProofError.mismatch(context: context, detail: difference) + } + + return baselineSignature + } + + /// Run the differential comparison through every mutation state, including the + /// initial state before the first mutation. + static func compareMutationSequence( + _ fixture: ScannerProofFixture, + baseline: Runner, + candidate: Runner + ) throws -> [ScanSignature] { + var signatures: [ScanSignature] = [] + signatures.reserveCapacity(fixture.mutations.count + 1) + + for stateIndex in 0...fixture.mutations.count { + signatures.append(try compare( + fixture, + baseline: baseline, + candidate: candidate, + context: "mutation state \(stateIndex)" + )) + + guard stateIndex < fixture.mutations.count else { break } + try fixture.mutations[stateIndex].apply(in: fixture.root) + } + + return signatures + } + + static func signature(of scan: StorageScan, root: URL) -> ScanSignature { + ScanSignature( + root: itemSignature(scan.rootItem, root: root), + retainedPaths: scan.retainedItems.map { relativePath($0.url, from: root) }.sorted(), + scannedItemCount: scan.scannedItemCount, + inaccessibleItemCount: scan.inaccessibleItemCount, + totalBytes: scan.totalBytes, + largestFiles: scan.largestFiles.map { relativePath($0.url, from: root) }.sorted(), + largestFolders: scan.largestFolders.map { relativePath($0.url, from: root) }.sorted(), + oldLargeFiles: scan.oldLargeFiles.map { relativePath($0.url, from: root) }.sorted(), + typeBreakdown: scan.typeBreakdown.map { + TypeSignature( + label: $0.label, + category: $0.category.rawValue, + fileCount: $0.fileCount, + totalBytes: $0.totalBytes + ) + }.sorted { $0.label < $1.label }, + categoryBreakdown: scan.categoryBreakdown.map { + CategorySignature( + category: $0.category.rawValue, + fileCount: $0.fileCount, + extensionCount: $0.extensionCount, + totalBytes: $0.totalBytes + ) + }.sorted { $0.category < $1.category }, + duplicateSizeGroups: scan.duplicateSizeGroups.map { + GroupSignature( + byteSize: $0.byteSize, + paths: $0.items.map { relativePath($0.url, from: root) }.sorted() + ) + }.sorted { + if $0.byteSize != $1.byteSize { + return $0.byteSize < $1.byteSize + } + return $0.paths.lexicographicallyPrecedes($1.paths) + }, + verifiedDuplicateGroups: scan.verifiedDuplicateGroups.map { + VerifiedGroupSignature( + checksum: $0.checksum, + byteSize: $0.byteSize, + paths: $0.items.map { relativePath($0.url, from: root) }.sorted() + ) + }.sorted { ($0.byteSize, $0.checksum) < ($1.byteSize, $1.checksum) }, + cleanupCandidates: scan.cleanupCandidates.map { + CandidateSignature( + kind: $0.kind.rawValue, + path: relativePath($0.item.url, from: root), + confidence: $0.confidence.rawValue, + reason: $0.reason, + reclaimableBytes: $0.reclaimableBytes + ) + }.sorted { ($0.path, $0.kind) < ($1.path, $1.kind) }, + duplicateCandidateItemLimit: scan.duplicateCandidateItemLimit, + duplicateCandidateItemsRetained: scan.duplicateCandidateItemsRetained, + duplicateCandidateItemsConsidered: scan.duplicateCandidateItemsConsidered, + duplicateCandidateEvictionCount: scan.duplicateCandidateEvictionCount, + duplicateCandidateLimitReached: scan.duplicateCandidateLimitReached, + snapshotBuildCount: scan.snapshotBuildCount, + duplicateVerificationBytesRead: scan.duplicateVerificationBytesRead, + isPartial: scan.isPartial + ) + } + + static func relativePath(_ url: URL, from root: URL) -> String { + let rootPath = root.standardizedFileURL.path + let itemPath = url.standardizedFileURL.path + guard itemPath != rootPath else { return "." } + guard itemPath.hasPrefix(rootPath + "/") else { return itemPath } + return String(itemPath.dropFirst(rootPath.count + 1)) + } + + private static func itemSignature(_ item: StorageItem, root: URL) -> ItemSignature { + ItemSignature( + path: relativePath(item.url, from: root), + kind: item.kind.rawValue, + byteSize: item.byteSize, + allocatedSize: item.allocatedSize, + immediateChildCount: item.immediateChildCount, + descendantCount: item.descendantCount, + isReadable: item.isReadable, + fileExtension: item.fileExtension, + children: item.children + .map { itemSignature($0, root: root) } + .sorted { $0.path < $1.path } + ) + } + + private static func firstDifference(_ baseline: ScanSignature, _ candidate: ScanSignature) -> String? { + guard baseline != candidate else { return nil } + + let baselinePaths = Set(baseline.root.allPaths) + let candidatePaths = Set(candidate.root.allPaths) + let baselineOnly = baselinePaths.subtracting(candidatePaths).sorted() + let candidateOnly = candidatePaths.subtracting(baselinePaths).sorted() + + return "baseline \(baseline.summary); candidate \(candidate.summary); " + + "baseline-only paths: \(baselineOnly.prefix(8)); candidate-only paths: \(candidateOnly.prefix(8))" + } +} + +struct ScanSignature: Equatable, Sendable { + let root: ItemSignature + let retainedPaths: [String] + let scannedItemCount: Int + let inaccessibleItemCount: Int + let totalBytes: Int64 + let largestFiles: [String] + let largestFolders: [String] + let oldLargeFiles: [String] + let typeBreakdown: [TypeSignature] + let categoryBreakdown: [CategorySignature] + let duplicateSizeGroups: [GroupSignature] + let verifiedDuplicateGroups: [VerifiedGroupSignature] + let cleanupCandidates: [CandidateSignature] + let duplicateCandidateItemLimit: Int + let duplicateCandidateItemsRetained: Int + let duplicateCandidateItemsConsidered: Int + let duplicateCandidateEvictionCount: Int + let duplicateCandidateLimitReached: Bool + let snapshotBuildCount: Int + let duplicateVerificationBytesRead: Int64 + let isPartial: Bool + + var treePaths: [String] { + root.allPaths.sorted() + } + + var summary: String { + "items=\(scannedItemCount), inaccessible=\(inaccessibleItemCount), bytes=\(totalBytes), retained=\(retainedPaths.count), tree=\(treePaths.count)" + } +} + +struct ItemSignature: Equatable, Sendable { + let path: String + let kind: String + let byteSize: Int64 + let allocatedSize: Int64 + let immediateChildCount: Int + let descendantCount: Int + let isReadable: Bool + let fileExtension: String? + let children: [ItemSignature] + + var allPaths: [String] { + [path] + children.flatMap(\.allPaths) + } + + var maxDepth: Int { + 1 + (children.map(\.maxDepth).max() ?? 0) + } + + var maxWidth: Int { + max(children.count, children.map(\.maxWidth).max() ?? 0) + } +} + +struct TypeSignature: Equatable, Sendable { + let label: String + let category: String + let fileCount: Int + let totalBytes: Int64 +} + +struct CategorySignature: Equatable, Sendable { + let category: String + let fileCount: Int + let extensionCount: Int + let totalBytes: Int64 +} + +struct GroupSignature: Equatable, Sendable { + let byteSize: Int64 + let paths: [String] +} + +struct VerifiedGroupSignature: Equatable, Sendable { + let checksum: String + let byteSize: Int64 + let paths: [String] +} + +struct CandidateSignature: Equatable, Sendable { + let kind: String + let path: String + let confidence: String + let reason: String + let reclaimableBytes: Int64 +} + +enum ScannerProofError: Error, CustomStringConvertible { + case mismatch(context: String, detail: String) + + var description: String { + switch self { + case let .mismatch(context, detail): + return "Scanner proof mismatch in \(context): \(detail)" + } + } +} + +struct ScannerProofFixture { + enum Kind: String, CaseIterable { + case deepAndWide + case mutationHeavy + case hardLinks + case cleanupCandidates + case permissionLoss + case volumeLoss + } + + enum FaultKind: String { + case permissionLoss + case volumeLoss + + var error: Error { + switch self { + case .permissionLoss: + return CocoaError(.fileReadNoPermission) + case .volumeLoss: + return CocoaError(.fileNoSuchFile) + } + } + } + + struct FaultInjection { + let relativePath: String + let kind: FaultKind + } + + struct HardLinkPair { + let source: URL + let alias: URL + } + + enum Mutation: CustomStringConvertible { + case create(relativePath: String, bytes: Int, seed: UInt8) + case delete(relativePath: String) + case rename(from: String, to: String) + case resize(relativePath: String, bytes: Int, seed: UInt8) + + var description: String { + switch self { + case let .create(path, bytes, _): return "create \(path) (\(bytes) bytes)" + case let .delete(path): return "delete \(path)" + case let .rename(from, to): return "rename \(from) -> \(to)" + case let .resize(path, bytes, _): return "resize \(path) (\(bytes) bytes)" + } + } + + func apply(in root: URL, fileManager: FileManager = FileManager()) throws { + switch self { + case let .create(relativePath, bytes, seed): + let url = root.appendingPathComponent(relativePath) + try fileManager.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try Self.write(url: url, bytes: bytes, seed: seed) + case let .delete(relativePath): + try fileManager.removeItem(at: root.appendingPathComponent(relativePath)) + case let .rename(from, to): + let destination = root.appendingPathComponent(to) + try fileManager.createDirectory( + at: destination.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try fileManager.moveItem( + at: root.appendingPathComponent(from), + to: destination + ) + case let .resize(relativePath, bytes, seed): + try Self.write(url: root.appendingPathComponent(relativePath), bytes: bytes, seed: seed) + } + } + + private static func write(url: URL, bytes: Int, seed: UInt8) throws { + try Data(repeating: seed, count: bytes).write(to: url, options: .atomic) + } + } + + let kind: Kind + let root: URL + let fault: FaultInjection? + let hardLinkPair: HardLinkPair? + let mutations: [Mutation] + + func makeFileManager() -> FileManager { + guard let fault else { return FileManager() } + return FaultInjectingFileManager( + failurePath: root.appendingPathComponent(fault.relativePath), + failure: fault.kind + ) + } + + func tearDown() { + try? FileManager().removeItem(at: root) + } + + static func make( + _ kind: Kind, + in parentDirectory: URL = FileManager.default.temporaryDirectory + ) throws -> ScannerProofFixture { + let fileManager = FileManager() + let root = parentDirectory.appendingPathComponent( + "StorageScopeProof-\(kind.rawValue)-\(UUID().uuidString)", + isDirectory: true + ) + try fileManager.createDirectory(at: root, withIntermediateDirectories: true) + + do { + switch kind { + case .deepAndWide: + try makeDeepAndWideFixture(root: root, fileManager: fileManager) + return ScannerProofFixture(kind: kind, root: root, fault: nil, hardLinkPair: nil, mutations: []) + case .mutationHeavy: + let mutations = try makeMutationFixture(root: root, fileManager: fileManager) + return ScannerProofFixture(kind: kind, root: root, fault: nil, hardLinkPair: nil, mutations: mutations) + case .hardLinks: + let pair = try makeHardLinkFixture(root: root, fileManager: fileManager) + return ScannerProofFixture(kind: kind, root: root, fault: nil, hardLinkPair: pair, mutations: []) + case .cleanupCandidates: + try makeCleanupCandidateFixture(root: root, fileManager: fileManager) + return ScannerProofFixture(kind: kind, root: root, fault: nil, hardLinkPair: nil, mutations: []) + case .permissionLoss: + let faultPath = try makeFaultFixture(root: root, fileManager: fileManager, folderName: "permission-lost") + return ScannerProofFixture( + kind: kind, + root: root, + fault: FaultInjection(relativePath: faultPath, kind: .permissionLoss), + hardLinkPair: nil, + mutations: [] + ) + case .volumeLoss: + let faultPath = try makeFaultFixture(root: root, fileManager: fileManager, folderName: "volume-disappears") + return ScannerProofFixture( + kind: kind, + root: root, + fault: FaultInjection(relativePath: faultPath, kind: .volumeLoss), + hardLinkPair: nil, + mutations: [] + ) + } + } catch { + try? fileManager.removeItem(at: root) + throw error + } + } + + private static func makeDeepAndWideFixture(root: URL, fileManager: FileManager) throws { + var deepDirectory = root.appendingPathComponent("deep", isDirectory: true) + for level in 0..<16 { + try fileManager.createDirectory(at: deepDirectory, withIntermediateDirectories: true) + try writeFile( + at: deepDirectory.appendingPathComponent("level-\(String(format: "%02d", level)).dat"), + bytes: 512 + level, + seed: UInt8(level + 1) + ) + deepDirectory = deepDirectory.appendingPathComponent("level-\(String(format: "%02d", level))", isDirectory: true) + } + + let wideRoot = root.appendingPathComponent("wide", isDirectory: true) + for bucket in 0..<8 { + let bucketURL = wideRoot.appendingPathComponent("bucket-\(String(format: "%02d", bucket))", isDirectory: true) + try fileManager.createDirectory(at: bucketURL, withIntermediateDirectories: true) + for index in 0..<64 { + try writeFile( + at: bucketURL.appendingPathComponent("file-\(String(format: "%03d", index)).bin"), + bytes: 1_024 + index + bucket, + seed: UInt8((index + bucket) % 200 + 1) + ) + } + } + } + + private static func makeMutationFixture(root: URL, fileManager: FileManager) throws -> [Mutation] { + let mutationRoot = root.appendingPathComponent("mutation", isDirectory: true) + for branch in 0..<4 { + let branchURL = mutationRoot.appendingPathComponent("branch-\(String(format: "%02d", branch))", isDirectory: true) + try fileManager.createDirectory(at: branchURL, withIntermediateDirectories: true) + for file in 0..<12 { + try writeFile( + at: branchURL.appendingPathComponent("file-\(String(format: "%02d", file)).dat"), + bytes: 1_000 + branch * 100 + file, + seed: UInt8((branch * 12 + file) % 200 + 1) + ) + } + } + + return [ + .create(relativePath: "mutation/incoming/new-00.dat", bytes: 4_096, seed: 80), + .resize(relativePath: "mutation/branch-00/file-00.dat", bytes: 8_192, seed: 81), + .rename(from: "mutation/branch-01/file-01.dat", to: "mutation/branch-01/renamed-01.dat"), + .delete(relativePath: "mutation/branch-02/file-02.dat"), + .create(relativePath: "mutation/branch-03/nested/new-03.dat", bytes: 2_048, seed: 82), + .rename(from: "mutation/branch-00", to: "mutation/renamed-branch-00"), + .delete(relativePath: "mutation/branch-03/nested/new-03.dat"), + .create(relativePath: "mutation/renamed-branch-00/roundtrip.dat", bytes: 3_072, seed: 83), + .resize(relativePath: "mutation/renamed-branch-00/roundtrip.dat", bytes: 6_144, seed: 84), + .create(relativePath: "mutation/branch-02/recreated-02.dat", bytes: 1_536, seed: 85), + .rename(from: "mutation/incoming", to: "mutation/received"), + .delete(relativePath: "mutation/received/new-00.dat") + ] + } + + private static func makeHardLinkFixture(root: URL, fileManager: FileManager) throws -> HardLinkPair { + let directory = root.appendingPathComponent("hard-links", isDirectory: true) + try fileManager.createDirectory(at: directory, withIntermediateDirectories: true) + + let source = directory.appendingPathComponent("original.bin") + let alias = directory.appendingPathComponent("alias.bin") + try writeFile(at: source, bytes: 32_768, seed: 91) + try fileManager.linkItem(at: source, to: alias) + try writeFile(at: directory.appendingPathComponent("different.bin"), bytes: 32_769, seed: 92) + + return HardLinkPair(source: source, alias: alias) + } + + private static func makeCleanupCandidateFixture(root: URL, fileManager: FileManager) throws { + let caches = root.appendingPathComponent("Caches", isDirectory: true) + let derivedData = root.appendingPathComponent("DerivedData", isDirectory: true) + let archives = root.appendingPathComponent("archives", isDirectory: true) + let installers = root.appendingPathComponent("installers", isDirectory: true) + try [caches, derivedData, archives, installers].forEach { + try fileManager.createDirectory(at: $0, withIntermediateDirectories: true) + } + + try writeFile(at: caches.appendingPathComponent("payload.bin"), bytes: 1_024, seed: 111) + try writeFile(at: derivedData.appendingPathComponent("project.o"), bytes: 2_048, seed: 112) + try writeFile(at: archives.appendingPathComponent("archive.zip"), bytes: 3_072, seed: 113) + try writeFile(at: installers.appendingPathComponent("installer.dmg"), bytes: 4_096, seed: 114) + try writeFile(at: root.appendingPathComponent("temporary.tmp"), bytes: 5_120, seed: 115) + try writeFile(at: root.appendingPathComponent("keep.txt"), bytes: 6_144, seed: 116) + } + + private static func makeFaultFixture( + root: URL, + fileManager: FileManager, + folderName: String + ) throws -> String { + let healthy = root.appendingPathComponent("healthy", isDirectory: true) + let faulted = root.appendingPathComponent(folderName, isDirectory: true) + try fileManager.createDirectory(at: healthy, withIntermediateDirectories: true) + try fileManager.createDirectory(at: faulted, withIntermediateDirectories: true) + try writeFile(at: healthy.appendingPathComponent("survives.txt"), bytes: 2_048, seed: 101) + try writeFile(at: faulted.appendingPathComponent("should-not-be-visible.txt"), bytes: 2_048, seed: 102) + return folderName + } + + private static func writeFile(at url: URL, bytes: Int, seed: UInt8) throws { + try Data(repeating: seed, count: bytes).write(to: url, options: .atomic) + } +} + +private final class FaultInjectingFileManager: FileManager { + private let failurePath: String + private let failure: ScannerProofFixture.FaultKind + + init(failurePath: URL, failure: ScannerProofFixture.FaultKind) { + self.failurePath = failurePath.standardizedFileURL.path + self.failure = failure + super.init() + } + + override func contentsOfDirectory( + at url: URL, + includingPropertiesForKeys keys: [URLResourceKey]?, + options mask: FileManager.DirectoryEnumerationOptions = [] + ) throws -> [URL] { + if url.standardizedFileURL.path == failurePath { + throw failure.error + } + return try super.contentsOfDirectory( + at: url, + includingPropertiesForKeys: keys, + options: mask + ) + } +} diff --git a/Tests/StorageScopeCoreTests/ScannerProofHarnessTests.swift b/Tests/StorageScopeCoreTests/ScannerProofHarnessTests.swift new file mode 100644 index 0000000..b8af4cb --- /dev/null +++ b/Tests/StorageScopeCoreTests/ScannerProofHarnessTests.swift @@ -0,0 +1,264 @@ +import Foundation +import Testing +@testable import StorageScopeCore + +struct ScannerProofHarnessTests { + @Test("differential harness covers deep and wide directory shapes") + func deepAndWideFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.deepAndWide) + defer { fixture.tearDown() } + + let signature = try ScannerProofHarness.compare( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner, + context: "deep and wide fixture" + ) + + #expect(signature.scannedItemCount > 500) + #expect(signature.root.maxDepth >= 17) + #expect(signature.root.maxWidth >= 64) + #expect(signature.inaccessibleItemCount == 0) + } + + @Test("differential harness compares every mutation-heavy state") + func mutationHeavyFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.mutationHeavy) + defer { fixture.tearDown() } + + let signatures = try ScannerProofHarness.compareMutationSequence( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner + ) + + #expect(signatures.count == fixture.mutations.count + 1) + #expect(signatures.first?.treePaths.contains("mutation/branch-00/file-00.dat") == true) + #expect(signatures.last?.treePaths.contains("mutation/renamed-branch-00/roundtrip.dat") == true) + #expect(signatures.last?.treePaths.contains("mutation/received/new-00.dat") == false) + #expect(signatures.first?.treePaths != signatures.last?.treePaths) + } + + @Test("differential harness preserves hard-link fixture semantics") + func hardLinkFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.hardLinks) + defer { fixture.tearDown() } + + let signature = try ScannerProofHarness.compare( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner, + context: "hard-link fixture" + ) + let pair = try #require(fixture.hardLinkPair) + + let sourceAttributes = try FileManager.default.attributesOfItem(atPath: pair.source.path) + let aliasAttributes = try FileManager.default.attributesOfItem(atPath: pair.alias.path) + let sourceFileNumber = try #require(sourceAttributes[.systemFileNumber] as? NSNumber) + let aliasFileNumber = try #require(aliasAttributes[.systemFileNumber] as? NSNumber) + + #expect(sourceFileNumber == aliasFileNumber) + #expect(signature.duplicateSizeGroups.contains { group in + group.paths.contains(ScannerProofHarness.relativePath(pair.source, from: fixture.root)) && + group.paths.contains(ScannerProofHarness.relativePath(pair.alias, from: fixture.root)) + }) + } + + @Test("compact worker keeps cleanup candidate parity") + func cleanupCandidateFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.cleanupCandidates) + defer { fixture.tearDown() } + + let signature = try ScannerProofHarness.compare( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner, + context: "cleanup-candidate fixture" + ) + + let candidateKeys = Set(signature.cleanupCandidates.map { "\($0.kind):\($0.path)" }) + #expect(candidateKeys.contains("cacheFolder:Caches")) + #expect(candidateKeys.contains("buildArtifact:DerivedData")) + #expect(candidateKeys.contains("archive:archives/archive.zip")) + #expect(candidateKeys.contains("diskImage:installers/installer.dmg")) + #expect(candidateKeys.contains("temporary:temporary.tmp")) + } + + @Test("permission-loss fixture becomes an inaccessible leaf without crashing") + func permissionLossFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.permissionLoss) + defer { fixture.tearDown() } + + let signature = try ScannerProofHarness.compare( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner, + context: "permission-loss fixture" + ) + let fault = try #require(fixture.fault) + let faultPath = fixture.root.appendingPathComponent(fault.relativePath) + let scan = try ScannerProofHarness.scan( + fixture, + runner: ScannerProofHarness.fixedWorkerRunner + ) + let item = try #require(scan.rootItem.flattened().first { + ScannerProofHarness.relativePath($0.url, from: fixture.root) == fault.relativePath + }) + + #expect(fault.kind == .permissionLoss) + #expect(signature.inaccessibleItemCount > 0) + #expect(item.kind == .inaccessible) + #expect(item.children.isEmpty) + #expect(FileManager.default.fileExists(atPath: faultPath.path)) + } + + @Test("volume-loss fixture becomes an inaccessible leaf without crashing") + func volumeLossFixtureHasStableParity() throws { + let fixture = try ScannerProofFixture.make(.volumeLoss) + defer { fixture.tearDown() } + + let signature = try ScannerProofHarness.compare( + fixture, + baseline: ScannerProofHarness.legacyRunner, + candidate: ScannerProofHarness.fixedWorkerRunner, + context: "volume-loss fixture" + ) + let fault = try #require(fixture.fault) + let scan = try ScannerProofHarness.scan( + fixture, + runner: ScannerProofHarness.fixedWorkerRunner + ) + let item = try #require(scan.rootItem.flattened().first { + ScannerProofHarness.relativePath($0.url, from: fixture.root) == fault.relativePath + }) + + #expect(fault.kind == .volumeLoss) + #expect(signature.inaccessibleItemCount > 0) + #expect(item.kind == .inaccessible) + #expect(item.children.isEmpty) + } + + @Test("fixed-worker walker pauses before work and resumes to completion") + func fixedWorkerPauseResumeCompletes() throws { + let fixture = try ScannerProofFixture.make(.deepAndWide) + defer { fixture.tearDown() } + + let cancellation = ScanCancellation() + let outcome = ProofScanOutcome() + let scanner = FileSystemScanner( + fileManager: FileManager(), + walkerMode: .fixedWorker + ) + let options = ScanOptions( + duplicateVerificationByteLimit: 0, + maxDuplicateVerificationFiles: 0 + ) + + cancellation.pause() + let thread = Thread { + do { + _ = try scanner.scan( + root: fixture.root, + options: options, + cancellation: cancellation + ) + outcome.set(.completed) + } catch { + outcome.set(.failed(error)) + } + } + thread.start() + + Thread.sleep(forTimeInterval: 0.05) + #expect(outcome.value == nil) + cancellation.resume() + waitForOutcome(outcome) + + switch outcome.value { + case .completed: + break + case .failed(let error): + Issue.record("Expected fixed-worker scan to resume, got \(error).") + case nil: + Issue.record("Fixed-worker scan did not complete after resume.") + } + } + + @Test("fixed-worker walker cancels while paused without deadlocking") + func fixedWorkerCancelWhilePausedTerminates() throws { + let fixture = try ScannerProofFixture.make(.deepAndWide) + defer { fixture.tearDown() } + + let cancellation = ScanCancellation() + let outcome = ProofScanOutcome() + let scanner = FileSystemScanner( + fileManager: FileManager(), + walkerMode: .fixedWorker + ) + let options = ScanOptions( + duplicateVerificationByteLimit: 0, + maxDuplicateVerificationFiles: 0 + ) + + cancellation.pause() + let thread = Thread { + do { + _ = try scanner.scan( + root: fixture.root, + options: options, + cancellation: cancellation + ) + outcome.set(.completed) + } catch { + outcome.set(.failed(error)) + } + } + thread.start() + + Thread.sleep(forTimeInterval: 0.05) + #expect(outcome.value == nil) + cancellation.cancel() + waitForOutcome(outcome) + + switch outcome.value { + case .failed(let error): + if case FileSystemScannerError.cancelled = error { + break + } + Issue.record("Expected FileSystemScannerError.cancelled, got \(error).") + case .completed: + Issue.record("Expected fixed-worker scan to cancel while paused.") + case nil: + Issue.record("Fixed-worker scan did not terminate after cancellation.") + } + } + + private func waitForOutcome(_ outcome: ProofScanOutcome) { + let deadline = Date().addingTimeInterval(5) + while outcome.value == nil, Date() < deadline { + Thread.sleep(forTimeInterval: 0.02) + } + } +} + +private enum ProofScanResult { + case completed + case failed(Error) +} + +private final class ProofScanOutcome: @unchecked Sendable { + private let lock = NSLock() + private var result: ProofScanResult? + + var value: ProofScanResult? { + lock.lock() + defer { lock.unlock() } + return result + } + + func set(_ result: ProofScanResult) { + lock.lock() + self.result = result + lock.unlock() + } +} diff --git a/Tests/StorageScopeTests/ScanStoreAppPerformanceProofTests.swift b/Tests/StorageScopeTests/ScanStoreAppPerformanceProofTests.swift new file mode 100644 index 0000000..4304102 --- /dev/null +++ b/Tests/StorageScopeTests/ScanStoreAppPerformanceProofTests.swift @@ -0,0 +1,116 @@ +import Foundation +import Testing +@testable import StorageScope +@testable import StorageScopeCore + +private let appPerformanceProofEnabled = + ProcessInfo.processInfo.environment["STORAGESCOPE_RUN_APP_PERF_PROOF"] == "1" + +@MainActor +@Suite("ScanStore app performance proof") +struct ScanStoreAppPerformanceProofTests { + @Test( + "release app path stays within 15 percent of callback-free scanner", + .enabled(if: appPerformanceProofEnabled) + ) + func appPathStaysWithinBenchmarkGate() async throws { + let environment = ProcessInfo.processInfo.environment + let itemCount = environment["STORAGESCOPE_APP_PERF_ITEMS"].flatMap(Int.init) ?? 100_000 + let runCount = max(1, environment["STORAGESCOPE_APP_PERF_RUNS"].flatMap(Int.init) ?? 3) + let root = try SyntheticBenchmarkFixture.create( + items: itemCount, + depth: 5, + duplicateRatio: 0.1 + ) + defer { SyntheticBenchmarkFixture.remove(root) } + + let store = ScanStore() + let options = store.makeScannerOptions() + var callbackFreeDurations: [TimeInterval] = [] + var appDurations: [TimeInterval] = [] + var scannerDurations: [TimeInterval] = [] + var snapshotCounts: [Int] = [] + + for runIndex in 0.. 0 }) + #expect( + ratio <= 1.15, + "App median \(appMedian)s exceeded callback-free median \(callbackFreeMedian)s by more than 15%" + ) + } + + private func measureCallbackFreeScan(root: URL, options: ScanOptions) throws -> TimeInterval { + let startedAt = Date() + _ = try FileSystemScanner().scan(root: root, options: options) + return Date().timeIntervalSince(startedAt) + } + + private func measureAppScan( + store: ScanStore, + root: URL + ) async throws -> (wallDuration: TimeInterval, scannerDuration: TimeInterval, snapshotCount: Int) { + let startedAt = Date() + store.scanDeveloperFixturePath(root.path) + let deadline = ContinuousClock.now + .seconds(60) + while store.isScanning, ContinuousClock.now < deadline { + try await Task.sleep(for: .milliseconds(10)) + } + + guard !store.isScanning else { + throw AppPerformanceProofError.timedOut + } + guard let scan = store.scan else { + throw AppPerformanceProofError.missingScan + } + if let errorMessage = store.errorMessage { + throw AppPerformanceProofError.scanFailed(errorMessage) + } + return ( + Date().timeIntervalSince(startedAt), + scan.enumerateDuration, + scan.snapshotBuildCount + ) + } + + private func median(_ values: [TimeInterval]) -> TimeInterval { + let sorted = values.sorted() + return sorted[sorted.count / 2] + } +} + +private enum AppPerformanceProofError: Error { + case timedOut + case missingScan + case scanFailed(String) +} diff --git a/script/benchmark_scan.sh b/script/benchmark_scan.sh index 8742b00..abeda40 100755 --- a/script/benchmark_scan.sh +++ b/script/benchmark_scan.sh @@ -4,4 +4,19 @@ set -euo pipefail ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" cd "$ROOT_DIR" -swift run StorageScopeBenchmark "$@" +CONFIGURATION="release" +ARGS=() + +while (($#)); do + case "$1" in + --debug) + CONFIGURATION="debug" + ;; + *) + ARGS+=("$1") + ;; + esac + shift +done + +STORAGESCOPE_BENCHMARK_CONFIGURATION="$CONFIGURATION" swift run -c "$CONFIGURATION" StorageScopeBenchmark "${ARGS[@]}"