Skip to content

Complete scanner performance Phase 2 #122

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
RasputinKaiser merged 1 commit into from
Jul 13, 2026
Merged

Complete scanner performance Phase 2 #122

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
51 changes: 29 additions & 22 deletions Sources/StorageScope/Stores/ScanStore.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -233,8 +233,8 @@ func setSelectedView(_ view: SmartView) {
private var cachedVerifiedDuplicateGroupsKey: DerivedCacheKey?
private var cachedVerifiedDuplicateGroups: [VerifiedDuplicateGroup] = []

/// Test observability for `items(for:)` cache. DEBUG-only so production pays nothing.
#if DEBUG
/// Test observability for `items(for:)` cache. Excluded from normal release builds.
#if DEBUG || STORAGESCOPE_TESTING
private(set) var itemsCacheHitCount: Int = 0
private(set) var itemsCacheMissCount: Int = 0

Expand Down Expand Up @@ -933,18 +933,7 @@ func setSelectedView(_ view: SmartView) {
progress = ScanProgress(scannedItemCount: 0, totalBytes: 0, currentPath: url.path)

let scanID = UUID()
let thresholds = ScanOptionPolicy.interactiveScanThresholds()
let (excludedComponents, excludedPrefixes) = Self.splitExcludedPaths(filters.excludedPaths)
let options = ScanOptions(
includeHidden: filters.includeHiddenFiles,
oldFileAgeDays: filters.oldFileAgeDays,
largeFileThreshold: thresholds.largeFileThreshold,
duplicateCandidateThreshold: Int64(filters.duplicateCandidateThresholdMB) * 1_000_000,
maxRankedResults: Self.rankedResultsCap,
excludeEnabled: filters.excludeFoldersEnabled,
excludedPathComponents: excludedComponents,
excludedAbsolutePrefixes: excludedPrefixes
)
let options = makeScannerOptions()
let optionsSnapshot = currentScanOptions
let scanCancellation = ScanCancellation()
activeScanID = scanID
Expand All @@ -967,11 +956,10 @@ func setSelectedView(_ view: SmartView) {
// Task.cancel() into the scanner's ScanCancellation handle AND finishes the
// stream so the consumer exits cleanly.
let (progressStream, progressContinuation) = AsyncStream<ScanTick>.makeStream()
// The scanner's `progress` and `onSnapshot` callbacks fire at the same
// throttled cadence (`onSnapshot` is invoked from inside `emitProgressLocked`
// right after the progress callback) but as two separate calls. Pair the
// latest snapshot with the next progress tick on this side rather than adding
// a second stream, so the consumer only has one `for await` loop to drain.
// Snapshots are emitted before their paired progress tick, but at a slower
// cadence than progress. Pair a newly staged snapshot with the next tick on
// this side rather than adding a second stream, so the consumer only has one
// `for await` loop to drain; intervening progress-only ticks carry nil.
let pendingSnapshotLock = NSLock()
var pendingSnapshot: StorageScan?
let result = try await withTaskCancellationHandler {
Expand Down Expand Up @@ -1043,10 +1031,11 @@ func setSelectedView(_ view: SmartView) {

scan = result
let cacheToPersist = hashCache
let pathsScanned = Set(result.retainedItems.map { $0.url.standardizedFileURL.path })
let retainedItems = result.retainedItems
let persistLog = Self.log
let persistSignpostID = Self.signpostID
Task.detached(priority: .utility) { [weak self] in
let pathsScanned = Set(retainedItems.map { $0.url.standardizedFileURL.path })
os_signpost(.begin, log: persistLog, name: "persist", signpostID: persistSignpostID,
"entries=%d", cacheToPersist.entryCount)
_ = cacheToPersist.purgeStale(except: pathsScanned)
Expand Down Expand Up @@ -1126,6 +1115,24 @@ func setSelectedView(_ view: SmartView) {
}
}

/// Builds the exact scanner configuration used by the interactive app path.
/// Internal so the opt-in release performance proof can compare ScanStore against
/// a callback-free FileSystemScanner without drifting to different thresholds.
func makeScannerOptions() -> ScanOptions {
let thresholds = ScanOptionPolicy.interactiveScanThresholds()
let (excludedComponents, excludedPrefixes) = Self.splitExcludedPaths(filters.excludedPaths)
return ScanOptions(
includeHidden: filters.includeHiddenFiles,
oldFileAgeDays: filters.oldFileAgeDays,
largeFileThreshold: thresholds.largeFileThreshold,
duplicateCandidateThreshold: Int64(filters.duplicateCandidateThresholdMB) * 1_000_000,
maxRankedResults: Self.rankedResultsCap,
excludeEnabled: filters.excludeFoldersEnabled,
excludedPathComponents: excludedComponents,
excludedAbsolutePrefixes: excludedPrefixes
)
}

func items(for view: SmartView) -> [StorageItem] {
guard let scan else {
return []
Expand All @@ -1147,7 +1154,7 @@ func setSelectedView(_ view: SmartView) {
ignoredCleanupCandidateIDs: cleanupRelevant ? ignoredCleanupCandidateIDs : []
)
if cachedItemsKey == key {
#if DEBUG
#if DEBUG || STORAGESCOPE_TESTING
itemsCacheHitCount &+= 1
#endif
return cachedItems
Expand All @@ -1158,7 +1165,7 @@ func setSelectedView(_ view: SmartView) {
defer {
os_signpost(.end, log: Self.log, name: "items_rebuild", signpostID: Self.signpostID)
}
#if DEBUG
#if DEBUG || STORAGESCOPE_TESTING
itemsCacheMissCount &+= 1
#endif

Expand Down
4 changes: 2 additions & 2 deletions Sources/StorageScope/Stores/SearchRecentsStore.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ final class SearchRecentsStore: ObservableObject {
let task = Task.detached(priority: .utility) { [writer = Self.writer] in
await writer.write(snapshot, generation: generation)
}
#if DEBUG
#if DEBUG || STORAGESCOPE_TESTING
Self.pendingPersistTasks.append(task)
#endif
}
Expand All @@ -81,7 +81,7 @@ final class SearchRecentsStore: ObservableObject {
}
}

#if DEBUG
#if DEBUG || STORAGESCOPE_TESTING
/// Drains detached `persist()` tasks from earlier tests so a test that
/// mutates `UserDefaults` directly can assert deterministically. Production
/// callers should never need this: the actor-serialized writer is already
Expand Down
9 changes: 9 additions & 0 deletions Sources/StorageScopeCore/Models/StorageScan.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,8 +25,11 @@ public struct StorageScan: Sendable {
public let duplicateCandidateItemLimit: Int
public let duplicateCandidateItemsRetained: Int
public let duplicateCandidateItemsConsidered: Int
public let duplicateCandidateEvictionCount: Int
public let duplicateCandidateLimitReached: Bool
public let snapshotBuildCount: Int
public let duplicateVerificationDuration: TimeInterval
public let duplicateVerificationBytesRead: Int64
public let enumerateDuration: TimeInterval
public let cleanupCandidates: [CleanupCandidate]
public let isPartial: Bool
Expand All @@ -51,8 +54,11 @@ public struct StorageScan: Sendable {
duplicateCandidateItemLimit: Int = 0,
duplicateCandidateItemsRetained: Int = 0,
duplicateCandidateItemsConsidered: Int = 0,
duplicateCandidateEvictionCount: Int = 0,
duplicateCandidateLimitReached: Bool = false,
snapshotBuildCount: Int = 0,
duplicateVerificationDuration: TimeInterval = 0,
duplicateVerificationBytesRead: Int64 = 0,
enumerateDuration: TimeInterval = 0,
cleanupCandidates: [CleanupCandidate],
isPartial: Bool = false
Expand All @@ -75,8 +81,11 @@ public struct StorageScan: Sendable {
self.duplicateCandidateItemLimit = duplicateCandidateItemLimit
self.duplicateCandidateItemsRetained = duplicateCandidateItemsRetained
self.duplicateCandidateItemsConsidered = duplicateCandidateItemsConsidered
self.duplicateCandidateEvictionCount = duplicateCandidateEvictionCount
self.duplicateCandidateLimitReached = duplicateCandidateLimitReached
self.snapshotBuildCount = snapshotBuildCount
self.duplicateVerificationDuration = duplicateVerificationDuration
self.duplicateVerificationBytesRead = duplicateVerificationBytesRead
self.enumerateDuration = enumerateDuration
self.cleanupCandidates = cleanupCandidates
self.isPartial = isPartial
Expand Down
Loading
Loading