From 745ffc6107fcf7ea9b3a0c42f15af54e3db7eb78 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 11:05:42 -0700 Subject: [PATCH 01/11] feat(compact): store the NTFS File Reference (frs+seq) inline per record MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add `file_ref: u64` to `CompactRecord` — the NTFS File Reference `(sequence_number << 48) | frs`. The FRS (low 48 bits) is the MFT slot; the sequence number (high 16 bits) disambiguates slot reuse, so together they uniquely identify a *file incarnation*. This is the identity key the planned delete-diff needs (to tell a modified file from a recycled slot) and the FRS the forensic view needs — both first-class, hot-path consumers, so the reference is stored inline rather than in a cold sidecar. `frs` + `sequence_number` already come out of the parser (they live on `MftIndex`'s `FileRecord`) and were simply discarded; `build_compact_index` now packs them via `CompactRecord::pack_file_reference`. The USN live-create path carries FRS only (USN strips the sequence number), so a freshly-created record gets seq 0 until the next full read stamps the real generation. ADS/hardlink expansions share the base record's reference. Cost: the row grows 80 -> 88 bytes (+10% index memory, ~+38 MB per 5M files). An A/B micro-bench (5M-record scan, min-of-12 x3 on macOS) shows no measurable scan regression — sequential scan within +/-2% noise, random-access walk equal-or-better — because a simple scan is not purely bandwidth-bound at this row size. Build cost is negligible (one packed field per record in the existing bulk loop; the MFT parse dominates). Cache format bumped v11 -> v12 (the bulk `bytemuck` row-size change invalidates older caches at the header check; the daemon rebuilds). --- crates/uffs-core/src/compact/builder.rs | 12 ++++++ crates/uffs-core/src/compact/record.rs | 43 +++++++++++++++++-- crates/uffs-core/src/compact_cache.rs | 7 ++- crates/uffs-core/src/compact_loader/apply.rs | 4 ++ crates/uffs-core/src/compact_mmap/tests.rs | 1 + crates/uffs-core/src/search/filters/tests.rs | 1 + .../src/search/filters/tests_malformed.rs | 1 + 7 files changed, 64 insertions(+), 5 deletions(-) diff --git a/crates/uffs-core/src/compact/builder.rs b/crates/uffs-core/src/compact/builder.rs index 8ce1a8475..52982c721 100644 --- a/crates/uffs-core/src/compact/builder.rs +++ b/crates/uffs-core/src/compact/builder.rs @@ -76,6 +76,10 @@ fn expand_ads_streams( created: record.stdinfo.created, modified: record.stdinfo.modified, accessed: record.stdinfo.accessed, + file_ref: CompactRecord::pack_file_reference( + record.frs.raw(), + record.sequence_number, + ), name_offset, flags: record.stdinfo.flags, parent_idx, @@ -163,6 +167,10 @@ fn expand_links_and_ads( created: record.stdinfo.created, modified: record.stdinfo.modified, accessed: record.stdinfo.accessed, + file_ref: CompactRecord::pack_file_reference( + record.frs.raw(), + record.sequence_number, + ), name_offset: link.name.offset, flags: record.stdinfo.flags, parent_idx: link_parent, @@ -233,6 +241,10 @@ pub fn build_compact_index( created: record.stdinfo.created, modified: record.stdinfo.modified, accessed: record.stdinfo.accessed, + file_ref: CompactRecord::pack_file_reference( + record.frs.raw(), + record.sequence_number, + ), name_offset: name_ref.offset, flags: record.stdinfo.flags, parent_idx, diff --git a/crates/uffs-core/src/compact/record.rs b/crates/uffs-core/src/compact/record.rs index 74d86b54c..6a0ad2b4b 100644 --- a/crates/uffs-core/src/compact/record.rs +++ b/crates/uffs-core/src/compact/record.rs @@ -1,14 +1,14 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! The 80-byte [`CompactRecord`] row type + the NTFS metafile-name allowlist. +//! The 88-byte [`CompactRecord`] row type + the NTFS metafile-name allowlist. //! //! Extracted from `compact.rs` (file-size decomposition); the public path //! `crate::compact::CompactRecord` is preserved via re-export. /// Compact per-record data for in-memory search, filter, and sort. /// -/// 80 bytes per record (76 data + 4 explicit tail padding). +/// 88 bytes per record (87 data + 1 explicit tail padding). /// Derives `bytemuck::Pod` + `Zeroable` so the entire record array can be /// serialized/deserialized as a single bulk `memcpy` — no per-field encoding. #[derive(Debug, Clone, Copy, Default, bytemuck::Pod, bytemuck::Zeroable)] @@ -29,6 +29,15 @@ pub struct CompactRecord { pub modified: i64, /// Last access time (Unix microseconds). pub accessed: i64, + /// NTFS **File Reference**: `(sequence_number << 48) | frs`. The FRS (low + /// 48 bits) is the MFT slot; the sequence number (high 16 bits) + /// disambiguates slot reuse, so together they uniquely identify a *file + /// incarnation* — the key a delete/forensic diff must use to tell a + /// modified file from a recycled slot. `0` for synthetic rows (aggregate + /// rollups, ADS/hardlink expansions that share the base record identity). + /// Populated by + /// [`build_compact_index`](crate::compact::build_compact_index). + pub file_ref: u64, // ── u32 fields (4-byte aligned) ─────────────────────────────── /// Byte offset into the names blob. @@ -69,6 +78,32 @@ pub struct CompactRecord { pub _pad: [u8; 1], } +/// Mask for the 48-bit FRS half of a [`CompactRecord::file_ref`]. +const FRS_MASK: u64 = 0x0000_FFFF_FFFF_FFFF; + +impl CompactRecord { + /// Pack an NTFS **File Reference** from its FRS (MFT slot) and sequence + /// number (slot-reuse generation): `(sequence_number << 48) | frs`. + #[must_use] + pub fn pack_file_reference(frs: u64, sequence_number: u16) -> u64 { + (frs & FRS_MASK) | (u64::from(sequence_number) << 48) + } + + /// The FRS (MFT slot, low 48 bits) of this record's file reference. + #[must_use] + pub const fn frs(&self) -> u64 { + self.file_ref & FRS_MASK + } + + /// The sequence number (slot-reuse generation, high 16 bits) of this + /// record's file reference. `>> 48` always fits `u16`, so the `try_from` + /// never fails; `unwrap_or` avoids an `unwrap`/`expect` for the lints. + #[must_use] + pub fn sequence_number(&self) -> u16 { + u16::try_from(self.file_ref >> 48).unwrap_or(u16::MAX) + } +} + /// The fixed set of reserved NTFS metafile names: the `$`-prefixed records at /// reserved FRS 0–15 and under the `$Extend` directory. An NTFS volume can /// only ever contain *these* specific metafiles. @@ -324,6 +359,6 @@ impl CompactRecord { // Compile-time size assertion. const _: () = assert!( - size_of::() == 80, - "CompactRecord must be exactly 80 bytes" + size_of::() == 88, + "CompactRecord must be exactly 88 bytes" ); diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index e114c93a5..f8e617d9e 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -151,7 +151,12 @@ const COMPACT_MAGIC: &[u8; 8] = b"UFFSCOM\0"; /// 10 caches are rejected at the header check so the daemon does a fresh MFT /// rebuild and writes a v10 cache; carrying them forward with an empty /// `frs_to_compact` would silently disable the surgical-patch path. -const COMPACT_VERSION: u16 = 11; +/// - v12: `file_ref: u64` (the NTFS File Reference `(seq << 48) | frs`) added +/// to `CompactRecord`, growing the row 80 → 88 bytes. It is the identity key +/// for delete-diff and the forensic view. The record array is a bulk +/// `bytemuck` memcpy, so the row-size change alone invalidates older caches +/// at the header version check (a fresh MFT rebuild writes v12). +const COMPACT_VERSION: u16 = 12; mod filters_io; pub mod parked; diff --git a/crates/uffs-core/src/compact_loader/apply.rs b/crates/uffs-core/src/compact_loader/apply.rs index ae4d2a567..5642fb767 100644 --- a/crates/uffs-core/src/compact_loader/apply.rs +++ b/crates/uffs-core/src/compact_loader/apply.rs @@ -138,6 +138,10 @@ pub(super) fn apply_create( created: staged.meta.created, modified: staged.meta.modified, accessed: staged.meta.accessed, + // USN strips the sequence number from the file reference + // (uffs_mft::usn), so a live-created record carries FRS only + // (seq = 0) until the next full read stamps the real generation. + file_ref: CompactRecord::pack_file_reference(uffs_mft::usize_to_u64(frs_usize), 0), name_offset: staged.name_offset, flags: staged.meta.flags, parent_idx: staged.parent_idx, diff --git a/crates/uffs-core/src/compact_mmap/tests.rs b/crates/uffs-core/src/compact_mmap/tests.rs index 0b91a102f..8c5a19a99 100644 --- a/crates/uffs-core/src/compact_mmap/tests.rs +++ b/crates/uffs-core/src/compact_mmap/tests.rs @@ -44,6 +44,7 @@ fn synth_record(seed: u32) -> CompactRecord { created: i64::from(seed), modified: i64::from(seed) + 1_i64, accessed: i64::from(seed) + 2_i64, + file_ref: 0, name_offset: seed, flags: seed, parent_idx: seed, diff --git a/crates/uffs-core/src/search/filters/tests.rs b/crates/uffs-core/src/search/filters/tests.rs index 29c592a3f..e64eb580e 100644 --- a/crates/uffs-core/src/search/filters/tests.rs +++ b/crates/uffs-core/src/search/filters/tests.rs @@ -24,6 +24,7 @@ fn test_record(name: &str, names: &mut Vec) -> CompactRecord { created: 100_000_000, modified: 200_000_000, accessed: 300_000_000, + file_ref: 0, name_offset: offset, flags: 0x20, // ARCHIVE parent_idx: u32::MAX, diff --git a/crates/uffs-core/src/search/filters/tests_malformed.rs b/crates/uffs-core/src/search/filters/tests_malformed.rs index 6dd4f516e..2a40de9d5 100644 --- a/crates/uffs-core/src/search/filters/tests_malformed.rs +++ b/crates/uffs-core/src/search/filters/tests_malformed.rs @@ -31,6 +31,7 @@ fn record_with_raw_name(raw: &[u8], names: &mut Vec) -> CompactRecord { created: 1, modified: 2, accessed: 3, + file_ref: 0, name_offset: offset, flags: 0x20, parent_idx: u32::MAX, From 2e9c5f46265cc8b6d9859047b8298f75383986fb Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 12:28:54 -0700 Subject: [PATCH 02/11] feat(diff): snapshot-diff engine keyed on the NTFS File Reference MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 2 of the delete-visibility slice: a pure, deterministic `diff_records(baseline, current) -> DeltaReport` that set-differences two compact record arrays by File Reference and classifies every real row as added, deleted, or modified. - **Deleted** — reference in baseline, absent from current. - **Added** — reference in current, absent from baseline. - **Modified** — same reference, changed `size` or `modified` timestamp. Keying on the full File Reference `(seq << 48) | frs` — not FRS alone — is what makes a delete-then-reuse of the same MFT slot classify correctly: a bumped sequence number yields a different reference, so the pair is a delete plus an add, never a false "modify". That case is the anchor test. Synthetic rows (`file_ref == 0`: aggregate rollups, `default()` placeholders) are excluded from both sides — a zero reference is not a unique identity, and no real file can carry it (FRS 0 is the `$MFT` metafile, already dropped from the compact index at build time). Hard links / ADS share one reference, so each of a file's names is reported at row granularity — the caller resolves each row index to a path against the correct side (deleted → baseline, added/modified → current). `DeltaReport` carries row indices, not resolved strings, because path reconstruction needs the whole index (the parent-chain walk); keeping the engine index-only makes it allocation-bounded and trivially unit-testable against synthetic index pairs. 10 tests cover add / delete / modify (size and mtime) / slot-reuse / synthetic-skip / hard-link fan-out / mixed. `diff_indexes(&DriveCompactIndex, ..)` is the thin wrapper over loaded indexes. The CLI/daemon surface (Phase 3) consumes this. --- crates/uffs-core/src/diff.rs | 306 +++++++++++++++++++++++++++++++++++ crates/uffs-core/src/lib.rs | 1 + 2 files changed, 307 insertions(+) create mode 100644 crates/uffs-core/src/diff.rs diff --git a/crates/uffs-core/src/diff.rs b/crates/uffs-core/src/diff.rs new file mode 100644 index 000000000..9644fc91c --- /dev/null +++ b/crates/uffs-core/src/diff.rs @@ -0,0 +1,306 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Snapshot diff: classify the delta between two full compact indexes. +//! +//! The `--newer` (timestamp) delta path can report files *created or modified* +//! after a date, but is structurally blind to *deletions* — a deleted file +//! simply stops appearing, and a timestamp cannot express "this is gone". The +//! USN journal has an explicit `FILE_DELETE` reason; the journal-free fallback +//! recovers delete visibility by comparing two full reads. +//! +//! This module is the deterministic core of that fallback: +//! [`diff_records`] set-differences a baseline against a current index by +//! **NTFS File Reference** — `(sequence_number << 48) | frs`, stored inline on +//! every [`CompactRecord`] as [`CompactRecord::file_ref`]. Keying on the FRS +//! (MFT slot) alone would misclassify a delete-then-reuse of the same slot as a +//! *modify*; the sequence number makes it exact — `(frs=N, seq=3)` in the +//! baseline and `(frs=N, seq=4)` in the current is a **delete of seq 3 plus an +//! add of seq 4**, not a modification. +//! +//! See `docs/architecture/delete-visibility-snapshot-diff.md` for the full +//! design (Mechanism 1: snapshot diff). + +use rustc_hash::{FxHashMap, FxHashSet}; + +use crate::compact::{CompactRecord, DriveCompactIndex}; + +/// The classified delta between a baseline and a current compact index. +/// +/// Every entry is a **row index** into the corresponding index's record array, +/// not a resolved path: path reconstruction needs the whole index (the +/// parent-chain walk), so the caller resolves each index via +/// [`crate::tree::resolve_path`] against the right side — `deleted` against the +/// baseline, `added` / `modified` against the current. +/// +/// Rows are reported at *name* granularity: a file with N hard links (which +/// share one File Reference) contributes N rows, so each affected path is +/// surfaced. Synthetic rows (aggregate rollups, `file_ref == 0`) are never +/// classified — see [`diff_records`]. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct DeltaReport { + /// Row indices into the **current** index whose File Reference is absent + /// from the baseline (newly created since the baseline). + pub added: Vec, + /// Row indices into the **baseline** index whose File Reference is absent + /// from the current (deleted since the baseline). + pub deleted: Vec, + /// Row indices into the **current** index whose File Reference is present + /// in the baseline but whose `size` or `modified` timestamp changed. + pub modified: Vec, +} + +impl DeltaReport { + /// Total number of classified rows across all three classes. + #[must_use] + pub const fn len(&self) -> usize { + self.added.len() + self.deleted.len() + self.modified.len() + } + + /// Whether the two indexes were identical at File-Reference granularity + /// (no adds, deletes, or in-place modifications). + #[must_use] + pub const fn is_empty(&self) -> bool { + self.added.is_empty() && self.deleted.is_empty() && self.modified.is_empty() + } +} + +/// The metadata a modify-detection compares: logical size and last-write time. +/// +/// Every row that shares a File Reference (hard links / ADS of one file) shares +/// this metadata — it comes from the single base MFT record — so keying the +/// baseline map on `file_ref` and storing the first row's `(size, modified)` is +/// consistent for all of that file's rows. +type Meta = (u64, i64); + +/// Whether a record participates in the diff. +/// +/// `file_ref == 0` marks a synthetic row — an aggregate rollup, an +/// unresolved/USN-fresh placeholder, or a `CompactRecord::default()`. A zero +/// reference is not a unique file identity (every synthetic row shares it), so +/// such rows are excluded from both sides of the diff. Real files can never +/// have `file_ref == 0`: that would require FRS 0 (the `$MFT` metafile itself), +/// which is excluded from the compact index at build time. +#[inline] +const fn is_real(rec: &CompactRecord) -> bool { + rec.file_ref != 0 +} + +/// Diff two compact record arrays, classifying every real row as added, +/// deleted, or modified by NTFS File Reference. +/// +/// - **Deleted** — File Reference in `baseline`, absent from `current`. +/// - **Added** — File Reference in `current`, absent from `baseline`. +/// - **Modified** — same File Reference in both, changed `size` or `modified`. +/// +/// A delete-then-reuse of the same MFT slot bumps the sequence number, so the +/// old and new File References differ and the pair is reported as a delete plus +/// an add — never as a modify. Synthetic rows (`file_ref == 0`) are skipped. +/// +/// Deterministic and allocation-bounded: two hash builds over the inputs plus +/// the three result vectors. Row indices are emitted in ascending array order. +#[must_use] +pub fn diff_records(baseline: &[CompactRecord], current: &[CompactRecord]) -> DeltaReport { + // Baseline: File Reference -> (size, modified) of the first row seen for it. + let mut base_meta: FxHashMap = + FxHashMap::with_capacity_and_hasher(baseline.len(), rustc_hash::FxBuildHasher); + for rec in baseline.iter().filter(|rec| is_real(rec)) { + base_meta + .entry(rec.file_ref) + .or_insert((rec.size, rec.modified)); + } + + // Current: the set of live File References (for the delete pass). + let mut current_refs: FxHashSet = + FxHashSet::with_capacity_and_hasher(current.len(), rustc_hash::FxBuildHasher); + for rec in current.iter().filter(|rec| is_real(rec)) { + current_refs.insert(rec.file_ref); + } + + let mut report = DeltaReport::default(); + + // Added + modified: walk the current rows. + for (idx, rec) in current.iter().enumerate() { + if !is_real(rec) { + continue; + } + match base_meta.get(&rec.file_ref) { + None => report.added.push(len_to_u32(idx)), + Some(&(base_size, base_modified)) => { + if base_size != rec.size || base_modified != rec.modified { + report.modified.push(len_to_u32(idx)); + } + } + } + } + + // Deleted: baseline rows whose File Reference vanished from the current. + for (idx, rec) in baseline.iter().enumerate() { + if is_real(rec) && !current_refs.contains(&rec.file_ref) { + report.deleted.push(len_to_u32(idx)); + } + } + + report +} + +/// Diff two loaded compact indexes. Thin wrapper over [`diff_records`] that +/// operates on their record arrays; see that function for the semantics. +#[must_use] +pub fn diff_indexes(baseline: &DriveCompactIndex, current: &DriveCompactIndex) -> DeltaReport { + diff_records(&baseline.records, ¤t.records) +} + +/// A record array index (bounded by the index size, which fits `u32` by +/// construction) narrowed to the `u32` the result vectors carry. +#[inline] +fn len_to_u32(idx: usize) -> u32 { + uffs_mft::len_to_u32(idx) +} + +#[cfg(test)] +mod tests { + use super::{DeltaReport, diff_records}; + use crate::compact::CompactRecord; + + /// Build a real (non-synthetic) record with the given File Reference parts + /// and the metadata the diff keys on. `name_offset` is set to `idx` only so + /// distinct rows are visibly distinct; the diff ignores it. + fn rec(frs: u64, seq: u16, size: u64, modified: i64) -> CompactRecord { + CompactRecord { + size, + modified, + file_ref: CompactRecord::pack_file_reference(frs, seq), + ..CompactRecord::default() + } + } + + #[test] + fn identical_indexes_produce_an_empty_delta() { + let baseline = [rec(10, 1, 100, 5), rec(11, 1, 200, 6)]; + let current = baseline; + let report = diff_records(&baseline, ¤t); + assert!(report.is_empty(), "no changes must yield an empty delta"); + assert_eq!(report.len(), 0); + } + + #[test] + fn pure_add_is_classified_added() { + let baseline = [rec(10, 1, 100, 5)]; + let current = [rec(10, 1, 100, 5), rec(12, 1, 50, 9)]; + let report = diff_records(&baseline, ¤t); + assert_eq!(report.added, vec![1], "the new row (idx 1) is an add"); + assert!(report.deleted.is_empty()); + assert!(report.modified.is_empty()); + } + + #[test] + fn pure_delete_is_classified_deleted() { + let baseline = [rec(10, 1, 100, 5), rec(11, 1, 200, 6)]; + let current = [rec(10, 1, 100, 5)]; + let report = diff_records(&baseline, ¤t); + assert_eq!(report.deleted, vec![1], "baseline idx 1 vanished"); + assert!(report.added.is_empty()); + assert!(report.modified.is_empty()); + } + + #[test] + fn changed_size_is_classified_modified() { + let baseline = [rec(10, 1, 100, 5)]; + let current = [rec(10, 1, 999, 5)]; + let report = diff_records(&baseline, ¤t); + assert_eq!(report.modified, vec![0], "same ref, changed size → modify"); + assert!(report.added.is_empty()); + assert!(report.deleted.is_empty()); + } + + #[test] + fn changed_mtime_is_classified_modified() { + let baseline = [rec(10, 1, 100, 5)]; + let current = [rec(10, 1, 100, 77)]; + let report = diff_records(&baseline, ¤t); + assert_eq!(report.modified, vec![0], "same ref, changed mtime → modify"); + } + + /// The anchor test: a delete-then-reuse of the *same MFT slot* bumps the + /// sequence number. FRS-only keying would call this a "modify"; keying on + /// the full File Reference makes it an exact delete + add. + #[test] + fn slot_reuse_is_delete_plus_add_not_modify() { + let baseline = [rec(10, 3, 100, 5)]; // (frs=10, seq=3) + let current = [rec(10, 4, 4096, 9)]; // same slot, seq bumped → different file + let report = diff_records(&baseline, ¤t); + assert_eq!(report.deleted, vec![0], "seq-3 incarnation was deleted"); + assert_eq!(report.added, vec![0], "seq-4 incarnation was added"); + assert!( + report.modified.is_empty(), + "slot reuse must NOT be reported as an in-place modify", + ); + } + + #[test] + fn synthetic_rows_file_ref_zero_are_ignored() { + // A default (file_ref == 0) row on each side plus one real unchanged + // file. Only the real file participates; the synthetic rows never + // classify, even though their default (size, modified) "match". + let baseline = [CompactRecord::default(), rec(10, 1, 100, 5)]; + let current = [ + CompactRecord::default(), + rec(10, 1, 100, 5), + CompactRecord::default(), + ]; + let report = diff_records(&baseline, ¤t); + assert!( + report.is_empty(), + "synthetic file_ref==0 rows must never be added/deleted/modified, got {report:?}", + ); + } + + #[test] + fn hard_links_sharing_a_reference_all_report_on_delete() { + // Two names (hard links) share one File Reference. Deleting the file + // drops both baseline rows; each is a distinct path, so both report. + let shared = rec(20, 2, 512, 3); + let baseline = [shared, shared]; + let current: [CompactRecord; 0] = []; + let report = diff_records(&baseline, ¤t); + assert_eq!( + report.deleted, + vec![0, 1], + "both hard-link rows of the deleted file must surface", + ); + } + + #[test] + fn mixed_delta_classifies_each_class_independently() { + // idx0 unchanged, idx1 deleted, plus one add and one in-place modify. + let baseline = [ + rec(10, 1, 100, 5), // unchanged + rec(11, 1, 200, 6), // deleted + rec(12, 1, 300, 7), // will be modified + ]; + let current = [ + rec(10, 1, 100, 5), // unchanged + rec(12, 1, 4096, 7), // idx1: modified (size changed) + rec(13, 1, 10, 8), // idx2: added + ]; + let report = diff_records(&baseline, ¤t); + assert_eq!(report.added, vec![2]); + assert_eq!(report.deleted, vec![1]); + assert_eq!(report.modified, vec![1]); + assert_eq!(report.len(), 3); + } + + #[test] + fn delta_report_len_and_is_empty_agree() { + let empty = DeltaReport::default(); + assert!(empty.is_empty()); + assert_eq!(empty.len(), 0); + let one = DeltaReport { + added: vec![0], + ..DeltaReport::default() + }; + assert!(!one.is_empty()); + assert_eq!(one.len(), 1); + } +} diff --git a/crates/uffs-core/src/lib.rs b/crates/uffs-core/src/lib.rs index 86a2bf7fa..1c44f0fa1 100644 --- a/crates/uffs-core/src/lib.rs +++ b/crates/uffs-core/src/lib.rs @@ -126,6 +126,7 @@ pub mod compact_loader; pub(crate) mod compact_mmap; pub mod compact_storage; pub(crate) mod compiled_pattern; +pub mod diff; mod error; mod export; pub mod extensions; From f3be94654e7eea1c81df6a00facbdd7ff8549831 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:12:25 -0700 Subject: [PATCH 03/11] feat(diff): resolve a snapshot delta to full paths (ResolvedDelta) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the presentation layer on top of the diff engine: `resolve_delta` runs `diff_indexes`, then walks each classified row's parent chain (`resolve_path`) to a full `C:\…` path with its size + mtime, returning a `ResolvedDelta { added, deleted, modified, truncated }` of `DeltaEntry`. Adds resolve against the current index, deletes against the baseline (each side owns its rows), and `limit` caps every class independently, flagging `truncated` when it drops rows so a caller never mistakes a capped list for the whole delta. This is the exact shape the daemon RPC serialises. Two tests over minimally-built resolvable drives pin it: add/delete/modify each resolve to the right full path with the right side's metadata (delete keeps the baseline size, modify the current size), and the limit path caps + sets `truncated`. --- crates/uffs-core/src/diff.rs | 215 ++++++++++++++++++++++++++++++++++- 1 file changed, 214 insertions(+), 1 deletion(-) diff --git a/crates/uffs-core/src/diff.rs b/crates/uffs-core/src/diff.rs index 9644fc91c..e3bba13c6 100644 --- a/crates/uffs-core/src/diff.rs +++ b/crates/uffs-core/src/diff.rs @@ -23,7 +23,8 @@ use rustc_hash::{FxHashMap, FxHashSet}; -use crate::compact::{CompactRecord, DriveCompactIndex}; +use crate::compact::{CompactRecord, DriveCompactIndex, MalformedRender}; +use crate::search::tree::resolve_path; /// The classified delta between a baseline and a current compact index. /// @@ -158,6 +159,94 @@ fn len_to_u32(idx: usize) -> u32 { uffs_mft::len_to_u32(idx) } +// ──────────────────────────────────────────────────────────────────────────── +// Path-resolved surface (what the daemon RPC / CLI consume) +// ──────────────────────────────────────────────────────────────────────────── + +/// One classified change with its row resolved to a full path and the metadata +/// a caller needs to render it. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct DeltaEntry { + /// Full path (`C:\Users\…\file.ext`), reconstructed by walking the parent + /// chain in the side the row belongs to (baseline for deletes, current for + /// adds/modifies). + pub path: String, + /// Logical file size in bytes. For a modify this is the *current* size; for + /// a delete it is the last-known size from the baseline. + pub size: u64, + /// Last-write time (Unix microseconds), from the same side as `path`. + pub modified: i64, +} + +/// A [`DeltaReport`] with every row index resolved to a full path + metadata — +/// the presentation-ready form the daemon returns over the wire. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct ResolvedDelta { + /// Files present in the current index but not the baseline (created). + pub added: Vec, + /// Files present in the baseline but not the current (deleted). + pub deleted: Vec, + /// Files in both whose `size` or `modified` timestamp changed. + pub modified: Vec, + /// `true` when `limit` capped at least one class (more rows exist than were + /// returned). `false` means every classified row is present. + pub truncated: bool, +} + +/// Diff two loaded indexes and resolve each classified row to a full path. +/// +/// `limit` caps **each class independently** (`0` = unlimited); `truncated` is +/// set when any class had more rows than `limit`. Adds and modifies resolve +/// against `current`; deletes resolve against `baseline` — see [`DeltaReport`] +/// for why each side owns its rows. +#[must_use] +pub fn resolve_delta( + baseline: &DriveCompactIndex, + current: &DriveCompactIndex, + limit: usize, +) -> ResolvedDelta { + let report = diff_indexes(baseline, current); + let mut truncated = false; + let added = resolve_class(current, &report.added, limit, &mut truncated); + let deleted = resolve_class(baseline, &report.deleted, limit, &mut truncated); + let modified = resolve_class(current, &report.modified, limit, &mut truncated); + ResolvedDelta { + added, + deleted, + modified, + truncated, + } +} + +/// Resolve one class's row indices against `drive`, capping at `limit` +/// (`0` = unlimited) and flagging `truncated` when the cap drops any rows. +fn resolve_class( + drive: &DriveCompactIndex, + indices: &[u32], + limit: usize, + truncated: &mut bool, +) -> Vec { + let prefix = format!("{}:\\", drive.letter); + let capped = if limit > 0 && indices.len() > limit { + *truncated = true; + indices.get(..limit).unwrap_or(indices) + } else { + indices + }; + capped + .iter() + .filter_map(|&raw_idx| { + let idx = raw_idx as usize; + let rec = drive.records.get(idx)?; + Some(DeltaEntry { + path: resolve_path(drive, idx, &prefix, MalformedRender::Lossy), + size: rec.size, + modified: rec.modified, + }) + }) + .collect() +} + #[cfg(test)] mod tests { use super::{DeltaReport, diff_records}; @@ -303,4 +392,128 @@ mod tests { assert!(!one.is_empty()); assert_eq!(one.len(), 1); } + + // ── Path-resolved surface ──────────────────────────────────────────── + + use alloc::sync::Arc; + use std::path::PathBuf; + + use uffs_text::case_fold::CaseFold; + + use super::resolve_delta; + use crate::compact::{ChildrenIndex, DriveCompactIndex, ExtensionIndex, IndexSource}; + use crate::compact_storage::ColumnStorage; + use crate::trigram::TrigramIndex; + + /// Shared names blob for the resolution fixtures: + /// `C`[0..1] `docs`[1..5] `a.txt`[5..10] `b.txt`[10..15] `c.txt`[15..20]. + const NAMES: &[u8] = b"Cdocsa.txtb.txtc.txt"; + + /// A leaf-file record under `docs` (idx 1) with the given identity + size. + fn file(name_offset: u32, first: u8, frs: u64, size: u64, modified: i64) -> CompactRecord { + CompactRecord { + size, + modified, + file_ref: CompactRecord::pack_file_reference(frs, 1), + name_offset, + parent_idx: 1, + name_len: 5, + name_first_byte: first, + ..CompactRecord::default() + } + } + + /// Build a resolvable drive: root `C` (idx0), dir `docs` (idx1), then the + /// given leaf files (idx2..). Root/dir carry no diff identity (`file_ref` + /// 0 / an unchanging dir ref), so only the leaves drive the delta. + fn drive(files: Vec) -> DriveCompactIndex { + let mut records = vec![ + CompactRecord { + name_offset: 0, + flags: 0x10, + parent_idx: u32::MAX, + name_len: 1, + name_first_byte: b'C', + ..CompactRecord::default() + }, + CompactRecord { + file_ref: CompactRecord::pack_file_reference(100, 1), + name_offset: 1, + flags: 0x10, + parent_idx: 0, + name_len: 4, + name_first_byte: b'd', + ..CompactRecord::default() + }, + ]; + records.extend(files); + let names = NAMES.to_vec(); + let fold = CaseFold::default_table(); + let trigram = TrigramIndex::build(&records, &names, fold); + let children = ChildrenIndex::build(&records); + let ext_index = ExtensionIndex::build(&records); + DriveCompactIndex { + letter: uffs_mft::platform::DriveLetter::C, + records: ColumnStorage::from_vec(records), + names: ColumnStorage::from_vec(names), + trigram: Arc::new(trigram), + children: Arc::new(children), + ext_index: Arc::new(ext_index), + fold, + ext_names: vec![Box::from("")], + source: IndexSource::MftFile(PathBuf::from("C:")), + source_epoch: 1, + bloom: None, + path_trie: None, + frs_to_compact: Vec::new(), + delta: None, + } + } + + #[test] + fn resolve_delta_classifies_and_resolves_full_paths() { + // baseline: a.txt (200), b.txt (201). + let baseline = drive(vec![ + file(5, b'a', 200, 100, 5), + file(10, b'b', 201, 200, 6), + ]); + // current: a.txt grew (modified), b.txt gone (delete), c.txt new (add). + let current = drive(vec![file(5, b'a', 200, 999, 5), file(15, b'c', 202, 50, 9)]); + + let delta = resolve_delta(&baseline, ¤t, 0); + + assert_eq!(delta.added.len(), 1, "c.txt is the only add"); + let added = delta.added.first().expect("one add"); + assert!(added.path.ends_with("docs\\c.txt"), "{:?}", added.path); + assert_eq!(added.size, 50); + + assert_eq!(delta.deleted.len(), 1, "b.txt is the only delete"); + let deleted = delta.deleted.first().expect("one delete"); + assert!(deleted.path.ends_with("docs\\b.txt"), "{:?}", deleted.path); + assert_eq!(deleted.size, 200, "delete carries the baseline size"); + + assert_eq!(delta.modified.len(), 1, "a.txt is the only modify"); + let modified = delta.modified.first().expect("one modify"); + assert!( + modified.path.ends_with("docs\\a.txt"), + "{:?}", + modified.path + ); + assert_eq!(modified.size, 999, "modify carries the current size"); + + assert!(!delta.truncated, "no limit → nothing truncated"); + } + + #[test] + fn resolve_delta_limit_caps_each_class_and_flags_truncation() { + // Two adds; a limit of 1 keeps one and marks the delta truncated. + let baseline = drive(vec![]); + let current = drive(vec![file(5, b'a', 200, 1, 1), file(10, b'b', 201, 2, 2)]); + let delta = resolve_delta(&baseline, ¤t, 1); + assert_eq!(delta.added.len(), 1, "limit 1 keeps a single add"); + assert!( + delta.truncated, + "dropping the second add must flag truncation" + ); + } } From 525a5fcd7f4e0373777338757fb7b353af1cd4ec Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:31:55 -0700 Subject: [PATCH 04/11] =?UTF-8?q?feat(diff):=20uffs=20--diff=20=E2=80=94?= =?UTF-8?q?=20daemon=20RPC=20surfacing=20the=20snapshot=20delete=20diff?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 3: wire the delete-visibility engine to a user-facing command. `uffs --diff --drive ` loads a baseline MFT capture and diffs it against the drive's LIVE in-memory index — the deletion-visible companion to `--newer`, which is structurally blind to deletes. Runs against the hot index the daemon already serves searches from, so it needs no cold rebuild. Layering (each layer verified where it can be): - uffs-core: `resolve_delta` already resolves the delta to full paths. - uffs-client: `diff_wire` (DiffParams / DiffResultWire / DiffEntryWire) wire types + a `UffsClientSync::diff` RPC helper. Round-trip tested. - uffs-daemon: `handle_diff` (own sibling file, 800-LOC policy) + `IndexManager::diff_snapshot` — snapshot the registry for the live current side, load the baseline off-thread, run the engine, map to the wire result. `DriveNotLoaded` -> ERR_NOT_READY, baseline-load failure -> ERR_INTERNAL. The pure wire-mapping is unit tested; the live path is Windows/daemon-only (no small MFT fixture exists to drive it on the host). - uffs-cli: `--diff` command (thin client) with arg parsing, a grouped human table (deleted rows carry their last-known size — the "what did I lose" figure), `--limit N`, and `--json`. Arg parsing fully tested. The `--diff` token is disjoint from every search flag, so `uffs diff` (bare) still searches for "diff"; only `--diff` as the first token is the command. --- crates/uffs-cli/src/args.rs | 31 +++ crates/uffs-cli/src/commands.rs | 2 + crates/uffs-cli/src/commands/diff.rs | 234 +++++++++++++++++++ crates/uffs-cli/src/dispatch.rs | 5 + crates/uffs-client/src/connect_sync_diff.rs | 30 +++ crates/uffs-client/src/lib.rs | 4 + crates/uffs-client/src/protocol/diff_wire.rs | 105 +++++++++ crates/uffs-client/src/protocol/mod.rs | 2 + crates/uffs-daemon/src/handler.rs | 7 + crates/uffs-daemon/src/handler_diff.rs | 77 ++++++ crates/uffs-daemon/src/index/diff.rs | 159 +++++++++++++ crates/uffs-daemon/src/index/mod.rs | 1 + 12 files changed, 657 insertions(+) create mode 100644 crates/uffs-cli/src/commands/diff.rs create mode 100644 crates/uffs-client/src/connect_sync_diff.rs create mode 100644 crates/uffs-client/src/protocol/diff_wire.rs create mode 100644 crates/uffs-daemon/src/handler_diff.rs create mode 100644 crates/uffs-daemon/src/index/diff.rs diff --git a/crates/uffs-cli/src/args.rs b/crates/uffs-cli/src/args.rs index a63e2a4ce..282a5fa0a 100644 --- a/crates/uffs-cli/src/args.rs +++ b/crates/uffs-cli/src/args.rs @@ -504,6 +504,7 @@ COMMANDS: --search Explicit search (same as the bare default) --stats [PATH] Show filesystem statistics --agg Run aggregate analytics + --diff Diff a baseline MFT snapshot vs the live index (deletes) --daemon Manage the UFFS daemon (start/stop/load/status) --mcp Manage the UFFS MCP server --update [ACTION] Self-update (snapshot/acquire/apply/doctor/recover) @@ -621,6 +622,36 @@ pub(crate) fn print_stats_help() { print!("{STATS_HELP}"); } +/// Help text for `uffs --diff`. +const DIFF_HELP: &str = "\ +uffs --diff — Snapshot delete-visibility diff + +Diff a baseline MFT capture against the drive's LIVE in-memory index and report +what was created, deleted, or modified since the baseline. The deletion-visible +companion to --newer (which can only see creates/modifies). The drive must be +loaded in a running daemon. + +USAGE: uffs --diff --drive [OPTIONS] + +ARGUMENTS: + Path to the baseline snapshot (raw MFT capture) to + diff the live index against. + +OPTIONS: + -d, --drive Drive letter the baseline covers (required, e.g. C). + -n, --limit Max entries per class (added/deleted/modified); 0 = all. + --json Emit the raw result as JSON instead of a table. + +EXAMPLE: + uffs --diff D:\\snapshots\\c_last_week.bin --drive C +"; + +/// Print diff help. +#[expect(clippy::print_stdout, reason = "intentional help output")] +pub(crate) fn print_diff_help() { + print!("{DIFF_HELP}"); +} + /// Help text for `uffs --agg`. const AGGREGATE_HELP: &str = "\ uffs --agg — Run aggregate analytics on the filesystem index diff --git a/crates/uffs-cli/src/commands.rs b/crates/uffs-cli/src/commands.rs index 8f00d7284..034360ba5 100644 --- a/crates/uffs-cli/src/commands.rs +++ b/crates/uffs-cli/src/commands.rs @@ -23,6 +23,8 @@ pub(crate) mod daemon_status; /// under the 800-LOC policy ceiling. Forward-looking: 8-D `forget` /// and 8-E `status_drives` will land their shims here as well. pub(crate) mod daemon_tiering; +/// `uffs --diff --drive ` — snapshot delete-visibility diff. +pub(crate) mod diff; /// Shared elevation gate for the mutating flows (uninstall / update): surface /// admin-only work up front and decide once (elevate / continue-without / /// abort) instead of failing mid-flow. Keeps both flows' elevation UX aligned. diff --git a/crates/uffs-cli/src/commands/diff.rs b/crates/uffs-cli/src/commands/diff.rs new file mode 100644 index 000000000..10d0281bd --- /dev/null +++ b/crates/uffs-cli/src/commands/diff.rs @@ -0,0 +1,234 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! `uffs --diff --drive ` — snapshot delete-visibility diff. +//! +//! Answers "what was created, deleted, or modified on a drive since a baseline +//! MFT capture" — the deletion-visible companion to `--newer`. Thin client: +//! parse args, fire the daemon's `diff` RPC (which diffs the baseline against +//! the drive's live in-memory index), render the classified delta. + +use anyhow::{Context as _, Result}; +use uffs_client::connect_sync::UffsClientSync; +use uffs_client::protocol::{DiffEntryWire, DiffParams, DiffResultWire}; +use uffs_mft::platform::DriveLetter; + +use crate::args::parse_drive_letter; + +/// Parsed `uffs --diff` invocation. +#[derive(Debug)] +struct DiffArgs { + /// Baseline snapshot path (raw MFT capture) to diff against. + baseline: String, + /// Drive letter the baseline covers and whose live index is the current + /// side. + drive: DriveLetter, + /// Max entries per class (0 = unlimited). + limit: u32, + /// Emit JSON instead of the human table. + json: bool, +} + +/// Run `uffs --diff --drive [--limit N] [--json]`. +/// +/// # Errors +/// +/// Returns an error on bad arguments, when the daemon is not running, or when +/// the `diff` RPC itself fails (drive not loaded / baseline unreadable). +pub(crate) fn run_diff(args: &[String]) -> Result<()> { + if args.iter().any(|arg| arg == "--help" || arg == "-h") { + crate::args::print_diff_help(); + return Ok(()); + } + + let parsed = parse_diff_args(args)?; + let mut client = UffsClientSync::connect_raw() + .map_err(|err| anyhow::anyhow!("Daemon is not running: {err}"))?; + + let params = DiffParams { + baseline: parsed.baseline, + drive: parsed.drive, + limit: parsed.limit, + }; + let result = client.diff(¶ms).with_context(|| "diff RPC failed")?; + + if parsed.json { + print_json(&result); + } else { + print_human(¶ms, &result); + } + Ok(()) +} + +/// Parse the `--diff` argument vector into a [`DiffArgs`]. +/// +/// The first non-flag token is the baseline path; `--drive`/`-d` is required; +/// `--limit`/`-n` and `--json` are optional. +fn parse_diff_args(args: &[String]) -> Result { + let mut baseline: Option = None; + let mut drive: Option = None; + let mut limit: u32 = 0; + let mut json = false; + + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--drive" | "-d" => { + let val = iter + .next() + .with_context(|| "`--drive` requires a drive letter (e.g. C)")?; + drive = Some(parse_drive_letter(val)?); + } + "--limit" | "-n" => { + let val = iter.next().with_context(|| "`--limit` requires a number")?; + limit = val + .parse::() + .with_context(|| format!("invalid --limit value '{val}'"))?; + } + "--json" => json = true, + other if other.starts_with('-') => { + anyhow::bail!("unknown flag '{other}'; see `uffs --diff --help`"); + } + other => { + if baseline.replace(other.to_owned()).is_some() { + anyhow::bail!("only one baseline path may be given; got a second: '{other}'"); + } + } + } + } + + let baseline_path = baseline.with_context( + || "missing baseline snapshot path; usage: uffs --diff --drive ", + )?; + let drive_letter = + drive.with_context(|| "missing `--drive `; the diff needs to know which drive")?; + Ok(DiffArgs { + baseline: baseline_path, + drive: drive_letter, + limit, + json, + }) +} + +/// Render the delta as a human-readable table grouped by change class. +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn print_human(params: &DiffParams, result: &DiffResultWire) { + println!( + "Diff of drive {} vs baseline {}:", + params.drive, params.baseline + ); + println!( + " deleted {}, added {}, modified {}{}", + result.deleted.len(), + result.added.len(), + result.modified.len(), + if result.truncated { + " (truncated — pass a larger --limit for the full list)" + } else { + "" + }, + ); + + print_section("Deleted", &result.deleted, true); + print_section("Added", &result.added, false); + print_section("Modified", &result.modified, false); +} + +/// Print one non-empty class section. `with_size` appends the byte size (the +/// "what did I lose" figure that matters most for deletes). +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn print_section(label: &str, entries: &[DiffEntryWire], with_size: bool) { + if entries.is_empty() { + return; + } + println!("\n{label}:"); + for entry in entries { + if with_size { + println!(" {} ({})", entry.path, human_bytes(entry.size)); + } else { + println!(" {}", entry.path); + } + } +} + +/// Emit the raw wire result as pretty JSON for scripting. +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn print_json(result: &DiffResultWire) { + match serde_json::to_string_pretty(result) { + Ok(json) => println!("{json}"), + Err(err) => println!("{{\"error\":\"failed to serialize diff result: {err}\"}}"), + } +} + +/// Humanise a byte count with binary units (integer arithmetic — no floats, +/// to satisfy the strict `clippy::float_arithmetic` gate). +fn human_bytes(bytes: u64) -> String { + const KIB: u64 = 1024; + const MIB: u64 = 1024 * KIB; + const GIB: u64 = 1024 * MIB; + if bytes >= GIB { + let whole = bytes / GIB; + let hundredths = (bytes % GIB).saturating_mul(100) / GIB; + format!("{whole}.{hundredths:02} GiB") + } else if bytes >= MIB { + format!("{} MiB", bytes / MIB) + } else if bytes >= KIB { + format!("{} KiB", bytes / KIB) + } else { + format!("{bytes} B") + } +} + +#[cfg(test)] +mod tests { + use super::parse_diff_args; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|item| (*item).to_owned()).collect() + } + + #[test] + fn parses_baseline_drive_and_limit() { + let parsed = + parse_diff_args(&args(&["C_old.bin", "--drive", "C", "--limit", "50"])).expect("parse"); + assert_eq!(parsed.baseline, "C_old.bin"); + assert_eq!(parsed.drive, uffs_mft::platform::DriveLetter::C); + assert_eq!(parsed.limit, 50); + assert!(!parsed.json); + } + + #[test] + fn drive_may_precede_the_positional_baseline() { + let parsed = parse_diff_args(&args(&["-d", "D", "snap.bin", "--json"])).expect("parse"); + assert_eq!(parsed.baseline, "snap.bin"); + assert_eq!(parsed.drive, uffs_mft::platform::DriveLetter::D); + assert_eq!(parsed.limit, 0, "no --limit → unlimited"); + assert!(parsed.json); + } + + #[test] + fn missing_drive_is_an_error() { + let err = parse_diff_args(&args(&["snap.bin"])).expect_err("must require --drive"); + assert!(err.to_string().contains("--drive"), "{err}"); + } + + #[test] + fn missing_baseline_is_an_error() { + let err = parse_diff_args(&args(&["--drive", "C"])).expect_err("must require baseline"); + assert!(err.to_string().contains("baseline"), "{err}"); + } + + #[test] + fn a_second_baseline_is_rejected() { + let err = parse_diff_args(&args(&["a.bin", "b.bin", "-d", "C"])) + .expect_err("two baselines must error"); + assert!(err.to_string().contains("second"), "{err}"); + } + + #[test] + fn unknown_flag_is_rejected() { + let err = parse_diff_args(&args(&["snap.bin", "-d", "C", "--bogus"])) + .expect_err("unknown flag must error"); + assert!(err.to_string().contains("unknown flag"), "{err}"); + } +} diff --git a/crates/uffs-cli/src/dispatch.rs b/crates/uffs-cli/src/dispatch.rs index 99d711e7e..9e02097b1 100644 --- a/crates/uffs-cli/src/dispatch.rs +++ b/crates/uffs-cli/src/dispatch.rs @@ -26,6 +26,8 @@ pub(crate) enum Command { Stats, /// `--agg `. Agg, + /// `--diff --drive `. + Diff, /// `--daemon `. Daemon, /// `--mcp `. @@ -46,6 +48,7 @@ impl Command { "--search" => Self::Search, "--stats" => Self::Stats, "--agg" | "--aggregate" => Self::Agg, + "--diff" => Self::Diff, "--daemon" => Self::Daemon, "--mcp" => Self::Mcp, // `--upgrade` is a HIDDEN alias for `--update` (winget/apt muscle @@ -67,6 +70,7 @@ const COMMAND_TOKENS: &[&str] = &[ "--stats", "--agg", "--aggregate", + "--diff", "--daemon", "--mcp", "--update", @@ -108,6 +112,7 @@ pub(crate) fn dispatch_command(command: Command, args: &[String]) -> Result<()> Command::Search => crate::run_search(args), Command::Stats => crate::run_stats(args), Command::Agg => crate::run_aggregate(args), + Command::Diff => commands::diff::run_diff(args), Command::Daemon => crate::run_daemon(args), Command::Mcp => commands::mcp_mgmt::mcp_from_args(args), Command::Update => commands::update::run_update(args), diff --git a/crates/uffs-client/src/connect_sync_diff.rs b/crates/uffs-client/src/connect_sync_diff.rs new file mode 100644 index 000000000..edb5d2763 --- /dev/null +++ b/crates/uffs-client/src/connect_sync_diff.rs @@ -0,0 +1,30 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Snapshot-diff RPC helper for [`crate::connect_sync::UffsClientSync`]. +//! +//! Paired with the daemon-side `handle_diff` in +//! `crates/uffs-daemon/src/handler.rs` and the wire types in +//! [`crate::protocol::diff_wire`]. Same typed-envelope dance as the tiering +//! cluster: serialise the params, fire the JSON-RPC, deserialise the result. + +use crate::connect_sync::UffsClientSync; +use crate::error::ClientError; +use crate::protocol::{DiffParams, DiffResultWire}; + +impl UffsClientSync { + /// Diff a baseline snapshot against the live index for a drive via the + /// daemon's `diff` RPC (delete-visible companion to `--newer`). + /// + /// # Errors + /// + /// Returns `ClientError` on I/O / protocol failure, or when the daemon + /// rejects the request (drive not loaded → `ERR_NOT_READY`; baseline + /// unreadable → `ERR_INTERNAL`), surfaced as [`ClientError::Protocol`]. + pub fn diff(&mut self, params: &DiffParams) -> Result { + let payload = + serde_json::to_value(params).map_err(|err| ClientError::Protocol(err.to_string()))?; + let result = self.send_request("diff", Some(payload))?; + serde_json::from_value(result).map_err(|err| ClientError::Protocol(err.to_string())) + } +} diff --git a/crates/uffs-client/src/lib.rs b/crates/uffs-client/src/lib.rs index 850708ff6..d72517008 100644 --- a/crates/uffs-client/src/lib.rs +++ b/crates/uffs-client/src/lib.rs @@ -127,6 +127,10 @@ pub mod connect_sync; /// `is_daemon_process`) — split off `connect_sync` to keep that file /// under the 800-LOC policy ceiling. pub(crate) mod connect_sync_autostart; +/// Snapshot-diff RPC helper (`diff`) — delete-visibility companion to +/// `--newer`. Split off `connect_sync` for module cohesion with the +/// [`protocol::diff_wire`] types it consumes. +pub(crate) mod connect_sync_diff; /// Platform-specific `platform_connect` impls and the `rpc_deadline` helper. /// /// Split `impl` blocks live on [`connect_sync::UffsClientSync`]; diff --git a/crates/uffs-client/src/protocol/diff_wire.rs b/crates/uffs-client/src/protocol/diff_wire.rs new file mode 100644 index 000000000..e2adba8f1 --- /dev/null +++ b/crates/uffs-client/src/protocol/diff_wire.rs @@ -0,0 +1,105 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Wire types for the `diff` method (snapshot delete-visibility diff). +//! +//! The CLI sends [`DiffParams`] naming a baseline snapshot + the drive it +//! covers; the daemon loads that baseline, diffs it against the live in-memory +//! index for the drive (`uffs_core::diff`), resolves every changed row to a +//! full path, and returns a [`DiffResultWire`]. Split into its own module (per +//! the 800-LOC policy and to keep `mod.rs` focused on the JSON-RPC envelope). + +use serde::{Deserialize, Serialize}; + +/// Parameters for the `diff` method. +/// +/// "What changed on `drive` between the `baseline` snapshot and now" — the +/// deletion-visible companion to `--newer`, which can only see +/// creates/modifies. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DiffParams { + /// Path to the baseline snapshot to diff against — a raw MFT capture + /// (`.bin`) the daemon can load into a compact index. The live index for + /// `drive` is the "current" side. + pub baseline: String, + /// Drive letter the baseline covers and whose live index is the current + /// side of the diff. + pub drive: uffs_mft::platform::DriveLetter, + /// Maximum entries returned **per class** (added / deleted / modified). + /// `0` = unlimited. When a class is capped, [`DiffResultWire::truncated`] + /// is set so the caller can tell a capped list from a complete one. + #[serde(default)] + pub limit: u32, +} + +/// One changed file in a [`DiffResultWire`]: a full path plus render metadata. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct DiffEntryWire { + /// Full path (`C:\Users\…\file.ext`). + pub path: String, + /// Logical size in bytes (current size for a modify; last-known baseline + /// size for a delete). + pub size: u64, + /// Last-write time in Unix microseconds, from the same snapshot as `path`. + pub modified: i64, +} + +/// Result of the `diff` method: the classified, path-resolved delta. +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct DiffResultWire { + /// Files present now but not in the baseline (created since it). + pub added: Vec, + /// Files present in the baseline but gone now (deleted since it). + pub deleted: Vec, + /// Files in both whose size or last-write time changed. + pub modified: Vec, + /// `true` when `limit` capped at least one class (more changes exist than + /// were returned). + pub truncated: bool, +} + +#[cfg(test)] +mod tests { + use super::{DiffEntryWire, DiffParams, DiffResultWire}; + + #[test] + fn diff_params_round_trip_through_json() { + let params = DiffParams { + baseline: r"D:\snapshots\c_2026-07-01.bin".to_owned(), + drive: uffs_mft::platform::DriveLetter::C, + limit: 500, + }; + let json = serde_json::to_value(¶ms).expect("serialize DiffParams"); + let back: DiffParams = serde_json::from_value(json).expect("deserialize DiffParams"); + assert_eq!(params, back); + } + + #[test] + fn diff_params_limit_defaults_to_zero_when_absent() { + // The CLI omits `limit` for an unlimited diff; it must default to 0. + let json = serde_json::json!({ "baseline": "x.bin", "drive": "C" }); + let params: DiffParams = serde_json::from_value(json).expect("deserialize"); + assert_eq!(params.limit, 0, "missing limit → unlimited (0)"); + } + + #[test] + fn diff_result_round_trips_through_json() { + let result = DiffResultWire { + added: vec![DiffEntryWire { + path: r"C:\new.txt".to_owned(), + size: 10, + modified: 9, + }], + deleted: vec![DiffEntryWire { + path: r"C:\gone.txt".to_owned(), + size: 200, + modified: 6, + }], + modified: vec![], + truncated: true, + }; + let json = serde_json::to_value(&result).expect("serialize DiffResultWire"); + let back: DiffResultWire = serde_json::from_value(json).expect("deserialize"); + assert_eq!(result, back); + } +} diff --git a/crates/uffs-client/src/protocol/mod.rs b/crates/uffs-client/src/protocol/mod.rs index e8a26cf54..cd024d601 100644 --- a/crates/uffs-client/src/protocol/mod.rs +++ b/crates/uffs-client/src/protocol/mod.rs @@ -13,6 +13,7 @@ pub mod aggregate_wire; pub mod cli_args; mod cli_args_helpers; +pub mod diff_wire; pub mod response; pub(crate) mod response_status; pub(crate) mod response_tiering; @@ -23,6 +24,7 @@ mod tests; pub use aggregate_wire::{ AggregateResultWire, AggregateSpecWire, BucketWire, DrilldownWire, SampleRowWire, StatsWire, }; +pub use diff_wire::{DiffEntryWire, DiffParams, DiffResultWire}; use serde::{Deserialize, Serialize}; // ──────────────────────────────────────────────────────────────────────────── diff --git a/crates/uffs-daemon/src/handler.rs b/crates/uffs-daemon/src/handler.rs index 9a8378dcf..8c490c514 100644 --- a/crates/uffs-daemon/src/handler.rs +++ b/crates/uffs-daemon/src/handler.rs @@ -39,6 +39,12 @@ mod blob; mod parse_search_params; use parse_search_params::ParseSearchParamsError; +// `handle_diff` lives in a sibling file for the same 800-LOC policy reason as +// `handler_blob.rs`; `#[path]` keeps it an `impl RequestHandler` method so the +// dispatcher above calls `self.handle_diff(...)` unchanged. +#[path = "handler_diff.rs"] +mod diff_handler; + /// Request handler holding shared daemon state. pub(crate) struct RequestHandler { /// Shared index manager. @@ -70,6 +76,7 @@ impl RequestHandler { "load_drive" => self.handle_load_drive(id, req).await, "refresh" => self.handle_refresh(id, req), "facet_values" => self.handle_facet_values(id, req).await, + "diff" => self.handle_diff(id, req).await, "keepalive" => self.handle_keepalive(id, req), "shutdown" => self.handle_shutdown(id, req), // Phase 8-B … 8-E — operator-driven memory tiering. diff --git a/crates/uffs-daemon/src/handler_diff.rs b/crates/uffs-daemon/src/handler_diff.rs new file mode 100644 index 000000000..5a9d03a3f --- /dev/null +++ b/crates/uffs-daemon/src/handler_diff.rs @@ -0,0 +1,77 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! The `diff` method handler for [`super::RequestHandler`]. +//! +//! Lifted out of `handler.rs` to keep that file under the 800-line policy +//! ceiling. Re-attached via `#[path = "handler_diff.rs"] mod diff_handler;` in +//! `handler.rs`, so `handle_diff` stays an `impl RequestHandler` method the +//! dispatcher calls as `self.handle_diff(...)`. +//! +//! Parsing + wire-mapping only: the classification, path resolution, baseline +//! load, and live-index snapshot all live in `IndexManager::diff_snapshot` +//! (`crate::index::diff`) and `uffs_core::diff`. + +use uffs_client::protocol::{ + DiffParams, ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_NOT_READY, RpcErrorResponse, RpcRequest, + RpcResponse, +}; + +use super::RequestHandler; +use crate::index::diff::DiffError; + +impl RequestHandler { + /// Handle the `diff` method — snapshot delete-visibility diff of a baseline + /// capture against the live index for a drive. + pub(super) async fn handle_diff(&self, id: u64, req: &RpcRequest) -> String { + let params: DiffParams = match req + .params + .as_ref() + .map(|val| serde_json::from_value(val.clone())) + { + Some(Ok(params)) => params, + Some(Err(err)) => { + return serde_json::to_string(&RpcErrorResponse::error( + Some(id), + ERR_INVALID_PARAMS, + &format!("diff: invalid params: {err}"), + )) + .unwrap_or_default(); + } + None => { + return serde_json::to_string(&RpcErrorResponse::error( + Some(id), + ERR_INVALID_PARAMS, + "diff: missing params (`baseline` + `drive` required)", + )) + .unwrap_or_default(); + } + }; + + match self.index.diff_snapshot(¶ms).await { + Ok(result) => { + let value = serde_json::to_value(&result).unwrap_or_default(); + serde_json::to_string(&RpcResponse::success(id, value)).unwrap_or_default() + } + Err(DiffError::DriveNotLoaded(letter)) => { + serde_json::to_string(&RpcErrorResponse::error( + Some(id), + ERR_NOT_READY, + &format!( + "diff: drive {letter} is not loaded; load it first \ + (`uffs --daemon load --drive {letter}`)" + ), + )) + .unwrap_or_default() + } + Err(DiffError::BaselineLoad { path, source }) => { + serde_json::to_string(&RpcErrorResponse::error( + Some(id), + ERR_INTERNAL, + &format!("diff: could not load baseline '{path}': {source}"), + )) + .unwrap_or_default() + } + } + } +} diff --git a/crates/uffs-daemon/src/index/diff.rs b/crates/uffs-daemon/src/index/diff.rs new file mode 100644 index 000000000..abbd05573 --- /dev/null +++ b/crates/uffs-daemon/src/index/diff.rs @@ -0,0 +1,159 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! Snapshot delete-visibility diff for [`IndexManager`] (RPC `diff`). +//! +//! The `diff` RPC answers "what was created, deleted, or modified on a drive +//! since a baseline snapshot" — the deletion-visible companion to `--newer`, +//! which is structurally blind to deletes. It loads the caller's baseline MFT +//! capture off-thread, diffs it against the **live in-memory index** for the +//! drive via [`uffs_core::diff::resolve_delta`] (so the "current" side is the +//! hot index the daemon already serves searches from), and returns the +//! classified, path-resolved delta. +//! +//! All the classification + path-resolution logic lives in `uffs_core::diff` +//! and is unit-tested there; this module is the daemon glue — snapshot the +//! registry, load the baseline, hand both to the engine, map the result onto +//! the wire type. + +use alloc::sync::Arc; +use std::path::PathBuf; + +use uffs_client::protocol::{DiffEntryWire, DiffParams, DiffResultWire}; +use uffs_core::compact::MftSource; +use uffs_core::diff::{DeltaEntry, ResolvedDelta, resolve_delta}; +use uffs_mft::platform::DriveLetter; + +use super::IndexManager; + +/// Why a `diff` request could not be served. Mapped to a JSON-RPC error by the +/// handler; kept data-only here so this module stays free of wire concerns. +pub(crate) enum DiffError { + /// The requested drive is not currently loaded in the live index, so there + /// is no "current" side to diff the baseline against. + DriveNotLoaded(DriveLetter), + /// The baseline snapshot at `path` could not be loaded into a compact + /// index (missing file, unreadable, not a valid MFT capture, …). + BaselineLoad { + /// The baseline path the caller supplied (echoed back in the message). + path: String, + /// The underlying load failure. + source: anyhow::Error, + }, +} + +impl IndexManager { + /// Diff a baseline snapshot against the live index for `params.drive`. + /// + /// # Errors + /// + /// Returns [`DiffError::DriveNotLoaded`] when the drive has no live index, + /// or [`DiffError::BaselineLoad`] when the baseline path cannot be loaded. + pub(crate) async fn diff_snapshot( + &self, + params: &DiffParams, + ) -> Result { + // Current side: the live, hot in-memory index for the drive. + let snap = self.snapshot().await; + let Some(current) = snap + .drives + .iter() + .find(|dr| dr.letter == params.drive) + .map(Arc::clone) + else { + return Err(DiffError::DriveNotLoaded(params.drive)); + }; + drop(snap); // We hold the one Arc we need; release the registry snapshot. + + // Baseline side: load the caller's capture and diff, both off the async + // runtime — the MFT parse is I/O + CPU heavy and the diff hashes over + // the whole record array. `no_cache = true` forces a fresh read of the + // baseline rather than reusing any persisted cache for that path. + let baseline_path = PathBuf::from(¶ms.baseline); + let drive = params.drive; + let limit = uffs_mft::u32_as_usize(params.limit); + let current_for_task = Arc::clone(¤t); + let outcome = tokio::task::spawn_blocking(move || { + let source = MftSource::File(baseline_path, Some(drive)); + let (baseline, _timing) = uffs_core::compact::load_drive(&source, true)?; + anyhow::Ok(resolve_delta(&baseline, ¤t_for_task, limit)) + }) + .await; + + match outcome { + Ok(Ok(resolved)) => Ok(to_wire(resolved)), + Ok(Err(source)) => Err(DiffError::BaselineLoad { + path: params.baseline.clone(), + source, + }), + Err(join_err) => Err(DiffError::BaselineLoad { + path: params.baseline.clone(), + source: join_err.into(), + }), + } + } +} + +/// Map the engine's [`ResolvedDelta`] onto the JSON-RPC wire result. +fn to_wire(resolved: ResolvedDelta) -> DiffResultWire { + DiffResultWire { + added: resolved.added.into_iter().map(entry_to_wire).collect(), + deleted: resolved.deleted.into_iter().map(entry_to_wire).collect(), + modified: resolved.modified.into_iter().map(entry_to_wire).collect(), + truncated: resolved.truncated, + } +} + +/// Map one resolved [`DeltaEntry`] onto its wire form. +fn entry_to_wire(entry: DeltaEntry) -> DiffEntryWire { + DiffEntryWire { + path: entry.path, + size: entry.size, + modified: entry.modified, + } +} + +#[cfg(test)] +mod tests { + use uffs_core::diff::{DeltaEntry, ResolvedDelta}; + + use super::{entry_to_wire, to_wire}; + + #[test] + fn to_wire_preserves_every_class_and_the_truncated_flag() { + let resolved = ResolvedDelta { + added: vec![DeltaEntry { + path: r"C:\new.txt".to_owned(), + size: 10, + modified: 9, + }], + deleted: vec![DeltaEntry { + path: r"C:\gone.txt".to_owned(), + size: 200, + modified: 6, + }], + modified: vec![], + truncated: true, + }; + let wire = to_wire(resolved); + assert_eq!(wire.added.len(), 1); + assert_eq!(wire.deleted.len(), 1); + assert!(wire.modified.is_empty()); + assert!(wire.truncated); + let added = wire.added.first().expect("one add"); + assert_eq!(added.path, r"C:\new.txt"); + assert_eq!(added.size, 10); + } + + #[test] + fn entry_to_wire_is_a_faithful_field_copy() { + let wire = entry_to_wire(DeltaEntry { + path: r"C:\a.txt".to_owned(), + size: 42, + modified: 7, + }); + assert_eq!(wire.path, r"C:\a.txt"); + assert_eq!(wire.size, 42); + assert_eq!(wire.modified, 7); + } +} diff --git a/crates/uffs-daemon/src/index/mod.rs b/crates/uffs-daemon/src/index/mod.rs index 7e13bc875..b813fbded 100644 --- a/crates/uffs-daemon/src/index/mod.rs +++ b/crates/uffs-daemon/src/index/mod.rs @@ -12,6 +12,7 @@ mod aggregation; mod constructors; +pub(crate) mod diff; mod dispatch; mod drives; pub(crate) mod forget_drive; From 724d0efa90505a4a1c0117e007f6c2621f49b357 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 14:34:59 -0700 Subject: [PATCH 05/11] docs(diff): fix intra-doc link to search::tree::resolve_path The DeltaReport doc linked `crate::tree::resolve_path`; the item lives at `crate::search::tree::resolve_path`. Trips the rustdoc broken-intra-doc-links gate (host + CI). No code change. --- crates/uffs-core/src/diff.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/crates/uffs-core/src/diff.rs b/crates/uffs-core/src/diff.rs index e3bba13c6..d9e6d72aa 100644 --- a/crates/uffs-core/src/diff.rs +++ b/crates/uffs-core/src/diff.rs @@ -31,8 +31,8 @@ use crate::search::tree::resolve_path; /// Every entry is a **row index** into the corresponding index's record array, /// not a resolved path: path reconstruction needs the whole index (the /// parent-chain walk), so the caller resolves each index via -/// [`crate::tree::resolve_path`] against the right side — `deleted` against the -/// baseline, `added` / `modified` against the current. +/// [`crate::search::tree::resolve_path`] against the right side — `deleted` +/// against the baseline, `added` / `modified` against the current. /// /// Rows are reported at *name* granularity: a file with N hard links (which /// share one File Reference) contributes N rows, so each affected path is From fe4839c5abef6950d270e16a11011ef931ca9a82 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 15:13:58 -0700 Subject: [PATCH 06/11] =?UTF-8?q?feat(deleted):=20uffs=20--deleted=20?= =?UTF-8?q?=E2=80=94=20forensic=20tombstone=20read=20(recently-deleted=20f?= =?UTF-8?q?iles)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Mechanism 2 of the delete-visibility slice: surface recently-deleted files with NO baseline required. When NTFS deletes a file it clears the record's in-use flag but leaves the bytes (name, parent, timestamps) until the MFT slot is reused; forensic parsing keeps those not-in-use records, and this command reconstructs each path by walking the still-present parent chain. `uffs --deleted --mft-file [--drive D] [--limit N] [--json]` — client-side, no daemon: reads the MFT capture (`load_raw_mft` forensic mode), forensic-parses every slot, and classifies the `is_deleted` records. Path reconstruction walks parent FRS through a whole-MFT map, so a delete still resolves through intermediate deleted directories; a parent whose slot was already reused yields a `…`-flagged partial path. Honest about its limits (in output + --help): best-effort (only deletes whose slot has not recycled are visible), the timestamp is the file's own last-write, not the deletion time, and reused-parent paths are unreliable. The classification + path resolution (`collect_tombstones` / `resolve_deleted_path`) are pure and unit-tested (deleted-only selection, resolution through a deleted parent, missing-parent → partial path, limit + truncation). Reuses `output::format_filetime_local` for wall-clock display. Live `--drive` (raw volume read via MftReader / broker) is a follow-up; the `--mft-file` path works on any offline capture today and is host-testable. --- crates/uffs-cli/src/args.rs | 35 ++ crates/uffs-cli/src/commands.rs | 3 + crates/uffs-cli/src/commands/deleted.rs | 446 +++++++++++++++++++++ crates/uffs-cli/src/commands/output/mod.rs | 5 +- crates/uffs-cli/src/dispatch.rs | 5 + 5 files changed, 493 insertions(+), 1 deletion(-) create mode 100644 crates/uffs-cli/src/commands/deleted.rs diff --git a/crates/uffs-cli/src/args.rs b/crates/uffs-cli/src/args.rs index 282a5fa0a..702a3969a 100644 --- a/crates/uffs-cli/src/args.rs +++ b/crates/uffs-cli/src/args.rs @@ -505,6 +505,7 @@ COMMANDS: --stats [PATH] Show filesystem statistics --agg Run aggregate analytics --diff Diff a baseline MFT snapshot vs the live index (deletes) + --deleted Forensic tombstone read: recently-deleted files from an MFT --daemon Manage the UFFS daemon (start/stop/load/status) --mcp Manage the UFFS MCP server --update [ACTION] Self-update (snapshot/acquire/apply/doctor/recover) @@ -652,6 +653,40 @@ pub(crate) fn print_diff_help() { print!("{DIFF_HELP}"); } +/// Help text for `uffs --deleted`. +const DELETED_HELP: &str = "\ +uffs --deleted — Forensic tombstone read (recently-deleted files) + +When NTFS deletes a file it clears the in-use flag but leaves the record (name, +parent, timestamps) intact until the MFT slot is reused. This surfaces those +not-in-use records as recently-deleted tombstones and reconstructs each path +from the surviving parent chain. No baseline needed. + +USAGE: uffs --deleted --mft-file [OPTIONS] + +OPTIONS: + --mft-file MFT capture to scan (required; a live --drive scan is + not wired yet). + -d, --drive Drive letter to label reconstructed paths with. + -n, --limit Max tombstones to print (0 = all). + --json Emit JSON instead of a table. + +LIMITS (best-effort by nature): + - Only deletes whose MFT slot has NOT been recycled are visible. + - The timestamp is the file's last-write time, NOT the deletion time. + - A path is unreliable if a parent directory's slot was itself reused + (such paths are prefixed with `…`). + +EXAMPLE: + uffs --deleted --mft-file C_mft.bin --drive C --limit 50 +"; + +/// Print deleted help. +#[expect(clippy::print_stdout, reason = "intentional help output")] +pub(crate) fn print_deleted_help() { + print!("{DELETED_HELP}"); +} + /// Help text for `uffs --agg`. const AGGREGATE_HELP: &str = "\ uffs --agg — Run aggregate analytics on the filesystem index diff --git a/crates/uffs-cli/src/commands.rs b/crates/uffs-cli/src/commands.rs index 034360ba5..aedec66ea 100644 --- a/crates/uffs-cli/src/commands.rs +++ b/crates/uffs-cli/src/commands.rs @@ -23,6 +23,9 @@ pub(crate) mod daemon_status; /// under the 800-LOC policy ceiling. Forward-looking: 8-D `forget` /// and 8-E `status_drives` will land their shims here as well. pub(crate) mod daemon_tiering; +/// `uffs --deleted --mft-file ` — forensic tombstone read (deleted +/// files). +pub(crate) mod deleted; /// `uffs --diff --drive ` — snapshot delete-visibility diff. pub(crate) mod diff; /// Shared elevation gate for the mutating flows (uninstall / update): surface diff --git a/crates/uffs-cli/src/commands/deleted.rs b/crates/uffs-cli/src/commands/deleted.rs new file mode 100644 index 000000000..6cef006c8 --- /dev/null +++ b/crates/uffs-cli/src/commands/deleted.rs @@ -0,0 +1,446 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! `uffs --deleted --mft-file ` — forensic tombstone read. +//! +//! When NTFS deletes a file it clears the record's in-use flag but leaves the +//! record bytes (name, parent, timestamps) intact until the MFT slot is +//! reallocated. This command reads an MFT capture with forensic parsing, +//! surfaces the not-in-use records as **recently-deleted tombstones**, and +//! reconstructs each path by walking the (still-present) parent chain. +//! +//! No baseline needed — this is the "what did I just delete, maybe still +//! recoverable" path (Mechanism 2 in +//! `docs/architecture/delete-visibility-snapshot-diff.md`). Honest limits: +//! best-effort (you only see deletes whose slot has not been recycled), no +//! true *deletion* time (the timestamp is the file's own last-write), and a +//! path is unreliable if a parent directory's slot was itself reused. + +use std::collections::HashMap; +use std::path::PathBuf; + +use anyhow::{Context as _, Result}; +use uffs_mft::parse::{ + ParseOptions, ParseResult, ParsedRecord, apply_fixup, parse_record_forensic, +}; +use uffs_mft::platform::DriveLetter; +use uffs_mft::raw::{LoadRawOptions, load_raw_mft}; + +use crate::args::parse_drive_letter; +use crate::commands::output::format_filetime_local; + +/// NTFS reserves File Record Segment 5 for the volume root directory; every +/// path walk terminates here. +const ROOT_FRS: u64 = 5; + +/// Parsed `uffs --deleted` invocation. +#[derive(Debug)] +struct DeletedArgs { + /// MFT capture to scan (raw `$MFT` dump). + mft_file: PathBuf, + /// Drive letter to label reconstructed paths with (default `X`). + drive: Option, + /// Max tombstones to print (0 = all). + limit: u32, + /// Emit JSON instead of the human table. + json: bool, +} + +/// One reconstructed deleted-file tombstone. +#[derive(Debug, Clone, PartialEq, Eq)] +struct Tombstone { + /// Reconstructed full path (best-effort — see module docs). + path: String, + /// Logical file size in bytes (from the surviving record). + size: u64, + /// The file's own last-write time (raw FILETIME) — NOT the deletion time. + modified: i64, + /// Whether the record is a directory. + is_dir: bool, + /// `true` when the parent chain resolved all the way to the volume root; + /// `false` when a parent FRS was missing (path is partial / prefixed `…`). + path_complete: bool, +} + +/// Run `uffs --deleted --mft-file [--drive D] [--limit N] [--json]`. +/// +/// # Errors +/// +/// Returns an error on bad arguments or when the MFT capture cannot be read. +pub(crate) fn run_deleted(args: &[String]) -> Result<()> { + if args.iter().any(|arg| arg == "--help" || arg == "-h") { + crate::args::print_deleted_help(); + return Ok(()); + } + + let parsed = parse_deleted_args(args)?; + let drive = parsed.drive.unwrap_or(DriveLetter::X); + + let options = LoadRawOptions { + header_only: false, + volume_letter: Some(drive), + forensic: true, + }; + let raw = load_raw_mft(&parsed.mft_file, &options) + .with_context(|| format!("failed to read MFT capture '{}'", parsed.mft_file.display()))?; + + // Forensic-parse every slot: this keeps the not-in-use (deleted) records + // that the default parser drops, and the live records we need to resolve + // deleted files' parent chains. + let capacity = usize::try_from(raw.record_count()).unwrap_or(0); + let mut records = Vec::with_capacity(capacity); + for (frs, data) in raw.iter_records() { + let mut record_buf = data.to_vec(); + let fixup_ok = apply_fixup(&mut record_buf); + if let ParseResult::Base(parsed_record) = + parse_record_forensic(&record_buf, frs, ParseOptions::FORENSIC, !fixup_ok) + { + records.push(parsed_record); + } + } + + let (tombstones, total, truncated) = + collect_tombstones(&records, drive, uffs_mft::u32_as_usize(parsed.limit)); + + if parsed.json { + print_json(&tombstones, total, truncated); + } else { + print_human(&tombstones, total, truncated, drive); + } + Ok(()) +} + +/// Parse the `--deleted` argument vector. +/// +/// `--mft-file ` is required; `--drive`, `--limit`, `--json` optional. +fn parse_deleted_args(args: &[String]) -> Result { + let mut mft_file: Option = None; + let mut drive: Option = None; + let mut limit: u32 = 0; + let mut json = false; + + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--mft-file" => { + let val = iter + .next() + .with_context(|| "`--mft-file` requires a path")?; + mft_file = Some(PathBuf::from(val)); + } + "--drive" | "-d" => { + let val = iter + .next() + .with_context(|| "`--drive` requires a drive letter (e.g. C)")?; + drive = Some(parse_drive_letter(val)?); + } + "--limit" | "-n" => { + let val = iter.next().with_context(|| "`--limit` requires a number")?; + limit = val + .parse::() + .with_context(|| format!("invalid --limit value '{val}'"))?; + } + "--json" => json = true, + other => anyhow::bail!("unknown argument '{other}'; see `uffs --deleted --help`"), + } + } + + let mft_path = mft_file.with_context(|| { + "missing `--mft-file `; a live `--drive` scan is not wired yet — \ + point at an MFT capture" + })?; + Ok(DeletedArgs { + mft_file: mft_path, + drive, + limit, + json, + }) +} + +/// Collect and path-resolve every deleted record. Returns the (capped) +/// tombstones, the total deleted count (before the cap), and whether the cap +/// dropped any. +fn collect_tombstones( + records: &[ParsedRecord], + drive: DriveLetter, + limit: usize, +) -> (Vec, usize, bool) { + // FRS -> (name, parent FRS) for *every* record, so a deleted file's parent + // chain resolves even through intermediate deleted directories. + let mut by_frs: HashMap = HashMap::with_capacity(records.len()); + for record in records { + by_frs.insert( + record.frs.raw(), + (record.name.as_str(), record.parent_frs.raw()), + ); + } + + let deleted: Vec<&ParsedRecord> = records.iter().filter(|rec| rec.is_deleted).collect(); + let total = deleted.len(); + let truncated = limit > 0 && total > limit; + let take = if truncated { limit } else { total }; + + let tombstones = deleted + .iter() + .take(take) + .map(|rec| { + let (path, complete) = + resolve_deleted_path(&rec.name, rec.parent_frs.raw(), &by_frs, drive); + Tombstone { + path, + size: rec.size, + modified: rec.std_info.modified, + is_dir: rec.is_directory, + path_complete: complete, + } + }) + .collect(); + + (tombstones, total, truncated) +} + +/// Reconstruct a deleted record's full path by walking `parent` up `by_frs` +/// until the volume root. Returns `(path, complete)`; `complete` is `false` +/// when a parent FRS is absent (the path is prefixed with `…` to flag it). +fn resolve_deleted_path( + name: &str, + parent: u64, + by_frs: &HashMap, + drive: DriveLetter, +) -> (String, bool) { + let mut parts: Vec<&str> = vec![name]; + let mut current = parent; + let mut complete = true; + + // Bounded walk: NTFS paths are far shallower than this, and the guard stops + // a cycle from a reused/self-referential parent slot. + for _ in 0_u32..256 { + if current == ROOT_FRS { + break; + } + let Some(&(parent_name, grandparent)) = by_frs.get(¤t) else { + complete = false; + break; + }; + parts.push(parent_name); + current = grandparent; + } + if current != ROOT_FRS { + complete = false; + } + + parts.reverse(); + let joined = parts.join("\\"); + let path = if complete { + format!("{drive}:\\{joined}") + } else { + format!("{drive}:\\…\\{joined}") + }; + (path, complete) +} + +/// Render the tombstones as a human-readable table. +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn print_human(tombstones: &[Tombstone], total: usize, truncated: bool, drive: DriveLetter) { + let complete = tombstones.iter().filter(|tomb| tomb.path_complete).count(); + println!( + "Deleted (tombstone) records on {drive} — best-effort; recoverable until the MFT slot \ + is reused:" + ); + println!( + " {total} tombstone(s){}; {complete} of the shown {} have a fully-resolved path", + if truncated { + " (showing the first --limit)" + } else { + "" + }, + tombstones.len(), + ); + if tombstones.is_empty() { + return; + } + println!(); + for tomb in tombstones { + let kind = if tomb.is_dir { " [dir]" } else { "" }; + println!( + " {} ({}, modified {}){kind}", + tomb.path, + human_bytes(tomb.size), + format_filetime_local(tomb.modified), + ); + } + println!( + "\nNote: the timestamp is the file's last-write time, not when it was deleted; \ + a `…`-prefixed path had a parent whose MFT slot was already reused." + ); +} + +/// Emit the tombstones as JSON for scripting. +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn print_json(tombstones: &[Tombstone], total: usize, truncated: bool) { + let rows: Vec = tombstones + .iter() + .map(|tomb| { + serde_json::json!({ + "path": tomb.path, + "size": tomb.size, + "modified": tomb.modified, + "is_dir": tomb.is_dir, + "path_complete": tomb.path_complete, + }) + }) + .collect(); + let doc = serde_json::json!({ + "total_deleted": total, + "truncated": truncated, + "tombstones": rows, + }); + match serde_json::to_string_pretty(&doc) { + Ok(json) => println!("{json}"), + Err(err) => println!("{{\"error\":\"failed to serialize tombstones: {err}\"}}"), + } +} + +/// Humanise a byte count with binary units (integer arithmetic — no floats). +fn human_bytes(bytes: u64) -> String { + const KIB: u64 = 1024; + const MIB: u64 = 1024 * KIB; + const GIB: u64 = 1024 * MIB; + if bytes >= GIB { + let whole = bytes / GIB; + let hundredths = (bytes % GIB).saturating_mul(100) / GIB; + format!("{whole}.{hundredths:02} GiB") + } else if bytes >= MIB { + format!("{} MiB", bytes / MIB) + } else if bytes >= KIB { + format!("{} KiB", bytes / KIB) + } else { + format!("{bytes} B") + } +} + +#[cfg(test)] +mod tests { + use uffs_mft::parse::ParsedRecord; + use uffs_mft::platform::DriveLetter; + + use super::{collect_tombstones, parse_deleted_args}; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|item| (*item).to_owned()).collect() + } + + /// Build a record: `frs`, `parent` FRS, name, size, deleted?, dir?. + fn record( + frs: u64, + parent: u64, + name: &str, + size: u64, + deleted: bool, + dir: bool, + ) -> ParsedRecord { + ParsedRecord { + frs: uffs_mft::frs::Frs::new(frs), + parent_frs: uffs_mft::frs::ParentFrs::new(parent), + name: name.to_owned(), + size, + is_deleted: deleted, + is_directory: dir, + ..ParsedRecord::default() + } + } + + #[test] + fn parses_required_mft_file_and_options() { + let parsed = parse_deleted_args(&args(&[ + "--mft-file", + "C.bin", + "-d", + "C", + "--limit", + "5", + "--json", + ])) + .expect("parse"); + assert_eq!(parsed.mft_file.to_str(), Some("C.bin")); + assert_eq!(parsed.drive, Some(DriveLetter::C)); + assert_eq!(parsed.limit, 5); + assert!(parsed.json); + } + + #[test] + fn missing_mft_file_is_an_error() { + let err = parse_deleted_args(&args(&["-d", "C"])).expect_err("must require --mft-file"); + assert!(err.to_string().contains("--mft-file"), "{err}"); + } + + #[test] + fn only_deleted_records_become_tombstones_with_resolved_paths() { + // Root(5) → docs(100) → [a.txt(200) deleted, live.txt(201) alive]. + let records = vec![ + record(100, ROOT_FRS_T, "docs", 0, false, true), + record(200, 100, "a.txt", 100, true, false), + record(201, 100, "live.txt", 50, false, false), + ]; + let (tombs, total, truncated) = collect_tombstones(&records, DriveLetter::C, 0); + assert_eq!(total, 1, "only a.txt is deleted"); + assert!(!truncated); + let tomb = tombs.first().expect("one tombstone"); + assert_eq!( + tomb.path, r"C:\docs\a.txt", + "resolved through the live parent" + ); + assert_eq!(tomb.size, 100); + assert!(tomb.path_complete); + } + + #[test] + fn a_deleted_dir_on_the_chain_still_resolves() { + // Root(5) → gone(100, deleted dir) → file(200, deleted). The deleted + // parent is still in the MFT, so the path reconstructs completely. + let records = vec![ + record(100, ROOT_FRS_T, "gone", 0, true, true), + record(200, 100, "file.txt", 10, true, false), + ]; + let (tombs, total, _) = collect_tombstones(&records, DriveLetter::C, 0); + assert_eq!(total, 2); + let file = tombs + .iter() + .find(|tomb| tomb.path.ends_with("file.txt")) + .expect("file tombstone"); + assert_eq!(file.path, r"C:\gone\file.txt"); + assert!(file.path_complete); + } + + #[test] + fn a_missing_parent_marks_the_path_incomplete() { + // The parent FRS (999) is not in the capture (slot reused / evicted). + let records = vec![record(200, 999, "orphan.log", 44, true, false)]; + let (tombs, _, _) = collect_tombstones(&records, DriveLetter::C, 0); + let tomb = tombs.first().expect("one tombstone"); + assert!(!tomb.path_complete, "missing parent → incomplete"); + assert!( + tomb.path.contains('…'), + "incomplete path is flagged: {}", + tomb.path + ); + assert!(tomb.path.ends_with("orphan.log")); + } + + #[test] + fn limit_caps_and_flags_truncation() { + let records = vec![ + record(200, ROOT_FRS_T, "a", 1, true, false), + record(201, ROOT_FRS_T, "b", 2, true, false), + record(202, ROOT_FRS_T, "c", 3, true, false), + ]; + let (tombs, total, truncated) = collect_tombstones(&records, DriveLetter::C, 2); + assert_eq!(total, 3, "total counts all deleted"); + assert_eq!(tombs.len(), 2, "cap keeps 2"); + assert!(truncated); + } + + /// Root FRS mirrored into the test module (the production const is private + /// to the parent module's non-test scope). + const ROOT_FRS_T: u64 = 5; +} diff --git a/crates/uffs-cli/src/commands/output/mod.rs b/crates/uffs-cli/src/commands/output/mod.rs index 9dc07915e..ecaf06192 100644 --- a/crates/uffs-cli/src/commands/output/mod.rs +++ b/crates/uffs-cli/src/commands/output/mod.rs @@ -753,7 +753,10 @@ fn format_filetime_with_tz(filetime: i64, tz_offset_secs: i32) -> String { /// fallback path was configured. CSV / parity / custom formatters /// take their offset from the config (`--tz-offset`) via /// [`format_filetime_with_tz`] instead. -fn format_filetime_local(filetime: i64) -> String { +/// +/// `pub(crate)` so the forensic `--deleted` command reuses the exact same +/// wall-clock rendering as the search table. +pub(crate) fn format_filetime_local(filetime: i64) -> String { format_filetime_with_tz(filetime, *LOCAL_TZ_OFFSET_SECS) } diff --git a/crates/uffs-cli/src/dispatch.rs b/crates/uffs-cli/src/dispatch.rs index 9e02097b1..0d99eca60 100644 --- a/crates/uffs-cli/src/dispatch.rs +++ b/crates/uffs-cli/src/dispatch.rs @@ -28,6 +28,8 @@ pub(crate) enum Command { Agg, /// `--diff --drive `. Diff, + /// `--deleted --mft-file `. + Deleted, /// `--daemon `. Daemon, /// `--mcp `. @@ -49,6 +51,7 @@ impl Command { "--stats" => Self::Stats, "--agg" | "--aggregate" => Self::Agg, "--diff" => Self::Diff, + "--deleted" => Self::Deleted, "--daemon" => Self::Daemon, "--mcp" => Self::Mcp, // `--upgrade` is a HIDDEN alias for `--update` (winget/apt muscle @@ -71,6 +74,7 @@ const COMMAND_TOKENS: &[&str] = &[ "--agg", "--aggregate", "--diff", + "--deleted", "--daemon", "--mcp", "--update", @@ -113,6 +117,7 @@ pub(crate) fn dispatch_command(command: Command, args: &[String]) -> Result<()> Command::Stats => crate::run_stats(args), Command::Agg => crate::run_aggregate(args), Command::Diff => commands::diff::run_diff(args), + Command::Deleted => commands::deleted::run_deleted(args), Command::Daemon => crate::run_daemon(args), Command::Mcp => commands::mcp_mgmt::mcp_from_args(args), Command::Update => commands::update::run_update(args), From 7ff393dd3354ac593b90f462d52cd7b4789983e8 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:55:05 -0700 Subject: [PATCH 07/11] fix(deleted): stream the tombstone scan so a large MFT doesn't OOM MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first cut materialized every parsed record (4.7M rich `ParsedRecord`s) plus a whole-MFT frs->name map on top of the 4.8 GB raw buffer — many GB of RAM, enough to get OOM-killed on a real volume. Rewrite to keep only the *deleted* records (a small fraction) in one pass, then resolve each parent on demand from the raw buffer with a small memoized cache. Peak memory is now ~the raw MFT plus the deleted subset, not a multiple of it. Path resolution is factored behind a `lookup` closure so it stays unit-tested with a plain map (no file I/O), and one reused fixup buffer replaces the per-record allocation. --- crates/uffs-cli/src/commands/deleted.rs | 275 ++++++++++++------------ 1 file changed, 134 insertions(+), 141 deletions(-) diff --git a/crates/uffs-cli/src/commands/deleted.rs b/crates/uffs-cli/src/commands/deleted.rs index 6cef006c8..c5363c942 100644 --- a/crates/uffs-cli/src/commands/deleted.rs +++ b/crates/uffs-cli/src/commands/deleted.rs @@ -15,16 +15,19 @@ //! best-effort (you only see deletes whose slot has not been recycled), no //! true *deletion* time (the timestamp is the file's own last-write), and a //! path is unreliable if a parent directory's slot was itself reused. +//! +//! Memory: the scan collects only the *deleted* records (a small fraction of +//! the MFT) and resolves each parent **on demand** from the raw buffer with a +//! small cache — it never materializes all N records, so peak memory is ~the +//! raw MFT plus the deleted subset, not a multiple of it. use std::collections::HashMap; use std::path::PathBuf; use anyhow::{Context as _, Result}; -use uffs_mft::parse::{ - ParseOptions, ParseResult, ParsedRecord, apply_fixup, parse_record_forensic, -}; +use uffs_mft::parse::{ParseOptions, ParseResult, apply_fixup, parse_record_forensic}; use uffs_mft::platform::DriveLetter; -use uffs_mft::raw::{LoadRawOptions, load_raw_mft}; +use uffs_mft::raw::{LoadRawOptions, RawMftData, load_raw_mft}; use crate::args::parse_drive_letter; use crate::commands::output::format_filetime_local; @@ -46,14 +49,28 @@ struct DeletedArgs { json: bool, } -/// One reconstructed deleted-file tombstone. +/// A deleted record captured during the scan, before path resolution. +struct DeletedEntry { + /// Parent directory FRS (start of the path walk). + parent: u64, + /// The deleted file's own name (leaf). + name: String, + /// Logical file size in bytes. + size: u64, + /// The file's own last-write time (raw FILETIME) — NOT the deletion time. + modified: i64, + /// Whether the record is a directory. + is_dir: bool, +} + +/// One reconstructed deleted-file tombstone (path-resolved, ready to render). #[derive(Debug, Clone, PartialEq, Eq)] struct Tombstone { /// Reconstructed full path (best-effort — see module docs). path: String, - /// Logical file size in bytes (from the surviving record). + /// Logical file size in bytes. size: u64, - /// The file's own last-write time (raw FILETIME) — NOT the deletion time. + /// The file's own last-write time (raw FILETIME). modified: i64, /// Whether the record is a directory. is_dir: bool, @@ -84,23 +101,51 @@ pub(crate) fn run_deleted(args: &[String]) -> Result<()> { let raw = load_raw_mft(&parsed.mft_file, &options) .with_context(|| format!("failed to read MFT capture '{}'", parsed.mft_file.display()))?; - // Forensic-parse every slot: this keeps the not-in-use (deleted) records - // that the default parser drops, and the live records we need to resolve - // deleted files' parent chains. - let capacity = usize::try_from(raw.record_count()).unwrap_or(0); - let mut records = Vec::with_capacity(capacity); + // Pass 1: forensic-parse every slot but KEEP only the deleted records. + // The default parser drops not-in-use records; forensic mode retains them. + // One reusable fixup buffer avoids a per-record allocation. + let mut deleted: Vec = Vec::new(); + let mut fixup_buf: Vec = Vec::new(); for (frs, data) in raw.iter_records() { - let mut record_buf = data.to_vec(); - let fixup_ok = apply_fixup(&mut record_buf); - if let ParseResult::Base(parsed_record) = - parse_record_forensic(&record_buf, frs, ParseOptions::FORENSIC, !fixup_ok) + fixup_buf.clear(); + fixup_buf.extend_from_slice(data); + let fixup_ok = apply_fixup(&mut fixup_buf); + if let ParseResult::Base(record) = + parse_record_forensic(&fixup_buf, frs, ParseOptions::FORENSIC, !fixup_ok) + && record.is_deleted { - records.push(parsed_record); + deleted.push(DeletedEntry { + parent: record.parent_frs.raw(), + name: record.name, + size: record.size, + modified: record.std_info.modified, + is_dir: record.is_directory, + }); } } - let (tombstones, total, truncated) = - collect_tombstones(&records, drive, uffs_mft::u32_as_usize(parsed.limit)); + let total = deleted.len(); + let limit = uffs_mft::u32_as_usize(parsed.limit); + let truncated = limit > 0 && total > limit; + let take = if truncated { limit } else { total }; + + // Pass 2: resolve each kept tombstone's path by walking parents on demand + // from the raw buffer, memoizing shared ancestors. + let mut parent_cache: HashMap> = HashMap::new(); + let mut lookup_buf: Vec = Vec::new(); + let mut tombstones: Vec = Vec::with_capacity(take); + for entry in deleted.iter().take(take) { + let (path, complete) = resolve_path(&entry.name, entry.parent, drive, |frs| { + lookup_parent(&raw, frs, &mut parent_cache, &mut lookup_buf) + }); + tombstones.push(Tombstone { + path, + size: entry.size, + modified: entry.modified, + is_dir: entry.is_dir, + path_complete: complete, + }); + } if parsed.json { print_json(&tombstones, total, truncated); @@ -110,6 +155,34 @@ pub(crate) fn run_deleted(args: &[String]) -> Result<()> { Ok(()) } +/// Resolve a parent record's `(name, its-parent FRS)` from the raw MFT, +/// memoizing the result (including a `None` miss) so shared ancestors are +/// parsed once. +fn lookup_parent( + raw: &RawMftData, + frs: u64, + cache: &mut HashMap>, + buf: &mut Vec, +) -> Option<(String, u64)> { + if let Some(cached) = cache.get(&frs) { + return cached.clone(); + } + let resolved = raw.get_record(frs).and_then(|data| { + buf.clear(); + buf.extend_from_slice(data); + let fixup_ok = apply_fixup(buf); + if let ParseResult::Base(record) = + parse_record_forensic(buf, frs, ParseOptions::FORENSIC, !fixup_ok) + { + Some((record.name, record.parent_frs.raw())) + } else { + None + } + }); + cache.insert(frs, resolved.clone()); + resolved +} + /// Parse the `--deleted` argument vector. /// /// `--mft-file ` is required; `--drive`, `--limit`, `--json` optional. @@ -157,58 +230,16 @@ fn parse_deleted_args(args: &[String]) -> Result { }) } -/// Collect and path-resolve every deleted record. Returns the (capped) -/// tombstones, the total deleted count (before the cap), and whether the cap -/// dropped any. -fn collect_tombstones( - records: &[ParsedRecord], - drive: DriveLetter, - limit: usize, -) -> (Vec, usize, bool) { - // FRS -> (name, parent FRS) for *every* record, so a deleted file's parent - // chain resolves even through intermediate deleted directories. - let mut by_frs: HashMap = HashMap::with_capacity(records.len()); - for record in records { - by_frs.insert( - record.frs.raw(), - (record.name.as_str(), record.parent_frs.raw()), - ); - } - - let deleted: Vec<&ParsedRecord> = records.iter().filter(|rec| rec.is_deleted).collect(); - let total = deleted.len(); - let truncated = limit > 0 && total > limit; - let take = if truncated { limit } else { total }; - - let tombstones = deleted - .iter() - .take(take) - .map(|rec| { - let (path, complete) = - resolve_deleted_path(&rec.name, rec.parent_frs.raw(), &by_frs, drive); - Tombstone { - path, - size: rec.size, - modified: rec.std_info.modified, - is_dir: rec.is_directory, - path_complete: complete, - } - }) - .collect(); - - (tombstones, total, truncated) -} - -/// Reconstruct a deleted record's full path by walking `parent` up `by_frs` -/// until the volume root. Returns `(path, complete)`; `complete` is `false` -/// when a parent FRS is absent (the path is prefixed with `…` to flag it). -fn resolve_deleted_path( +/// Reconstruct a deleted record's full path by walking `parent` up via +/// `lookup` until the volume root. Returns `(path, complete)`; `complete` is +/// `false` when a parent FRS is absent (the path is prefixed with `…`). +fn resolve_path( name: &str, parent: u64, - by_frs: &HashMap, drive: DriveLetter, + mut lookup: impl FnMut(u64) -> Option<(String, u64)>, ) -> (String, bool) { - let mut parts: Vec<&str> = vec![name]; + let mut parts: Vec = vec![name.to_owned()]; let mut current = parent; let mut complete = true; @@ -218,7 +249,7 @@ fn resolve_deleted_path( if current == ROOT_FRS { break; } - let Some(&(parent_name, grandparent)) = by_frs.get(¤t) else { + let Some((parent_name, grandparent)) = lookup(current) else { complete = false; break; }; @@ -321,33 +352,21 @@ fn human_bytes(bytes: u64) -> String { #[cfg(test)] mod tests { - use uffs_mft::parse::ParsedRecord; + use std::collections::HashMap; + use uffs_mft::platform::DriveLetter; - use super::{collect_tombstones, parse_deleted_args}; + use super::{parse_deleted_args, resolve_path}; fn args(list: &[&str]) -> Vec { list.iter().map(|item| (*item).to_owned()).collect() } - /// Build a record: `frs`, `parent` FRS, name, size, deleted?, dir?. - fn record( - frs: u64, - parent: u64, - name: &str, - size: u64, - deleted: bool, - dir: bool, - ) -> ParsedRecord { - ParsedRecord { - frs: uffs_mft::frs::Frs::new(frs), - parent_frs: uffs_mft::frs::ParentFrs::new(parent), - name: name.to_owned(), - size, - is_deleted: deleted, - is_directory: dir, - ..ParsedRecord::default() - } + /// A `by_frs` map → the `lookup` closure `resolve_path` expects. + fn lookup_from( + map: &HashMap, + ) -> impl FnMut(u64) -> Option<(String, u64)> + '_ { + move |frs| map.get(&frs).cloned() } #[test] @@ -375,69 +394,43 @@ mod tests { } #[test] - fn only_deleted_records_become_tombstones_with_resolved_paths() { - // Root(5) → docs(100) → [a.txt(200) deleted, live.txt(201) alive]. - let records = vec![ - record(100, ROOT_FRS_T, "docs", 0, false, true), - record(200, 100, "a.txt", 100, true, false), - record(201, 100, "live.txt", 50, false, false), - ]; - let (tombs, total, truncated) = collect_tombstones(&records, DriveLetter::C, 0); - assert_eq!(total, 1, "only a.txt is deleted"); - assert!(!truncated); - let tomb = tombs.first().expect("one tombstone"); - assert_eq!( - tomb.path, r"C:\docs\a.txt", - "resolved through the live parent" - ); - assert_eq!(tomb.size, 100); - assert!(tomb.path_complete); + fn resolves_through_a_live_parent_to_the_root() { + // Root(5) → docs(100). A deleted a.txt(parent 100) resolves fully. + let mut map = HashMap::new(); + map.insert(100_u64, ("docs".to_owned(), ROOT_FRS_T)); + let (path, complete) = resolve_path("a.txt", 100, DriveLetter::C, lookup_from(&map)); + assert_eq!(path, r"C:\docs\a.txt"); + assert!(complete); } #[test] - fn a_deleted_dir_on_the_chain_still_resolves() { - // Root(5) → gone(100, deleted dir) → file(200, deleted). The deleted - // parent is still in the MFT, so the path reconstructs completely. - let records = vec![ - record(100, ROOT_FRS_T, "gone", 0, true, true), - record(200, 100, "file.txt", 10, true, false), - ]; - let (tombs, total, _) = collect_tombstones(&records, DriveLetter::C, 0); - assert_eq!(total, 2); - let file = tombs - .iter() - .find(|tomb| tomb.path.ends_with("file.txt")) - .expect("file tombstone"); - assert_eq!(file.path, r"C:\gone\file.txt"); - assert!(file.path_complete); + fn resolves_through_a_deleted_parent_still_in_the_mft() { + // The parent dir `gone`(100) is itself deleted but its record survives, + // so lookup still returns it and the path reconstructs completely. + let mut map = HashMap::new(); + map.insert(100_u64, ("gone".to_owned(), ROOT_FRS_T)); + let (path, complete) = resolve_path("file.txt", 100, DriveLetter::C, lookup_from(&map)); + assert_eq!(path, r"C:\gone\file.txt"); + assert!(complete); } #[test] fn a_missing_parent_marks_the_path_incomplete() { - // The parent FRS (999) is not in the capture (slot reused / evicted). - let records = vec![record(200, 999, "orphan.log", 44, true, false)]; - let (tombs, _, _) = collect_tombstones(&records, DriveLetter::C, 0); - let tomb = tombs.first().expect("one tombstone"); - assert!(!tomb.path_complete, "missing parent → incomplete"); - assert!( - tomb.path.contains('…'), - "incomplete path is flagged: {}", - tomb.path - ); - assert!(tomb.path.ends_with("orphan.log")); + // Parent 999 is not in the capture (slot reused / evicted). + let map: HashMap = HashMap::new(); + let (path, complete) = resolve_path("orphan.log", 999, DriveLetter::C, lookup_from(&map)); + assert!(!complete, "missing parent → incomplete"); + assert!(path.contains('…'), "incomplete path is flagged: {path}"); + assert!(path.ends_with("orphan.log")); } #[test] - fn limit_caps_and_flags_truncation() { - let records = vec![ - record(200, ROOT_FRS_T, "a", 1, true, false), - record(201, ROOT_FRS_T, "b", 2, true, false), - record(202, ROOT_FRS_T, "c", 3, true, false), - ]; - let (tombs, total, truncated) = collect_tombstones(&records, DriveLetter::C, 2); - assert_eq!(total, 3, "total counts all deleted"); - assert_eq!(tombs.len(), 2, "cap keeps 2"); - assert!(truncated); + fn a_file_directly_under_root_needs_no_lookup() { + let map: HashMap = HashMap::new(); + let (path, complete) = + resolve_path("boot.ini", ROOT_FRS_T, DriveLetter::C, lookup_from(&map)); + assert_eq!(path, r"C:\boot.ini"); + assert!(complete); } /// Root FRS mirrored into the test module (the production const is private From 62819e3b355d481a9abbea4cae1e2163d9b090ba Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 17:55:48 -0700 Subject: [PATCH 08/11] feat(diff): route --diff through the full search pipeline (filter deleted files) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replaces the bespoke `--diff` output with a search over the baseline's DELETED set, so deleted files are filterable/sortable by every criterion a normal search supports — pattern, `--ext`, `--newer`/`--older`, `--min-size`, `--type`, sort, projection, and every output format (table/csv/json). The key insight: the old MFT snapshot is a full capture, so each deleted file is a complete record. The mechanism, high-reuse: - uffs-core: a `SearchFilters.deleted` filter (mirrors `malformed`) that matches the `DELETED` tombstone bit (0x8000) on `CompactRecord.flags`. - uffs-client: `SearchParams.diff_baseline` + a `--diff ` search flag (so `uffs --diff C_old.bin --drive C '*.txt' --newer 30d` just works); a bare diff defaults the pattern to `*` (list all deleted). - uffs-daemon: `search()` splits into a thin entry + `run_search_over(params, override_index)`. `diff_search` loads the baseline off-thread, diffs it against the live index by File Reference (`diff_indexes`), marks the vanished rows `DELETED`, wraps the baseline as a `DriveIndex`, and runs the normal pipeline over it with the deleted-only filter forced on. The registry warm-up / dispatch-accounting is skipped for the (non-shard) baseline. Setup errors (no drive / unloaded / bad baseline) map to JSON-RPC errors. `--diff` is now a search flag, not a command, so `uffs diff` (bare) still searches for "diff". Retires the old bespoke diff surface (diff_wire wire types, the `diff` RPC + client helper, the `--diff` command). The `deleted` filter, `diff_baseline` parsing (compose + bare-star default), and the DELETED-tombstone match are unit-tested; the live round-trip is Windows-only. --- crates/uffs-cli/src/args.rs | 34 +-- crates/uffs-cli/src/commands.rs | 2 - crates/uffs-cli/src/commands/diff.rs | 234 ------------------ crates/uffs-cli/src/dispatch.rs | 5 - crates/uffs-client/src/connect_sync_diff.rs | 30 --- crates/uffs-client/src/lib.rs | 4 - crates/uffs-client/src/protocol/cli_args.rs | 15 ++ .../src/protocol/cli_args_helpers.rs | 40 +++ crates/uffs-client/src/protocol/diff_wire.rs | 105 -------- crates/uffs-client/src/protocol/mod.rs | 14 +- crates/uffs-core/src/search/filters/mod.rs | 28 +++ crates/uffs-core/src/search/filters/tests.rs | 35 +++ crates/uffs-daemon/src/handler.rs | 15 +- crates/uffs-daemon/src/handler_diff.rs | 87 ++++--- crates/uffs-daemon/src/index/diff.rs | 187 ++++++-------- crates/uffs-daemon/src/index/search.rs | 52 +++- 16 files changed, 306 insertions(+), 581 deletions(-) delete mode 100644 crates/uffs-cli/src/commands/diff.rs delete mode 100644 crates/uffs-client/src/connect_sync_diff.rs delete mode 100644 crates/uffs-client/src/protocol/diff_wire.rs diff --git a/crates/uffs-cli/src/args.rs b/crates/uffs-cli/src/args.rs index 702a3969a..0d6b4b4ea 100644 --- a/crates/uffs-cli/src/args.rs +++ b/crates/uffs-cli/src/args.rs @@ -504,7 +504,6 @@ COMMANDS: --search Explicit search (same as the bare default) --stats [PATH] Show filesystem statistics --agg Run aggregate analytics - --diff Diff a baseline MFT snapshot vs the live index (deletes) --deleted Forensic tombstone read: recently-deleted files from an MFT --daemon Manage the UFFS daemon (start/stop/load/status) --mcp Manage the UFFS MCP server @@ -529,6 +528,9 @@ COMMON OPTIONS: --columns Columns to output (default: all) --newer Modified after date/duration --older Modified before date/duration + --diff Search the DELETED set vs a baseline MFT capture + (combine with any filter: --diff C_old.bin --drive C + '*.txt' --newer 30d). Needs the drive loaded. --min-size Minimum file size (e.g. 100KB, 10MB) --max-size Maximum file size --profile Show timing breakdown @@ -623,36 +625,6 @@ pub(crate) fn print_stats_help() { print!("{STATS_HELP}"); } -/// Help text for `uffs --diff`. -const DIFF_HELP: &str = "\ -uffs --diff — Snapshot delete-visibility diff - -Diff a baseline MFT capture against the drive's LIVE in-memory index and report -what was created, deleted, or modified since the baseline. The deletion-visible -companion to --newer (which can only see creates/modifies). The drive must be -loaded in a running daemon. - -USAGE: uffs --diff --drive [OPTIONS] - -ARGUMENTS: - Path to the baseline snapshot (raw MFT capture) to - diff the live index against. - -OPTIONS: - -d, --drive Drive letter the baseline covers (required, e.g. C). - -n, --limit Max entries per class (added/deleted/modified); 0 = all. - --json Emit the raw result as JSON instead of a table. - -EXAMPLE: - uffs --diff D:\\snapshots\\c_last_week.bin --drive C -"; - -/// Print diff help. -#[expect(clippy::print_stdout, reason = "intentional help output")] -pub(crate) fn print_diff_help() { - print!("{DIFF_HELP}"); -} - /// Help text for `uffs --deleted`. const DELETED_HELP: &str = "\ uffs --deleted — Forensic tombstone read (recently-deleted files) diff --git a/crates/uffs-cli/src/commands.rs b/crates/uffs-cli/src/commands.rs index aedec66ea..cf4b26cdb 100644 --- a/crates/uffs-cli/src/commands.rs +++ b/crates/uffs-cli/src/commands.rs @@ -26,8 +26,6 @@ pub(crate) mod daemon_tiering; /// `uffs --deleted --mft-file ` — forensic tombstone read (deleted /// files). pub(crate) mod deleted; -/// `uffs --diff --drive ` — snapshot delete-visibility diff. -pub(crate) mod diff; /// Shared elevation gate for the mutating flows (uninstall / update): surface /// admin-only work up front and decide once (elevate / continue-without / /// abort) instead of failing mid-flow. Keeps both flows' elevation UX aligned. diff --git a/crates/uffs-cli/src/commands/diff.rs b/crates/uffs-cli/src/commands/diff.rs deleted file mode 100644 index 10d0281bd..000000000 --- a/crates/uffs-cli/src/commands/diff.rs +++ /dev/null @@ -1,234 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! `uffs --diff --drive ` — snapshot delete-visibility diff. -//! -//! Answers "what was created, deleted, or modified on a drive since a baseline -//! MFT capture" — the deletion-visible companion to `--newer`. Thin client: -//! parse args, fire the daemon's `diff` RPC (which diffs the baseline against -//! the drive's live in-memory index), render the classified delta. - -use anyhow::{Context as _, Result}; -use uffs_client::connect_sync::UffsClientSync; -use uffs_client::protocol::{DiffEntryWire, DiffParams, DiffResultWire}; -use uffs_mft::platform::DriveLetter; - -use crate::args::parse_drive_letter; - -/// Parsed `uffs --diff` invocation. -#[derive(Debug)] -struct DiffArgs { - /// Baseline snapshot path (raw MFT capture) to diff against. - baseline: String, - /// Drive letter the baseline covers and whose live index is the current - /// side. - drive: DriveLetter, - /// Max entries per class (0 = unlimited). - limit: u32, - /// Emit JSON instead of the human table. - json: bool, -} - -/// Run `uffs --diff --drive [--limit N] [--json]`. -/// -/// # Errors -/// -/// Returns an error on bad arguments, when the daemon is not running, or when -/// the `diff` RPC itself fails (drive not loaded / baseline unreadable). -pub(crate) fn run_diff(args: &[String]) -> Result<()> { - if args.iter().any(|arg| arg == "--help" || arg == "-h") { - crate::args::print_diff_help(); - return Ok(()); - } - - let parsed = parse_diff_args(args)?; - let mut client = UffsClientSync::connect_raw() - .map_err(|err| anyhow::anyhow!("Daemon is not running: {err}"))?; - - let params = DiffParams { - baseline: parsed.baseline, - drive: parsed.drive, - limit: parsed.limit, - }; - let result = client.diff(¶ms).with_context(|| "diff RPC failed")?; - - if parsed.json { - print_json(&result); - } else { - print_human(¶ms, &result); - } - Ok(()) -} - -/// Parse the `--diff` argument vector into a [`DiffArgs`]. -/// -/// The first non-flag token is the baseline path; `--drive`/`-d` is required; -/// `--limit`/`-n` and `--json` are optional. -fn parse_diff_args(args: &[String]) -> Result { - let mut baseline: Option = None; - let mut drive: Option = None; - let mut limit: u32 = 0; - let mut json = false; - - let mut iter = args.iter(); - while let Some(arg) = iter.next() { - match arg.as_str() { - "--drive" | "-d" => { - let val = iter - .next() - .with_context(|| "`--drive` requires a drive letter (e.g. C)")?; - drive = Some(parse_drive_letter(val)?); - } - "--limit" | "-n" => { - let val = iter.next().with_context(|| "`--limit` requires a number")?; - limit = val - .parse::() - .with_context(|| format!("invalid --limit value '{val}'"))?; - } - "--json" => json = true, - other if other.starts_with('-') => { - anyhow::bail!("unknown flag '{other}'; see `uffs --diff --help`"); - } - other => { - if baseline.replace(other.to_owned()).is_some() { - anyhow::bail!("only one baseline path may be given; got a second: '{other}'"); - } - } - } - } - - let baseline_path = baseline.with_context( - || "missing baseline snapshot path; usage: uffs --diff --drive ", - )?; - let drive_letter = - drive.with_context(|| "missing `--drive `; the diff needs to know which drive")?; - Ok(DiffArgs { - baseline: baseline_path, - drive: drive_letter, - limit, - json, - }) -} - -/// Render the delta as a human-readable table grouped by change class. -#[expect(clippy::print_stdout, reason = "CLI user-facing output")] -fn print_human(params: &DiffParams, result: &DiffResultWire) { - println!( - "Diff of drive {} vs baseline {}:", - params.drive, params.baseline - ); - println!( - " deleted {}, added {}, modified {}{}", - result.deleted.len(), - result.added.len(), - result.modified.len(), - if result.truncated { - " (truncated — pass a larger --limit for the full list)" - } else { - "" - }, - ); - - print_section("Deleted", &result.deleted, true); - print_section("Added", &result.added, false); - print_section("Modified", &result.modified, false); -} - -/// Print one non-empty class section. `with_size` appends the byte size (the -/// "what did I lose" figure that matters most for deletes). -#[expect(clippy::print_stdout, reason = "CLI user-facing output")] -fn print_section(label: &str, entries: &[DiffEntryWire], with_size: bool) { - if entries.is_empty() { - return; - } - println!("\n{label}:"); - for entry in entries { - if with_size { - println!(" {} ({})", entry.path, human_bytes(entry.size)); - } else { - println!(" {}", entry.path); - } - } -} - -/// Emit the raw wire result as pretty JSON for scripting. -#[expect(clippy::print_stdout, reason = "CLI user-facing output")] -fn print_json(result: &DiffResultWire) { - match serde_json::to_string_pretty(result) { - Ok(json) => println!("{json}"), - Err(err) => println!("{{\"error\":\"failed to serialize diff result: {err}\"}}"), - } -} - -/// Humanise a byte count with binary units (integer arithmetic — no floats, -/// to satisfy the strict `clippy::float_arithmetic` gate). -fn human_bytes(bytes: u64) -> String { - const KIB: u64 = 1024; - const MIB: u64 = 1024 * KIB; - const GIB: u64 = 1024 * MIB; - if bytes >= GIB { - let whole = bytes / GIB; - let hundredths = (bytes % GIB).saturating_mul(100) / GIB; - format!("{whole}.{hundredths:02} GiB") - } else if bytes >= MIB { - format!("{} MiB", bytes / MIB) - } else if bytes >= KIB { - format!("{} KiB", bytes / KIB) - } else { - format!("{bytes} B") - } -} - -#[cfg(test)] -mod tests { - use super::parse_diff_args; - - fn args(list: &[&str]) -> Vec { - list.iter().map(|item| (*item).to_owned()).collect() - } - - #[test] - fn parses_baseline_drive_and_limit() { - let parsed = - parse_diff_args(&args(&["C_old.bin", "--drive", "C", "--limit", "50"])).expect("parse"); - assert_eq!(parsed.baseline, "C_old.bin"); - assert_eq!(parsed.drive, uffs_mft::platform::DriveLetter::C); - assert_eq!(parsed.limit, 50); - assert!(!parsed.json); - } - - #[test] - fn drive_may_precede_the_positional_baseline() { - let parsed = parse_diff_args(&args(&["-d", "D", "snap.bin", "--json"])).expect("parse"); - assert_eq!(parsed.baseline, "snap.bin"); - assert_eq!(parsed.drive, uffs_mft::platform::DriveLetter::D); - assert_eq!(parsed.limit, 0, "no --limit → unlimited"); - assert!(parsed.json); - } - - #[test] - fn missing_drive_is_an_error() { - let err = parse_diff_args(&args(&["snap.bin"])).expect_err("must require --drive"); - assert!(err.to_string().contains("--drive"), "{err}"); - } - - #[test] - fn missing_baseline_is_an_error() { - let err = parse_diff_args(&args(&["--drive", "C"])).expect_err("must require baseline"); - assert!(err.to_string().contains("baseline"), "{err}"); - } - - #[test] - fn a_second_baseline_is_rejected() { - let err = parse_diff_args(&args(&["a.bin", "b.bin", "-d", "C"])) - .expect_err("two baselines must error"); - assert!(err.to_string().contains("second"), "{err}"); - } - - #[test] - fn unknown_flag_is_rejected() { - let err = parse_diff_args(&args(&["snap.bin", "-d", "C", "--bogus"])) - .expect_err("unknown flag must error"); - assert!(err.to_string().contains("unknown flag"), "{err}"); - } -} diff --git a/crates/uffs-cli/src/dispatch.rs b/crates/uffs-cli/src/dispatch.rs index 0d99eca60..ef6614486 100644 --- a/crates/uffs-cli/src/dispatch.rs +++ b/crates/uffs-cli/src/dispatch.rs @@ -26,8 +26,6 @@ pub(crate) enum Command { Stats, /// `--agg `. Agg, - /// `--diff --drive `. - Diff, /// `--deleted --mft-file `. Deleted, /// `--daemon `. @@ -50,7 +48,6 @@ impl Command { "--search" => Self::Search, "--stats" => Self::Stats, "--agg" | "--aggregate" => Self::Agg, - "--diff" => Self::Diff, "--deleted" => Self::Deleted, "--daemon" => Self::Daemon, "--mcp" => Self::Mcp, @@ -73,7 +70,6 @@ const COMMAND_TOKENS: &[&str] = &[ "--stats", "--agg", "--aggregate", - "--diff", "--deleted", "--daemon", "--mcp", @@ -116,7 +112,6 @@ pub(crate) fn dispatch_command(command: Command, args: &[String]) -> Result<()> Command::Search => crate::run_search(args), Command::Stats => crate::run_stats(args), Command::Agg => crate::run_aggregate(args), - Command::Diff => commands::diff::run_diff(args), Command::Deleted => commands::deleted::run_deleted(args), Command::Daemon => crate::run_daemon(args), Command::Mcp => commands::mcp_mgmt::mcp_from_args(args), diff --git a/crates/uffs-client/src/connect_sync_diff.rs b/crates/uffs-client/src/connect_sync_diff.rs deleted file mode 100644 index edb5d2763..000000000 --- a/crates/uffs-client/src/connect_sync_diff.rs +++ /dev/null @@ -1,30 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Snapshot-diff RPC helper for [`crate::connect_sync::UffsClientSync`]. -//! -//! Paired with the daemon-side `handle_diff` in -//! `crates/uffs-daemon/src/handler.rs` and the wire types in -//! [`crate::protocol::diff_wire`]. Same typed-envelope dance as the tiering -//! cluster: serialise the params, fire the JSON-RPC, deserialise the result. - -use crate::connect_sync::UffsClientSync; -use crate::error::ClientError; -use crate::protocol::{DiffParams, DiffResultWire}; - -impl UffsClientSync { - /// Diff a baseline snapshot against the live index for a drive via the - /// daemon's `diff` RPC (delete-visible companion to `--newer`). - /// - /// # Errors - /// - /// Returns `ClientError` on I/O / protocol failure, or when the daemon - /// rejects the request (drive not loaded → `ERR_NOT_READY`; baseline - /// unreadable → `ERR_INTERNAL`), surfaced as [`ClientError::Protocol`]. - pub fn diff(&mut self, params: &DiffParams) -> Result { - let payload = - serde_json::to_value(params).map_err(|err| ClientError::Protocol(err.to_string()))?; - let result = self.send_request("diff", Some(payload))?; - serde_json::from_value(result).map_err(|err| ClientError::Protocol(err.to_string())) - } -} diff --git a/crates/uffs-client/src/lib.rs b/crates/uffs-client/src/lib.rs index d72517008..850708ff6 100644 --- a/crates/uffs-client/src/lib.rs +++ b/crates/uffs-client/src/lib.rs @@ -127,10 +127,6 @@ pub mod connect_sync; /// `is_daemon_process`) — split off `connect_sync` to keep that file /// under the 800-LOC policy ceiling. pub(crate) mod connect_sync_autostart; -/// Snapshot-diff RPC helper (`diff`) — delete-visibility companion to -/// `--newer`. Split off `connect_sync` for module cohesion with the -/// [`protocol::diff_wire`] types it consumes. -pub(crate) mod connect_sync_diff; /// Platform-specific `platform_connect` impls and the `rpc_deadline` helper. /// /// Split `impl` blocks live on [`connect_sync::UffsClientSync`]; diff --git a/crates/uffs-client/src/protocol/cli_args.rs b/crates/uffs-client/src/protocol/cli_args.rs index d153dd17a..6ec1f44d4 100644 --- a/crates/uffs-client/src/protocol/cli_args.rs +++ b/crates/uffs-client/src/protocol/cli_args.rs @@ -93,6 +93,10 @@ impl SearchParams { raw.agg_page_size = Some(parse_u16("--agg-page-size", &pv)?); } "--attr" => raw.attr = Some(flag_val(&arg, "--attr", &mut iter)?), + // Snapshot delete-visibility diff: search the deleted set of the + // given baseline MFT capture vs the live index. Every other flag + // then filters/shapes that set like a normal search. + "--diff" => raw.diff_baseline = Some(flag_val(&arg, "--diff", &mut iter)?), "--newer" => raw.newer = Some(flag_val(&arg, "--newer", &mut iter)?), "--older" => raw.older = Some(flag_val(&arg, "--older", &mut iter)?), "--newer-created" => { @@ -313,6 +317,9 @@ struct RawCliArgs { malformed: Option, /// WI-4.4: `Some(true)` from `--malformed-path`. malformed_path: Option, + /// Snapshot-diff baseline path from `--diff `: turns the query + /// into a search over the deleted set of that baseline vs the live index. + diff_baseline: Option, profile: bool, benchmark: bool, no_cache: bool, @@ -768,7 +775,15 @@ impl RawCliArgs { // separately and overrides this field directly on the // passthrough `SearchParams`. output_drive_targets: drives, + // Snapshot-diff baseline (set by the `--diff` command path, which + // parses the remaining flags as a normal search); a plain search + // leaves it unset. + diff_baseline: self.diff_baseline.clone(), }; + // A diff with no explicit pattern lists every deleted file. + if params.diff_baseline.is_some() && params.pattern.is_empty() { + "*".clone_into(&mut params.pattern); + } params.populate_canonical_fields(); Ok(params) } diff --git a/crates/uffs-client/src/protocol/cli_args_helpers.rs b/crates/uffs-client/src/protocol/cli_args_helpers.rs index 1a3c7854b..ee404430f 100644 --- a/crates/uffs-client/src/protocol/cli_args_helpers.rs +++ b/crates/uffs-client/src/protocol/cli_args_helpers.rs @@ -505,6 +505,46 @@ mod cli_args_error_tests { },); } + /// `--diff ` sets `diff_baseline` and composes with a plain + /// pattern + the `*.ext` ext-glob sugar, so a snapshot-diff is just a + /// search over the deleted set. + #[test] + fn from_cli_args_diff_baseline_composes_with_filters() { + use crate::protocol::SearchParams; + // `*.txt` is UFFS ext-glob sugar → pattern `*` + an ext filter, so the + // diff's deleted set is filtered to `.txt`; a plain `report` pattern + // would survive verbatim. Use the latter to pin pattern composition. + let args = vec![ + "report".to_owned(), + "--diff".to_owned(), + "C_old.bin".to_owned(), + "--drive".to_owned(), + "C".to_owned(), + ]; + let params = SearchParams::from_cli_args(&args).expect("valid diff search"); + assert_eq!(params.diff_baseline.as_deref(), Some("C_old.bin")); + assert_eq!( + params.pattern, "report", + "the pattern still filters the deleted set" + ); + } + + /// A bare `--diff ` (no pattern) defaults the pattern to `*` so + /// it lists every deleted file. + #[test] + fn from_cli_args_bare_diff_defaults_pattern_to_star() { + use crate::protocol::SearchParams; + let args = vec![ + "--diff".to_owned(), + "C_old.bin".to_owned(), + "--drive".to_owned(), + "C".to_owned(), + ]; + let params = SearchParams::from_cli_args(&args).expect("valid bare diff"); + assert_eq!(params.diff_baseline.as_deref(), Some("C_old.bin")); + assert_eq!(params.pattern, "*", "a bare diff lists all deleted files"); + } + /// End-to-end: a second positional argument after the pattern /// surfaces as `UnexpectedArgument`. #[test] diff --git a/crates/uffs-client/src/protocol/diff_wire.rs b/crates/uffs-client/src/protocol/diff_wire.rs deleted file mode 100644 index e2adba8f1..000000000 --- a/crates/uffs-client/src/protocol/diff_wire.rs +++ /dev/null @@ -1,105 +0,0 @@ -// SPDX-License-Identifier: MPL-2.0 -// Copyright (c) 2025-2026 SKY, LLC. - -//! Wire types for the `diff` method (snapshot delete-visibility diff). -//! -//! The CLI sends [`DiffParams`] naming a baseline snapshot + the drive it -//! covers; the daemon loads that baseline, diffs it against the live in-memory -//! index for the drive (`uffs_core::diff`), resolves every changed row to a -//! full path, and returns a [`DiffResultWire`]. Split into its own module (per -//! the 800-LOC policy and to keep `mod.rs` focused on the JSON-RPC envelope). - -use serde::{Deserialize, Serialize}; - -/// Parameters for the `diff` method. -/// -/// "What changed on `drive` between the `baseline` snapshot and now" — the -/// deletion-visible companion to `--newer`, which can only see -/// creates/modifies. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DiffParams { - /// Path to the baseline snapshot to diff against — a raw MFT capture - /// (`.bin`) the daemon can load into a compact index. The live index for - /// `drive` is the "current" side. - pub baseline: String, - /// Drive letter the baseline covers and whose live index is the current - /// side of the diff. - pub drive: uffs_mft::platform::DriveLetter, - /// Maximum entries returned **per class** (added / deleted / modified). - /// `0` = unlimited. When a class is capped, [`DiffResultWire::truncated`] - /// is set so the caller can tell a capped list from a complete one. - #[serde(default)] - pub limit: u32, -} - -/// One changed file in a [`DiffResultWire`]: a full path plus render metadata. -#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] -pub struct DiffEntryWire { - /// Full path (`C:\Users\…\file.ext`). - pub path: String, - /// Logical size in bytes (current size for a modify; last-known baseline - /// size for a delete). - pub size: u64, - /// Last-write time in Unix microseconds, from the same snapshot as `path`. - pub modified: i64, -} - -/// Result of the `diff` method: the classified, path-resolved delta. -#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] -pub struct DiffResultWire { - /// Files present now but not in the baseline (created since it). - pub added: Vec, - /// Files present in the baseline but gone now (deleted since it). - pub deleted: Vec, - /// Files in both whose size or last-write time changed. - pub modified: Vec, - /// `true` when `limit` capped at least one class (more changes exist than - /// were returned). - pub truncated: bool, -} - -#[cfg(test)] -mod tests { - use super::{DiffEntryWire, DiffParams, DiffResultWire}; - - #[test] - fn diff_params_round_trip_through_json() { - let params = DiffParams { - baseline: r"D:\snapshots\c_2026-07-01.bin".to_owned(), - drive: uffs_mft::platform::DriveLetter::C, - limit: 500, - }; - let json = serde_json::to_value(¶ms).expect("serialize DiffParams"); - let back: DiffParams = serde_json::from_value(json).expect("deserialize DiffParams"); - assert_eq!(params, back); - } - - #[test] - fn diff_params_limit_defaults_to_zero_when_absent() { - // The CLI omits `limit` for an unlimited diff; it must default to 0. - let json = serde_json::json!({ "baseline": "x.bin", "drive": "C" }); - let params: DiffParams = serde_json::from_value(json).expect("deserialize"); - assert_eq!(params.limit, 0, "missing limit → unlimited (0)"); - } - - #[test] - fn diff_result_round_trips_through_json() { - let result = DiffResultWire { - added: vec![DiffEntryWire { - path: r"C:\new.txt".to_owned(), - size: 10, - modified: 9, - }], - deleted: vec![DiffEntryWire { - path: r"C:\gone.txt".to_owned(), - size: 200, - modified: 6, - }], - modified: vec![], - truncated: true, - }; - let json = serde_json::to_value(&result).expect("serialize DiffResultWire"); - let back: DiffResultWire = serde_json::from_value(json).expect("deserialize"); - assert_eq!(result, back); - } -} diff --git a/crates/uffs-client/src/protocol/mod.rs b/crates/uffs-client/src/protocol/mod.rs index cd024d601..1450bee53 100644 --- a/crates/uffs-client/src/protocol/mod.rs +++ b/crates/uffs-client/src/protocol/mod.rs @@ -13,7 +13,6 @@ pub mod aggregate_wire; pub mod cli_args; mod cli_args_helpers; -pub mod diff_wire; pub mod response; pub(crate) mod response_status; pub(crate) mod response_tiering; @@ -24,7 +23,6 @@ mod tests; pub use aggregate_wire::{ AggregateResultWire, AggregateSpecWire, BucketWire, DrilldownWire, SampleRowWire, StatsWire, }; -pub use diff_wire::{DiffEntryWire, DiffParams, DiffResultWire}; use serde::{Deserialize, Serialize}; // ──────────────────────────────────────────────────────────────────────────── @@ -548,6 +546,17 @@ pub struct SearchParams { /// empty because the MFT path is a separate wire selector. #[serde(default, skip_serializing_if = "Vec::is_empty")] pub output_drive_targets: Vec, + + // ── Snapshot-diff (delete visibility) ────────────────────────── + /// When set, this is a **snapshot-diff search**: the daemon loads the + /// baseline MFT capture at this path, marks the records whose File + /// Reference vanished from the live index (the deletes), and runs this + /// search over the **baseline** restricted to those deleted rows. Every + /// other field (pattern, `ext`, `newer`/`older`, `min_size`, sort, + /// projection, output format) then filters/shapes the deleted set exactly + /// like a normal search. `None` = ordinary live search. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub diff_baseline: Option, } /// Default-true helper for serde. @@ -622,6 +631,7 @@ impl Default for SearchParams { output_tz_offset_hours: None, output_format: None, output_drive_targets: Vec::new(), + diff_baseline: None, } } } diff --git a/crates/uffs-core/src/search/filters/mod.rs b/crates/uffs-core/src/search/filters/mod.rs index 6ea1c7dd6..db0d988ae 100644 --- a/crates/uffs-core/src/search/filters/mod.rs +++ b/crates/uffs-core/src/search/filters/mod.rs @@ -150,6 +150,16 @@ pub struct SearchFilters { /// UTF-8 and would match nothing). pub malformed: Option, + /// Filter on whether the record is a **deleted tombstone** — its + /// `FileFlags::DELETED` bit (`0x8000`, bit 15, UFFS-internal) is set. + /// `Some(true)` keeps only deleted records; + /// `Some(false)` only live ones; `None` = no filter. + /// + /// Populated by the snapshot-diff path, which marks the baseline rows that + /// vanished from the current index before running the normal search over + /// the baseline — so deleted files are filterable by every other criterion. + pub deleted: Option, + /// Render ill-formed names with greppable `` markers instead of /// the default U+FFFD (`�`). A display-only option (not a filter): it /// selects [`crate::compact::MalformedRender`] for the resolved path + name @@ -400,6 +410,9 @@ impl SearchFilters { // predicate compiler (it is not a legacy positional param), so the // param-based constructor leaves it disabled. malformed: None, + // Set by the snapshot-diff path (marks vanished baseline rows), + // not a legacy positional param; disabled for a normal search. + deleted: None, // Display-only; the daemon sets it from the request's // `normalize_malformed` flag, so it defaults off here. normalize_malformed: false, @@ -634,6 +647,12 @@ impl SearchFilters { if self.attr_exclude != 0 && (rec.flags & self.attr_exclude) != 0 { return false; } + if let Some(want_deleted) = self.deleted { + let is_deleted = (rec.flags & DELETED_TOMBSTONE_FLAG) != 0; + if is_deleted != want_deleted { + return false; + } + } if let Some(min) = self.min_descendants && rec.descendants < min { @@ -827,8 +846,17 @@ impl SearchFilters { // gate (`has_filters = !is_empty()`) skip `matches_record`, so the // filter silently no-ops on `uffs "*" --malformed`. && self.malformed.is_none() + // A deleted-tombstone toggle is a real filter — same match-all-gate + // reasoning as `malformed` above. + && self.deleted.is_none() } } +/// UFFS-internal "deleted tombstone" bit — mirrors +/// `uffs_mft::flags::FileFlags::DELETED` (0x8000, bit 15, reserved in NTFS). +/// The snapshot-diff path sets it on a baseline record whose File Reference +/// vanished from the current index; `SearchFilters::deleted` filters on it. +const DELETED_TOMBSTONE_FLAG: u32 = 0x8000; + #[cfg(test)] mod tests; diff --git a/crates/uffs-core/src/search/filters/tests.rs b/crates/uffs-core/src/search/filters/tests.rs index e64eb580e..ae175a1f2 100644 --- a/crates/uffs-core/src/search/filters/tests.rs +++ b/crates/uffs-core/src/search/filters/tests.rs @@ -1220,6 +1220,41 @@ fn is_ext_only_false_with_type_filter() { // Attribute presets // ═══════════════════════════════════════════════════════════════════════════ +#[test] +fn deleted_filter_selects_only_tombstoned_records() { + let mut names = Vec::new(); + let live = test_record("live.txt", &mut names); // flags = 0x20 (ARCHIVE) + let mut gone = test_record("gone.txt", &mut names); + gone.flags |= 0x8000; // FileFlags::DELETED tombstone bit + + let fold = CaseFold::default_table(); + + // `--deleted` (Some(true)) is a real filter and keeps only the tombstone. + let only_deleted = SearchFilters { + deleted: Some(true), + ..Default::default() + }; + assert!( + !only_deleted.is_empty(), + "--deleted must register as active" + ); + assert!(only_deleted.matches_record(&gone, &names, &mut Vec::new(), fold)); + assert!(!only_deleted.matches_record(&live, &names, &mut Vec::new(), fold)); + + // Some(false) keeps only live records. + let only_live = SearchFilters { + deleted: Some(false), + ..Default::default() + }; + assert!(!only_live.matches_record(&gone, &names, &mut Vec::new(), fold)); + assert!(only_live.matches_record(&live, &names, &mut Vec::new(), fold)); + + // None = no filter: both pass. + let no_filter = SearchFilters::default(); + assert!(no_filter.matches_record(&gone, &names, &mut Vec::new(), fold)); + assert!(no_filter.matches_record(&live, &names, &mut Vec::new(), fold)); +} + #[path = "tests_ext.rs"] mod tests_ext; diff --git a/crates/uffs-daemon/src/handler.rs b/crates/uffs-daemon/src/handler.rs index 8c490c514..211365a98 100644 --- a/crates/uffs-daemon/src/handler.rs +++ b/crates/uffs-daemon/src/handler.rs @@ -39,9 +39,10 @@ mod blob; mod parse_search_params; use parse_search_params::ParseSearchParamsError; -// `handle_diff` lives in a sibling file for the same 800-LOC policy reason as -// `handler_blob.rs`; `#[path]` keeps it an `impl RequestHandler` method so the -// dispatcher above calls `self.handle_diff(...)` unchanged. +// `diff_search_response` (the snapshot-diff error mapping) lives in a sibling +// file for the same 800-LOC policy reason as `handler_blob.rs`; `#[path]` keeps +// it an `impl RequestHandler` method the dispatcher calls as +// `self.diff_search_response(...)`. #[path = "handler_diff.rs"] mod diff_handler; @@ -76,7 +77,6 @@ impl RequestHandler { "load_drive" => self.handle_load_drive(id, req).await, "refresh" => self.handle_refresh(id, req), "facet_values" => self.handle_facet_values(id, req).await, - "diff" => self.handle_diff(id, req).await, "keepalive" => self.handle_keepalive(id, req), "shutdown" => self.handle_shutdown(id, req), // Phase 8-B … 8-E — operator-driven memory tiering. @@ -118,7 +118,12 @@ impl RequestHandler { } } - let mut response = self.index.search(&search_params).await; + // A `--diff ` routes to the snapshot-diff path (which may + // early-return a JSON-RPC error); everything else is a live search. + let mut response = match self.search_or_diff(id, &search_params).await { + Ok(resp) => resp, + Err(error_json) => return error_json, + }; // Row count captured up-front for logging and threshold // dispatch: both blob-packing and shmem-rows routing may // replace the payload variant in-place, at which point diff --git a/crates/uffs-daemon/src/handler_diff.rs b/crates/uffs-daemon/src/handler_diff.rs index 5a9d03a3f..35addce61 100644 --- a/crates/uffs-daemon/src/handler_diff.rs +++ b/crates/uffs-daemon/src/handler_diff.rs @@ -1,60 +1,59 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! The `diff` method handler for [`super::RequestHandler`]. +//! Snapshot-diff error mapping for [`super::RequestHandler`]. //! //! Lifted out of `handler.rs` to keep that file under the 800-line policy -//! ceiling. Re-attached via `#[path = "handler_diff.rs"] mod diff_handler;` in -//! `handler.rs`, so `handle_diff` stays an `impl RequestHandler` method the -//! dispatcher calls as `self.handle_diff(...)`. +//! ceiling. Re-attached via `#[path = "handler_diff.rs"] mod diff_handler;`, so +//! `diff_search_response` stays an `impl RequestHandler` method the search +//! handler calls as `self.diff_search_response(...)`. //! -//! Parsing + wire-mapping only: the classification, path resolution, baseline -//! load, and live-index snapshot all live in `IndexManager::diff_snapshot` -//! (`crate::index::diff`) and `uffs_core::diff`. +//! The diff itself lives in `IndexManager::diff_search` (`crate::index::diff`); +//! this only maps its [`crate::index::diff::DiffError`] setup failures onto the +//! JSON-RPC error envelope. +use uffs_client::protocol::response::SearchResponse; use uffs_client::protocol::{ - DiffParams, ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_NOT_READY, RpcErrorResponse, RpcRequest, - RpcResponse, + ERR_INTERNAL, ERR_INVALID_PARAMS, ERR_NOT_READY, RpcErrorResponse, SearchParams, }; use super::RequestHandler; use crate::index::diff::DiffError; impl RequestHandler { - /// Handle the `diff` method — snapshot delete-visibility diff of a baseline - /// capture against the live index for a drive. - pub(super) async fn handle_diff(&self, id: u64, req: &RpcRequest) -> String { - let params: DiffParams = match req - .params - .as_ref() - .map(|val| serde_json::from_value(val.clone())) - { - Some(Ok(params)) => params, - Some(Err(err)) => { - return serde_json::to_string(&RpcErrorResponse::error( - Some(id), - ERR_INVALID_PARAMS, - &format!("diff: invalid params: {err}"), - )) - .unwrap_or_default(); - } - None => { - return serde_json::to_string(&RpcErrorResponse::error( - Some(id), - ERR_INVALID_PARAMS, - "diff: missing params (`baseline` + `drive` required)", - )) - .unwrap_or_default(); - } - }; + /// Resolve a search request to its response: a snapshot diff when + /// `params.diff_baseline` is set (via [`Self::diff_search_response`], which + /// may yield a JSON-RPC error string), or an ordinary live search. + pub(super) async fn search_or_diff( + &self, + id: u64, + params: &SearchParams, + ) -> Result { + if params.diff_baseline.is_some() { + self.diff_search_response(id, params).await + } else { + Ok(self.index.search(params).await) + } + } - match self.index.diff_snapshot(¶ms).await { - Ok(result) => { - let value = serde_json::to_value(&result).unwrap_or_default(); - serde_json::to_string(&RpcResponse::success(id, value)).unwrap_or_default() - } + /// Run a snapshot-diff search, returning the response or a pre-serialized + /// JSON-RPC error string for the setup failures (no drive / drive not + /// loaded / baseline unreadable). + async fn diff_search_response( + &self, + id: u64, + params: &SearchParams, + ) -> Result { + match self.index.diff_search(params).await { + Ok(response) => Ok(response), + Err(DiffError::NoDrive) => Err(serde_json::to_string(&RpcErrorResponse::error( + Some(id), + ERR_INVALID_PARAMS, + "diff: `--drive ` is required (which live drive to diff against)", + )) + .unwrap_or_default()), Err(DiffError::DriveNotLoaded(letter)) => { - serde_json::to_string(&RpcErrorResponse::error( + Err(serde_json::to_string(&RpcErrorResponse::error( Some(id), ERR_NOT_READY, &format!( @@ -62,15 +61,15 @@ impl RequestHandler { (`uffs --daemon load --drive {letter}`)" ), )) - .unwrap_or_default() + .unwrap_or_default()) } Err(DiffError::BaselineLoad { path, source }) => { - serde_json::to_string(&RpcErrorResponse::error( + Err(serde_json::to_string(&RpcErrorResponse::error( Some(id), ERR_INTERNAL, &format!("diff: could not load baseline '{path}': {source}"), )) - .unwrap_or_default() + .unwrap_or_default()) } } } diff --git a/crates/uffs-daemon/src/index/diff.rs b/crates/uffs-daemon/src/index/diff.rs index abbd05573..cc68ec06d 100644 --- a/crates/uffs-daemon/src/index/diff.rs +++ b/crates/uffs-daemon/src/index/diff.rs @@ -1,39 +1,49 @@ // SPDX-License-Identifier: MPL-2.0 // Copyright (c) 2025-2026 SKY, LLC. -//! Snapshot delete-visibility diff for [`IndexManager`] (RPC `diff`). +//! Snapshot delete-visibility diff for [`IndexManager`], surfaced through the +//! **full search pipeline** (RPC `search` with a `diff_baseline`). //! -//! The `diff` RPC answers "what was created, deleted, or modified on a drive -//! since a baseline snapshot" — the deletion-visible companion to `--newer`, -//! which is structurally blind to deletes. It loads the caller's baseline MFT -//! capture off-thread, diffs it against the **live in-memory index** for the -//! drive via [`uffs_core::diff::resolve_delta`] (so the "current" side is the -//! hot index the daemon already serves searches from), and returns the -//! classified, path-resolved delta. +//! A diff answers "what was deleted on a drive since a baseline snapshot" — the +//! deletion-visible companion to `--newer`. Rather than a bespoke output, it +//! reuses everything a normal search does: pattern, `--ext`, +//! `--newer`/`--older`, `--min-size`, sort, projection, and every output format +//! all filter/shape the deleted set. The mechanism: //! -//! All the classification + path-resolution logic lives in `uffs_core::diff` -//! and is unit-tested there; this module is the daemon glue — snapshot the -//! registry, load the baseline, hand both to the engine, map the result onto -//! the wire type. +//! 1. Load the baseline MFT capture into a compact index (off the async +//! runtime). +//! 2. Diff it against the drive's **live** in-memory index by File Reference +//! ([`uffs_core::diff::diff_indexes`]) to find the rows that vanished. +//! 3. Mark those baseline rows with the `DELETED` flag. +//! 4. Run the normal search pipeline ([`IndexManager::run_search_over`]) over +//! the marked baseline, with a forced `deleted-only` filter. use alloc::sync::Arc; use std::path::PathBuf; -use uffs_client::protocol::{DiffEntryWire, DiffParams, DiffResultWire}; +use uffs_client::protocol::SearchParams; +use uffs_client::protocol::response::SearchResponse; use uffs_core::compact::MftSource; -use uffs_core::diff::{DeltaEntry, ResolvedDelta, resolve_delta}; +use uffs_core::search::backend::DriveIndex; use uffs_mft::platform::DriveLetter; use super::IndexManager; +/// UFFS-internal "deleted tombstone" bit — mirrors +/// `uffs_mft::flags::FileFlags::DELETED` (0x8000, bit 15, reserved in NTFS) and +/// `uffs_core::search::filters`'s `DELETED_TOMBSTONE_FLAG`. Set on a baseline +/// record whose File Reference vanished from the current index. +const DELETED_FLAG: u32 = 0x8000; + /// Why a `diff` request could not be served. Mapped to a JSON-RPC error by the /// handler; kept data-only here so this module stays free of wire concerns. pub(crate) enum DiffError { - /// The requested drive is not currently loaded in the live index, so there - /// is no "current" side to diff the baseline against. + /// No `--drive` was supplied, so there is no live index to diff against. + NoDrive, + /// The requested drive is not currently loaded in the live index. DriveNotLoaded(DriveLetter), /// The baseline snapshot at `path` could not be loaded into a compact - /// index (missing file, unreadable, not a valid MFT capture, …). + /// index. BaselineLoad { /// The baseline path the caller supplied (echoed back in the message). path: String, @@ -43,117 +53,72 @@ pub(crate) enum DiffError { } impl IndexManager { - /// Diff a baseline snapshot against the live index for `params.drive`. + /// Run a snapshot-diff search: diff `params.diff_baseline` against the live + /// index for `params.drives[0]`, then search the deleted set with the full + /// filter/sort/output pipeline. /// /// # Errors /// - /// Returns [`DiffError::DriveNotLoaded`] when the drive has no live index, - /// or [`DiffError::BaselineLoad`] when the baseline path cannot be loaded. - pub(crate) async fn diff_snapshot( + /// [`DiffError::NoDrive`] when no drive is given, + /// [`DiffError::DriveNotLoaded`] when it has no live index, or + /// [`DiffError::BaselineLoad`] when the baseline path cannot be loaded. + pub(crate) async fn diff_search( &self, - params: &DiffParams, - ) -> Result { + params: &SearchParams, + ) -> Result { + let drive = *params.drives.first().ok_or(DiffError::NoDrive)?; + let baseline_path = params.diff_baseline.clone().unwrap_or_default(); + // Current side: the live, hot in-memory index for the drive. - let snap = self.snapshot().await; - let Some(current) = snap + let snapshot = self.snapshot().await; + let Some(current) = snapshot .drives .iter() - .find(|dr| dr.letter == params.drive) + .find(|dr| dr.letter == drive) .map(Arc::clone) else { - return Err(DiffError::DriveNotLoaded(params.drive)); + return Err(DiffError::DriveNotLoaded(drive)); }; - drop(snap); // We hold the one Arc we need; release the registry snapshot. + drop(snapshot); - // Baseline side: load the caller's capture and diff, both off the async - // runtime — the MFT parse is I/O + CPU heavy and the diff hashes over - // the whole record array. `no_cache = true` forces a fresh read of the - // baseline rather than reusing any persisted cache for that path. - let baseline_path = PathBuf::from(¶ms.baseline); - let drive = params.drive; - let limit = uffs_mft::u32_as_usize(params.limit); - let current_for_task = Arc::clone(¤t); + // Load the baseline, diff it against the live index, and mark the + // vanished rows — all off the async runtime (MFT parse + a hash-diff). + let load_path = baseline_path.clone(); let outcome = tokio::task::spawn_blocking(move || { - let source = MftSource::File(baseline_path, Some(drive)); - let (baseline, _timing) = uffs_core::compact::load_drive(&source, true)?; - anyhow::Ok(resolve_delta(&baseline, ¤t_for_task, limit)) + let source = MftSource::File(PathBuf::from(&load_path), Some(drive)); + let (mut baseline, _timing) = uffs_core::compact::load_drive(&source, true)?; + let report = uffs_core::diff::diff_indexes(&baseline, ¤t); + let records = baseline.records.as_mut_slice(); + for &idx in &report.deleted { + if let Some(record) = records.get_mut(idx as usize) { + record.flags |= DELETED_FLAG; + } + } + anyhow::Ok(baseline) }) .await; - match outcome { - Ok(Ok(resolved)) => Ok(to_wire(resolved)), - Ok(Err(source)) => Err(DiffError::BaselineLoad { - path: params.baseline.clone(), - source, - }), - Err(join_err) => Err(DiffError::BaselineLoad { - path: params.baseline.clone(), - source: join_err.into(), - }), - } - } -} - -/// Map the engine's [`ResolvedDelta`] onto the JSON-RPC wire result. -fn to_wire(resolved: ResolvedDelta) -> DiffResultWire { - DiffResultWire { - added: resolved.added.into_iter().map(entry_to_wire).collect(), - deleted: resolved.deleted.into_iter().map(entry_to_wire).collect(), - modified: resolved.modified.into_iter().map(entry_to_wire).collect(), - truncated: resolved.truncated, - } -} - -/// Map one resolved [`DeltaEntry`] onto its wire form. -fn entry_to_wire(entry: DeltaEntry) -> DiffEntryWire { - DiffEntryWire { - path: entry.path, - size: entry.size, - modified: entry.modified, - } -} - -#[cfg(test)] -mod tests { - use uffs_core::diff::{DeltaEntry, ResolvedDelta}; - - use super::{entry_to_wire, to_wire}; - - #[test] - fn to_wire_preserves_every_class_and_the_truncated_flag() { - let resolved = ResolvedDelta { - added: vec![DeltaEntry { - path: r"C:\new.txt".to_owned(), - size: 10, - modified: 9, - }], - deleted: vec![DeltaEntry { - path: r"C:\gone.txt".to_owned(), - size: 200, - modified: 6, - }], - modified: vec![], - truncated: true, + let baseline = match outcome { + Ok(Ok(index)) => index, + Ok(Err(source)) => { + return Err(DiffError::BaselineLoad { + path: baseline_path, + source, + }); + } + Err(join_err) => { + return Err(DiffError::BaselineLoad { + path: baseline_path, + source: join_err.into(), + }); + } }; - let wire = to_wire(resolved); - assert_eq!(wire.added.len(), 1); - assert_eq!(wire.deleted.len(), 1); - assert!(wire.modified.is_empty()); - assert!(wire.truncated); - let added = wire.added.first().expect("one add"); - assert_eq!(added.path, r"C:\new.txt"); - assert_eq!(added.size, 10); - } - #[test] - fn entry_to_wire_is_a_faithful_field_copy() { - let wire = entry_to_wire(DeltaEntry { - path: r"C:\a.txt".to_owned(), - size: 42, - modified: 7, + // Search the marked baseline through the normal pipeline; the override + // forces the deleted-only filter (see `run_search_over`). + let index = Arc::new(DriveIndex { + drives: vec![Arc::new(baseline)], }); - assert_eq!(wire.path, r"C:\a.txt"); - assert_eq!(wire.size, 42); - assert_eq!(wire.modified, 7); + Ok(self.run_search_over(params, Some(index)).await) } } diff --git a/crates/uffs-daemon/src/index/search.rs b/crates/uffs-daemon/src/index/search.rs index d7b795ff1..c578ca639 100644 --- a/crates/uffs-daemon/src/index/search.rs +++ b/crates/uffs-daemon/src/index/search.rs @@ -11,6 +11,7 @@ //! Search execution: query dispatch, profile construction, and drive info. +use alloc::sync::Arc; use core::sync::atomic::Ordering; use std::time::Instant; @@ -18,14 +19,29 @@ use uffs_client::protocol::response::{ DriveProfile, SearchPayload, SearchProfile, SearchResponse, SearchRow, }; use uffs_client::protocol::{SearchFilterMode, SearchParams, SearchResponseMode}; -use uffs_core::search::backend::{FilterMode, PhaseTimings, SearchRequest, SortSpec, search_index}; +use uffs_core::search::backend::{ + DriveIndex, FilterMode, PhaseTimings, SearchRequest, SortSpec, search_index, +}; use uffs_core::search::field::FieldId; use uffs_core::search::filters::{SearchFilterParams, SearchFilters}; use super::IndexManager; impl IndexManager { - /// Execute a search query (updates perf counters). + /// Execute a live search query over the registry snapshot (updates perf + /// counters). Snapshot-diff searches (`params.diff_baseline`) are routed by + /// the handler to [`Self::diff_search`] instead, so they can surface setup + /// errors (missing baseline / unloaded drive) as JSON-RPC errors. + pub(crate) async fn search(&self, params: &SearchParams) -> SearchResponse { + self.run_search_over(params, None).await + } + + /// Run the search pipeline over either the live registry snapshot + /// (`snapshot_override == None`) or a caller-supplied index — the marked + /// baseline built by [`Self::diff_search`]. When an override is present the + /// query is a snapshot diff: the `deleted-only` filter is forced on and the + /// registry-specific warm-up / dispatch-accounting is skipped (the baseline + /// is not a registry shard). /// /// When `params.profile` is `true`, populates `SearchResponse::profile` /// with a per-phase timing breakdown so the CLI can print it. @@ -37,7 +53,12 @@ impl IndexManager { clippy::cognitive_complexity, reason = "search filter application with many predicate branches" )] - pub(crate) async fn search(&self, params: &SearchParams) -> SearchResponse { + pub(crate) async fn run_search_over( + &self, + params: &SearchParams, + snapshot_override: Option>, + ) -> SearchResponse { + let is_diff = snapshot_override.is_some(); // Acquire a concurrency permit — blocks if too many searches // are already in flight. The effective cap is // `max(2, (cpus × 26) / (drives × 10))` by default (see @@ -149,6 +170,12 @@ impl IndexManager { // path (size / descendant bounds). Self::compile_predicates_into_filters(&mut filters, &effective_params.predicates); + // Snapshot-diff: the override index carries the baseline with its + // vanished rows pre-marked `DELETED`; restrict the search to those. + if is_diff { + filters.deleted = Some(true); + } + // Phase 3 Commit C — promote any Parked/Cold shards in the // touched set before we snapshot the active subset. Fast // path (single read-lock acquisition, no work) when every @@ -164,18 +191,27 @@ impl IndexManager { // skip the promote (zero-RAM-touch contract). Empty // `ext_terms` short-circuits to the Phase-3 always-promote // behaviour. - self.ensure_warm_for_dispatch(&effective_params.drives, &filters.extensions) - .await; + // Registry warm-up only applies to live shards; a diff searches the + // caller's baseline index, which is not in the registry. + if !is_diff { + self.ensure_warm_for_dispatch(&effective_params.drives, &filters.extensions) + .await; + } // ── Snapshot the index (< 1 μs) ──────────────────────────── let t_lock = profiling.then(Instant::now); - let snapshot = self.snapshot().await; + let snapshot = match snapshot_override { + Some(baseline) => baseline, + None => self.snapshot().await, + }; // Phase 1 of memory-tiering: record this dispatch on every // active shard so `DriveStats::decay_ema` (consumed by Phase 6 // adaptive-TTL) accumulates a real signal. See // `crate::cache::DriveStats` and the `record_search_dispatch` - // doc comment. - self.record_search_dispatch().await; + // doc comment. Skipped for a diff (the baseline is not a shard). + if !is_diff { + self.record_search_dispatch().await; + } let lock_us = t_lock.map_or(0, |ts| ts.elapsed().as_micros()); // Snapshot per-drive info (only when profiling). From 429955ba272c5095c37dca7548305d89dd1eb4be Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:23:49 -0700 Subject: [PATCH 09/11] feat(cli): wire uffs --snapshot + live uffs --deleted --drive C Closes the two capture-side gaps so the delete-visibility workflow is first-class end to end on Windows. - `uffs --snapshot --drive C --out C_baseline.bin` captures the live MFT to a baseline file (the step 1 that `--diff` needs). Thin wrapper over the same proven `MftReader::open` + `save_raw_to_file` the `uffs-mft save` diagnostic uses; `--no-compress`, `--compression-level`, `--raw` mirror it. - `uffs --deleted --drive C` now scans the LIVE volume (in addition to `--mft-file`): reads the raw MFT via the fast parallel reader (`MftReader::read_raw`) into an in-memory `RawMftData`, then runs the exact same forensic tombstone pass. `--mft-file` XOR `--drive` is required. The live MFT read is Windows + elevated; both commands return an actionable error on other platforms (and point at the `--mft-file` offline path). Arg parsing for both is host-unit-tested; the Windows I/O compiles clean under xwin clippy and reuses library primitives already shipping in `uffs-mft`. --- crates/uffs-cli/src/args.rs | 39 ++++- crates/uffs-cli/src/commands.rs | 2 + crates/uffs-cli/src/commands/deleted.rs | 109 ++++++++++--- crates/uffs-cli/src/commands/snapshot.rs | 193 +++++++++++++++++++++++ crates/uffs-cli/src/dispatch.rs | 5 + 5 files changed, 324 insertions(+), 24 deletions(-) create mode 100644 crates/uffs-cli/src/commands/snapshot.rs diff --git a/crates/uffs-cli/src/args.rs b/crates/uffs-cli/src/args.rs index 0d6b4b4ea..1a626b7ac 100644 --- a/crates/uffs-cli/src/args.rs +++ b/crates/uffs-cli/src/args.rs @@ -505,6 +505,7 @@ COMMANDS: --stats [PATH] Show filesystem statistics --agg Run aggregate analytics --deleted Forensic tombstone read: recently-deleted files from an MFT + --snapshot Capture the live MFT to a baseline file (Windows, for --diff) --daemon Manage the UFFS daemon (start/stop/load/status) --mcp Manage the UFFS MCP server --update [ACTION] Self-update (snapshot/acquire/apply/doctor/recover) @@ -625,6 +626,34 @@ pub(crate) fn print_stats_help() { print!("{STATS_HELP}"); } +/// Help text for `uffs --snapshot`. +const SNAPSHOT_HELP: &str = "\ +uffs --snapshot — Capture the live MFT to a baseline file + +Save the drive's current MFT so a later `uffs --diff --drive C` can +report what was deleted since. Reads the live NTFS MFT: Windows + Administrator. + +USAGE: uffs --snapshot --drive --out [OPTIONS] + +OPTIONS: + -d, --drive Drive to capture (required, e.g. C). + -o, --out Output .bin path (required). + --no-compress Store uncompressed (default: zstd-compressed). + --compression-level zstd level 1-22 (default 3). + --raw Headerless raw dump for other MFT tools; implies + --no-compress and is NOT loadable by `uffs --diff`. + +EXAMPLE: + uffs --snapshot --drive C --out C_baseline.bin + uffs --diff C_baseline.bin --drive C '*.txt' # later: what .txt was deleted +"; + +/// Print snapshot help. +#[expect(clippy::print_stdout, reason = "intentional help output")] +pub(crate) fn print_snapshot_help() { + print!("{SNAPSHOT_HELP}"); +} + /// Help text for `uffs --deleted`. const DELETED_HELP: &str = "\ uffs --deleted — Forensic tombstone read (recently-deleted files) @@ -634,12 +663,14 @@ parent, timestamps) intact until the MFT slot is reused. This surfaces those not-in-use records as recently-deleted tombstones and reconstructs each path from the surviving parent chain. No baseline needed. -USAGE: uffs --deleted --mft-file [OPTIONS] +USAGE: uffs --deleted (--mft-file | --drive ) [OPTIONS] + +SOURCE (one required): + --mft-file Offline MFT capture to scan. + -d, --drive Live volume scan (Windows, elevated). With --mft-file, + just labels reconstructed paths. OPTIONS: - --mft-file MFT capture to scan (required; a live --drive scan is - not wired yet). - -d, --drive Drive letter to label reconstructed paths with. -n, --limit Max tombstones to print (0 = all). --json Emit JSON instead of a table. diff --git a/crates/uffs-cli/src/commands.rs b/crates/uffs-cli/src/commands.rs index cf4b26cdb..48d09ac15 100644 --- a/crates/uffs-cli/src/commands.rs +++ b/crates/uffs-cli/src/commands.rs @@ -30,6 +30,8 @@ pub(crate) mod deleted; /// admin-only work up front and decide once (elevate / continue-without / /// abort) instead of failing mid-flow. Keeps both flows' elevation UX aligned. pub(crate) mod elevation; +/// `uffs --snapshot --drive C --out FILE` — capture the live MFT to a baseline. +pub(crate) mod snapshot; // Index and info subcommands were merged into other modules. /// MCP server management subcommands. pub(crate) mod mcp_mgmt; diff --git a/crates/uffs-cli/src/commands/deleted.rs b/crates/uffs-cli/src/commands/deleted.rs index c5363c942..266938421 100644 --- a/crates/uffs-cli/src/commands/deleted.rs +++ b/crates/uffs-cli/src/commands/deleted.rs @@ -36,12 +36,16 @@ use crate::commands::output::format_filetime_local; /// path walk terminates here. const ROOT_FRS: u64 = 5; -/// Parsed `uffs --deleted` invocation. +/// Parsed `uffs --deleted` invocation. The source is either an offline capture +/// (`--mft-file`) or the live drive (`--drive`, Windows). #[derive(Debug)] struct DeletedArgs { - /// MFT capture to scan (raw `$MFT` dump). - mft_file: PathBuf, - /// Drive letter to label reconstructed paths with (default `X`). + /// Offline MFT capture to scan (raw `$MFT` dump). Mutually exclusive with a + /// live-drive scan; when both are given the file wins and `drive` only + /// labels paths. + mft_file: Option, + /// Drive letter: the live source (Windows) when `mft_file` is absent, or + /// just the path label otherwise. Defaults to `X` for labelling. drive: Option, /// Max tombstones to print (0 = all). limit: u32, @@ -93,13 +97,20 @@ pub(crate) fn run_deleted(args: &[String]) -> Result<()> { let parsed = parse_deleted_args(args)?; let drive = parsed.drive.unwrap_or(DriveLetter::X); - let options = LoadRawOptions { - header_only: false, - volume_letter: Some(drive), - forensic: true, + // Source the raw MFT from either an offline capture or the live drive. + let raw = match &parsed.mft_file { + Some(path) => { + let options = LoadRawOptions { + header_only: false, + volume_letter: Some(drive), + forensic: true, + }; + load_raw_mft(path, &options) + .with_context(|| format!("failed to read MFT capture '{}'", path.display()))? + } + None => read_live_raw(drive) + .with_context(|| format!("failed to read the live MFT of drive {drive}"))?, }; - let raw = load_raw_mft(&parsed.mft_file, &options) - .with_context(|| format!("failed to read MFT capture '{}'", parsed.mft_file.display()))?; // Pass 1: forensic-parse every slot but KEEP only the deleted records. // The default parser drops not-in-use records; forensic mode retains them. @@ -185,7 +196,8 @@ fn lookup_parent( /// Parse the `--deleted` argument vector. /// -/// `--mft-file ` is required; `--drive`, `--limit`, `--json` optional. +/// A source is required: `--mft-file ` (offline) or `--drive ` (live, +/// Windows). `--limit`, `--json` optional. fn parse_deleted_args(args: &[String]) -> Result { let mut mft_file: Option = None; let mut drive: Option = None; @@ -218,18 +230,64 @@ fn parse_deleted_args(args: &[String]) -> Result { } } - let mft_path = mft_file.with_context(|| { - "missing `--mft-file `; a live `--drive` scan is not wired yet — \ - point at an MFT capture" - })?; + if mft_file.is_none() && drive.is_none() { + anyhow::bail!( + "missing a source: pass `--mft-file ` (offline capture) or \ + `--drive ` (live volume, Windows)" + ); + } Ok(DeletedArgs { - mft_file: mft_path, + mft_file, drive, limit, json, }) } +/// Read the live raw MFT of `drive` into an in-memory [`RawMftData`] (Windows). +/// +/// Uses the fast parallel reader ([`uffs_mft::MftReader::read_raw`]); the +/// header is synthesised from the returned record size (the raw bytes carry no +/// UFFS header). `iter_records` only consults `record_size`/`record_count`, so +/// this is a faithful in-memory equivalent of an offline capture. +#[cfg(windows)] +fn read_live_raw(drive: DriveLetter) -> Result { + use uffs_mft::MftReader; + use uffs_mft::raw::RawMftHeader; + + let reader = MftReader::open(drive) + .with_context(|| format!("failed to open drive {drive}: (needs Administrator)"))?; + let (data, record_size) = reader.read_raw()?; + let data_len = uffs_mft::usize_to_u64(data.len()); + let record_count = if record_size == 0 { + 0 + } else { + data_len / u64::from(record_size) + }; + let header = RawMftHeader { + // Format version of the equivalent on-disk capture; unused by + // `iter_records`, set for a well-formed in-memory header. + version: 3, + flags: 0, + record_size, + record_count, + original_size: data_len, + compressed_size: 0, + volume_letter: drive, + reserved_allocated_bytes: 0, + }; + Ok(RawMftData { header, data }) +} + +/// Non-Windows stub: reading a live volume is Windows-only. +#[cfg(not(windows))] +fn read_live_raw(drive: DriveLetter) -> Result { + anyhow::bail!( + "a live `--drive {drive}` scan reads the NTFS volume directly and requires \ + Windows (elevated); on other platforms use `--mft-file `" + ) +} + /// Reconstruct a deleted record's full path by walking `parent` up via /// `lookup` until the volume root. Returns `(path, complete)`; `complete` is /// `false` when a parent FRS is absent (the path is prefixed with `…`). @@ -381,16 +439,27 @@ mod tests { "--json", ])) .expect("parse"); - assert_eq!(parsed.mft_file.to_str(), Some("C.bin")); + assert_eq!( + parsed.mft_file.as_deref().and_then(std::path::Path::to_str), + Some("C.bin") + ); assert_eq!(parsed.drive, Some(DriveLetter::C)); assert_eq!(parsed.limit, 5); assert!(parsed.json); } #[test] - fn missing_mft_file_is_an_error() { - let err = parse_deleted_args(&args(&["-d", "C"])).expect_err("must require --mft-file"); - assert!(err.to_string().contains("--mft-file"), "{err}"); + fn a_bare_drive_is_a_valid_live_source() { + // `-d C` with no --mft-file is a live scan (Windows); parsing accepts it. + let parsed = parse_deleted_args(&args(&["-d", "C"])).expect("live drive parses"); + assert!(parsed.mft_file.is_none()); + assert_eq!(parsed.drive, Some(DriveLetter::C)); + } + + #[test] + fn no_source_is_an_error() { + let err = parse_deleted_args(&args(&["--limit", "5"])).expect_err("must require a source"); + assert!(err.to_string().contains("source"), "{err}"); } #[test] diff --git a/crates/uffs-cli/src/commands/snapshot.rs b/crates/uffs-cli/src/commands/snapshot.rs new file mode 100644 index 000000000..b405fd51d --- /dev/null +++ b/crates/uffs-cli/src/commands/snapshot.rs @@ -0,0 +1,193 @@ +// SPDX-License-Identifier: MPL-2.0 +// Copyright (c) 2025-2026 SKY, LLC. + +//! `uffs --snapshot --drive C --out FILE` — capture the live MFT to a file. +//! +//! The first step of the snapshot-diff workflow: save a baseline MFT capture +//! now so `uffs --diff --drive C` can later surface what was deleted. +//! Thin wrapper over the same proven library primitives the `uffs-mft save` +//! diagnostic uses (`MftReader::open` + `save_raw_to_file`). +//! +//! Reading the live MFT requires Windows + Administrator; on other platforms +//! the command returns an actionable error rather than failing obscurely. + +use std::path::PathBuf; + +use anyhow::{Context as _, Result}; +use uffs_mft::platform::DriveLetter; + +use crate::args::parse_drive_letter; + +/// Parsed `uffs --snapshot` invocation. +#[derive(Debug)] +struct SnapshotArgs { + /// Drive whose live MFT to capture. + drive: DriveLetter, + /// Output `.bin` path. + out: PathBuf, + /// Disable zstd compression (default: compressed). + no_compress: bool, + /// Headerless raw mode (compatible with other MFT tools; implies + /// no-compress). NOT loadable by `uffs --diff`, which needs the UFFS + /// header. + raw: bool, + /// zstd compression level (1-22). + compression_level: i32, +} + +/// Run `uffs --snapshot --drive --out [--no-compress] [--raw]`. +/// +/// # Errors +/// +/// Returns an error on bad arguments, on a non-Windows host, or when the live +/// MFT read / file write fails. +pub(crate) fn run_snapshot(args: &[String]) -> Result<()> { + if args.iter().any(|arg| arg == "--help" || arg == "-h") { + crate::args::print_snapshot_help(); + return Ok(()); + } + let parsed = parse_snapshot_args(args)?; + capture(&parsed) +} + +/// Parse the `--snapshot` argument vector. `--drive` + `--out` are required. +fn parse_snapshot_args(args: &[String]) -> Result { + let mut drive: Option = None; + let mut out: Option = None; + let mut no_compress = false; + let mut raw = false; + let mut compression_level: i32 = 3; + + let mut iter = args.iter(); + while let Some(arg) = iter.next() { + match arg.as_str() { + "--drive" | "-d" => { + let val = iter + .next() + .with_context(|| "`--drive` requires a drive letter (e.g. C)")?; + drive = Some(parse_drive_letter(val)?); + } + "--out" | "-o" => { + let val = iter.next().with_context(|| "`--out` requires a path")?; + out = Some(PathBuf::from(val)); + } + "--no-compress" => no_compress = true, + "--raw" => raw = true, + "--compression-level" => { + let val = iter + .next() + .with_context(|| "`--compression-level` requires a number (1-22)")?; + compression_level = val + .parse::() + .with_context(|| format!("invalid --compression-level '{val}'"))?; + } + other => anyhow::bail!("unknown argument '{other}'; see `uffs --snapshot --help`"), + } + } + + let drive_letter = + drive.with_context(|| "missing `--drive ` (which drive to capture)")?; + let out_path = out.with_context(|| "missing `--out ` (where to write the capture)")?; + Ok(SnapshotArgs { + drive: drive_letter, + out: out_path, + no_compress, + raw, + compression_level, + }) +} + +/// Read the live MFT for `args.drive` and write it to `args.out` (Windows). +#[cfg(windows)] +#[expect(clippy::print_stdout, reason = "CLI user-facing output")] +fn capture(args: &SnapshotArgs) -> Result<()> { + use uffs_mft::{MftReader, SaveRawOptions}; + + let reader = MftReader::open(args.drive) + .with_context(|| format!("failed to open drive {}: (needs Administrator)", args.drive))?; + let options = SaveRawOptions { + // `--raw` (headerless, for other tools) implies no compression. + compress: !args.no_compress && !args.raw, + compression_level: args.compression_level, + volume_letter: args.drive, + raw_compat: args.raw, + reserved_allocated_bytes: 0, + }; + let header = reader + .save_raw_to_file(&args.out, &options) + .with_context(|| format!("failed to write snapshot to {}", args.out.display()))?; + + println!( + "Saved MFT snapshot of {}: -> {} ({} records)", + args.drive, + args.out.display(), + header.record_count, + ); + if args.raw { + println!( + " Format: raw (headerless) — NOT loadable by `uffs --diff`; drop --raw for that." + ); + } else { + println!( + " Diff it later with: uffs --diff {} --drive {}", + args.out.display(), + args.drive, + ); + } + Ok(()) +} + +/// Non-Windows stub: the live MFT read is Windows-only. Every field is consumed +/// by the Windows path; the error references them all so the struct has no dead +/// fields on non-Windows hosts. +#[cfg(not(windows))] +fn capture(args: &SnapshotArgs) -> Result<()> { + anyhow::bail!( + "uffs --snapshot ({} -> {}, no_compress={}, raw={}, level={}) reads the live NTFS MFT \ + and requires Windows (elevated). On other platforms, diff two existing captures with \ + `uffs --diff --mft-file ` instead.", + args.drive, + args.out.display(), + args.no_compress, + args.raw, + args.compression_level, + ) +} + +#[cfg(test)] +mod tests { + use super::parse_snapshot_args; + + fn args(list: &[&str]) -> Vec { + list.iter().map(|item| (*item).to_owned()).collect() + } + + #[test] + fn parses_drive_out_and_options() { + let parsed = parse_snapshot_args(&args(&[ + "--drive", + "C", + "--out", + "C_base.bin", + "--no-compress", + ])) + .expect("parse"); + assert_eq!(parsed.drive, uffs_mft::platform::DriveLetter::C); + assert_eq!(parsed.out.to_str(), Some("C_base.bin")); + assert!(parsed.no_compress); + assert!(!parsed.raw); + assert_eq!(parsed.compression_level, 3_i32); + } + + #[test] + fn missing_drive_is_an_error() { + let err = parse_snapshot_args(&args(&["--out", "x.bin"])).expect_err("needs --drive"); + assert!(err.to_string().contains("--drive"), "{err}"); + } + + #[test] + fn missing_out_is_an_error() { + let err = parse_snapshot_args(&args(&["--drive", "C"])).expect_err("needs --out"); + assert!(err.to_string().contains("--out"), "{err}"); + } +} diff --git a/crates/uffs-cli/src/dispatch.rs b/crates/uffs-cli/src/dispatch.rs index ef6614486..ab611acc3 100644 --- a/crates/uffs-cli/src/dispatch.rs +++ b/crates/uffs-cli/src/dispatch.rs @@ -28,6 +28,8 @@ pub(crate) enum Command { Agg, /// `--deleted --mft-file `. Deleted, + /// `--snapshot --drive C --out `. + Snapshot, /// `--daemon `. Daemon, /// `--mcp `. @@ -49,6 +51,7 @@ impl Command { "--stats" => Self::Stats, "--agg" | "--aggregate" => Self::Agg, "--deleted" => Self::Deleted, + "--snapshot" => Self::Snapshot, "--daemon" => Self::Daemon, "--mcp" => Self::Mcp, // `--upgrade` is a HIDDEN alias for `--update` (winget/apt muscle @@ -71,6 +74,7 @@ const COMMAND_TOKENS: &[&str] = &[ "--agg", "--aggregate", "--deleted", + "--snapshot", "--daemon", "--mcp", "--update", @@ -113,6 +117,7 @@ pub(crate) fn dispatch_command(command: Command, args: &[String]) -> Result<()> Command::Stats => crate::run_stats(args), Command::Agg => crate::run_aggregate(args), Command::Deleted => commands::deleted::run_deleted(args), + Command::Snapshot => commands::snapshot::run_snapshot(args), Command::Daemon => crate::run_daemon(args), Command::Mcp => commands::mcp_mgmt::mcp_from_args(args), Command::Update => commands::update::run_update(args), From 2e3c163c03caa43ca551563c8ca33d943731964c Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 18:34:02 -0700 Subject: [PATCH 10/11] docs(delete-visibility): document --snapshot / --diff / --deleted MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - forensics-diagnostics.md: new "Delete Visibility (uffs CLI)" section covering the three commands, both mechanisms, and their honest limits. - cli-overview.md: --diff as a forensic filter (§2) + --snapshot / --deleted commands (§6), cross-linked to the forensic guide. - delete-visibility-snapshot-diff.md: status → implemented; note which phases shipped and what's still deferred (slice-8.6 reconciliation, baseline-retention ring). No new forensic doc created — engine/12-forensics-diagnostics.md already existed and is the right home. --- .../delete-visibility-snapshot-diff.md | 17 +++++++- .../engine/12-forensics-diagnostics.md | 41 ++++++++++++++++++- docs/user-manual/cli-overview.md | 28 +++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/docs/architecture/delete-visibility-snapshot-diff.md b/docs/architecture/delete-visibility-snapshot-diff.md index bfcef904b..6032c9651 100644 --- a/docs/architecture/delete-visibility-snapshot-diff.md +++ b/docs/architecture/delete-visibility-snapshot-diff.md @@ -5,7 +5,22 @@ Copyright (c) 2025-2026 SKY, LLC. # Delete visibility for the `--newer` fallback (snapshot diff + tombstone read) -**Status:** design sketch / proposed slice +**Status:** **implemented** — both mechanisms shipped. +- **Snapshot diff** (Mechanism 1): the NTFS File Reference `(seq, frs)` is + persisted inline on every `CompactRecord` (`file_ref`), `uffs_core::diff` + computes the delta, and `--diff ` is a search flag that runs the + full filter/sort/output pipeline over the deleted set (daemon + `IndexManager::diff_search`). `uffs --snapshot` captures the baseline. +- **Tombstone read** (Mechanism 2): `uffs --deleted` (live `--drive` or + offline `--mft-file`) surfaces not-in-use records via forensic parsing. +- User-facing surface is documented in + [engine/12-forensics-diagnostics.md](engine/12-forensics-diagnostics.md#delete-visibility-uffs-cli). + +Phases 1–3 of the plan below are done; the phased text is retained as the +design record. Not built: promoting the diff into the daemon's slice-8.6 +reconciliation, and a dedicated per-drive baseline-retention ring (today the +baseline is a capture file the user manages). + **Motivating gap:** the `--newer` (timestamp) delta path can report files *created or modified* after a date, but it is structurally blind to *deletions*. A deleted file simply stops appearing; a timestamp cannot express diff --git a/docs/architecture/engine/12-forensics-diagnostics.md b/docs/architecture/engine/12-forensics-diagnostics.md index 52225e6ef..2ae5d4d8b 100644 --- a/docs/architecture/engine/12-forensics-diagnostics.md +++ b/docs/architecture/engine/12-forensics-diagnostics.md @@ -45,7 +45,7 @@ When forensic mode is enabled, additional columns are available: ### Forensic Use Cases -- **Deleted file recovery**: Find recently deleted files whose MFT records haven't been reused +- **Deleted file recovery**: Find recently deleted files whose MFT records haven't been reused (surfaced by `uffs --deleted` / `uffs --diff` — see [Delete Visibility](#delete-visibility-uffs-cli)) - **Corruption detection**: Identify records with torn writes or disk errors - **Timeline analysis**: Use `$STANDARD_INFORMATION` vs `$FILE_NAME` timestamp discrepancies to detect anti-forensic timestamp manipulation - **Extension record analysis**: Understand file fragmentation across MFT records @@ -76,6 +76,45 @@ For each 1KB record in MFT: --- +## Delete Visibility (`uffs` CLI) + +The forensic engine above powers three user-facing `uffs` commands for answering "what was deleted". They use the two mechanisms in `docs/architecture/delete-visibility-snapshot-diff.md`: **snapshot diff** (compare two states) and **tombstone read** (surface not-in-use records). Every live MFT read needs Windows + Administrator; the offline (`--mft-file`) paths run anywhere. + +### `uffs --snapshot` — capture a baseline + +```bash +uffs --snapshot --drive C --out C_baseline.bin # zstd-compressed, UFFS header +uffs --snapshot --drive C --out C_base.bin --no-compress +``` + +Thin wrapper over the same `MftReader::open` + `save_raw_to_file` primitives as `uffs-mft save` (below), but part of the main `uffs` binary so the diff workflow needs no second tool. `--raw` produces a headerless dump for other MFT tools — but that form is **not** loadable by `uffs --diff`. + +### `uffs --diff` — snapshot diff, full-filter (primary) + +`--diff ` is a **search flag**, not a separate command: it diffs the baseline capture against the drive's live in-memory index (by NTFS File Reference — `(sequence_number << 48) | frs` — so a delete-then-reuse of an MFT slot is a delete + an add, never a false "modify"), marks the vanished rows, and runs the **normal search pipeline** over the deleted set. So deleted files filter/sort by every criterion a normal search has: + +```bash +uffs --diff C_baseline.bin --drive C # all deleted files +uffs --diff C_baseline.bin --drive C '*.txt' --newer 30d --min-size 1MB +uffs --diff C_baseline.bin --drive C 'report*' --sort -size -n 100 --format csv +``` + +This is the reliable, journal-free delete delta (offline captures, non-Windows, wrapped/disabled USN). It reports the *net* delete between two states, not every intermediate delete the way USN's continuous log does. The drive must be loaded in a running daemon (it supplies the live "current" side). + +### `uffs --deleted` — tombstone read (no baseline) + +Surfaces recently-deleted files straight from not-in-use MFT records — no baseline needed — reconstructing each path from the surviving parent chain: + +```bash +uffs --deleted --drive C # live volume (Windows, elevated) +uffs --deleted --mft-file C_old.bin # or an offline capture +uffs --deleted --drive C --limit 50 --json +``` + +Best-effort by nature: only deletes whose MFT slot has not been recycled are visible, the timestamp is the file's own last-write (not the deletion time), and a path is unreliable (flagged with a leading `…`) if a parent directory's slot was itself reused. The scan streams only the deleted records, so it stays memory-bounded on large volumes. + +--- + ## The `uffs-mft` Utility Binary The `uffs-mft` binary (from the `uffs-mft` crate) provides MFT-specific operations beyond search: diff --git a/docs/user-manual/cli-overview.md b/docs/user-manual/cli-overview.md index 6b846886d..57ba57b60 100644 --- a/docs/user-manual/cli-overview.md +++ b/docs/user-manual/cli-overview.md @@ -129,6 +129,7 @@ All filters are detailed in the [Filters guide](filters.md). Summary: | `--well-formed` | Forensic | Only valid names (inverse of `--malformed`) | | `--malformed-path` | Forensic | Match when any path segment is ill-formed | | `--normalize-malformed` | Forensic | Display: render corrupt code units as `` not `�` | +| `--diff ` | Forensic | Search the **deleted** set vs a baseline MFT capture; composes with every filter above ([Delete Visibility](../architecture/engine/12-forensics-diagnostics.md#delete-visibility-uffs-cli)) | | `-n, --limit ` | Limit | Max results (0 = unlimited) | --- @@ -230,6 +231,33 @@ uffs --agg count # Simple total count > **Full guide:** [Aggregation](aggregation.md) +### `uffs --snapshot` — capture a baseline MFT + +Save the drive's current MFT to a file so a later `uffs --diff` can report +what was deleted since. Reads the live NTFS MFT: **Windows + Administrator**. + +```bash +uffs --snapshot --drive C --out C_baseline.bin # zstd-compressed (default) +uffs --snapshot --drive C --out C_base.bin --no-compress +``` + +### `uffs --deleted` — forensic tombstone read + +Surface recently-deleted files straight from not-in-use MFT records — **no +baseline needed** — reconstructing each path from the surviving parent chain. +Best-effort: only deletes whose MFT slot hasn't been recycled are visible, and +the timestamp is the file's own last-write, not the deletion time. + +```bash +uffs --deleted --drive C # live volume (Windows, elevated) +uffs --deleted --mft-file C_old.bin # or an offline capture +uffs --deleted --drive C --limit 50 --json +``` + +> Deleting *with filters* since a baseline? Use the `--diff` search flag (§2) +> instead — it runs the full filter/sort/output pipeline over the deleted set. +> **Full guide:** [Delete Visibility](../architecture/engine/12-forensics-diagnostics.md#delete-visibility-uffs-cli) + ### `uffs --daemon` Manage the UFFS background daemon. The daemon starts automatically on From 89431ca94345e938d4dea67878f02d57b91ea7e6 Mon Sep 17 00:00:00 2001 From: Robert M1 <50460704+githubrobbi@users.noreply.github.com> Date: Mon, 13 Jul 2026 22:33:17 -0700 Subject: [PATCH 11/11] fix(parse): persist the MFT sequence number so the snapshot diff sees deletes MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compact `file_ref` is the NTFS File Reference `(sequence_number << 48) | frs`, but the parsers never set `FileRecord.sequence_number` — so `file_ref` degraded to the FRS (slot number) alone. MFT slot numbers are stable across delete-then-reuse, so `diff_indexes` found the same set of `frs` values in baseline and current and reported **0 deletions**. The `ntfs` module even noted "only consumes the FRS half; the sequence-number extraction is [not done]". Unit tests passed because they built synthetic records with `sequence_number` already set, hiding the gap. Fix: the two parsers that populate the compact index now copy the base record's header sequence number onto the record — - `process_record` (unified single-pass parser used by both the live read and the offline `.bin` load), and - `parse_record_to_index` (the USN live-update path). Cache version 12 -> 13: v12 caches were written with `file_ref == frs` (seq 0); rejecting them forces a rebuild that captures the real sequence number, so live diffs work without a manual cache clear. Regression test pins that `process_record` persists the header seq. --- crates/uffs-core/src/compact_cache.rs | 7 +++++- crates/uffs-mft/src/io/parser/mod.rs | 28 +++++++++++++++++++++++ crates/uffs-mft/src/io/parser/unified.rs | 10 ++++++++ crates/uffs-mft/src/parse/direct_index.rs | 5 ++++ 4 files changed, 49 insertions(+), 1 deletion(-) diff --git a/crates/uffs-core/src/compact_cache.rs b/crates/uffs-core/src/compact_cache.rs index f8e617d9e..d7ceb9d12 100644 --- a/crates/uffs-core/src/compact_cache.rs +++ b/crates/uffs-core/src/compact_cache.rs @@ -156,7 +156,12 @@ const COMPACT_MAGIC: &[u8; 8] = b"UFFSCOM\0"; /// for delete-diff and the forensic view. The record array is a bulk /// `bytemuck` memcpy, so the row-size change alone invalidates older caches /// at the header version check (a fresh MFT rebuild writes v12). -const COMPACT_VERSION: u16 = 12; +/// - v13: the MFT parsers now persist the record's `sequence_number`, so +/// `file_ref` carries the real slot-reuse generation instead of `0`. v12 +/// caches were written with `file_ref == frs` (seq 0), which makes the +/// snapshot diff blind to deletions (MFT slot numbers are stable across +/// reuse); rejecting them forces a rebuild that captures the sequence number. +const COMPACT_VERSION: u16 = 13; mod filters_io; pub mod parked; diff --git a/crates/uffs-mft/src/io/parser/mod.rs b/crates/uffs-mft/src/io/parser/mod.rs index 2324bcac2..a4d61af13 100644 --- a/crates/uffs-mft/src/io/parser/mod.rs +++ b/crates/uffs-mft/src/io/parser/mod.rs @@ -45,6 +45,34 @@ mod tests { assert!(!parse_record_to_fragment(&[0_u8; 3], 42, &mut fragment)); } + /// Regression pin: `process_record` must persist the header's NTFS + /// **sequence number** onto the record. Without it `file_ref` degrades to + /// the FRS alone, and the snapshot diff — keyed on the File Reference — + /// goes blind to deletions (MFT slot numbers are stable across reuse). + #[test] + fn process_record_persists_the_sequence_number() { + // Minimal in-use base FILE record; sequence_number is a u16 at header + // offset 0x10. No attributes needed — the record is created regardless + // and the fix copies the header seq onto it before the attribute loop. + let mut record = RecordBuilder::new(56).build(); + record + .get_mut(16..18) + .expect("header has a sequence-number field at 0x10") + .copy_from_slice(&0x1234_u16.to_le_bytes()); + + let mut index = MftIndex::new(crate::platform::DriveLetter::C); + let mut name_buf = String::new(); + process_record(&record, 42, &mut index, &mut name_buf); + + let rec = index + .find(crate::frs::Frs::new(42)) + .expect("process_record must create the base record"); + assert_eq!( + rec.sequence_number, 0x1234, + "the header sequence number must be persisted onto the record", + ); + } + // ── WI-5.2 panic-resistance corpus ────────────────────────────── // // The daemon builds with `panic = "abort"`: a single parser panic on a diff --git a/crates/uffs-mft/src/io/parser/unified.rs b/crates/uffs-mft/src/io/parser/unified.rs index 08e893e60..8718ecca0 100644 --- a/crates/uffs-mft/src/io/parser/unified.rs +++ b/crates/uffs-mft/src/io/parser/unified.rs @@ -345,6 +345,16 @@ pub fn process_record(data: &[u8], frs: u64, index: &mut MftIndex, name_buf: &mu let frs_base_typed = crate::frs::Frs::new(frs_base); let base_ri = u32_as_usize(index.ensure_record(frs_base_typed)); + // Persist the NTFS sequence number (slot-reuse generation) from the base + // record's own header. With the FRS it forms the File Reference the compact + // index packs into `file_ref`; the snapshot diff needs it to tell a + // delete-then-reuse of an MFT slot apart from an unchanged file (the slot + // number alone is stable across reuse). Extension records carry their own + // sequence, so only a base record's header sets the file's sequence. + if header.is_base_record() { + index.records[base_ri].sequence_number = header.sequence_number; + } + // ── Attribute loop ───────────────────────────────────────────────── let mut offset = usize::from(header.first_attribute_offset); let max_offset = core::cmp::min(u32_as_usize(header.bytes_in_use), data.len()); diff --git a/crates/uffs-mft/src/parse/direct_index.rs b/crates/uffs-mft/src/parse/direct_index.rs index 77cce4960..52b6f25ed 100644 --- a/crates/uffs-mft/src/parse/direct_index.rs +++ b/crates/uffs-mft/src/parse/direct_index.rs @@ -720,6 +720,11 @@ pub fn parse_record_to_index(data: &[u8], frs: u64, index: &mut crate::index::Mf // Now get or create the record in the index - no more index mutations // after this. let record = index.get_or_create(crate::frs::Frs::new(frs)); + // Persist the NTFS sequence number (slot-reuse generation). Together with + // the FRS it forms the File Reference the compact index packs into + // `file_ref`; without it a delete-then-reuse of an MFT slot is invisible to + // the snapshot diff (the slot number alone is stable across reuse). + record.sequence_number = header.sequence_number; record.stdinfo = std_info; record.first_stream.size = SizeInfo { length: default_size,