Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions crates/watcher/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -13,3 +13,4 @@ log.workspace = true
anyhow.workspace = true
notify-debouncer-full.workspace = true
futures.workspace = true
walkdir.workspace = true
61 changes: 60 additions & 1 deletion crates/watcher/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,17 @@ use notify_debouncer_full::{
};
use warpui_core::{Entity, ModelContext};

/// Maximum number of directories allowed in a single recursive watch registration.
/// On Linux, the `notify` crate's inotify backend creates one watch descriptor per
/// directory. Each watch consumes memory for internal bookkeeping (path storage,
/// event buffers, HashMap entries). When a directory tree contains hundreds of
/// thousands of subdirectories (e.g., deep `node_modules` trees or monorepos),
/// the cumulative memory can reach 10+ GB.
///
/// When a directory tree exceeds this threshold, the watch falls back to
/// `RecursiveMode::NonRecursive` to prevent excessive memory usage.
const MAX_RECURSIVE_WATCH_DIRECTORIES: usize = 100_000;

#[derive(Debug)]
enum BackgroundFileWatcherCommand {
AddPath {
Expand Down Expand Up @@ -68,9 +79,34 @@ impl BackgroundFileWatcher {
response,
recursive_mode,
} => {
// On Linux, recursive watches create one inotify descriptor
// per directory, which can consume 10+ GB of memory for very
// large trees. Guard against this by checking the directory
// count before registering.
let effective_mode = if recursive_mode == RecursiveMode::Recursive {
match estimate_directory_count(
&path,
MAX_RECURSIVE_WATCH_DIRECTORIES,
) {
count if count > MAX_RECURSIVE_WATCH_DIRECTORIES => {
log::warn!(
"Directory tree at {} contains >{} directories \
(sampled {count}). Falling back to non-recursive \
watch to prevent excessive memory usage.",
path.display(),
MAX_RECURSIVE_WATCH_DIRECTORIES,
);
RecursiveMode::NonRecursive
}
_ => RecursiveMode::Recursive,
}
} else {
recursive_mode
};

let _ = response.send(
self.notifier
.watch_filtered(path, recursive_mode, filter)
.watch_filtered(path, effective_mode, filter)
.inspect_err(|err| {
log::warn!("Failed to watch path: {err:?}");
})
Expand Down Expand Up @@ -383,3 +419,26 @@ fn deduplicate_and_merge_raw_notifier_events(

Ok(update)
}

/// Estimates the number of directories under `root` by walking the tree up to
/// `limit + 1` entries. Returns as soon as the count exceeds `limit`, avoiding
/// a full traversal of very large trees.
///
/// Symlinks are *not* followed to avoid infinite loops. Permission errors and
/// other I/O failures on individual entries are silently skipped.
fn estimate_directory_count(root: &Path, limit: usize) -> usize {
let mut count: usize = 0;
for entry in walkdir::WalkDir::new(root)
.follow_links(false)
.into_iter()
.filter_map(|e| e.ok())
{
if entry.file_type().is_dir() {
count += 1;
if count > limit {
return count;
}
}
}
count
}