Implement fine grain locking for build-dir

src/cargo/core/compiler/build_runner/compilation_files.rs

@@ -10,6 +10,7 @@ use std::sync::Arc;
 use tracing::debug;
 
 use super::{BuildContext, BuildRunner, CompileKind, FileFlavor, Layout};
+use crate::core::compiler::layout::BuildUnitLockLocation;
 use crate::core::compiler::{CompileMode, CompileTarget, CrateType, FileType, Unit};
 use crate::core::{Target, TargetKind, Workspace};
 use crate::util::{self, CargoResult, OnceExt, StableHasher};
@@ -281,6 +282,13 @@ impl<'a, 'gctx: 'a> CompilationFiles<'a, 'gctx> {
         self.layout(unit.kind).build_dir().fingerprint(&dir)
     }
 
+    /// The path of the partial and full locks for a given build unit
+    /// when fine grain locking is enabled.
+    pub fn build_unit_lock(&self, unit: &Unit) -> BuildUnitLockLocation {
+        let dir = self.pkg_dir(unit);
+        self.layout(unit.kind).build_dir().build_unit_lock(&dir)
+    }
+
     /// Directory where incremental output for the given unit should go.
     pub fn incremental_dir(&self, unit: &Unit) -> &Path {
         self.layout(unit.kind).build_dir().incremental()

src/cargo/core/compiler/build_runner/mod.rs

@@ -6,15 +6,18 @@ use std::sync::{Arc, Mutex};
 
 use crate::core::PackageId;
 use crate::core::compiler::compilation::{self, UnitOutput};
+use crate::core::compiler::locking::LockingMode;
 use crate::core::compiler::{self, Unit, UserIntent, artifact};
 use crate::util::cache_lock::CacheLockMode;
 use crate::util::errors::CargoResult;
+use crate::util::rlimit;
 use annotate_snippets::{Level, Message};
 use anyhow::{Context as _, bail};
 use cargo_util::paths;
 use filetime::FileTime;
 use itertools::Itertools;
 use jobserver::Client;
+use tracing::{debug, warn};
 
 use super::custom_build::{self, BuildDeps, BuildScriptOutputs, BuildScripts};
 use super::fingerprint::{Checksum, Fingerprint};
@@ -88,6 +91,10 @@ pub struct BuildRunner<'a, 'gctx> {
     /// because the target has a type error. This is in an Arc<Mutex<..>>
     /// because it is continuously updated as the job progresses.
     pub failed_scrape_units: Arc<Mutex<HashSet<UnitHash>>>,
+
+    /// The locking mode to use for this build.
+    /// We use fine grain by default, but fallback to coarse grain for some systems.
+    pub locking_mode: LockingMode,
 }
 
 impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
@@ -110,6 +117,12 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             }
         };
 
+        let locking_mode = if bcx.gctx.cli_unstable().fine_grain_locking {
+            determine_locking_mode(&bcx)?
+        } else {
+            LockingMode::Coarse
+        };
+
         Ok(Self {
             bcx,
             compilation: Compilation::new(bcx)?,
@@ -127,6 +140,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             lto: HashMap::new(),
             metadata_for_doc_units: HashMap::new(),
             failed_scrape_units: Arc::new(Mutex::new(HashSet::new())),
+            locking_mode,
         })
     }
 
@@ -368,7 +382,13 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
             | UserIntent::Doctest
             | UserIntent::Bench => true,
         };
-        let host_layout = Layout::new(self.bcx.ws, None, &dest, must_take_artifact_dir_lock)?;
+        let host_layout = Layout::new(
+            self.bcx.ws,
+            None,
+            &dest,
+            must_take_artifact_dir_lock,
+            &self.locking_mode,
+        )?;
         let mut targets = HashMap::new();
         for kind in self.bcx.all_kinds.iter() {
             if let CompileKind::Target(target) = *kind {
@@ -377,6 +397,7 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
                     Some(target),
                     &dest,
                     must_take_artifact_dir_lock,
+                    &self.locking_mode,
                 )?;
                 targets.insert(target, layout);
             }
@@ -727,3 +748,49 @@ impl<'a, 'gctx> BuildRunner<'a, 'gctx> {
         }
     }
 }
+
+/// Determines if we have enough resources to safely use fine grain locking.
+/// This function will raises the number of max number file descriptors the current process
+/// can have open at once to make sure we are able to compile without running out of fds.
+///
+/// If we cannot acquire a safe max number of file descriptors, we fallback to coarse grain
+/// locking.
+pub fn determine_locking_mode(bcx: &BuildContext<'_, '_>) -> CargoResult<LockingMode> {
+    let total_units = bcx.unit_graph.keys().len() as u64;
+
+    // This is a bit arbitrary but if we do not have at least 10 times the remaining file
+    // descriptors than total build units there is a chance we could hit the limit.
+    // This is a fairly conservative estimate to make sure we don't hit max fd errors.
+    let safe_threshold = total_units * 10;
+
+    let Ok(mut limit) = rlimit::get_max_file_descriptors() else {
+        return Ok(LockingMode::Coarse);
+    };
+
+    if limit.soft_limit >= safe_threshold {
+        // The limit is higher or equal to what we deemed safe, so
+        // there is no need to raise the limit.
+        return Ok(LockingMode::Fine);
+    }
+
+    let display_fd_warning = || -> CargoResult<()> {
+        bcx.gctx.shell().verbose(|shell| shell.warn("ulimit was to low to safely enable fine grain locking, falling back to coarse grain locking"))
+    };
+
+    if limit.hard_limit < safe_threshold {
+        // The max we could raise the limit to is still not enough to safely compile.
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    limit.soft_limit = safe_threshold;
+
+    debug!("raising fd limit to {safe_threshold}");
+    if let Err(err) = rlimit::set_max_file_descriptors(limit) {
+        warn!("failed to raise max fds: {err}");
+        display_fd_warning()?;
+        return Ok(LockingMode::Coarse);
+    }
+
+    return Ok(LockingMode::Fine);
+}

src/cargo/core/compiler/layout.rs

@@ -103,6 +103,7 @@
 
 use crate::core::Workspace;
 use crate::core::compiler::CompileTarget;
+use crate::core::compiler::locking::LockingMode;
 use crate::util::flock::is_on_nfs_mount;
 use crate::util::{CargoResult, FileLock};
 use cargo_util::paths;
@@ -128,6 +129,7 @@ impl Layout {
         target: Option<CompileTarget>,
         dest: &str,
         must_take_artifact_dir_lock: bool,
+        build_dir_locking_mode: &LockingMode,
     ) -> CargoResult<Layout> {
         let is_new_layout = ws.gctx().cli_unstable().build_dir_new_layout;
         let mut root = ws.target_dir();
@@ -155,11 +157,18 @@ impl Layout {
         {
             None
         } else {
-            Some(build_dest.open_rw_exclusive_create(
-                ".cargo-lock",
-                ws.gctx(),
-                "build directory",
-            )?)
+            match build_dir_locking_mode {
+                LockingMode::Fine => Some(build_dest.open_ro_shared_create(
+                    ".cargo-lock",
+                    ws.gctx(),
+                    "build directory",
+                )?),
+                LockingMode::Coarse => Some(build_dest.open_rw_exclusive_create(
+                    ".cargo-lock",
+                    ws.gctx(),
+                    "build directory",
+                )?),
+            }
         };
         let build_root = build_root.into_path_unlocked();
         let build_dest = build_dest.as_path_unlocked();
@@ -361,6 +370,14 @@ impl BuildDirLayout {
             self.build().join(pkg_dir)
         }
     }
+    /// Fetch the lock paths for a build unit
+    pub fn build_unit_lock(&self, pkg_dir: &str) -> BuildUnitLockLocation {
+        let dir = self.build_unit(pkg_dir);
+        BuildUnitLockLocation {
+            partial: dir.join("partial.lock"),
+            full: dir.join("full.lock"),
+        }
+    }
     /// Fetch the artifact path.
     pub fn artifact(&self) -> &Path {
         &self.artifact
@@ -375,3 +392,10 @@ impl BuildDirLayout {
         Ok(&self.tmp)
     }
 }
+
+/// See [crate::core::compiler::locking] module docs for details about build system locking
+/// structure.
+pub struct BuildUnitLockLocation {
+    pub partial: PathBuf,
+    pub full: PathBuf,
+}