Feat: Out-of-tree cargo workspaces (was: Handle package.workspace key in child Cargo.toml)

MODULE.bazel

@@ -102,6 +102,7 @@ use_repo(
     "cui__cargo_toml-0.22.1",
     "cui__cfg-expr-0.18.0",
     "cui__clap-4.5.37",
+    "cui__clean-path-0.2.1",
     "cui__crates-index-3.7.0",
     "cui__glob-0.3.2",
     "cui__hex-0.4.3",

crate_universe/Cargo.toml

@@ -96,6 +96,7 @@ tracing-subscriber = "0.3.19"
 url = "2.5.4"
 walkdir = "2.5.0"
 glob = "0.3.2"
+clean-path = "0.2.1"
 
 [dev-dependencies]
 maplit = "1.0.2"

crate_universe/extensions.bzl

@@ -4,7 +4,7 @@ Crate Universe is a set of Bazel rule for generating Rust targets using Cargo.
 
 This doc describes using crate_universe with bzlmod.
 
-If you're using a WORKSPACE file, please see [the WORKSPACE equivalent of this doc](crate_universe.html).
+If you're using a WORKSPACE file, please see [the WORKSPACE equivalent of this doc](crate_universe_workspace.html).
 
 There are some examples of using crate_universe with bzlmod in the [example folder](https://github.com/bazelbuild/rules_rust/examples/bzlmod).
 
@@ -74,7 +74,7 @@ will not be able to pull from your private registry.
 
 The generated crates_repository contains helper macros which make collecting dependencies for Bazel targets simpler.
 Notably, the all_crate_deps and aliases macros (
-see [Dependencies API](https://bazelbuild.github.io/rules_rust/crate_universe.html#dependencies-api)) commonly allow the
+see [Dependencies API](https://bazelbuild.github.io/rules_rust/crate_universe_workspace.html#dependencies-api)) commonly allow the
 Cargo.toml files to be the single source of truth for dependencies.
 Since these macros come from the generated repository, the dependencies and alias definitions
 they return will automatically update BUILD targets. In your BUILD files,
@@ -137,7 +137,7 @@ CARGO_BAZEL_REPIN=1 bazel sync --only=crates
 
 This will result in all dependencies being updated for a project. The `CARGO_BAZEL_REPIN`
 environment variable can also be used to customize how dependencies are updated.
-For more details about repin, [please refer to the documentation](https://bazelbuild.github.io/rules_rust/crate_universe.html#crates_vendor).
+For more details about repin, [please refer to the documentation](https://bazelbuild.github.io/rules_rust/crate_universe_workspace.html#crates_vendor).
 
 ### Direct Dependencies
 

crate_universe/private/crates_repository.bzl

@@ -307,7 +307,9 @@ CARGO_BAZEL_REPIN=1 CARGO_BAZEL_REPIN_ONLY=crate_index bazel sync --only=crate_i
         "lockfile": attr.label(
             doc = (
                 "The path to a file to use for reproducible renderings. " +
-                "If set, this file must exist within the workspace (but can be empty) before this rule will work."
+                "If set, this file must exist within the workspace (but can be empty) before this rule will work." +
+                "If you already have a `MODULE.bazel.lock` file, you don't need this." +
+                "If you don't have a `MODULE.bazel.lock` file, the `lockfile` will save you generation time."
             ),
         ),
         "manifests": attr.label_list(

crate_universe/src/cli/splice.rs

@@ -100,6 +100,7 @@ pub fn splice(opt: SpliceOptions) -> Result<()> {
     let resolver_data = TreeResolver::new(cargo.clone())
         .generate(
             manifest_path.as_path_buf(),
+            &splicer.member_dirs(),
             &config.supported_platform_triples,
         )
         .context("Failed to generate features")?;

crate_universe/src/cli/vendor.rs

@@ -228,6 +228,7 @@ pub fn vendor(opt: VendorOptions) -> anyhow::Result<()> {
 
     let resolver_data = TreeResolver::new(cargo.clone()).generate(
         manifest_path.as_path_buf(),
+        &splicer.member_dirs(),
         &config.supported_platform_triples,
     )?;
 

crate_universe/src/context.rs

@@ -187,7 +187,7 @@ impl Context {
             // If no package prefix is present, attempt to apply the workspace prefix
             // since workspace members would not have shown up with their own label
             None => match workspace_prefix {
-                Some(prefix) => PathBuf::from(prefix).join(package_path_diff),
+                Some(prefix) => clean_path::clean(PathBuf::from(prefix).join(package_path_diff)),
                 None => package_path_diff,
             },
         };

crate_universe/src/metadata/cargo_tree_resolver.rs

@@ -6,7 +6,8 @@ use std::path::{Path, PathBuf};
 use std::process::Child;
 
 use anyhow::{anyhow, bail, Context, Result};
-use camino::Utf8Path;
+use camino::{Utf8Path, Utf8PathBuf};
+use clean_path::Clean;
 use semver::Version;
 use serde::{Deserialize, Serialize};
 use tracing::{debug, trace};
@@ -17,6 +18,7 @@ use crate::metadata::cargo_bin::Cargo;
 use crate::select::{Select, SelectableScalar};
 use crate::utils::symlink::symlink;
 use crate::utils::target_triple::TargetTriple;
+use crate::utils::PathCleanUtf8;
 
 /// A list platform triples that support host tools
 ///
@@ -308,6 +310,7 @@ impl TreeResolver {
     pub(crate) fn generate(
         &self,
         pristine_manifest_path: &Utf8Path,
+        member_dirs: &BTreeMap<Utf8PathBuf, BTreeSet<Utf8PathBuf>>,
         target_triples: &BTreeSet<TargetTriple>,
     ) -> Result<TreeResolverMetadata> {
         debug!(
@@ -320,6 +323,7 @@ impl TreeResolver {
         let manifest_path_with_transitive_proc_macros = self
             .copy_project_with_explicit_deps_on_all_transitive_proc_macros(
                 pristine_manifest_path,
+                member_dirs,
                 &tempdir.path().join("explicit_proc_macro_deps"),
             )
             .context("Failed to copy project with proc macro deps made direct")?;
@@ -451,8 +455,17 @@ impl TreeResolver {
     fn copy_project_with_explicit_deps_on_all_transitive_proc_macros(
         &self,
         pristine_manifest_path: &Utf8Path,
+        member_dirs: &BTreeMap<Utf8PathBuf, BTreeSet<Utf8PathBuf>>,
         output_dir: &Path,
     ) -> Result<PathBuf> {
+        if member_dirs.len() != 1 {
+            anyhow::bail!("Failed: The workspace has multiple roots");
+        }
+        let (root, members) = member_dirs.first_key_value().unwrap();
+        let output_dir = output_dir.join(root);
+        let output_dir = output_dir.as_path();
+
+        // check size of members_dir then pop refs to single elem
         if !output_dir.exists() {
             std::fs::create_dir_all(output_dir)?;
         }
@@ -471,6 +484,22 @@ impl TreeResolver {
                 })?;
             }
         }
+        for member_dir in members {
+            let source_path = pristine_root.join(member_dir).clean();
+            let destination = output_dir.join(member_dir).clean();
+            if destination.exists() {
+                continue;
+            }
+            if let Some(parent) = destination.parent() {
+                std::fs::create_dir_all(parent)?;
+            }
+            symlink(source_path.as_std_path(), &destination).with_context(|| {
+                format!(
+                    "Failed to create symlink {:?} pointing at {:?}",
+                    destination, source_path
+                )
+            })?;
+        }
         std::fs::copy(
             pristine_root.join("Cargo.lock"),
             output_dir.join("Cargo.lock"),

crate_universe/src/metadata/workspace_discoverer.rs

@@ -1,5 +1,6 @@
 use std::collections::{BTreeMap, BTreeSet};
 
+use crate::utils::PathCleanUtf8;
 use anyhow::{anyhow, bail, Context, Result};
 use camino::{Utf8Path, Utf8PathBuf};
 use cargo_toml::Manifest;
@@ -52,10 +53,6 @@ fn discover_workspaces_with_cache(
             discovered_workspaces
                 .workspaces_to_members
                 .insert(workspace_parent, BTreeSet::new());
-        } else {
-            discovered_workspaces
-                .non_workspaces
-                .insert(cargo_toml_path.clone());
         }
     }
 
@@ -85,6 +82,39 @@ fn discover_workspaces_with_cache(
             })
             .transpose()?;
 
+        workspace_manifest.workspace.as_ref().and_then(|workspace| {
+            let excludes: Vec<_> = workspace
+                .exclude
+                .iter()
+                .map(|exclude| {
+                    workspace_path
+                        .parent()
+                        .unwrap()
+                        .join(exclude)
+                        .clean()
+                        .join("Cargo.toml")
+                })
+                .collect();
+            for member in workspace.members.iter() {
+                let member_pattern = workspace_path.parent()?.to_string() + "/" + member;
+                for entry in glob::glob(&member_pattern).unwrap() {
+                    let maybe_member_cargo_toml = Utf8Path::from_path(&entry.unwrap())
+                        .unwrap()
+                        .clean()
+                        .join("Cargo.toml");
+                    if excludes.contains(&maybe_member_cargo_toml) {
+                        continue;
+                    }
+                    discovered_workspaces
+                        .workspaces_to_members
+                        .get_mut(&workspace_path)
+                        .unwrap()
+                        .insert(maybe_member_cargo_toml);
+                }
+            }
+            Some(())
+        });
+
         'per_child: for entry in walkdir::WalkDir::new(workspace_path.parent().unwrap())
             .follow_links(false)
             .follow_root_links(false)
@@ -135,8 +165,12 @@ fn discover_workspaces_with_cache(
                                 explicit_workspace_path.display()
                             )
                         })?;
-                    actual_workspace_path =
-                        child_path.parent().unwrap().join(explicit_workspace_path);
+                    actual_workspace_path = child_path
+                        .parent()
+                        .unwrap()
+                        .join(explicit_workspace_path)
+                        .clean()
+                        .join("Cargo.toml");
                 }
             }
             if !discovered_workspaces