Expand workspace to cover all crates and enable rust testing (#1684)

Summary:

Expand Github rust testing to the whole workspace of crates in monarch.
Tests that do not pass in Github are marked as fb-only for now. Many of them can
be fixed easily, but we can turn on the majority of tests right away.

Reviewed By: colin2328

Differential Revision: D85676520

Author: dulinriley

.config/nextest.toml

@@ -0,0 +1,7 @@
+[profile.ci]
+# Do not cancel the test run on the first failure.
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"
+

.github/workflows/test-gpu-rust.yml

@@ -56,9 +56,16 @@ jobs:
 
         # Run GPU Rust tests
         echo "Running OSS Rust tests..."
-        # TODO: fix broken tests, then update to `cargo test --no-fail-fast`
-        cargo test -p monarch_rdma
         # Uses cargo nextest to run tests in separate processes, which better matches
         # internal buck test behavior.
-        # TODO: increase coverage to more crates.
-        cargo nextest run -p hyperactor --no-fail-fast
+        # The CI profile is configured in .config/nextest.toml
+        # Exclude filter is for packages that don't build in Github Actions yet.
+        # * monarch_messages: monarch/target/debug/deps/monarch_messages-...:
+        #   /lib64/libm.so.6: version `GLIBC_2.29' not found
+        #   (required by /meta-pytorch/monarch/libtorch/lib/libtorch_cpu.so)
+        cargo nextest run --workspace --profile ci \
+          --exclude monarch_messages \
+          --exclude monarch_tensor_worker \
+          --exclude monarch_simulator_lib
+        # Copy the test results to the expected location
+        cp target/nextest/ci/junit.xml "${RUNNER_TEST_RESUILTS_DIR:-test-results}/junit.xml"

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "build_utils",
     "controller",
     "cuda-sys",
     "erased_lifetime",
@@ -10,11 +11,22 @@ members = [
     "hyperactor_multiprocess",
     "hyperactor_mesh",
     "hyperactor_mesh_macros",
-    "ndslice",
+    "hyperactor_telemetry",
+    "monarch_conda",
     "monarch_extension",
-    "monarch_tensor_worker",
+    "monarch_hyperactor",
+    "monarch_messages",
+    "monarch_perfetto_trace",
     "monarch_rdma",
+    "monarch_simulator",
+    "monarch_tensor_worker",
+    "monarch_types",
     "nccl-sys",
+    "ndslice",
+    "preempt_rwlock",
     "rdmaxcel-sys",
+    "serde_multipart",
+    "timed_test",
     "torch-sys",
+    "torch-sys-cuda",
 ]

scripts/common-setup.sh

@@ -24,7 +24,8 @@ setup_conda_environment() {
 install_system_dependencies() {
     echo "Installing system dependencies..."
     dnf update -y
-    dnf install clang-devel libunwind libunwind-devel -y
+    # Protobuf compiler is required for the tracing-perfetto-sdk-schema crate.
+    dnf install clang-devel libunwind libunwind-devel protobuf-compiler -y
 }
 
 # Install and configure Rust nightly toolchain