Expand workspace to cover all crates and enable rust testing (#1684)

Summary:
Pull Request resolved: #1684

Expand Github rust testing to the whole workspace of crates in monarch.
Tests that do not pass in Github are marked as fb-only for now. Many of them can
be fixed easily, but we can turn on the majority of tests right away.

Reviewed By: colin2328

Differential Revision: D85676520

Author: dulinriley

.config/nextest.toml

@@ -0,0 +1,6 @@
+[profile.ci]
+# Do not cancel the test run on the first failure.
+fail-fast = false
+
+[profile.ci.junit]
+path = "junit.xml"

.github/workflows/test-gpu-rust.yml

@@ -56,9 +56,21 @@ jobs:
 
         # Run GPU Rust tests
         echo "Running OSS Rust tests..."
-        # TODO: fix broken tests, then update to `cargo test --no-fail-fast`
-        cargo test -p monarch_rdma
         # Uses cargo nextest to run tests in separate processes, which better matches
         # internal buck test behavior.
-        # TODO: increase coverage to more crates.
-        cargo nextest run -p hyperactor --no-fail-fast
+        # The CI profile is configured in .config/nextest.toml
+        # Exclude filter is for packages that don't build in Github Actions yet.
+        # * monarch_messages: monarch/target/debug/deps/monarch_messages-...:
+        #   /lib64/libm.so.6: version `GLIBC_2.29' not found
+        #   (required by /meta-pytorch/monarch/libtorch/lib/libtorch_cpu.so)
+        cargo nextest run --workspace --profile ci \
+          --exclude monarch_messages \
+          --exclude monarch_tensor_worker \
+          --exclude monarch_simulator_lib \
+          --exclude torch-sys \
+          --exclude torch-sys-cuda
+        # Copy the test results to the expected location
+        # TODO: error in pytest-results-action, TypeError: results.testsuites.testsuite.testcase is not iterable
+        # Don't try to parse these results for now.
+        # mkdir -p "${RUNNER_TEST_RESULTS_DIR:-test-results}"
+        # cp target/nextest/ci/junit.xml "${RUNNER_TEST_RESULTS_DIR:-test-results}/junit.xml"

Cargo.toml

@@ -1,6 +1,7 @@
 [workspace]
 resolver = "2"
 members = [
+    "build_utils",
     "controller",
     "cuda-sys",
     "erased_lifetime",
@@ -10,11 +11,22 @@ members = [
     "hyperactor_multiprocess",
     "hyperactor_mesh",
     "hyperactor_mesh_macros",
-    "ndslice",
+    "hyperactor_telemetry",
+    "monarch_conda",
     "monarch_extension",
-    "monarch_tensor_worker",
+    "monarch_hyperactor",
+    "monarch_messages",
+    "monarch_perfetto_trace",
     "monarch_rdma",
+    "monarch_simulator",
+    "monarch_tensor_worker",
+    "monarch_types",
     "nccl-sys",
+    "ndslice",
+    "preempt_rwlock",
     "rdmaxcel-sys",
+    "serde_multipart",
+    "timed_test",
     "torch-sys",
+    "torch-sys-cuda",
 ]

controller/Cargo.toml

@@ -36,4 +36,8 @@ tracing = { version = "0.1.41", features = ["attributes", "valuable"] }
 
 [dev-dependencies]
 monarch_types = { version = "0.0.0", path = "../monarch_types" }
+timed_test = { version = "0.0.0", path = "../timed_test" }
 torch-sys = { version = "0.0.0", path = "../torch-sys" }
+
+[lints]
+rust = { unexpected_cfgs = { check-cfg = ["cfg(fbcode_build)"], level = "warn" } }

controller/src/lib.rs

@@ -660,6 +660,7 @@ mod tests {
     use monarch_messages::worker::CallFunctionParams;
     use monarch_messages::worker::WorkerMessage;
     use monarch_types::PyTree;
+    use timed_test::async_timed_test;
     use torch_sys::RValue;
 
     use super::*;
@@ -1838,7 +1839,9 @@ mod tests {
 
     hyperactor::remote!(PanickingActor);
 
-    #[tokio::test]
+    #[async_timed_test(timeout_secs = 30)]
+    // times out (both internal and external).
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_supervision_fault() {
         // Start system actor.
         let timeout: Duration = Duration::from_secs(6);

hyperactor/Cargo.toml

@@ -100,3 +100,6 @@ tracing-test = { version = "0.2.3", features = ["no-env-filter"] }
 [features]
 default = []
 stdio-write-probe = []
+
+[lints]
+rust = { unexpected_cfgs = { check-cfg = ["cfg(fbcode_build)"], level = "warn" } }

hyperactor/src/channel.rs

@@ -1143,7 +1143,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }))
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_dial_serve() {
         for addr in addrs() {
             let (listen_addr, mut rx) = crate::channel::serve::<i32>(addr).unwrap();
@@ -1155,7 +1155,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Server(Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" }))
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_send() {
         let config = crate::config::global::lock();
 

hyperactor/src/channel/net.rs

@@ -2572,7 +2572,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_basic() {
         let (addr, mut rx) = tcp::serve::<u64>("[::1]:0".parse().unwrap()).unwrap();
         {
@@ -2605,7 +2605,7 @@ mod tests {
     // The message size is limited by CODEC_MAX_FRAME_LENGTH.
     #[async_timed_test(timeout_secs = 5)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_message_size() {
         let default_size_in_bytes = 100 * 1024 * 1024;
         // Use temporary config for this test
@@ -2635,7 +2635,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_flush() {
         let config = config::global::lock();
         // Set a large value to effectively prevent acks from being sent except
@@ -2659,7 +2659,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[tokio::test]
     // TODO: OSS: failed to retrieve ipv6 address
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_meta_tls_basic() {
         let addr = ChannelAddr::any(ChannelTransport::MetaTls(TlsMode::IpV6));
         let meta_addr = match addr {
@@ -3273,7 +3273,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[tokio::test]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_tx_delivery_timeout() {
         // This link always fails to connect.
         let link = MockLink::<u64>::fail_connects();
@@ -3699,15 +3699,15 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_exceeded_limit_with_connected_link() {
         verify_ack_exceeded_limit(false).await;
     }
 
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 30)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_ack_exceeded_limit_with_broken_link() {
         verify_ack_exceeded_limit(true).await;
     }
@@ -3878,7 +3878,7 @@ mod tests {
 
     #[async_timed_test(timeout_secs = 300)]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: Listen(Tcp([::1]:0), Os { code: 99, kind: AddrNotAvailable, message: "Cannot assign requested address" })
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_tcp_throughput() {
         let config = config::global::lock();
         let _guard =
@@ -3930,7 +3930,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[async_timed_test(timeout_secs = 60)]
     // TODO: OSS: The logs_assert function returned an error: expected log not found
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_net_tx_closed_on_server_reject() {
         let link = MockLink::<u64>::new();
         let receiver_storage = link.receiver_storage();

hyperactor/src/config.rs

@@ -314,7 +314,7 @@ mod tests {
     #[tracing_test::traced_test]
     #[test]
     // TODO: OSS: The logs_assert function returned an error: missing log lines: {"# export HYPERACTOR_DEFAULT_ENCODING=serde_multipart", ...}
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     fn test_from_env() {
         // Set environment variables
         // SAFETY: TODO: Audit that the environment access only happens in single-threaded code.

hyperactor/src/host.rs

@@ -1280,7 +1280,7 @@ mod tests {
 
     #[tokio::test]
     // TODO: OSS: called `Result::unwrap()` on an `Err` value: ReadFailed { manifest_path: "/meta-pytorch/monarch/target/debug/deps/hyperactor-0e1fe83af739d976.resources.json", source: Os { code: 2, kind: NotFound, message: "No such file or directory" } }
-    #[cfg_attr(not(feature = "fb"), ignore)]
+    #[cfg_attr(not(fbcode_build), ignore)]
     async fn test_process_proc_manager() {
         hyperactor_telemetry::initialize_logging(crate::clock::ClockKind::default());