test failed in CI: integration_tests::multicast::groups::test_multicast_group_ip_version_conflict #9588
Description
This test failed on a CI run on #9508:
https://github.com/oxidecomputer/omicron/pull/9508/checks?check_run_id=59374263412
Log showing the specific test failure:
https://buildomat.eng.oxide.computer/wg/0/details/01KE2EPXB6TRKCR0HT80ZD85SF/n2ArXfzCcyOmyjY6ZjVvhDAJCobUnrc2mjGTwLNRrNWEXtmj/01KE2EQBH3VRT7F1N7EE19ZCV3#S8333
Excerpt from the log showing the failure:
thread 'integration_tests::multicast::groups::test_multicast_group_ip_version_conflict' (2) panicked at nexus/tests/integration_tests/multicast/mod.rs:982:13:
group v4-explicit-pool-group was not deleted within 120.150239515s
stack backtrace:
0: __rustc::rust_begin_unwind
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/panicking.rs:698:5
1: core::panicking::panic_fmt
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/panicking.rs:75:14
2: {async_fn#0}
at ./tests/integration_tests/multicast/mod.rs:982:13
3: {async_fn#0}
at ./tests/integration_tests/multicast/groups.rs:3068:62
4: {async_block#0}
at ./tests/integration_tests/multicast/groups.rs:2912:1
5: poll<&mut dyn core::future::future::Future<Output=()>>
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9
6: poll<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/future/future.rs:133:9
7: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:742:70
8: with_budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/task/coop/mod.rs:167:5
9: budget<core::task::poll::Poll<()>, tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure#0}::{closure#0}::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/task/coop/mod.rs:133:5
10: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:742:25
11: tokio::runtime::scheduler::current_thread::Context::enter
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:432:19
12: {closure#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:741:44
13: tokio::runtime::scheduler::current_thread::CoreGuard::enter::{{closure}}
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:829:68
14: tokio::runtime::context::scoped::Scoped<T>::set
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/context/scoped.rs:40:9
15: tokio::runtime::context::set_scheduler::{{closure}}
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/context.rs:176:38
16: try_with<tokio::runtime::context::Context, tokio::runtime::context::set_scheduler::{closure_env#0}<(alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>), tokio::runtime::scheduler::current_thread::{impl#8}::enter::{closure_env#0}<tokio::runtime::scheduler::current_thread::{impl#8}::block_on::{closure_env#0}<core::pin::Pin<&mut core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>>, core::option::Option<()>>>, (alloc::boxed::Box<tokio::runtime::scheduler::current_thread::Core, alloc::alloc::Global>, core::option::Option<()>)>
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:315:12
17: std::thread::local::LocalKey<T>::with
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/std/src/thread/local.rs:279:20
18: tokio::runtime::context::set_scheduler
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/context.rs:176:17
19: tokio::runtime::scheduler::current_thread::CoreGuard::enter
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:829:27
20: tokio::runtime::scheduler::current_thread::CoreGuard::block_on
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:729:24
21: {closure#0}<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:200:33
22: tokio::runtime::context::runtime::enter_runtime
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/context/runtime.rs:65:16
23: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/scheduler/current_thread/mod.rs:188:9
24: tokio::runtime::runtime::Runtime::block_on_inner
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/runtime.rs:368:52
25: block_on<core::pin::Pin<&mut dyn core::future::future::Future<Output=()>>>
at /home/build/.cargo/registry/src/index.crates.io-1949cf8c6b5b557f/tokio-1.48.0/src/runtime/runtime.rs:342:18
26: test_multicast_group_ip_version_conflict
at ./tests/integration_tests/multicast/groups.rs:2912:1
27: test_all::integration_tests::multicast::groups::test_multicast_group_ip_version_conflict::{{closure}}
at ./tests/integration_tests/multicast/groups.rs:2915:2
28: core::ops::function::FnOnce::call_once
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5
29: core::ops::function::FnOnce::call_once
at /rustc/ed61e7d7e242494fb7057f2657300d9e77bb4fcb/library/core/src/ops/function.rs:250:5
note: Some details are omitted, run with `RUST_BACKTRACE=full` for a verbose backtrace.
WARN: dropped CockroachInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: temporary directory leaked: "/var/tmp/omicron_tmp/.tmpNaTAfb"
If you would like to access the database for debugging, run the following:
# Run the database
cargo xtask db-dev run --no-populate --store-dir "/var/tmp/omicron_tmp/.tmpNaTAfb/data"
# Access the database. Note the port may change if you run multiple databases.
cockroach sql --host=localhost:32221 --insecure
WARN: dropped ClickHouse process without cleaning it up first (there may still be a child process running (PID 626) and a temporary directory leaked, /var/tmp/omicron_tmp/test_all-78c4e1c509ca422a-test_multicast_group_ip_version_conflict.614.1-clickhouse-VsWbyF)
failed to clean up ClickHouse data dir:
- /var/tmp/omicron_tmp/test_all-78c4e1c509ca422a-test_multicast_group_ip_version_conflict.614.1-clickhouse-VsWbyF: File exists (os error 17)
WARN: dropped DendriteInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: dendrite temporary directory leaked: /var/tmp/omicron_tmp/.tmppCl3yj
WARN: dropped MgdInstance without cleaning it up first (there may still be a child process running and a temporary directory leaked)
WARN: mgd temporary directory leaked: /var/tmp/omicron_tmp/.tmpGZaAU6