Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
478 changes: 317 additions & 161 deletions pegaflow-server/src/fd_channel.rs

Large diffs are not rendered by default.

28 changes: 24 additions & 4 deletions pegaflow-server/src/http_server.rs
Original file line number Diff line number Diff line change
Expand Up @@ -82,8 +82,17 @@ async fn cleanup_handler(
) -> impl IntoResponse {
match query.id {
None => {
let removed_tensors = state.registry.clear().await;
let removed_instances = state.engine.unregister_all_instances();
let engine = Arc::clone(&state.engine);
let registry = state.registry.clone();
let (removed_instances, removed_tensors) = tokio::spawn(async move {
let cleanup = registry.clear().await;
let removed_instances = engine.unregister_all_instances();
let removed_tensors = cleanup.tensor_count();
registry.finish_cleanup(cleanup).await;
(removed_instances, removed_tensors)
})
.await
.expect("cleanup-all task failed");

if !removed_instances.is_empty() || removed_tensors > 0 {
warn!(
Expand All @@ -104,8 +113,19 @@ async fn cleanup_handler(
)
}
Some(instance_id) => {
let removed_tensors = state.registry.drop_instance(instance_id.clone()).await;
match state.engine.unregister_instance(&instance_id) {
let engine = Arc::clone(&state.engine);
let registry = state.registry.clone();
let cleanup_id = instance_id.clone();
let (removed_tensors, unregister) = tokio::spawn(async move {
let cleanup = registry.drop_instance(cleanup_id.clone()).await;
let removed_tensors = cleanup.tensor_count();
let unregister = engine.unregister_instance(&cleanup_id);
registry.finish_cleanup(cleanup).await;
(removed_tensors, unregister)
})
.await
.expect("instance cleanup task failed");
match unregister {
Ok(()) => {
warn!(
"Cleanup instance {}: {} CUDA tensor(s) released",
Expand Down
26 changes: 23 additions & 3 deletions pegaflow-server/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -94,7 +94,7 @@ pub struct Cli {
pub enable_prometheus: bool,

/// UDS path for native clients to send VMM allocation fds (SCM_RIGHTS). Empty disables.
#[arg(long, default_value = "/tmp/pegaflow-fd.sock")]
#[arg(long, default_value = "")]
pub fd_socket_path: String,

/// Init torch CUDA registry (vLLM path). False = torch-free, native VMM only.
Expand Down Expand Up @@ -306,7 +306,7 @@ fn init_cuda_driver() -> Result<(), std::io::Error> {
.map_err(|err| std::io::Error::other(format!("failed to initialize CUDA driver: {err}")))
}

fn detect_cuda_devices() -> Result<Vec<i32>, std::io::Error> {
fn detect_python_cuda_devices() -> Result<Vec<i32>, std::io::Error> {
Python::attach(|py| -> pyo3::PyResult<Vec<i32>> {
let torch = py.import("torch")?;
let cuda = torch.getattr("cuda")?;
Expand All @@ -331,6 +331,15 @@ fn detect_cuda_devices() -> Result<Vec<i32>, std::io::Error> {
})
}

fn detect_native_cuda_devices() -> Result<Vec<i32>, std::io::Error> {
let device_count = cudarc::driver::CudaContext::device_count().map_err(|err| {
std::io::Error::other(format!(
"failed to detect CUDA devices with the CUDA driver: {err}"
))
})?;
Ok((0..device_count).collect())
}

fn init_python_cuda(device_ids: &[i32]) -> Result<(), std::io::Error> {
if device_ids.is_empty() {
return Err(std::io::Error::other("no CUDA devices to initialize"));
Expand Down Expand Up @@ -473,7 +482,11 @@ pub fn run() -> Result<(), Box<dyn Error>> {
// Determine which devices to initialize
let devices = if cli.devices.is_empty() {
// Auto-detect all available devices
let detected = detect_cuda_devices()?;
let detected = if cli.python_registry {
detect_python_cuda_devices()?
} else {
detect_native_cuda_devices()?
};
info!(
"Auto-detected {} CUDA device(s): {:?}",
detected.len(),
Expand Down Expand Up @@ -779,6 +792,13 @@ mod tests {
);
}

#[test]
fn cli_defaults_native_fd_side_channel_off() {
let cli = Cli::try_parse_from(["pegaflow-server"]).unwrap();

assert!(cli.fd_socket_path.is_empty());
}

#[test]
fn cli_nics_accepts_comma_separated_values() {
let cli =
Expand Down
Loading