test(e2e): add simple e2e test with docker to test /readyz

Signed-off-by: Adrien Langou <alangou@nvidia.com>

Author: alangou

crates/openshell-server/tests/health_endpoint_integration.rs

@@ -0,0 +1,104 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+use bytes::Bytes;
+use http_body_util::{BodyExt, Empty};
+use hyper::{Request, StatusCode};
+use hyper_util::rt::TokioIo;
+use openshell_server::{Store, health_router};
+use serde_json::Value;
+use std::sync::Arc;
+use tokio::net::TcpListener;
+
+async fn start_health_server(
+    store: Arc<Store>,
+) -> (std::net::SocketAddr, tokio::task::JoinHandle<()>) {
+    let listener = TcpListener::bind("127.0.0.1:0")
+        .await
+        .expect("bind ephemeral health test listener");
+    let addr = listener
+        .local_addr()
+        .expect("resolve local address for health test listener");
+
+    let server = tokio::spawn(async move {
+        let _ = axum::serve(listener, health_router(store).into_make_service()).await;
+    });
+
+    (addr, server)
+}
+
+async fn http_get_json(addr: std::net::SocketAddr, path: &str) -> (StatusCode, Value) {
+    let stream = tokio::net::TcpStream::connect(addr)
+        .await
+        .expect("connect test HTTP client");
+    let (mut sender, conn) = hyper::client::conn::http1::Builder::new()
+        .handshake(TokioIo::new(stream))
+        .await
+        .expect("handshake HTTP/1 test client");
+    tokio::spawn(async move {
+        let _ = conn.await;
+    });
+
+    let req = Request::builder()
+        .method("GET")
+        .uri(format!("http://{addr}{path}"))
+        .body(Empty::<Bytes>::new())
+        .expect("build HTTP request");
+    let resp = sender.send_request(req).await.expect("send HTTP request");
+    let status = resp.status();
+    let bytes = resp
+        .into_body()
+        .collect()
+        .await
+        .expect("collect response body")
+        .to_bytes();
+    let body = if bytes.is_empty() {
+        Value::Null
+    } else {
+        serde_json::from_slice(&bytes).expect("response body must be valid JSON")
+    };
+    (status, body)
+}
+
+#[tokio::test]
+async fn readyz_reports_healthy_when_database_is_reachable() {
+    let store = Arc::new(
+        Store::connect("sqlite::memory:")
+            .await
+            .expect("connect in-memory sqlite store for health integration test"),
+    );
+    let (addr, server) = start_health_server(store.clone()).await;
+
+    let (status, body) = http_get_json(addr, "/readyz").await;
+    assert_eq!(status, StatusCode::OK);
+    assert_eq!(body["status"], "healthy");
+    assert_eq!(body["checks"]["database"]["status"], "healthy");
+
+    server.abort();
+}
+
+#[cfg(feature = "test-support")]
+#[tokio::test]
+async fn readyz_reports_database_health_transition_after_close() {
+    let store = Arc::new(
+        Store::connect("sqlite::memory:")
+            .await
+            .expect("connect in-memory sqlite store for health integration test"),
+    );
+    let (addr, server) = start_health_server(store.clone()).await;
+
+    let (status, body) = http_get_json(addr, "/readyz").await;
+    assert_eq!(status, StatusCode::OK);
+    assert_eq!(body["status"], "healthy");
+    assert_eq!(body["checks"]["database"]["status"], "healthy");
+
+    store.close().await;
+
+    let (status, body) = http_get_json(addr, "/readyz").await;
+    assert_eq!(status, StatusCode::SERVICE_UNAVAILABLE);
+    assert_eq!(body["status"], "unhealthy");
+    assert_eq!(body["checks"]["database"]["status"], "unhealthy");
+    assert_eq!(body["checks"]["database"]["error"], "database unavailable");
+
+    server.abort();
+}

e2e/rust/Cargo.lock

@@ -45,6 +45,11 @@ name = "gateway_resume"
 path = "tests/gateway_resume.rs"
 required-features = ["e2e-docker"]
 
+[[test]]
+name = "readyz_health"
+path = "tests/readyz_health.rs"
+required-features = ["e2e-docker"]
+
 [[test]]
 name = "websocket_conformance"
 path = "tests/websocket_conformance.rs"
@@ -77,6 +82,10 @@ required-features = ["e2e-gpu"]
 
 [dependencies]
 base64 = "0.22"
+bytes = "1"
+http-body-util = "0.1"
+hyper = { version = "1", features = ["client", "http1"] }
+hyper-util = { version = "0.1", features = ["tokio"] }
 tokio = { version = "1.43", features = ["full"] }
 tempfile = "3"
 sha1 = "0.10"

e2e/rust/Cargo.toml

@@ -0,0 +1,93 @@
+// SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+#![cfg(feature = "e2e-docker")]
+
+use bytes::Bytes;
+use http_body_util::{BodyExt, Empty};
+use hyper::Request;
+use hyper_util::rt::TokioIo;
+use serde_json::Value;
+use std::time::{Duration, Instant};
+use tokio::net::TcpStream;
+
+fn health_port_from_env() -> Option<u16> {
+    std::env::var("OPENSHELL_E2E_HEALTH_PORT")
+        .ok()
+        .and_then(|value| value.parse::<u16>().ok())
+}
+
+async fn http_get_json(port: u16, path: &str) -> Result<(u16, Value), String> {
+    let stream = TcpStream::connect(("127.0.0.1", port))
+        .await
+        .map_err(|err| format!("connect health endpoint :{port}: {err}"))?;
+    let (mut sender, conn) = hyper::client::conn::http1::Builder::new()
+        .handshake(TokioIo::new(stream))
+        .await
+        .map_err(|err| format!("handshake health HTTP/1 client :{port}: {err}"))?;
+    tokio::spawn(async move {
+        let _ = conn.await;
+    });
+
+    let req = Request::builder()
+        .method("GET")
+        .uri(format!("http://127.0.0.1:{port}{path}"))
+        .body(Empty::<Bytes>::new())
+        .map_err(|err| format!("build health request {path}: {err}"))?;
+    let resp = sender
+        .send_request(req)
+        .await
+        .map_err(|err| format!("send health request {path} to :{port}: {err}"))?;
+    let status_code = resp.status().as_u16();
+    let bytes = resp
+        .into_body()
+        .collect()
+        .await
+        .map_err(|err| format!("read health response body {path}: {err}"))?
+        .to_bytes();
+    let json = serde_json::from_slice::<Value>(&bytes)
+        .map_err(|err| format!("health endpoint {path} did not return valid JSON: {err}"))?;
+
+    Ok((status_code, json))
+}
+
+#[tokio::test]
+async fn readyz_reports_healthy_database_check() {
+    let Some(port) = health_port_from_env() else {
+        eprintln!(
+            "Skipping /readyz e2e: OPENSHELL_E2E_HEALTH_PORT is not set (existing-endpoint mode)"
+        );
+        return;
+    };
+
+    let deadline = Instant::now() + Duration::from_secs(20);
+    let timeout_detail = loop {
+        let observation = match http_get_json(port, "/readyz").await {
+            Ok((status, payload)) => {
+                let ready = status == 200
+                    && payload["status"] == "healthy"
+                    && payload["checks"]["database"]["status"] == "healthy";
+                if ready {
+                    assert!(
+                        payload["checks"]["database"]["latency_ms"].is_number(),
+                        "readyz payload should include checks.database.latency_ms: {payload}"
+                    );
+                    assert!(
+                        payload["checks"]["database"]["error"].is_null(),
+                        "readyz payload should not include checks.database.error when healthy: {payload}"
+                    );
+                    return;
+                }
+                format!("unexpected /readyz response status={status} payload={payload}")
+            }
+            Err(err) => err,
+        };
+
+        if Instant::now() >= deadline {
+            break observation;
+        }
+
+        tokio::time::sleep(Duration::from_secs(1)).await;
+    };
+    panic!("timed out waiting for /readyz healthy response after 20s: {timeout_detail}");
+}