From 19b893e15cff5b1a24da8123745204be2a417c81 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 24 Jul 2026 05:48:13 -0400 Subject: [PATCH] feat: add Kubernetes deployment path Signed-off-by: Francisco Javier Arceo --- Cargo.lock | 1 + .../src/executor/modes/conversation.rs | 9 + .../src/storage/conversation.rs | 26 ++ .../src/storage/types/errors.rs | 4 + crates/agentic-server/Cargo.toml | 1 + .../agentic-server/benches/gateway_bench.rs | 1 + crates/agentic-server/benches/proxy_bench.rs | 1 + crates/agentic-server/src/app.rs | 2 + .../agentic-server/src/handler/http/models.rs | 54 +++- crates/agentic-server/src/server.rs | 1 + crates/agentic-server/tests/common/mod.rs | 7 +- crates/agentic-server/tests/health_test.rs | 65 ++++- .../tests/kubernetes_manifests_test.rs | 239 ++++++++++++++++++ .../tests/responses_websocket_test.rs | 1 + deploy/kubernetes/configmap.yaml | 15 ++ deploy/kubernetes/deployment.yaml | 112 ++++++++ deploy/kubernetes/kustomization.yaml | 11 + deploy/kubernetes/namespace.yaml | 7 + .../network-policy-ingress.example.yaml | 26 ++ deploy/kubernetes/network-policy.yaml | 16 ++ deploy/kubernetes/pod-disruption-budget.yaml | 14 + deploy/kubernetes/secret.example.yaml | 12 + deploy/kubernetes/service-account.yaml | 9 + deploy/kubernetes/service.yaml | 19 ++ docs/deploying/container.md | 3 + docs/deploying/kubernetes.md | 191 ++++++++++++++ mkdocs.yaml | 1 + 27 files changed, 832 insertions(+), 16 deletions(-) create mode 100644 crates/agentic-server/tests/kubernetes_manifests_test.rs create mode 100644 deploy/kubernetes/configmap.yaml create mode 100644 deploy/kubernetes/deployment.yaml create mode 100644 deploy/kubernetes/kustomization.yaml create mode 100644 deploy/kubernetes/namespace.yaml create mode 100644 deploy/kubernetes/network-policy-ingress.example.yaml create mode 100644 deploy/kubernetes/network-policy.yaml create mode 100644 deploy/kubernetes/pod-disruption-budget.yaml create mode 100644 deploy/kubernetes/secret.example.yaml create mode 100644 deploy/kubernetes/service-account.yaml create mode 100644 deploy/kubernetes/service.yaml create mode 100644 docs/deploying/kubernetes.md diff --git a/Cargo.lock b/Cargo.lock index 7a7d66e6..8eaa871f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -24,6 +24,7 @@ dependencies = [ "reqwest 0.12.28", "serde", "serde_json", + "serde_yml", "thiserror", "tokio", "tokio-tungstenite", diff --git a/crates/agentic-server-core/src/executor/modes/conversation.rs b/crates/agentic-server-core/src/executor/modes/conversation.rs index 3229bd29..ee0a413a 100644 --- a/crates/agentic-server-core/src/executor/modes/conversation.rs +++ b/crates/agentic-server-core/src/executor/modes/conversation.rs @@ -18,6 +18,15 @@ impl ConversationHandler { Self { store } } + /// Verifies that the underlying conversation store is available. + /// + /// # Errors + /// + /// Returns `ExecutorError` when the database health query fails or times out. + pub async fn check_health(&self) -> ExecutorResult<()> { + self.store.check_health().await.map_err(ExecutorError::Storage) + } + /// Gets an existing conversation or creates one. /// /// Reads `conversation_id` from `ctx.original_request`. diff --git a/crates/agentic-server-core/src/storage/conversation.rs b/crates/agentic-server-core/src/storage/conversation.rs index 192250e1..2a19839b 100644 --- a/crates/agentic-server-core/src/storage/conversation.rs +++ b/crates/agentic-server-core/src/storage/conversation.rs @@ -2,12 +2,15 @@ use std::convert::TryFrom; use std::sync::Arc; +use std::time::Duration; use super::models::{conversation, item, response}; use super::pool::DbPool; use super::types::{ConversationData, InOutItem, ResponseMetadata, StorageError, StoreResult}; use crate::utils::common::{serialize_to_string, uuid7_str}; +const DATABASE_HEALTH_TIMEOUT_SECONDS: u64 = 2; + /// Conversation storage operations. #[derive(Clone, Debug)] pub struct ConversationStore { @@ -36,6 +39,29 @@ impl ConversationStore { self.pool.as_deref().ok_or(StorageError::NotConfigured) } + /// Verifies that configured storage can execute a bounded query. + /// + /// Disabled storage has no required database dependency and is considered healthy. + /// + /// # Errors + /// + /// Returns an error when the query fails or exceeds the health-check deadline. + pub async fn check_health(&self) -> StoreResult<()> { + let Some(pool) = self.pool.as_deref() else { + return Ok(()); + }; + let query = sqlx::query("SELECT 1").execute(pool); + match tokio::time::timeout(Duration::from_secs(DATABASE_HEALTH_TIMEOUT_SECONDS), query).await { + Ok(result) => { + result?; + Ok(()) + } + Err(_) => Err(StorageError::HealthCheckTimeout { + timeout_seconds: DATABASE_HEALTH_TIMEOUT_SECONDS, + }), + } + } + /// Creates a new conversation. /// /// # Errors diff --git a/crates/agentic-server-core/src/storage/types/errors.rs b/crates/agentic-server-core/src/storage/types/errors.rs index 4bf5fb06..16e06cca 100644 --- a/crates/agentic-server-core/src/storage/types/errors.rs +++ b/crates/agentic-server-core/src/storage/types/errors.rs @@ -26,6 +26,10 @@ pub enum StorageError { #[error("storage not configured or disabled")] NotConfigured, + /// Database health check exceeded its deadline. + #[error("database health check timed out after {timeout_seconds} seconds")] + HealthCheckTimeout { timeout_seconds: u64 }, + /// Serialization or deserialization of data failed. /// /// Wraps `serde_json::Error` and automatically converts from it via `#[from]`. diff --git a/crates/agentic-server/Cargo.toml b/crates/agentic-server/Cargo.toml index 2d0f3564..5e74e188 100644 --- a/crates/agentic-server/Cargo.toml +++ b/crates/agentic-server/Cargo.toml @@ -30,6 +30,7 @@ criterion.workspace = true futures.workspace = true reqwest = { workspace = true, features = ["json"] } serde_json.workspace = true +serde_yml = "0.0.12" tokio = { workspace = true, features = ["test-util"] } tokio-tungstenite.workspace = true uuid = { version = "1", features = ["v7"] } diff --git a/crates/agentic-server/benches/gateway_bench.rs b/crates/agentic-server/benches/gateway_bench.rs index 488e51b0..0117f7e1 100644 --- a/crates/agentic-server/benches/gateway_bench.rs +++ b/crates/agentic-server/benches/gateway_bench.rs @@ -175,6 +175,7 @@ async fn spawn_gateway(llm_url: &str) -> (Arc, String) { shutdown_token: CancellationToken::new(), websocket_tracker: WebSocketTracker::default(), llm_api_base: config.llm_api_base, + skip_llm_ready_check: config.skip_llm_ready_check, openai_api_key: config.openai_api_key, }; diff --git a/crates/agentic-server/benches/proxy_bench.rs b/crates/agentic-server/benches/proxy_bench.rs index 4dce033b..5d4e9081 100644 --- a/crates/agentic-server/benches/proxy_bench.rs +++ b/crates/agentic-server/benches/proxy_bench.rs @@ -95,6 +95,7 @@ async fn spawn_gateway(config: Config) -> String { shutdown_token: CancellationToken::new(), websocket_tracker: WebSocketTracker::default(), llm_api_base: config.llm_api_base, + skip_llm_ready_check: config.skip_llm_ready_check, openai_api_key: config.openai_api_key, }; let server_config = ServerConfig::from_env(); diff --git a/crates/agentic-server/src/app.rs b/crates/agentic-server/src/app.rs index c2771c57..4e26b7fe 100644 --- a/crates/agentic-server/src/app.rs +++ b/crates/agentic-server/src/app.rs @@ -113,6 +113,8 @@ pub struct AppState { pub websocket_tracker: WebSocketTracker, /// vLLM base URL — used by the `/ready` health probe. pub llm_api_base: String, + /// Whether `/ready` should omit the upstream health check. + pub skip_llm_ready_check: bool, /// Server-configured API key; used as fallback when the request carries no /// `Authorization` header on the executor path. pub openai_api_key: Option, diff --git a/crates/agentic-server/src/handler/http/models.rs b/crates/agentic-server/src/handler/http/models.rs index cc9251bc..8ff9404f 100644 --- a/crates/agentic-server/src/handler/http/models.rs +++ b/crates/agentic-server/src/handler/http/models.rs @@ -105,28 +105,54 @@ pub async fn health() -> impl IntoResponse { StatusCode::OK } -pub async fn ready(State(state): State) -> impl IntoResponse { +async fn upstream_is_ready(state: &AppState) -> bool { let base = state.llm_api_base.trim_end_matches('/'); let url = format!("{base}/health"); - let client = reqwest::Client::builder() - .timeout(std::time::Duration::from_secs(2)) - .build(); - - let Ok(client) = client else { - return StatusCode::SERVICE_UNAVAILABLE; - }; + let mut request = state.exec_ctx.client.get(&url); + if let Some(key) = state + .openai_api_key + .as_deref() + .map(str::trim) + .filter(|key| !key.is_empty()) + { + request = request.bearer_auth(key); + } - match client.get(&url).send().await { - Ok(resp) if resp.status().is_success() => StatusCode::OK, - Ok(resp) => { + match tokio::time::timeout(std::time::Duration::from_secs(2), request.send()).await { + Ok(Ok(resp)) if resp.status().is_success() => true, + Ok(Ok(resp)) => { warn!("LLM backend not ready: {}", resp.status()); - StatusCode::SERVICE_UNAVAILABLE + false } - Err(e) => { + Ok(Err(e)) => { warn!("LLM backend unreachable: {e}"); - StatusCode::SERVICE_UNAVAILABLE + false } + Err(_) => { + warn!("LLM backend readiness check timed out"); + false + } + } +} + +async fn configured_upstream_is_ready(state: &AppState) -> bool { + state.skip_llm_ready_check || upstream_is_ready(state).await +} + +pub async fn ready(State(state): State) -> impl IntoResponse { + let (upstream_ready, database_result) = tokio::join!( + configured_upstream_is_ready(&state), + state.exec_ctx.conv_handler.check_health() + ); + if let Err(error) = &database_result { + warn!("database not ready: {error}"); + } + + if upstream_ready && database_result.is_ok() { + StatusCode::OK + } else { + StatusCode::SERVICE_UNAVAILABLE } } diff --git a/crates/agentic-server/src/server.rs b/crates/agentic-server/src/server.rs index c9cd19f5..52f9cb68 100644 --- a/crates/agentic-server/src/server.rs +++ b/crates/agentic-server/src/server.rs @@ -25,6 +25,7 @@ async fn build_state(config: &Config, shutdown_token: CancellationToken) -> Resu shutdown_token, websocket_tracker: WebSocketTracker::default(), llm_api_base: config.llm_api_base.clone(), + skip_llm_ready_check: config.skip_llm_ready_check, openai_api_key: config.openai_api_key.clone(), }) } diff --git a/crates/agentic-server/tests/common/mod.rs b/crates/agentic-server/tests/common/mod.rs index 983ae205..d8253041 100644 --- a/crates/agentic-server/tests/common/mod.rs +++ b/crates/agentic-server/tests/common/mod.rs @@ -26,8 +26,12 @@ pub fn test_config(llm_url: &str) -> Config { } pub fn test_state(config: &Config) -> AppState { + test_state_with_conversation_store(config, ConversationStore::disabled()) +} + +pub fn test_state_with_conversation_store(config: &Config, conversation_store: ConversationStore) -> AppState { let exec_ctx = ExecutionContext::new( - ConversationHandler::new(ConversationStore::disabled()), + ConversationHandler::new(conversation_store), ResponseHandler::new(ResponseStore::disabled()), Arc::new(reqwest::Client::new()), config.llm_api_base.clone(), @@ -40,6 +44,7 @@ pub fn test_state(config: &Config) -> AppState { shutdown_token: CancellationToken::new(), websocket_tracker: WebSocketTracker::default(), llm_api_base: config.llm_api_base.clone(), + skip_llm_ready_check: config.skip_llm_ready_check, openai_api_key: config.openai_api_key.clone(), } } diff --git a/crates/agentic-server/tests/health_test.rs b/crates/agentic-server/tests/health_test.rs index 72d6dcb6..342e42db 100644 --- a/crates/agentic-server/tests/health_test.rs +++ b/crates/agentic-server/tests/health_test.rs @@ -1,7 +1,14 @@ mod common; use agentic_core::config::Config; -use common::{spawn_gateway, spawn_mock_llm, test_config, test_state}; +use agentic_core::storage::{ConversationStore, create_pool}; +use axum::Router; +use axum::http::HeaderMap; +use axum::response::IntoResponse; +use axum::routing::get; +use common::{spawn_gateway, spawn_mock_llm, test_config, test_state, test_state_with_conversation_store}; +use http::StatusCode; +use tokio::net::TcpListener; fn test_config_no_key(llm_url: &str) -> Config { Config { @@ -45,3 +52,59 @@ async fn test_ready_returns_503_when_llm_unreachable() { let resp = reqwest::get(format!("{gw_url}/ready")).await.unwrap(); assert_eq!(resp.status(), 503); } + +#[tokio::test] +async fn test_ready_skips_upstream_when_configured() { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let dead_addr = listener.local_addr().unwrap(); + drop(listener); + let config = Config { + skip_llm_ready_check: true, + ..test_config_no_key(&format!("http://{dead_addr}")) + }; + let (gw_url, _gateway) = spawn_gateway(test_state(&config)).await; + + let resp = reqwest::get(format!("{gw_url}/ready")).await.unwrap(); + + assert_eq!(resp.status(), 200); +} + +#[tokio::test] +async fn test_ready_authenticates_upstream_health_check() { + let app = Router::new().route( + "/health", + get(|headers: HeaderMap| async move { + if headers.get("authorization").and_then(|value| value.to_str().ok()) == Some("Bearer test-key") { + StatusCode::OK.into_response() + } else { + StatusCode::UNAUTHORIZED.into_response() + } + }), + ); + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let addr = listener.local_addr().unwrap(); + let upstream = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() }); + + let (gw_url, gateway) = spawn_gateway(test_state(&test_config(&format!("http://{addr}")))).await; + let resp = reqwest::get(format!("{gw_url}/ready")).await.unwrap(); + + assert_eq!(resp.status(), 200); + upstream.abort(); + gateway.abort(); +} + +#[tokio::test] +async fn test_ready_returns_503_when_database_is_unreachable() { + let (llm_url, upstream) = spawn_mock_llm().await; + let pool = create_pool(Some("sqlite::memory:")).await.unwrap(); + let store = ConversationStore::new(pool.clone()); + pool.close().await; + let state = test_state_with_conversation_store(&test_config(&llm_url), store); + let (gw_url, gateway) = spawn_gateway(state).await; + + let resp = reqwest::get(format!("{gw_url}/ready")).await.unwrap(); + + assert_eq!(resp.status(), 503); + upstream.abort(); + gateway.abort(); +} diff --git a/crates/agentic-server/tests/kubernetes_manifests_test.rs b/crates/agentic-server/tests/kubernetes_manifests_test.rs new file mode 100644 index 00000000..dcf59627 --- /dev/null +++ b/crates/agentic-server/tests/kubernetes_manifests_test.rs @@ -0,0 +1,239 @@ +use std::fs; +use std::path::PathBuf; + +use serde_yml::Value; + +fn manifest_path(name: &str) -> PathBuf { + PathBuf::from(env!("CARGO_MANIFEST_DIR")) + .join("../../deploy/kubernetes") + .join(name) +} + +fn parse_manifest(name: &str) -> Value { + let path = manifest_path(name); + let contents = fs::read_to_string(&path).unwrap_or_else(|error| panic!("read {}: {error}", path.display())); + serde_yml::from_str(&contents).unwrap_or_else(|error| panic!("parse {}: {error}", path.display())) +} + +fn named<'a>(values: &'a Value, name: &str) -> &'a Value { + values + .as_sequence() + .unwrap_or_else(|| panic!("expected sequence containing {name}")) + .iter() + .find(|value| value["name"].as_str() == Some(name)) + .unwrap_or_else(|| panic!("missing entry named {name}")) +} + +fn string_list(value: &Value) -> Vec<&str> { + value + .as_sequence() + .expect("expected string sequence") + .iter() + .map(|entry| entry.as_str().expect("expected string entry")) + .collect() +} + +#[test] +fn kustomization_includes_only_non_secret_base_resources() { + let manifest = parse_manifest("kustomization.yaml"); + let resources = string_list(&manifest["resources"]); + + assert_eq!( + resources, + [ + "namespace.yaml", + "service-account.yaml", + "configmap.yaml", + "deployment.yaml", + "service.yaml", + "network-policy.yaml", + "pod-disruption-budget.yaml", + ] + ); + assert!(!resources.contains(&"secret.example.yaml")); + assert!(!resources.contains(&"ingress.example.yaml")); + assert!(!resources.contains(&"network-policy-ingress.example.yaml")); +} + +#[test] +fn deployment_is_replicated_hardened_and_probe_driven() { + let manifest = parse_manifest("deployment.yaml"); + assert_eq!(manifest["apiVersion"].as_str(), Some("apps/v1")); + assert_eq!(manifest["kind"].as_str(), Some("Deployment")); + assert_eq!(manifest["spec"]["replicas"].as_u64(), Some(2)); + assert_eq!( + manifest["spec"]["strategy"]["rollingUpdate"]["maxUnavailable"].as_u64(), + Some(0) + ); + + let pod_spec = &manifest["spec"]["template"]["spec"]; + assert_eq!(pod_spec["serviceAccountName"].as_str(), Some("agentic-api")); + assert_eq!(pod_spec["automountServiceAccountToken"].as_bool(), Some(false)); + assert!( + pod_spec["terminationGracePeriodSeconds"] + .as_u64() + .is_some_and(|seconds| seconds >= 10) + ); + assert_eq!( + pod_spec["securityContext"]["seccompProfile"]["type"].as_str(), + Some("RuntimeDefault") + ); + assert!(pod_spec["securityContext"]["runAsUser"].is_null()); + assert_eq!( + pod_spec["affinity"]["podAntiAffinity"]["preferredDuringSchedulingIgnoredDuringExecution"][0] + ["podAffinityTerm"]["topologyKey"] + .as_str(), + Some("kubernetes.io/hostname") + ); + + let container = named(&pod_spec["containers"], "agentic-api"); + assert_eq!(string_list(&container["args"]), ["--llm-ready-timeout-s", "300"]); + assert_eq!(container["ports"][0]["name"].as_str(), Some("http")); + assert_eq!(container["ports"][0]["containerPort"].as_u64(), Some(9000)); + let startup_budget_seconds = container["startupProbe"]["failureThreshold"] + .as_u64() + .zip(container["startupProbe"]["periodSeconds"].as_u64()) + .map(|(failures, period)| failures * period) + .expect("startup probe must define a numeric failure threshold and period"); + assert!(startup_budget_seconds > 600); + assert!( + manifest["spec"]["progressDeadlineSeconds"] + .as_u64() + .is_some_and(|deadline| deadline > startup_budget_seconds), + "deployment progress deadline must exceed the startup probe budget" + ); + assert_eq!( + container["envFrom"][0]["configMapRef"]["name"].as_str(), + Some("agentic-api") + ); + + let database_url = named(&container["env"], "DATABASE_URL"); + assert_eq!( + database_url["valueFrom"]["secretKeyRef"]["name"].as_str(), + Some("agentic-api") + ); + assert_eq!( + database_url["valueFrom"]["secretKeyRef"]["key"].as_str(), + Some("DATABASE_URL") + ); + assert_ne!( + database_url["valueFrom"]["secretKeyRef"]["optional"].as_bool(), + Some(true) + ); + + let openai_api_key = named(&container["env"], "OPENAI_API_KEY"); + assert_eq!( + openai_api_key["valueFrom"]["secretKeyRef"]["optional"].as_bool(), + Some(true) + ); + + for (probe, path) in [ + ("startupProbe", "/health"), + ("livenessProbe", "/health"), + ("readinessProbe", "/ready"), + ] { + assert_eq!(container[probe]["httpGet"]["path"].as_str(), Some(path)); + assert_eq!(container[probe]["httpGet"]["port"].as_str(), Some("http")); + assert!( + container[probe]["timeoutSeconds"] + .as_u64() + .is_some_and(|seconds| seconds > 0) + ); + } + + assert_eq!( + container["securityContext"]["allowPrivilegeEscalation"].as_bool(), + Some(false) + ); + assert_eq!( + container["securityContext"]["readOnlyRootFilesystem"].as_bool(), + Some(true) + ); + assert_eq!( + container["securityContext"]["capabilities"]["drop"][0].as_str(), + Some("ALL") + ); + assert_eq!( + string_list(&container["lifecycle"]["preStop"]["exec"]["command"]), + ["/bin/sh", "-c", "sleep 5"] + ); + + for class in ["requests", "limits"] { + assert!(container["resources"][class]["cpu"].is_string()); + assert!(container["resources"][class]["memory"].is_string()); + } +} + +#[test] +fn service_config_and_disruption_budget_match_the_workload() { + let service = parse_manifest("service.yaml"); + assert_eq!(service["spec"]["type"].as_str(), Some("ClusterIP")); + assert_eq!( + service["spec"]["selector"]["app.kubernetes.io/name"].as_str(), + Some("agentic-api") + ); + assert_eq!(service["spec"]["ports"][0]["targetPort"].as_str(), Some("http")); + + let config = parse_manifest("configmap.yaml"); + assert_eq!(config["data"]["GATEWAY_HOST"].as_str(), Some("0.0.0.0")); + assert_eq!(config["data"]["GATEWAY_PORT"].as_str(), Some("9000")); + assert_eq!(config["data"]["SKIP_LLM_READY_CHECK"].as_str(), Some("false")); + assert!( + config["data"]["CORS_ALLOWED_ORIGINS"] + .as_str() + .is_some_and(|origins| origins.starts_with("https://")) + ); + assert!( + config["data"]["LLM_API_BASE"] + .as_str() + .is_some_and(|url| url.starts_with("https://")) + ); + assert!(config["data"]["DATABASE_URL"].is_null()); + assert!(config["data"]["OPENAI_API_KEY"].is_null()); + + let budget = parse_manifest("pod-disruption-budget.yaml"); + assert_eq!(budget["apiVersion"].as_str(), Some("policy/v1")); + assert_eq!(budget["spec"]["minAvailable"].as_u64(), Some(1)); + assert_eq!( + budget["spec"]["selector"]["matchLabels"]["app.kubernetes.io/name"].as_str(), + Some("agentic-api") + ); +} + +#[test] +fn secret_example_documents_required_credentials() { + let secret = parse_manifest("secret.example.yaml"); + assert_eq!(secret["kind"].as_str(), Some("Secret")); + assert_eq!(secret["metadata"]["name"].as_str(), Some("agentic-api")); + assert!( + secret["stringData"]["DATABASE_URL"] + .as_str() + .is_some_and(|url| url.starts_with("postgresql://")) + ); + assert!(secret["stringData"]["OPENAI_API_KEY"].as_str().is_some()); +} + +#[test] +fn network_access_is_denied_by_default_and_ingress_is_opt_in() { + let default_deny = parse_manifest("network-policy.yaml"); + assert_eq!(default_deny["apiVersion"].as_str(), Some("networking.k8s.io/v1")); + assert_eq!(default_deny["kind"].as_str(), Some("NetworkPolicy")); + assert_eq!( + default_deny["spec"]["podSelector"]["matchLabels"]["app.kubernetes.io/name"].as_str(), + Some("agentic-api") + ); + assert_eq!(string_list(&default_deny["spec"]["policyTypes"]), ["Ingress"]); + assert!(default_deny["spec"]["ingress"].as_sequence().is_some_and(Vec::is_empty)); + + let ingress_access = parse_manifest("network-policy-ingress.example.yaml"); + let ingress_rule = &ingress_access["spec"]["ingress"][0]; + assert_eq!( + ingress_rule["from"][0]["namespaceSelector"]["matchLabels"]["kubernetes.io/metadata.name"].as_str(), + Some("ingress-nginx") + ); + assert_eq!( + ingress_rule["from"][0]["podSelector"]["matchLabels"]["app.kubernetes.io/name"].as_str(), + Some("ingress-nginx") + ); + assert_eq!(ingress_rule["ports"][0]["port"].as_u64(), Some(9000)); +} diff --git a/crates/agentic-server/tests/responses_websocket_test.rs b/crates/agentic-server/tests/responses_websocket_test.rs index df8bc1cc..3fff7cd7 100644 --- a/crates/agentic-server/tests/responses_websocket_test.rs +++ b/crates/agentic-server/tests/responses_websocket_test.rs @@ -274,6 +274,7 @@ async fn storage_backed_state_with_web_search(llm_url: &str, web_search_base_url shutdown_token: CancellationToken::new(), websocket_tracker: WebSocketTracker::default(), llm_api_base: config.llm_api_base, + skip_llm_ready_check: config.skip_llm_ready_check, openai_api_key: config.openai_api_key, }; StorageBackedState { state, _db: db } diff --git a/deploy/kubernetes/configmap.yaml b/deploy/kubernetes/configmap.yaml new file mode 100644 index 00000000..88836e97 --- /dev/null +++ b/deploy/kubernetes/configmap.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +data: + GATEWAY_HOST: "0.0.0.0" + GATEWAY_PORT: "9000" + LLM_API_BASE: "https://vllm.example.com" + SKIP_LLM_READY_CHECK: "false" + CORS_ALLOWED_ORIGINS: "https://agentic-api.example.com" + RUST_LOG: "agentic_server=info,agentic_core=info" diff --git a/deploy/kubernetes/deployment.yaml b/deploy/kubernetes/deployment.yaml new file mode 100644 index 00000000..66c290bc --- /dev/null +++ b/deploy/kubernetes/deployment.yaml @@ -0,0 +1,112 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +spec: + replicas: 2 + revisionHistoryLimit: 3 + progressDeadlineSeconds: 720 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + template: + metadata: + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api + spec: + serviceAccountName: agentic-api + automountServiceAccountToken: false + terminationGracePeriodSeconds: 30 + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + podAffinityTerm: + topologyKey: kubernetes.io/hostname + labelSelector: + matchLabels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + containers: + - name: agentic-api + image: agentic-api:dev + imagePullPolicy: IfNotPresent + args: + - --llm-ready-timeout-s + - "300" + ports: + - name: http + containerPort: 9000 + protocol: TCP + envFrom: + - configMapRef: + name: agentic-api + env: + - name: DATABASE_URL + valueFrom: + secretKeyRef: + name: agentic-api + key: DATABASE_URL + - name: OPENAI_API_KEY + valueFrom: + secretKeyRef: + name: agentic-api + key: OPENAI_API_KEY + optional: true + startupProbe: + httpGet: + path: /health + port: http + periodSeconds: 5 + timeoutSeconds: 2 + failureThreshold: 132 + livenessProbe: + httpGet: + path: /health + port: http + periodSeconds: 10 + timeoutSeconds: 2 + failureThreshold: 3 + readinessProbe: + httpGet: + path: /ready + port: http + periodSeconds: 5 + timeoutSeconds: 3 + failureThreshold: 3 + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - sleep 5 + resources: + requests: + cpu: 100m + memory: 128Mi + limits: + cpu: "1" + memory: 1Gi + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + drop: + - ALL diff --git a/deploy/kubernetes/kustomization.yaml b/deploy/kubernetes/kustomization.yaml new file mode 100644 index 00000000..3dba52ba --- /dev/null +++ b/deploy/kubernetes/kustomization.yaml @@ -0,0 +1,11 @@ +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agentic-api +resources: + - namespace.yaml + - service-account.yaml + - configmap.yaml + - deployment.yaml + - service.yaml + - network-policy.yaml + - pod-disruption-budget.yaml diff --git a/deploy/kubernetes/namespace.yaml b/deploy/kubernetes/namespace.yaml new file mode 100644 index 00000000..3b601e00 --- /dev/null +++ b/deploy/kubernetes/namespace.yaml @@ -0,0 +1,7 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/part-of: agentic-api diff --git a/deploy/kubernetes/network-policy-ingress.example.yaml b/deploy/kubernetes/network-policy-ingress.example.yaml new file mode 100644 index 00000000..d93fac5b --- /dev/null +++ b/deploy/kubernetes/network-policy-ingress.example.yaml @@ -0,0 +1,26 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: agentic-api-from-ingress + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + policyTypes: + - Ingress + ingress: + - from: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: ingress-nginx + podSelector: + matchLabels: + app.kubernetes.io/name: ingress-nginx + ports: + - protocol: TCP + port: 9000 diff --git a/deploy/kubernetes/network-policy.yaml b/deploy/kubernetes/network-policy.yaml new file mode 100644 index 00000000..5e4f5e30 --- /dev/null +++ b/deploy/kubernetes/network-policy.yaml @@ -0,0 +1,16 @@ +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: agentic-api-default-deny + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + policyTypes: + - Ingress + ingress: [] diff --git a/deploy/kubernetes/pod-disruption-budget.yaml b/deploy/kubernetes/pod-disruption-budget.yaml new file mode 100644 index 00000000..4201046b --- /dev/null +++ b/deploy/kubernetes/pod-disruption-budget.yaml @@ -0,0 +1,14 @@ +apiVersion: policy/v1 +kind: PodDisruptionBudget +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +spec: + minAvailable: 1 + selector: + matchLabels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway diff --git a/deploy/kubernetes/secret.example.yaml b/deploy/kubernetes/secret.example.yaml new file mode 100644 index 00000000..334d5e43 --- /dev/null +++ b/deploy/kubernetes/secret.example.yaml @@ -0,0 +1,12 @@ +apiVersion: v1 +kind: Secret +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +type: Opaque +stringData: + DATABASE_URL: "postgresql://agentic-api:replace-me@postgres.example.com:5432/agentic_api?sslmode=require" + OPENAI_API_KEY: "replace-me" diff --git a/deploy/kubernetes/service-account.yaml b/deploy/kubernetes/service-account.yaml new file mode 100644 index 00000000..66d9a064 --- /dev/null +++ b/deploy/kubernetes/service-account.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +automountServiceAccountToken: false diff --git a/deploy/kubernetes/service.yaml b/deploy/kubernetes/service.yaml new file mode 100644 index 00000000..cc509eec --- /dev/null +++ b/deploy/kubernetes/service.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: Service +metadata: + name: agentic-api + labels: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + app.kubernetes.io/part-of: agentic-api +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: agentic-api + app.kubernetes.io/component: gateway + ports: + - name: http + port: 9000 + targetPort: http + protocol: TCP + appProtocol: http diff --git a/docs/deploying/container.md b/docs/deploying/container.md index ecb736df..067144ec 100644 --- a/docs/deploying/container.md +++ b/docs/deploying/container.md @@ -88,3 +88,6 @@ spec: ``` Mount writable storage at `/var/lib/agentic-api` only when using SQLite. PostgreSQL deployments do not need a persistent filesystem for the gateway. + +For a replicated PostgreSQL deployment, use the +[Kubernetes manifests and operational guide](kubernetes.md). diff --git a/docs/deploying/kubernetes.md b/docs/deploying/kubernetes.md new file mode 100644 index 00000000..a6452b01 --- /dev/null +++ b/docs/deploying/kubernetes.md @@ -0,0 +1,191 @@ +# Deploy agentic-api on Kubernetes + +The repository includes a Kustomize-compatible base in `deploy/kubernetes`. It runs two stateless gateway replicas +against managed PostgreSQL and an external OpenAI-compatible inference service. The Service is `ClusterIP` because the +gateway does not authenticate inbound callers yet. Put an authenticated application boundary in front of it before +allowing traffic from outside the cluster. + +## Prepare the image and configuration + +Build and publish the production image described in the [container guide](container.md). The base uses +`agentic-api:dev` so it also works with images loaded directly into a local cluster. Before deploying to a shared +cluster, replace that image with an immutable registry digest in an environment-specific Kustomize overlay such as +`deploy/kubernetes/overlays/production/kustomization.yaml`: + +```yaml +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +namespace: agentic-api +resources: + - ../.. +images: + - name: agentic-api + newName: registry.example.com/vllm/agentic-api + digest: sha256:replace-with-the-published-digest +``` + +Change `LLM_API_BASE` and `CORS_ALLOWED_ORIGINS` in `deploy/kubernetes/configmap.yaml` or patch them in the same +overlay. Keep `SKIP_LLM_READY_CHECK=false` when the inference service exposes `/health`. The recurring upstream check +uses `OPENAI_API_KEY` as a bearer credential when configured. For a provider without `/health`, set +`SKIP_LLM_READY_CHECK=true`; `/ready` will continue checking PostgreSQL but will omit the upstream check. Add a small +Responses API request as an external monitor in that configuration. + +Create the namespace and Secret before applying the workload: + +```console +kubectl apply -f deploy/kubernetes/namespace.yaml +kubectl --namespace agentic-api create secret generic agentic-api \ + --from-literal=DATABASE_URL='postgresql://agentic-api:replace-me@postgres.example.com:5432/agentic_api?sslmode=require' \ + --from-literal=OPENAI_API_KEY='replace-me' +``` + +Omit `OPENAI_API_KEY` when the inference service does not require a fallback credential. The request's +`Authorization` or Anthropic-compatible `x-api-key` header still takes precedence where the protocol requires it. +`secret.example.yaml` documents the expected keys but is deliberately excluded from the Kustomize base. Do not add +real credentials to that file or commit a generated Secret. Kubernetes Secrets are only base64-encoded by default; +enable encryption at rest and restrict Secret access in the cluster. + +## Deploy and inspect the gateway + +Apply the base or the environment overlay: + +```console +kubectl apply -k deploy/kubernetes +kubectl --namespace agentic-api rollout status deployment/agentic-api +kubectl --namespace agentic-api get pods,service,networkpolicy,poddisruptionbudget +``` + +ConfigMap and Secret values are injected as environment variables and do not update inside existing pods. After +changing either object without otherwise changing the pod template, run +`kubectl --namespace agentic-api rollout restart deployment/agentic-api` and wait for the rollout to finish. + +The base defines: + +- two gateway replicas with a rolling update that keeps the current replicas available; +- a `ClusterIP` Service on port `9000`; +- startup and liveness probes on `/health`, plus a readiness probe on `/ready`; +- a 30-second pod termination grace period for endpoint propagation and the gateway's bounded eight-second drain; +- CPU and memory requests and limits that should be tuned from observed traffic; +- a preferred hostname anti-affinity rule for the two gateway replicas; +- a non-root, read-only runtime with no Linux capabilities or mounted service-account token; +- a default-deny ingress NetworkPolicy; and +- a PodDisruptionBudget that retains one replica during voluntary disruption. + +The startup probe allows up to eleven minutes. The Deployment bounds the upstream startup wait at five minutes, +leaving about six minutes for the database connection and embedded migrations before Kubernetes restarts the pod. The +process does not bind its port until both steps succeed. `/health` then reports process liveness without depending on +remote services. `/ready` runs bounded PostgreSQL and inference checks concurrently and removes a pod from Service +endpoints when either required dependency fails. + +## Choose a migration policy + +By default, every new pod applies the embedded SQLx migrations before starting the HTTP server. PostgreSQL migration +locking serializes concurrent migration attempts, and a failed migration keeps the pod out of service. Keep migrations +backward-compatible with the previous gateway version so a rolling update can run old and new replicas together. + +For a supervisor-managed schema: + +1. apply the repository migrations in a release job before updating the Deployment; +2. verify the target schema and any required compatibility upgrades; +3. set `AGENTIC_API_SCHEMA_READY=1` in the gateway ConfigMap; and +4. roll out the gateway only after the release job succeeds. + +The runtime image intentionally contains no shell database client, and the gateway has no migration-only command, so +the base does not pretend to provide an in-image migration Job. Use a trusted database migration image or deployment +controller. Never set `AGENTIC_API_SCHEMA_READY` merely to bypass a failed migration. + +## Put an authenticated edge in front + +Do not use `OPENAI_API_KEY` as a caller password. It is an upstream inference credential. Exposing the gateway directly +would let anonymous callers spend that credential and read or write unscoped stored state. + +The repository deliberately does not include a ready-to-apply Ingress while inbound caller authentication remains +deployment-specific. Create the Ingress in an environment overlay only after an identity-aware proxy or authenticated +application service protects every `/v1/*` HTTP and WebSocket route. For ingress-nginx, configure external +authentication and include settings equivalent to: + +```yaml +metadata: + annotations: + nginx.ingress.kubernetes.io/auth-url: "https://auth.example.com/verify" + nginx.ingress.kubernetes.io/proxy-buffering: "off" + nginx.ingress.kubernetes.io/proxy-http-version: "1.1" + nginx.ingress.kubernetes.io/proxy-read-timeout: "3600" + nginx.ingress.kubernetes.io/proxy-send-timeout: "3600" +``` + +Use TLS and appropriate connection and request limits. The authentication boundary must consume and strip caller +`Authorization` and `x-api-key` headers before forwarding to the gateway; otherwise the gateway treats them as +upstream inference credentials instead of using its configured fallback. Use mTLS, a trusted non-forwarded identity +mechanism, or another platform control between the edge and gateway until inbound and upstream credentials are +separated in the gateway. + +The base NetworkPolicy denies all pod-network ingress to the gateway. When the authenticated boundary is an +ingress-nginx controller, copy `network-policy-ingress.example.yaml` into the overlay with the authenticated Ingress. +Its selectors +admit only pods labeled `app.kubernetes.io/name=ingress-nginx` in the `ingress-nginx` namespace. Patch both selectors +when the controller uses different labels or a different namespace. Keep the default-deny policy in place, and verify +that the cluster's network plugin enforces NetworkPolicy before relying on it as a security boundary. + +The portable base does not restrict egress because the DNS resolver, managed PostgreSQL addresses, ports, and external +inference destinations are environment-specific. Add a default-deny egress policy and explicit DNS, database, and +inference allowances in the production overlay when the cluster's network plugin can express those destinations. + +SSE streams need buffering disabled from the client through every proxy. WebSocket clients should use `wss`, send +keepalive pings, and reconnect with backoff after a rollout or node disruption. Stored continuation state is in +PostgreSQL, so reconnects do not require session affinity; clients can continue with a conversation or previous +response ID through any ready replica. + +Kubernetes begins endpoint removal at the same time as graceful pod termination. The five-second `preStop` delay gives +controllers time to observe the terminating endpoint before `SIGTERM` reaches the gateway. The gateway then stops +accepting new requests, drains active HTTP requests and WebSockets for up to eight seconds, and exits. The 30-second +grace period includes both phases. Verify the external load balancer's endpoint propagation and retry behavior. + +Preferred anti-affinity encourages the scheduler to place replicas on different nodes without making a single-node +development cluster unschedulable. Use hard topology-spread constraints across nodes or zones when production +availability requires them. The PodDisruptionBudget only limits voluntary disruptions; it cannot prevent simultaneous +loss of co-located replicas during a node failure. + +## Verify persistence and transport + +Reach the private Service through a temporary port-forward in a separate terminal: + +```console +kubectl --namespace agentic-api port-forward service/agentic-api 9000:9000 +curl --fail http://127.0.0.1:9000/health +curl --fail http://127.0.0.1:9000/ready +``` + +Send a stored Responses request, save its response ID, restart the Deployment, and continue with +`previous_response_id`. A successful continuation after the rollout verifies that PostgreSQL, rather than a pod +filesystem, owns the state: + +```console +first_response_id=$( + curl --fail --silent --show-error http://127.0.0.1:9000/v1/responses \ + --header "Content-Type: application/json" \ + --data '{"model":"Qwen/Qwen3-30B-A3B-FP8","input":"Reply with READY","store":true}' | + jq --exit-status --raw-output .id +) + +kubectl --namespace agentic-api rollout restart deployment/agentic-api +kubectl --namespace agentic-api rollout status deployment/agentic-api +``` + +The port-forward exits when the selected pod terminates. Start the same `kubectl port-forward` command again after the +rollout, then continue the request: + +```console +curl --fail --silent --show-error http://127.0.0.1:9000/v1/responses \ + --header "Content-Type: application/json" \ + --data "$(jq --null-input --arg id "$first_response_id" '{ + model: "Qwen/Qwen3-30B-A3B-FP8", + input: "What word did you return?", + previous_response_id: $id, + store: true + }')" +``` + +Repeat the check through the authenticated ingress for HTTP streaming and WebSockets before a production release. +Size PostgreSQL pools, gateway replicas, and inference capacity together; adding gateway replicas cannot compensate +for a saturated database or inference service. diff --git a/mkdocs.yaml b/mkdocs.yaml index f50ee255..0f9042b2 100644 --- a/mkdocs.yaml +++ b/mkdocs.yaml @@ -96,6 +96,7 @@ nav: - Getting Started: developing/getting-started.md - Deploying: - Container: deploying/container.md + - Kubernetes: deploying/kubernetes.md - Community: community/index.md extra: