diff --git a/Cargo.toml b/Cargo.toml index 2f4b3bb..a542dfb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,8 +9,8 @@ repository = "https://github.com/Mesh-LLM/openai-endpoint" [dependencies] anyhow = "1" mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", branch = "codex/plugin-cli-commands" } +serde_json = "1" tokio = { version = "1", features = ["full"] } [dev-dependencies] reqwest = { version = "0.12", features = ["json"] } -serde_json = "1" diff --git a/README.md b/README.md index f5a2df4..6d1e72c 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ mesh-llm plugins install Mesh-LLM/openai-endpoint ## Configure -Point the plugin at the external server: +Point the plugin at one external server: ```toml [[plugin]] @@ -39,6 +39,55 @@ url = "http://localhost:8000/v1" mesh-llm passes `url` to the plugin as `MESH_LLM_PLUGIN_URL`. If neither config nor environment is set, it defaults to `http://localhost:8000/v1`. +### Multiple endpoints + +The same plugin instance can advertise one or more OpenAI-compatible servers. +The existing single-URL form remains supported. For multiple servers, use +`urls`: + +```toml +[[plugin]] +name = "openai-endpoint" +urls = [ + "http://localhost:8000/v1", + "http://gpu-box:8000/v1", +] +``` + +If `urls` is set, the plugin receives the values through +`MESH_LLM_PLUGIN_URLS`. A single endpoint keeps the endpoint ID +`openai-endpoint`; multiple endpoints are assigned `openai-endpoint-1`, +`openai-endpoint-2`, and so on. Newline-separated URLs are also accepted when +setting the environment variable directly. Each endpoint is health-checked +independently, and the mesh routes a model to the healthy endpoint that +advertises it. + +### Authentication suggestions + +Authentication is not currently applied by this plugin. The plugin advertises +the endpoint address, while the mesh host performs the model health probe and +inference request, so adding a token only inside this process would not secure +those host-side requests. + +Recommended options, in order of practicality: + +1. Add host-managed per-endpoint credentials. Extend the endpoint configuration + with an auth reference such as `bearer_env = "GPU_BOX_API_KEY"`, keep the + secret out of the manifest, and have the host attach the header to both + `/v1/models` health probes and inference requests. This supports different + credentials per endpoint and secret rotation without exposing tokens in + URLs, logs, or endpoint metadata. +2. Use a local authenticated gateway or sidecar today. Point this plugin at + Envoy, an OAuth2 proxy, or another local gateway that injects credentials, + and let the gateway forward to the protected upstream. This requires no + mesh protocol change and keeps credentials outside the plugin URL. +3. For OAuth2/OIDC, add a host-side token provider with caching, expiry-aware + refresh, and separate scopes per endpoint. The same provider should be used + by health checks and inference forwarding. + +Avoid putting bearer tokens in query strings or URL user-info, since endpoint +addresses can appear in manifests, diagnostics, and logs. + ## Build ```bash diff --git a/src/lib.rs b/src/lib.rs index 53bdfc9..61160dd 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,57 +1,164 @@ -use anyhow::Result; +use anyhow::{Context, Result, bail}; use mesh_llm_plugin::{ - PluginMetadata, PluginRuntime, PluginStartupPolicy, capability, plugin_server_info, + DeclarativePluginBuilder, PluginMetadata, PluginRuntime, PluginStartupPolicy, capability, + plugin_server_info, }; +use serde_json::Value; +use std::collections::HashSet; pub const VERSION: &str = env!("CARGO_PKG_VERSION"); const DEFAULT_BASE_URL: &str = "http://localhost:8000/v1"; const PLUGIN_ID: &str = "openai-endpoint"; -fn base_url() -> String { - std::env::var("MESH_LLM_PLUGIN_URL") +#[derive(Clone, Debug, PartialEq, Eq)] +struct EndpointSpec { + id: String, + url: String, +} + +fn configured_endpoint_specs() -> Result> { + let value = std::env::var("MESH_LLM_PLUGIN_URLS") .ok() - .map(|value| value.trim().to_string()) - .filter(|value| !value.is_empty()) - .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()) + .filter(|value| !value.trim().is_empty()) + .or_else(|| { + std::env::var("MESH_LLM_PLUGIN_URL") + .ok() + .filter(|value| !value.trim().is_empty()) + }) + .unwrap_or_else(|| DEFAULT_BASE_URL.to_string()); + endpoint_specs_from_value(&value) +} + +/// Parse the value passed through the plugin URL environment variables. +/// +/// A plain URL remains the single-endpoint format. For multiple endpoints, +/// use a JSON array of strings or objects with an optional stable `id`: +/// `["http://one/v1", {"id":"two", "url":"http://two/v1"}]`. +fn endpoint_specs_from_value(value: &str) -> Result> { + let value = value.trim(); + if value.is_empty() { + bail!("at least one endpoint URL must be configured"); + } + + let urls = if value.starts_with('[') { + let entries = serde_json::from_str::(value) + .context("parse plugin JSON endpoint list")? + .as_array() + .cloned() + .context("plugin JSON URL value must be an array")?; + entries + .into_iter() + .enumerate() + .map(|(index, entry)| { + let (id, url) = match entry { + Value::String(url) => (None, url), + Value::Object(object) => { + let id = object.get("id").and_then(Value::as_str).map(str::to_string); + let url = object + .get("url") + .and_then(Value::as_str) + .with_context(|| { + format!("endpoint {index} object must contain a string `url`") + })? + .trim() + .to_string(); + (id, url) + } + _ => bail!("endpoint {index} must be a URL string or an object"), + }; + Ok((id, url)) + }) + .collect::>>()? + } else { + // Newlines are accepted for convenient environment-variable based + // configuration while keeping commas valid inside URLs. + value + .lines() + .map(|url| (None, url.trim().to_string())) + .filter(|(_, url)| !url.is_empty()) + .collect() + }; + + if urls.is_empty() { + bail!("at least one endpoint URL must be configured"); + } + + let endpoint_count = urls.len(); + let specs = urls + .into_iter() + .enumerate() + .map(|(index, (id, url))| { + let url = url.trim().to_string(); + if url.is_empty() { + bail!("endpoint {index} URL must not be empty"); + } + let id = id.unwrap_or_else(|| { + if endpoint_count == 1 { + PLUGIN_ID.to_string() + } else { + format!("{PLUGIN_ID}-{}", index + 1) + } + }); + let id = id.trim().to_string(); + if id.is_empty() { + bail!("endpoint {index} ID must not be empty"); + } + Ok(EndpointSpec { id, url }) + }) + .collect::>>()?; + + let mut ids = HashSet::new(); + for spec in &specs { + if !ids.insert(&spec.id) { + bail!("duplicate endpoint ID '{}'", spec.id); + } + } + Ok(specs) } -fn build_plugin(name: String) -> mesh_llm_plugin::SimplePlugin { - let base_url = base_url(); - let health_url = base_url.clone(); +fn build_plugin(name: String) -> Result { + let endpoints = configured_endpoint_specs()?; + let endpoint_summary = endpoints + .iter() + .map(|endpoint| format!("{}={}", endpoint.id, endpoint.url)) + .collect::>() + .join(", "); - mesh_llm_plugin::plugin! { - metadata: PluginMetadata::new( - name, + let metadata = PluginMetadata::new( + name, + VERSION, + plugin_server_info( + "mesh-openai-endpoint", VERSION, - plugin_server_info( - "mesh-openai-endpoint", - VERSION, - "OpenAI-Compatible Endpoint Plugin", - "Routes inference to an external OpenAI-compatible server (vLLM, TGI, Ollama, etc.).", - Some( - "Set MESH_LLM_PLUGIN_URL to point at any server \ - that speaks the OpenAI /v1/chat/completions API.", - ), + "OpenAI-Compatible Endpoint Plugin", + "Routes inference to external OpenAI-compatible servers (vLLM, TGI, Ollama, etc.).", + Some( + "Set MESH_LLM_PLUGIN_URL for one endpoint or MESH_LLM_PLUGIN_URLS \ + for multiple endpoint URLs.", ), ), - startup_policy: PluginStartupPolicy::Any, - provides: [ - capability("endpoint:inference"), - capability("endpoint:inference/openai_compatible"), - ], - inference: [ - mesh_llm_plugin::inference::openai_http(PLUGIN_ID, base_url.clone()) + ); + let mut builder = DeclarativePluginBuilder::new(metadata) + .startup_policy(PluginStartupPolicy::Any) + .provide(capability("endpoint:inference")) + .provide(capability("endpoint:inference/openai_compatible")); + for endpoint in endpoints { + builder = builder.inference_item( + mesh_llm_plugin::inference::openai_http(endpoint.id, endpoint.url) .managed_by_plugin(false), - ], - health: move |_context| { - let health_url = health_url.clone(); - Box::pin(async move { Ok(format!("base_url={health_url}")) }) - }, + ); } + builder = builder.customize(move |plugin| { + plugin.with_health(move |_context| { + let endpoint_summary = endpoint_summary.clone(); + Box::pin(async move { Ok(format!("endpoints={endpoint_summary}")) }) + }) + }); + Ok(builder.build()) } async fn run_plugin(name: String) -> Result<()> { - PluginRuntime::run(build_plugin(name)).await + PluginRuntime::run(build_plugin(name)?).await } pub fn run_main() -> i32 { @@ -80,7 +187,7 @@ mod tests { #[test] fn manifest_declares_external_openai_endpoint() { - let plugin = build_plugin(PLUGIN_ID.to_string()); + let plugin = build_plugin(PLUGIN_ID.to_string()).expect("build plugin"); let manifest = plugin.manifest().expect("manifest"); assert!( @@ -100,8 +207,9 @@ mod tests { return Ok(()); } - let base_url = std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| base_url()); - let plugin = build_plugin(PLUGIN_ID.to_string()); + let base_url = + std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.to_string()); + let plugin = build_plugin(PLUGIN_ID.to_string())?; let manifest = plugin.manifest().context("plugin manifest")?; let endpoint = manifest .endpoints @@ -180,4 +288,48 @@ mod tests { fn endpoint_url(base_url: &str, tail: &str) -> String { format!("{}/{}", base_url.trim_end_matches('/'), tail) } + + #[test] + fn endpoint_urls_keep_single_endpoint_id_compatible() { + assert_eq!( + endpoint_specs_from_value("http://localhost:8000/v1").unwrap(), + vec![EndpointSpec { + id: PLUGIN_ID.to_string(), + url: "http://localhost:8000/v1".to_string(), + }] + ); + } + + #[test] + fn endpoint_urls_accept_json_array_with_optional_ids() { + let specs = endpoint_specs_from_value( + r#"["http://one:8000/v1", {"id":"remote", "url":"https://two.example/v1"}]"#, + ) + .unwrap(); + assert_eq!(specs[0].id, "openai-endpoint-1"); + assert_eq!(specs[0].url, "http://one:8000/v1"); + assert_eq!(specs[1].id, "remote"); + assert_eq!(specs[1].url, "https://two.example/v1"); + } + + #[test] + fn endpoint_urls_accept_newline_separated_values() { + let specs = endpoint_specs_from_value(" http://one/v1\n\nhttp://two/v1 ").unwrap(); + assert_eq!( + specs + .iter() + .map(|spec| spec.url.as_str()) + .collect::>(), + vec!["http://one/v1", "http://two/v1"] + ); + } + + #[test] + fn endpoint_urls_reject_duplicate_ids() { + let error = endpoint_specs_from_value( + r#"[{"id":"same", "url":"http://one/v1"}, "http://two/v1", {"id":"same", "url":"http://three/v1"}]"#, + ) + .expect_err("duplicate IDs should be rejected"); + assert!(error.to_string().contains("duplicate endpoint ID 'same'")); + } }