Support multiple configured endpoint URLs

Cargo.toml

@@ -9,8 +9,8 @@ repository = "https://github.com/Mesh-LLM/openai-endpoint"
 [dependencies]
 anyhow = "1"
 mesh-llm-plugin = { git = "https://github.com/Mesh-LLM/mesh-llm.git", branch = "codex/plugin-cli-commands" }
+serde_json = "1"
 tokio = { version = "1", features = ["full"] }
 
 [dev-dependencies]
 reqwest = { version = "0.12", features = ["json"] }
-serde_json = "1"

README.md

@@ -18,7 +18,7 @@ mesh-llm plugins install Mesh-LLM/openai-endpoint
 
 ## Configure
 
-Point the plugin at the external server:
+Point the plugin at one external server:
 
 ```toml
 [[plugin]]
@@ -39,6 +39,55 @@ url = "http://localhost:8000/v1"
 mesh-llm passes `url` to the plugin as `MESH_LLM_PLUGIN_URL`. If neither config
 nor environment is set, it defaults to `http://localhost:8000/v1`.
 
+### Multiple endpoints
+
+The same plugin instance can advertise one or more OpenAI-compatible servers.
+The existing single-URL form remains supported. For multiple servers, use
+`urls`:
+
+```toml
+[[plugin]]
+name = "openai-endpoint"
+urls = [
+  "http://localhost:8000/v1",
+  "http://gpu-box:8000/v1",
+]
+```
+
+If `urls` is set, the plugin receives the values through
+`MESH_LLM_PLUGIN_URLS`. A single endpoint keeps the endpoint ID
+`openai-endpoint`; multiple endpoints are assigned `openai-endpoint-1`,
+`openai-endpoint-2`, and so on. Newline-separated URLs are also accepted when
+setting the environment variable directly. Each endpoint is health-checked
+independently, and the mesh routes a model to the healthy endpoint that
+advertises it.
+
+### Authentication suggestions
+
+Authentication is not currently applied by this plugin. The plugin advertises
+the endpoint address, while the mesh host performs the model health probe and
+inference request, so adding a token only inside this process would not secure
+those host-side requests.
+
+Recommended options, in order of practicality:
+
+1. Add host-managed per-endpoint credentials. Extend the endpoint configuration
+   with an auth reference such as `bearer_env = "GPU_BOX_API_KEY"`, keep the
+   secret out of the manifest, and have the host attach the header to both
+   `/v1/models` health probes and inference requests. This supports different
+   credentials per endpoint and secret rotation without exposing tokens in
+   URLs, logs, or endpoint metadata.
+2. Use a local authenticated gateway or sidecar today. Point this plugin at
+   Envoy, an OAuth2 proxy, or another local gateway that injects credentials,
+   and let the gateway forward to the protected upstream. This requires no
+   mesh protocol change and keeps credentials outside the plugin URL.
+3. For OAuth2/OIDC, add a host-side token provider with caching, expiry-aware
+   refresh, and separate scopes per endpoint. The same provider should be used
+   by health checks and inference forwarding.
+
+Avoid putting bearer tokens in query strings or URL user-info, since endpoint
+addresses can appear in manifests, diagnostics, and logs.
+
 ## Build
 
 ```bash

src/lib.rs

@@ -1,57 +1,164 @@
-use anyhow::Result;
+use anyhow::{Context, Result, bail};
 use mesh_llm_plugin::{
-    PluginMetadata, PluginRuntime, PluginStartupPolicy, capability, plugin_server_info,
+    DeclarativePluginBuilder, PluginMetadata, PluginRuntime, PluginStartupPolicy, capability,
+    plugin_server_info,
 };
+use serde_json::Value;
+use std::collections::HashSet;
 
 pub const VERSION: &str = env!("CARGO_PKG_VERSION");
 const DEFAULT_BASE_URL: &str = "http://localhost:8000/v1";
 const PLUGIN_ID: &str = "openai-endpoint";
 
-fn base_url() -> String {
-    std::env::var("MESH_LLM_PLUGIN_URL")
+#[derive(Clone, Debug, PartialEq, Eq)]
+struct EndpointSpec {
+    id: String,
+    url: String,
+}
+
+fn configured_endpoint_specs() -> Result<Vec<EndpointSpec>> {
+    let value = std::env::var("MESH_LLM_PLUGIN_URLS")
         .ok()
-        .map(|value| value.trim().to_string())
-        .filter(|value| !value.is_empty())
-        .unwrap_or_else(|| DEFAULT_BASE_URL.to_string())
+        .filter(|value| !value.trim().is_empty())
+        .or_else(|| {
+            std::env::var("MESH_LLM_PLUGIN_URL")
+                .ok()
+                .filter(|value| !value.trim().is_empty())
+        })
+        .unwrap_or_else(|| DEFAULT_BASE_URL.to_string());
+    endpoint_specs_from_value(&value)
+}
+
+/// Parse the value passed through the plugin URL environment variables.
+///
+/// A plain URL remains the single-endpoint format. For multiple endpoints,
+/// use a JSON array of strings or objects with an optional stable `id`:
+/// `["http://one/v1", {"id":"two", "url":"http://two/v1"}]`.
+fn endpoint_specs_from_value(value: &str) -> Result<Vec<EndpointSpec>> {
+    let value = value.trim();
+    if value.is_empty() {
+        bail!("at least one endpoint URL must be configured");
+    }
+
+    let urls = if value.starts_with('[') {
+        let entries = serde_json::from_str::<Value>(value)
+            .context("parse plugin JSON endpoint list")?
+            .as_array()
+            .cloned()
+            .context("plugin JSON URL value must be an array")?;
+        entries
+            .into_iter()
+            .enumerate()
+            .map(|(index, entry)| {
+                let (id, url) = match entry {
+                    Value::String(url) => (None, url),
+                    Value::Object(object) => {
+                        let id = object.get("id").and_then(Value::as_str).map(str::to_string);
+                        let url = object
+                            .get("url")
+                            .and_then(Value::as_str)
+                            .with_context(|| {
+                                format!("endpoint {index} object must contain a string `url`")
+                            })?
+                            .trim()
+                            .to_string();
+                        (id, url)
+                    }
+                    _ => bail!("endpoint {index} must be a URL string or an object"),
+                };
+                Ok((id, url))
+            })
+            .collect::<Result<Vec<_>>>()?
+    } else {
+        // Newlines are accepted for convenient environment-variable based
+        // configuration while keeping commas valid inside URLs.
+        value
+            .lines()
+            .map(|url| (None, url.trim().to_string()))
+            .filter(|(_, url)| !url.is_empty())
+            .collect()
+    };
+
+    if urls.is_empty() {
+        bail!("at least one endpoint URL must be configured");
+    }
+
+    let endpoint_count = urls.len();
+    let specs = urls
+        .into_iter()
+        .enumerate()
+        .map(|(index, (id, url))| {
+            let url = url.trim().to_string();
+            if url.is_empty() {
+                bail!("endpoint {index} URL must not be empty");
+            }
+            let id = id.unwrap_or_else(|| {
+                if endpoint_count == 1 {
+                    PLUGIN_ID.to_string()
+                } else {
+                    format!("{PLUGIN_ID}-{}", index + 1)
+                }
+            });
+            let id = id.trim().to_string();
+            if id.is_empty() {
+                bail!("endpoint {index} ID must not be empty");
+            }
+            Ok(EndpointSpec { id, url })
+        })
+        .collect::<Result<Vec<_>>>()?;
+
+    let mut ids = HashSet::new();
+    for spec in &specs {
+        if !ids.insert(&spec.id) {
+            bail!("duplicate endpoint ID '{}'", spec.id);
+        }
+    }
+    Ok(specs)
 }
 
-fn build_plugin(name: String) -> mesh_llm_plugin::SimplePlugin {
-    let base_url = base_url();
-    let health_url = base_url.clone();
+fn build_plugin(name: String) -> Result<mesh_llm_plugin::SimplePlugin> {
+    let endpoints = configured_endpoint_specs()?;
+    let endpoint_summary = endpoints
+        .iter()
+        .map(|endpoint| format!("{}={}", endpoint.id, endpoint.url))
+        .collect::<Vec<_>>()
+        .join(", ");
 
-    mesh_llm_plugin::plugin! {
-        metadata: PluginMetadata::new(
-            name,
+    let metadata = PluginMetadata::new(
+        name,
+        VERSION,
+        plugin_server_info(
+            "mesh-openai-endpoint",
             VERSION,
-            plugin_server_info(
-                "mesh-openai-endpoint",
-                VERSION,
-                "OpenAI-Compatible Endpoint Plugin",
-                "Routes inference to an external OpenAI-compatible server (vLLM, TGI, Ollama, etc.).",
-                Some(
-                    "Set MESH_LLM_PLUGIN_URL to point at any server \
-                     that speaks the OpenAI /v1/chat/completions API.",
-                ),
+            "OpenAI-Compatible Endpoint Plugin",
+            "Routes inference to external OpenAI-compatible servers (vLLM, TGI, Ollama, etc.).",
+            Some(
+                "Set MESH_LLM_PLUGIN_URL for one endpoint or MESH_LLM_PLUGIN_URLS \
+                 for multiple endpoint URLs.",
             ),
         ),
-        startup_policy: PluginStartupPolicy::Any,
-        provides: [
-            capability("endpoint:inference"),
-            capability("endpoint:inference/openai_compatible"),
-        ],
-        inference: [
-            mesh_llm_plugin::inference::openai_http(PLUGIN_ID, base_url.clone())
+    );
+    let mut builder = DeclarativePluginBuilder::new(metadata)
+        .startup_policy(PluginStartupPolicy::Any)
+        .provide(capability("endpoint:inference"))
+        .provide(capability("endpoint:inference/openai_compatible"));
+    for endpoint in endpoints {
+        builder = builder.inference_item(
+            mesh_llm_plugin::inference::openai_http(endpoint.id, endpoint.url)
                 .managed_by_plugin(false),
-        ],
-        health: move |_context| {
-            let health_url = health_url.clone();
-            Box::pin(async move { Ok(format!("base_url={health_url}")) })
-        },
+        );
     }
+    builder = builder.customize(move |plugin| {
+        plugin.with_health(move |_context| {
+            let endpoint_summary = endpoint_summary.clone();
+            Box::pin(async move { Ok(format!("endpoints={endpoint_summary}")) })
+        })
+    });
+    Ok(builder.build())
 }
 
 async fn run_plugin(name: String) -> Result<()> {
-    PluginRuntime::run(build_plugin(name)).await
+    PluginRuntime::run(build_plugin(name)?).await
 }
 
 pub fn run_main() -> i32 {
@@ -80,7 +187,7 @@ mod tests {
 
     #[test]
     fn manifest_declares_external_openai_endpoint() {
-        let plugin = build_plugin(PLUGIN_ID.to_string());
+        let plugin = build_plugin(PLUGIN_ID.to_string()).expect("build plugin");
         let manifest = plugin.manifest().expect("manifest");
 
         assert!(
@@ -100,8 +207,9 @@ mod tests {
             return Ok(());
         }
 
-        let base_url = std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| base_url());
-        let plugin = build_plugin(PLUGIN_ID.to_string());
+        let base_url =
+            std::env::var("MESH_LLM_PLUGIN_URL").unwrap_or_else(|_| DEFAULT_BASE_URL.to_string());
+        let plugin = build_plugin(PLUGIN_ID.to_string())?;
         let manifest = plugin.manifest().context("plugin manifest")?;
         let endpoint = manifest
             .endpoints
@@ -180,4 +288,48 @@ mod tests {
     fn endpoint_url(base_url: &str, tail: &str) -> String {
         format!("{}/{}", base_url.trim_end_matches('/'), tail)
     }
+
+    #[test]
+    fn endpoint_urls_keep_single_endpoint_id_compatible() {
+        assert_eq!(
+            endpoint_specs_from_value("http://localhost:8000/v1").unwrap(),
+            vec![EndpointSpec {
+                id: PLUGIN_ID.to_string(),
+                url: "http://localhost:8000/v1".to_string(),
+            }]
+        );
+    }
+
+    #[test]
+    fn endpoint_urls_accept_json_array_with_optional_ids() {
+        let specs = endpoint_specs_from_value(
+            r#"["http://one:8000/v1", {"id":"remote", "url":"https://two.example/v1"}]"#,
+        )
+        .unwrap();
+        assert_eq!(specs[0].id, "openai-endpoint-1");
+        assert_eq!(specs[0].url, "http://one:8000/v1");
+        assert_eq!(specs[1].id, "remote");
+        assert_eq!(specs[1].url, "https://two.example/v1");
+    }
+
+    #[test]
+    fn endpoint_urls_accept_newline_separated_values() {
+        let specs = endpoint_specs_from_value(" http://one/v1\n\nhttp://two/v1 ").unwrap();
+        assert_eq!(
+            specs
+                .iter()
+                .map(|spec| spec.url.as_str())
+                .collect::<Vec<_>>(),
+            vec!["http://one/v1", "http://two/v1"]
+        );
+    }
+
+    #[test]
+    fn endpoint_urls_reject_duplicate_ids() {
+        let error = endpoint_specs_from_value(
+            r#"[{"id":"same", "url":"http://one/v1"}, "http://two/v1", {"id":"same", "url":"http://three/v1"}]"#,
+        )
+        .expect_err("duplicate IDs should be rejected");
+        assert!(error.to_string().contains("duplicate endpoint ID 'same'"));
+    }
 }