diff --git a/docs/modules/harness/observability.md b/docs/modules/harness/observability.md index cc1abdb..fbd436c 100644 --- a/docs/modules/harness/observability.md +++ b/docs/modules/harness/observability.md @@ -320,6 +320,35 @@ Redaction policies should handle: Redacted events should preserve structural fields such as ids, event kinds, timings, and counters so traces remain useful. +## Payload Capture + +The event stream is **payload-free by default**: `AgentEvent::ModelCompleted` +and `AgentEvent::ToolCompleted` carry only ids and usage, never the prompt, +completion, tool arguments, or tool result. This keeps journals and exporters +safe for privacy-sensitive deployments without any configuration. + +`RunPolicy::capture` (a `PayloadCapture { model_io, tool_io }`) opts a run into +carrying those payloads onto the completion events: + +- `model_io` — attaches the request messages (`input`) and the model completion + (`output`) to every `ModelCompleted`. +- `tool_io` — attaches the tool arguments (`input`) and the tool result + (`output`) to every `ToolCompleted`. + +Capture is snapshotted inside the agent loop before the request/call value is +moved into the model or tool wrap onion, so it never perturbs execution. Because +captured payloads ride the ordinary event pipeline, a `RedactingSink` still +masks them before they reach a journal or exporter, and the durable journal +replays them verbatim. + +Downstream, the Langfuse exporter reads these fields to populate the +Input/Output panels of a generation (`generation-create`) and a tool +observation (`tool-create`). With capture disabled the fields are absent and the +observation renders with ids, timing, and usage only. The harness exporter also +defaults the trace-level metadata from the run lineage (root/first run ids and +parent), mirroring the graph exporter, so a trace is correlatable even when the +caller passes no metadata. + ## Graph Events State-graph event payloads should be rich enough for a UI to render run diff --git a/src/harness/agent_loop/mod.rs b/src/harness/agent_loop/mod.rs index 4e0de75..3d2efcf 100644 --- a/src/harness/agent_loop/mod.rs +++ b/src/harness/agent_loop/mod.rs @@ -444,6 +444,14 @@ impl AgentHarness { model: binding.model, streaming, }; + // Snapshot the request messages for observability before `request` + // is moved into the model-wrap onion, gated by the capture policy so + // payload-free runs never serialize prompt text. + let captured_input = self + .policy + .capture + .model_io + .then(|| serde_json::to_value(&request.messages).unwrap_or(Value::Null)); let mut response = self .middleware .run_wrapped_model(ctx, state, request, &base) @@ -466,9 +474,16 @@ impl AgentHarness { let record = ctx.emit(AgentEvent::UsageRecorded { usage }); status.set_last_event(record.id); } + let captured_output = self + .policy + .capture + .model_io + .then(|| serde_json::to_value(&response.message).unwrap_or(Value::Null)); let record = ctx.emit(AgentEvent::ModelCompleted { call_id, usage: response.usage, + input: captured_input, + output: captured_output, }); status.set_last_event(record.id); @@ -628,6 +643,10 @@ impl AgentHarness { }); status.set_last_event(record.id); + // Snapshot the arguments for observability before `call` is + // moved into the tool-wrap onion, gated by the capture policy. + let captured_input = self.policy.capture.tool_io.then(|| call.arguments.clone()); + // The real tool call is the innermost base of the tool-wrap // onion (same before -> wrap -> after ordering as the model // path): lifecycle `before_tool` ran above, the wrap onion runs @@ -646,9 +665,16 @@ impl AgentHarness { run.tool_calls += 1; status.tool_calls = run.tool_calls; status.active_tool_calls.retain(|c| c != &tool_call_id); + let captured_output = self + .policy + .capture + .tool_io + .then(|| Value::String(result.content.clone())); let record = ctx.emit(AgentEvent::ToolCompleted { call_id: tool_call_id, tool_name, + input: captured_input, + output: captured_output, }); status.set_last_event(record.id); diff --git a/src/harness/events/types.rs b/src/harness/events/types.rs index e512411..be79030 100644 --- a/src/harness/events/types.rs +++ b/src/harness/events/types.rs @@ -75,6 +75,18 @@ pub enum AgentEvent { /// Token usage reported by the provider, when available. #[serde(default, skip_serializing_if = "Option::is_none")] usage: Option, + /// The request messages sent to the model, captured only when + /// [`PayloadCapture::model_io`][crate::harness::runtime::PayloadCapture::model_io] + /// is enabled. `None` in the default payload-free mode. Populated so an + /// exporter can render the prompt in a generation's Input panel. + #[serde(default, skip_serializing_if = "Option::is_none")] + input: Option, + /// The model completion (assistant message), captured only when + /// [`PayloadCapture::model_io`][crate::harness::runtime::PayloadCapture::model_io] + /// is enabled. `None` in the default payload-free mode. Populated so an + /// exporter can render the completion in a generation's Output panel. + #[serde(default, skip_serializing_if = "Option::is_none")] + output: Option, }, /// A tool-selection middleware filtered the model-visible tool set before a @@ -103,6 +115,18 @@ pub enum AgentEvent { call_id: CallId, /// Name of the tool that was invoked. tool_name: String, + /// The arguments the tool was invoked with, captured only when + /// [`PayloadCapture::tool_io`][crate::harness::runtime::PayloadCapture::tool_io] + /// is enabled. `None` in the default payload-free mode. Populated so an + /// exporter can render the call in a tool observation's Input panel. + #[serde(default, skip_serializing_if = "Option::is_none")] + input: Option, + /// The tool result content, captured only when + /// [`PayloadCapture::tool_io`][crate::harness::runtime::PayloadCapture::tool_io] + /// is enabled. `None` in the default payload-free mode. Populated so an + /// exporter can render the result in a tool observation's Output panel. + #[serde(default, skip_serializing_if = "Option::is_none")] + output: Option, }, /// The model called a tool that is not registered, and the run's diff --git a/src/harness/observability/langfuse.rs b/src/harness/observability/langfuse.rs index 60cf070..ba47698 100644 --- a/src/harness/observability/langfuse.rs +++ b/src/harness/observability/langfuse.rs @@ -156,6 +156,7 @@ impl LangfuseClient { .first() .map(|obs| iso_ms(obs.ts_ms)) .unwrap_or_else(|| iso_ms(now_ms())); + let metadata = trace_metadata(&trace, observations); let mut batch = Vec::with_capacity(observations.len() + 1); batch.push(json!({ @@ -172,7 +173,7 @@ impl LangfuseClient { "release": trace.release, "version": trace.version, "tags": if trace.tags.is_empty() { Value::Null } else { json!(trace.tags) }, - "metadata": if trace.metadata.is_null() { Value::Null } else { trace.metadata }, + "metadata": metadata, })), })); @@ -277,6 +278,33 @@ fn resolve_trace_id( }) } +/// Builds the trace-level metadata, defaulting useful run-lineage coordinates +/// (root/first run ids and thread, mirroring what the graph exporter folds in) +/// so a harness trace is correlatable even when the caller passes no metadata. +/// Any caller-supplied [`LangfuseTraceConfig::metadata`] keys are merged on top +/// and win on collision. Returns [`Value::Null`] only when there is nothing to +/// attach, so [`clean_nulls`] drops the field entirely. +fn trace_metadata(trace: &LangfuseTraceConfig, observations: &[AgentObservation]) -> Value { + let mut metadata = serde_json::Map::new(); + if let Some(first) = observations.first() { + metadata.insert("root_run_id".to_string(), json!(first.root_run_id.as_str())); + metadata.insert("run_id".to_string(), json!(first.run_id.as_str())); + if let Some(parent) = &first.parent_run_id { + metadata.insert("parent_run_id".to_string(), json!(parent.as_str())); + } + } + if let Value::Object(extra) = &trace.metadata { + for (k, v) in extra { + metadata.insert(k.clone(), v.clone()); + } + } + if metadata.is_empty() { + Value::Null + } else { + Value::Object(metadata) + } +} + fn observation_event(trace_id: &str, obs: &AgentObservation) -> Value { let timestamp = iso_ms(obs.ts_ms); let metadata = json!({ @@ -287,7 +315,12 @@ fn observation_event(trace_id: &str, obs: &AgentObservation) -> Value { "event": obs.event, }); match &obs.event { - AgentEvent::ModelCompleted { call_id, usage } => json!({ + AgentEvent::ModelCompleted { + call_id, + usage, + input, + output, + } => json!({ "id": obs.event_id.as_str(), "timestamp": timestamp, "type": "generation-create", @@ -298,10 +331,17 @@ fn observation_event(trace_id: &str, obs: &AgentObservation) -> Value { "startTime": timestamp, "endTime": timestamp, "usage": usage.map(langfuse_usage), + "input": input, + "output": output, "metadata": metadata, })), }), - AgentEvent::ToolCompleted { call_id, tool_name } => json!({ + AgentEvent::ToolCompleted { + call_id, + tool_name, + input, + output, + } => json!({ "id": obs.event_id.as_str(), "timestamp": timestamp, "type": "tool-create", @@ -311,6 +351,8 @@ fn observation_event(trace_id: &str, obs: &AgentObservation) -> Value { "name": tool_name, "startTime": timestamp, "endTime": timestamp, + "input": input, + "output": output, "metadata": metadata, })), }), @@ -469,6 +511,8 @@ mod tests { total_tokens: 7, ..Default::default() }), + input: None, + output: None, }, ), ], @@ -479,8 +523,82 @@ mod tests { assert_eq!(events[0]["type"], "trace-create"); assert_eq!(events[0]["body"]["id"], "root-1"); assert_eq!(events[0]["body"]["userId"], "user-1"); + // Trace metadata is defaulted from run lineage even without a caller value. + assert_eq!(events[0]["body"]["metadata"]["root_run_id"], "root-1"); + assert_eq!(events[0]["body"]["metadata"]["run_id"], "run-1"); assert_eq!(events[2]["type"], "generation-create"); assert_eq!(events[2]["body"]["id"], "model-call"); assert_eq!(events[2]["body"]["usage"]["input"], 3); + // Payload-free generation: no input/output body fields. + assert!(events[2]["body"].get("input").is_none()); + assert!(events[2]["body"].get("output").is_none()); + } + + #[test] + fn populates_generation_and_tool_io_when_captured() { + let client = + LangfuseClient::proxy("https://backend.test/telemetry/langfuse/ingestion", "t") + .unwrap(); + let batch = client + .build_ingestion_batch( + LangfuseTraceConfig::default(), + &[ + obs( + 0, + AgentEvent::ModelCompleted { + call_id: CallId::new("model-call"), + usage: None, + input: Some(json!([{ "role": "user", "content": "hi" }])), + output: Some(json!({ "content": "hello there" })), + }, + ), + obs( + 1, + AgentEvent::ToolCompleted { + call_id: CallId::new("tool-call"), + tool_name: "lookup".to_string(), + input: Some(json!({ "query": "weather" })), + output: Some(json!("sunny")), + }, + ), + ], + ) + .unwrap(); + + let events = batch["batch"].as_array().unwrap(); + assert_eq!(events[1]["type"], "generation-create"); + assert_eq!(events[1]["body"]["input"][0]["content"], "hi"); + assert_eq!(events[1]["body"]["output"]["content"], "hello there"); + assert_eq!(events[2]["type"], "tool-create"); + assert_eq!(events[2]["body"]["input"]["query"], "weather"); + assert_eq!(events[2]["body"]["output"], "sunny"); + } + + #[test] + fn merges_caller_trace_metadata_over_defaults() { + let client = + LangfuseClient::proxy("https://backend.test/telemetry/langfuse/ingestion", "t") + .unwrap(); + let batch = client + .build_ingestion_batch( + LangfuseTraceConfig { + metadata: json!({ "deployment": "prod", "root_run_id": "override" }), + ..Default::default() + }, + &[obs( + 0, + AgentEvent::RunStarted { + run_id: RunId::new("run-1"), + thread_id: None, + }, + )], + ) + .unwrap(); + + let meta = &batch["batch"].as_array().unwrap()[0]["body"]["metadata"]; + assert_eq!(meta["deployment"], "prod"); + // Caller keys win on collision with the defaulted lineage. + assert_eq!(meta["root_run_id"], "override"); + assert_eq!(meta["run_id"], "run-1"); } } diff --git a/src/harness/observability/test.rs b/src/harness/observability/test.rs index 1d44ed3..ae378cf 100644 --- a/src/harness/observability/test.rs +++ b/src/harness/observability/test.rs @@ -117,6 +117,8 @@ fn agent_latency_metrics_include_model_tool_and_run_elapsed() { AgentEvent::ModelCompleted { call_id: model_id.clone(), usage: None, + input: None, + output: None, }, ), obs( @@ -133,6 +135,8 @@ fn agent_latency_metrics_include_model_tool_and_run_elapsed() { AgentEvent::ToolCompleted { call_id: tool_id.clone(), tool_name: "lookup".to_string(), + input: None, + output: None, }, ), obs( diff --git a/src/harness/runtime/types.rs b/src/harness/runtime/types.rs index 55fb792..18c7893 100644 --- a/src/harness/runtime/types.rs +++ b/src/harness/runtime/types.rs @@ -73,6 +73,53 @@ pub enum UnknownToolPolicy { }, } +/// Controls whether the agent loop captures model and tool **payloads** +/// (prompt/completion text, tool arguments/results) onto the +/// [`AgentEvent::ModelCompleted`][crate::harness::events::AgentEvent::ModelCompleted] +/// and [`AgentEvent::ToolCompleted`][crate::harness::events::AgentEvent::ToolCompleted] +/// events it emits. +/// +/// The observability layer is **payload-free by default**: events carry only +/// ids, counters, and usage so privacy-sensitive deployments never journal or +/// export prompt text or tool I/O. Opt in per-family here to surface the request +/// messages + completion (`model_io`) and tool arguments + result (`tool_io`) so +/// downstream exporters — notably the Langfuse exporter — can populate the +/// Input/Output panels of a generation or tool observation. +/// +/// Captured payloads flow through the same event pipeline as everything else, so +/// a [`RedactingSink`][crate::harness::observability::RedactingSink] configured +/// with secret substrings still masks them before they reach a journal or +/// exporter. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct PayloadCapture { + /// Capture the request messages and the model completion onto every + /// [`AgentEvent::ModelCompleted`][crate::harness::events::AgentEvent::ModelCompleted]. + pub model_io: bool, + /// Capture the tool arguments and the tool result onto every + /// [`AgentEvent::ToolCompleted`][crate::harness::events::AgentEvent::ToolCompleted]. + pub tool_io: bool, +} + +impl PayloadCapture { + /// A capture policy with both model and tool I/O enabled. + /// + /// Prefer this only when the observability pipeline is trusted (or a + /// [`RedactingSink`][crate::harness::observability::RedactingSink] is in + /// place), since it journals and can export full prompt/completion text and + /// tool arguments/results. + pub const fn all() -> Self { + Self { + model_io: true, + tool_io: true, + } + } + + /// `true` when neither model nor tool payloads are captured (the default). + pub const fn is_disabled(&self) -> bool { + !self.model_io && !self.tool_io + } +} + #[derive(Clone, Debug, PartialEq)] pub struct RunPolicy { /// Hard run limits enforced fail-closed by the agent loop. @@ -85,6 +132,11 @@ pub struct RunPolicy { pub fallback: Option, /// Response format attached to every model request when set. pub default_response_format: Option, + /// Whether the loop captures model/tool payloads onto completion events. + /// + /// Defaults to [`PayloadCapture::default`] (payload-free), preserving the + /// privacy-preserving behavior where events carry only ids and usage. + pub capture: PayloadCapture, /// Default caching policy for the run. /// /// The loop consults [`CachePolicy::response_cache_enabled`] only when a @@ -102,6 +154,7 @@ impl Default for RunPolicy { retry: RetryPolicy::default(), fallback: None, default_response_format: None, + capture: PayloadCapture::default(), // Caching defaults ON, but is gated by an attached `ResponseCache`, // so a harness with no cache never caches regardless of this flag. cache: CachePolicy { diff --git a/src/harness/testkit/test.rs b/src/harness/testkit/test.rs index 6f561aa..2fadd59 100644 --- a/src/harness/testkit/test.rs +++ b/src/harness/testkit/test.rs @@ -327,6 +327,8 @@ fn make_trajectory() -> Vec { AgentEvent::ModelCompleted { call_id: CallId::new("c1"), usage: None, + input: None, + output: None, }, AgentEvent::ToolStarted { call_id: CallId::new("t1"), @@ -335,6 +337,8 @@ fn make_trajectory() -> Vec { AgentEvent::ToolCompleted { call_id: CallId::new("t1"), tool_name: "search".into(), + input: None, + output: None, }, AgentEvent::ModelStarted { call_id: CallId::new("c2"), @@ -343,6 +347,8 @@ fn make_trajectory() -> Vec { AgentEvent::ModelCompleted { call_id: CallId::new("c2"), usage: None, + input: None, + output: None, }, AgentEvent::RunCompleted { run_id: RunId::new("r1"), diff --git a/src/harness/testkit/types.rs b/src/harness/testkit/types.rs index 36ef3bf..35fab91 100644 --- a/src/harness/testkit/types.rs +++ b/src/harness/testkit/types.rs @@ -253,7 +253,12 @@ pub struct EventRecorder { /// let events = vec![ /// AgentEvent::RunStarted { run_id: RunId::new("r1"), thread_id: None }, /// AgentEvent::ModelStarted { call_id: CallId::new("c1"), model: "gpt".into() }, -/// AgentEvent::ModelCompleted { call_id: CallId::new("c1"), usage: None }, +/// AgentEvent::ModelCompleted { +/// call_id: CallId::new("c1"), +/// usage: None, +/// input: None, +/// output: None, +/// }, /// AgentEvent::RunCompleted { run_id: RunId::new("r1") }, /// ]; /// let traj = Trajectory::from_events(events); diff --git a/tests/e2e_observability.rs b/tests/e2e_observability.rs index b310ec2..b822dfd 100644 --- a/tests/e2e_observability.rs +++ b/tests/e2e_observability.rs @@ -27,7 +27,7 @@ use tinyagents::harness::ids::{ExecutionStatus, RunId}; use tinyagents::harness::message::{AssistantMessage, ContentBlock, Message}; use tinyagents::harness::model::ModelResponse; use tinyagents::harness::providers::MockModel; -use tinyagents::harness::runtime::AgentHarness; +use tinyagents::harness::runtime::{AgentHarness, PayloadCapture, RunPolicy}; use tinyagents::harness::testkit::FakeTool; use tinyagents::harness::tool::ToolCall; use tinyagents::harness::usage::Usage; @@ -184,6 +184,93 @@ async fn harness_run_journals_redacted_replayable_events() { assert_eq!(tail.first().unwrap().offset, 2); } +#[tokio::test] +async fn harness_run_with_capture_exports_generation_and_tool_io() { + // With PayloadCapture enabled, a real harness run journals model/tool + // payloads on its completion events; feeding those durable observations + // through the Langfuse exporter populates the generation Input/Output and + // the tool-create Input/Output panels — the fix for tinyhumansai/tinyagents#6. + let mut harness: AgentHarness<()> = AgentHarness::new(); + harness + .register_model( + "capture-model", + Arc::new(MockModel::with_responses(vec![ + tool_call_response("call-1", "lookup", json!({ "q": "weather" })), + text_response("all done"), + ])), + ) + .set_default_model("capture-model") + .register_tool(Arc::new(FakeTool::returning("lookup", "tool-output"))) + .with_policy(RunPolicy { + capture: PayloadCapture::all(), + ..Default::default() + }); + + let journal: Arc = Arc::new(InMemoryEventJournal::new()); + let run_id = RunId::new("run-capture"); + let ctx: RunContext<()> = RunContext::new(RunConfig::new(run_id.as_str()), ()); + ctx.events + .subscribe(Arc::new(JournalSink::new(journal.clone(), run_id.clone()))); + + harness + .invoke_in_context_with_status(&(), ctx, vec![Message::user("please look up")]) + .await + .expect("run succeeds"); + + let observations = journal + .read_from(run_id.as_str(), 0) + .await + .expect("replay journal"); + + // The captured payloads survive on the durable observations. + let model_completed = observations + .iter() + .find(|o| matches!(o.event, AgentEvent::ModelCompleted { .. })) + .expect("a model completion is journaled"); + match &model_completed.event { + AgentEvent::ModelCompleted { input, output, .. } => { + assert!(input.is_some(), "captured model input rides the event"); + assert!(output.is_some(), "captured model output rides the event"); + } + _ => unreachable!(), + } + + // Export through the harness Langfuse client and assert the body carries I/O. + let client = LangfuseClient::proxy("https://backend.test", "tok").expect("client"); + let batch = client + .build_ingestion_batch(LangfuseTraceConfig::default(), &observations) + .expect("build batch"); + let events = batch["batch"].as_array().expect("batch array"); + + // Trace metadata is defaulted from the run lineage even with no caller value. + assert_eq!(events[0]["type"], "trace-create"); + assert_eq!(events[0]["body"]["metadata"]["root_run_id"], "run-capture"); + + let generations: Vec<_> = events + .iter() + .filter(|e| e["type"] == "generation-create") + .collect(); + assert_eq!(generations.len(), 2, "one generation per model call"); + assert!( + generations + .iter() + .all(|g| !g["body"]["input"].is_null() && !g["body"]["output"].is_null()), + "every generation Input/Output panel is populated" + ); + // The final generation's completion carries the answer text. + assert_eq!( + generations[1]["body"]["output"]["content"][0]["text"], + "all done" + ); + + let tool = events + .iter() + .find(|e| e["type"] == "tool-create") + .expect("a tool observation"); + assert_eq!(tool["body"]["input"]["q"], "weather"); + assert_eq!(tool["body"]["output"], "tool-output"); +} + /// A two-node line graph over `i32` with overwrite semantics: `a -> b`. fn line_graph() -> tinyagents::CompiledGraph { GraphBuilder::::overwrite() diff --git a/tests/e2e_orchestrator_subagents.rs b/tests/e2e_orchestrator_subagents.rs index c985b18..554dd0e 100644 --- a/tests/e2e_orchestrator_subagents.rs +++ b/tests/e2e_orchestrator_subagents.rs @@ -171,6 +171,8 @@ async fn orchestrator_resolves_and_runs_only_the_chosen_subagents() -> Result<() sink.emit(AgentEvent::ToolCompleted { call_id: CallId::new(call_id), tool_name: name.clone(), + input: None, + output: None, }); Ok::<(String, String), TinyAgentsError>((name, result.content)) } diff --git a/tests/e2e_registry_observability_contracts.rs b/tests/e2e_registry_observability_contracts.rs index f27d78d..d46731a 100644 --- a/tests/e2e_registry_observability_contracts.rs +++ b/tests/e2e_registry_observability_contracts.rs @@ -171,6 +171,8 @@ fn component_metadata_and_event_kinds_are_stable_serializable_contracts() { AgentEvent::ModelCompleted { call_id: CallId::new("model-1"), usage: None, + input: None, + output: None, }, AgentEvent::ToolStarted { call_id: CallId::new("tool-1"), @@ -179,6 +181,8 @@ fn component_metadata_and_event_kinds_are_stable_serializable_contracts() { AgentEvent::ToolCompleted { call_id: CallId::new("tool-1"), tool_name: "lookup".into(), + input: None, + output: None, }, AgentEvent::StateUpdate, AgentEvent::MiddlewareStarted { name: "mw".into() }, @@ -300,6 +304,8 @@ async fn event_sinks_journals_and_status_stores_preserve_run_lineage() { journal.append(AgentEvent::ToolCompleted { call_id: CallId::new("tool-1"), tool_name: "lookup".into(), + input: None, + output: None, }); assert_eq!(journal.len(), 2); assert_eq!(journal.replay_from(1)[0].event.kind(), "tool.completed");