From 335b1b850838836de2e72285500061949fee8e98 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 17:38:44 -0700 Subject: [PATCH 1/2] fix(stella-pipeline,stella-protocol): record verifier degradation per candidate in a fan-out A best-of-N fan-out whose verifier died reported one once-per-run prose caveat for N candidates, and nothing recorded WHICH candidates were judged by the deterministic heuristic. Each candidate now emits one structured ProofStep::VerdictDegraded fact (1-based ordinal + stated reason), keyed by candidate-local state so concurrent candidates cannot race a shared flag; the once-per-run transcript warning is unchanged. ProofStep moves to stella-protocol/src/proof.rs (the crate's ladder.rs pattern, re-exported so every existing path still resolves) because event.rs sits at its file-size ceiling; pipeline.rs stays under its own ceiling by consolidating the two once-per-run AtomicBool notices into VerifierNotices in verifier_stage.rs. Refs #1787 --- crates/stella-pipeline/src/pipeline.rs | 31 ++-- .../src/pipeline/fanout_stage.rs | 8 +- .../src/pipeline/verifier_stage.rs | 86 ++++++++-- crates/stella-pipeline/src/replay.rs | 5 + crates/stella-protocol/src/event.rs | 101 +----------- crates/stella-protocol/src/lib.rs | 1 + crates/stella-protocol/src/proof.rs | 154 ++++++++++++++++++ crates/stella-protocol/tests/wire_contract.rs | 4 + crates/stella-tui/src/proof.rs | 8 + 9 files changed, 267 insertions(+), 131 deletions(-) create mode 100644 crates/stella-protocol/src/proof.rs diff --git a/crates/stella-pipeline/src/pipeline.rs b/crates/stella-pipeline/src/pipeline.rs index 75af7d4ab..831a67ea0 100644 --- a/crates/stella-pipeline/src/pipeline.rs +++ b/crates/stella-pipeline/src/pipeline.rs @@ -129,6 +129,7 @@ pub use run_error::{PipelineError, PipelineRunError}; use run_error::{RoleResolveError, WitnessAuthorIndependence}; use stage_budget::{PipelineBudgetAbort, Spend, budget_abort}; use task_frame::TaskFrame; +use verifier_stage::{VerdictDegradation, VerifierNotices}; use witness_stage::{BoundHookRunner, WitnessAuthoring}; /// Make a diff that verification hands downstream *incapable of lying*. /// @@ -692,6 +693,8 @@ struct CandidateState { /// warrant (the #1701 recurrence a projection method used to guard /// against by hand). signals: ChangeSignals, + /// This candidate's verdict-degradation record (#1787) — see [`VerdictDegradation`]. + degradation: VerdictDegradation, /// Ends the execute turn as soon as the tracked test goes fail→pass. /// /// `None` whenever there is nothing to watch: no configured test command, @@ -874,12 +877,8 @@ pub struct Pipeline<'a> { /// other. Starts at [`RECEIPT_SEQ_ALLOCATED_BASE`], above the seats the /// engine's worker and summarizer reserve. raw_call_seq: AtomicU64, - /// Whether the verifier's same-family degradation caveat (L-M8) has been - /// surfaced this run — see [`Pipeline::warn_verifier_caveat`]. - verifier_caveat_warned: AtomicBool, - /// Whether a verdict's silent degradation to the deterministic heuristic - /// has been surfaced this run — see [`Pipeline::warn_verifier_fallback`]. - verifier_fallback_warned: AtomicBool, + /// The once-per-run verifier notices — see [`VerifierNotices`]. + verifier_notices: VerifierNotices, /// Whether more than one candidate is currently writing to [`Self::events`] /// — set for the duration of a concurrent best-of-N fan-out and false /// everywhere else. @@ -932,8 +931,7 @@ impl<'a> Pipeline<'a> { config, configured_test, raw_call_seq: AtomicU64::new(RECEIPT_SEQ_ALLOCATED_BASE), - verifier_caveat_warned: AtomicBool::new(false), - verifier_fallback_warned: AtomicBool::new(false), + verifier_notices: VerifierNotices::default(), shared_event_lane: AtomicBool::new(false), started: std::time::Instant::now(), } @@ -1715,15 +1713,12 @@ impl<'a> Pipeline<'a> { if let Some(view) = view.as_ref() { engine = engine.with_steering(view); } + // Authoring is `None`: a shared-tree run has no workspace to graft + // into and no pristine snapshot to author blind in, so it never + // buys a witness. The ordinal is 1-based, like the start notice. results.push( - self.run_candidate( - frame, - // A shared-tree run has no workspace to graft into and no - // pristine snapshot to author blind in, so it never buys a - // witness — exactly as before, when it was passed `None`. - None, &engine, surface, spend, - ) - .await, + self.run_candidate(i + 1, frame, None, &engine, surface, spend) + .await, ); } results @@ -1964,6 +1959,7 @@ impl<'a> Pipeline<'a> { async fn run_candidate( &self, + candidate: u32, frame: TaskFrame<'_>, authoring: Option>, engine: &Engine<'_>, @@ -2056,6 +2052,7 @@ impl<'a> Pipeline<'a> { ), final_text: String::new(), signals: ChangeSignals::default(), + degradation: VerdictDegradation::new(candidate), flip_halt, oracle, oracle_trace, @@ -2763,7 +2760,7 @@ impl<'a> Pipeline<'a> { }, ); match self - .verifier(prompt, &inputs, spend.budget, spend.total) + .verifier(&mut state.degradation, prompt, &inputs, spend) .await { Ok(verdict) => { diff --git a/crates/stella-pipeline/src/pipeline/fanout_stage.rs b/crates/stella-pipeline/src/pipeline/fanout_stage.rs index 307d0b7c0..f8a13c91c 100644 --- a/crates/stella-pipeline/src/pipeline/fanout_stage.rs +++ b/crates/stella-pipeline/src/pipeline/fanout_stage.rs @@ -199,6 +199,11 @@ impl<'a> Pipeline<'a> { let mut cost_usd = 0.0; let result = self .run_isolated_candidate( + // 1-based, the ordinal every candidate-facing + // surface speaks (`candidate_start_notice`, + // `ProofStep::VerdictDegraded`). `n` is a `u32` + // config knob, so the index always fits. + index as u32 + 1, frame, worker, authoring, @@ -246,6 +251,7 @@ impl<'a> Pipeline<'a> { /// than in a loop iteration shared with nobody. async fn run_isolated_candidate( &self, + candidate: u32, frame: TaskFrame<'_>, worker: &ResolvedRole<'_>, authoring: Option>, @@ -286,7 +292,7 @@ impl<'a> Pipeline<'a> { if let Some(view) = view.as_ref() { engine = engine.with_steering(view); } - self.run_candidate(frame, authoring, &engine, surface, spend) + self.run_candidate(candidate, frame, authoring, &engine, surface, spend) .await } } diff --git a/crates/stella-pipeline/src/pipeline/verifier_stage.rs b/crates/stella-pipeline/src/pipeline/verifier_stage.rs index 6ce223bc7..abffab377 100644 --- a/crates/stella-pipeline/src/pipeline/verifier_stage.rs +++ b/crates/stella-pipeline/src/pipeline/verifier_stage.rs @@ -11,6 +11,45 @@ use super::*; +/// The once-per-run verifier notices, grouped where their emitters live: both +/// flags describe the run's *configuration* (a same-family caveat, a dead or +/// non-compliant verifier), which cannot change mid-run, so each is surfaced +/// to the transcript exactly once however many calls observe it. +#[derive(Default)] +pub(super) struct VerifierNotices { + /// Whether the router's same-family degradation caveat (L-M8) has been + /// surfaced — see [`Pipeline::warn_verifier_caveat`]. + caveat_warned: AtomicBool, + /// Whether a verdict's degradation to the deterministic heuristic has + /// been surfaced — see [`Pipeline::warn_verifier_fallback`]. + fallback_warned: AtomicBool, +} + +/// One candidate's verdict-degradation record: its ordinal (1-based, +/// [`ProofStep::Oracle`]'s `run` convention) and whether the degradation fact +/// has been emitted yet. +/// +/// Per candidate rather than per run, because a best-of-N fan-out degrading N +/// times used to leave one prose caveat and no record of *which* candidates +/// the heuristic judged (#1787) — and per candidate rather than per round, +/// because the escalation loop can hit the same dead verifier on every +/// revision and the fact does not change. Plain candidate-local state, never +/// a shared flag: candidates run concurrently, and resetting a run-wide flag +/// at candidate boundaries would race. +pub(super) struct VerdictDegradation { + candidate: u32, + recorded: bool, +} + +impl VerdictDegradation { + pub(super) fn new(candidate: u32) -> Self { + Self { + candidate, + recorded: false, + } + } +} + impl<'a> Pipeline<'a> { /// One distress-guidance call: best-effort and never a verdict — the /// failure it reacts to is already deterministic, so the verifier's job @@ -76,10 +115,10 @@ impl<'a> Pipeline<'a> { /// call can fail. pub(super) async fn verifier( &self, + degradation: &mut VerdictDegradation, prompt: ManagementPrompt, inputs: &LadderInputs, - budget: &mut BudgetGuard, - total: &mut f64, + spend: &mut Spend<'_>, ) -> Result { self.emit(AgentEvent::Stage { name: StageKind::Verdict, @@ -89,6 +128,7 @@ impl<'a> Pipeline<'a> { // Verifier unresolvable → conservative heuristic verdict (L-E11). Err(_) => { self.warn_verifier_fallback( + degradation, "the verifier role is unresolvable (no routable provider); check the \ `pipeline_verifier_model` provider and its credential", ); @@ -117,8 +157,8 @@ impl<'a> Pipeline<'a> { overrides: &self.config.role_overrides.verifier, timeout: self.config.engine.model_timeout, }, - budget, - total, + spend.budget, + spend.total, ) .await { @@ -136,6 +176,7 @@ impl<'a> Pipeline<'a> { } None => { self.warn_verifier_fallback( + degradation, "the verifier's response did not follow the verdict protocol", ); Ok(heuristic_fallback(inputs)) @@ -143,7 +184,7 @@ impl<'a> Pipeline<'a> { }, Err(RawCallError::Budget(abort)) => Err(abort), Err(RawCallError::Provider | RawCallError::Timeout) => { - self.warn_verifier_fallback("the verifier call failed or timed out"); + self.warn_verifier_fallback(degradation, "the verifier call failed or timed out"); Ok(heuristic_fallback(inputs)) } } @@ -160,25 +201,40 @@ impl<'a> Pipeline<'a> { /// *checks* also call and which must stay silent (its doc contract). fn warn_verifier_caveat(&self, resolved: &ResolvedRole<'_>) { if let Some(caveat) = &resolved.caveat - && !self.verifier_caveat_warned.swap(true, Ordering::Relaxed) + && !self + .verifier_notices + .caveat_warned + .swap(true, Ordering::Relaxed) { self.warn(caveat.clone()); } } - /// Surface a verdict's degradation to the deterministic heuristic — once - /// per run, like the caveat above, because the escalation loop can hit - /// the same dead verifier several times and the transcript needs the fact, - /// not an echo. The ladder rung (`HeuristicFallback`) records *that* it - /// happened on every round either way; this is the prose account of *why*, - /// which used to be silent (a configured-on pipeline must never degrade - /// without saying so and naming a way out). - fn warn_verifier_fallback(&self, why: &str) { - if !self.verifier_fallback_warned.swap(true, Ordering::Relaxed) { + /// Surface a verdict's degradation to the deterministic heuristic — the + /// prose warning once per run, like the caveat above, because the + /// escalation loop can hit the same dead verifier several times and the + /// transcript needs the fact, not an echo; and the structured + /// [`ProofStep::VerdictDegraded`] fact once per candidate, because the + /// run-wide warning cannot say *which* of a fan-out's candidates the + /// heuristic judged (#1787). The ladder rung (`HeuristicFallback`) + /// records *that* it happened on every round either way. + fn warn_verifier_fallback(&self, degradation: &mut VerdictDegradation, why: &str) { + if !self + .verifier_notices + .fallback_warned + .swap(true, Ordering::Relaxed) + { self.warn(format!( "the verifier could not render a model verdict — {why}; this round's verdict \ falls back to a deterministic heuristic" )); } + if !degradation.recorded { + degradation.recorded = true; + self.emit_proof(ProofStep::VerdictDegraded { + candidate: degradation.candidate, + reason: why.to_string(), + }); + } } } diff --git a/crates/stella-pipeline/src/replay.rs b/crates/stella-pipeline/src/replay.rs index 303138256..1585fbc28 100644 --- a/crates/stella-pipeline/src/replay.rs +++ b/crates/stella-pipeline/src/replay.rs @@ -659,6 +659,11 @@ pub fn event_signature(event: &AgentEvent) -> String { ProofStep::Oracle { passed, tree, .. } => { format!("proof:oracle:{tree:?}:{passed}") } + // The reason is prose about the outage, not the shape of the + // proof; which candidate degraded is structural. + ProofStep::VerdictDegraded { candidate, .. } => { + format!("proof:verdict_degraded:{candidate}") + } } } AgentEvent::ToolStart { call } => format!("tool_start:{}", call.name), diff --git a/crates/stella-protocol/src/event.rs b/crates/stella-protocol/src/event.rs index 633c29467..e2a31824d 100644 --- a/crates/stella-protocol/src/event.rs +++ b/crates/stella-protocol/src/event.rs @@ -84,6 +84,9 @@ use crate::context_event::CompiledContextFrameBuilt; // against. Re-exported rather than imported so `event::LadderSnapshot` — the // path these types had before the move — still resolves for every reader. pub use crate::ladder::{LadderRung, LadderSnapshot, OracleObservation, ProofTree}; +// The proof-step vocabulary moved to `crate::proof` the same way (#1787), with +// the same contract: `event::ProofStep` still resolves for every reader. +pub use crate::proof::ProofStep; use crate::subagent_event::SubAgentPhase; use crate::tool::{ToolCall, ToolOutput}; @@ -1227,104 +1230,6 @@ pub struct VerdictEvidence { pub ladder: Option>, } -/// One step of the proof a turn builds for its own work, in the order the -/// pipeline makes the observation. Carried by [`AgentEvent::Proof`]. -/// -/// Additive in one direction only: an older reader that does not know the -/// `proof` type tag preserves the whole event via [`AgentEvent::Unknown`], -/// but a reader that knows `Proof` and meets a future `kind` fails the whole -/// event — this nested enum is closed, with no `Unknown` step (see the -/// module docs on nested vocabularies). -#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] -#[serde(tag = "kind", rename_all = "snake_case")] -pub enum ProofStep { - /// What assurance this turn is going to buy, stated by triage **before** - /// any of it happens. - /// - /// Emitted first, and the reason the rail can be honest at all. Every - /// other step reports something that *did* happen, so a turn where the - /// answer is "we decided not to" produced no steps and left the surface - /// with nothing to say — which is exactly the case that dominates in - /// practice. A declared plan turns that silence into a statement: the - /// witness row reads "waived by triage" from the first second of the - /// turn instead of implying a test is still coming. - Assurance { - /// Whether an independently authored witness test was called for. - witness: bool, - /// Whether a model verifier was called for on inconclusive evidence. - /// - /// Aliased: this field shipped as `judge`, and every recorded session - /// and golden fixture spells it that way. Renaming it without the - /// alias makes those streams unparseable — which is exactly what the - /// golden fixtures caught. - #[serde(alias = "judge")] - verifier: bool, - }, - /// The warrant read the diff and answered "does this change need a test". - /// Emitted once per candidate, before any witness is bought — a change - /// with nothing to prove is a *stated* outcome here, never silence. - Warrant { - required: bool, - /// The stated reason when no test is warranted; `None` when one is. - #[serde(default, skip_serializing_if = "Option::is_none")] - reason: Option, - /// Size of the change the answer was read from. - diff_lines: u32, - }, - /// An independent model authored the failing witness, and its bytes are - /// pinned: any later change to `path` fails the candidate closed. - WitnessAuthored { - path: String, - /// The command that arms the flip oracle. - command: String, - /// The accepted artifact's fingerprint — what tamper exclusion compares - /// against for the rest of the run. - fingerprint: String, - }, - /// A warranted witness could not be *produced* (no independent author, an - /// author that got stuck, a failed graft). The work stands; it is simply - /// unproven, and saying so is the point. - WitnessUnavailable { reason: String }, - /// **Every** evidence channel was blind, so the ladder abstained rather - /// than judging: no flip oracle, no test result, a working tree the diff - /// probe could not read, and no recorded file change. - /// - /// Distinct from a failing verdict, and the distinction is the point. The - /// turn this exists for ended with a verifier asserting a file "likely does - /// not exist" while the file sat in the container — a claim about the - /// *instruments* delivered as a claim about the *work* (#973). Without a - /// step of its own, an abstention reaches the rail as `✓ passed · model - /// verifier`, which is the same silent outcome in the other direction. - VerificationUnavailable { reason: String }, - /// The flip oracle observed one run of the tracked command against one - /// tree. A fail in `Baseline` followed by a pass in `Candidate` is the - /// flip; anything else is not. - /// - /// `run`/`runs_required`/`seed` are the witness surface's replay facts - /// (D6) — which candidate replay this observation is, how many the flip - /// requires, and the deterministic seed the replay pinned. All additive - /// (`serde(default)`): observations recorded before they existed parse - /// with each absent, and an emitter that has no replay discipline (a - /// single-run oracle) simply omits them. - Oracle { - command: String, - passed: bool, - tree: ProofTree, - /// Which candidate replay this observation is (1-based). `None` on - /// baseline runs and on single-run oracles. - #[serde(default, skip_serializing_if = "Option::is_none")] - run: Option, - /// How many passing candidate replays the flip requires, when the - /// oracle runs more than one. - #[serde(default, skip_serializing_if = "Option::is_none")] - runs_required: Option, - /// The deterministic seed the replay pinned, when one was pinned. - #[serde(default, skip_serializing_if = "Option::is_none")] - seed: Option, - }, -} - /// What a `ScopeReview` gate presents for approval before a large plan /// executes (L-E5). /// diff --git a/crates/stella-protocol/src/lib.rs b/crates/stella-protocol/src/lib.rs index 754c5d527..a65323655 100644 --- a/crates/stella-protocol/src/lib.rs +++ b/crates/stella-protocol/src/lib.rs @@ -56,6 +56,7 @@ pub mod context_event; pub mod error; pub mod event; pub mod ladder; +pub mod proof; pub mod provider; pub mod receipt; pub mod role; diff --git a/crates/stella-protocol/src/proof.rs b/crates/stella-protocol/src/proof.rs new file mode 100644 index 000000000..f908ce6f1 --- /dev/null +++ b/crates/stella-protocol/src/proof.rs @@ -0,0 +1,154 @@ +// SPDX-License-Identifier: AGPL-3.0-only +// Copyright (c) 2026 Oxagen, Inc. Commercial licensing: licensing@oxagen.sh + +//! The proof rail's wire vocabulary: the steps a turn emits as it buys +//! assurance for its own work. +//! +//! Split out of [`crate::event`] (which carries the proof *event*) the same +//! way [`crate::ladder`] was: the step vocabulary grows on its own schedule — +//! [`ProofStep::VerdictDegraded`] joined for #1787 — while the event envelope +//! does not. Re-exported from the crate root and from [`crate::event`], so +//! `stella_protocol::ProofStep` and `event::ProofStep` are unchanged. + +use serde::{Deserialize, Serialize}; + +use crate::ladder::ProofTree; + +/// One step of the proof a turn builds for its own work, in the order the +/// pipeline makes the observation. Carried by [`crate::event::AgentEvent::Proof`]. +/// +/// Additive in one direction only: an older reader that does not know the +/// `proof` type tag preserves the whole event via +/// [`crate::event::AgentEvent::Unknown`], but a reader that knows `Proof` and +/// meets a future `kind` fails the whole event — this nested enum is closed, +/// with no `Unknown` step (see [`crate::event`]'s module docs on nested +/// vocabularies). +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(feature = "schema", derive(schemars::JsonSchema))] +#[serde(tag = "kind", rename_all = "snake_case")] +pub enum ProofStep { + /// What assurance this turn is going to buy, stated by triage **before** + /// any of it happens. + /// + /// Emitted first, and the reason the rail can be honest at all. Every + /// other step reports something that *did* happen, so a turn where the + /// answer is "we decided not to" produced no steps and left the surface + /// with nothing to say — which is exactly the case that dominates in + /// practice. A declared plan turns that silence into a statement: the + /// witness row reads "waived by triage" from the first second of the + /// turn instead of implying a test is still coming. + Assurance { + /// Whether an independently authored witness test was called for. + witness: bool, + /// Whether a model verifier was called for on inconclusive evidence. + /// + /// Aliased: this field shipped as `judge`, and every recorded session + /// and golden fixture spells it that way. Renaming it without the + /// alias makes those streams unparseable — which is exactly what the + /// golden fixtures caught. + #[serde(alias = "judge")] + verifier: bool, + }, + /// The warrant read the diff and answered "does this change need a test". + /// Emitted once per candidate, before any witness is bought — a change + /// with nothing to prove is a *stated* outcome here, never silence. + Warrant { + required: bool, + /// The stated reason when no test is warranted; `None` when one is. + #[serde(default, skip_serializing_if = "Option::is_none")] + reason: Option, + /// Size of the change the answer was read from. + diff_lines: u32, + }, + /// An independent model authored the failing witness, and its bytes are + /// pinned: any later change to `path` fails the candidate closed. + WitnessAuthored { + path: String, + /// The command that arms the flip oracle. + command: String, + /// The accepted artifact's fingerprint — what tamper exclusion compares + /// against for the rest of the run. + fingerprint: String, + }, + /// A warranted witness could not be *produced* (no independent author, an + /// author that got stuck, a failed graft). The work stands; it is simply + /// unproven, and saying so is the point. + WitnessUnavailable { reason: String }, + /// **Every** evidence channel was blind, so the ladder abstained rather + /// than judging: no flip oracle, no test result, a working tree the diff + /// probe could not read, and no recorded file change. + /// + /// Distinct from a failing verdict, and the distinction is the point. The + /// turn this exists for ended with a verifier asserting a file "likely does + /// not exist" while the file sat in the container — a claim about the + /// *instruments* delivered as a claim about the *work* (#973). Without a + /// step of its own, an abstention reaches the rail as `✓ passed · model + /// verifier`, which is the same silent outcome in the other direction. + VerificationUnavailable { reason: String }, + /// The flip oracle observed one run of the tracked command against one + /// tree. A fail in `Baseline` followed by a pass in `Candidate` is the + /// flip; anything else is not. + /// + /// `run`/`runs_required`/`seed` are the witness surface's replay facts + /// (D6) — which candidate replay this observation is, how many the flip + /// requires, and the deterministic seed the replay pinned. All additive + /// (`serde(default)`): observations recorded before they existed parse + /// with each absent, and an emitter that has no replay discipline (a + /// single-run oracle) simply omits them. + Oracle { + command: String, + passed: bool, + tree: ProofTree, + /// Which candidate replay this observation is (1-based). `None` on + /// baseline runs and on single-run oracles. + #[serde(default, skip_serializing_if = "Option::is_none")] + run: Option, + /// How many passing candidate replays the flip requires, when the + /// oracle runs more than one. + #[serde(default, skip_serializing_if = "Option::is_none")] + runs_required: Option, + /// The deterministic seed the replay pinned, when one was pinned. + #[serde(default, skip_serializing_if = "Option::is_none")] + seed: Option, + }, + /// This candidate's model verdict degraded to the deterministic heuristic + /// ([`crate::LadderRung::HeuristicFallback`]): the verifier role was + /// unresolvable, its response did not follow the verdict protocol, or the + /// call failed outright. + /// + /// Emitted once per candidate, keyed by ordinal, because the once-per-run + /// prose warning cannot say *which* of a best-of-N fan-out's candidates + /// were judged by the heuristic — N candidates degrading used to leave one + /// caveat and no record (#1787). The ladder rung records *that* a round + /// degraded; this step records *whose* verdict and why. + VerdictDegraded { + /// Which candidate degraded (1-based, [`ProofStep::Oracle::run`]'s + /// convention). + candidate: u32, + /// The stated reason a model verdict could not be rendered. + reason: String, + }, +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Invariant 4: the new step round-trips byte-for-byte, and its wire + /// spelling is pinned — a rename of the tag or a field breaks every + /// recorded stream that carries it. + #[test] + fn verdict_degraded_round_trips_and_pins_its_wire_shape() { + let step = ProofStep::VerdictDegraded { + candidate: 2, + reason: "the verifier call failed or timed out".into(), + }; + let json = serde_json::to_string(&step).unwrap(); + assert_eq!( + json, + r#"{"kind":"verdict_degraded","candidate":2,"reason":"the verifier call failed or timed out"}"# + ); + let parsed: ProofStep = serde_json::from_str(&json).unwrap(); + assert_eq!(parsed, step); + } +} diff --git a/crates/stella-protocol/tests/wire_contract.rs b/crates/stella-protocol/tests/wire_contract.rs index 0982330ed..4aa5f02f9 100644 --- a/crates/stella-protocol/tests/wire_contract.rs +++ b/crates/stella-protocol/tests/wire_contract.rs @@ -437,6 +437,10 @@ fn all_proof_steps() -> Vec { runs_required: Some(3), seed: Some(7741), }, + ProofStep::VerdictDegraded { + candidate: 2, + reason: "the verifier call failed or timed out".into(), + }, ] } diff --git a/crates/stella-tui/src/proof.rs b/crates/stella-tui/src/proof.rs index 3546708a9..ee7616c28 100644 --- a/crates/stella-tui/src/proof.rs +++ b/crates/stella-tui/src/proof.rs @@ -228,6 +228,11 @@ impl ProofState { } } } + // Per-candidate provenance for the traces view and replay (#1787); + // the rail's verdict row already states the *winning* verdict's + // heuristic degradation via its evidence summary, so no row folds + // from it here. + ProofStep::VerdictDegraded { .. } => {} } } @@ -639,6 +644,9 @@ pub(crate) fn proof_trace(step: &stella_protocol::ProofStep) -> String { ProofTree::Candidate => "new", } ), + ProofStep::VerdictDegraded { candidate, reason } => { + format!("verdict degraded (candidate {candidate}): {reason}") + } } } From 2b323f47aaa757cfb69e9f0cc2c335c1ae468f58 Mon Sep 17 00:00:00 2001 From: Mac Anderson Date: Thu, 6 Aug 2026 17:54:59 -0700 Subject: [PATCH 2/2] test(stella-pipeline): witness the per-candidate degradation facts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A two-candidate fan-out with a tokenless verifier records one VerdictDegraded fact per candidate (ordinals 1 and 2) beside the single once-per-run transcript warning, and a candidate that degrades on both of its rounds records exactly one fact — per candidate, never per round. CandidateSlot groups the ordinal with its workspace so the fan-out driver stays under the argument-count lint structurally rather than by allow. Regenerates docs/wire for the new ProofStep variant. Refs #1787 --- .../src/pipeline/fanout_stage.rs | 26 ++- .../pipeline/tests/management_accounting.rs | 10 +- .../pipeline/tests/verification_hardening.rs | 167 ++++++++++++++++++ docs/wire/agentevent.d.ts | 22 ++- docs/wire/agentevent.schema.json | 27 ++- docs/wire/serveframe.d.ts | 22 ++- docs/wire/serveframe.schema.json | 27 ++- 7 files changed, 279 insertions(+), 22 deletions(-) diff --git a/crates/stella-pipeline/src/pipeline/fanout_stage.rs b/crates/stella-pipeline/src/pipeline/fanout_stage.rs index f8a13c91c..e378d4b75 100644 --- a/crates/stella-pipeline/src/pipeline/fanout_stage.rs +++ b/crates/stella-pipeline/src/pipeline/fanout_stage.rs @@ -73,6 +73,14 @@ struct FinishedCandidate { cost_usd: f64, } +/// One candidate's identity in the fan-out: its 1-based ordinal — the number +/// every candidate-facing surface speaks (`candidate_start_notice`, +/// [`ProofStep::VerdictDegraded`]) — and the isolated workspace it runs in. +struct CandidateSlot<'w> { + ordinal: u32, + ws: &'w dyn CandidateWorkspace, +} + /// A [`CandidateWorkspacePort`] that lets exactly one `create` run at a time. /// /// Wrapping the port rather than serializing at the call sites is what makes @@ -199,15 +207,15 @@ impl<'a> Pipeline<'a> { let mut cost_usd = 0.0; let result = self .run_isolated_candidate( - // 1-based, the ordinal every candidate-facing - // surface speaks (`candidate_start_notice`, - // `ProofStep::VerdictDegraded`). `n` is a `u32` - // config knob, so the index always fits. - index as u32 + 1, + CandidateSlot { + // `n` is a `u32` config knob, so the index + // always fits. + ordinal: index as u32 + 1, + ws, + }, frame, worker, authoring, - ws, fan, &mut Spend { budget: &mut allowance, @@ -251,14 +259,14 @@ impl<'a> Pipeline<'a> { /// than in a loop iteration shared with nobody. async fn run_isolated_candidate( &self, - candidate: u32, + slot: CandidateSlot<'_>, frame: TaskFrame<'_>, worker: &ResolvedRole<'_>, authoring: Option>, - ws: &dyn CandidateWorkspace, fan: Option<&SteeringFanOut<'_>>, spend: &mut Spend<'_>, ) -> CandidateResult { + let CandidateSlot { ordinal, ws } = slot; let bound_hook_runner = self.hooks.map(|(_, runner)| BoundHookRunner { inner: runner, cwd: ws.root(), @@ -292,7 +300,7 @@ impl<'a> Pipeline<'a> { if let Some(view) = view.as_ref() { engine = engine.with_steering(view); } - self.run_candidate(candidate, frame, authoring, &engine, surface, spend) + self.run_candidate(ordinal, frame, authoring, &engine, surface, spend) .await } } diff --git a/crates/stella-pipeline/src/pipeline/tests/management_accounting.rs b/crates/stella-pipeline/src/pipeline/tests/management_accounting.rs index be1cee08f..8b1219a63 100644 --- a/crates/stella-pipeline/src/pipeline/tests/management_accounting.rs +++ b/crates/stella-pipeline/src/pipeline/tests/management_accounting.rs @@ -266,7 +266,15 @@ async fn late_verdict_is_abandoned_and_falls_back_to_the_heuristic() { }; let verdict = pipeline - .verifier(prompt, &inputs, &mut budget, &mut total) + .verifier( + &mut super::super::verifier_stage::VerdictDegradation::new(1), + prompt, + &inputs, + &mut Spend { + budget: &mut budget, + total: &mut total, + }, + ) .await .expect("a wedged verifier is never a run-ending failure"); diff --git a/crates/stella-pipeline/src/pipeline/tests/verification_hardening.rs b/crates/stella-pipeline/src/pipeline/tests/verification_hardening.rs index 19d5ef836..f2c3438fb 100644 --- a/crates/stella-pipeline/src/pipeline/tests/verification_hardening.rs +++ b/crates/stella-pipeline/src/pipeline/tests/verification_hardening.rs @@ -1259,3 +1259,170 @@ async fn a_second_deterministic_failure_fires_guidance_even_when_not_consecutive "the guidance call is an honest Verifier stage in the stream" ); } + +/// #1787 witness, fan-out half: TWO isolated candidates escalate to a +/// verifier whose replies never carry a verdict token, and the stream records +/// one structured [`ProofStep::VerdictDegraded`] fact per candidate — the +/// once-per-run prose warning cannot say which candidates the heuristic +/// judged, and before the fact existed nothing did. +#[tokio::test] +async fn a_two_candidate_fanout_records_which_candidates_degraded() { + // triage, then per candidate one worker turn and one verifier escalation. + // Every post-triage reply is deliberately tokenless: candidates run + // concurrently, so whichever call pops which reply, no verifier can parse + // a verdict out of it — the degradation is scripted independent of + // completion order. + let provider = ScriptedProvider::new(vec![ + text_result("single"), + text_result("worked on it"), + text_result("Here is my assessment of the change."), + text_result("worked on it"), + text_result("Here is my assessment of the change."), + ]); + let log = Arc::new(std::sync::Mutex::new(Vec::new())); + let port = FakeWorkspacePort::new( + vec![ + Ok(FakeWorkspace::new(0, vec![], Ok(vec![]), log.clone())), + Ok(FakeWorkspace::new(1, vec![], Ok(vec![]), log.clone())), + ], + log, + ); + // No test command and no witness author resolvable, so each candidate's + // ladder is inconclusive over its diff and escalates to the verifier. + let config = PipelineConfig { + candidates: Some(2), + max_revisions: 0, + diff_diagnostic: Some(DiagnosticInvocation::GitDiff), + distress_guidance: false, + ..PipelineConfig::default() + }; + + let (outcome, events, _messages) = run_isolated(&provider, &port, config, "Fix the bug").await; + outcome.expect("the run proceeds to a (failed) verdict"); + + let mut degraded: Vec = events + .iter() + .filter_map(|event| match event { + AgentEvent::Proof { + step: ProofStep::VerdictDegraded { candidate, .. }, + } => Some(*candidate), + _ => None, + }) + .collect(); + degraded.sort_unstable(); + assert_eq!( + degraded, + vec![1, 2], + "each candidate's degradation is recorded, keyed by ordinal" + ); + let warnings = events + .iter() + .filter(|event| { + matches!(event, AgentEvent::Error { message, .. } + if message.contains("falls back to a deterministic heuristic")) + }) + .count(); + assert_eq!( + warnings, 1, + "the transcript warning stays once per run; the per-candidate record is the proof step" + ); +} + +/// #1787 witness, dedup half: ONE candidate whose escalation hits the same +/// non-compliant verifier on the first round AND on the revision records its +/// degradation fact exactly once — per candidate, not per round (the ladder +/// rung already counts rounds). +#[tokio::test] +async fn a_candidate_degrading_on_every_round_records_one_fact() { + // triage; worker; tokenless verdict; revision turn; tokenless verdict. + let provider = ScriptedProvider::new(vec![ + text_result("single"), + text_result("done"), + text_result("Here is my assessment of the change."), + text_result("revised"), + text_result("Here is my assessment of the change."), + ]); + let resolver = OneProvider(&provider); + // Red baseline, then a timed-out observation on every round: no flip, no + // touched-test result, a diff — the ladder escalates each time. + let runner = ScriptedRunner::scripted( + vec![TestScript::Fail, TestScript::TimeOut, TestScript::TimeOut], + "@@ -1 +1 @@\n-old\n+new", + ); + let tools = EmptyTools; + let recall = NoContextRecall; + let repo = NoRepoStructure; + let repo_status = SeqRepoStatus::new(vec![vec![], vec![]]); + let approvals = AutoApproveGate; + let sleeper = NoopSleeper; + let router = router(); + let (tx, mut rx) = mpsc::unbounded_channel(); + let config = PipelineConfig { + test_command: Some("cargo test -p x".into()), + diff_diagnostic: Some(DiagnosticInvocation::GitDiff), + max_revisions: 1, + distress_guidance: false, + ..PipelineConfig::default() + }; + let pipeline = Pipeline::new( + PipelinePorts { + router: &router, + providers: &resolver, + tools: &tools, + recall: &recall, + repo: &repo, + repo_status: &repo_status, + touches: &NoFileTouches, + diagnostics: &runner, + tests: &runner, + lint: None, + mutation: None, + coverage: None, + approvals: &approvals, + sleeper: &sleeper, + hooks: None, + candidate_workspaces: None, + mcp_prefetch: None, + steering: None, + }, + tx, + config, + ); + + let mut messages = vec![CompletionMessage::system("sys")]; + let mut budget = BudgetGuard::new(BudgetMode::Off, None, None); + let outcome = pipeline + .run("Fix the failing test", &mut messages, &mut budget) + .await + .expect("the run proceeds to a (failed) verdict"); + assert!( + matches!(outcome.status, PipelineStatus::VerificationFailed { .. }), + "both rounds degraded to the failing heuristic: {:?}", + outcome.status + ); + + // Both scripted verdict calls were really made — without this, a run that + // never reached the second escalation would pass the one-fact assertion + // below for the wrong reason. + assert_eq!( + provider.prompts().len(), + 5, + "triage, worker, verdict, revision, verdict" + ); + + let events = drain(&mut rx); + let facts: Vec = events + .iter() + .filter_map(|event| match event { + AgentEvent::Proof { + step: ProofStep::VerdictDegraded { candidate, .. }, + } => Some(*candidate), + _ => None, + }) + .collect(); + assert_eq!( + facts, + vec![1], + "two degraded rounds, one candidate, one fact" + ); +} diff --git a/docs/wire/agentevent.d.ts b/docs/wire/agentevent.d.ts index 2776d85dc..659a5de1a 100644 --- a/docs/wire/agentevent.d.ts +++ b/docs/wire/agentevent.d.ts @@ -665,13 +665,14 @@ export type PrStatus = "draft" | "open" | "merged" | "closed"; /** * One step of the proof a turn builds for its own work, in the order the - * pipeline makes the observation. Carried by [`AgentEvent::Proof`]. + * pipeline makes the observation. Carried by [`crate::event::AgentEvent::Proof`]. * * Additive in one direction only: an older reader that does not know the - * `proof` type tag preserves the whole event via [`AgentEvent::Unknown`], - * but a reader that knows `Proof` and meets a future `kind` fails the whole - * event — this nested enum is closed, with no `Unknown` step (see the - * module docs on nested vocabularies). + * `proof` type tag preserves the whole event via + * [`crate::event::AgentEvent::Unknown`], but a reader that knows `Proof` and + * meets a future `kind` fails the whole event — this nested enum is closed, + * with no `Unknown` step (see [`crate::event`]'s module docs on nested + * vocabularies). */ export type ProofStep = { kind: "assurance"; @@ -736,6 +737,17 @@ export type ProofStep = { */ seed?: number | null; tree: ProofTree; +} | { + /** + * Which candidate degraded (1-based, [`ProofStep::Oracle::run`]'s + * convention). + */ + candidate: number; + kind: "verdict_degraded"; + /** + * The stated reason a model verdict could not be rendered. + */ + reason: string; }; /** diff --git a/docs/wire/agentevent.schema.json b/docs/wire/agentevent.schema.json index 94cb56153..1984fc9ff 100644 --- a/docs/wire/agentevent.schema.json +++ b/docs/wire/agentevent.schema.json @@ -954,7 +954,7 @@ ] }, "ProofStep": { - "description": "One step of the proof a turn builds for its own work, in the order the\npipeline makes the observation. Carried by [`AgentEvent::Proof`].\n\nAdditive in one direction only: an older reader that does not know the\n`proof` type tag preserves the whole event via [`AgentEvent::Unknown`],\nbut a reader that knows `Proof` and meets a future `kind` fails the whole\nevent — this nested enum is closed, with no `Unknown` step (see the\nmodule docs on nested vocabularies).", + "description": "One step of the proof a turn builds for its own work, in the order the\npipeline makes the observation. Carried by [`crate::event::AgentEvent::Proof`].\n\nAdditive in one direction only: an older reader that does not know the\n`proof` type tag preserves the whole event via\n[`crate::event::AgentEvent::Unknown`], but a reader that knows `Proof` and\nmeets a future `kind` fails the whole event — this nested enum is closed,\nwith no `Unknown` step (see [`crate::event`]'s module docs on nested\nvocabularies).", "oneOf": [ { "description": "What assurance this turn is going to buy, stated by triage **before**\nany of it happens.\n\nEmitted first, and the reason the rail can be honest at all. Every\nother step reports something that *did* happen, so a turn where the\nanswer is \"we decided not to\" produced no steps and left the surface\nwith nothing to say — which is exactly the case that dominates in\npractice. A declared plan turns that silence into a statement: the\nwitness row reads \"waived by triage\" from the first second of the\nturn instead of implying a test is still coming.", @@ -1122,6 +1122,31 @@ "tree" ], "type": "object" + }, + { + "description": "This candidate's model verdict degraded to the deterministic heuristic\n([`crate::LadderRung::HeuristicFallback`]): the verifier role was\nunresolvable, its response did not follow the verdict protocol, or the\ncall failed outright.\n\nEmitted once per candidate, keyed by ordinal, because the once-per-run\nprose warning cannot say *which* of a best-of-N fan-out's candidates\nwere judged by the heuristic — N candidates degrading used to leave one\ncaveat and no record (#1787). The ladder rung records *that* a round\ndegraded; this step records *whose* verdict and why.", + "properties": { + "candidate": { + "description": "Which candidate degraded (1-based, [`ProofStep::Oracle::run`]'s\nconvention).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "verdict_degraded", + "type": "string" + }, + "reason": { + "description": "The stated reason a model verdict could not be rendered.", + "type": "string" + } + }, + "required": [ + "kind", + "candidate", + "reason" + ], + "type": "object" } ] }, diff --git a/docs/wire/serveframe.d.ts b/docs/wire/serveframe.d.ts index 301b2436f..111ccf436 100644 --- a/docs/wire/serveframe.d.ts +++ b/docs/wire/serveframe.d.ts @@ -1381,13 +1381,14 @@ export type PrStatus = "draft" | "open" | "merged" | "closed"; /** * One step of the proof a turn builds for its own work, in the order the - * pipeline makes the observation. Carried by [`AgentEvent::Proof`]. + * pipeline makes the observation. Carried by [`crate::event::AgentEvent::Proof`]. * * Additive in one direction only: an older reader that does not know the - * `proof` type tag preserves the whole event via [`AgentEvent::Unknown`], - * but a reader that knows `Proof` and meets a future `kind` fails the whole - * event — this nested enum is closed, with no `Unknown` step (see the - * module docs on nested vocabularies). + * `proof` type tag preserves the whole event via + * [`crate::event::AgentEvent::Unknown`], but a reader that knows `Proof` and + * meets a future `kind` fails the whole event — this nested enum is closed, + * with no `Unknown` step (see [`crate::event`]'s module docs on nested + * vocabularies). */ export type ProofStep = { kind: "assurance"; @@ -1452,6 +1453,17 @@ export type ProofStep = { */ seed?: number | null; tree: ProofTree; +} | { + /** + * Which candidate degraded (1-based, [`ProofStep::Oracle::run`]'s + * convention). + */ + candidate: number; + kind: "verdict_degraded"; + /** + * The stated reason a model verdict could not be rendered. + */ + reason: string; }; /** diff --git a/docs/wire/serveframe.schema.json b/docs/wire/serveframe.schema.json index f0057d01c..a900f1e69 100644 --- a/docs/wire/serveframe.schema.json +++ b/docs/wire/serveframe.schema.json @@ -2500,7 +2500,7 @@ ] }, "ProofStep": { - "description": "One step of the proof a turn builds for its own work, in the order the\npipeline makes the observation. Carried by [`AgentEvent::Proof`].\n\nAdditive in one direction only: an older reader that does not know the\n`proof` type tag preserves the whole event via [`AgentEvent::Unknown`],\nbut a reader that knows `Proof` and meets a future `kind` fails the whole\nevent — this nested enum is closed, with no `Unknown` step (see the\nmodule docs on nested vocabularies).", + "description": "One step of the proof a turn builds for its own work, in the order the\npipeline makes the observation. Carried by [`crate::event::AgentEvent::Proof`].\n\nAdditive in one direction only: an older reader that does not know the\n`proof` type tag preserves the whole event via\n[`crate::event::AgentEvent::Unknown`], but a reader that knows `Proof` and\nmeets a future `kind` fails the whole event — this nested enum is closed,\nwith no `Unknown` step (see [`crate::event`]'s module docs on nested\nvocabularies).", "oneOf": [ { "description": "What assurance this turn is going to buy, stated by triage **before**\nany of it happens.\n\nEmitted first, and the reason the rail can be honest at all. Every\nother step reports something that *did* happen, so a turn where the\nanswer is \"we decided not to\" produced no steps and left the surface\nwith nothing to say — which is exactly the case that dominates in\npractice. A declared plan turns that silence into a statement: the\nwitness row reads \"waived by triage\" from the first second of the\nturn instead of implying a test is still coming.", @@ -2668,6 +2668,31 @@ "tree" ], "type": "object" + }, + { + "description": "This candidate's model verdict degraded to the deterministic heuristic\n([`crate::LadderRung::HeuristicFallback`]): the verifier role was\nunresolvable, its response did not follow the verdict protocol, or the\ncall failed outright.\n\nEmitted once per candidate, keyed by ordinal, because the once-per-run\nprose warning cannot say *which* of a best-of-N fan-out's candidates\nwere judged by the heuristic — N candidates degrading used to leave one\ncaveat and no record (#1787). The ladder rung records *that* a round\ndegraded; this step records *whose* verdict and why.", + "properties": { + "candidate": { + "description": "Which candidate degraded (1-based, [`ProofStep::Oracle::run`]'s\nconvention).", + "format": "uint32", + "minimum": 0, + "type": "integer" + }, + "kind": { + "const": "verdict_degraded", + "type": "string" + }, + "reason": { + "description": "The stated reason a model verdict could not be rendered.", + "type": "string" + } + }, + "required": [ + "kind", + "candidate", + "reason" + ], + "type": "object" } ] },