diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d48e65c..03fd4aa 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: - uses: actions/checkout@v6 with: repository: Nanako0129/TokenBar - ref: 4dfed5ffab26e2707a8cd82ee99864520b75892b + ref: 55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218 path: tokenbar-mac - uses: actions/checkout@v6 with: @@ -44,14 +44,16 @@ jobs: cd "$TOKENBAR_MAC_CANONICAL" TZ=Asia/Taipei swift run crosscheck-harness \ "$TOKENBAR_WINDOWS/crosscheck/fixtures" \ - "$TOKENBAR_WINDOWS/crosscheck/swift-out" + "$TOKENBAR_WINDOWS/crosscheck/swift-out" \ + -AppleLanguages "(en)" ) ( cd "$TOKENBAR_MAC_CANONICAL" TZ=Asia/Taipei swift run crosscheck-harness \ "$TOKENBAR_WINDOWS/crosscheck/fixtures" \ "$TOKENBAR_WINDOWS/crosscheck/swift-out" \ - provider-quota-pace-v3 + provider-quota-pace-v3 \ + -AppleLanguages "(en)" ) - uses: actions/upload-artifact@v4 with: diff --git a/README.md b/README.md index fde22d0..94d7819 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ a native WinUI 3 shell. | Layer | Path | Notes | |---|---|---| | Rust core | `crates/tb_core_ffi` + `vendor/tokscale-core` | Copied from the macOS repo (single sync source — see `vendor/tokscale-core/SYNC.md`). C ABI, JSON envelope, built as `cdylib` for P/Invoke | -| C ABI contract | `include/ctb.h` | 10 entry points, `{"ok":true,"data":…}` / `{"ok":false,"err":…}` | +| C ABI contract | `include/ctb.h` | 11 entry points, `{"ok":true,"data":…}` / `{"ok":false,"err":…}` | | Interop | `src/TokenBar.Interop` | `net10.0`, platform-neutral — P/Invoke facade + envelope decode | | Logic | `src/TokenBar.Core` | `net10.0`, platform-neutral — C# port of the macOS `TokenBarCore` | | Shell | `src/TokenBar.App` | WinUI 3, unpackaged. Windows-only build (not in the slnx): `dotnet build src/TokenBar.App/TokenBar.App.csproj -c Release -p:Platform=x64 -p:RuntimeIdentifier=win-x64` | diff --git a/crates/tb_core_ffi/src/filter_parity_probe.rs b/crates/tb_core_ffi/src/filter_parity_probe.rs new file mode 100644 index 0000000..9437c53 --- /dev/null +++ b/crates/tb_core_ffi/src/filter_parity_probe.rs @@ -0,0 +1,711 @@ +//! Source-generation-aware parity diagnostics for the hourly and Agents +//! report filters. +//! +//! The two reports are deliberately run through one synchronous orchestration +//! seam. Production uses the real local-source token and report functions; +//! tests inject closures so a fixture can mutate a source at an exact stage +//! without sleeping or relying on a racing background writer. + +use serde::Serialize; +use serde_json::Value; + +use crate::{agents_report, hourly_report, usage_graph, LocalSourceContext}; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) enum ProbeStatus { + #[serde(rename = "match")] + Match, + #[serde(rename = "mismatch")] + Mismatch, + #[serde(rename = "sourceChanged")] + SourceChanged, + #[serde(rename = "tokenUnavailable")] + TokenUnavailable, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ReportAggregate { + pub entry_count: i64, + pub input: i64, + pub output: i64, + pub cache_read: i64, + pub cache_write: i64, + pub reasoning: i64, + pub total_tokens: i64, + pub message_count: i64, + pub total_cost: f64, +} + +impl ReportAggregate { + fn delta(unfiltered: &Self, full: &Self) -> Self { + Self { + entry_count: full.entry_count.saturating_sub(unfiltered.entry_count), + input: full.input.saturating_sub(unfiltered.input), + output: full.output.saturating_sub(unfiltered.output), + cache_read: full.cache_read.saturating_sub(unfiltered.cache_read), + cache_write: full.cache_write.saturating_sub(unfiltered.cache_write), + reasoning: full.reasoning.saturating_sub(unfiltered.reasoning), + total_tokens: full.total_tokens.saturating_sub(unfiltered.total_tokens), + message_count: full.message_count.saturating_sub(unfiltered.message_count), + total_cost: full.total_cost - unfiltered.total_cost, + } + } + + fn matches(&self, other: &Self) -> bool { + // Pricing refreshes independently from the local-source generation + // token. Keep cost in the diagnostic delta, but do not turn a price-only + // refresh between scans into a filter mismatch. + self.entry_count == other.entry_count + && self.input == other.input + && self.output == other.output + && self.cache_read == other.cache_read + && self.cache_write == other.cache_write + && self.reasoning == other.reasoning + && self.total_tokens == other.total_tokens + && self.message_count == other.message_count + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct ReportParity { + pub status: ProbeStatus, + // A report can be intentionally skipped after a known source change or a + // failed token probe. Keep the keys present as JSON nulls so the envelope + // remains schema-stable without inventing zero-valued evidence. + pub unfiltered: Option, + pub full: Option, + pub delta: Option, +} + +impl ReportParity { + fn skipped(status: ProbeStatus) -> Self { + Self { + status, + unfiltered: None, + full: None, + delta: None, + } + } + + fn from_reports( + status: ProbeStatus, + unfiltered: ReportAggregate, + full: ReportAggregate, + ) -> Self { + let delta = ReportAggregate::delta(&unfiltered, &full); + Self { + status, + unfiltered: Some(unfiltered), + full: Some(full), + delta: Some(delta), + } + } +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +#[serde(rename_all = "camelCase")] +pub(crate) struct FilterParityPayload { + pub hourly: ReportParity, + pub agents: ReportParity, + pub present_client_count: i64, +} + +type TokenFn<'a> = dyn FnMut(&LocalSourceContext) -> Result + 'a; +type GraphFn<'a> = dyn FnMut(&LocalSourceContext) -> Result + 'a; +type ReportFn<'a> = dyn FnMut(&LocalSourceContext, Option<&[String]>) -> Result + 'a; + +/// Run the production parity probe using one context and a fresh graph. +pub(crate) fn run(context: &LocalSourceContext) -> Result { + let mut token = |context: &LocalSourceContext| { + tokscale_core::local_source_change_token(&context.parse_options(None, None)) + }; + let mut graph = |context: &LocalSourceContext| usage_graph::run(context, ""); + let mut hourly = |context: &LocalSourceContext, clients: Option<&[String]>| { + hourly_report::run(context, "", clients.map(|values| values.to_vec())) + }; + let mut agents = |context: &LocalSourceContext, clients: Option<&[String]>| { + agents_report::run(context, "", clients.map(|values| values.to_vec())) + }; + + run_with(context, &mut token, &mut graph, &mut hourly, &mut agents) +} + +/// Testable orchestration seam. The callbacks are synchronous by design: a +/// test can mutate a recognized temporary source immediately before returning +/// from one stage, making each token boundary deterministic. +pub(crate) fn run_with( + context: &LocalSourceContext, + token: &mut TokenFn<'_>, + graph: &mut GraphFn<'_>, + hourly: &mut ReportFn<'_>, + agents: &mut ReportFn<'_>, +) -> Result { + let token0 = token(context).ok(); + // Never use graph_cached here: the present-client list must come from the + // same fresh graph generation that the first token brackets. + let graph = graph(context)?; + let clients = graph_clients(&graph)?; + let present_client_count = i64::try_from(clients.len()) + .map_err(|_| "present client count exceeds supported range".to_string())?; + let token1 = token(context).ok(); + + if let Some(status) = boundary_status(&[token0, token1]) { + return Ok(FilterParityPayload { + hourly: ReportParity::skipped(status), + agents: ReportParity::skipped(status), + present_client_count, + }); + } + + let hourly_unfiltered = parse_hourly(hourly(context, None)?)?; + let token2 = token(context).ok(); + if let Some(status) = boundary_status(&[token0, token1, token2]) { + return Ok(FilterParityPayload { + hourly: ReportParity { + status, + unfiltered: Some(hourly_unfiltered), + full: None, + delta: None, + }, + agents: ReportParity::skipped(status), + present_client_count, + }); + } + + let hourly_full = parse_hourly(hourly(context, Some(&clients))?)?; + let token3 = token(context).ok(); + if let Some(status) = boundary_status(&[token0, token1, token2, token3]) { + let delta = ReportAggregate::delta(&hourly_unfiltered, &hourly_full); + return Ok(FilterParityPayload { + hourly: ReportParity { + status, + unfiltered: Some(hourly_unfiltered), + full: Some(hourly_full), + delta: Some(delta), + }, + agents: ReportParity::skipped(status), + present_client_count, + }); + } + + let hourly_status = if hourly_unfiltered.matches(&hourly_full) { + ProbeStatus::Match + } else { + ProbeStatus::Mismatch + }; + + let agents_unfiltered = parse_agents(agents(context, None)?)?; + let token4 = token(context).ok(); + if let Some(status) = boundary_status(&[token0, token1, token2, token3, token4]) { + return Ok(FilterParityPayload { + hourly: ReportParity::from_reports(hourly_status, hourly_unfiltered, hourly_full), + agents: ReportParity { + status, + unfiltered: Some(agents_unfiltered), + full: None, + delta: None, + }, + present_client_count, + }); + } + + let agents_full = parse_agents(agents(context, Some(&clients))?)?; + let token5 = token(context).ok(); + let agents_status = boundary_status(&[token0, token1, token2, token3, token4, token5]) + .unwrap_or_else(|| { + if agents_unfiltered.matches(&agents_full) { + ProbeStatus::Match + } else { + ProbeStatus::Mismatch + } + }); + + Ok(FilterParityPayload { + hourly: ReportParity::from_reports(hourly_status, hourly_unfiltered, hourly_full), + agents: ReportParity::from_reports(agents_status, agents_unfiltered, agents_full), + present_client_count, + }) +} + +fn boundary_status(tokens: &[Option]) -> Option { + // A known adjacent difference is stronger evidence than a later missing + // token. This keeps a real source mutation classified as sourceChanged + // even if a subsequent probe also fails. + if tokens + .windows(2) + .any(|window| matches!((window[0], window[1]), (Some(left), Some(right)) if left != right)) + { + Some(ProbeStatus::SourceChanged) + } else if tokens.iter().any(Option::is_none) { + Some(ProbeStatus::TokenUnavailable) + } else { + None + } +} + +fn graph_clients(graph: &Value) -> Result, String> { + let clients = graph + .get("summary") + .and_then(|summary| summary.get("clients")) + .and_then(Value::as_array) + .ok_or_else(|| "graph summary clients are missing".to_string())?; + clients + .iter() + .map(|client| { + client + .as_str() + .map(str::to_owned) + .ok_or_else(|| "graph summary contains an invalid client".to_string()) + }) + .collect() +} + +fn required_i64(value: &Value, field: &str) -> Result { + value + .get(field) + .and_then(Value::as_i64) + .ok_or_else(|| format!("report field {field} is missing or invalid")) +} + +fn required_cost(value: &Value) -> Result { + let cost = value + .get("totalCost") + .and_then(Value::as_f64) + .ok_or_else(|| "report total cost is missing or invalid".to_string())?; + if cost.is_finite() { + Ok(cost) + } else { + Err("report total cost is not finite".to_string()) + } +} + +fn aggregate_entries(value: &Value, message_field: &str) -> Result<(ReportAggregate, i64), String> { + let entries = value + .get("entries") + .and_then(Value::as_array) + .ok_or_else(|| "report entries are missing or invalid".to_string())?; + let entry_count = i64::try_from(entries.len()) + .map_err(|_| "report entry count exceeds supported range".to_string())?; + let mut aggregate = ReportAggregate { + entry_count, + input: 0, + output: 0, + cache_read: 0, + cache_write: 0, + reasoning: 0, + total_tokens: 0, + message_count: 0, + total_cost: required_cost(value)?, + }; + + for entry in entries { + aggregate.input = aggregate + .input + .saturating_add(required_i64(entry, "input")?); + aggregate.output = aggregate + .output + .saturating_add(required_i64(entry, "output")?); + aggregate.cache_read = aggregate + .cache_read + .saturating_add(required_i64(entry, "cacheRead")?); + aggregate.cache_write = aggregate + .cache_write + .saturating_add(required_i64(entry, "cacheWrite")?); + aggregate.reasoning = aggregate + .reasoning + .saturating_add(required_i64(entry, "reasoning")?); + aggregate.message_count = aggregate + .message_count + .saturating_add(required_i64(entry, message_field)?); + } + aggregate.total_tokens = aggregate + .input + .saturating_add(aggregate.output) + .saturating_add(aggregate.cache_read) + .saturating_add(aggregate.cache_write) + .saturating_add(aggregate.reasoning); + + Ok((aggregate, entry_count)) +} + +fn parse_hourly(value: Value) -> Result { + Ok(aggregate_entries(&value, "messageCount")?.0) +} + +fn parse_agents(value: Value) -> Result { + let (mut aggregate, _) = aggregate_entries(&value, "messages")?; + // The report's top-level message total is the authoritative aggregate used + // by the existing Swift DTO. Requiring it also catches a mapper that drops + // an entry's message count while still serializing valid rows. + aggregate.message_count = required_i64(&value, "totalMessages")?; + Ok(aggregate) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::fs::OpenOptions; + use std::io::Write; + use std::path::PathBuf; + use std::rc::Rc; + + fn context() -> LocalSourceContext { + LocalSourceContext { + home_dir: Some(PathBuf::from("fixture-home")), + } + } + + fn graph() -> Value { + serde_json::json!({"summary": {"clients": ["claude", "synthetic"]}}) + } + + fn hourly(input: i64, messages: i64, cost: f64) -> Value { + serde_json::json!({ + "entries": [{ + "input": input, "output": 20, "cacheRead": 3, + "cacheWrite": 4, "reasoning": 5, "messageCount": messages + }], + "totalCost": cost + }) + } + + fn agents(input: i64, messages: i64, cost: f64) -> Value { + serde_json::json!({ + "entries": [{ + "input": input, "output": 20, "cacheRead": 3, + "cacheWrite": 4, "reasoning": 5, "messages": messages + }], + "totalCost": cost, "totalMessages": messages + }) + } + + fn run_fixture( + tokens: Vec>, + hourly_values: Vec>, + agent_values: Vec>, + ) -> FilterParityPayload { + let mut tokens = tokens.into_iter(); + let mut hourly_values = hourly_values.into_iter(); + let mut agent_values = agent_values.into_iter(); + let mut token = |_context: &LocalSourceContext| tokens.next().unwrap(); + let mut graph_fn = |_context: &LocalSourceContext| Ok(graph()); + let mut hourly_fn = |_context: &LocalSourceContext, _clients: Option<&[String]>| { + hourly_values.next().unwrap() + }; + let mut agents_fn = |_context: &LocalSourceContext, _clients: Option<&[String]>| { + agent_values.next().unwrap() + }; + run_with( + &context(), + &mut token, + &mut graph_fn, + &mut hourly_fn, + &mut agents_fn, + ) + .unwrap() + } + + #[test] + fn stable_equal_reports_match_and_include_delta() { + let result = run_fixture( + vec![Ok(1); 6], + vec![Ok(hourly(10, 2, 0.25)), Ok(hourly(10, 2, 0.25))], + vec![Ok(agents(10, 2, 0.25)), Ok(agents(10, 2, 0.25))], + ); + assert_eq!(result.hourly.status, ProbeStatus::Match); + assert_eq!(result.agents.status, ProbeStatus::Match); + assert_eq!(result.hourly.delta.unwrap().total_tokens, 0); + assert_eq!(result.agents.delta.unwrap().message_count, 0); + } + + #[test] + fn stable_different_reports_are_mismatch() { + let result = run_fixture( + vec![Ok(1); 6], + vec![Ok(hourly(10, 2, 0.25)), Ok(hourly(11, 2, 0.25))], + vec![Ok(agents(10, 2, 0.25)), Ok(agents(10, 2, 0.25))], + ); + assert_eq!(result.hourly.status, ProbeStatus::Mismatch); + assert_eq!(result.agents.status, ProbeStatus::Match); + assert_eq!(result.hourly.delta.unwrap().input, 1); + } + + #[test] + fn stable_source_ignores_independent_pricing_refresh() { + let result = run_fixture( + vec![Ok(1); 6], + vec![Ok(hourly(10, 2, 0.25)), Ok(hourly(10, 2, 0.75))], + vec![Ok(agents(10, 2, 0.25)), Ok(agents(10, 2, 1.25))], + ); + assert_eq!(result.hourly.status, ProbeStatus::Match); + assert_eq!(result.agents.status, ProbeStatus::Match); + assert_eq!(result.hourly.delta.unwrap().total_cost, 0.5); + assert_eq!(result.agents.delta.unwrap().total_cost, 1.0); + } + + #[test] + fn every_relevant_boundary_is_source_changed_and_later_scans_short_circuit() { + for changed_at in 1..=5 { + let mut tokens: Vec> = vec![Ok(1); 6]; + // A single adjacent boundary differs while the remaining values + // stay equal, making the expected source generation unambiguous. + tokens[changed_at] = Ok(100 + changed_at as u64); + let calls = Rc::new(RefCell::new(Vec::new())); + let token_calls = Rc::clone(&calls); + let token_index = Rc::new(RefCell::new(0usize)); + let token_counter = Rc::clone(&token_index); + let mut token = { + let mut values = tokens.into_iter(); + move |_context: &LocalSourceContext| { + let index = *token_counter.borrow(); + *token_counter.borrow_mut() += 1; + token_calls.borrow_mut().push(format!("token{index}")); + values.next().unwrap() + } + }; + let graph_calls = Rc::clone(&calls); + let mut graph_fn = |_context: &LocalSourceContext| { + graph_calls.borrow_mut().push("graph".to_string()); + Ok(graph()) + }; + let hourly_calls = Rc::clone(&calls); + let mut hourly_fn = |_context: &LocalSourceContext, clients: Option<&[String]>| { + hourly_calls.borrow_mut().push(if clients.is_some() { + "hourly-full".to_string() + } else { + "hourly-nil".to_string() + }); + Ok(hourly(10, 2, 0.25)) + }; + let agents_calls = Rc::clone(&calls); + let mut agents_fn = |_context: &LocalSourceContext, clients: Option<&[String]>| { + agents_calls.borrow_mut().push(if clients.is_some() { + "agents-full".to_string() + } else { + "agents-nil".to_string() + }); + Ok(agents(10, 2, 0.25)) + }; + let result = run_with( + &context(), + &mut token, + &mut graph_fn, + &mut hourly_fn, + &mut agents_fn, + ) + .unwrap(); + assert_eq!( + result.hourly.status, + if changed_at <= 3 { + ProbeStatus::SourceChanged + } else { + ProbeStatus::Match + }, + "boundary {changed_at}" + ); + assert_eq!( + result.agents.status, + ProbeStatus::SourceChanged, + "boundary {changed_at}" + ); + let expected = match changed_at { + 1 => vec!["token0", "graph", "token1"], + 2 => vec!["token0", "graph", "token1", "hourly-nil", "token2"], + 3 => vec![ + "token0", + "graph", + "token1", + "hourly-nil", + "token2", + "hourly-full", + "token3", + ], + 4 => vec![ + "token0", + "graph", + "token1", + "hourly-nil", + "token2", + "hourly-full", + "token3", + "agents-nil", + "token4", + ], + 5 => vec![ + "token0", + "graph", + "token1", + "hourly-nil", + "token2", + "hourly-full", + "token3", + "agents-nil", + "token4", + "agents-full", + "token5", + ], + _ => unreachable!(), + }; + assert_eq!( + calls + .borrow() + .iter() + .map(String::as_str) + .collect::>(), + expected, + "stage call log for boundary {changed_at}" + ); + } + } + + #[test] + fn known_change_wins_over_later_missing_token() { + assert_eq!( + boundary_status(&[Some(1), Some(2), None]), + Some(ProbeStatus::SourceChanged) + ); + assert_eq!( + boundary_status(&[Some(1), None, Some(2)]), + Some(ProbeStatus::TokenUnavailable) + ); + } + + #[test] + fn token_failure_is_token_unavailable_not_outer_failure() { + let result = run_fixture(vec![Err("probe failed".to_string()); 6], vec![], vec![]); + assert_eq!(result.hourly.status, ProbeStatus::TokenUnavailable); + assert_eq!(result.agents.status, ProbeStatus::TokenUnavailable); + } + + #[test] + fn graph_failure_remains_an_outer_error() { + let mut token = |_context: &LocalSourceContext| Ok(1); + let mut graph_fn = |_context: &LocalSourceContext| Err("graph failed".to_string()); + let mut hourly_fn = + |_context: &LocalSourceContext, _clients: Option<&[String]>| Ok(hourly(1, 1, 0.0)); + let mut agents_fn = + |_context: &LocalSourceContext, _clients: Option<&[String]>| Ok(agents(1, 1, 0.0)); + let result = run_with( + &context(), + &mut token, + &mut graph_fn, + &mut hourly_fn, + &mut agents_fn, + ); + assert!(result.is_err()); + } + + #[test] + fn hourly_stays_stable_when_only_the_later_agents_boundary_changes() { + let result = run_fixture( + vec![Ok(1), Ok(1), Ok(1), Ok(1), Ok(1), Ok(2)], + vec![Ok(hourly(10, 2, 0.25)), Ok(hourly(10, 2, 0.25))], + vec![Ok(agents(10, 2, 0.25)), Ok(agents(10, 2, 0.25))], + ); + assert_eq!(result.hourly.status, ProbeStatus::Match); + assert_eq!(result.agents.status, ProbeStatus::SourceChanged); + } + + #[test] + fn controlled_append_between_stages_is_source_changed_and_skips_later_scans() { + let root = std::env::temp_dir().join(format!( + "tokenbar-filter-parity-stage-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let source = root.join(".codex/sessions/session.jsonl"); + std::fs::create_dir_all(source.parent().unwrap()).unwrap(); + std::fs::write(&source, b"seed\n").unwrap(); + let options = tokscale_core::LocalParseOptions { + home_dir: Some(root.to_string_lossy().into_owned()), + use_env_roots: false, + clients: Some(vec!["codex".to_string()]), + ..Default::default() + }; + let mut token = + |_context: &LocalSourceContext| tokscale_core::local_source_change_token(&options); + let calls = Rc::new(RefCell::new(Vec::new())); + let graph_calls = Rc::clone(&calls); + let mut graph_fn = |_context: &LocalSourceContext| { + graph_calls.borrow_mut().push("graph".to_string()); + Ok(graph()) + }; + let hourly_calls = Rc::clone(&calls); + let mut hourly_fn = |_context: &LocalSourceContext, clients: Option<&[String]>| { + hourly_calls.borrow_mut().push(if clients.is_some() { + "hourly-full".to_string() + } else { + "hourly-nil".to_string() + }); + if clients.is_none() { + let mut file = OpenOptions::new().append(true).open(&source).unwrap(); + file.write_all(b"size-changing append\n").unwrap(); + file.flush().unwrap(); + } + Ok(hourly(10, 2, 0.25)) + }; + let agents_calls = Rc::clone(&calls); + let mut agents_fn = |_context: &LocalSourceContext, _clients: Option<&[String]>| { + agents_calls.borrow_mut().push("agents".to_string()); + Ok(agents(10, 2, 0.25)) + }; + + let result = run_with( + &LocalSourceContext { + home_dir: Some(root.clone()), + }, + &mut token, + &mut graph_fn, + &mut hourly_fn, + &mut agents_fn, + ) + .unwrap(); + let _ = std::fs::remove_dir_all(root); + + assert_eq!(result.hourly.status, ProbeStatus::SourceChanged); + assert_eq!(result.agents.status, ProbeStatus::SourceChanged); + assert_eq!( + calls + .borrow() + .iter() + .map(String::as_str) + .collect::>(), + vec!["graph", "hourly-nil"] + ); + } + + #[test] + fn controlled_append_changes_the_real_source_token_by_size() { + let root = std::env::temp_dir().join(format!( + "tokenbar-filter-parity-{}-{}", + std::process::id(), + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_nanos() + )); + let source = root.join(".codex/sessions/session.jsonl"); + std::fs::create_dir_all(source.parent().unwrap()).unwrap(); + std::fs::write(&source, b"seed\n").unwrap(); + let options = tokscale_core::LocalParseOptions { + home_dir: Some(root.to_string_lossy().into_owned()), + use_env_roots: false, + clients: Some(vec!["codex".to_string()]), + ..Default::default() + }; + let before = tokscale_core::local_source_change_token(&options).unwrap(); + let mut file = OpenOptions::new().append(true).open(&source).unwrap(); + file.write_all(b"size-changing append\n").unwrap(); + file.flush().unwrap(); + let after = tokscale_core::local_source_change_token(&options).unwrap(); + let _ = std::fs::remove_dir_all(root); + assert_ne!(before, after); + } +} diff --git a/crates/tb_core_ffi/src/lib.rs b/crates/tb_core_ffi/src/lib.rs index 83c69b7..8c060fe 100644 --- a/crates/tb_core_ffi/src/lib.rs +++ b/crates/tb_core_ffi/src/lib.rs @@ -25,6 +25,7 @@ mod agent_quota_history; mod agent_storage_windows; mod agent_usage; mod agents_report; +mod filter_parity_probe; mod hourly_report; mod model_report; mod opencode_integrations; @@ -466,6 +467,29 @@ pub unsafe extern "C" fn tb_agents_report( }) } +/// Source-generation-aware filter parity diagnostic. The graph is always +/// freshly computed to derive the present-client list; the hourly and Agents +/// reports are then bracketed by one opaque local-source token sequence. The +/// payload intentionally exposes only bounded aggregates and classifications, +/// never source paths or raw local data. +#[no_mangle] +pub extern "C" fn tb_filter_parity_probe() -> *mut c_char { + // Keep this boundary panic-safe without forwarding panic text that could + // contain a private source path, model, or provider value. + LazyLock::force(&RAYON_INIT); + let result = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| { + let context = LocalSourceContext::current(); + filter_parity_probe::run(&context) + })); + match result { + Ok(Ok(payload)) => envelope( + serde_json::to_value(payload) + .map_err(|_| "filter parity probe serialization failed".to_string()), + ), + Ok(Err(_)) | Err(_) => envelope(Err("filter parity probe failed".to_string())), + } +} + /// Live per-(client, agent, model) trace buckets over the trailing /// `window_secs`. Field names are snake_case (`tokens_per_min`), matching the /// Tauri `TraceBucket` serialization the frontend consumes. diff --git a/crosscheck/README.md b/crosscheck/README.md index 2832f3a..06b49e0 100644 --- a/crosscheck/README.md +++ b/crosscheck/README.md @@ -13,7 +13,7 @@ diagnostic only because serializer formatting and line endings differ by host. | Surface | Canonical source | |---|---| | Legacy `usage-pace.json` and `format.json` | macOS commit `2ed256ee` | -| Provider quota pace v3 | final merged Native PR #102 commit `4dfed5ffab26e2707a8cd82ee99864520b75892b` | +| Provider quota pace v3 | issue-107 Native baseline `55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218` | Run the Swift side from a clean archive or worktree at the exact tested provider-v3 merged SHA. Do not use an unrelated dirty macOS checkout as the @@ -166,8 +166,8 @@ projection differences. ## Running -Set `TOKENBAR_MAC_CANONICAL` to a clean archive or worktree at final merged -Native PR #102 commit `4dfed5ffab26e2707a8cd82ee99864520b75892b`. +Set `TOKENBAR_MAC_CANONICAL` to a clean archive or worktree at the issue-107 +Native baseline `55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218`. `Package.swift` links `target/release/libtb_core_ffi.a` by a path relative to the canonical macOS repo root. Build the Rust static library first, and run the @@ -176,7 +176,7 @@ unrelated checkout's `target/` artifact. ```bash export TOKENBAR_WINDOWS="$(pwd)" -export TOKENBAR_MAC_CANONICAL="${TMPDIR:-/tmp}/tokenbar-mac-4dfed5ff" +export TOKENBAR_MAC_CANONICAL="${TMPDIR:-/tmp}/tokenbar-mac-55ca05d3" ( cd "$TOKENBAR_MAC_CANONICAL" @@ -195,7 +195,8 @@ mkdir -p crosscheck/swift-out crosscheck/csharp-out TZ=Asia/Taipei swift run crosscheck-harness \ "$TOKENBAR_WINDOWS/crosscheck/fixtures" \ "$TOKENBAR_WINDOWS/crosscheck/swift-out" \ - provider-quota-pace-v3 + provider-quota-pace-v3 \ + -AppleLanguages "(en)" ) TZ=Asia/Taipei dotnet run \ @@ -224,7 +225,8 @@ mkdir -p crosscheck/swift-out crosscheck/csharp-out cd "$TOKENBAR_MAC_CANONICAL" TZ=Asia/Taipei swift run crosscheck-harness \ "$TOKENBAR_WINDOWS/crosscheck/fixtures" \ - "$TOKENBAR_WINDOWS/crosscheck/swift-out" + "$TOKENBAR_WINDOWS/crosscheck/swift-out" \ + -AppleLanguages "(en)" ) TZ=Asia/Taipei dotnet run \ diff --git a/include/ctb.h b/include/ctb.h index 562536e..6d808f6 100644 --- a/include/ctb.h +++ b/include/ctb.h @@ -68,6 +68,17 @@ char *tb_hourly_report(const char *year, const char *clients); // Per-(sub-)agent report (AgentsReport). `clients` as in tb_hourly_report. char *tb_agents_report(const char *year, const char *clients); +// Source-generation-aware hourly/Agents filter parity diagnostic. The graph +// client list is derived from a fresh graph and all reports are bracketed by +// one opaque local-source token sequence. The success payload contains only +// lower-camel status values (match/mismatch/sourceChanged/tokenUnavailable), +// bounded report aggregates, and presentClientCount; it never exposes source +// paths, raw messages, cache data, credentials, providers, models, agents, or +// workspaces. A token probe failure is a successful tokenUnavailable result; +// graph/report/mapping/serialization failures use the normal outer error +// envelope. All calls are blocking and must be made off the main thread. +char *tb_filter_parity_probe(void); + // Live trace buckets over the trailing window (array of TraceBucket; // snake_case fields, e.g. tokens_per_min). Lazily re-parses at most every 10s. char *tb_usage_trace(int64_t window_secs); diff --git a/src/TokenBar.Core.Tests/DtoDecodeTests.cs b/src/TokenBar.Core.Tests/DtoDecodeTests.cs index 26097bf..f429c78 100644 --- a/src/TokenBar.Core.Tests/DtoDecodeTests.cs +++ b/src/TokenBar.Core.Tests/DtoDecodeTests.cs @@ -157,6 +157,60 @@ public void TokensPerMinDecodesCamelCase() Assert.Equal(42.5, v.Value); } + [Fact] + public void FilterParityProbeDecodesStatusesNullableAggregatesAndBoundedSummary() + { + var payload = TbCore.DecodeEnvelope( + """ + {"ok":true,"data":{ + "hourly":{ + "status":"match", + "unfiltered":{"entryCount":1,"input":2,"output":3,"cacheRead":4, + "cacheWrite":5,"reasoning":6,"totalTokens":20,"messageCount":7, + "totalCost":1.0}, + "full":{"entryCount":1,"input":2,"output":3,"cacheRead":4, + "cacheWrite":5,"reasoning":6,"totalTokens":20,"messageCount":7, + "totalCost":1.5}, + "delta":{"entryCount":0,"input":0,"output":0,"cacheRead":0, + "cacheWrite":0,"reasoning":0,"totalTokens":0,"messageCount":0, + "totalCost":0.5} + }, + "agents":{"status":"sourceChanged","unfiltered":null,"full":null,"delta":null}, + "presentClientCount":2 + }} + """); + + Assert.Equal(FilterParityStatus.Match, payload.Hourly.Status); + Assert.Equal(FilterParityStatus.SourceChanged, payload.Agents.Status); + Assert.Null(payload.Agents.Delta); + Assert.Equal(2, payload.PresentClientCount); + Assert.Equal( + "hourly=MATCH entriesΔ=0 tokensΔ=0 messagesΔ=0 costΔ=0.50; " + + "agents=SOURCE_CHANGED / SKIP", + payload.SmokeSummary); + } + + [Fact] + public void FilterParityProbeRejectsUnknownOrNumericStatus() + { + Assert.Throws(() => TbCore.DecodeEnvelope( + """ + {"ok":true,"data":{ + "hourly":{"status":"changed","unfiltered":null,"full":null,"delta":null}, + "agents":{"status":"match","unfiltered":null,"full":null,"delta":null}, + "presentClientCount":0 + }} + """)); + Assert.Throws(() => TbCore.DecodeEnvelope( + """ + {"ok":true,"data":{ + "hourly":{"status":0,"unfiltered":null,"full":null,"delta":null}, + "agents":{"status":"match","unfiltered":null,"full":null,"delta":null}, + "presentClientCount":0 + }} + """)); + } + [Fact] public void ModelReportDecodesLowercaseKQuirkAndOptionals() { diff --git a/src/TokenBar.Interop/FilterParityProbe.cs b/src/TokenBar.Interop/FilterParityProbe.cs new file mode 100644 index 0000000..040163f --- /dev/null +++ b/src/TokenBar.Interop/FilterParityProbe.cs @@ -0,0 +1,67 @@ +using System.Globalization; +using System.Text.Json.Serialization; + +namespace TokenBar.Interop; + +public enum FilterParityStatus +{ + [JsonStringEnumMemberName("match")] + Match, + [JsonStringEnumMemberName("mismatch")] + Mismatch, + [JsonStringEnumMemberName("sourceChanged")] + SourceChanged, + [JsonStringEnumMemberName("tokenUnavailable")] + TokenUnavailable, +} + +public sealed record FilterParityAggregate( + long EntryCount, + long Input, + long Output, + long CacheRead, + long CacheWrite, + long Reasoning, + long TotalTokens, + long MessageCount, + double TotalCost); + +public sealed record FilterParityReport( + FilterParityStatus Status, + FilterParityAggregate? Unfiltered = null, + FilterParityAggregate? Full = null, + FilterParityAggregate? Delta = null) +{ + public string SmokeSummary + { + get + { + var label = Status switch + { + FilterParityStatus.Match => "MATCH", + FilterParityStatus.Mismatch => "MISMATCH", + FilterParityStatus.SourceChanged => "SOURCE_CHANGED / SKIP", + FilterParityStatus.TokenUnavailable => "TOKEN_UNAVAILABLE / SKIP", + _ => throw new InvalidOperationException("unknown filter parity status"), + }; + if (Status is not (FilterParityStatus.Match or FilterParityStatus.Mismatch) || + Delta is null) + { + return label; + } + + var cost = Delta.TotalCost.ToString("F2", CultureInfo.InvariantCulture); + return $"{label} entriesΔ={Delta.EntryCount} tokensΔ={Delta.TotalTokens} " + + $"messagesΔ={Delta.MessageCount} costΔ={cost}"; + } + } +} + +public sealed record FilterParityProbe( + FilterParityReport Hourly, + FilterParityReport Agents, + long PresentClientCount) +{ + public string SmokeSummary => + $"hourly={Hourly.SmokeSummary}; agents={Agents.SmokeSummary}"; +} diff --git a/src/TokenBar.Interop/NativeMethods.cs b/src/TokenBar.Interop/NativeMethods.cs index 6cd5a87..5dfea7c 100644 --- a/src/TokenBar.Interop/NativeMethods.cs +++ b/src/TokenBar.Interop/NativeMethods.cs @@ -33,6 +33,9 @@ internal static partial class NativeMethods [LibraryImport(Lib, StringMarshalling = StringMarshalling.Utf8)] internal static partial nint tb_agents_report(string? year, string? clients); + [LibraryImport(Lib)] + internal static partial nint tb_filter_parity_probe(); + [LibraryImport(Lib)] internal static partial nint tb_usage_trace(long windowSecs); diff --git a/src/TokenBar.Interop/TbCore.cs b/src/TokenBar.Interop/TbCore.cs index e142815..6e55702 100644 --- a/src/TokenBar.Interop/TbCore.cs +++ b/src/TokenBar.Interop/TbCore.cs @@ -24,6 +24,10 @@ public static class TbCore { RespectRequiredConstructorParameters = true, RespectNullableAnnotations = true, + Converters = + { + new JsonStringEnumConverter(allowIntegerValues: false), + }, }; public static ProbeResult Probe() @@ -69,6 +73,11 @@ public static HourlyReport HourlyReport(string? year = null, IReadOnlyList? clients = null) => Unwrap(NativeMethods.tb_agents_report(year, JoinClients(clients))); + /// Source-generation-aware nil/full parity diagnostic for the + /// hourly and Agents report filters. + public static FilterParityProbe FilterParityProbe() => + Unwrap(NativeMethods.tb_filter_parity_probe()); + private static string? JoinClients(IReadOnlyList? clients) => clients is { Count: > 0 } ? string.Join(',', clients) : null; diff --git a/src/TokenBar.Smoke/Program.cs b/src/TokenBar.Smoke/Program.cs index e216671..80622eb 100644 --- a/src/TokenBar.Smoke/Program.cs +++ b/src/TokenBar.Smoke/Program.cs @@ -121,6 +121,8 @@ void Step(string name, Func body) $"claudeOnlyMessages={filtered.TotalMessages}"; }); +Step("tb_filter_parity", () => TbCore.FilterParityProbe().SmokeSummary); + Step("tb_usage_trace", () => { var buckets = TbCore.UsageTrace(600); diff --git a/vendor/README.md b/vendor/README.md index c2d47d5..edf643a 100644 --- a/vendor/README.md +++ b/vendor/README.md @@ -30,7 +30,7 @@ The Copilot duplicate-span contribution reached upstream in issue [#938](https:/ M24 PR [#86](https://github.com/Nanako0129/TokenBar/pull/86) is closed unmerged as fidelity evidence; its [close rationale](https://github.com/Nanako0129/TokenBar/pull/86#issuecomment-5047332480) records the public stop decision. Current-head Codex found four independent security/state-machine defects, and the first local fix round was rejected by fresh security verification because process-local Warp source/revocation state could race with the shared cross-process singleton app cache: one process handling a 401 or Disconnect for scope B could delete another process's valid scope-A cache. Holding the refresh lock through remote I/O violates the approved network boundary; releasing it exposes the cache-ownership conflict. The second consecutive review round therefore triggered the M24 stop condition. No review fix was pushed, `63a44d7c` moved from `TAKE` to `DEFER`, and the resulting checkpoint was `78/2/0/17/13/1`. -The approved replacement delivery graph removes M24 from the cache dependency chain without reviving Warp: `M26-A → M26-B → M19-BP → M19-BQ → PT0 → M19-B0 → M19-B1 → D2`. M26-A merged in PR [#90](https://github.com/Nanako0129/TokenBar/pull/90) at [`95c819c7`](https://github.com/Nanako0129/TokenBar/commit/95c819c7cf6532be7276b64386490b1b03a0c1ae); M26-B merged in PR [#91](https://github.com/Nanako0129/TokenBar/pull/91) at [`cc52c3b9`](https://github.com/Nanako0129/TokenBar/commit/cc52c3b9e01836cbf1c7ab7c77b4dd8b5e2df7b2), moving `cd07bf78` to `DEFER` because its Devin residual remains excluded. M19-BP, M19-BQ, and PT0 then merged in PRs [#92](https://github.com/Nanako0129/TokenBar/pull/92), [#93](https://github.com/Nanako0129/TokenBar/pull/93), and [#94](https://github.com/Nanako0129/TokenBar/pull/94) without changing the ledger. M19-B0 merged in PR [#99](https://github.com/Nanako0129/TokenBar/pull/99) at [`f820b06f`](https://github.com/Nanako0129/TokenBar/commit/f820b06fd99a53cada8495338bd7d58898525a7b), followed by the Native review fixes in PR [#101](https://github.com/Nanako0129/TokenBar/pull/101) at [`7abd205a`](https://github.com/Nanako0129/TokenBar/commit/7abd205ab692950c9b4574a14da72f01733d1935). M19-B1 is validating the exact Windows sync; Warp remains `parse_local: false`. +The approved replacement delivery graph removes M24 from the cache dependency chain without reviving Warp: `M26-A → M26-B → M19-BP → M19-BQ → PT0 → M19-B0 → M19-B1 → D2`. M26-A merged in PR [#90](https://github.com/Nanako0129/TokenBar/pull/90) at [`95c819c7`](https://github.com/Nanako0129/TokenBar/commit/95c819c7cf6532be7276b64386490b1b03a0c1ae); M26-B merged in PR [#91](https://github.com/Nanako0129/TokenBar/pull/91) at [`cc52c3b9`](https://github.com/Nanako0129/TokenBar/commit/cc52c3b9e01836cbf1c7ab7c77b4dd8b5e2df7b2), moving `cd07bf78` to `DEFER` because its Devin residual remains excluded. M19-BP, M19-BQ, and PT0 then merged in PRs [#92](https://github.com/Nanako0129/TokenBar/pull/92), [#93](https://github.com/Nanako0129/TokenBar/pull/93), and [#94](https://github.com/Nanako0129/TokenBar/pull/94) without changing the ledger. M19-B0 merged in PR [#99](https://github.com/Nanako0129/TokenBar/pull/99) at [`f820b06f`](https://github.com/Nanako0129/TokenBar/commit/f820b06fd99a53cada8495338bd7d58898525a7b), followed by the Native review fixes in PR [#101](https://github.com/Nanako0129/TokenBar/pull/101) at [`7abd205a`](https://github.com/Nanako0129/TokenBar/commit/7abd205ab692950c9b4574a14da72f01733d1935). M19-B1 Native PR [#102](https://github.com/Nanako0129/TokenBar/pull/102) merged at [`4dfed5ff`](https://github.com/Nanako0129/TokenBar/commit/4dfed5ffab26e2707a8cd82ee99864520b75892b), and exact Windows PR [#7](https://github.com/Nanako0129/TokenBar-Windows/pull/7) merged at [`f29d3b35`](https://github.com/Nanako0129/TokenBar-Windows/commit/f29d3b3546440081e20a917c44cf11f6cf3d80ff). D2 records the docs-only closeout; Warp remains `parse_local: false`. The immutable audited set is the 111 hashes produced in a clean upstream clone: @@ -190,7 +190,7 @@ flowchart TD BP --> BQ[M19-BQ Claude probe] BQ --> PT0[PT0 provider transport] PT0 --> W0[M19-B0 Native secure storage] - W0 --> W1[M19-B1 Windows final sync] + W0 --> W1[M19-B1 merged — Native PR #102 + Windows PR #7] W1 --> D2[D2 final docs checkpoint] ``` @@ -212,13 +212,14 @@ flowchart TD | M26-A | PR #90 merged at `95c819c7`; `ae36db5c: TAKE → ALREADY_VENDORED`; format-1 identity-aware shards become active while M24 remains excluded | | M26-B | PR #91 merged at `cc52c3b9`; generic format-2 metadata and Claude cached-parent recovery landed, while `cd07bf78: TAKE → DEFER` leaves the Devin residual and Pi excluded | | M19-BP / M19-BQ / PT0 | PRs #92/#93/#94 merged shared roots, the Claude version probe, and provider transport without changing counts | -| M19-B0 / M19-B1 | M19-B0 merged in PR #99 with Native review fixes in PR #101; M19-B1 validates the exact Native → Windows shared-core sync and keeps Windows shared-tree local patches empty; counts unchanged | +| M19-B0 / M19-B1 | M19-B0 merged in PR #99 with Native review fixes in PR #101. M19-B1 Native PR #102 merged at `4dfed5ff`; exact Windows PR #7 merged at `f29d3b35`, pins that SHA, keeps Windows shared-tree local patches empty, and passes hosted plus real ARM64 runtime gates; counts unchanged | +| D2 | Final docs-only checkpoint; records the landed cross-repository evidence without changing runtime, ledger, or cache | -The current checkpoint is `ALREADY_VENDORED 79`, `TAKE 0`, `ADAPT_FOR_STREAMING 0`, `DEFER 18`, `SKIP 13`, and `SUPERSEDED 1`, total 111. M24 remains deferred and outside the cache dependency graph; M26-A, M26-B, M19-BP, M19-BQ, PT0, and M19-B0 are merged, so the approved remaining sequence is M19-B1, then D2. Every later transition must start from this actual ledger, regenerate all six sets, and rerun duplicate and symmetric-difference checks. +The current checkpoint is `ALREADY_VENDORED 79`, `TAKE 0`, `ADAPT_FOR_STREAMING 0`, `DEFER 18`, `SKIP 13`, and `SUPERSEDED 1`, total 111. M24 remains deferred and outside the cache dependency graph; M26-A, M26-B, M19-BP, M19-BQ, PT0, M19-B0, and M19-B1 are merged, and D2 closes this approved sequence in documentation. Every later runtime transition must start from this actual ledger, regenerate all six sets, and rerun duplicate and symmetric-difference checks. M26-A made shard format 1 active at `source-message-cache-v2//shard-XX.bin`; M26-B makes format 2 the active cache contract by persisting related-file paths and existence state and reusing cached Claude parent candidates only after the primary fingerprint matches. Existing format-1 shards are locally stale and cold-rebuild under format 2. The former schema-32 `source-message-cache.bin` remains inert provenance: the shard reader does not load, migrate, rewrite, or delete it. Per-client parser versions remain independent of this global storage-format bump. -Public issue #45 is the designated remote inventory. M25 merged in PR #75; M22 PR #72 is closed unmerged; M23-H merged in PR #82 at `1a8ee0c6`; M23-D merged in PR #83 at `f99d9274`; M26-A merged in PR #90 at `95c819c7`; M26-B merged in PR #91 at `cc52c3b9`; M19-BP, M19-BQ, and PT0 merged in PRs #92/#93/#94; M19-B0 and its Native review fixes merged in PRs #99/#101; PR #74 is closed unmerged at `e274f2ad`; and M24 PR #86 is closed unmerged as the latest fidelity evidence. The private Project tracks executable milestones only; it does not duplicate the 111 commit rows. +Public issue #45 is the designated remote inventory. M25 merged in PR #75; M22 PR #72 is closed unmerged; M23-H merged in PR #82 at `1a8ee0c6`; M23-D merged in PR #83 at `f99d9274`; M26-A merged in PR #90 at `95c819c7`; M26-B merged in PR #91 at `cc52c3b9`; M19-BP, M19-BQ, and PT0 merged in PRs #92/#93/#94; M19-B0 and its Native review fixes merged in PRs #99/#101; M19-B1 merged through Native PR #102 at `4dfed5ff` and Windows PR #7 at `f29d3b35`; PR #74 is closed unmerged at `e274f2ad`; and M24 PR #86 is closed unmerged as fidelity evidence. The private Project tracks executable milestones only; it does not duplicate the 111 commit rows. ## Cherry-picked upstream commits (ahead of baseline) @@ -369,15 +370,17 @@ TokenBar factors the retry loop into an injected replacement/sleep helper compil ## Recovered Windows downstream commits -The Windows port started from TokenBar [`2ed256ee`](https://github.com/Nanako0129/TokenBar/commit/2ed256eea7f6761e85198e3bc584e08a8d3d8ac1), then accumulated and validated the vendor-only sequence recorded in the [issue #45 handoff](https://github.com/Nanako0129/TokenBar/issues/45#issuecomment-5002865759). These commits are now recovered into this repository so it remains the canonical source for the next Windows sync; the detailed Windows build, hostile-environment, profile-manifest, and FFI evidence stays in that handoff rather than being duplicated here. +The Windows port started from TokenBar [`2ed256ee`](https://github.com/Nanako0129/TokenBar/commit/2ed256eea7f6761e85198e3bc584e08a8d3d8ac1), then accumulated and validated the vendor-only sequence recorded in the [issue #45 handoff](https://github.com/Nanako0129/TokenBar/issues/45#issuecomment-5002865759). These commits were recovered into this repository before M19-B1 so Native remained canonical. Windows PR #7 now pins exact Native merge `4dfed5ff`; the detailed historical Windows build, hostile-environment, profile-manifest, and FFI evidence stays in the public handoff rather than being duplicated here. | Commits | What | Files | Cache / upstream status | |---|---|---|---| | `e5200634` → `db2a96a3` → `15f418ee` → `e807f333` → `0979cdb0` → `26b892a6` → `6ac77c03` → `e91fb2c6` → `fbecb99c` → `3c8bfc52` → `aec5bd88` | Makes core tests hermetic with panic-safe environment/current-directory guards, serial coordination, isolated cache/XDG roots, platform-safe JSON/path fixtures, and explicit-home parser/scanner fixtures while retaining dedicated positive environment cases. Production fixes release the temporary cache writer before Windows atomic replacement, reopen the final cache read/write for the durability sync, derive Windows explicit-home config and Zed paths from supplied `/AppData/Roaming/tokscale` and `/AppData/Local/Zed/threads/threads.db` while limiting process known-folder fallbacks to env-aware scans, and compare extra-path warnings against the supplied scan home. | `src/clients.rs`, `src/lib.rs`, `src/message_cache.rs`, `src/pricing/cache.rs`, `src/scanner.rs`, `src/sessions/claudecode.rs`, `src/sessions/opencode.rs` | No serialized parser output, cache layout, FFI, or public API change; `CACHE_SCHEMA_VERSION` remained 29 at that Windows-recovery checkpoint; M20 later advances the Native source to schema 30. The upstream-applicable portions are merged: cache PR [#914](https://github.com/junhoyeo/tokscale/pull/914) (`163ec570`) maps the handle ordering to upstream's shard writer, explicit-home PR [#916](https://github.com/junhoyeo/tokscale/pull/916) (`a2f7cef5`) matches the supplied-home and warning semantics including positive `AppData` roots, and core hermetic PR [#917](https://github.com/junhoyeo/tokscale/pull/917) (`a0929482`) carries the same fixture-isolation failure class on upstream's current test layout. The remaining structural differences are TokenBar-specific cache and test adaptations, not missing upstream behavior. M26-A later replaced the monolithic writer with the active shard writer, so M19-B1-R2 reapplies the same upstream-approved Windows handle ordering to that new persistence seam without a schema change. | -M19-B1-R2 also closes the other Windows portability gaps exposed by the exact downstream sync. The format-2 shard writer now drops its temporary `BufWriter` before `fs_atomic::replace_file` and reopens the installed shard read/write for the durability sync; Copilot Desktop rejects non-directory `session-state` entries before probing `events.jsonl`; the cc-mirror JSON fixture uses `serde_json`; and the Kiro scanner fixture compares native `PathBuf` values. Hosted Windows test fixtures pin `TOKSCALE_CONFIG_DIR` to each sandbox because Windows Known Folders ignore HOME/XDG, open mtime targets with writable handles, and drop manually serialized stale-shard writers before readers or replacement. In the same Native canonical repair, `tb_core_ffi` restores cfg-gated PowerShell/CIM Antigravity process and PID-bound listener discovery. These changes affect `src/lib.rs`, `src/message_cache.rs`, `src/sessions/copilot_desktop.rs`, `src/scanner.rs`, and `crates/tb_core_ffi/src/agent_antigravity.rs`; they do not change parser output, parser versions, cache identity, `CACHE_FORMAT_VERSION = 2`, FFI/C ABI, or the inert schema-32 monolith. Windows consumes them through the next byte-identical Native sync rather than a downstream shared-tree patch. +M19-B1-R2 also closed the other Windows portability gaps exposed by the exact downstream sync. The format-2 shard writer now drops its temporary `BufWriter` before `fs_atomic::replace_file` and reopens the installed shard read/write for the durability sync; Copilot Desktop rejects non-directory `session-state` entries before probing `events.jsonl`; the cc-mirror JSON fixture uses `serde_json`; and the Kiro scanner fixture compares native `PathBuf` values. Hosted Windows test fixtures pin `TOKSCALE_CONFIG_DIR` to each sandbox because Windows Known Folders ignore HOME/XDG, open mtime targets with writable handles, and drop manually serialized stale-shard writers before readers or replacement. In the same Native canonical repair, `tb_core_ffi` restores cfg-gated PowerShell/CIM Antigravity process and PID-bound listener discovery. These changes affect `src/lib.rs`, `src/message_cache.rs`, `src/sessions/copilot_desktop.rs`, `src/scanner.rs`, and `crates/tb_core_ffi/src/agent_antigravity.rs`; they do not change parser output, parser versions, cache identity, `CACHE_FORMAT_VERSION = 2`, FFI/C ABI, or the inert schema-32 monolith. Windows consumed them byte-identically in PR #7 rather than retaining a downstream shared-tree patch. -The PR #102 current-head review adds two shared-tree corrections before that sync. Codex OAuth credential write-back now commits its staged `auth.json` through the same `fs_atomic::replace_file` primitive, so Windows receives the bounded 5/32 retry policy while the existing unique sibling temp, create-new semantics, file sync, Unix directory sync, live-token revalidation, JSON sibling preservation, receipt/lineage ordering, and rollback boundary stay unchanged; Windows-only fixtures cover transient success and persistent-lock failure without losing the original file. Copilot duplicate OTEL spans now merge the earliest start with the latest known endpoint instead of pairing the earliest start with the largest independent duration. Because `duration_ms` is serialized parser output, the Copilot parser version advances **3→4** and a same-fingerprint stale-v3 shard regression proves the old duration is rejected and rebuilt. Hosted Windows shipping-cache fixtures now build related-file fingerprints, seed, and query with the scanner-returned path spelling, preserving the production cache key and related-path identity when explicit-home resolution uses `/` but `TempDir::join` constructs the same physical path with `\`. `CACHE_FORMAT_VERSION` remains 2, every other parser namespace stays warm, FFI/C ABI is unchanged, and the schema-32 monolith remains inert. +The PR #102 review added shared-tree corrections before the final sync. Codex OAuth credential write-back now commits its staged `auth.json` through the same `fs_atomic::replace_file` primitive, so Windows receives the bounded 5/32 retry policy while the existing unique sibling temp, create-new semantics, file sync, Unix directory sync, live-token revalidation, JSON sibling preservation, receipt/lineage ordering, and rollback boundary stay unchanged; Windows-only fixtures cover transient success and persistent-lock failure without losing the original file. Copilot duplicate OTEL spans now merge the earliest start with the latest known endpoint instead of pairing the earliest start with the largest independent duration. Because `duration_ms` is serialized parser output, the Copilot parser version advances **3→4** and a same-fingerprint stale-v3 shard regression proves the old duration is rejected and rebuilt. Hosted Windows shipping-cache fixtures now build related-file fingerprints, seed, and query with the scanner-returned path spelling, preserving the production cache key and related-path identity when explicit-home resolution uses `/` but `TempDir::join` constructs the same physical path with `\`. + +The final Native review round also made Copilot's optional quota reset date lossy without weakening required quota-row validation or poisoning a valid last-good snapshot, and replaced substring transport classification with typed nested DNS/TLS source inspection plus bounded allowlisted diagnostics. Native PR #102 and Windows PR #7 both received current-head Codex `+1` results and clean review threads. Windows hosted x64、Swift cross-check、ARM64 cross-package CI and a separate real ARM64 run of all 351 Rust tests、12 provider-v3 CrossCheck cases、PE checks與synthetic WinUI startup passed against the exact merged source. `CACHE_FORMAT_VERSION` remains 2, every non-Copilot parser namespace stays warm, FFI/C ABI and ledger counts are unchanged, and the schema-32 monolith remains inert. ## Upstream fixes reported but not yet vendored @@ -397,6 +400,7 @@ these are upstream fixes a sync will *gain*.) | PR #3 (perf): streaming per-file aggregation replaces materialize-then-aggregate for the graph/model/monthly/hourly reports — `StreamingAggregator` + `SessionizeAccumulator` folded by `scan_messages_streaming` in one cache-aware pass (no full-history `Vec`). Each client lane owns its dedup set (follow-up `0752e35`: prevents cross-client `dedup_key` collisions). | `src/aggregator.rs`, `src/lib.rs`, `src/sessionize.rs`, `tests/streaming_snapshot.rs` | not yet forwarded to junhoyeo/tokscale | | #6 (fix): the **agents report** now folds over `scan_messages_streaming` too — new `get_agents_report` (mirrors `get_model_report`, `resolve_report_clients` + a single streaming pass into `AgentAccumulator`), so it shares the one deduped/per-client-gated/priced stream as every other report (resolves the issue #6 divergence: agents no longer over-counts copilot/codebuff/kimi/cursor/warp/… duplicate `dedup_key`s, and scans the same client set). `parse_local_unified_messages` survives as public API only (footgun-documented, no in-repo callers). `crates/tb_core_ffi/src/agents_report.rs` is now a thin mapper like `model_report.rs` (no longer byte-identical to the archived Tauri original — accepted). | `src/lib.rs`, `crates/tb_core_ffi/src/agents_report.rs` | not yet forwarded to junhoyeo/tokscale | | #35/#36 (fix): **two-level client filter for the hourly & agents reports.** TokenBar passes the user's displayed client slice into `ReportOptions.clients` so shared hour/agent buckets carry only the selected clients' totals (a Swift membership filter can't split a mixed fold). But `scan_messages_streaming` selects scanner *lanes* from that list, and a `cc-mirror/` id (produced during Claude-lane parsing, #659) is not a lane — requesting it alone would scan nothing. New `split_report_client_filter` / `report_message_client_passes` helpers split the request into (a) lanes to scan (each `cc-mirror/*` → its producing `claude` lane) and (b) an EXACT client-id set the fold keeps; `get_agents_report` **and** `get_hourly_report` call the split + gate `msg_filter` on it. Unlike the scan's built-in `retain_for_requested_clients`, requesting `claude` here excludes the distinct `cc-mirror/*` variants (they are their own client ids in the graph/model/daily payloads); the `synthetic` special-case is preserved. `get_hourly_report` was previously byte-upstream — this makes it a local patch; **re-apply on any re-vendor of `lib.rs`.** | `src/lib.rs`, `crates/tb_core_ffi/src/lib.rs`, `crates/tb_core_ffi/src/{hourly,agents}_report.rs` | not yet forwarded to junhoyeo/tokscale | +| [TokenBar #107](https://github.com/Nanako0129/TokenBar/issues/107) (verification): the hermetic source-aware filter-parity fixture protects the local #6 and #35/#36 report seams without changing vendor production code. It builds one isolated source generation, checks hourly and Agents `nil` versus full-client reports across cold and warm cache paths, and pins canonical client discovery, exact `cc-mirror/` gating, duplicate scan-root deduplication, unattributed `Main` usage, entries, token lanes, messages, and cost. The app-owned FFI smoke separately classifies a generation change instead of treating two different live snapshots as a filter mismatch. No parser output, cache format/schema, public vendor API, or 111-row classification changes. **Retain this test on any re-vendor while TokenBar owns these report adaptations.** | `tests/filter_parity.rs` | TokenBar-only regression for local streaming/report adaptations; not an upstream production patch | | #5 (feat): discover Claude desktop "Cowork" (local-agent-mode) transcripts. `discover_cowork_project_roots()` recurses `~/Library/Application Support/Claude/local-agent-mode-sessions/**/.claude/projects` and feeds the roots into `built_in_extra_scan_paths_for` as `ClientId::Claude`. Returns the per-session `projects` roots only, so the sibling `audit.jsonl` (a mirror of the same `usage` records) is never scanned — scanning it would double-count. | `src/scanner.rs` | Covered by closed-unmerged upstream PR #708 together with `` filtering; revive that PR rather than opening a duplicate. | | pricing (fix): **cache-rate backfill** — `choose_best_source_result` is wrapped so the chosen pricing source has any missing cache read/write rates grafted from the runner-up source (`backfill_cache_costs` + `prefer_litellm_over_openrouter`). Without it, a provider-hint that selects an entry lacking cache rates (e.g. an OpenRouter row) bills cache reads at $0 (fable-5 showed $11.50 instead of $45). Upstream `#658`/`#707` do **not** subsume this: `has_any_usable_pricing` is an `.any()` gate, so a row that prices everything *except* cache still passes through unfilled — the root cause stays ours to fix. | `src/pricing/lookup.rs` | not yet forwarded to junhoyeo/tokscale | | pricing (perf): **in-memory auto-refresh** — `PRICING_SERVICE` is a `RwLock>` with `IN_MEMORY_TTL = 3600s` (was a never-refreshing `OnceCell`), so prices re-read the file cache / network roughly hourly instead of being frozen for the process lifetime; adds `pricing_cached_at()` (Models card "Prices updated …"). Spans `mod.rs` + the additive `litellm::cached_at` / `cache::cache_timestamp` helpers. | `src/pricing/mod.rs`, `src/pricing/litellm.rs`, `src/pricing/cache.rs` | not yet forwarded to junhoyeo/tokscale | diff --git a/vendor/tokscale-core/SYNC.md b/vendor/tokscale-core/SYNC.md index 976e684..45ff585 100644 --- a/vendor/tokscale-core/SYNC.md +++ b/vendor/tokscale-core/SYNC.md @@ -3,19 +3,18 @@ | Field | Value | |---|---| | Source repo | [Nanako0129/TokenBar](https://github.com/Nanako0129/TokenBar) | -| Copied commit | [`4dfed5ffab26e2707a8cd82ee99864520b75892b`](https://github.com/Nanako0129/TokenBar/commit/4dfed5ffab26e2707a8cd82ee99864520b75892b) | -| Copied on | 2026-07-27 | +| Copied commit | [`55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218`](https://github.com/Nanako0129/TokenBar/commit/55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218) | +| Copied on | 2026-07-28 | -`4dfed5ffab26e2707a8cd82ee99864520b75892b` is the final merged Native PR -#102 commit on `main`. In addition to the Antigravity discovery, format-2 writer -lifecycle, malformed Copilot Desktop entry, and fixture portability -corrections, this source retries Codex credential replacement, merges duplicate -Copilot span endpoints, advances only the Copilot parser identity to 4, and -makes shipping cache fixtures build fingerprints, seed, and query with the -scanner-returned path spelling. The post-review merge also preserves valid -Copilot quota rows when the optional reset field has a non-string type and -emits DNS/TLS transport categories from typed error sources without exposing -free-form error text. This is the canonical post-merge M19-B checkpoint. +`55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218` is the final rebase-merged Native +PR #111 commit on `main`. It preserves the post-merge M19-B checkpoint and adds +the issue-107 source-generation-aware filter-parity diagnostic: one Rust-owned +probe uses a fresh graph and brackets hourly and Agents nil/full reports with +opaque local-source tokens. Source movement is classified as `sourceChanged` +instead of a filter mismatch; independently refreshed cost remains diagnostic +but cannot decide parity. A hermetic vendor fixture covers exact client gates, +synthetic traffic, duplicate Codebuff roots, unattributed `Main`, cold/warm +cache, and inherited scanner-root isolation. The active cache is format 2 at `source-message-cache-v2`; format-1 shards are stale and rebuild cold under format 2. The legacy schema-32 monolith @@ -29,6 +28,30 @@ tracked runtime vendor file is byte-identical to Native except this Windows-only fixture at `Fixtures/CrossCheck/provider-quota-pace-v3.json` and the Windows cross-check copy are byte-identical to the same Native fixture. +## Verification + +The macOS-side downstream gate passed against this exact sync: + +- `crates/` is byte-identical to Native `55ca05d3`; `vendor/` is + byte-identical after excluding Native-only `vendor/AGENTS.md` and this + Windows-only `SYNC.md`; the C header and both provider-v3 fixture copies also + match byte-for-byte. +- `Cargo.lock` is unchanged. The two new Rust fixture/source files pass a + focused `rustfmt --check`; the exact shared tree was not reformatted. +- Focused parity tests pass (`tb_core_ffi` 10/10 and vendor fixture 1/1). +- `scripts/check.sh` passes with locked dependencies: workspace check, all Rust + tests (including 319 FFI and 1,290 vendor unit tests), release build, locked + .NET restore, solution build with zero warnings/errors, all 287 Core tests, + and the 11-entry P/Invoke smoke. +- The live smoke observed source movement while local sessions were changing, + so both parity reports correctly returned `sourceChanged` and skipped + comparison. Credential/network-backed agent usage was intentionally disabled + with `TB_SMOKE_SKIP_NETWORK=1`; pricing was cache-only. + +This is local macOS-side evidence. Hosted Windows x64 and ARM64 cross-package +checks remain CI-owned, and this record does not claim real ARM64 runtime +validation for this sync. + ## Sync procedure Run from a clean Windows checkout with `TOKENBAR_NATIVE` pointing to a clean @@ -37,7 +60,7 @@ then update its commit, date, and verification fields before committing: ```bash : "${TOKENBAR_NATIVE:?set TOKENBAR_NATIVE to a clean Native checkout}" -source_commit=4dfed5ffab26e2707a8cd82ee99864520b75892b +source_commit=55ca05d3a0a5bf0f02ed20f46bb3e73e65a07218 stage="$(mktemp -d)" sync_record="$(mktemp)" trap 'rm -rf "$stage"; rm -f "$sync_record"' EXIT diff --git a/vendor/tokscale-core/tests/filter_parity.rs b/vendor/tokscale-core/tests/filter_parity.rs new file mode 100644 index 0000000..b4a004b --- /dev/null +++ b/vendor/tokscale-core/tests/filter_parity.rs @@ -0,0 +1,403 @@ +//! Stable nil/full report parity fixture for the Native source-aware smoke +//! probe. This is test-only coverage: no vendored production behavior changes. +//! +//! The fixture intentionally combines the source identities that are most +//! likely to expose a two-level client-filter regression: a canonical client, +//! a Claude-produced `cc-mirror/*` id, a Synthetic gateway message, duplicate +//! canonical scan roots, and unattributed messages that fold into `Main`. +//! Reports are checked cold and warm so cache reuse cannot create a false +//! source-generation mismatch. + +use std::ffi::{OsStr, OsString}; +use std::path::Path; + +use tokscale_core::{ + generate_local_graph_report, get_agents_report, get_hourly_report, AgentReport, GraphResult, + HourlyReport, ReportOptions, +}; + +struct EnvGuard { + previous: Vec<(&'static str, Option)>, +} + +impl EnvGuard { + fn isolate(values: &[(&'static str, &OsStr)], removed: &[&'static str]) -> Self { + let previous = values + .iter() + .map(|(key, _)| *key) + .chain(removed.iter().copied()) + .map(|key| (key, std::env::var_os(key))) + .collect(); + unsafe { + for (key, value) in values { + std::env::set_var(key, value); + } + for key in removed { + std::env::remove_var(key); + } + } + Self { previous } + } +} + +impl Drop for EnvGuard { + fn drop(&mut self) { + unsafe { + for (key, value) in self.previous.drain(..) { + match value { + Some(value) => std::env::set_var(key, value), + None => std::env::remove_var(key), + } + } + } + } +} + +#[derive(Debug, Clone, PartialEq)] +struct Aggregate { + entry_count: usize, + input: i64, + output: i64, + cache_read: i64, + cache_write: i64, + reasoning: i64, + total_tokens: i64, + message_count: i64, + total_cost: f64, +} + +fn hourly_aggregate(report: &HourlyReport) -> Aggregate { + let mut result = Aggregate { + entry_count: report.entries.len(), + input: 0, + output: 0, + cache_read: 0, + cache_write: 0, + reasoning: 0, + total_tokens: 0, + message_count: 0, + total_cost: report.total_cost, + }; + for entry in &report.entries { + result.input = result.input.saturating_add(entry.input); + result.output = result.output.saturating_add(entry.output); + result.cache_read = result.cache_read.saturating_add(entry.cache_read); + result.cache_write = result.cache_write.saturating_add(entry.cache_write); + result.reasoning = result.reasoning.saturating_add(entry.reasoning); + result.message_count = result + .message_count + .saturating_add(i64::from(entry.message_count)); + } + result.total_tokens = result + .input + .saturating_add(result.output) + .saturating_add(result.cache_read) + .saturating_add(result.cache_write) + .saturating_add(result.reasoning); + result +} + +fn agents_aggregate(report: &AgentReport) -> Aggregate { + let mut result = Aggregate { + entry_count: report.entries.len(), + input: 0, + output: 0, + cache_read: 0, + cache_write: 0, + reasoning: 0, + total_tokens: 0, + message_count: i64::from(report.total_messages), + total_cost: report.total_cost, + }; + for entry in &report.entries { + result.input = result.input.saturating_add(entry.input); + result.output = result.output.saturating_add(entry.output); + result.cache_read = result.cache_read.saturating_add(entry.cache_read); + result.cache_write = result.cache_write.saturating_add(entry.cache_write); + result.reasoning = result.reasoning.saturating_add(entry.reasoning); + } + result.total_tokens = result + .input + .saturating_add(result.output) + .saturating_add(result.cache_read) + .saturating_add(result.cache_write) + .saturating_add(result.reasoning); + result +} + +#[derive(Debug, Clone, PartialEq)] +struct ReportSet { + hourly_nil: Aggregate, + hourly_full: Aggregate, + agents_nil: Aggregate, + agents_full: Aggregate, +} + +fn options(home: &Path, clients: Option>) -> ReportOptions { + ReportOptions { + home_dir: Some(home.to_string_lossy().into_owned()), + // Keep this fixture rooted in its temporary home while still allowing + // the duplicate TOKSCALE_EXTRA_DIRS task below to exercise canonical + // scan-root deduplication. + use_env_roots: true, + clients, + ..Default::default() + } +} + +fn reports(home: &Path, clients: &[String]) -> ReportSet { + let hourly_nil = hourly_report(home, None); + let hourly_full = hourly_report(home, Some(clients.to_vec())); + let agents_nil = agents_report(home, None); + let agents_full = agents_report(home, Some(clients.to_vec())); + ReportSet { + hourly_nil: hourly_aggregate(&hourly_nil), + hourly_full: hourly_aggregate(&hourly_full), + agents_nil: agents_aggregate(&agents_nil), + agents_full: agents_aggregate(&agents_full), + } +} + +fn hourly_report(home: &Path, clients: Option>) -> HourlyReport { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime + .block_on(get_hourly_report(options(home, clients))) + .unwrap() +} + +fn agents_report(home: &Path, clients: Option>) -> AgentReport { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + runtime + .block_on(get_agents_report(options(home, clients))) + .unwrap() +} + +fn write_fixture(home: &Path) { + // Canonical client, duplicated under two project files with one exact + // message id. The scanner sees the same root through TOKSCALE_EXTRA_DIRS, + // while the streaming dedup gate sees the same canonical message key. + for project in ["proj-a", "proj-b"] { + let dir = home.join(format!(".config/manicode/projects/{project}")); + std::fs::create_dir_all(&dir).unwrap(); + std::fs::write( + dir.join("chat-messages.json"), + r#"[{"role":"assistant","id":"CANONICAL-DUP","metadata":{"model":"claude-sonnet-4","usage":{"inputTokens":200,"outputTokens":80}},"credits":0.02}]"#, + ) + .unwrap(); + } + + // Plain Claude plus a cc-mirror variant produced by the Claude lane. Both + // are unattributed and therefore must remain in the single Agents Main + // bucket without the exact variant gate leaking plain Claude. + let claude_dir = home.join(".claude/projects/plain"); + std::fs::create_dir_all(&claude_dir).unwrap(); + std::fs::write( + claude_dir.join("conversation.jsonl"), + r#"{"type":"assistant","timestamp":"2024-12-01T10:00:00.000Z","requestId":"req_plain","message":{"id":"msg_plain","model":"claude-3-5-sonnet","usage":{"input_tokens":100,"output_tokens":50}}}"#, + ) + .unwrap(); + let variant = home.join(".cc-mirror/kimi-code"); + let variant_config = variant.join("config"); + let variant_project = variant_config.join("projects/proj"); + std::fs::create_dir_all(&variant_project).unwrap(); + std::fs::write( + variant.join("variant.json"), + serde_json::json!({ + "name": "kimi-code", + "provider": "kimi", + "configDir": variant_config, + }) + .to_string(), + ) + .unwrap(); + std::fs::write( + variant_project.join("session.jsonl"), + r#"{"type":"assistant","timestamp":"2024-12-01T11:00:00.000Z","requestId":"req_variant","message":{"id":"msg_variant","model":"claude-3-5-sonnet","usage":{"input_tokens":300,"output_tokens":70}}}"#, + ) + .unwrap(); + + // Kimi is another canonical client with no agent attribution. + let kimi_dir = home.join(".kimi/sessions/group/session"); + std::fs::create_dir_all(&kimi_dir).unwrap(); + std::fs::write( + kimi_dir.join("wire.jsonl"), + concat!( + "{\"type\":\"metadata\",\"protocol_version\":\"1.3\"}\n", + "{\"timestamp\":1770983410.0,\"message\":{\"type\":\"StatusUpdate\",\"payload\":{\"token_usage\":{\"input_other\":100,\"output\":50,\"input_cache_read\":3,\"input_cache_creation\":4},\"message_id\":\"KIMI-1\"}}}" + ), + ) + .unwrap(); + + // Synthetic gateway traffic rides the OpenCode lane. Its canonical client + // remains `opencode`; the synthetic matcher must therefore agree in both + // the nil and full-list report paths without adding a fake client id. + let opencode_dir = home.join(".local/share/opencode/storage/message/project-1"); + std::fs::create_dir_all(&opencode_dir).unwrap(); + std::fs::write( + opencode_dir.join("msg_synthetic.json"), + r#"{"id":"synthetic-1","sessionID":"synthetic-session","role":"assistant","modelID":"hf:deepseek-ai/DeepSeek-V3-0324","providerID":"unknown","cost":0,"tokens":{"input":10,"output":5,"reasoning":2,"cache":{"read":3,"write":4}},"time":{"created":1733011200000}}"#, + ) + .unwrap(); +} + +fn graph_clients(home: &Path) -> Vec { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + let graph: GraphResult = runtime + .block_on(generate_local_graph_report(options(home, None))) + .unwrap(); + graph.summary.clients +} + +#[test] +#[serial_test::serial] +fn source_aware_filter_parity_fixture_is_stable_cold_and_warm() { + let cache_home = tempfile::TempDir::new().unwrap(); + let source_home = tempfile::TempDir::new().unwrap(); + let extra_dir = format!( + "codebuff:{}", + source_home + .path() + .join(".config/manicode/projects") + .display() + ); + let _env = EnvGuard::isolate( + &[ + ("HOME", cache_home.path().as_os_str()), + ("TOKSCALE_CONFIG_DIR", cache_home.path().as_os_str()), + ("TOKSCALE_PRICING_CACHE_ONLY", OsStr::new("1")), + ("TOKSCALE_EXTRA_DIRS", OsStr::new(&extra_dir)), + ( + "XDG_DATA_HOME", + source_home.path().join(".local/share").as_os_str(), + ), + ], + &[ + "APPDATA", + "CODEBUFF_DATA_DIR", + "CODEX_HOME", + "COPILOT_OTEL_FILE_EXPORTER_PATH", + "GEMINI_CLI_HOME", + "GJC_CODING_AGENT_DIR", + "GOOSE_PATH_ROOT", + "GROK_HOME", + "HERMES_HOME", + "JCODE_HOME", + "KIMI_CODE_HOME", + "LOCALAPPDATA", + "TOKSCALE_HEADLESS_DIR", + "XDG_CONFIG_HOME", + ], + ); + write_fixture(source_home.path()); + + let clients = graph_clients(source_home.path()); + assert!(clients.iter().any(|client| client == "codebuff")); + assert!(clients.iter().any(|client| client == "claude")); + assert!(clients + .iter() + .any(|client| client.starts_with("cc-mirror/"))); + assert!(clients.iter().any(|client| client == "kimi")); + assert!(clients.iter().any(|client| client == "opencode")); + + let cold = reports(source_home.path(), &clients); + assert_eq!( + cold.hourly_nil, cold.hourly_full, + "cold hourly nil/full parity" + ); + assert_eq!( + cold.agents_nil, cold.agents_full, + "cold Agents nil/full parity" + ); + assert!(cold.agents_nil.entry_count >= 1); + assert!(cold.agents_nil.message_count >= 1, "Main must retain usage"); + + // Explicit slices prove the report producer, rather than a shared mixed + // bucket, applies the requested client filter before aggregation. The + // exact cc-mirror and synthetic seams are otherwise easy for nil/full + // parity to miss because both paths could be wrong in the same way. + let variant = reports(source_home.path(), &["cc-mirror/kimi-code".to_string()]); + assert_eq!( + (variant.hourly_full.input, variant.hourly_full.output), + (300, 70) + ); + assert_eq!( + (variant.agents_full.input, variant.agents_full.output), + (300, 70) + ); + + let claude = reports(source_home.path(), &["claude".to_string()]); + assert_eq!( + (claude.hourly_full.input, claude.hourly_full.output), + (100, 50) + ); + assert_eq!( + (claude.agents_full.input, claude.agents_full.output), + (100, 50) + ); + + let codebuff = reports(source_home.path(), &["codebuff".to_string()]); + assert_eq!( + (codebuff.hourly_full.input, codebuff.hourly_full.output), + (200, 80) + ); + assert_eq!( + (codebuff.agents_full.input, codebuff.agents_full.output), + (200, 80) + ); + + let synthetic = hourly_report(source_home.path(), Some(vec!["synthetic".to_string()])); + assert_eq!( + synthetic.entries.len(), + 1, + "synthetic filter returns gateway row" + ); + assert_eq!( + (synthetic.entries[0].input, synthetic.entries[0].output), + (10, 5) + ); + + let synthetic_agents = agents_report(source_home.path(), Some(vec!["synthetic".to_string()])); + assert_eq!( + synthetic_agents.entries.len(), + 1, + "synthetic filter returns one Agents row" + ); + assert_eq!( + ( + synthetic_agents.entries[0].input, + synthetic_agents.entries[0].output + ), + (10, 5) + ); + + let all_agents = agents_report(source_home.path(), None); + let main = all_agents + .entries + .iter() + .find(|entry| entry.agent == "Main") + .expect("unattributed usage must remain in Main"); + assert_eq!((main.input, main.output), (710, 255)); + + let warm = reports(source_home.path(), &clients); + assert_eq!( + warm, cold, + "warm cache must preserve cold report aggregates" + ); + assert_eq!( + warm.hourly_nil, warm.hourly_full, + "warm hourly nil/full parity" + ); + assert_eq!( + warm.agents_nil, warm.agents_full, + "warm Agents nil/full parity" + ); +}