diff --git a/.gitignore b/.gitignore index 139e033..1349ca5 100644 --- a/.gitignore +++ b/.gitignore @@ -8,12 +8,8 @@ __pycache__/ /artifacts/ /.audit/ /REPORT.md -/evidence.jsonl -/candidates.jsonl -/chats.jsonl -/judgments.jsonl -/task-source.jsonl -/task-inventory-draft.jsonl -/task-inventory.jsonl -/read-attributions.jsonl -/effect-judgments.jsonl +/facts.json +/facts.md +/sample.json +/judgments.json +*.zip diff --git a/README.md b/README.md index bc31cc6..740ed75 100644 --- a/README.md +++ b/README.md @@ -1,229 +1,124 @@ # Context Tree Value Audit -`context-tree-value-audit` 0.5.0 is an explicit-only Skill for task-first, -evidence-first analysis of Context Tree decision value for the current First -Tree Runtime when its native historical evidence is supported. It reconstructs -single-Agent-owned continuous Tasks from authorized Chats, records whether a -Tree Read is observed or unresolved, and judges zero or more independent -Effects without a minimum sample gate. - -The 0.2 series renamed the installable Skill from -`context-tree-insights` to `context-tree-value-audit`. Replace the old Skill -directory during upgrade; do not install both names because they represent one -explicit audit capability, not two independent workflows. - -Version 0.5 replaces the combined schema-v3 Task judgment with three ordered -schema-v4 artifacts: frozen Task inventory, Read attributions, and Effect -judgments. Do not reuse schema-v1/v2/v3 judgment files. Reconstruct Tasks from -the message-only projection, freeze the inventory digest, then rerun Read and -Effect analysis. No migration or compatibility reader is provided. - -The audit core remains separate from First Tree core. Codex, Claude Code, and -Claude Code TUI use their existing native local transcripts. Cursor and Kimi -Code remain unsupported for historical value audits because their existing -local records cannot yet prove complete, Chat-bound Tree reads; affected Reads -are unresolved. -There is no shared Tree-read CLI, generic tool abstraction, runtime event, -database table, schedule, Context Tree write, or Web surface. Each run covers -one First Tree Agent, one managed workspace, one current Runtime, and one bound -Tree. - -## Safety and interpretation - -- Invocation is explicit only: `$context-tree-value-audit` in Codex or - `/context-tree-value-audit` in Claude Code / Claude Code TUI. -- The invoking human may authorize the current Chat, exact Chat UUIDs, or all - Chats visible to this one current Agent. The Skill trusts that explicit - scope and never broadens it or crosses to another Agent. -- `Chat UUID @ Agent UUID` remains the authorization and trace-mapping unit; - a single-Agent-owned continuous work episode is the Task judgment and - counting unit. -- Short continuations, status prompts, context-dependent questions, repeated - review/fix requests, and phases of one delivery do not become separate Tasks. -- Clear Tasks carry source-backed objective and same-Agent outcome messages; - work owned by another Agent remains context until the audited Agent receives - or visibly accepts an objective. That acceptance is an analyst judgment from - work messages; the reporter does not infer the addressee of arbitrary prose. -- Task reconstruction uses a message-only projection that excludes - collector-derived Reads, passages, Tree-mention indexes, receipts, choice - projections, and Effect judgments; original work-message content remains - unchanged. Its normalized inventory is frozen before Read attribution. -- Local Runtime evidence is preflighted against authorized Chat and Agent IDs - before complete recorded output is scanned. -- Missing, cleaned, ambiguous, malformed, truncated, or unsupported traces are - coverage gaps. -- A valid `contextDecision` is projected minimally. Absence is unknown; - malformed metadata is diagnostic and never blocks Chat export; repository - identities must be remote and credential-free. -- A Tree read is evidence of explicit activity, not semantic use or causal - value by itself. -- Single-file reads and statically closed read-only composites are recovered; - dynamic or unknown shapes stay unresolved, and unsafe shapes are rejected. -- A read attempt with one completed, non-empty, attributable result and no - explicit failure signal may become candidate evidence. Missing, failed, - duplicate, pending, or out-of-window results stay unresolved. -- Read is only `observed` or `unresolved`; unresolved is never counted as - unused. -- Each Task has zero or more Effects; each is only `confirmed`, `constrained`, - `redirected`, or `conflicted`. -- A decision receipt may support an Effect but cannot create one by itself. -- There is no fixed Task quota, task-type gate, or saturation state. -- The output is a sampled evidence report, not causal proof, ROI, or a - global effectiveness rate. - -The audit writes only private local artifacts in the invoking Agent workspace. -Never commit real Chat exports, traces, passages, task judgments, evidence -JSONL, reports, or production-derived artifacts. - -## Repository layout +A First Tree Skill that answers two questions about one agent's Context Tree +usage: + +1. **Observed exposure** — which nodes have a recorded read, which have none, + and what write events reached the feed? +2. **Influence** — for a random sample of reads, did the read change what the + agent did next? + +Exposure counts recorded events — a lower bound, not complete activity. +Influence is sampled, judged by a model, and +**every claimed effect must survive an adversarial pass that tries to explain the +same choice without the Tree**. The report publishes the refutation rate, and +withholds the influence numbers entirely when most claims are refuted. + +## Why it looks like this + +Version 1.0 is a rewrite around `first-tree tree io`, the agent-scoped feed of +durable Context Tree IO. Exposure is now a **recorded fact** rather than +something reconstructed from local runtime transcripts. + +The previous releases mined Codex/Claude transcripts and statically parsed shell +commands to guess which nodes had been read. That approach reached only ~9% +exact attribution on its own pilot, supported two runtimes, and cost roughly a +quarter of the codebase. The runtime already records the same facts at +tool-execution time, across every runtime, in a table that outlives session +timelines — so the audit consumes that instead. + +Deleting the reconstruction layer removed the reason for most of the rest. +Because the analysis unit is now a recorded read rather than an +analyst-reconstructed "task", there are no task boundaries to draw, so the +blinding protocol, the frozen inventory, the digest, and the exclusion taxonomy +all lost their purpose. What remains is a small deterministic layer plus one +judgment. + +There is no schema migration from 0.x. Artifacts from earlier versions are not +readable and should be discarded. + +## Known recording gaps + +The audit prints these in every report, and you should repeat them whenever you +quote a number: + +- **Read telemetry is best-effort, and pipeline shell reads are not recorded at + all.** `cat NODE.md | head -40` produces no event, and that is a common way to + read a long file. **Exposure is a lower bound, never a rate**, and "no observed + read" never means "never read". +- **Search reads are directory-granular.** `Grep` / `Glob` record one event for + the search root, not one per matched node, so any node under a recorded search + root is excluded from the no-observed-read list. +- **Write events are telemetry-only.** They miss merge commits and worktree edits + outside the bound path. Complete write activity comes from the Tree + repository's git history, not from this feed. +- **Node text is a candidate snapshot.** It is reconstructed from the checkout + HEAD observed at read time, which is not a promise that the working file + matched that commit. + +These gaps live in First Tree's recording layer, not in this skill. + +## Install + +Package the Skill directory as a ZIP and upload it as a **Team Skill Resource**, +then bind it to the agents that should have it. First Tree materializes it into +each agent's runtime skill root and manages its version; there is no manual copy, +per-runtime install, or rollback ceremony. -```text -skills/context-tree-value-audit/ - SKILL.md - VERSION - agents/openai.yaml - references/ - evidence-schema.md - runtime-evidence-adapters.md - task-analysis-schema.md - scripts/context_tree_value_audit.py -projections/claude/context-tree-value-audit/ - SKILL.md -tests/ -evals/manual-behavior-checklist.md +```bash +cd skills && zip -r ../context-tree-value-audit.zip context-tree-value-audit ``` -`skills/context-tree-value-audit` is the canonical payload. The small Claude -projection supplies Claude's manual-invocation metadata and delegates to the -canonical payload. Tests, evaluation material, and repository documentation -stay outside both. +The Skill stays explicit-invocation only: Codex `allow_implicit_invocation: +false` and Claude `disable-model-invocation: true` keep it out of ordinary tasks. -## Install into one Agent workspace +## Use -This repository does not install or enable the Skill automatically. Project -the Skill directory into one selected Agent workspace. For a fresh install: +A human invokes `/context-tree-value-audit` (Claude) or +`$context-tree-value-audit` (Codex). Everything after that runs without further +human input; the human reads the report at the end. ```bash -CTVA_REPO="/absolute/path/to/context-tree-insights" -CTVA_AGENT_WORKSPACE="/absolute/path/to/selected/agent/workspace" -CTVA_SKILLS_ROOT="$CTVA_AGENT_WORKSPACE/.agents/skills" -CTVA_DESTINATION="$CTVA_SKILLS_ROOT/context-tree-value-audit" -CTVA_CLAUDE_ROOT="$CTVA_AGENT_WORKSPACE/.claude/skills" -CTVA_CLAUDE_DESTINATION="$CTVA_CLAUDE_ROOT/context-tree-value-audit" -CTVA_CLAUDE_SOURCE="$CTVA_REPO/projections/claude/context-tree-value-audit" - -test ! -e "$CTVA_DESTINATION" -test ! -e "$CTVA_CLAUDE_DESTINATION" -test ! -L "$CTVA_CLAUDE_DESTINATION" -mkdir -p "$CTVA_SKILLS_ROOT" "$CTVA_CLAUDE_ROOT" -cp -R "$CTVA_REPO/skills/context-tree-value-audit" "$CTVA_DESTINATION" -cp -R "$CTVA_CLAUDE_SOURCE" "$CTVA_CLAUDE_DESTINATION" -diff -qr "$CTVA_CLAUDE_SOURCE" "$CTVA_CLAUDE_DESTINATION" -test -f "$CTVA_CLAUDE_DESTINATION/SKILL.md" -python3 "$CTVA_REPO/scripts/validate_skill.py" -``` +# 1. Facts — recorded events only, no sampling, no judgment +python3 scripts/context_tree_value_audit.py facts \ + --tree-root /path/to/context-tree --since 2026-07-01T00:00:00Z -For an upgrade from the old 0.2.x name, move the exact legacy payload to a -recoverable directory outside every Skill discovery root, then install and -compare the new payload: +# 2. Sample — uniform over observed file reads of normal content +python3 scripts/context_tree_value_audit.py sample \ + --tree-root /path/to/context-tree --size 40 --seed 1 --output sample.json -```bash -CTVA_REPO="/absolute/path/to/context-tree-insights" -CTVA_AGENT_WORKSPACE="/absolute/path/to/selected/agent/workspace" -CTVA_SKILLS_ROOT="$CTVA_AGENT_WORKSPACE/.agents/skills" -CTVA_OLD="$CTVA_SKILLS_ROOT/context-tree-insights" -CTVA_NEW="$CTVA_SKILLS_ROOT/context-tree-value-audit" -CTVA_SOURCE="$CTVA_REPO/skills/context-tree-value-audit" -CTVA_QUARANTINE="$CTVA_AGENT_WORKSPACE/.skill-quarantine/context-tree-insights" -CTVA_CLAUDE_ROOT="$CTVA_AGENT_WORKSPACE/.claude/skills" -CTVA_OLD_CLAUDE="$CTVA_CLAUDE_ROOT/context-tree-insights" -CTVA_NEW_CLAUDE="$CTVA_CLAUDE_ROOT/context-tree-value-audit" -CTVA_OLD_CLAUDE_TARGET="../../.agents/skills/context-tree-insights" -CTVA_CLAUDE_SOURCE="$CTVA_REPO/projections/claude/context-tree-value-audit" -CTVA_QUARANTINE_CLAUDE="$CTVA_AGENT_WORKSPACE/.skill-quarantine/context-tree-insights.claude-link" - -test -f "$CTVA_OLD/SKILL.md" -test "$(sed -n 's/^name:[[:space:]]*//p' "$CTVA_OLD/SKILL.md")" = "context-tree-insights" -test ! -e "$CTVA_NEW" -test ! -e "$CTVA_NEW_CLAUDE" -test ! -L "$CTVA_NEW_CLAUDE" -test ! -e "$CTVA_QUARANTINE" -test ! -e "$CTVA_QUARANTINE_CLAUDE" -test ! -L "$CTVA_QUARANTINE_CLAUDE" -mkdir -p "$(dirname "$CTVA_QUARANTINE")" "$CTVA_CLAUDE_ROOT" -if test -e "$CTVA_OLD_CLAUDE" || test -L "$CTVA_OLD_CLAUDE"; then - test -L "$CTVA_OLD_CLAUDE" - test "$(readlink "$CTVA_OLD_CLAUDE")" = "$CTVA_OLD_CLAUDE_TARGET" - mv "$CTVA_OLD_CLAUDE" "$CTVA_QUARANTINE_CLAUDE" -fi -mv "$CTVA_OLD" "$CTVA_QUARANTINE" -cp -R "$CTVA_SOURCE" "$CTVA_NEW" -cp -R "$CTVA_CLAUDE_SOURCE" "$CTVA_NEW_CLAUDE" -diff -qr "$CTVA_SOURCE" "$CTVA_NEW" -diff -qr "$CTVA_CLAUDE_SOURCE" "$CTVA_NEW_CLAUDE" -test -f "$CTVA_NEW_CLAUDE/SKILL.md" -python3 "$CTVA_REPO/scripts/validate_skill.py" -test ! -e "$CTVA_OLD" +# 3. The model judges each case, then tries to refute every claim. + +# 4. Report +python3 scripts/context_tree_value_audit.py report \ + --tree-root /path/to/context-tree \ + --sample sample.json --judgments judgments.json --output REPORT.md ``` -To roll back, move the new payload aside and restore the quarantined directory: +`facts` alone is useful and carries no judgment risk. The **no-observed-read +list** is an evidence gap to take to a human, not a deletion proposal. -```bash -test -d "$CTVA_QUARANTINE" -test -d "$CTVA_NEW" -test -d "$CTVA_NEW_CLAUDE" -diff -qr "$CTVA_CLAUDE_SOURCE" "$CTVA_NEW_CLAUDE" -mv "$CTVA_NEW_CLAUDE" "$CTVA_QUARANTINE.failed-new.claude" -mv "$CTVA_NEW" "$CTVA_QUARANTINE.failed-new" -mv "$CTVA_QUARANTINE" "$CTVA_OLD" -if test -L "$CTVA_QUARANTINE_CLAUDE"; then - mv "$CTVA_QUARANTINE_CLAUDE" "$CTVA_OLD_CLAUDE" -fi -``` +`report` refuses to publish effect counts unless the sample came from the same +Tree identity, window, and eligible read population, so a stale sample cannot +produce influence numbers against a feed it was never drawn from. -Start a new Runtime session after a successful install, upgrade, or rollback, -then confirm the intended single Skill name is callable. The Codex -`allow_implicit_invocation: false` policy and Claude -`disable-model-invocation: true` frontmatter keep the Skill out of ordinary -tasks. Pin a reviewed commit or release when installing for another Agent. - -## Pipeline - -The Skill orchestrates six operational steps across three analysis stages: - -1. `export-chats` resolves explicit authorization and exports visible records. -2. `collect` maps authorized Chats to supported native local evidence, - classifies every in-window Tree-read attempt into a conserving four-state - grammar, reconstructs exact or read-only-composite evidence plus visible - choices, and distinguishes local default-branch matches from unverified - sources. Unsupported Runtime history produces unresolved Reads. -3. `task-source` removes every collector-derived Tree/Read/Effect projection - and keeps only authorized work messages. It does not redact literal - Tree/Read/Effect discussion from the work itself. -4. The Agent reconstructs Tasks from that projection; `freeze-tasks` validates - and writes a digest-bound `task-inventory.jsonl`. -5. The Agent writes digest-bound Read attributions, then zero-or-more Effect - judgments for each clear Task without changing the frozen inventory. -6. `report` validates Task sources, weak fragment-only objectives, source - ownership, windows, cross-Chat linkage, Read/choice timing, inventory - digests, deduplication, multi-Effect conservation, and creates - `evidence.jsonl` and `REPORT.md`. An optional hash-anchored reviewed baseline - remains separate from the current rerun. - -There is no default time window. `--days` is an optional data-acquisition -bound. Every available Task in the authorized bound is reported; sample size -limits the conclusion rather than whether a report can be produced. - -Detailed commands and schemas are in -[`SKILL.md`](skills/context-tree-value-audit/SKILL.md), -[`evidence-schema.md`](skills/context-tree-value-audit/references/evidence-schema.md), -[runtime-evidence-adapters.md](skills/context-tree-value-audit/references/runtime-evidence-adapters.md), -and -[`task-analysis-schema.md`](skills/context-tree-value-audit/references/task-analysis-schema.md). +Use `--events-file` to replay a captured `tree io --json` payload instead of +calling the CLI. -## Validate +## Layout -Run the deterministic floor before publishing: +```text +skills/context-tree-value-audit/ + SKILL.md the workflow and judgment rules + VERSION + agents/openai.yaml + references/judging-effects.md rubric, worked examples, refutation guide + scripts/context_tree_value_audit.py +tests/ +evals/manual-behavior-checklist.md +``` + +## Validate ```bash python3 scripts/validate_skill.py @@ -231,7 +126,16 @@ python3 -m compileall -q skills tests scripts python3 -m unittest discover -s tests -v ``` -Then execute -[`evals/manual-behavior-checklist.md`](evals/manual-behavior-checklist.md) -against a designated pilot Agent with authorized disposable or sanitized -records. Model-backed evaluation remains outside the deterministic gate. +Then run [`evals/manual-behavior-checklist.md`](evals/manual-behavior-checklist.md) +against a pilot agent with authorized records. + +## Safety + +Read-only. It reads the agent's own IO feed and the bound Tree; it never writes +Tree content, Chats, git state, or product state. Artifacts stay private to the +invoking agent's workspace at mode `0600` and must never be committed — they +contain Tree content and chat-derived material. + +The output is a sampled evidence report. It is not causal proof, an +effectiveness rate, or ROI. Missing evidence is unknown, never proof that a node +went unread or that the Tree went unused. diff --git a/evals/manual-behavior-checklist.md b/evals/manual-behavior-checklist.md index 6499a0e..1e35f39 100644 --- a/evals/manual-behavior-checklist.md +++ b/evals/manual-behavior-checklist.md @@ -1,339 +1,89 @@ # Manual Behavior Checklist -Use this checklist before admitting a `context-tree-value-audit` revision. Run it -in designated First Tree Agent workspaces against disposable or sanitized -records. Never commit generated artifacts. - -For a 0.5.0 upgrade, confirm the installed payload exposes only -`$context-tree-value-audit`; the superseded `$context-tree-insights` directory -must not remain as a second callable Skill. Exercise the README's minimal -move/copy/compare flow and confirm the old payload is outside both Skill -discovery roots. Confirm the Claude manual-invocation projection delegates to -the new canonical payload. - -## Run record - -Record in a private note: - -- date, tester, repository commit, and installed Skill commit; -- Agent name/UUID, managed workspace, and bound Tree; -- First Tree binary/channel and authorization mode; -- exact Chat UUIDs when using `explicit_chat`; -- optional acquisition bound; -- artifact directory; -- clear Task count and acquisition bound; -- current Runtime and evidence-adapter version. - -Capture the Tree HEAD, `git status --short`, and initial artifact directory -listing. Do not copy real trace passages into the test note. - -## 1. Explicit routing and scope - -In a fresh Chat, explicitly invoke `$context-tree-value-audit` in Codex or -`/context-tree-value-audit` in Claude and authorize either all Chats for the -current Agent or exact Chat UUIDs. - -Pass when: - -- the Skill loads only after explicit invocation; -- the scope contains one exact Agent, workspace, Tree, and authorization mode; -- the immutable runtime Agent slug is used for CLI selection even when the - workspace `displayName` differs; -- a missing or malformed runtime slug, or a runtime/workspace UUID mismatch, - fails closed; -- current underscore/trailing-separator Agent names and the complete - historical `[a-z0-9_-]{1,100}` grammar, including leading `-` / `_`, remain - selectable for still-runnable grandfathered Agents, while names outside the - producer grammar fail closed; -- the First Tree CLI's producer-owned local binding resolution of the runtime - slug must return the same UUID as later `chat --agent` calls; the Skill does - not reimplement the local YAML parser or persist/enumerate other Agents into - the audit scope; -- `explicit_agent` is used only after the human explicitly asks for all Chats - visible to the current Agent; -- `explicit_chat` is used for the invoking Chat or exact supplied Chat UUIDs; -- neither mode carries human, organization, or extra authorization-context - fields, and neither can cross to another Agent or workspace; -- another Agent, workspace, Tree, or unlisted Chat fails closed; -- only one private artifact directory is created; -- ordinary tasks do not load the Skill or scan history; -- `policy.allow_implicit_invocation` remains `false`. - -## 2. Runtime evidence adapters remain isolated - -Run the supported matrix with sanitized records: - -- Codex root session JSONL; -- Claude Code root transcript JSONL; -- Claude Code TUI through the same Claude transcript family. - -Pass when: - -- the selected adapter exactly matches `FIRST_TREE_PROVIDER`; -- missing or invalid `FIRST_TREE_PROVIDER` fails closed instead of defaulting - to an adapter; -- every adapter maps one authorized Chat, current Agent, workspace, and bound - Tree before accepting complete output; -- Claude sidechain/subagent rows, tool-result echoes, and later session/Chat - drift do not become root evidence; -- Cursor and Kimi Code produce explicit - `historical_evidence_not_supported` gaps, no read IDs, and unresolved Reads; -- no First Tree Runtime handler, local state schema, server, database, or Web - surface is changed for the audit; -- equal-timestamp results appended before their calls remain unresolved; -- a missing, cleaned, malformed, or unmapped source remains unresolved; -- the audit core emits the same candidate and Task schema for every Runtime. - -## 3. Collector safety remains intact - -Use sanitized traces covering: - -- canonical current-context plus matching mirror; -- conflicting mirror, mirror-only, missing Chat ID, and two canonical Chat IDs; -- successful direct file read; -- `functions.exec` with one literal nested read, `Promise.all`, multiple - literal reads, static `for`, multi-path operands, safe pipelines, - predicates, labels, hierarchy selectors, and line counts; -- dynamic interpolation, unknown programs, stdin, suffix lookalikes, failed - output, and pending/completed continuations; -- null-sink output, file output, git mutation, network programs, literal - non-Tree paths, and Git commands with implicit configured-helper potential, - including index-backed revision resolution and unsafe global options; -- a valid `first-tree tree tree --no-pull` selector and exact help form beside - the default-refresh form and a mutating First Tree namespace that merely - contains the same `tree tree` token pair; -- ordinary whitelisted `rg` options beside `--file`, `--ignore-file`, and an - unknown option, including both separate-value and `--option=value` forms, - with accepted calls requiring `--no-config --no-ignore` and both - implicit-config and implicit-ignore forms rejected; -- bare allowed readers beside `./rg`, `/tmp/rg`, path-qualified - `first-tree-staging`, and another path-qualified reader basename; -- bare and forced-paging `bat` forms, plus a no-config/never-page-looking - form, all rejected because `bat` is outside the accepted reader grammar; -- benign-looking and file-valued `find` forms rejected alongside explicit - mutating actions because `find` is outside the accepted diagnostic grammar; -- bare and recursive/dereferencing `ls` forms rejected, plus pipeline - `head`/`tail`/`nl` help, follow, and unknown options rejected by a closed - filter grammar; -- Claude Tree-reading `tool_use` rows with a missing result, duplicate result, - duplicate call ID, a result after the acquisition end, and the same pairing - failures wholly before the acquisition start; -- another Tree path and an unauthorized trace with a unique sentinel. - -Pass when: - -- authorization preflight happens before full content scanning; -- only the exact authorized root Codex trace is mapped; -- exact and statically closed read-only composite reads of the bound Tree are - retained, with nested output sliced when provider blocks preserve it; -- each in-window Tree-read attempt lands in exactly one of - `accepted_exact`, `accepted_read_only_composite`, `unresolved_opaque`, or - `rejected_unsafe`, and the four counts conserve the total; -- calls outside the acquisition window do not contaminate attempt counts or - coverage gaps; a call started in-window with a result after the acquisition - end remains one unresolved attempt; -- dynamic/unknown calls remain unresolved; mutation, file output, network - access, and Tree-external reads are rejected; -- unauthorized, unresolved, and unsafe sentinels never appear in output; -- deterministic labels are removed from passages; unseparated dynamic - diagnostics produce no read ID, even when the command grammar itself is - accepted; -- wrapper aliases, output mutation, reversed forwarding, callback side - effects, duplicate properties, unsafe or config-driven Git diagnostics, - and `rg` patterns that merely look like Markdown paths all fail closed; -- initial calls and exact `write_stdin`/`wait` continuations form one attempt; -- an explicit direct-reader error envelope produces no read ID; -- a missing or duplicate continuation result keeps the whole parent read - unresolved even if a later continuation appears successful; -- every attributable in-window Claude Tree-reading call stays in the attempt - denominator; missing, failed, pending, duplicate, incomplete, or - out-of-window pairing is `unresolved_opaque` and produces no read ID; -- gaps remain diagnostic rather than being turned into negative exposure; -- artifacts keep `0700`/`0600` permissions and opaque local identities. - -## 4. Receipt present, absent, and malformed - -Export three sanitized messages: - -1. a valid `metadata.contextDecision` v1 with unrelated metadata and extra - receipt fields; -2. no `contextDecision`; -3. a malformed `contextDecision`. - -Pass when: - -- the valid receipt is saved as the minimal v1 projection only; -- unrelated message metadata and extra receipt fields are not copied; -- credential-bearing URLs, local paths, loopback hosts, and non-repository - identities are omitted without echoing their raw values; -- receipt absence stays unknown and adds no negative gap; -- malformed receipt is omitted with `context_decision_invalid`; -- no malformed receipt blocks the message, Chat export, or audit. - -## 5. Pure Task reconstruction and freeze - -Use one Chat containing one objective across planning, implementation, review, -QA, correction, and final delivery, followed by a genuinely independent -objective and outcome. Include short continuation/status prompts; polite or -deictic variants such as `please continue`, `status please`, `修一下吧`, -`please continue fixing it`, `please fix it`, `continue the work`, -`继续修一下`, `修复一下`, `修这个`, and `帮忙修下`; edge-decorated forms -such as `“please continue”`, `please continue 🙏`, and `请继续（谢谢）`; -one or more leading Agent mentions; and concrete objectives that contain -`continue` while naming an independent scope and deliverable. - -Pass when: - -- `task-source.jsonl` contains authorized work messages but no - collector-derived Reads, passages, Tree-mention indexes, decision receipts, - choice projections, or Effect judgments; -- original work-message content remains byte-for-byte intact even when the - work itself discusses Tree, Read, or Effect, and those literal terms are not - treated as audit evidence during Task reconstruction; -- all phases and continuations for the first deliverable remain one Task; -- it produces two Task rows with separate source messages and windows; -- clear Tasks have a concrete objective, independent same-Agent outcome, - bounded source fragments, and explicit objective/outcome sources; -- material scope and primary deliverable may clarify the boundary but are not - required for research, diagnosis, review, or decision Tasks; -- weak prompt variants and weak normalized objectives are merged or excluded, - while concrete `continue` objectives remain eligible; -- one Agent message cannot simultaneously serve as objective and terminal - outcome; -- every outcome source is a non-empty current-Agent message, so a later human - follow-up cannot extend the episode or become the bound Effect outcome; -- work owned by another Agent remains context until the audited Agent receives - or visibly accepts an objective; -- a candidate missing any clear-Task gate is excluded with the deterministic - structured exclusion kind; -- excluded candidates contain no clear-Task sources, Read, or Effect judgment; -- `freeze-tasks` writes one normalized inventory digest to every row; -- changing any frozen Task row invalidates the digest; -- Read/Effect fields in a Task inventory draft are rejected. - -## 6. Cross-Chat handoff merge - -Use two Chats for one PR/MR/Issue or a visible handoff. - -Pass when: - -- one Task may contain both source fragments when each carries the same - `work_item`, `explicit_handoff`, or `same_objective_delivery` linkage; -- the same fragments without linkage are rejected; -- unrelated Chats cannot be merged by topical similarity alone. - -## 7. Observed and unresolved Read - -Create one Task with attributable Task-window reads and one historical Task -whose trace coverage cannot resolve the Read. - -Pass when: - -- the first is `observed` with valid Read IDs; -- the second is `unresolved` with a reason; -- unresolved Read has no Read IDs and an empty Effect list; -- no `not_observed`, `unused`, or negative-value state is emitted; -- Reads outside the frozen Task window or source Chats are - rejected; -- one Read cannot be attributed to different Tasks; -- unresolved Tasks appear in coverage counts, never an unused denominator. - -## 8. Zero-or-more Effects - -Prepare valid examples of `confirmed`, `constrained`, `redirected`, and -`conflicted`, including one Task with two distinct Effects supported by the -same Read, plus observed-Read Tasks with no Effect. - -Pass when: - -- unknown Effects, confidence tiers, and numeric weights are rejected; -- one Task can carry multiple Effects and the report separately counts Effect - Tasks and total Effects; -- every Effect has observed Task Reads, later same-Agent choices, an outcome - message inside the Task, and a summary; -- the Effect outcome message is not earlier than any cited Read completion - or choice; -- post-choice Reads and out-of-window choices are rejected; -- an empty Effect list requires one short `effect_reason`; -- a decision receipt alone does not create an Effect; -- superseded task types, sampling fields, `verified` / `probable`, rubrics, - and support fields are rejected; -- the same Read may support multiple distinct choices; -- one choice cannot be reused across Effects; -- unresolved Reads cannot have Effects. - -## 9. Sample handling - -Run sanitized reports with 1, 16, and 44 clear Tasks. - -Pass when: - -- every available clear Task is reported without a minimum quota; -- no task-type coverage, batch expansion, or saturation state is emitted; -- `--days`, when supplied, remains only an acquisition bound; -- a Chat with messages inside the bound but later activity after `window.end` - is fetched and filtered rather than omitted. - -## 10. Report conservation and language - -Pass when the report includes: - -- a complete frozen Task inventory with objective, optional scope/deliverable, - outcome, source IDs, Read, zero-or-more Effects, and evidence summary; -- the Task inventory digest and every clear Task's objective/outcome sources; -- a complete excluded-candidate inventory with exclusion kind, observed scope, - and reason; -- observed and unresolved Read Tasks; -- Effect Tasks, total Effects, and observed Reads without an Effect; -- the four-Effect distribution; -- authorized Chat/message/trace coverage and gaps; -- the four-class in-window Tree-read attempt conservation table. - -Verify: - -- observed + unresolved = clear Tasks; -- Effect Tasks + observed Read without Effect = observed Read Tasks; -- the four Effect counts sum to total Effects; -- total Effects is greater than or equal to Effect Tasks; -- the report does not output a global effectiveness rate; -- Read counts, receipts, and unresolved gaps are not represented as causal - value or non-value. -- the report labels itself sampled and does not imply remote provenance, - causal proof, or ROI; -- an optional reviewed baseline is hash-anchored, internally conserving, and - rendered separately; it does not alter current Reads or Effects. - -For the 0.2.1 historical pilot rerun, the 210 in-window calls must all remain -accounted for. Compare the new result with the grammar-only overlay -(`19 exact + 145 read-only composite + 42 opaque + 4 unsafe`). Any deliberate -delta must name the command-shape class and the stricter attribution or safety -reason. Do not tune the parser merely to reproduce the target counts. - -After collection, redo Task-level Read and Effect judgment. Never reuse -the old blanket `unresolved` / empty-Effect rows as negative cases, and never -let the current rerun erase the separately reviewed 37 positive effect Tasks. - -## 11. No product or source mutation - -Compare pre-run and post-run state. - -Pass when: - -- bound Tree HEAD, files, and `git status --short` are unchanged; -- repository files and Agent configuration are unchanged; -- existing Chats, traces, schedules, receipts, database, and product state are - unchanged; -- no runtime event, message-path validation, raw IO API, Context Tab, Tree - write, schedule, or other-provider adapter was invoked; -- the only intentional audit writes are private files beneath the artifact - directory; -- the run remains limited to the current Agent, one workspace, one Tree, and - local Codex traces. +Run this against a pilot agent with authorized records before admitting a +revision. Never commit generated artifacts. + +Record: date, tester, repo commit, agent name/UUID, bound Tree, window used, +sample size and seed, and the refutation rate. + +## 1. Routing and scope + +- The Skill loads only after an explicit `/context-tree-value-audit` or + `$context-tree-value-audit` invocation; ordinary tasks do not load it. +- `policy.allow_implicit_invocation` is `false` and the single canonical + `SKILL.md` keeps `disable-model-invocation: true`. There is no separate Claude + projection directory; one payload serves every runtime. +- The run touches only this agent's own feed. There is no option, prompt, or + workaround that widens it to another agent. +- Artifacts land in a private workspace directory at mode `0600` and are not + committed. + +## 2. Facts are observed, and say so + +- `facts` runs with no sample and no judgment. +- Exposure is reported as counts. **No percentage of total work appears + anywhere**, because unrecorded reads make any such rate false. +- All recording gaps are printed: read telemetry best-effort with pipeline reads + unrecorded, search reads directory-granular, write events telemetry-only, and + node text a candidate snapshot. +- The no-observed-read list excludes nodes under a recorded search root. Verify + with a `Grep` of a tree directory: no node beneath it may be listed. +- That list is labelled an evidence gap and carries **no** removal or merge + recommendation. +- The list excludes `AGENTS.md`, `members/`, and `raw-context/`. +- A node read via `Read` in the window does not appear in the list. +- Exposure is reported as counts only; no percentage of total work appears. + +## 3. Sampling is uniform and reproducible + +- The same `--seed` yields the same cases; a different seed yields different + ones. +- Only file-level reads of normal content are eligible. +- The analyst does not hand-pick or re-roll cases to find interpretable ones. +- Case material carries a **candidate snapshot** from the checkout HEAD observed + at read time, labelled `head_commit_snapshot`, with its caveat text. It never + claims to be exactly what the agent read. Verify the honest fallbacks + (`current_working_copy`, `unavailable`) too. +- Editing `cases` in a valid sample — even keeping identity, window, and + population digest — makes `report` refuse to publish effect counts. +- An events feed from another Tree, or a changed branch binding, is excluded + rather than credited to the current Tree. +- A window with no IO at all reports zero counts instead of failing, provided + `--tree-root` pins the Tree. + +## 4. Judgment and the adversarial pass + +- Every sampled case has exactly one judgment; a missing one fails the report. +- `"effect": null` is used freely when no specific later choice is visible. +- **Every claimed effect carries an explicit `refuted` boolean.** Omitting it + fails the report — verify this by deleting one and rerunning. +- `refuted: true` without a `refutation` string fails. +- The refutation pass actually looks for a pre-read human instruction. Construct + a case where the human asked for the outcome before the read and confirm it is + refuted rather than counted. +- Unknown effect types, confidence tiers, and numeric weights are rejected. + +## 5. Report conservation and the reliability floor + +- Effect counts equal the upheld claims; refuted claims are excluded from the + distribution. +- With more than half the claims refuted, the report **withholds the influence + numbers**, says the run is unreliable, and still prints the exposure section. + Verify with a seeded judgment file. +- With a healthy rate, the effect distribution and per-effect summaries appear. +- The report never prints a global effectiveness rate, causal claim, or ROI. +- The report states that missing evidence is unknown, not proof of non-use. + +## 6. No mutation + +Compare before and after: + +- bound Tree HEAD, files, and `git status --short` unchanged; +- Chats, agent configuration, schedules, and product state unchanged; +- the only writes are private files under the artifact directory. ## Sign-off -Record pass/fail and the observable artifact or transcript location for every -section. A failure in explicit routing, authorization, trace preflight, -Agent/workspace/Tree isolation, Task reconstruction, Read semantics, -Effect validation, conservation, or no-mutation behavior -blocks the revision. +Record pass/fail per section. A failure in routing, scope, Tree-identity isolation, +the search guard on the no-observed-read list, sample conservation, the +mandatory adversarial pass, or the reliability floor blocks the revision. diff --git a/projections/claude/context-tree-value-audit/SKILL.md b/projections/claude/context-tree-value-audit/SKILL.md deleted file mode 100644 index 4deaa93..0000000 --- a/projections/claude/context-tree-value-audit/SKILL.md +++ /dev/null @@ -1,11 +0,0 @@ ---- -name: context-tree-value-audit -description: Run the manual, read-only Context Tree decision value audit for this First Tree Agent only when a human explicitly invokes /context-tree-value-audit. -disable-model-invocation: true ---- - -# Context Tree Value Audit - -Read and follow -`../../../.agents/skills/context-tree-value-audit/SKILL.md` completely. Treat -that canonical payload directory as `CTVA_SKILL_DIR`. diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 074be8d..e444900 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -12,32 +12,20 @@ SKILL_ROOT = ROOT / "skills" / "context-tree-value-audit" SKILL_MD = SKILL_ROOT / "SKILL.md" OPENAI_YAML = SKILL_ROOT / "agents" / "openai.yaml" -CLAUDE_SKILL_MD = ( - ROOT / "projections" / "claude" / "context-tree-value-audit" / "SKILL.md" -) EXPECTED_FILES = ( SKILL_MD, OPENAI_YAML, - CLAUDE_SKILL_MD, SKILL_ROOT / "VERSION", SKILL_ROOT / "scripts" / "context_tree_value_audit.py", - SKILL_ROOT / "references" / "evidence-schema.md", - SKILL_ROOT / "references" / "task-analysis-schema.md", - SKILL_ROOT / "references" / "runtime-evidence-adapters.md", + SKILL_ROOT / "references" / "judging-effects.md", ) FORBIDDEN_PATH_FRAGMENTS = ("/Users/", "\\Users\\") FORBIDDEN_ARTIFACT_NAMES = { "REPORT.md", - "evidence.jsonl", - "candidates.jsonl", - "chats.jsonl", - "judgments.jsonl", - "task-judgments.jsonl", - "task-source.jsonl", - "task-inventory-draft.jsonl", - "task-inventory.jsonl", - "read-attributions.jsonl", - "effect-judgments.jsonl", + "facts.json", + "facts.md", + "sample.json", + "judgments.json", } @@ -73,28 +61,20 @@ def validate() -> None: skill_text = SKILL_MD.read_text(encoding="utf-8") frontmatter = parse_frontmatter(skill_text) - if set(frontmatter) != {"name", "description"}: - fail("SKILL.md frontmatter must contain only name and description.") + if set(frontmatter) != {"name", "description", "disable-model-invocation"}: + fail( + "SKILL.md frontmatter must contain exactly name, description, and " + "disable-model-invocation." + ) if frontmatter["name"] != SKILL_ROOT.name: fail("Skill name must match its directory name.") if not frontmatter["description"]: fail("Skill description must not be empty.") - claude_frontmatter = parse_frontmatter( - CLAUDE_SKILL_MD.read_text(encoding="utf-8") - ) - if set(claude_frontmatter) != { - "name", - "description", - "disable-model-invocation", - }: - fail( - "Claude projection frontmatter must contain name, description, " - "and disable-model-invocation." - ) - if claude_frontmatter["name"] != SKILL_ROOT.name: - fail("Claude projection name must match the canonical Skill name.") - if claude_frontmatter["disable-model-invocation"] != "true": - fail("Claude projection must disable model invocation.") + # One payload serves every runtime: the Team Skill Resource bundle is + # materialized into each runtime's own skill root, so a Claude-specific + # projection directory would install a second, divergent copy. + if frontmatter["disable-model-invocation"] != "true": + fail("SKILL.md must disable model invocation.") version = (SKILL_ROOT / "VERSION").read_text(encoding="utf-8").strip() if re.fullmatch(r"[0-9]+\.[0-9]+\.[0-9]+", version) is None: fail("VERSION must contain one semantic version.") diff --git a/skills/context-tree-value-audit/SKILL.md b/skills/context-tree-value-audit/SKILL.md index ca41719..170878e 100644 --- a/skills/context-tree-value-audit/SKILL.md +++ b/skills/context-tree-value-audit/SKILL.md @@ -1,307 +1,188 @@ --- name: context-tree-value-audit -description: Audit how Context Tree reads affected complete work Tasks when a human explicitly invokes $context-tree-value-audit in Codex or /context-tree-value-audit in Claude. The audit is evidence-first, manual, read-only, limited to explicitly authorized Chats for the current First Tree Agent, and reports the available sample without a minimum Task quota. Do not use for ordinary task reads, stored-tree audits, Tree writes, generic Chat analytics, monitoring, or another Agent. +description: Audit whether Context Tree reads changed what an agent did, when a human explicitly invokes /context-tree-value-audit or $context-tree-value-audit. Reads the agent's own durable Context Tree IO feed, reports node-level observed exposure and nodes with no observed read, then judges a random read sample with a mandatory adversarial pass. Do not use for ordinary task reads, stored-tree quality audits, Tree writes, or another agent. +disable-model-invocation: true --- # Context Tree Value Audit ## Capability -Run a manual and read-only retrospective that reconstructs single-Agent-owned -continuous Tasks from authorized Chats, determines whether a Tree Read is -observed or unresolved, and reports zero or more independent Effects where a -Read reasonably confirmed, constrained, redirected, or conflicted with later -choices. +Answer two questions about one agent's own Context Tree usage: -Keep three analysis stages strictly ordered: +1. **Observed exposure** — which nodes have a recorded read, which have none, + and what write events reached the feed? *Recorded events only — a lower + bound, never a complete picture.* +2. **Influence** — for a random sample of reads, did the read change what the + agent did next? *Sampled, and every claim must survive an attempt to refute + it.* -- reconstruct Task from message-only work evidence; -- freeze the Task inventory before any Read is visible to the analyst; -- attribute Reads, then analyze zero or more Effects without changing Task - boundaries. - -`Chat UUID @ Agent UUID` remains the authorization and evidence-source unit. -Task is the work unit. Effect is the independently counted value unit. The -bundled script performs deterministic projection, freezing, reference -validation, deduplication, conservation checks, and reporting. A Read or -decision receipt is evidence, not server-verified causality. +Exposure comes from `context_tree_io_events`, which the runtime records at +tool-execution time and which outlives session timelines. That recording is +best-effort: pipeline shell reads produce no event at all, and write telemetry +misses merge and out-of-path worktree commits. Influence is the only +judgment in the run, and it is deliberately adversarial: the model that claims +an effect must then try to explain the same choice **without** the Tree. ## Gate the run -Proceed only when a human explicitly invokes `$context-tree-value-audit` in -Codex or `/context-tree-value-audit` in Claude and asks for this value audit. -Do not trigger from an ordinary task, a normal Context Tree read, a stored-tree -quality audit, or an implicit analytics request. +Proceed only when a human explicitly invokes `/context-tree-value-audit` +(Claude) or `$context-tree-value-audit` (Codex) and asks for this audit. Do not +trigger from an ordinary task, a normal Tree read, or a stored-tree quality +audit. -Keep the run: +The run is read-only. Do not modify Chats, Tree content, git state, agent +configuration, or product state. Write only inside a private artifact directory +in this agent's workspace, and never commit those artifacts. -- manual and read-only; -- limited to one invoking Agent, one managed workspace, and one bound Context - Tree; -- limited to the invoking Agent's supported local Runtime evidence; -- confined to a new private artifact directory inside the invoking Agent - workspace. +The audit covers **this agent only**. The feed is self-scoped by the server; do +not attempt to widen it. -Do not modify Chat history, traces, Tree content, git state, schedules, agent -configuration, databases, or product state. The visible reply and provider's -automatic trace append are not audit writes. Do not override the current -Runtime provider, invoke another Runtime adapter, or scan another Agent. +## Step 1 — Facts (no sampling, no judgment) -## Authorize the source scope +```bash +python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" facts \ + --tree-root "/absolute/path/to/bound/context-tree" \ + --since 2026-07-01T00:00:00Z \ + --output "$CTVA_ARTIFACT_DIR/facts.md" +``` -Resolve the exact current Agent UUID and bound Tree root from the managed -workspace identity. Resolve the immutable Agent `name` slug used by mentions, -URLs, CLI selectors, and local mirror paths from `FIRST_TREE_AGENT_SLUG`, and -cross-check `FIRST_TREE_AGENT_ID` and the First Tree CLI's producer-owned local -binding resolution of that slug against the workspace UUID. The local listing -is used only for this one identity check and is never persisted or promoted -into the authorized source scope. `displayName` is a mutable UI label and must -never be used as the CLI selector. Reject -symlinks, missing or malformed runtime identity, an Agent mismatch, an unbound -Tree, or more than one workspace or Tree. +`--since` / `--until` / `--chat` are optional filters. Add `--json` for the +structured form when you want to inspect counts directly. -Choose exactly one mode from the human's explicit request: +This step alone is worth reporting. It gives node-level read distribution, +observed write events, and the **list of nodes with no observed read**. -1. `explicit_agent`: all Chats visible to this one current Agent, only when the - human explicitly asks for the current Agent's full Chat scope; -2. `explicit_chat`: exact Chat UUIDs for this Agent, or the explicitly - authorized invoking Chat resolved from runtime `chatId`. +Treat that list as an **evidence gap, not a finding**. A node lands there when no +event reached the feed, which also happens for every pipeline shell read and +whenever best-effort telemetry drops a call. **Never present it as a deletion or +merge proposal.** It is a prompt to ask a human whether a node still earns its +place — nothing stronger. -Trust the human's explicit scope. Do not broaden it, mix modes, infer another -Agent, or scan across workspaces. Ask the human only when scope is ambiguous. +It is conservative on top of that: a node inside a recorded search root is +excluded, because `Grep`/`Glob` record only the search root. -Write `scope.json`: +Observed write events are telemetry-only and therefore incomplete. If someone +needs the real write picture, point them at the Tree repository's git history. -```json -{ - "schema_version": 1, - "agents": [ - { - "name": "current-agent", - "agent_id": "00000000-0000-0000-0000-000000000001", - "authorization": "explicit_agent" - } - ], - "chats": [] -} +## Step 2 — Sample + +```bash +python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" sample \ + --tree-root "/absolute/path/to/bound/context-tree" \ + --since 2026-07-01T00:00:00Z \ + --size 40 --seed 1 \ + --output "$CTVA_ARTIFACT_DIR/sample.json" ``` -or: +Sampling is uniform over recorded file-level reads of normal content. **Do not +hand-pick cases.** Choosing the reads whose surrounding conversation is easiest +to interpret biases the result toward whatever is legible, not toward what is +true. + +Each case carries the read (node path, time, chat) and a **candidate snapshot** +of the node's text, reconstructed from the checkout HEAD observed at read time. + +That snapshot is not guaranteed to be what the agent saw: the recorded commit is +the HEAD observed for the read, not proof that the working file matched it. If +the agent was reading uncommitted Tree edits, the real text is unrecoverable. +Each case states which of `head_commit_snapshot` / `current_working_copy` / +`unavailable` it is. **When the snapshot may not match what was read, do not +claim an effect that depends on specific wording** — use `null`. + +## Step 3 — Judge each case, then try to refute it + +For every case, read the surrounding work in that Chat around the read time — +what the agent said and did after it — and answer one question: + +> **If the agent had not read this node, would the later choice have been +> different?** + +Record one judgment object per case: ```json { - "schema_version": 1, - "agents": [], - "chats": [ - { - "chat_id": "00000000-0000-0000-0000-000000000000", - "agent": "current-agent", - "agent_id": "00000000-0000-0000-0000-000000000001", - "authorization": "explicit_chat" - } - ] + "read_id": "…", + "effect": { "type": "redirected", "summary": "Dropped the password-login plan and used the OAuth provider the node requires." }, + "refuted": false } ``` -Every row must name the same current Agent identity. Do not add human, -organization, or other authorization-context fields; explicit scope is the -complete authorization model. +Use `"effect": null` when nothing defensible is visible. **Uncertainty is not an +effect** — if you cannot point at a specific later choice, the answer is `null`. -## Collect deterministic evidence +Effect types: -Read [references/evidence-schema.md](references/evidence-schema.md) and -[references/runtime-evidence-adapters.md](references/runtime-evidence-adapters.md). -Locate the Skill directory, create a private timestamped artifact directory, -and keep all inputs and outputs inside it. Require directory mode `0700` and -file mode `0600`. Set `FIRST_TREE_BIN` for a channel-specific executable. +- `confirmed` — removed real hesitation and justified keeping the plan; +- `constrained` — ruled an option out or narrowed the acceptable boundary; +- `redirected` — changed the intended approach; +- `conflicted` — surfaced a conflict that still needed resolving. -```bash -python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" export-chats \ - --artifact-root "$CTVA_ARTIFACT_DIR" \ - --scope "$CTVA_ARTIFACT_DIR/scope.json" \ - --agent-workspace "AGENT_UUID=/absolute/current/agent/workspace" \ - --output "$CTVA_ARTIFACT_DIR/chats.jsonl" - -python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" collect \ - --artifact-root "$CTVA_ARTIFACT_DIR" \ - --chats "$CTVA_ARTIFACT_DIR/chats.jsonl" \ - --agent-workspace "AGENT_UUID=/absolute/current/agent/workspace" \ - --tree-root "/absolute/current/agent/bound/context-tree" \ - --output "$CTVA_ARTIFACT_DIR/candidates.jsonl" -``` +### The adversarial pass is mandatory -There is no default lookback. Use `--days N` only when the human explicitly -wants a time-based acquisition ceiling. It limits data fetching; it does not -create a minimum sample requirement. Use `--now` for reproducible reruns. -Normally let the collector resolve the local evidence root from -`FIRST_TREE_PROVIDER`; use `--trace-root` only for an explicitly resolved root -for that same Runtime. - -Before fully scanning a trace, require bounded metadata/current-context -preflight to establish one authorized `chatId` for the exact workspace. Never -search arbitrary full traces to discover an authorized Chat. Classify every -in-window call that references bound-Tree Markdown exactly once as -`accepted_exact`, `accepted_read_only_composite`, `unresolved_opaque`, or -`rejected_unsafe`; the four counts must conserve the attempt total. Allow only -statically closed read-only wrappers, paths, programs, and forwarded outputs. -Keep exact output/continuation pairing and reject writes, mutation, network -access, and literal Tree-external reads. Unknown or dynamic shapes remain -unresolved rather than becoming negative exposure. Missing, failed, duplicate, -pending, incomplete, or out-of-window results are unresolved and produce no -accepted read evidence. A unique, completed, non-empty, attributable result -with no explicit failure signal may remain candidate evidence even when its -provider has no separate positive-success flag. - -For message metadata: - -- project only a valid `metadata.contextDecision` v1 into - `decision_receipt`; -- treat receipt absence as unknown; -- omit malformed receipts and add `context_decision_invalid`; -- never fail Chat export because analysis metadata is malformed. - -## Reconstruct Tasks, then attribute Reads, then judge Effects - -Read -[references/task-analysis-schema.md](references/task-analysis-schema.md). - -First project a message-only source. This output deliberately excludes -collector-derived Reads, passages, Tree-mention indexes, decision receipts, -choice projections, and Effect judgments. It retains original work-message -content verbatim, including literal Tree, Read, or Effect discussion when that -discussion is part of the objective: +For **every** case where you claimed an effect, run a second, separate pass with +one job: -```bash -python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" task-source \ - --artifact-root "$CTVA_ARTIFACT_DIR" \ - --agent-workspace "AGENT_UUID=/absolute/current/agent/workspace" \ - --candidates "$CTVA_ARTIFACT_DIR/candidates.jsonl" \ - --output "$CTVA_ARTIFACT_DIR/task-source.jsonl" -``` +> **Find an explanation for that same choice that does not need the Tree.** -During Task reconstruction, read only `task-source.jsonl`. Do not inspect -`candidates.jsonl`, trace evidence, Reads, receipts, choices, or any anticipated -Effect. Write one schema-v4 `task-inventory-draft.jsonl` row for every clear -Task or excluded candidate, then freeze it: +Look hardest for the most common one: **the human already said it.** If the +human asked for OAuth in the same Chat, an agent "switching to OAuth" after +reading an auth node is explained by the instruction, not the node. Also check +whether the agent had already committed to the choice before the read, and +whether the choice follows from the code it was editing. -```bash -python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" freeze-tasks \ - --artifact-root "$CTVA_ARTIFACT_DIR" \ - --agent-workspace "AGENT_UUID=/absolute/current/agent/workspace" \ - --task-source "$CTVA_ARTIFACT_DIR/task-source.jsonl" \ - --task-inventory-draft "$CTVA_ARTIFACT_DIR/task-inventory-draft.jsonl" \ - --task-inventory-output "$CTVA_ARTIFACT_DIR/task-inventory.jsonl" +If such an explanation exists, set `"refuted": true` and record it: + +```json +{ "read_id": "…", "effect": { … }, "refuted": true, "refutation": "The human asked for OAuth explicitly two messages before the read." } ``` -A clear Task is one independently judgeable continuous work episode owned by -the audited Agent. It needs a concrete objective, independently judgeable -outcome, bounded source fragments, and explicit objective/outcome sources. -Material scope and primary deliverable help clarify boundaries but are -optional. Otherwise mark the candidate excluded with a structured exclusion -kind. - -Short continuations, status prompts, context-dependent questions, merge -approval, repeated review/fix requests, and ordinary phase transitions are not -separate Tasks. Merge plan → implementation → review → QA → final delivery, -plus corrections to the same deliverable, into one episode. Split only when -there is a new objective, material scope or deliverable change, independent -outcome, and unambiguous source boundary. - -For a single-Agent audit, another Agent's work is context until this Agent -receives or visibly accepts an objective. A later independent review, takeover, -verification gate, or orchestration objective may form a new episode only when -it passes every clear-Task gate. -Judge that acceptance from the work messages themselves. The reporter checks -source identity, ordering, and a same-Agent outcome, but does not infer who -arbitrary prose addresses. - -One Chat may contain multiple Tasks. Merge across Chats only for one PR/MR/ -Issue, a visible handoff, or the same objective and primary delivery, and -record the explicit shared linkage. - -After freeze, do not edit Task rows or boundaries. Write exactly one -schema-v4 `read-attributions.jsonl` row for every clear Task. Each row carries -the frozen `inventory_sha256`. - -Read is only: - -- `observed`, with attributable Task-window reads; -- `unresolved`, with a reason explaining the evidence gap, no reads, and no - Effect. - -Do not invent `not_observed`. Missing telemetry and receipt absence are -unknown, not proof of non-use. - -Then write exactly one schema-v4 `effect-judgments.jsonl` row for every clear -Task, carrying the same inventory digest. Each Task has zero or more Effects; -each Effect has exactly one type: `confirmed`, `constrained`, `redirected`, or -`conflicted`. Record one only when all four conditions hold: - -1. a real Read contains a relevant normal Tree decision or constraint; -2. the Read completes before the cited choice; -3. the later same-Agent choice or outcome reasonably shows one of the four - effects; -4. no more direct user instruction or other evidence fully explains the - result. - -Every Effect needs attributed Read IDs, later same-Agent choice message IDs, -a same-Agent outcome message, and a concise summary. The same Read may support -multiple distinct choices, but one choice cannot be reused across Effects. If -the evidence is insufficient, use an empty Effect list and record one short -reason. Do not add confidence tiers, support levels, numeric weights, -`verified`, or `probable`. A decision receipt may support the judgment but -cannot create an Effect by itself. - -Report every available Task in the authorized acquisition bound. There is no -minimum Task quota, task-type coverage gate, batch-expansion rule, or saturation -state. State the sample size and evidence gaps so readers can limit the -conclusion to the sampled scope. - -## Validate and report +Do this pass **without reusing the reasoning that produced the claim**. Judge the +material again from the refuter's side. A claim you cannot attack is worth +something; a claim you never attacked is worth nothing. + +`refuted` is required on every claimed effect — the reporter rejects a judgment +file that skips it. + +## Step 4 — Report ```bash python3 "$CTVA_SKILL_DIR/scripts/context_tree_value_audit.py" report \ - --artifact-root "$CTVA_ARTIFACT_DIR" \ - --agent-workspace "AGENT_UUID=/absolute/current/agent/workspace" \ - --candidates "$CTVA_ARTIFACT_DIR/candidates.jsonl" \ - --task-inventory "$CTVA_ARTIFACT_DIR/task-inventory.jsonl" \ - --read-attributions "$CTVA_ARTIFACT_DIR/read-attributions.jsonl" \ - --effect-judgments "$CTVA_ARTIFACT_DIR/effect-judgments.jsonl" \ - --evidence-output "$CTVA_ARTIFACT_DIR/evidence.jsonl" \ - --report-output "$CTVA_ARTIFACT_DIR/REPORT.md" + --tree-root "/absolute/path/to/bound/context-tree" \ + --since 2026-07-01T00:00:00Z \ + --sample "$CTVA_ARTIFACT_DIR/sample.json" \ + --judgments "$CTVA_ARTIFACT_DIR/judgments.json" \ + --output "$CTVA_ARTIFACT_DIR/REPORT.md" ``` -Optionally supply a v4 `--reviewed-baseline` when an independently reviewed -earlier case set exists. The reporter keeps its hash-anchored Task and Effect -counts separate from the current rerun. - -The deterministic pipeline rejects old combined judgment schemas, Task drafts -that contain Read/Effect fields, weak fragment-only objectives, missing or -invalid objective/outcome sources, reused identity sources, changed inventory -digests, Reads or choices outside the frozen Task, unauthorized source -messages, Task-window violations, unlinked cross-Chat merges, duplicated Reads -across Tasks, reused choices across Effects, invalid Effects, invalid outcome -messages, superseded judgment fields, and non-conserving aggregates. - -The report must include: - -- a complete frozen inventory of clear Tasks and excluded candidates with - structured exclusion reasons; -- observed and unresolved Read Tasks; -- Effect Tasks, total Effects, and observed Reads without an Effect; -- the four-Effect distribution over total Effects; -- every clear Task's Read and zero-or-more Effect results; -- every excluded Task's reason; -- authorized Chat, message, trace, and coverage-gap counts; -- the four-class in-window Tree-read attempt conservation table; -- explicit language that unresolved and receipt absence are unknown; -- a separately labeled, evidence-anchored historical baseline when supplied, - without merging it into the current rerun; -- no global effectiveness rate. - -Because all authorized Chats are not an eligible value denominator, return local links -to `REPORT.md` and `evidence.jsonl`, the acquisition bound if one was supplied, -authorization mode, sample size, and any material coverage gap. Keep -artifacts private in the invoking Agent workspace and never commit them. -Describe the result as a sampled evidence report, not causal proof, ROI, or an -effectiveness rate. +`report` refuses to publish effect counts unless the sample provably came from +the same Tree identity, the same window, and the same eligible read population. +New events arrive continuously, so if it tells you the population changed, +re-run `sample` rather than working around it. + +The report shows observed exposure, the no-observed-read list, observed write +events, the effect distribution, and the **refutation rate**. + +**When more than half of the claimed effects are refuted, the reporter withholds +the influence numbers** and says the run is unreliable. That is the intended +behavior, not a failure: a number nobody can defend is worse than no number. +Report the exposure section and investigate the judgment step. + +## What to tell the human + +Give them the report path and, in prose: + +- the observed exposure counts and the no-observed-read list, stated as + **recorded events and evidence gaps**, never as complete activity; +- the refutation rate, which says how much the influence numbers are worth; +- the two known recording gaps, so nobody reads adoption as a rate: + - pipeline shell reads (`cat NODE.md | head -40`) are **not recorded**, so + exposure is a lower bound; + - `Grep` / `Glob` record the search *directory*, not the matched nodes; + - write events are telemetry-only and miss merge and out-of-path commits; + - node text is a candidate snapshot from the observed HEAD, not necessarily + what the agent read. + +Never present the output as causal proof, an effectiveness rate, or ROI. Missing +evidence is unknown — it is never proof that a node went unread or that the Tree +went unused. diff --git a/skills/context-tree-value-audit/VERSION b/skills/context-tree-value-audit/VERSION index 8f0916f..3eefcb9 100644 --- a/skills/context-tree-value-audit/VERSION +++ b/skills/context-tree-value-audit/VERSION @@ -1 +1 @@ -0.5.0 +1.0.0 diff --git a/skills/context-tree-value-audit/agents/openai.yaml b/skills/context-tree-value-audit/agents/openai.yaml index c43ae9f..b759bc0 100644 --- a/skills/context-tree-value-audit/agents/openai.yaml +++ b/skills/context-tree-value-audit/agents/openai.yaml @@ -1,6 +1,6 @@ interface: display_name: "Context Tree Value Audit" - short_description: "Audit frozen Task, Read, and Effect evidence" - default_prompt: "Use $context-tree-value-audit to reconstruct and freeze Tasks before attributing Context Tree Reads and zero-or-more Effects, using the available authorized sample." + short_description: "Audit Context Tree exposure and influence from recorded IO" + default_prompt: "Use $context-tree-value-audit to report which Context Tree nodes this agent actually opened, which were never opened, and — for a random sample of reads — whether the read changed a later choice, with every claimed effect put through an adversarial pass." policy: allow_implicit_invocation: false diff --git a/skills/context-tree-value-audit/references/evidence-schema.md b/skills/context-tree-value-audit/references/evidence-schema.md deleted file mode 100644 index 3fa38dc..0000000 --- a/skills/context-tree-value-audit/references/evidence-schema.md +++ /dev/null @@ -1,329 +0,0 @@ -# Collector Evidence Schema - -This reference defines the deterministic collector boundary. Task -reconstruction and semantic judgment live in -[task-analysis-schema.md](task-analysis-schema.md). - -## Authorization unit - -One collector unit is: - -```text -CHAT_UUID@AGENT_UUID -``` - -Every unit in one run uses the same invoking Agent UUID, managed workspace, and -bound Tree. Allowed authorization values are: - -- `explicit_agent`: the human explicitly authorized all Chats visible to this - one current Agent; -- `explicit_chat`: the human supplied exact Chat UUIDs for the current Agent or - authorized the invoking Chat resolved from runtime `chatId`. - -These values record explicit scope. A run uses one mode only, never broadens -the requested scope, and never crosses to another Agent or workspace. No -human, organization, or extra authorization-context fields are used. The -Chat-Agent pair remains the authorization, source, and trace-mapping unit; it -is not the value-counting unit. - -The Agent name in scope and evidence is the immutable lowercase slug from the -invoking runtime's `FIRST_TREE_AGENT_SLUG`. The runtime's -`FIRST_TREE_AGENT_ID` must equal the UUID in the managed workspace identity. -The First Tree CLI must resolve that slug to the same UUID through its -producer-owned local binding loader—the same mapping used by later -`chat --agent` commands. The local Agent listing exists only in memory for this -identity preflight; it is not persisted, analyzed, or used to broaden consent. -Workspace `displayName` is a mutable human-facing label and is never a CLI -selector or authorization identity. The consumer accepts both the current -1-64 character Agent-name grammar and the complete historical -`[a-z0-9_-]{1,100}` grammar for still-runnable grandfathered Agent names, -including legacy names with a leading `-` or `_`. It does not reinterpret -arbitrary filesystem text as a selector or parse the local YAML mirror -independently of its producer. - -## Chat export - -`export-chats` writes authorized visible messages. Each message contains only -its ID, timestamp, sender identity/kind, visible content, and an optional -`decision_receipt`. - -When `metadata.contextDecision` is a valid v1 receipt, the export retains only: - -```json -{ - "version": 1, - "effect": "constrained", - "summary": "The Tree narrowed the acceptable implementation.", - "evidence": [ - { - "repoUrl": "https://github.com/example/context-tree", - "commit": "0123456789abcdef0123456789abcdef01234567", - "nodePath": "system/example.md", - "heading": "Decision" - } - ] -} -``` - -No other message metadata is copied. Valid effects are `confirmed`, -`constrained`, `redirected`, and `conflicted`; evidence contains one to three -rows, a 40-character Git commit, and a relative Markdown node path. -`repoUrl` must be a remote repository identity with at least an owner/group -and repository path. HTTP(S) identities must not contain user info; SSH -identities may use only the conventional `git` user; local paths, `file:` -URLs, loopback or unspecified hosts, non-canonical numeric hosts, credentials, -query strings, and fragments are malformed. Host validation is purely -syntactic and never performs DNS resolution; percent-encoded hosts and hosts -made entirely from decimal or hexadecimal numeric components are rejected. -Rejected values are never echoed into artifacts or diagnostics. - -Receipt absence is unknown and creates no negative diagnostic. A malformed -receipt is omitted and adds `context_decision_invalid` to the Chat coverage -gaps. It never blocks Chat export or the full audit. - -## Candidate evidence - -`collect` emits one JSON object per authorized Chat-Agent unit: - -```json -{ - "schema_version": 1, - "audit_id": "CHAT_UUID@AGENT_UUID", - "chat": { - "chat_id": "CHAT_UUID", - "title": "Chat topic", - "authorization": "explicit_chat", - "source_agent": "agent-name", - "source_agent_id": "AGENT_UUID", - "message_count": 12 - }, - "window": { - "start": null, - "end": "RFC3339" - }, - "tree_identity": "tree-opaque-hash", - "tree_source_snapshot": { - "status": "local_default_branch", - "branch": "main", - "commit": "0123456789abcdef0123456789abcdef01234567" - }, - "runtime_provider": "claude-code", - "candidate_status": "candidate", - "mapped_trace_files": ["trace-opaque-hash"], - "collector_diagnostics": { - "in_window_tree_read_attempts": 1, - "attempt_status_counts": { - "accepted_exact": 1, - "accepted_read_only_composite": 0, - "unresolved_opaque": 0, - "rejected_unsafe": 0 - }, - "attempt_reason_counts": {} - }, - "reads": [ - { - "read_id": "stable-id", - "timestamp": "RFC3339", - "completed_at": "RFC3339", - "session_file": "trace-opaque-hash", - "call_id": "provider-call-id", - "nested_call_index": null, - "tool_name": "exec_command", - "runtime_provider": "claude-code", - "reader_agent_id": "AGENT_UUID", - "tree_identity": "tree-opaque-hash", - "node_paths": ["system/example.md"], - "read_components": [ - { - "reader": "sed", - "node_paths": ["system/example.md"] - } - ], - "read_mode": "isolated", - "output_attribution": "exact", - "auxiliary_output_possible": false, - "content_class_hint": "normal", - "command": "normalized read descriptor", - "command_truncated": false, - "passage": "actual recorded tool output", - "passage_truncated": false, - "success": true, - "tree_source": { - "status": "default_branch_match", - "branch": "main", - "commit": "0123456789abcdef0123456789abcdef01234567", - "node_path": "system/example.md" - } - } - ], - "visible_messages": [ - { - "message_id": "message-id", - "created_at": "RFC3339", - "sender_id": "sender-id", - "content": "visible message", - "decision_receipt": null - } - ], - "visible_choice_candidates": [ - { - "message_id": "message-id", - "created_at": "RFC3339", - "sender_id": "AGENT_UUID", - "content": "visible later output" - } - ], - "visible_tree_mentions": [], - "coverage_gaps": [] -} -``` - -`window.start` is `null` unless the human supplied `--days`. The option is an -acquisition bound only; it does not decide sample size or the stopping rule. -A Chat whose latest activity is after `window.end` must still be fetched and -filtered because it may contain messages inside the historical window. A Chat -may be skipped from its summary timestamp only when that timestamp is strictly -before `window.start`. - -`tree_identity` is a deterministic opaque identity for the exact -Agent/workspace-bound Tree. The audit-row and read-level values must match. -`tree_source_snapshot` records the locally available `origin/HEAD` snapshot at -collection time without fetching or changing git state. If it cannot be -established, its status is `unavailable` with a conservative reason. Reporting -checks the recorded snapshot and each read's `tree_source` for internal -consistency; a later advance of the bound Tree does not invalidate an -otherwise reproducible historical artifact. - -Each read's `tree_source` is `default_branch_match` only when its one recorded -node passage matches that node in the local snapshot. Every other read is -`unverified_source`. This is a lightweight local-content classification, not -remote provenance or causal proof. - -`runtime_provider` is the canonical local evidence adapter. Schema-v1 -artifacts produced by 0.2.x omitted it because that release was Codex-only; -the reporter interprets that legacy omission as `codex`. - -`mapped_trace_files` and the compatibility-named `session_file` field contain -opaque evidence identities, never local filesystem paths. Each local Runtime -source must pass its adapter's bounded preflight for the exact workspace, one -authorized Chat, and the current Agent before complete outputs may be scanned. -The exact sources and support matrix are defined in -[runtime-evidence-adapters.md](runtime-evidence-adapters.md). - -`collector_diagnostics` counts command-shape decisions, not effects. Every -in-window call whose payload can be tied to bound-Tree Markdown is classified -exactly once: - -- `accepted_exact` — one statically closed content read with one completed, - non-empty, exactly attributable output and no explicit failure signal; -- `accepted_read_only_composite` — a statically closed read-only wrapper, - multi-file read, sequence, loop, or pipeline with one completed, non-empty, - attributable result and no explicit failure signal; -- `unresolved_opaque` — dynamic interpolation, unknown program, incomplete - path closure, or output attribution that cannot be proved; -- `rejected_unsafe` — mutation, file output, network access, or a proven - Tree-external read. - -The four status counts must sum to `in_window_tree_read_attempts`. Reason -counts conserve the unresolved and rejected calls. The acquisition window is -applied to call start time before diagnostics, so an older call cannot -contaminate the current run's gaps. A call that starts in-window but whose -result completes after the acquisition end remains one `unresolved_opaque` -attempt; it does not disappear from the denominator. - -`visible_messages` is the source for the later message-only `task-source` -projection. That projection keeps only message identity, time, sender, and -unchanged content; it removes decision receipts and every collector-derived -Tree-mention, Read, choice, and Effect projection before Task reconstruction. -Literal Tree/Read/Effect discussion inside a work message remains part of that -message and cannot by itself establish that a Read or Effect occurred. -`visible_choice_candidates` is used only after the Task inventory is frozen and -contains visible messages authored by the audited Agent; human or other-Agent -messages cannot establish that Agent's Effect. - -`outside_candidate_set` means collection found neither a successful qualifying -Tree read nor a visible Tree-influence signal. It is not evidence of no -Read or no value. A later Task reconstruction may still mark the historical -Read unresolved. - -## Read evidence - -The collector retains conservative trace rules while recognizing real -read-only command shapes: - -- bounded metadata/current-context preflight happens before full trace scan; -- one trace must map unambiguously to one authorized Chat and exact workspace; -- documented direct readers and statically extractable - `functions.exec`/`exec_command` content reads are accepted; -- an outer `functions.exec` assignment may omit only the final JavaScript - semicolon when it still forwards the same nested result's `.output` directly; - every nested command and workdir must remain literal; -- a call is paired with its exact output and continuations; an attributable - in-window provider call with a missing, duplicate, or otherwise invalid - result remains one `unresolved_opaque` attempt instead of disappearing from - the attempt denominator or being counted as accepted; -- explicit multi-file reads, multiple read statements, static `for` loops, - single-branch literal filesystem guards, read-only pipelines, filesystem - predicates, hierarchy selectors, labels, and line counts may coexist at the - command-classification layer; -- a hierarchy selector must parse as the exact `first-tree tree tree` command - path with only its documented read options and explicit `--no-pull` (apart - from an exact standalone help form), and `rg` accepts only a closed option - grammar with explicit `--no-config --no-ignore`; implicit refresh, - configuration/ignore sources, file-valued, external-program, unknown, and - Tree-external options never become accepted read-only diagnostics; -- shell readers and diagnostics use exact bare executable tokens; a - path-qualified executable is not trusted merely because its basename - matches an allowed reader; -- `bat` is not an accepted content reader because its unbound configuration - and paging environment can launch an external pager; the remaining content - readers are limited to closed `cat`, `head`, `nl`, `sed`, and `tail` - grammars with no external-helper options; -- `find` is not an accepted diagnostic because its broad, platform-dependent - grammar includes actions and file-valued inputs that cannot be proven - Tree-contained from the historical command record; -- `ls` is not an accepted diagnostic because recursive/dereferencing options - can traverse a Tree-contained symlink into an external directory; -- pipeline `head`, `tail`, and `nl` filters accept only exact formatting/range - option names with validated values; help, version, follow, unknown, and - output-replacing forms remain unresolved; -- Git diagnostics are not accepted as statically closed reads: unbound - system, global, repository, and environment configuration can activate - external diff, text-conversion, filesystem-monitor, or equivalent helpers, - including while resolving index-backed revisions; -- conditional guards with dynamic values, alternate branches, nested control, - or an unsafe body stay unresolved or rejected; -- null-sink diagnostic output is allowed, while file output is rejected; -- nested shell calls are kept as separate read slices when provider output - preserves that boundary; otherwise the read is marked - `output_attribution: aggregate` and receives an attribution gap; -- deterministic static labels are removed only when exactly attributable; - an accepted command that mixes unseparated dynamic diagnostic output with - Tree output receives an attribution gap and produces no read ID; -- `auxiliary_output_possible` records that a composite contained safe - auxiliary operations, but persisted passage bytes have already passed the - attribution gate; -- dynamic, unknown, stdin, lookalike, cross-tree, missing-result, - duplicate-call/result, failed, pending, incomplete, out-of-window, mutating, - or network shapes are unresolved or rejected and produce no read ID; -- historical output is never replaced with the current Tree file. - -`read_components` conserves `node_paths`. `read_mode: isolated` implies -`output_attribution: exact`; `read_only_composite` uses aggregate attribution -unless provider-native output blocks permit safe nested-call slicing. An -accepted command shape is still only candidate evidence: the Task auditor -must verify that the recorded passage actually contains decision-bearing Tree -content before recording an observed Read or Effect. - -When one outer orchestration call forwards multiple provider-native output -blocks, the collector emits one read row per attributable nested shell call -and sets `nested_call_index`. The outer call still contributes exactly one -four-state attempt classification; read-row count is therefore not required to -equal attempt count. - -`content_class_hint` is path-based triage, not a semantic verdict. A qualifying -effect still requires the Agent to judge a current decision, constraint, -rationale, or cross-domain relationship in normal content. - -Missing, cleaned, malformed, truncated, unsupported, or unmapped Runtime -evidence remains a coverage gap. It does not become a negative Read. diff --git a/skills/context-tree-value-audit/references/judging-effects.md b/skills/context-tree-value-audit/references/judging-effects.md new file mode 100644 index 0000000..1d7b576 --- /dev/null +++ b/skills/context-tree-value-audit/references/judging-effects.md @@ -0,0 +1,124 @@ +# Judging an Effect + +This reference is for step 3 of the audit: deciding whether one recorded read +changed what the agent did next, and then trying to prove yourself wrong. + +## The only question + +> **If the agent had not read this node, would the later choice have been +> different?** + +Everything below is scaffolding for answering that honestly. + +## Material you have + +Each sampled case gives you a read (node path, chat, timestamp) and a +**candidate snapshot** of the node's text, taken from the checkout HEAD observed +at read time. That is not a promise of what the agent saw — if it was reading +uncommitted Tree edits, the real text is unrecoverable. Each case says which of +`head_commit_snapshot`, `current_working_copy`, or `unavailable` it is. + +**When the snapshot may not be what was read, do not claim an effect that turns +on specific wording.** Use `null`. + +You supply the rest by reading that Chat around the read time: what the agent +said, what it did, and what the human asked for. + +Read the window **before** the read too. Most false positives come from +skipping that. + +## The four types + +| Type | The shape it takes | +| --- | --- | +| `confirmed` | The agent was hesitating between staying the course and changing; the node removed the hesitation and it stayed. | +| `constrained` | The agent had options open; the node ruled one out or narrowed the boundary. | +| `redirected` | The agent was heading toward A; after the read it went to B. | +| `conflicted` | The node and the situation disagreed, and the agent had to surface or resolve that. | + +`confirmed` is the weakest of the four and the easiest to over-claim. "The agent +did something consistent with the node" is not `confirmed` — consistency is not +influence. Ask whether there was any real uncertainty for the node to remove. If +the agent would obviously have done the same thing anyway, the answer is `null`. + +## When the answer is `null` + +Use `null` — no effect — whenever: + +- you cannot point at a specific later choice, only at a general vibe; +- the agent read the node and then did something unrelated; +- the read happened after the choice was already made; +- you would have to argue for it. + +`null` is a normal, common, healthy outcome. A sample where every read produced +an effect is evidence that the judgment is broken, not that the Tree is +excellent. + +## The adversarial pass + +For every effect you claimed, run a **separate** pass whose only job is: + +> **Explain that same choice without the Tree.** + +Do not reuse the reasoning that produced the claim. Come at the material fresh, +from the refuter's side. + +The explanations to hunt for, in order of how often they turn out to be the real +cause: + +1. **The human already said it.** Scan the Chat before the read. If the person + asked for the outcome, the node did not cause it — the instruction did. This + is by far the most common refutation. +2. **The agent had already committed.** If it announced the approach before the + read, a later read cannot have redirected it. +3. **The code forced it.** If the surrounding code, framework, or existing + pattern only permits one answer, the node is decoration. +4. **A different node or document did the work.** If several sources say the + same thing, this specific read is not load-bearing. +5. **The node is generic.** If the passage would fit any project, it cannot + explain a project-specific choice. + +If any of these holds, set `refuted: true` and write the explanation down. A +refuted case is not a failure of the audit — it is the audit working. + +## Worked examples + +**Refuted — the human said it.** +The human asks "add login to the admin panel, use OAuth." The agent reads +`system/auth.md` (which requires OAuth), then says "switched from password login +to OAuth." Tempting `redirected`. **Refuted:** the instruction already specified +OAuth. The node is consistent with the outcome but did not cause it. + +**Upheld — redirected.** +The human asks only "add login to the admin panel." The agent starts sketching a +username/password table, reads `system/auth.md`, then says "this needs to go +through the shared OAuth provider, dropping the local password table." No +instruction named OAuth; the approach changed after the read. **Upheld: +`redirected`.** + +**Upheld — constrained.** +The agent lists three storage options, reads a node that forbids a second state +source, and drops one option with that reason. No one told it to. **Upheld: +`constrained`.** + +**Null — consistency, not influence.** +The agent reads a node about naming conventions and then writes code that +follows those conventions, which it was already following everywhere else. +Nothing changed. **`null`.** + +**Null — read after the fact.** +The agent implements the change, then reads the node while writing its summary. +The read cannot have influenced a choice that preceded it. **`null`.** + +## What the refutation rate means + +The report divides refuted claims by total claims. + +- **Low** — the surviving effects were attacked and held. The numbers mean + something. +- **High (over half)** — the reporter withholds the influence numbers entirely + and says so. That is correct behavior. A number that cannot survive its own + refuter is worse than no number, because someone will quote it. + +If you find the rate is high, do not re-judge the cases more leniently to bring +it down. The rate is the finding. diff --git a/skills/context-tree-value-audit/references/runtime-evidence-adapters.md b/skills/context-tree-value-audit/references/runtime-evidence-adapters.md deleted file mode 100644 index 29be860..0000000 --- a/skills/context-tree-value-audit/references/runtime-evidence-adapters.md +++ /dev/null @@ -1,78 +0,0 @@ -# Runtime Evidence Adapters - -The audit core is Runtime-neutral. Authorization, Task reconstruction, Read -judgment, Effect judgment, and reporting do not change by provider. Only -historical Tree-read evidence recovery varies. - -The collector requires and resolves the current Runtime from -`FIRST_TREE_PROVIDER`; absence or an unknown value fails closed. An explicit -`--runtime-provider` is only an equality assertion and must match it. -`claude-code-tui` uses the `claude-code` transcript adapter because both -persist the same transcript family. - -## Support matrix - -| Runtime | Historical evidence source | Status | -| --- | --- | --- | -| Codex | Root session JSONL in the local Codex sessions directory | Supported | -| Claude Code | Root project transcript JSONL with complete `tool_use` / `tool_result` blocks (`CLAUDE_CONFIG_DIR` when set) | Supported | -| Claude Code TUI | Claude project transcript JSONL (`CLAUDE_CONFIG_DIR` when set) | Supported through the Claude adapter | -| Cursor | Existing native records do not retain complete attributable tool output | Unsupported; affected Reads unresolved | -| Kimi Code | Native wire lacks a durable First Tree Chat/Agent binding boundary | Unsupported; affected Reads unresolved | - -Supported adapters consume evidence already produced by each Runtime. This -Skill does not modify Runtime handlers or introduce a shared Tree-read CLI, -sidecar, general tool abstraction, or provider-neutral full-output ledger. - -## Shared acceptance boundary - -Every adapter must establish: - -- one authorized Chat and the exact current Agent UUID; -- one local session associated with the managed workspace; -- one paired, completed, non-empty tool result with no explicit failure signal; -- a statically attributable Markdown read inside the bound Tree; -- timestamps inside the acquisition window. - -The shared read grammar then classifies the attempt as exact, read-only -composite, unresolved, or unsafe. Provider-native output never bypasses Tree -path isolation, mutation checks, or output-attribution checks. A unique, -completed, non-empty, attributable result with no explicit failure signal may -be candidate evidence even when the provider has no separate positive-success -flag. Missing, failed, duplicate, pending, incomplete, or out-of-window -results remain unresolved and produce no read IDs. - -## Provider-specific notes - -### Codex - -Bounded session metadata and the runtime-injected current Chat context are -checked before the full trace is scanned. Calls, outputs, and continuations use -the existing Codex grammar. - -### Claude Code and Claude Code TUI - -Only root transcript rows are considered; sidechain/subagent rows are ignored. -The adapter uses `$CLAUDE_CONFIG_DIR/projects` when the Runtime sets a custom -Claude configuration root, otherwise `~/.claude/projects`. -One session ID, one workspace, and one authorized current Chat context must be -recoverable before tool blocks are paired, and the full scan must keep the -same session and Chat. Only canonical external-human rows can establish Chat -identity; tool results, compact summaries, meta rows, and other echoed text -cannot. Legacy per-Chat work directories may provide the Chat identity when no -injected context exists. - -### Cursor and Kimi Code - -The collector recognizes these Runtime names but does not infer a historical -Read from incomplete native records. It emits a provider-specific -`historical_evidence_not_supported` coverage gap, no read IDs, and leaves every -affected Task Read unresolved. Adding audit-only persistence to First Tree Runtime -handlers is outside this Skill's scope. - -## Missing evidence - -Missing roots, cleaned files, malformed identity, missing or duplicate calls -or results, failed or pending results, incomplete call/result pairs, truncated -output, or unsupported historical sessions remain `unresolved`. They never -become accepted evidence, proof of no Tree use, or zero effect. diff --git a/skills/context-tree-value-audit/references/task-analysis-schema.md b/skills/context-tree-value-audit/references/task-analysis-schema.md deleted file mode 100644 index 5ae8b2f..0000000 --- a/skills/context-tree-value-audit/references/task-analysis-schema.md +++ /dev/null @@ -1,285 +0,0 @@ -# Task, Read, and Effect Schema - -The audit uses three ordered stages. Task reconstruction is completed and -frozen before Tree Reads are visible to the analyst. Read attribution then -uses that frozen inventory. Effect analysis runs last and may not change either -earlier artifact. - -`Chat UUID @ Agent UUID` remains the authorization, trace-mapping, and source -unit. Task is the work unit. Effect is the independently counted value unit. - -## Stage 1: Task reconstruction - -Run `task-source` after collection. It projects only authorized work messages: -no collector-derived Reads, passages, Tree-mention indexes, decision receipts, -choice projections, or Effect judgments are present. Original message content -is unchanged and may literally discuss Tree, Read, or Effect when that -discussion is part of the work. Reconstruct Tasks only from this message-only -artifact and do not use those literal terms as evidence that a Read or Effect -occurred. - -A Task is one continuous work episode in which the audited Agent accepted a -concrete objective and produced an independently judgeable outcome. Scope and -primary deliverable may clarify the boundary but are optional. - -Write one schema-v4 `task-inventory-draft.jsonl` row for every clear Task or -excluded candidate: - -```json -{ - "schema_version": 4, - "task_id": "stable-local-task-id", - "status": "clear", - "objective": "Choose the state authority", - "object_scope": "state persistence", - "primary_deliverable": "A decision selecting one state source", - "outcome": "Kept the existing state source", - "started_at": "RFC3339", - "ended_at": "RFC3339", - "source_fragments": [ - { - "audit_id": "CHAT_UUID@AGENT_UUID", - "message_ids": [ - "objective-message-id", - "continuation-message-id", - "outcome-message-id" - ] - } - ], - "objective_source_message_ids": ["objective-message-id"], - "outcome_source_message_ids": ["outcome-message-id"] -} -``` - -`objective`, `outcome`, source fragments, and objective/outcome message IDs are -required. `object_scope` and `primary_deliverable` may be null. `started_at` -must equal the earliest objective source; `ended_at` must equal the latest -outcome source. Outcome sources must be non-empty messages authored by the -audited Agent and must strictly follow all objective sources. - -The Task inventory must not contain `episode`, ownership categories, -continuation IDs, boundary rationale, Read, Effect, confidence, support, -sampling, or task-type fields. Assignment, transfer, and visible acceptance are -all expressed by the objective source messages rather than a separate -ownership taxonomy. -The analyst is responsible for judging from those work messages that the -audited Agent received or accepted the objective. The reporter validates source -identity, ordering, and the audited Agent's outcome; it does not infer the -semantic addressee of arbitrary message prose. - -Use an excluded row when a defensible Task cannot be reconstructed: - -```json -{ - "schema_version": 4, - "task_id": "stable-local-candidate-id", - "status": "excluded", - "objective": null, - "object_scope": "unclear scope", - "outcome": null, - "started_at": "RFC3339", - "ended_at": "RFC3339", - "source_fragments": [ - { - "audit_id": "CHAT_UUID@AGENT_UUID", - "message_ids": ["message-id"] - } - ], - "exclusion_kind": "missing_objective", - "exclusion_reason": "No defensible objective and outcome boundary." -} -``` - -Allowed exclusion kinds: - -- `greeting_or_acknowledgement`; -- `status_ping_or_continuation`; -- `context_dependent_clarification`; -- `missing_objective`; -- `missing_outcome`; -- `ownership_not_established`; -- `automatic_or_provider_only`; -- `ambiguous_boundary`; -- `non_independent_subphase`. - -Apply this precedence when more than one fits: - -1. form-only exclusions; -2. `ownership_not_established`; -3. `missing_objective`; -4. `missing_outcome`; -5. `ambiguous_boundary`. - -An excluded row is an observed candidate, not a Task. It contains no clear-Task -sources, primary deliverable, Read, or Effect analysis. - -### Task boundaries - -Keep planning, implementation, review, QA, corrections, merge approval, status -questions, and short continuations inside one Task when they serve the same -objective and outcome. Split only when a new objective, material scope or -deliverable change, independent outcome, and unambiguous source boundary are -all present. - -Weak fragments such as `continue`, `status`, `why`, `please continue`, -`please fix it`, `继续`, `做了吗`, `修一下`, and mention-decorated equivalents -cannot establish a Task objective by themselves. Merge them into a visible -parent episode or exclude them. A concrete message that contains `continue` -but names its objective and outcome remains eligible. - -For a single-Agent audit, another Agent's work is context until the audited -Agent receives or visibly accepts an objective. Another Agent's later message -cannot serve as this Agent's outcome. - -Merge fragments across Chats only when every fragment carries the same -explicit linkage: - -```json -{ - "linkage": { - "kind": "work_item", - "key": "repo#123" - } -} -``` - -Allowed kinds are `work_item`, `explicit_handoff`, and -`same_objective_delivery`. Topical similarity alone is insufficient. - -### Freeze - -`freeze-tasks` validates the draft against the message-only Task source, -normalizes row order, computes one SHA-256 digest, and writes -`task-inventory.jsonl`. Every frozen row carries the same -`inventory_sha256`. - -The digest is an internal integrity fence, not a user-facing evidence concept. -If Task reconstruction changes, create a new frozen inventory and rerun Read -and Effect analysis. Never edit the frozen inventory during later stages. - -## Stage 2: Read attribution - -Write exactly one schema-v4 `read-attributions.jsonl` row for every clear Task: - -```json -{ - "schema_version": 4, - "inventory_sha256": "64-lowercase-hex", - "task_id": "stable-local-task-id", - "status": "observed", - "read_ids": ["read-id"], - "reason": null -} -``` - -Read has two states: - -- `observed` — one or more attributable Task-window Read IDs exist; -- `unresolved` — historical evidence cannot resolve the Read; `read_ids` is - empty and `reason` explains the gap. - -Zero attributed Reads are represented as unresolved unless the available -historical evidence can support a stronger interpretation. Do not emit -`not_observed`, `unused`, or another negative-value state. Missing telemetry -and receipt absence remain unknown. - -Every observed Read must belong to one source Chat, start and complete inside -the frozen Task window, and be attributed to only one Task. Read attribution -cannot create, delete, merge, split, or resize a Task. - -Collector command classification is not a Read by itself. For aggregate or -read-only-composite evidence, inspect the retained passage and component paths. -If actual Tree content cannot be attributed, keep the Task unresolved. - -## Stage 3: Effect analysis - -Write exactly one schema-v4 `effect-judgments.jsonl` row for every clear Task: - -```json -{ - "schema_version": 4, - "inventory_sha256": "64-lowercase-hex", - "task_id": "stable-local-task-id", - "effects": [ - { - "type": "constrained", - "read_ids": ["read-id"], - "choice_message_ids": ["choice-message-id"], - "outcome_message_id": "outcome-message-id", - "summary": "The constraint ruled out a second state source." - } - ], - "effect_reason": null -} -``` - -A Task has zero or more Effects. Each Effect independently binds supporting -Reads, one or more later same-Agent choices, a same-Agent outcome message, and -one summary. Valid types are: - -- `confirmed` — removed material uncertainty and justified keeping a choice; -- `constrained` — ruled out an option or narrowed the acceptable boundary; -- `redirected` — changed the intended approach; -- `conflicted` — exposed a conflict that still required resolution. - -Record an Effect only when: - -1. a real Read contains relevant normal Tree content; -2. every cited Read completes before the earliest cited choice; -3. the later same-Agent choice or outcome reasonably shows the Effect; -4. no more direct user instruction or other evidence fully explains it. - -The same Read may support multiple distinct choices and therefore multiple -Effects. One choice message cannot be reused across Effects. Outcome messages -may be shared when distinct choices converge on one later result. Effects may -bind intermediate same-Agent outcomes inside the Task; they do not have to use -the Task's terminal outcome source. - -When no Effect is defensible, set `"effects": []` and include one short -`effect_reason`. An unresolved Read must have no Effects. Do not add -`verified`, `probable`, confidence tiers, support levels, rubrics, or numeric -weights. A `contextDecision` receipt may support judgment but cannot create an -Effect by itself. - -## Separately reviewed historical baseline - -An optional schema-v4 baseline remains separate from the current rerun: - -```json -{ - "schema_version": 4, - "basis": "separately_reviewed_task_cases", - "reviewed_at": "RFC3339", - "evidence_anchor": { - "artifact_id": "opaque-reviewed-artifact-id", - "sha256": "64-lowercase-hex" - }, - "clear_tasks": 44, - "effect_tasks": 16, - "effects": 19, - "effect_counts": { - "confirmed": 3, - "constrained": 9, - "redirected": 6, - "conflicted": 1 - } -} -``` - -Effect counts conserve `effects`; `effect_tasks` cannot exceed clear Tasks or -total Effects. - -## Reporting - -Report every available Task in the authorized acquisition bound. There is no -minimum Task quota, task-type gate, batch expansion, or saturation status. - -The report conserves: - -- observed Read Tasks + unresolved Read Tasks = clear Tasks; -- Effect Tasks + observed Read without Effect = observed Read Tasks; -- the four Effect counts = total Effects; -- total Effects is greater than or equal to Effect Tasks. - -Always show both Effect Tasks and total Effects. State sample size and evidence -gaps. Never output a global effectiveness rate, causal claim, or ROI. diff --git a/skills/context-tree-value-audit/scripts/context_tree_value_audit.py b/skills/context-tree-value-audit/scripts/context_tree_value_audit.py index 577933d..d5089fa 100755 --- a/skills/context-tree-value-audit/scripts/context_tree_value_audit.py +++ b/skills/context-tree-value-audit/scripts/context_tree_value_audit.py @@ -1,7825 +1,966 @@ #!/usr/bin/env python3 -"""Build three-stage Context Tree audits from First Tree Chats and runtime evidence. - -Collection remains deliberately conservative and read-only. Semantic value is -judged only after a message-only Task inventory has been frozen. +"""Context Tree value audit built on the durable `context_tree_io_events` feed. + +Exposure is a recorded fact, not an inference: `first-tree tree io` returns the +calling agent's own Context Tree reads and writes, captured at tool-execution +time by the runtime. This script does the deterministic half of the audit — +aggregate the feed, pick a sample, assemble case material — and leaves exactly +one judgment to the model: did a given read change what the agent did next. + +Two recording gaps are known and are carried into every report rather than +silently absorbed: + +* shell reads that go through a pipeline (`cat NODE.md | head -40`) are not + recorded, so adoption is a LOWER BOUND, never a rate; +* `Grep` / `Glob` record one directory-level event for the search root, not one + per matched node, so a node inside a searched directory is never reported as + having no observed read. """ from __future__ import annotations import argparse -import ast import hashlib -import ipaddress import json import os +import random import re -import shlex import shutil -import stat import subprocess import sys -import tempfile -import unicodedata from collections import Counter -from dataclasses import dataclass +from dataclasses import dataclass, field from datetime import datetime, timedelta, timezone from pathlib import Path from typing import Any, Iterable, Iterator, Mapping, Sequence -from urllib.parse import urlsplit SCHEMA_VERSION = 1 -TASK_INVENTORY_SCHEMA_VERSION = 4 -TASK_ANALYSIS_SCHEMA_VERSION = 4 -AUTHORIZATION_VALUES = {"explicit_agent", "explicit_chat"} -EFFECT_VALUES = {"confirmed", "constrained", "redirected", "conflicted"} -READ_STATUS_VALUES = {"observed", "unresolved"} -TASK_STATUS_VALUES = {"clear", "excluded"} -TASK_EXCLUSION_VALUES = { - "greeting_or_acknowledgement", - "status_ping_or_continuation", - "context_dependent_clarification", - "missing_objective", - "missing_outcome", - "ownership_not_established", - "automatic_or_provider_only", - "ambiguous_boundary", - "non_independent_subphase", -} -WEAK_TASK_OBJECTIVES = { - "continue", - "do", - "done", - "status", - "why", - "proceed", - "go ahead", - "keep going", - "carry on", - "try again", - "what do you mean", - "继续", - "继续做", - "接着做", - "做了吗", - "你在干啥", - "进展呢", - "地址呢", - "为什么", - "什么意思", - "你这个修复什么", - "你这个在修复什么", - "那这个呢", - "再检查", - "修一下", - "重新看", - "按刚才说的改", -} -WEAK_TASK_PREFIX_WRAPPERS = { - "please", - "kindly", - "could you", - "can you", - "would you", - "just", - "simply", - "请", - "请你", - "麻烦", - "麻烦你", - "劳烦", - "劳烦你", - "帮忙", - "帮我", - "能否", - "可以", -} -WEAK_TASK_SUFFIX_WRAPPERS = { - "please", - "thanks", - "thank you", - "谢谢", - "辛苦了", - "可以吗", - "好吗", - "行吗", - "一下", - "下", - "吧", - "呢", - "呀", - "啊", - "嘛", - "吗", - "哈", -} -WEAK_TASK_ANCHORED_PATTERNS = ( - r"(?:continue\s+)?(?:fixing|working\s+on|with)\s+(?:it|this|that)", - r"(?:fix|check|review|try)\s+(?:it|this|that)(?:\s+again)?", - r"continue\s+(?:(?:the\s+)?(?:work|task|job)|it|this|that)", - r"继续(?:修|改|处理|弄)(?:一下|下)?", - r"继续(?:这个|它|这项工作|该工作|任务|这个任务)", - r"(?:再)?(?:修|修复|改|检查|看|处理|重试)(?:一下|下)?", - r"(?:修|修复|改|检查|看|处理)(?:这个|它)(?:一下|下)?", -) -LINKAGE_VALUES = { - "work_item", - "explicit_handoff", - "same_objective_delivery", -} -TRACE_PREFLIGHT_MAX_BYTES = 512 * 1024 -TRACE_PREFLIGHT_MAX_LINES = 512 -RUNTIME_PROVIDER_VALUES = { - "codex", - "claude-code", - "claude-code-tui", - "cursor", - "kimi-code", -} -SUPPORTED_EVIDENCE_PROVIDERS = {"codex", "claude-code"} -PURE_READ_COMMANDS = {"cat", "head", "nl", "sed", "tail"} -EXEC_COMMAND_TOOLS = {"exec_command", "functions.exec_command"} -EXEC_ORCHESTRATION_TOOLS = {"exec", "functions.exec"} -DIRECT_READ_TOOLS = { - "read_file", - "view_file", - "functions.read_file", - "functions.view_file", -} -SHELL_CONTINUATION_TOOLS = {"write_stdin", "functions.write_stdin"} -CELL_CONTINUATION_TOOLS = {"wait", "functions.wait"} -MUTATING_TOOLS = { - "apply_patch", - "functions.apply_patch", -} -READ_ATTEMPT_STATUSES = ( - "accepted_exact", - "accepted_read_only_composite", - "unresolved_opaque", - "rejected_unsafe", -) -KNOWN_UNSAFE_PROGRAMS = { - "bash", - "bat", - "chmod", - "chown", - "cp", - "curl", - "dd", - "eval", - "install", - "ln", - "mv", - "nc", - "perl", - "python", - "python3", - "rm", - "rsync", - "scp", - "sh", - "source", - "ssh", - "tee", - "truncate", - "wget", - "xargs", - "zsh", -} -SAFE_GIT_DIAGNOSTICS: set[str] = set() -CONFIG_DRIVEN_GIT_DIAGNOSTICS = { - "diff", - "log", - "show", - "status", -} -MUTATING_GIT_COMMANDS = { - "add", - "am", - "apply", - "bisect", - "branch", - "checkout", - "cherry-pick", - "clean", - "clone", - "commit", - "fetch", - "gc", - "init", - "merge", - "mv", - "pull", - "push", - "rebase", - "reset", - "restore", - "revert", - "rm", - "stash", - "submodule", - "switch", - "tag", - "worktree", -} -UNSAFE_GIT_OPTIONS = { - "--config-env", - "--exec-path", - "--ext-diff", - "--no-index", - "--output", - "--textconv", -} -UNSAFE_FIND_ACTIONS = { - "-delete", - "-exec", - "-execdir", - "-fls", - "-fprint", - "-fprint0", - "-fprintf", - "-ok", - "-okdir", -} -UNSAFE_RG_OPTIONS = { - "--file", - "--generate", - "--hostname-bin", - "--ignore-file", - "--pre", - "--pre-glob", - "--replace", - "-f", - "-r", -} -RG_PATTERN_OPTIONS = {"-e", "--regexp"} -RG_VALUE_OPTIONS = { - "-A", - "--after-context", - "-B", - "--before-context", - "-C", - "--context", - "--color", - "--colors", - "-E", - "--encoding", - "--engine", - "-g", - "--glob", - "--iglob", - "-j", - "--threads", - "-m", - "--max-count", - "--max-depth", - "--max-filesize", - "--sort", - "--sortr", - "-t", - "--type", - "-T", - "--type-not", -} -RG_FLAG_OPTIONS = { - "-0", - "--null", - "-a", - "--text", - "-c", - "--count", - "--count-matches", - "--column", - "--crlf", - "-F", - "--fixed-strings", - "--files", - "--files-with-matches", - "--files-without-match", - "-H", - "--with-filename", - "--heading", - "--hidden", - "-i", - "--ignore-case", - "-I", - "--no-filename", - "-l", - "-n", - "--line-number", - "--mmap", - "--multiline-dotall", - "-N", - "--no-line-number", - "--no-config", - "--no-heading", - "--no-ignore", - "--no-ignore-dot", - "--no-ignore-exclude", - "--no-ignore-files", - "--no-ignore-global", - "--no-ignore-messages", - "--no-ignore-parent", - "--no-ignore-vcs", - "--no-messages", - "--no-require-git", - "--no-unicode", - "--null-data", - "--one-file-system", - "--passthru", - "--pcre2", - "-S", - "--smart-case", - "--stats", - "--trim", - "--type-list", - "-U", - "--multiline", - "--unicode", - "-v", - "--invert-match", - "-w", - "--word-regexp", - "-x", - "--line-regexp", -} -_ARTIFACT_LEXICAL_ROOTS: dict[Path, Path] = {} -UUID_PATTERN = r"[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}" -# Current Agent names use the tighter 1-64 grammar with an alphanumeric first -# character. First Tree still runs older names created under -# `[a-z0-9_-]{1,100}`, including leading separators, so an audit must accept -# that complete path-safe grandfathered grammar and let the producer-owned -# local binding UUID check establish the exact identity. -AGENT_SLUG_PATTERN = re.compile(r"[a-z0-9_-]{1,100}") -CHAT_CONTEXT_PATTERN = re.compile( - r"", - re.UNICODE, -) -CHAT_ID_PATTERN = re.compile(rf'"chatId"\s*:\s*"({UUID_PATTERN})"', re.UNICODE) -TREE_MENTION_PATTERN = re.compile( - r"context[\s-]+tree|tree\s+(?:node|节点|decision|决策|constraint|约束|rationale|现行|current)|" - r"(?:^|[\s\"'`(])(?:[^\s/\"'`()]+/)+[^\s\"'`()]+\.md(?=$|[\s\"'`,;:)])", - re.IGNORECASE | re.UNICODE, -) -SHELL_SESSION_PATTERN = re.compile( - r"\A\s*Script running with session ID\s+([0-9]+)\s*\Z", - re.IGNORECASE, -) -CELL_SESSION_PATTERN = re.compile( - r"\A\s*Script running with cell ID\s+([A-Za-z0-9_.:-]+)\s*\Z", - re.IGNORECASE, -) +EFFECT_TYPES = ("confirmed", "constrained", "redirected", "conflicted") +ACTIONS = ("read", "write") +TARGET_KINDS = ("file", "directory", "repo") +# Path-based triage only. A node being "normal" does not make a read valuable; +# it only means the node is the kind of content a decision can come from. +NON_NORMAL_PREFIXES = ("members/", "raw-context/") +NON_NORMAL_NAMES = ("AGENTS.md", "CLAUDE.md", "README.md") +SHA_RE = re.compile(r"\A[0-9a-f]{40}\Z") class AuditError(RuntimeError): - """Raised for invalid input or an incomplete deterministic audit step.""" + """Invalid input, or a deterministic step that cannot complete honestly.""" -@dataclass(frozen=True) -class Window: - start: datetime | None - end: datetime +# ── time ────────────────────────────────────────────────────────────────── -@dataclass(frozen=True) -class ScopedChat: - chat_id: str - agent: str - agent_id: str - authorization: str +def parse_time(value: str, *, field_name: str) -> datetime: + text = value.strip() + if text.endswith("Z"): + text = f"{text[:-1]}+00:00" + try: + parsed = datetime.fromisoformat(text) + except ValueError as error: + raise AuditError(f"{field_name} is not an RFC 3339 timestamp: {value}") from error + if parsed.tzinfo is None: + raise AuditError(f"{field_name} must carry a timezone: {value}") + return parsed.astimezone(timezone.utc) -@dataclass(frozen=True) -class ScopedAgent: - name: str - agent_id: str - authorization: str +def iso(value: datetime) -> str: + return value.astimezone(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") -@dataclass(frozen=True) -class Scope: - agents: tuple[ScopedAgent, ...] - chats: tuple[ScopedChat, ...] +# ── CLI bridge ──────────────────────────────────────────────────────────── -@dataclass(frozen=True) -class WorkspaceIdentity: - agent_name: str - agent_display_name: str - agent_id: str - workspace_lexical: Path - workspace: Path - bound_tree_root: Path +def resolve_binary(explicit: str | None) -> str: + requested = explicit or os.environ.get("FIRST_TREE_BIN") + candidates = [requested] if requested else ["first-tree", "first-tree-staging"] + for candidate in candidates: + if not candidate: + continue + if os.sep in candidate: + path = Path(candidate).expanduser() + if path.is_file() and os.access(path, os.X_OK): + return str(path.resolve()) + continue + found = shutil.which(candidate) + if found: + return found + raise AuditError("No First Tree CLI found; set FIRST_TREE_BIN or pass --first-tree-bin.") -@dataclass(frozen=True) -class TracePreflight: - path: Path - trace_id: str - audit_id: str - agent_id: str - workspace: Path - runtime_provider: str - provider_session_id: str | None = None +def run_cli_json(binary: str, args: Sequence[str]) -> Any: + command = [binary, "--json", *args] + completed = subprocess.run(command, capture_output=True, text=True, check=False) + if completed.returncode != 0: + raise AuditError( + f"Read-only First Tree command failed: {' '.join(args)} " + f"(exit {completed.returncode}; output withheld)." + ) + out = completed.stdout.strip() + if not out: + raise AuditError(f"First Tree command produced no JSON: {' '.join(args)}") + try: + payload = json.loads(out) + except json.JSONDecodeError as error: + raise AuditError(f"Invalid JSON from {' '.join(args)}: {error}") from error + if not isinstance(payload, dict) or payload.get("ok") is not True: + raise AuditError(f"First Tree command did not succeed: {' '.join(args)}") + return payload.get("data") -@dataclass(frozen=True) -class ReadComponent: - reader: str - node_paths: tuple[str, ...] +def fetch_io_events( + binary: str, + *, + since: datetime | None, + until: datetime | None, + chat_id: str | None, + agent: str | None, + page_limit: int = 200, + max_pages: int = 200, +) -> list[dict[str, Any]]: + """Page the agent-scoped IO feed. Fails closed rather than truncating.""" + events: list[dict[str, Any]] = [] + cursor: str | None = None + seen_cursors: set[str] = set() + for _ in range(max_pages): + args = ["tree", "io", "--limit", str(page_limit)] + if since is not None: + args += ["--since", iso(since)] + if until is not None: + args += ["--until", iso(until)] + if chat_id: + args += ["--chat", chat_id] + if agent: + args += [f"--agent={agent}"] + if cursor: + args += ["--cursor", cursor] + data = run_cli_json(binary, args) + if not isinstance(data, dict): + raise AuditError("tree io returned no data object.") + items = data.get("items") + if not isinstance(items, list): + raise AuditError("tree io returned no items array.") + events.extend(item for item in items if isinstance(item, dict)) + next_cursor = data.get("nextCursor") + if not isinstance(next_cursor, str) or not next_cursor: + return events + if next_cursor in seen_cursors: + raise AuditError("tree io pagination repeated a cursor.") + seen_cursors.add(next_cursor) + cursor = next_cursor + raise AuditError( + f"tree io did not finish within {max_pages} pages; narrow --since/--until." + ) -@dataclass(frozen=True) -class ReadPlan: - node_paths: tuple[str, ...] - components: tuple[ReadComponent, ...] - command: str - mode: str - auxiliary_output_possible: bool = False - auxiliary_literals: tuple[str, ...] = () - output_requires_separation: bool = False +# ── event model ─────────────────────────────────────────────────────────── @dataclass(frozen=True) -class ReadAssessment: - plan: ReadPlan | None - status: str | None - reason: str | None - subplans: tuple[ReadPlan, ...] = () +class TreeIdentity: + """Canonical identity of one bound Context Tree.""" + repo: str + branch: str -@dataclass(frozen=True) -class ShellSegment: - tokens: tuple[str, ...] - input_mode: str - output_discarded: bool = False + def label(self) -> str: + return f"{self.repo}#{self.branch}" -def parse_datetime(value: str, *, field: str = "timestamp") -> datetime: - normalized = value.strip() - if normalized.endswith("Z"): - normalized = f"{normalized[:-1]}+00:00" - try: - parsed = datetime.fromisoformat(normalized) - except ValueError as error: - raise AuditError(f"Invalid {field}: {value}") from error - if parsed.tzinfo is None: - raise AuditError(f"{field} must include a timezone: {value}") - return parsed.astimezone(timezone.utc) +def canonical_repo(url: str) -> str: + """Normalize a git remote so ssh/https/`.git` spellings compare equal. + The origin PORT is deliberately preserved. For a Self-Managed GitLab the + instance origin — port included — is the authority boundary, so folding + `git.example:8443` and `git.example:9443` together would credit one + instance's reads against another's Tree. + """ + text = url.strip().rstrip("/") + text = re.sub(r"\.git\Z", "", text) + # URL forms first: `ssh://git@host:22/org/tree` must not be read as scp. + match = re.fullmatch(r"(?:https?|git|ssh)://(?:[^@/]+@)?([^/]+)/(.+)", text) + if match: + return f"{match.group(1).lower()}/{match.group(2).strip('/').lower()}" + # scp-like `git@host:org/tree`; the part after `:` is a path, not a port. + scp = re.fullmatch(r"git@([^:/]+):(.+)", text) + if scp: + return f"{scp.group(1).lower()}/{scp.group(2).strip('/').lower()}" + return text.lower() -def isoformat(value: datetime) -> str: - return value.astimezone(timezone.utc).isoformat(timespec="milliseconds").replace("+00:00", "Z") +@dataclass(frozen=True) +class IoEvent: + event_id: str + chat_id: str + action: str + source: str + target_kind: str + target_path: str + tree_repo_url: str + tree_branch: str + tree_head_commit: str | None + created_at: datetime + + @property + def identity(self) -> TreeIdentity: + return TreeIdentity(repo=canonical_repo(self.tree_repo_url), branch=self.tree_branch) + + @property + def is_normal_content(self) -> bool: + path = self.target_path + if path in NON_NORMAL_NAMES or Path(path).name in NON_NORMAL_NAMES: + return False + return not path.startswith(NON_NORMAL_PREFIXES) + + +def normalize_event(raw: Mapping[str, Any]) -> IoEvent: + def text(key: str) -> str: + value = raw.get(key) + if not isinstance(value, str) or not value.strip(): + raise AuditError(f"IO event is missing `{key}`.") + return value.strip() + + action = text("action") + if action not in ACTIONS: + raise AuditError(f"IO event has an unknown action: {action}") + target_kind = text("targetKind") + if target_kind not in TARGET_KINDS: + raise AuditError(f"IO event has an unknown targetKind: {target_kind}") + commit = raw.get("treeHeadCommit") + return IoEvent( + event_id=text("id"), + chat_id=text("chatId"), + action=action, + source=text("source"), + target_kind=target_kind, + target_path=text("targetPath"), + tree_repo_url=text("treeRepoUrl"), + tree_branch=text("treeBranch"), + tree_head_commit=commit if isinstance(commit, str) and SHA_RE.match(commit) else None, + created_at=parse_time(text("createdAt"), field_name="IO event createdAt"), + ) + + +def tree_identity_of_root(tree_root: Path) -> TreeIdentity | None: + """Canonical identity of the local checkout, or None when unprovable.""" + remote = git_text(tree_root, ["remote", "get-url", "origin"]) + branch = git_text(tree_root, ["rev-parse", "--abbrev-ref", "HEAD"]) + if not remote: + return None + if not branch or branch == "HEAD": + upstream = git_text(tree_root, ["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"]) + branch = upstream.split("/", 1)[-1] if upstream else None + if not branch: + return None + return TreeIdentity(repo=canonical_repo(remote), branch=branch) + + +def select_events_for_tree( + events: Sequence[IoEvent], + expected: TreeIdentity | None, +) -> tuple[list[IoEvent], TreeIdentity]: + """Keep only events provably belonging to one Tree; fail closed otherwise. + + A binding can change, and a captured feed can mix Trees. Aggregating by + path alone would credit reads of one Tree against another's nodes, so an + event that cannot be matched to the target identity is excluded rather + than assumed. + """ + identities = {event.identity for event in events} + if expected is not None: + # "No Tree IO in this window" is a legitimate, reportable audit result + # once the target Tree is provable, so absence must not fail the run. + return [event for event in events if event.identity == expected], expected + if not events: + raise AuditError( + "The IO feed contained no events and no --tree-root was given, so the audited " + "Context Tree cannot be identified." + ) + if expected is None: + if len(identities) != 1: + raise AuditError( + "The feed mixes Context Trees (" + + ", ".join(sorted(item.label() for item in identities)) + + "); pass --tree-root so the audit can pin one identity." + ) + return list(events), next(iter(identities)) + return list(events), next(iter(identities)) + + +# ── aggregation (no sampling, no judgment) ──────────────────────────────── + + +@dataclass +class Aggregate: + reads: list[IoEvent] = field(default_factory=list) + writes: list[IoEvent] = field(default_factory=list) + chats_with_read: set[str] = field(default_factory=set) + chats_seen: set[str] = field(default_factory=set) + node_reads: Counter[str] = field(default_factory=Counter) + searched_dirs: set[str] = field(default_factory=set) + node_writes: Counter[str] = field(default_factory=Counter) + source_counts: Counter[str] = field(default_factory=Counter) + + +def aggregate(events: Iterable[IoEvent]) -> Aggregate: + agg = Aggregate() + for event in events: + agg.chats_seen.add(event.chat_id) + agg.source_counts[f"{event.action}:{event.source}"] += 1 + if event.action == "write": + agg.writes.append(event) + if event.target_kind == "file": + agg.node_writes[event.target_path] += 1 + continue + agg.reads.append(event) + agg.chats_with_read.add(event.chat_id) + if event.target_kind == "file": + agg.node_reads[event.target_path] += 1 + else: + # Directory / repo events name a search root, not a node. They + # cannot credit a specific node, but they DO forbid reporting any + # node beneath them as having no observed read. + agg.searched_dirs.add("" if event.target_kind == "repo" else event.target_path.rstrip("/")) + return agg -def resolve_window(days: int | None, now_text: str | None) -> Window: - if days is not None and days <= 0: - raise AuditError("--days must be greater than zero.") - end = parse_datetime(now_text, field="--now") if now_text else datetime.now(timezone.utc) - return Window(start=end - timedelta(days=days) if days is not None else None, end=end) +def covered_by_search(node_path: str, searched_dirs: set[str]) -> bool: + """True when some recorded search root contains this node.""" + if "" in searched_dirs: # a repo-level event covers everything + return True + parts = Path(node_path).parts + for index in range(len(parts)): + if "/".join(parts[:index]) in searched_dirs: + return True + return False -def in_window(value: str | None, window: Window) -> bool: - if not value: - return False - try: - timestamp = parse_datetime(value) - except AuditError: - return False - return timestamp <= window.end and (window.start is None or timestamp >= window.start) +def unobserved_nodes(tree_root: Path, agg: Aggregate) -> list[str]: + """Nodes with no observed read event and no search root recorded above them. -def strictly_before_window(value: Any, window: Window) -> bool: - if not isinstance(value, str) or not value or window.start is None: - return False - try: - timestamp = parse_datetime(value) - except AuditError: - return False - return timestamp < window.start + This is an evidence gap, NOT a claim that the node was never read. Pipeline + shell reads are never recorded at all, and read telemetry is best-effort, so + absence here means "no event reached the feed" and nothing more. + Deliberately conservative on top of that: a node inside a recorded search + root is excluded even though the search may never have opened it. + """ + candidates: list[str] = [] + for path in sorted(tree_root.rglob("*.md")): + if path.is_symlink() or not path.is_file(): + continue + try: + relative = path.relative_to(tree_root).as_posix() + except ValueError: + continue + if any(part.startswith(".") for part in Path(relative).parts): + continue + if relative in NON_NORMAL_NAMES or relative.startswith(NON_NORMAL_PREFIXES): + continue + if agg.node_reads.get(relative): + continue + if covered_by_search(relative, agg.searched_dirs): + continue + candidates.append(relative) + return candidates -def window_start_text(window: Window) -> str | None: - return isoformat(window.start) if window.start is not None else None +# ── case material for the judgment step ─────────────────────────────────── -def read_json(path: Path) -> Any: - try: - return json.loads(path.read_text(encoding="utf-8")) - except (OSError, json.JSONDecodeError) as error: - raise AuditError(f"Could not read JSON from {path}: {error}") from error +def git_text(tree_root: Path, args: Sequence[str]) -> str | None: + completed = subprocess.run( + ["git", "-C", str(tree_root), *args], + capture_output=True, + text=True, + check=False, + env={**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0"}, + ) + out = completed.stdout.strip() + return out if completed.returncode == 0 and out else None -def iter_jsonl(path: Path) -> Iterator[dict[str, Any]]: - try: - with path.open("r", encoding="utf-8") as handle: - for line_number, line in enumerate(handle, start=1): - if not line.strip(): - continue - try: - value = json.loads(line) - except json.JSONDecodeError as error: - raise AuditError(f"Invalid JSONL at {path}:{line_number}: {error}") from error - if not isinstance(value, dict): - raise AuditError(f"Expected a JSON object at {path}:{line_number}.") - yield value - except OSError as error: - raise AuditError(f"Could not read {path}: {error}") from error +def git_show(tree_root: Path, commit: str, node_path: str) -> str | None: + completed = subprocess.run( + ["git", "-C", str(tree_root), "show", f"{commit}:{node_path}"], + capture_output=True, + text=True, + check=False, + env={**os.environ, "GIT_CONFIG_NOSYSTEM": "1", "GIT_TERMINAL_PROMPT": "0"}, + ) + return completed.stdout if completed.returncode == 0 else None -def resolve_artifact_root(value: str, workspace_identity: WorkspaceIdentity) -> Path: - raw = Path(value).expanduser() - lexical = Path(os.path.abspath(raw)) - if lexical.is_symlink(): - raise AuditError("--artifact-root must not be a symbolic link.") - try: - lexical_relative = lexical.relative_to(workspace_identity.workspace_lexical) - except ValueError as error: - raise AuditError( - "--artifact-root must be a strict descendant of the authorized Agent workspace." - ) from error - if not lexical_relative.parts: - raise AuditError( - "--artifact-root must be a dedicated directory below the authorized Agent workspace." - ) - current = workspace_identity.workspace_lexical - for part in lexical_relative.parts: - current = current / part - if current.is_symlink(): - raise AuditError("--artifact-root must not traverse a symbolic link.") - try: - prospective = lexical.resolve(strict=False) - resolved_relative = prospective.relative_to(workspace_identity.workspace) - except (OSError, ValueError) as error: - raise AuditError( - "--artifact-root must resolve inside the authorized Agent workspace." - ) from error - if not resolved_relative.parts: - raise AuditError( - "--artifact-root must be a dedicated directory below the authorized Agent workspace." - ) - try: - lexical.mkdir(mode=0o700, parents=True, exist_ok=True) - root = lexical.resolve(strict=True) - except OSError as error: - raise AuditError(f"Could not prepare --artifact-root {lexical}: {error}") from error - try: - root_stat = root.stat() - except OSError as error: - raise AuditError(f"Could not inspect --artifact-root {root}: {error}") from error - if not stat.S_ISDIR(root_stat.st_mode): - raise AuditError(f"--artifact-root must be a directory: {root}") - try: - final_relative = root.relative_to(workspace_identity.workspace) - except ValueError as error: - raise AuditError( - "--artifact-root must resolve inside the authorized Agent workspace." - ) from error - if not final_relative.parts: - raise AuditError( - "--artifact-root must be a dedicated directory below the authorized Agent workspace." - ) - if hasattr(os, "geteuid") and root_stat.st_uid != os.geteuid(): - raise AuditError("--artifact-root must be owned by the current user.") - try: - os.chmod(root, 0o700) - except OSError as error: - raise AuditError(f"Could not restrict --artifact-root permissions: {error}") from error - _ARTIFACT_LEXICAL_ROOTS[root] = lexical - return root +def node_content_at_read(tree_root: Path, event: IoEvent, max_chars: int) -> dict[str, Any]: + """Recover the node's likely text at read time. -def artifact_path(root: Path, value: str, *, field: str, must_exist: bool) -> Path: - raw = Path(value).expanduser() - candidate = raw if raw.is_absolute() else root / raw - lexical = Path(os.path.abspath(candidate)) - lexical_root: Path | None = None - allowed_lexical_roots = (root, _ARTIFACT_LEXICAL_ROOTS.get(root, root)) - for allowed_root in allowed_lexical_roots: - try: - lexical.relative_to(allowed_root) - except ValueError: - continue - lexical_root = allowed_root - break - if lexical_root is None: - raise AuditError(f"{field} must stay lexically inside --artifact-root.") - lexical_relative = lexical.relative_to(lexical_root) - current = lexical_root - for part in lexical_relative.parts: - current = current / part - if current.is_symlink(): - raise AuditError(f"{field} must not traverse a symbolic link.") - try: - resolved = lexical.resolve(strict=must_exist) - resolved.relative_to(root) - except (OSError, ValueError) as error: - raise AuditError(f"{field} must resolve inside --artifact-root {root}: {candidate}") from error - if must_exist: - if not resolved.is_file() or resolved.is_symlink(): - raise AuditError(f"{field} must be a regular file inside --artifact-root: {resolved}") - else: + `treeHeadCommit` is the checkout HEAD observed for the read, NOT a promise + that the working file matched that commit. An agent reading uncommitted + Tree edits saw something this cannot reconstruct, so the status stays + `head_commit_snapshot` and the analyst is told to treat it as a candidate. + """ + if event.tree_head_commit: + content = git_show(tree_root, event.tree_head_commit, event.target_path) + if content is not None: + body, truncated = clip(content, max_chars) + return { + "status": "head_commit_snapshot", + "commit": event.tree_head_commit, + "content": body, + "truncated": truncated, + "caveat": ( + "Node text at the checkout HEAD observed for this read. If the agent read " + "uncommitted edits, it saw different text." + ), + } + current = tree_root / event.target_path + if current.is_file() and not current.is_symlink(): try: - resolved.parent.mkdir(mode=0o700, parents=True, exist_ok=True) - parent = resolved.parent.resolve(strict=True) - parent.relative_to(root) - except (OSError, ValueError) as error: - raise AuditError(f"Could not prepare {field} inside --artifact-root: {resolved}") from error - if resolved.exists() and (not resolved.is_file() or resolved.is_symlink()): - raise AuditError(f"{field} must be a regular file path: {resolved}") - return resolved - + body, truncated = clip(current.read_text(encoding="utf-8"), max_chars) + except (OSError, UnicodeDecodeError): + return {"status": "unavailable", "reason": "node_unreadable"} + return { + "status": "current_working_copy", + "content": body, + "truncated": truncated, + "caveat": ( + "Node text as it stands now, not at read time. The node may have changed since." + ), + } + return {"status": "unavailable", "reason": "node_absent_at_audit_time"} -def require_distinct_paths(paths: Mapping[str, Path]) -> None: - reverse: dict[Path, list[str]] = {} - for field, path in paths.items(): - reverse.setdefault(path, []).append(field) - duplicates = {path: fields for path, fields in reverse.items() if len(fields) > 1} - if duplicates: - detail = "; ".join(f"{path}: {', '.join(fields)}" for path, fields in duplicates.items()) - raise AuditError(f"Artifact inputs and outputs must be distinct ({detail}).") +def clip(text: str, limit: int) -> tuple[str, bool]: + return (text, False) if len(text) <= limit else (text[:limit], True) -def atomic_write(path: Path, text: str) -> None: - temporary: str | None = None - try: - if path.is_symlink(): - raise AuditError(f"Refusing to replace symbolic-link artifact: {path}") - with tempfile.NamedTemporaryFile( - "w", - encoding="utf-8", - dir=path.parent, - prefix=f".{path.name}.", - delete=False, - ) as handle: - os.fchmod(handle.fileno(), 0o600) - handle.write(text) - handle.flush() - os.fsync(handle.fileno()) - temporary = handle.name - if path.is_symlink(): - raise AuditError(f"Refusing to replace symbolic-link artifact: {path}") - os.replace(temporary, path) - os.chmod(path, 0o600) - temporary = None - except AuditError: - raise - except OSError as error: - raise AuditError(f"Could not write {path}: {error}") from error - finally: - if temporary is not None: - try: - Path(temporary).unlink() - except OSError: - pass +def eligible_reads(reads: Sequence[IoEvent]) -> list[IoEvent]: + return [event for event in reads if event.target_kind == "file" and event.is_normal_content] -def write_jsonl(path: Path, rows: Iterable[Mapping[str, Any]]) -> None: - text = "".join(f"{json.dumps(row, ensure_ascii=False, sort_keys=True)}\n" for row in rows) - atomic_write(path, text) +def population_digest(events: Sequence[IoEvent]) -> str: + """Digest of the exact eligible population a sample was drawn from. -def write_text(path: Path, text: str) -> None: - atomic_write(path, text) + The report re-fetches the feed, and new events arrive between the two + steps. Without this fence a stale or hand-written sample could produce + effect counts against a population it was never drawn from. + """ + payload = "\n".join(sorted(event.event_id for event in events)) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() -def require_string(value: Any, field: str) -> str: - if not isinstance(value, str) or not value.strip(): - raise AuditError(f"{field} must be a non-empty string.") - return value.strip() +def sample_reads(reads: Sequence[IoEvent], size: int, seed: int) -> list[IoEvent]: + """Uniform sample over recorded reads. + Sampling is random on purpose. Picking the cases whose surrounding decision + stream is still available would bias the sample toward recent, still-live + sessions; instead we sample first and report how many cases turned out to + be unresolvable. + """ + eligible = eligible_reads(reads) + if size >= len(eligible): + return list(eligible) + return random.Random(seed).sample(eligible, size) -def validate_authorization(value: Any, field: str) -> str: - authorization = require_string(value, field) - if authorization not in AUTHORIZATION_VALUES: - raise AuditError(f"{field} must be one of: {', '.join(sorted(AUTHORIZATION_VALUES))}.") - return authorization +# ── report ──────────────────────────────────────────────────────────────── -def require_uuid(value: Any, field: str) -> str: - result = require_string(value, field) - if re.fullmatch(UUID_PATTERN, result) is None: - raise AuditError(f"{field} must be a UUID.") - return result +KNOWN_GAPS = [ + "Read telemetry is best-effort. Shell reads that pass through a pipeline (for example " + "`cat NODE.md | head -40`) produce no event at all, so every read count here is a lower bound and " + "no rate can be derived from it.", + "`Grep` / `Glob` record one directory-level event for the search root rather than one event per " + "matched node, so a node inside a recorded search root is never listed as unobserved.", + "Write telemetry misses merge commits and worktree edits outside the bound path. Complete write " + "activity comes from the Tree repository's git history, not from this feed.", + "A node's text is reconstructed from the checkout HEAD observed at read time. That is a candidate " + "snapshot: an agent reading uncommitted edits saw text this audit cannot recover.", +] -def audit_id(chat_id: str, agent_id: str) -> str: - return f"{chat_id}@{agent_id}" +def render_report( + *, + generated_at: datetime, + window_start: datetime | None, + window_end: datetime | None, + agg: Aggregate, + unobserved: list[str] | None, + judgments: Sequence[Mapping[str, Any]] | None, + sample_size: int, +) -> str: + lines: list[str] = [ + "# Context Tree Value Audit", + "", + f"Generated: {iso(generated_at)}", + "Window: " + + (f"{iso(window_start)} – " if window_start else "unbounded – ") + + (iso(window_end) if window_end else "now"), + "", + "This is an evidence report over one agent's own recorded Context Tree IO. " + "It is not causal proof, an effectiveness rate, or ROI.", + "", + "## Observed exposure", + "", + "| Measure | Count |", + "| --- | ---: |", + f"| Chats with at least one observed Tree read | {len(agg.chats_with_read)} |", + f"| Chats with any observed Tree IO | {len(agg.chats_seen)} |", + f"| Observed reads | {len(agg.reads)} |", + f"| Observed write events (telemetry) | {len(agg.writes)} |", + f"| Distinct nodes with an observed read | {len(agg.node_reads)} |", + "", + "Every number here counts **observed events**, not activity. Read telemetry is " + "best-effort and pipeline shell reads are never recorded, so these are lower bounds " + "and no percentage of total work can be derived from them.", + "", + ] + if agg.node_reads: + lines += ["## Most-read nodes (observed)", "", "| Node | Observed reads |", "| --- | ---: |"] + for path, count in agg.node_reads.most_common(15): + lines.append(f"| `{path}` | {count} |") + lines.append("") -def load_scope(path: Path) -> Scope: - raw = read_json(path) - if not isinstance(raw, dict) or raw.get("schema_version") != SCHEMA_VERSION: - raise AuditError(f"{path} must be a schema_version {SCHEMA_VERSION} scope object.") + if unobserved is not None: + lines += [ + "## Nodes with no observed read", + "", + f"{len(unobserved)} normal node(s) had no file-level read event and sat under no recorded " + "search root in this window.", + "", + "**This is an evidence gap, not a finding.** A node appears here when no event reached the " + "feed — which also happens for every pipeline shell read, and whenever best-effort read " + "telemetry drops a call. Do not treat this list as a removal or merge proposal; it is a " + "starting point for asking a human whether a node is still earning its place.", + "", + ] + if unobserved: + lines += ["```"] + [f"{path}" for path in unobserved[:100]] + ["```", ""] + if len(unobserved) > 100: + lines.append(f"…and {len(unobserved) - 100} more.\n") + else: + lines.append("Every normal node had an observed read or sat under a recorded search root.\n") - agents: list[ScopedAgent] = [] - for index, value in enumerate(raw.get("agents", [])): - if not isinstance(value, dict): - raise AuditError(f"agents[{index}] must be an object.") - authorization = validate_authorization(value.get("authorization"), f"agents[{index}].authorization") - if authorization != "explicit_agent": - raise AuditError( - f"agents[{index}].authorization must be explicit_agent." - ) - agents.append( - ScopedAgent( - name=require_string(value.get("name"), f"agents[{index}].name"), - agent_id=require_uuid(value.get("agent_id"), f"agents[{index}].agent_id"), - authorization=authorization, - ) - ) + if agg.node_writes: + lines += [ + "## Observed write events (telemetry only)", + "", + "Write telemetry does not see merge commits or worktree edits outside the bound path, so " + "this is **not** the complete set of Tree writes. For complete write activity, use the " + "git-derived write history of the Tree repository.", + "", + "| Node | Observed write events |", + "| --- | ---: |", + ] + for path, count in agg.node_writes.most_common(15): + lines.append(f"| `{path}` | {count} |") + lines.append("") - chats: list[ScopedChat] = [] - for index, value in enumerate(raw.get("chats", [])): - if not isinstance(value, dict): - raise AuditError(f"chats[{index}] must be an object.") - chat_id = require_uuid(value.get("chat_id"), f"chats[{index}].chat_id") - agent = require_string(value.get("agent"), f"chats[{index}].agent") - authorization = validate_authorization(value.get("authorization"), f"chats[{index}].authorization") - if authorization != "explicit_chat": - raise AuditError( - f"chats[{index}].authorization must be explicit_chat." - ) - chats.append( - ScopedChat( - chat_id=chat_id, - agent=agent, - agent_id=require_uuid(value.get("agent_id"), f"chats[{index}].agent_id"), - authorization=authorization, - ) - ) + if judgments is not None: + effects = [item for item in judgments if item.get("effect")] + refuted = [item for item in effects if item.get("refuted") is True] + upheld = [item for item in effects if item.get("refuted") is not True] + counts = Counter(str(item["effect"]["type"]) for item in upheld) + rate = (len(refuted) / len(effects)) if effects else None + lines += [ + "## Influence (sampled)", + "", + "| Measure | Count |", + "| --- | ---: |", + f"| Sampled reads | {sample_size} |", + f"| Judged as influencing a later choice | {len(effects)} |", + f"| Refuted by the adversarial pass | {len(refuted)} |", + f"| Upheld | {len(upheld)} |", + "", + ] + if rate is None: + lines.append("No influence was claimed in this sample, so there is no refutation rate.\n") + else: + lines.append(f"**Refutation rate: {len(refuted)}/{len(effects)} ({rate:.0%})**\n") + if rate > 0.5: + lines += [ + "> ⚠️ More than half of the claimed effects were refuted. **The influence numbers in this " + "run are not reliable and are withheld.** Treat the exposure section above as the only " + "usable output, and investigate the judgment step before quoting any effect count.", + "", + ] + else: + lines += ["| Effect | Upheld |", "| --- | ---: |"] + for effect_type in EFFECT_TYPES: + lines.append(f"| {effect_type} | {counts[effect_type]} |") + lines.append("") + for item in upheld: + effect = item["effect"] + lines += [ + f"- **{effect['type']}** — {effect.get('summary', '').strip()}", + f" - read `{item['read_id']}` on `{item['target_path']}` at {item['read_at']}", + ] + if upheld: + lines.append("") - if bool(agents) == bool(chats): - raise AuditError( - "Scope must choose exactly one mode: one explicit_agent entry or one-or-more explicit_chat entries." - ) - if agents and len(agents) != 1: - raise AuditError("explicit_agent scope must contain exactly one Agent.") - if len({(chat.chat_id, chat.agent_id) for chat in chats}) != len(chats): - raise AuditError("The scope contains duplicate Chat and audited-Agent pairs.") - identities = { - (agent.name, agent.agent_id) for agent in agents - } | { - (chat.agent, chat.agent_id) for chat in chats - } - if len(identities) != 1: - raise AuditError("Scope must name one exact Agent name and UUID.") - if "authorization_context" in raw: - raise AuditError( - "Scope must not contain authorization_context; explicit scope is " - "the complete authorization model." - ) - return Scope(agents=tuple(agents), chats=tuple(chats)) + lines += ["## Known recording gaps", ""] + lines += [f"- {gap}" for gap in KNOWN_GAPS] + lines += [ + "", + "Missing evidence is unknown. It is never proof that a node went unread or that the Tree went unused.", + "", + ] + return "\n".join(lines) -def run_first_tree_data(binary: str, arguments: Sequence[str]) -> Any: - command = [binary, "--json", *arguments] - completed = subprocess.run(command, capture_output=True, check=False, text=True) - if completed.returncode != 0: - raise AuditError( - f"Read-only First Tree command failed: {' '.join(command)} " - f"(exit {completed.returncode}; output withheld)." - ) - output = completed.stdout.strip() - if not output: - raise AuditError(f"Read-only First Tree command produced no JSON: {' '.join(command)}") - try: - payload = json.loads(output) - except json.JSONDecodeError as error: - raise AuditError(f"Invalid JSON from {' '.join(command)}: {error}") from error - if not isinstance(payload, dict) or payload.get("ok") is not True: - raise AuditError(f"First Tree command did not succeed: {' '.join(command)}") - return payload.get("data") +# ── commands ────────────────────────────────────────────────────────────── -def run_first_tree_json(binary: str, arguments: Sequence[str]) -> dict[str, Any]: - data = run_first_tree_data(binary, arguments) - if not isinstance(data, dict): - raise AuditError(f"First Tree command returned no data object: {' '.join(arguments)}") - return data +def resolve_tree_root(value: str | None) -> Path: + raw = Path(value).expanduser() if value else None + if raw is None: + raise AuditError("--tree-root is required to resolve node paths and content.") + if not raw.is_absolute() or raw.is_symlink(): + raise AuditError("--tree-root must be an absolute, non-symlink directory.") + root = raw.resolve(strict=False) + if not root.is_dir(): + raise AuditError(f"--tree-root is not a directory: {root}") + return root -def resolve_first_tree_binary(cli_value: str | None) -> str: - """Resolve one executable without invoking a shell or probing product state.""" - requested = cli_value or os.environ.get("FIRST_TREE_BIN") - candidates = [requested] if requested else ["first-tree", "first-tree-staging"] - for candidate in candidates: - if not candidate: - continue - if os.path.sep in candidate: - raw = Path(candidate).expanduser() - try: - resolved = raw.resolve(strict=True) - except OSError: - continue - if resolved.is_file() and os.access(resolved, os.X_OK): - return str(resolved) - continue - discovered = shutil.which(candidate) - if discovered: - return discovered - if requested: - raise AuditError( - "The explicitly configured First Tree binary is missing or not executable." - ) - raise AuditError( - "Could not find a First Tree CLI; set FIRST_TREE_BIN or pass --first-tree-bin." +def load_events(args: argparse.Namespace) -> list[IoEvent]: + if args.events_file: + path = Path(args.events_file).expanduser() + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AuditError(f"Could not read --events-file: {error}") from error + raw_items = payload.get("items") if isinstance(payload, dict) else payload + if not isinstance(raw_items, list): + raise AuditError("--events-file must contain an items array.") + else: + binary = resolve_binary(args.first_tree_bin) + since = parse_time(args.since, field_name="--since") if args.since else None + until = parse_time(args.until, field_name="--until") if args.until else None + if since and until and since > until: + raise AuditError("--since must not be after --until.") + raw_items = fetch_io_events( + binary, + since=since, + until=until, + chat_id=args.chat, + agent=args.agent, + ) + return [normalize_event(item) for item in raw_items if isinstance(item, dict)] + + +def write_output(path: str | None, text: str) -> None: + if not path: + sys.stdout.write(text) + return + target = Path(path).expanduser() + target.parent.mkdir(parents=True, exist_ok=True, mode=0o700) + target.write_text(text, encoding="utf-8") + os.chmod(target, 0o600) + + +def command_facts(args: argparse.Namespace) -> None: + events = load_events(args) + tree_root = resolve_tree_root(args.tree_root) if args.tree_root else None + expected = tree_identity_of_root(tree_root) if tree_root else None + if tree_root is not None and expected is None: + raise AuditError( + "Could not establish the bound Tree's repository and branch identity from --tree-root, " + "so IO events cannot be proven to belong to it." + ) + selected, identity = select_events_for_tree(events, expected) + agg = aggregate(selected) + unobserved = unobserved_nodes(tree_root, agg) if tree_root else None + window_start = parse_time(args.since, field_name="--since") if args.since else None + window_end = parse_time(args.until, field_name="--until") if args.until else None + generated_at = parse_time(args.now, field_name="--now") if args.now else datetime.now(timezone.utc) + if args.json: + write_output( + args.output, + json.dumps( + { + "schema_version": SCHEMA_VERSION, + "generated_at": iso(generated_at), + "tree_identity": identity.label(), + "excluded_events_from_other_trees": len(events) - len(selected), + "reads": len(agg.reads), + "writes": len(agg.writes), + "chats_with_read": sorted(agg.chats_with_read), + "node_reads": dict(agg.node_reads.most_common()), + "node_writes": dict(agg.node_writes.most_common()), + "searched_dirs": sorted(agg.searched_dirs), + "unobserved_nodes": unobserved, + "source_counts": dict(sorted(agg.source_counts.items())), + "known_gaps": KNOWN_GAPS, + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + ) + return + write_output( + args.output, + render_report( + generated_at=generated_at, + window_start=window_start, + window_end=window_end, + agg=agg, + unobserved=unobserved, + judgments=None, + sample_size=0, + ), ) -def verify_cli_agent_identity(binary: str, agent_slug: str, agent_id: str) -> None: - """Cross-check the runtime slug through the CLI's local binding resolver.""" - command = [binary, "agent", "list"] - environment = os.environ.copy() - environment.pop("FIRST_TREE_JSON", None) - completed = subprocess.run( - command, - capture_output=True, - check=False, - text=True, - env=environment, - ) - if completed.returncode != 0: +def command_sample(args: argparse.Namespace) -> None: + events = load_events(args) + tree_root = resolve_tree_root(args.tree_root) + expected = tree_identity_of_root(tree_root) + if expected is None: raise AuditError( - "The invoking Agent slug could not be resolved from local First Tree bindings " - f"(exit {completed.returncode}; output withheld)." + "Could not establish the bound Tree's repository and branch identity from --tree-root." ) - resolved_ids = re.findall( - rf"^[ \t]*{re.escape(agent_slug)}[ \t]+runtime:[ \t]+" - rf"[^ \t\r\n]+[ \t]+uuid:[ \t]+({UUID_PATTERN})[ \t]*$", - f"{completed.stdout}\n{completed.stderr}", - re.MULTILINE, - ) - if len(resolved_ids) != 1 or require_uuid( - resolved_ids[0], "First Tree CLI Agent UUID" - ) != agent_id: - raise AuditError( - "The invoking Agent's local CLI binding does not resolve to the " - "authorized workspace UUID." + selected, identity = select_events_for_tree(events, expected) + agg = aggregate(selected) + eligible = eligible_reads(agg.reads) + chosen = sample_reads(agg.reads, args.size, args.seed) + cases = [ + { + "read_id": event.event_id, + "chat_id": event.chat_id, + "read_at": iso(event.created_at), + "target_path": event.target_path, + "source": event.source, + "node": node_content_at_read(tree_root, event, args.max_content_chars), + } + for event in chosen + ] + write_output( + args.output, + json.dumps( + { + "schema_version": SCHEMA_VERSION, + "tree_identity": identity.label(), + "window": {"since": args.since, "until": args.until, "chat": args.chat}, + "eligible_reads": len(eligible), + "population_sha256": population_digest(eligible), + "sample_size": len(cases), + "seed": args.seed, + "cases": cases, + }, + ensure_ascii=False, + indent=2, ) + + "\n", + ) -def parse_agent_workspace(value: str) -> WorkspaceIdentity: - agent_id_text, separator, workspace_text = value.partition("=") - agent_id = require_uuid(agent_id_text, "--agent-workspace Agent UUID") - if not separator or not workspace_text.strip(): - raise AuditError("--agent-workspace must use AGENT_UUID=/absolute/workspace syntax.") - raw_workspace = Path(workspace_text).expanduser() - if not raw_workspace.is_absolute(): - raise AuditError("--agent-workspace must name an absolute workspace path.") - workspace_lexical = Path(os.path.abspath(raw_workspace)) - if workspace_lexical.is_symlink(): - raise AuditError("Authorized Agent workspace must not be a symbolic link.") - try: - workspace = workspace_lexical.resolve(strict=True) - except OSError as error: - raise AuditError(f"Could not resolve the authorized Agent workspace: {error}") from error - if not workspace.is_dir(): - raise AuditError("Authorized Agent workspace is not a directory.") - - runtime_dir = workspace / ".first-tree-workspace" - identity_path = runtime_dir / "identity.json" - if runtime_dir.is_symlink() or identity_path.is_symlink(): - raise AuditError("Managed workspace identity must not traverse a symbolic link.") - try: - resolved_identity = identity_path.resolve(strict=True) - resolved_identity.relative_to(workspace) - except (OSError, ValueError) as error: - raise AuditError("Managed workspace identity is missing or outside the workspace.") from error - if not resolved_identity.is_file(): - raise AuditError("Managed workspace identity is not a regular file.") - identity = read_json(resolved_identity) - if ( - not isinstance(identity, dict) - or identity.get("agentId") != agent_id - or identity.get("type") != "agent" - ): - raise AuditError(f"Managed workspace identity does not match Agent {agent_id}.") - agent_display_name = identity.get("displayName") - if not isinstance(agent_display_name, str) or not agent_display_name.strip(): - raise AuditError("Managed workspace identity does not declare an Agent display name.") - runtime_agent_id_text = os.environ.get("FIRST_TREE_AGENT_ID") - runtime_agent_slug = os.environ.get("FIRST_TREE_AGENT_SLUG") - if not runtime_agent_id_text: - raise AuditError( - "FIRST_TREE_AGENT_ID is required to bind the audit to the invoking runtime Agent." - ) - runtime_agent_id = require_uuid(runtime_agent_id_text, "FIRST_TREE_AGENT_ID") - if runtime_agent_id != agent_id: +def validate_judgments(raw: Any, cases: Mapping[str, Mapping[str, Any]]) -> list[dict[str, Any]]: + if not isinstance(raw, list): + raise AuditError("Judgments must be a JSON array.") + seen: set[str] = set() + out: list[dict[str, Any]] = [] + for index, item in enumerate(raw): + where = f"judgments[{index}]" + if not isinstance(item, dict): + raise AuditError(f"{where} must be an object.") + read_id = item.get("read_id") + if not isinstance(read_id, str) or read_id not in cases: + raise AuditError(f"{where}.read_id does not match a sampled case.") + if read_id in seen: + raise AuditError(f"{where} duplicates read_id {read_id}.") + seen.add(read_id) + effect = item.get("effect") + row: dict[str, Any] = { + "read_id": read_id, + "target_path": cases[read_id]["target_path"], + "read_at": cases[read_id]["read_at"], + "effect": None, + "refuted": False, + } + if effect is None: + out.append(row) + continue + if not isinstance(effect, dict): + raise AuditError(f"{where}.effect must be an object or null.") + effect_type = effect.get("type") + if effect_type not in EFFECT_TYPES: + raise AuditError(f"{where}.effect.type must be one of: {', '.join(EFFECT_TYPES)}.") + summary = effect.get("summary") + if not isinstance(summary, str) or not summary.strip(): + raise AuditError(f"{where}.effect.summary is required.") + refuted = item.get("refuted") + if not isinstance(refuted, bool): + raise AuditError( + f"{where}.refuted must be true or false — every claimed effect must go through the " + "adversarial pass." + ) + if refuted and not str(item.get("refutation", "")).strip(): + raise AuditError(f"{where}.refutation is required when refuted is true.") + row["effect"] = {"type": effect_type, "summary": summary.strip()} + row["refuted"] = refuted + if refuted: + row["refutation"] = str(item["refutation"]).strip() + out.append(row) + missing = set(cases) - seen + if missing: raise AuditError( - "FIRST_TREE_AGENT_ID does not match the authorized workspace Agent UUID." + "Every sampled case needs a judgment; missing: " + ", ".join(sorted(missing)) + "." ) - if ( - not isinstance(runtime_agent_slug, str) - or AGENT_SLUG_PATTERN.fullmatch(runtime_agent_slug) is None - ): + return out + + +def command_report(args: argparse.Namespace) -> None: + events = load_events(args) + tree_root = resolve_tree_root(args.tree_root) if args.tree_root else None + expected = tree_identity_of_root(tree_root) if tree_root else None + if tree_root is not None and expected is None: raise AuditError( - "FIRST_TREE_AGENT_SLUG must contain the invoking Agent's lowercase CLI selector." + "Could not establish the bound Tree's repository and branch identity from --tree-root." ) - tree_value = identity.get("contextTreePath") - if not isinstance(tree_value, str) or not tree_value.strip(): - raise AuditError("Managed workspace identity does not declare a bound Context Tree.") - raw_tree = Path(tree_value).expanduser() - if not raw_tree.is_absolute() or raw_tree.is_symlink(): - raise AuditError("Bound Context Tree must be an absolute, non-symbolic-link path.") + selected, identity = select_events_for_tree(events, expected) + agg = aggregate(selected) + unobserved = unobserved_nodes(tree_root, agg) if tree_root else None + try: - bound_tree = raw_tree.resolve(strict=True) - except OSError as error: - raise AuditError(f"Could not resolve the bound Context Tree: {error}") from error - if not bound_tree.is_dir(): - raise AuditError("Bound Context Tree is not a directory.") - return WorkspaceIdentity( - agent_name=runtime_agent_slug, - agent_display_name=agent_display_name.strip(), - agent_id=agent_id, - workspace_lexical=workspace_lexical, - workspace=workspace, - bound_tree_root=bound_tree, + sample_payload = json.loads(Path(args.sample).expanduser().read_text(encoding="utf-8")) + judgment_payload = json.loads(Path(args.judgments).expanduser().read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as error: + raise AuditError(f"Could not read sample or judgments: {error}") from error + if not isinstance(sample_payload, dict): + raise AuditError("--sample must be a sample object produced by the `sample` command.") + raw_cases = sample_payload.get("cases") + if not isinstance(raw_cases, list): + raise AuditError("--sample must contain a cases array.") + + # The sample must provably come from this same feed, Tree, and window. + # Effect counts drawn from a different population are not evidence. + if sample_payload.get("tree_identity") != identity.label(): + raise AuditError( + f"--sample was drawn from {sample_payload.get('tree_identity')!r}, but this run audits " + f"{identity.label()!r}. Re-run `sample` against the intended Tree." + ) + sample_window = sample_payload.get("window") + current_window = {"since": args.since, "until": args.until, "chat": args.chat} + if sample_window != current_window: + raise AuditError( + "--sample was drawn over a different acquisition window " + f"({sample_window!r} vs {current_window!r}); re-run `sample` and `report` over the same one." + ) + eligible = eligible_reads(agg.reads) + current_digest = population_digest(eligible) + if sample_payload.get("population_sha256") != current_digest: + raise AuditError( + "The eligible read population changed since --sample was drawn, so the sample is no longer " + "uniform over it. Re-run `sample` (new events arrive continuously) before reporting." + ) + if sample_payload.get("eligible_reads") != len(eligible): + raise AuditError("--sample reports a different eligible population size than this feed holds.") + + # A matching population digest only proves the population is unchanged. It + # says nothing about whether `cases` really is the draw. Recompute the draw + # from the recorded seed and size and require an exact match, so a + # hand-picked or edited case list cannot yield effect counts. + seed = sample_payload.get("seed") + size = sample_payload.get("sample_size") + if not isinstance(seed, int) or not isinstance(size, int) or size < 0: + raise AuditError("--sample must record the integer `seed` and `sample_size` it was drawn with.") + expected_draw = sample_reads(agg.reads, size, seed) + if [item.event_id for item in expected_draw] != [str(case.get("read_id")) for case in raw_cases]: + raise AuditError( + "--sample cases are not the uniform draw for their recorded seed and size. Re-run `sample` " + "instead of editing or hand-picking cases." + ) + by_id = {item.event_id: item for item in expected_draw} + for case in raw_cases: + source = by_id[str(case["read_id"])] + if case.get("target_path") != source.target_path or case.get("read_at") != iso(source.created_at): + raise AuditError( + f"--sample case {case.get('read_id')!r} does not match the recorded event it names." + ) + cases = { + str(case["read_id"]): case + for case in raw_cases + if isinstance(case, dict) and isinstance(case.get("read_id"), str) + } + judgments = validate_judgments(judgment_payload, cases) + generated_at = parse_time(args.now, field_name="--now") if args.now else datetime.now(timezone.utc) + write_output( + args.output, + render_report( + generated_at=generated_at, + window_start=parse_time(args.since, field_name="--since") if args.since else None, + window_end=parse_time(args.until, field_name="--until") if args.until else None, + agg=agg, + unobserved=unobserved, + judgments=judgments, + sample_size=len(cases), + ), ) -def paginated_items(binary: str, arguments: Sequence[str], *, agent: str | None) -> list[dict[str, Any]]: - items: list[dict[str, Any]] = [] - cursor: str | None = None - seen_cursors: set[str] = set() - while True: - page_args = [*arguments, "-l", "100"] - if cursor is not None: - page_args.extend(["--cursor", cursor]) - if agent is not None: - # The historical Agent-name grammar permits leading `-`, including - # option-looking names such as `--json`. The `--option=value` - # form keeps the selector bound to this argument instead of - # allowing the CLI parser to reinterpret it as another option. - page_args.append(f"--agent={agent}") - data = run_first_tree_json(binary, page_args) - page_items = data.get("items") - if not isinstance(page_items, list): - raise AuditError(f"Expected data.items from {' '.join(page_args)}.") - for item in page_items: - if isinstance(item, dict): - items.append(item) - next_cursor = data.get("nextCursor") - if not isinstance(next_cursor, str) or not next_cursor: - break - if next_cursor in seen_cursors: - raise AuditError(f"Pagination cursor repeated for {' '.join(page_args)}.") - seen_cursors.add(next_cursor) - cursor = next_cursor - return items - +# ── parser ──────────────────────────────────────────────────────────────── -def supported_repository_identity(value: Any) -> str | None: - if not isinstance(value, str): - return None - repository = value.strip() - if ( - not repository - or any(character.isspace() or ord(character) < 32 for character in repository) - or repository.startswith(("/", "./", "../", "~")) - ): - return None - scp_match = re.fullmatch( - r"git@(?P[A-Za-z0-9.-]+):(?P[^?#]+)", - repository, +def add_feed_options(parser: argparse.ArgumentParser) -> None: + parser.add_argument("--since", help="RFC 3339 lower bound on event time.") + parser.add_argument("--until", help="RFC 3339 upper bound on event time.") + parser.add_argument("--chat", help="Restrict to one Chat UUID.") + parser.add_argument("--agent", help="Local agent name (defaults to FIRST_TREE_AGENT_ID).") + parser.add_argument("--first-tree-bin", help="First Tree CLI executable.") + parser.add_argument( + "--events-file", + help="Read a captured `tree io --json` payload instead of calling the CLI (tests, replay).", ) - if scp_match is not None: - host = scp_match.group("host") - path = scp_match.group("path") - else: - try: - parsed = urlsplit(repository) - parsed_host = parsed.hostname - parsed.port - except ValueError: - return None - if parsed.scheme not in {"http", "https", "ssh", "git"}: - return None - if parsed.query or parsed.fragment or not parsed_host: - return None - if parsed.password is not None: - return None - if parsed.scheme in {"http", "https", "git"} and parsed.username is not None: - return None - if parsed.scheme == "ssh" and parsed.username not in {None, "git"}: - return None - host = parsed_host - path = parsed.path.lstrip("/") - - lowered_host = host.lower().rstrip(".") - if lowered_host == "localhost" or "%" in lowered_host: - return None - try: - address = ipaddress.ip_address(lowered_host) - except ValueError: - numeric_components = lowered_host.split(".") - if re.fullmatch(r"[0-9.]+", lowered_host) or all( - re.fullmatch(r"(?:[0-9]+|0x[0-9a-f]+)", component) - for component in numeric_components - ): - return None - address = None - if address is not None: - mapped_address = getattr(address, "ipv4_mapped", None) - if ( - address.is_loopback - or address.is_unspecified - or ( - mapped_address is not None - and (mapped_address.is_loopback or mapped_address.is_unspecified) - ) - ): - return None - path_parts = path.removesuffix(".git").split("/") - if ( - len(path_parts) < 2 - or any(part in {"", ".", ".."} for part in path_parts) - ): - return None - return repository + parser.add_argument("--output", help="Write here instead of stdout (mode 0600).") + parser.add_argument("--now", help="Fixed RFC 3339 generation time for reproducible output.") -def git_output(tree_root: Path, arguments: Sequence[str]) -> str | None: - completed = subprocess.run( - ["git", "-C", str(tree_root), *arguments], - capture_output=True, - check=False, - text=True, - env={ - **os.environ, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_TERMINAL_PROMPT": "0", - }, +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Context Tree value audit over the durable agent IO feed.", ) - if completed.returncode != 0: - return None - return completed.stdout.strip() + sub = parser.add_subparsers(dest="command", required=True) - -def resolve_tree_source_snapshot(tree_root: Path) -> dict[str, Any]: - repository_root = git_output(tree_root, ["rev-parse", "--show-toplevel"]) - if ( - not isinstance(repository_root, str) - or Path(repository_root).resolve(strict=False) - != tree_root.resolve(strict=False) - ): - return { - "status": "unavailable", - "reason": "bound_tree_is_not_repository_root", - } - remote_head = git_output( - tree_root, - ["symbolic-ref", "refs/remotes/origin/HEAD"], + facts = sub.add_parser( + "facts", help="Aggregate observed exposure, node distribution, and observed write events." ) - if not isinstance(remote_head, str): - return { - "status": "unavailable", - "reason": "default_branch_ref_unavailable", - } - match = re.fullmatch(r"refs/remotes/([^/]+)/(.+)", remote_head) - if match is None: - return { - "status": "unavailable", - "reason": "default_branch_ref_malformed", - } - _, branch = match.groups() - commit = git_output(tree_root, ["rev-parse", "--verify", remote_head]) - if ( - not isinstance(commit, str) - or re.fullmatch(r"[0-9a-fA-F]{40}", commit) is None - ): - return { - "status": "unavailable", - "reason": "default_branch_identity_unavailable", - } - return { - "status": "local_default_branch", - "branch": branch, - "commit": commit.lower(), - } - - -def git_file_at_commit( - tree_root: Path, - commit: str, - node_path: str, -) -> str | None: - completed = subprocess.run( - ["git", "-C", str(tree_root), "show", f"{commit}:{node_path}"], - capture_output=True, - check=False, - text=True, - env={ - **os.environ, - "GIT_CONFIG_NOSYSTEM": "1", - "GIT_TERMINAL_PROMPT": "0", - }, - ) - return completed.stdout if completed.returncode == 0 else None - - -def read_tree_source( - read: Mapping[str, Any], - tree_root: Path, - snapshot: Mapping[str, Any], -) -> dict[str, Any]: - unverified = {"status": "unverified_source"} - if snapshot.get("status") != "local_default_branch": - return unverified - node_paths = read.get("node_paths") - passage = read.get("passage") - if ( - not isinstance(node_paths, list) - or len(node_paths) != 1 - or not isinstance(node_paths[0], str) - or not isinstance(passage, str) - or not passage.strip() - ): - return unverified - snapshot_commit = snapshot.get("commit") - if not isinstance(snapshot_commit, str): - return unverified - node_path = node_paths[0] - default_content = git_file_at_commit(tree_root, snapshot_commit, node_path) - if default_content is None or passage.strip() not in default_content: - return unverified - return { - "status": "default_branch_match", - "branch": snapshot["branch"], - "commit": snapshot_commit, - "node_path": node_path, - } - - -def normalize_context_decision(value: Any) -> dict[str, Any] | None: - """Return the minimal valid contextDecision v1 projection. - - The message path is intentionally tolerant: malformed analysis metadata is - diagnosed by the caller, never allowed to block Chat export. - """ - if not isinstance(value, Mapping): - return None - effect = value.get("effect") - if ( - value.get("version") != 1 - or not isinstance(effect, str) - or effect not in EFFECT_VALUES - ): - return None - summary = value.get("summary") - evidence = value.get("evidence") - if not isinstance(summary, str) or not summary.strip(): - return None - if not isinstance(evidence, list) or not 1 <= len(evidence) <= 3: - return None - projected_evidence: list[dict[str, Any]] = [] - for item in evidence: - if not isinstance(item, Mapping): - return None - repo_url = supported_repository_identity(item.get("repoUrl")) - commit = item.get("commit") - node_path = item.get("nodePath") - heading = item.get("heading") - normalized_node_path = node_path.strip() if isinstance(node_path, str) else "" - if ( - repo_url is None - or not isinstance(commit, str) - or re.fullmatch(r"[0-9a-fA-F]{40}", commit) is None - or not normalized_node_path - or "\\" in normalized_node_path - or Path(normalized_node_path).is_absolute() - or ".." in Path(normalized_node_path).parts - or not normalized_node_path.endswith(".md") - or (heading is not None and (not isinstance(heading, str) or not heading.strip())) - ): - return None - projected = { - "repoUrl": repo_url, - "commit": commit.lower(), - "nodePath": normalized_node_path, - } - if heading is not None: - projected["heading"] = heading.strip() - projected_evidence.append(projected) - return { - "version": 1, - "effect": effect, - "summary": summary.strip(), - "evidence": projected_evidence, - } - - -def message_record(value: Mapping[str, Any]) -> dict[str, Any]: - created_at = value.get("createdAt") or value.get("created_at") - content = value.get("content") - record = { - "message_id": value.get("id") or value.get("message_id"), - "created_at": created_at if isinstance(created_at, str) else None, - "sender_id": value.get("senderId") or value.get("sender_id"), - "sender_kind": value.get("sender_kind"), - "content": content if isinstance(content, str) else payload_text(content), - } - metadata = value.get("metadata") - raw_receipt: Any = None - receipt_present = False - if isinstance(metadata, Mapping) and "contextDecision" in metadata: - raw_receipt = metadata.get("contextDecision") - receipt_present = True - elif "decision_receipt" in value: - raw_receipt = value.get("decision_receipt") - receipt_present = True - if receipt_present: - receipt = normalize_context_decision(raw_receipt) - if receipt is None: - record["_context_decision_invalid"] = True - else: - record["decision_receipt"] = receipt - return record - - -def normalize_message_records( - values: Iterable[Mapping[str, Any]], - *, - window: Window | None = None, -) -> tuple[list[dict[str, Any]], bool]: - records: list[dict[str, Any]] = [] - invalid_receipt = False - for value in values: - record = message_record(value) - invalid = record.pop("_context_decision_invalid", False) is True - if invalid and ( - window is None - or ( - isinstance(record.get("created_at"), str) - and in_window(record["created_at"], window) - ) - ): - invalid_receipt = True - records.append(record) - return records, invalid_receipt - - -def export_chats(args: argparse.Namespace) -> None: - workspace_identity = parse_agent_workspace(args.agent_workspace) - artifact_root = resolve_artifact_root(args.artifact_root, workspace_identity) - scope_path = artifact_path(artifact_root, args.scope, field="--scope", must_exist=True) - output_path = artifact_path(artifact_root, args.output, field="--output", must_exist=False) - require_distinct_paths({"--scope": scope_path, "--output": output_path}) - scope = load_scope(scope_path) - scoped_identities = { - (agent.name, agent.agent_id) for agent in scope.agents - } | { - (chat.agent, chat.agent_id) for chat in scope.chats - } - scoped_agent_name, scoped_agent_id = next(iter(scoped_identities)) - if ( - workspace_identity.agent_name != scoped_agent_name - or workspace_identity.agent_id != scoped_agent_id - ): - raise AuditError( - "--agent-workspace identity must match the one exact Agent name and UUID in --scope." - ) - first_tree_binary = resolve_first_tree_binary(args.first_tree_bin) - verify_cli_agent_identity( - first_tree_binary, - workspace_identity.agent_name, - workspace_identity.agent_id, - ) - window = resolve_window(args.days, args.now) - chat_sources: dict[tuple[str, str], dict[str, Any]] = {} - - for scoped_agent in scope.agents: - for chat in paginated_items( - first_tree_binary, - ["chat", "list"], - agent=scoped_agent.name, - ): - chat_id = chat.get("id") - if ( - not isinstance(chat_id, str) - or re.fullmatch(UUID_PATTERN, chat_id) is None - ): - continue - if strictly_before_window(chat.get("lastMessageAt"), window): - continue - key = (chat_id, scoped_agent.agent_id) - chat_sources.setdefault( - key, - { - "agent": scoped_agent.name, - "agent_id": scoped_agent.agent_id, - "authorization": scoped_agent.authorization, - "chat": chat, - "exact_chat": False, - }, - ) - - for scoped_chat in scope.chats: - key = (scoped_chat.chat_id, scoped_chat.agent_id) - chat_sources[key] = { - "agent": scoped_chat.agent, - "agent_id": scoped_chat.agent_id, - "authorization": scoped_chat.authorization, - "chat": {}, - "exact_chat": True, - } - - exported: list[dict[str, Any]] = [] - for (chat_id, source_agent_id), source in sorted(chat_sources.items()): - history = paginated_items( - first_tree_binary, - ["chat", "history", chat_id], - agent=source["agent"], - ) - messages, invalid_receipt = normalize_message_records(history, window=window) - messages = [ - message - for message in messages - if isinstance(message.get("created_at"), str) and in_window(message["created_at"], window) - ] - messages.sort(key=lambda message: (message["created_at"], str(message.get("message_id") or ""))) - if not messages and not source["exact_chat"]: - continue - chat_metadata = source["chat"] - title = chat_metadata.get("topic") or chat_metadata.get("title") or chat_id - exported.append( - { - "schema_version": SCHEMA_VERSION, - "audit_id": audit_id(chat_id, source_agent_id), - "chat_id": chat_id, - "title": title, - "authorization": source["authorization"], - "source_agent": source["agent"], - "source_agent_id": source["agent_id"], - "window": {"start": window_start_text(window), "end": isoformat(window.end)}, - "messages": messages, - "coverage_gaps": sorted( - (["context_decision_invalid"] if invalid_receipt else []) - + ([] if messages else ["no_visible_messages_in_window"]) - ), - } - ) - - write_jsonl(output_path, exported) - - -def payload_text(value: Any) -> str: - if isinstance(value, str): - return value - if isinstance(value, list): - return "\n".join(filter(None, (payload_text(item) for item in value))) - if not isinstance(value, dict): - return "" - values = [] - for key in ("text", "content", "output", "message"): - if key in value: - text = payload_text(value[key]) - if text: - values.append(text) - return "\n".join(values) - - -def chat_ids_from_text(text: str) -> list[str]: - chat_ids: list[str] = [] - for block in CHAT_CONTEXT_PATTERN.findall(text): - chat_ids.extend(match.group(1) for match in CHAT_ID_PATTERN.finditer(block)) - return chat_ids - - -def parse_tool_arguments(payload: Mapping[str, Any]) -> dict[str, Any]: - raw = payload.get("arguments") - if isinstance(raw, dict): - return dict(raw) - if isinstance(raw, str): - try: - parsed = json.loads(raw) - except json.JSONDecodeError: - return {"raw": raw} - return parsed if isinstance(parsed, dict) else {"raw": raw} - raw_input = payload.get("input") - if isinstance(raw_input, dict): - return dict(raw_input) - if isinstance(raw_input, str): - try: - parsed = json.loads(raw_input) - except json.JSONDecodeError: - return {"raw": raw_input} - return parsed if isinstance(parsed, dict) else {"raw": raw_input} - return {} - - -def tool_raw(payload: Mapping[str, Any]) -> str: - arguments = parse_tool_arguments(payload) - return "\n".join( - filter( - None, - ( - str(payload.get("name") or ""), - payload_text(payload.get("input")), - payload_text(payload.get("arguments")), - payload_text(arguments), - ), - ) - ) - - -def relative_tree_path(candidate: Path, tree_roots: Sequence[Path]) -> str | None: - try: - resolved = candidate.expanduser().resolve(strict=False) - except OSError: - return None - for tree_root in tree_roots: - try: - relative = resolved.relative_to(tree_root) - except ValueError: - continue - if relative.suffix.lower() == ".md" and ".." not in relative.parts and relative.parts: - return relative.as_posix() - return None - - -def extract_node_paths( - payload: Mapping[str, Any], - tree_roots: Sequence[Path], - default_workdir: Path, -) -> list[str]: - raw = tool_raw(payload).replace("\\/", "/") - paths: set[str] = set() - - for tree_root in tree_roots: - root_text = tree_root.as_posix().rstrip("/") - absolute_pattern = re.compile( - rf"{re.escape(root_text)}/(?P[^\"'`\r\n;|&<>]+?\.md)(?=$|[\s\"'`,;:)|&<>])", - re.UNICODE, - ) - for match in absolute_pattern.finditer(raw): - relative = match.group("relative").strip() - candidate = relative_tree_path(tree_root / relative, tree_roots) - if candidate is not None: - paths.add(candidate) - - arguments = parse_tool_arguments(payload) - workdir_value = arguments.get("workdir") - workdirs = {default_workdir} - if isinstance(workdir_value, str) and workdir_value.strip(): - workdirs.add(Path(workdir_value).expanduser()) - for match in re.finditer(r"""workdir\s*:\s*["'`]([^"'`]+)["'`]""", raw, re.UNICODE): - workdirs.add(Path(match.group(1)).expanduser()) - - def string_values(value: Any) -> Iterator[str]: - if isinstance(value, str): - yield value - elif isinstance(value, Mapping): - for child in value.values(): - yield from string_values(child) - elif isinstance(value, (list, tuple)): - for child in value: - yield from string_values(child) - - for value in string_values(arguments): - try: - tokens = shlex.split(value) - except ValueError: - tokens = value.split() - for token in tokens: - cleaned = token.strip(" \t\r\n\"'`,;:()[]{}") - if not cleaned.lower().endswith(".md") or any(character.isspace() for character in cleaned): - continue - token_path = Path(cleaned).expanduser() - candidates = [token_path] if token_path.is_absolute() else [workdir / token_path for workdir in workdirs] - for candidate in candidates: - relative = relative_tree_path(candidate, tree_roots) - if relative is not None: - paths.add(relative) - - relative_pattern = re.compile( - r"""(?]+/)+[^\s/"'`(){}[\],;:|&<>]+\.md)""", - re.UNICODE, - ) - for match in relative_pattern.finditer(raw): - token_path = Path(match.group(1)) - for workdir in workdirs: - relative = relative_tree_path(workdir / token_path, tree_roots) - if relative is not None: - paths.add(relative) - return sorted(paths) - - -def has_literal_non_tree_markdown( - payload: Mapping[str, Any], - tree_root: Path, -) -> bool: - raw = tool_raw(payload).replace("\\/", "/") - for match in re.finditer( - r"(?]+\.md)" - r"(?=$|[\s\"'`,;:)|&<>])", - raw, - re.UNICODE, - ): - candidate = Path(match.group(1)).expanduser().resolve(strict=False) - if not path_is_within(candidate, tree_root): - return True - return False - - -def path_is_within(path: Path, root: Path) -> bool: - try: - path.expanduser().resolve(strict=False).relative_to(root) - except (OSError, ValueError): - return False - return True - - -def normalize_shell_newlines(command: str) -> str | None: - """Turn unquoted newlines into shell separators without changing quotes.""" - output: list[str] = [] - quote: str | None = None - escaped = False - for character in command: - if escaped: - output.append(character) - escaped = False - continue - if character == "\\" and quote != "'": - output.append(character) - escaped = True - continue - if quote is None and character in {"'", '"'}: - quote = character - output.append(character) - continue - if quote == character: - quote = None - output.append(character) - continue - if quote is None and character in {"\n", "\r"}: - output.append(";") - continue - output.append(character) - if quote is not None or escaped: - return None - return "".join(output) - - -def shell_segments( - command: str, -) -> tuple[list[ShellSegment] | None, str | None]: - normalized = normalize_shell_newlines(command) - if normalized is None: - return None, "unresolved_shell_syntax" - try: - lexer = shlex.shlex( - normalized, - posix=True, - punctuation_chars=";&|<>()`", - ) - lexer.whitespace_split = True - lexer.commenters = "" - tokens = list(lexer) - except ValueError: - return None, "unresolved_shell_syntax" - if not tokens: - return None, "unresolved_empty_command" - - segments: list[ShellSegment] = [] - current: list[str] = [] - input_mode = "none" - output_discarded = False - index = 0 - while index < len(tokens): - token = tokens[index] - if token in {";", "&&", "||", "|"}: - if not current: - return None, "unresolved_shell_syntax" - segments.append( - ShellSegment( - tokens=tuple(current), - input_mode=input_mode, - output_discarded=output_discarded, - ) - ) - current = [] - input_mode = "pipe" if token == "|" else "none" - output_discarded = False - index += 1 - continue - if ( - token in {">", ">>", "&>"} - and index + 1 < len(tokens) - and tokens[index + 1] == "/dev/null" - ): - descriptor = "1" - if current and current[-1] in {"0", "1", "2"}: - descriptor = current.pop() - if descriptor in {"1", "&"}: - output_discarded = True - index += 2 - continue - if ( - token in {"0", "1", "2"} - and index + 2 < len(tokens) - and tokens[index + 1] == ">&" - and tokens[index + 2] in {"0", "1", "2"} - ): - index += 3 - continue - if token and any(character in token for character in "<>"): - return None, "unsafe_shell_redirection" - if token == "&": - return None, "unsafe_background_shell" - if token in {"(", ")"}: - return None, "unresolved_shell_grouping" - if token == "`" or "$(" in token: - return None, "unresolved_command_substitution" - current.append(token) - index += 1 - if not current: - return None, "unresolved_shell_syntax" - segments.append( - ShellSegment( - tokens=tuple(current), - input_mode=input_mode, - output_discarded=output_discarded, - ) - ) - return segments, None - - -def expand_static_for_loop( - segments: Sequence[ShellSegment], -) -> tuple[list[ShellSegment] | None, str | None]: - control_indexes = [ - index - for index, segment in enumerate(segments) - if segment.tokens - and segment.tokens[0] in {"for", "do", "done"} - ] - if not control_indexes: - return list(segments), None - for_indexes = [ - index - for index in control_indexes - if segments[index].tokens[0] == "for" - ] - done_indexes = [ - index - for index in control_indexes - if segments[index].tokens[0] == "done" - ] - if len(for_indexes) != 1 or len(done_indexes) != 1: - return None, "unresolved_shell_loop" - for_index = for_indexes[0] - done_index = done_indexes[0] - if done_index <= for_index: - return None, "unresolved_shell_loop" - header = segments[for_index] - if ( - header.input_mode != "none" - or len(header.tokens) < 4 - or header.tokens[2] != "in" - ): - return None, "unresolved_shell_loop" - variable = header.tokens[1] - if re.fullmatch(r"[A-Za-z_][A-Za-z0-9_]*", variable) is None: - return None, "unresolved_shell_loop" - values = list(header.tokens[3:]) - if ( - not values - or tokens_have_dynamic_expansion(values) - or any(not value.lower().endswith(".md") for value in values) - ): - return None, "unresolved_shell_loop" - - body = list(segments[for_index + 1 : done_index]) - if body and body[0].tokens and body[0].tokens[0] == "do": - first = body.pop(0) - if len(first.tokens) > 1: - body.insert( - 0, - ShellSegment( - tokens=first.tokens[1:], - input_mode=first.input_mode, - output_discarded=first.output_discarded, - ), - ) - if not body or any( - segment.tokens - and segment.tokens[0] in {"for", "do", "done"} - for segment in body - ): - return None, "unresolved_shell_loop" - - expanded: list[ShellSegment] = list(segments[:for_index]) - variable_forms = {f"${variable}", f"${{{variable}}}"} - for value in values: - for segment in body: - if any(token in variable_forms for token in segment.tokens): - expanded.append( - ShellSegment( - tokens=tuple( - value if token in variable_forms else token - for token in segment.tokens - ), - input_mode=segment.input_mode, - output_discarded=segment.output_discarded, - ) - ) - else: - expanded.append(segment) - suffix = list(segments[done_index + 1 :]) - if any( - segment.tokens - and segment.tokens[0] in {"for", "do", "done"} - for segment in suffix - ): - return None, "unresolved_shell_loop" - expanded.extend(suffix) - return expanded, None - - -def expand_literal_if_guard( - segments: Sequence[ShellSegment], -) -> tuple[list[ShellSegment] | None, str | None]: - """Flatten one literal if/test guard while preserving its guarded body.""" - control_indexes = [ - index - for index, segment in enumerate(segments) - if segment.tokens - and segment.tokens[0] in {"if", "then", "elif", "else", "fi"} - ] - if not control_indexes: - return list(segments), None - if any( - segments[index].tokens[0] in {"elif", "else"} - for index in control_indexes - ): - return None, "unresolved_shell_conditional" - if_indexes = [ - index - for index in control_indexes - if segments[index].tokens[0] == "if" - ] - fi_indexes = [ - index - for index in control_indexes - if segments[index].tokens[0] == "fi" - ] - then_indexes = [ - index - for index in control_indexes - if segments[index].tokens[0] == "then" - ] - if ( - len(if_indexes) != 1 - or len(fi_indexes) != 1 - or len(then_indexes) != 1 - ): - return None, "unresolved_shell_conditional" - if_index = if_indexes[0] - then_index = then_indexes[0] - fi_index = fi_indexes[0] - if not if_index < then_index < fi_index: - return None, "unresolved_shell_conditional" - - header = segments[if_index] - terminator = segments[fi_index] - if ( - header.input_mode != "none" - or header.output_discarded - or len(header.tokens) < 2 - or terminator.tokens != ("fi",) - or terminator.input_mode != "none" - or terminator.output_discarded - ): - return None, "unresolved_shell_conditional" - guard_tokens = header.tokens[1:] - if Path(guard_tokens[0]).name not in {"test", "["}: - return None, "unresolved_shell_conditional" - - body = list(segments[then_index:fi_index]) - first = body.pop(0) - if len(first.tokens) > 1: - body.insert( - 0, - ShellSegment( - tokens=first.tokens[1:], - input_mode=first.input_mode, - output_discarded=first.output_discarded, - ), - ) - if not body or any( - segment.tokens - and segment.tokens[0] in {"if", "then", "elif", "else", "fi"} - for segment in body - ): - return None, "unresolved_shell_conditional" - - prefix = list(segments[:if_index]) - suffix = list(segments[fi_index + 1 :]) - if any( - segment.tokens - and segment.tokens[0] in {"if", "then", "elif", "else", "fi"} - for segment in (*prefix, *suffix) - ): - return None, "unresolved_shell_conditional" - return [ - *prefix, - ShellSegment( - tokens=guard_tokens, - input_mode="none", - output_discarded=False, - ), - *body, - *suffix, - ], None - - -def markdown_token_path( - token: str, - workdir: Path, - tree_roots: Sequence[Path], -) -> str | None: - cleaned = token.strip(" \t\r\n\"'`,;:()[]{}") - if ( - not cleaned.lower().endswith(".md") - or any(character.isspace() for character in cleaned) - or any(character in cleaned for character in "*?[]{}") - ): - return None - candidate = Path(cleaned).expanduser() - return relative_tree_path( - candidate if candidate.is_absolute() else workdir / candidate, - tree_roots, - ) - - -def markdown_read_component( - tokens: Sequence[str], - workdir: Path, - tree_roots: Sequence[Path], -) -> ReadComponent | None: - if not tokens: - return None - executable = Path(tokens[0]).name - if executable not in PURE_READ_COMMANDS: - return None - arguments = list(tokens[1:]) - if not arguments or "-" in arguments: - return None - - paths: list[str] = [] - if executable == "sed": - expressions: list[str] = [] - index = 0 - while index < len(arguments): - token = arguments[index] - if token in {"-n", "--quiet", "--silent"}: - index += 1 - continue - if token in {"-e", "--expression"}: - if index + 1 >= len(arguments): - return None - expressions.append(arguments[index + 1]) - index += 2 - continue - if token.startswith("--expression="): - expressions.append(token.split("=", 1)[1]) - index += 1 - continue - if token.startswith("-"): - return None - node_path = markdown_token_path(token, workdir, tree_roots) - if node_path is not None: - paths.append(node_path) - elif not expressions: - expressions.append(token) - else: - return None - index += 1 - if not expressions or any( - re.fullmatch(r"(?:\d+|\$)(?:,(?:\d+|\$))?p", expression) is None - for expression in expressions - ): - return None - else: - option_arguments = { - "head": {"-c", "--bytes", "-n", "--lines"}, - "tail": {"-c", "--bytes", "-n", "--lines"}, - "nl": {"-b", "--body-numbering", "-d", "--section-delimiter", "-f", - "--footer-numbering", "-h", "--header-numbering", "-i", - "--line-increment", "-l", "--join-blank-lines", "-n", - "--number-format", "-s", "--number-separator", "-v", - "--starting-line-number", "-w", "--number-width"}, - } - needs_value = option_arguments.get(executable, set()) - no_value_options = { - "cat": { - "-A", "--show-all", "-b", "--number-nonblank", "-e", "-E", - "--show-ends", "-n", "--number", "-s", "--squeeze-blank", - "-t", "-T", "--show-tabs", "-u", "-v", "--show-nonprinting", - }, - "head": {"-q", "--quiet", "--silent", "-v", "--verbose", "-z", "--zero-terminated"}, - "tail": {"-q", "--quiet", "--silent", "-v", "--verbose", "-z", "--zero-terminated"}, - "nl": {"-p", "--no-renumber"}, - }.get(executable, set()) - index = 0 - after_options = False - while index < len(arguments): - token = arguments[index] - if not after_options and token == "--": - after_options = True - index += 1 - continue - if not after_options and token in needs_value: - if index + 1 >= len(arguments): - return None - index += 2 - continue - if not after_options and any( - token.startswith(f"{option}=") - for option in needs_value - if option.startswith("--") - ): - index += 1 - continue - if ( - not after_options - and executable in {"head", "tail"} - and ( - re.fullmatch(r"-[cn]\+?\d+", token) - or re.fullmatch(r"-\d+", token) - ) - ): - index += 1 - continue - if not after_options and token in no_value_options: - index += 1 - continue - if ( - not after_options - and executable == "cat" - and re.fullmatch(r"-[AbeEnstTuv]+", token) - ): - index += 1 - continue - if not after_options and token.startswith("-"): - return None - if executable in {"head", "tail"} and re.fullmatch(r"\+?\d+", token): - index += 1 - continue - node_path = markdown_token_path(token, workdir, tree_roots) - if node_path is None: - return None - paths.append(node_path) - index += 1 - - if not paths: - return None - return ReadComponent(reader=executable, node_paths=tuple(sorted(set(paths)))) - - -def diagnostic_path(token: str, workdir: Path) -> Path | None: - if token in {".", ".."} or token.startswith(("./", "../", "/")): - candidate = Path(token).expanduser() - return (candidate if candidate.is_absolute() else workdir / candidate).resolve( - strict=False - ) - return None - - -def tokens_have_dynamic_expansion(tokens: Sequence[str]) -> bool: - return any("$" in token or "`" in token for token in tokens) - - -def safe_pipeline_filter(tokens: Sequence[str]) -> bool: - """Accept filters that consume only a preceding, already-safe pipe.""" - if not tokens or tokens_have_dynamic_expansion(tokens): - return False - executable = Path(tokens[0]).name - arguments = list(tokens[1:]) - if executable == "sed": - expressions: list[str] = [] - index = 0 - while index < len(arguments): - token = arguments[index] - if token in {"-n", "--quiet", "--silent"}: - index += 1 - continue - if token in {"-e", "--expression"}: - if index + 1 >= len(arguments): - return False - expressions.append(arguments[index + 1]) - index += 2 - continue - if token.startswith("--expression="): - expressions.append(token.split("=", 1)[1]) - index += 1 - continue - if token.startswith("-"): - return False - if expressions: - return False - expressions.append(token) - index += 1 - return bool(expressions) and all( - re.fullmatch(r"(?:\d+|\$)(?:,(?:\d+|\$))?p", expression) - is not None - for expression in expressions - ) - if executable in {"head", "tail"}: - index = 0 - while index < len(arguments): - argument = arguments[index] - if argument in {"-c", "--bytes", "-n", "--lines"}: - if ( - index + 1 >= len(arguments) - or re.fullmatch(r"[+-]?\d+", arguments[index + 1]) - is None - ): - return False - index += 2 - continue - if any( - argument.startswith(f"{option}=") - and re.fullmatch( - r"[+-]?\d+", - argument.split("=", 1)[1], - ) - is not None - for option in {"--bytes", "--lines"} - ): - index += 1 - continue - if ( - re.fullmatch(r"-[cn][+-]?\d+", argument) - or re.fullmatch(r"[+-]\d+", argument) - ): - index += 1 - continue - if argument in { - "-q", - "--quiet", - "--silent", - "-v", - "--verbose", - "-z", - "--zero-terminated", - }: - index += 1 - continue - return False - return True - if executable == "nl": - needs_value = { - "-b", - "--body-numbering", - "-d", - "--section-delimiter", - "-f", - "--footer-numbering", - "-h", - "--header-numbering", - "-i", - "--line-increment", - "-l", - "--join-blank-lines", - "-n", - "--number-format", - "-s", - "--number-separator", - "-v", - "--starting-line-number", - "-w", - "--number-width", - } - index = 0 - while index < len(arguments): - argument = arguments[index] - if argument in {"-p", "--no-renumber"}: - index += 1 - continue - if argument in needs_value: - if index + 1 >= len(arguments): - return False - index += 2 - continue - if any( - argument.startswith(f"{option}=") - and bool(argument.split("=", 1)[1]) - for option in needs_value - if option.startswith("--") - ): - index += 1 - continue - return False - return True - if executable == "wc": - return all( - argument in { - "-c", - "--bytes", - "-l", - "--lines", - "-m", - "--chars", - "-w", - "--words", - "-L", - "--max-line-length", - } - for argument in arguments - ) - return False - - -def safe_test_command( - tokens: Sequence[str], - workdir: Path, - workspace: Path, - tree_root: Path, -) -> bool: - if not tokens: - return False - executable = Path(tokens[0]).name - arguments = list(tokens[1:]) - if executable == "[": - if not arguments or arguments[-1] != "]": - return False - arguments = arguments[:-1] - elif executable != "test": - return False - if tokens_have_dynamic_expansion(arguments): - return False - allowed_operators = { - "!", - "-a", - "-d", - "-e", - "-f", - "-h", - "-L", - "-n", - "-o", - "-r", - "-s", - "-w", - "-x", - "-z", - } - saw_predicate = False - for argument in arguments: - if argument in allowed_operators: - if argument.startswith("-") and argument not in {"-a", "-o"}: - saw_predicate = True - continue - candidate_path = Path(argument).expanduser() - candidate = ( - candidate_path - if candidate_path.is_absolute() - else workdir / candidate_path - ).resolve(strict=False) - if not ( - path_is_within(candidate, tree_root) - ): - return False - return saw_predicate - - -def safe_static_output(tokens: Sequence[str]) -> bool: - return static_output_literal(tokens) is not None - - -def shell_backslash_expansion(value: str) -> str | None: - output: list[str] = [] - index = 0 - replacements = { - "\\": "\\", - "a": "\a", - "b": "\b", - "f": "\f", - "n": "\n", - "r": "\r", - "t": "\t", - "v": "\v", - } - while index < len(value): - if value[index] != "\\": - output.append(value[index]) - index += 1 - continue - if index + 1 >= len(value): - return None - escaped = value[index + 1] - replacement = replacements.get(escaped) - if replacement is None: - return None - output.append(replacement) - index += 2 - return "".join(output) - - -def static_output_literal(tokens: Sequence[str]) -> str | None: - """Render the small, deterministic label subset accepted in composites.""" - if not tokens or tokens_have_dynamic_expansion(tokens[1:]): - return None - executable = Path(tokens[0]).name - arguments = list(tokens[1:]) - if executable == "echo": - newline = True - if arguments and arguments[0] == "-n": - newline = False - arguments.pop(0) - if any(argument.startswith("-") for argument in arguments): - return None - return " ".join(arguments) + ("\n" if newline else "") - if executable != "printf" or not arguments: - return None - - format_value = shell_backslash_expansion(arguments.pop(0)) - if format_value is None: - return None - marker = "\0PERCENT\0" - protected = format_value.replace("%%", marker) - if re.search(r"%(?!s)", protected): - return None - placeholders = protected.count("%s") - if placeholders != len(arguments): - return None - rendered = protected - for argument in arguments: - rendered = rendered.replace("%s", argument, 1) - return rendered.replace(marker, "%") - - -def safe_wc_command( - tokens: Sequence[str], - workdir: Path, - tree_root: Path, - *, - pipe_input: bool, -) -> bool: - if not tokens or Path(tokens[0]).name != "wc": - return False - paths = 0 - for argument in tokens[1:]: - if argument == "-l": - continue - if argument.startswith("-"): - return False - candidate = Path(argument).expanduser() - resolved = ( - candidate if candidate.is_absolute() else workdir / candidate - ).resolve(strict=False) - if not path_is_within(resolved, tree_root): - return False - paths += 1 - return pipe_input or paths > 0 - - -def safe_tree_cli( - tokens: Sequence[str], - workdir: Path, - tree_root: Path, -) -> bool: - if len(tokens) < 3: - return False - executable = tokens[0] - if executable not in {"first-tree", "first-tree-staging"}: - return False - if ( - tuple(tokens[1:3]) != ("tree", "tree") - or tokens_have_dynamic_expansion(tokens[3:]) - ): - return False - - arguments = list(tokens[3:]) - if arguments in (["-h"], ["--help"]): - return True - - path_operand: str | None = None - no_pull = False - index = 0 - after_options = False - while index < len(arguments): - argument = arguments[index] - if not after_options and argument == "--": - after_options = True - index += 1 - continue - if not after_options and argument == "--no-pull": - no_pull = True - index += 1 - continue - if not after_options and argument in {"-L", "--level"}: - if ( - index + 1 >= len(arguments) - or re.fullmatch(r"\d+", arguments[index + 1]) is None - ): - return False - index += 2 - continue - if not after_options and argument.startswith("--level="): - if re.fullmatch(r"\d+", argument.split("=", 1)[1]) is None: - return False - index += 1 - continue - if not after_options and argument in {"-P", "--pattern"}: - if index + 1 >= len(arguments) or not arguments[index + 1]: - return False - index += 2 - continue - if not after_options and argument.startswith("--pattern="): - if not argument.split("=", 1)[1]: - return False - index += 1 - continue - if not after_options and argument.startswith("-"): - return False - if path_operand is not None or any( - character in argument for character in "*?[]{}" - ): - return False - path_operand = argument - index += 1 - - if not no_pull: - return False - if path_operand is None: - return path_is_within(workdir, tree_root) - candidate = Path(path_operand).expanduser() - resolved = ( - candidate if candidate.is_absolute() else workdir / candidate - ).resolve(strict=False) - return path_is_within(resolved, tree_root) - - -def git_command_parts( - tokens: Sequence[str], - workdir: Path, -) -> tuple[Path, str, list[str]] | None: - if not tokens or Path(tokens[0]).name != "git": - return None - arguments = list(tokens[1:]) - git_workdir = workdir - index = 0 - if len(arguments) >= 2 and arguments[0] == "-C": - candidate = Path(arguments[1]).expanduser() - git_workdir = ( - candidate if candidate.is_absolute() else workdir / candidate - ).resolve(strict=False) - index = 2 - if index >= len(arguments) or arguments[index].startswith("-"): - return None - return git_workdir, arguments[index], arguments[index + 1 :] - - -def git_has_unsafe_option(arguments: Sequence[str]) -> bool: - return any( - argument == "-c" - or argument.startswith("-c") - or argument in UNSAFE_GIT_OPTIONS - or any( - argument.startswith(f"{option}=") - for option in UNSAFE_GIT_OPTIONS - ) - for argument in arguments - ) - - -def safe_git_diagnostic( - tokens: Sequence[str], - workdir: Path, - tree_root: Path, -) -> bool: - parts = git_command_parts(tokens, workdir) - if parts is None: - return False - git_workdir, subcommand, arguments = parts - if ( - not path_is_within(git_workdir, tree_root) - or subcommand not in SAFE_GIT_DIAGNOSTICS - or git_has_unsafe_option(arguments) - or tokens_have_dynamic_expansion(arguments) - ): - return False - if subcommand == "remote": - if not arguments: - return True - if all(argument in {"-v", "--verbose"} for argument in arguments): - return True - if arguments[0] == "get-url": - remainder = list(arguments[1:]) - while remainder and remainder[0] in {"--all", "--push"}: - remainder.pop(0) - return len(remainder) == 1 and not remainder[0].startswith("-") - return False - for argument in arguments: - if argument.startswith(("../", "./", "/")): - candidate = Path(argument).expanduser() - resolved = ( - candidate - if candidate.is_absolute() - else git_workdir / candidate - ).resolve(strict=False) - if not path_is_within(resolved, tree_root): - return False - return True - - -def parse_rg_arguments( - tokens: Sequence[str], -) -> tuple[list[str], bool, bool] | None: - if not tokens or tokens[0] != "rg": - return None - arguments = list(tokens[1:]) - if tokens_have_dynamic_expansion(arguments): - return None - explicit_pattern = False - files_mode = False - no_config = False - no_ignore = False - positionals: list[str] = [] - index = 0 - after_options = False - while index < len(arguments): - argument = arguments[index] - if not after_options and argument == "--": - after_options = True - index += 1 - continue - if not after_options and argument in RG_PATTERN_OPTIONS: - if index + 1 >= len(arguments): - return None - explicit_pattern = True - index += 2 - continue - if not after_options and any( - argument.startswith(f"{option}=") - for option in RG_PATTERN_OPTIONS - if option.startswith("--") - ): - explicit_pattern = True - index += 1 - continue - if not after_options and argument in RG_VALUE_OPTIONS: - if index + 1 >= len(arguments): - return None - index += 2 - continue - if not after_options and any( - argument.startswith(f"{option}=") - for option in RG_VALUE_OPTIONS - if option.startswith("--") - ): - index += 1 - continue - if not after_options and argument in RG_FLAG_OPTIONS: - files_mode = files_mode or argument in {"--files", "--type-list"} - no_config = no_config or argument == "--no-config" - no_ignore = no_ignore or argument == "--no-ignore" - index += 1 - continue - if not after_options and argument.startswith("-"): - return None - positionals.append(argument) - index += 1 - - if not no_config or not no_ignore or (files_mode and explicit_pattern): - return None - path_operands = ( - positionals - if explicit_pattern or files_mode - else positionals[1:] - ) - if any(path == "-" for path in path_operands): - return None - return path_operands, explicit_pattern, files_mode - - -def rg_read_component( - tokens: Sequence[str], - workdir: Path, - tree_roots: Sequence[Path], -) -> ReadComponent | None: - parsed = parse_rg_arguments(tokens) - if parsed is None: - return None - path_operands, _explicit_pattern, files_mode = parsed - if files_mode: - return None - paths = { - path - for argument in path_operands - if (path := markdown_token_path(argument, workdir, tree_roots)) - is not None - } - if not paths: - return None - return ReadComponent(reader="rg", node_paths=tuple(sorted(paths))) - - -def diagnostic_markdown_paths( - tokens: Sequence[str], - workdir: Path, - tree_root: Path, -) -> set[str]: - if not tokens: - return set() - executable = Path(tokens[0]).name - if executable not in {"[", "test", "wc"}: - return set() - return { - path - for argument in tokens[1:] - if (path := markdown_token_path(argument, workdir, (tree_root,))) - is not None - } - - -def safe_read_only_diagnostic( - tokens: Sequence[str], - workdir: Path, - workspace: Path, - tree_root: Path, -) -> bool: - if not tokens: - return False - executable = Path(tokens[0]).name - arguments = list(tokens[1:]) - if executable == "pwd": - return all(argument in {"-L", "-P"} for argument in arguments) and ( - path_is_within(workdir, workspace) - or path_is_within(workdir, tree_root) - ) - if executable == "true": - return not arguments - if executable in {"test", "["}: - return safe_test_command( - tokens, - workdir, - workspace, - tree_root, - ) - if executable in {"echo", "printf"}: - return safe_static_output(tokens) - if executable == "wc": - return safe_wc_command( - tokens, - workdir, - tree_root, - pipe_input=False, - ) - if executable in {"first-tree", "first-tree-staging"}: - return safe_tree_cli(tokens, workdir, tree_root) - if executable == "git": - return safe_git_diagnostic(tokens, workdir, tree_root) - if executable == "rg": - if not path_is_within(workdir, tree_root) or any( - argument in UNSAFE_RG_OPTIONS - or any(argument.startswith(f"{option}=") for option in UNSAFE_RG_OPTIONS) - for argument in arguments - ): - return False - parsed = parse_rg_arguments(tokens) - if parsed is None: - return False - path_operands, _explicit_pattern, _files_mode = parsed - for argument in path_operands: - candidate_path = Path(argument).expanduser() - candidate = ( - candidate_path - if candidate_path.is_absolute() - else workdir / candidate_path - ).resolve(strict=False) - if not path_is_within(candidate, tree_root): - return False - return True - return False - - -def command_workdir(payload: Mapping[str, Any], default_workdir: Path) -> Path: - value = parse_tool_arguments(payload).get("workdir") - if not isinstance(value, str) or not value.strip(): - return default_workdir - candidate = Path(value).expanduser() - return (candidate if candidate.is_absolute() else default_workdir / candidate).resolve( - strict=False - ) - - -def rejected_assessment(reason: str) -> ReadAssessment: - status = ( - "rejected_unsafe" - if reason.startswith("unsafe_") - else "unresolved_opaque" - ) - return ReadAssessment(plan=None, status=status, reason=reason) - - -def accepted_assessment( - plan: ReadPlan, - *, - subplans: Sequence[ReadPlan] | None = None, -) -> ReadAssessment: - return ReadAssessment( - plan=plan, - status=( - "accepted_exact" - if plan.mode == "isolated" - else "accepted_read_only_composite" - ), - reason=None, - subplans=tuple(subplans or (plan,)), - ) - - -def unsafe_shell_reason(tokens: Sequence[str]) -> str | None: - if not tokens: - return None - executable = Path(tokens[0]).name - arguments = list(tokens[1:]) - if tokens[0] != executable: - return "unsafe_path_qualified_program" - if executable in KNOWN_UNSAFE_PROGRAMS: - return f"unsafe_program_{executable}" - if executable == "sed" and any( - argument == "-i" - or argument.startswith("-i") - or argument.startswith("--in-place") - for argument in arguments - ): - return "unsafe_sed_in_place" - if executable == "find" and any( - argument in UNSAFE_FIND_ACTIONS for argument in arguments - ): - return "unsafe_find_action" - if executable == "find": - return "unsafe_find_unbound_grammar" - if executable == "ls": - return "unsafe_ls_unbound_grammar" - if executable == "rg" and any( - argument in UNSAFE_RG_OPTIONS - or any(argument.startswith(f"{option}=") for option in UNSAFE_RG_OPTIONS) - for argument in arguments - ): - return "unsafe_rg_option" - if executable == "git": - index = 2 if len(arguments) >= 2 and arguments[0] == "-C" else 0 - if git_has_unsafe_option(arguments): - return "unsafe_git_option" - if ( - index < len(arguments) - and arguments[index] in CONFIG_DRIVEN_GIT_DIAGNOSTICS - ): - return "unsafe_git_configured_helper" - if index < len(arguments) and arguments[index] in MUTATING_GIT_COMMANDS: - return "unsafe_git_mutation" - return "unsafe_git_unbound_configuration" - return None - - -def shell_command_assessment( - tool_name: str, - payload: Mapping[str, Any], - tree_root: Path, - default_workdir: Path, - *, - allow_diagnostic_plan: bool = False, -) -> ReadAssessment: - node_paths = extract_node_paths(payload, (tree_root,), default_workdir) - arguments = parse_tool_arguments(payload) - command = arguments.get("cmd") - if not isinstance(command, str) or not command.strip(): - return ( - rejected_assessment("unresolved_missing_literal_command") - if node_paths or allow_diagnostic_plan - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - segments, reason = shell_segments(command) - if segments is None: - return ( - rejected_assessment(reason or "unresolved_shell_syntax") - if node_paths or allow_diagnostic_plan - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - segments, reason = expand_static_for_loop(segments) - if segments is None: - return ( - rejected_assessment(reason or "unresolved_shell_loop") - if node_paths or allow_diagnostic_plan - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - segments, reason = expand_literal_if_guard(segments) - if segments is None: - return ( - rejected_assessment(reason or "unresolved_shell_conditional") - if node_paths or allow_diagnostic_plan - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - - workdir = command_workdir(payload, default_workdir) - current_workdir = workdir - components: list[ReadComponent] = [] - diagnostic_paths: set[str] = set() - diagnostic_count = 0 - auxiliary_literals: list[str] = [] - output_requires_separation = False - deferred_unsafe_reason: str | None = None - for shell_segment in segments: - segment = list(shell_segment.tokens) - if not segment: - return rejected_assessment("unresolved_shell_syntax") - executable = Path(segment[0]).name - unsafe_reason = unsafe_shell_reason(segment) - if unsafe_reason is not None: - if node_paths or components or allow_diagnostic_plan: - return rejected_assessment(unsafe_reason) - deferred_unsafe_reason = unsafe_reason - continue - if executable in {"first-tree", "first-tree-staging"} and not safe_tree_cli( - segment, - current_workdir, - tree_root, - ): - if node_paths or components or allow_diagnostic_plan: - return rejected_assessment("unsafe_first_tree_command") - return ReadAssessment(None, None, "not_a_tree_markdown_read") - literal_non_tree_path = False - for argument in segment[1:]: - if not argument.startswith(("/", "./", "../")): - continue - candidate_path = Path(argument).expanduser() - candidate = ( - candidate_path - if candidate_path.is_absolute() - else current_workdir / candidate_path - ).resolve(strict=False) - if not ( - argument.lower().endswith(".md") - or candidate.exists() - ): - # A leading slash can be a search/sed expression. If it is not - # a Markdown operand or an existing path, later program-specific - # validation decides whether the shape is merely opaque. - continue - if not path_is_within(candidate, tree_root): - literal_non_tree_path = True - break - if literal_non_tree_path: - if node_paths or components or allow_diagnostic_plan: - return rejected_assessment("unsafe_literal_non_tree_path") - deferred_unsafe_reason = "unsafe_literal_non_tree_path" - continue - if executable in PURE_READ_COMMANDS and "-" in segment[1:]: - if shell_segment.input_mode == "pipe" and safe_pipeline_filter(segment): - diagnostic_count += 1 - continue - return rejected_assessment("unresolved_stdin_tree_read") - if executable == "cd": - if shell_segment.input_mode == "pipe" or len(segment) != 2: - return rejected_assessment("unresolved_shell_cd") - candidate = Path(segment[1]).expanduser() - current_workdir = ( - candidate if candidate.is_absolute() else current_workdir / candidate - ).resolve(strict=False) - if not path_is_within(current_workdir, tree_root): - return rejected_assessment("unsafe_cross_tree_workdir") - diagnostic_count += 1 - continue - - component = markdown_read_component( - segment, - current_workdir, - (tree_root,), - ) - if component is not None: - if deferred_unsafe_reason is not None: - return rejected_assessment(deferred_unsafe_reason) - if shell_segment.output_discarded: - diagnostic_count += 1 - diagnostic_paths.update(component.node_paths) - else: - components.append(component) - continue - - if executable == "rg": - if not safe_read_only_diagnostic( - segment, - current_workdir, - default_workdir, - tree_root, - ): - return rejected_assessment("unsafe_or_unresolved_rg") - component = rg_read_component( - segment, - current_workdir, - (tree_root,), - ) - if component is not None: - if shell_segment.output_discarded: - diagnostic_count += 1 - diagnostic_paths.update(component.node_paths) - else: - components.append(component) - else: - diagnostic_count += 1 - if not shell_segment.output_discarded: - output_requires_separation = True - continue - - if shell_segment.input_mode == "pipe" and ( - safe_pipeline_filter(segment) - or safe_wc_command( - segment, - current_workdir, - tree_root, - pipe_input=True, - ) - ): - diagnostic_count += 1 - diagnostic_paths.update( - diagnostic_markdown_paths( - segment, - current_workdir, - tree_root, - ) - ) - if executable == "wc" and not shell_segment.output_discarded: - output_requires_separation = True - continue - - if safe_read_only_diagnostic( - segment, - current_workdir, - default_workdir, - tree_root, - ): - diagnostic_count += 1 - diagnostic_paths.update( - diagnostic_markdown_paths( - segment, - current_workdir, - tree_root, - ) - ) - if not shell_segment.output_discarded: - if executable in {"echo", "printf"}: - literal = static_output_literal(segment) - if literal is None: - return rejected_assessment("unresolved_static_output") - auxiliary_literals.append(literal) - elif executable not in {"test", "[", "true"}: - output_requires_separation = True - continue - if executable in {"for", "do", "done"}: - return rejected_assessment("unresolved_shell_loop") - if not node_paths and not components and not allow_diagnostic_plan: - return ReadAssessment(None, None, "not_a_tree_markdown_read") - return rejected_assessment("unresolved_unknown_program") - - if not components: - if deferred_unsafe_reason is not None and allow_diagnostic_plan: - return rejected_assessment(deferred_unsafe_reason) - if allow_diagnostic_plan and diagnostic_count > 0: - return accepted_assessment( - ReadPlan( - node_paths=(), - components=(), - command=( - f"{tool_name.rsplit('.', 1)[-1]} " - "read_only_diagnostic" - ), - mode="read_only_composite", - auxiliary_output_possible=True, - auxiliary_literals=tuple(auxiliary_literals), - output_requires_separation=output_requires_separation, - ) - ) - return ( - rejected_assessment("unresolved_tree_path_without_content_reader") - if node_paths - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - recovered_paths = tuple( - sorted({path for component in components for path in component.node_paths}) - ) - if ( - node_paths - and set(recovered_paths) | diagnostic_paths != set(node_paths) - ): - return rejected_assessment("unresolved_node_path_attribution") - mode = ( - "isolated" - if len(segments) == 1 - and len(components) == 1 - and len(recovered_paths) == 1 - and diagnostic_count == 0 - else "read_only_composite" - ) - descriptor = ( - f"{tool_name.rsplit('.', 1)[-1]} {mode} " - + " ".join(recovered_paths) - ) - return accepted_assessment( - ReadPlan( - node_paths=recovered_paths, - components=tuple(components), - command=descriptor, - mode=mode, - auxiliary_output_possible=diagnostic_count > 0, - auxiliary_literals=tuple(auxiliary_literals), - output_requires_separation=output_requires_separation, - ) - ) - - -def raw_orchestration_source(payload: Mapping[str, Any]) -> str | None: - for key in ("input", "arguments"): - value = payload.get(key) - if isinstance(value, str): - try: - parsed = json.loads(value) - except json.JSONDecodeError: - return value - if isinstance(parsed, str): - return parsed - if isinstance(parsed, Mapping): - for field in ("code", "js", "source"): - candidate = parsed.get(field) - if isinstance(candidate, str): - return candidate - return None - - -def matching_js_delimiter( - source: str, - start: int, - opening: str, - closing: str, -) -> tuple[str, int] | None: - if start >= len(source) or source[start] != opening: - return None - depth = 0 - quote: str | None = None - escaped = False - for index in range(start, len(source)): - character = source[index] - if quote is not None: - if escaped: - escaped = False - elif character == "\\": - escaped = True - elif character == quote: - quote = None - continue - if character in {"'", '"', "`"}: - quote = character - continue - if character == opening: - depth += 1 - elif character == closing: - depth -= 1 - if depth == 0: - return source[start + 1 : index], index + 1 - return None - - -def parse_js_string_literal(source: str, start: int) -> tuple[str, int] | None: - if start >= len(source) or source[start] not in {"'", '"', "`"}: - return None - quote = source[start] - escaped = False - for index in range(start + 1, len(source)): - character = source[index] - if escaped: - escaped = False - continue - if character == "\\": - escaped = True - continue - if character != quote: - continue - literal = source[start : index + 1] - if quote == "`": - body = literal[1:-1] - if "${" in body: - return None - return body, index + 1 - try: - value = ast.literal_eval(literal) - except (SyntaxError, ValueError): - return None - return (value, index + 1) if isinstance(value, str) else None - return None - - -def split_js_top_level(source: str, delimiter: str) -> list[str] | None: - parts: list[str] = [] - start = 0 - depths = {"(": 0, "[": 0, "{": 0} - closing = {")": "(", "]": "[", "}": "{"} - quote: str | None = None - escaped = False - for index, character in enumerate(source): - if quote is not None: - if escaped: - escaped = False - elif character == "\\": - escaped = True - elif character == quote: - quote = None - continue - if character in {"'", '"', "`"}: - quote = character - continue - if character in depths: - depths[character] += 1 - continue - if character in closing: - opening = closing[character] - if depths[opening] == 0: - return None - depths[opening] -= 1 - continue - if character == delimiter and all(depth == 0 for depth in depths.values()): - parts.append(source[start:index]) - start = index + 1 - if quote is not None or any(depth != 0 for depth in depths.values()): - return None - parts.append(source[start:]) - return parts - - -def js_top_level_colon(source: str) -> int | None: - parts = split_js_top_level(source, ":") - if parts is None or len(parts) != 2: - return None - return len(parts[0]) - - -def parse_js_object_properties( - source: str, -) -> tuple[dict[str, str] | None, str | None]: - entries = split_js_top_level(source, ",") - if entries is None: - return None, "unresolved_exec_dynamic_arguments" - properties: dict[str, str] = {} - for index, raw_entry in enumerate(entries): - entry = raw_entry.strip() - if not entry: - if index == len(entries) - 1: - continue - return None, "unresolved_exec_dynamic_arguments" - if entry.startswith("..."): - return None, "unresolved_exec_dynamic_arguments" - colon = js_top_level_colon(entry) - if colon is None: - return None, "unresolved_exec_dynamic_arguments" - raw_key = entry[:colon].strip() - raw_value = entry[colon + 1 :].strip() - if re.fullmatch(r"[A-Za-z_$][A-Za-z0-9_$]*", raw_key): - key = raw_key - else: - parsed_key = parse_js_string_literal(raw_key, 0) - if parsed_key is None or parsed_key[1] != len(raw_key): - return None, "unresolved_exec_dynamic_arguments" - key = parsed_key[0] - if key in properties or key in {"__proto__", "constructor", "prototype"}: - return None, "unresolved_exec_dynamic_arguments" - properties[key] = raw_value - return properties, None - - -def exact_js_string(source: str) -> str | None: - parsed = parse_js_string_literal(source, 0) - if parsed is None or source[parsed[1] :].strip(): - return None - return parsed[0] - - -def exact_safe_js_value(source: str) -> bool: - value = source.strip() - if exact_js_string(value) is not None: - return True - if value in {"true", "false", "null"}: - return True - if re.fullmatch(r"(?:0|[1-9][0-9]*)", value): - return True - if value.startswith("[") and value.endswith("]"): - entries = split_js_top_level(value[1:-1], ",") - if entries is None: - return False - return all( - not entry.strip() or exact_js_string(entry.strip()) is not None - for entry in entries - ) - return False - - -def orchestration_source_skeleton( - source: str, -) -> tuple[str | None, list[str] | None]: - """Replace literal exec calls while retaining every surrounding JS byte.""" - call_bodies: list[str] = [] - output: list[str] = [] - cursor = 0 - pattern = re.compile(r"\btools\.exec_command\s*\(") - while (match := pattern.search(source, cursor)) is not None: - opening_index = match.end() - 1 - balanced = matching_js_delimiter(source, opening_index, "(", ")") - if balanced is None: - return None, None - call_body, end = balanced - output.append(source[cursor : match.start()]) - output.append("__EXEC_CALL__") - call_bodies.append(call_body) - cursor = end - output.append(source[cursor:]) - if not call_bodies: - return None, None - return "".join(output), call_bodies - - -def orchestration_wrapper_shape( - skeleton: str, - nested_count: int, -) -> str | None: - identifier = r"[A-Za-z_][A-Za-z0-9_]*" - call = r"__EXEC_CALL__" - - single_forward = re.fullmatch( - rf"\s*(?:const|let|var)\s+(?P{identifier})\s*=\s*" - rf"await\s+{call}\s*;\s*" - rf"text\s*\(\s*(?P=result)\.output\s*\)\s*;?\s*", - skeleton, - ) - if single_forward is not None and nested_count == 1: - return "sequential" - - sequential = re.fullmatch( - rf"\s*(?P(?:(?:const|let|var)\s+{identifier}\s*=\s*" - rf"await\s+{call}\s*;\s*)+)" - rf"(?P(?:text\s*\(\s*{identifier}\.output\s*\)\s*;\s*)+)", - skeleton, - ) - if sequential is not None: - assignments = re.findall( - rf"(?:const|let|var)\s+({identifier})\s*=\s*" - rf"await\s+{call}", - sequential.group("decls"), - ) - forwarded = re.findall( - rf"text\s*\(\s*({identifier})\.output\s*\)", - sequential.group("forwards"), - ) - if ( - len(assignments) == nested_count - and len(set(assignments)) == len(assignments) - and assignments == forwarded - ): - return "sequential" - - promise_prefix = ( - rf"\s*(?:const|let|var)\s+(?P{identifier})\s*=\s*" - rf"await\s+Promise\.all\s*\(\s*\[\s*" - rf"(?P{call}(?:\s*,\s*{call})*\s*,?)" - rf"\s*\]\s*\)\s*;\s*" - ) - callback = re.fullmatch( - promise_prefix - + rf"(?P=results)\.(?:forEach|map)\s*\(\s*" - rf"(?:\(\s*)?(?P{identifier})(?:\s*\))?\s*=>\s*" - rf"text\s*\(\s*(?P=item)\.output\s*\)\s*\)\s*;?\s*", - skeleton, - ) - if callback is not None and callback.group("calls").count(call) == nested_count: - return "promise" - - for_of = re.fullmatch( - promise_prefix - + rf"for\s*\(\s*(?:const|let|var)\s+(?P{identifier})\s+" - rf"of\s+(?P=results)\s*\)\s*\{{\s*" - rf"text\s*\(\s*(?P=item)\.output\s*\)\s*;\s*\}}\s*", - skeleton, - ) - if for_of is not None and for_of.group("calls").count(call) == nested_count: - return "promise" - return None - - -def orchestration_command_payloads( - payload: Mapping[str, Any], - default_workdir: Path, -) -> tuple[list[dict[str, Any]] | None, str | None, str | None]: - source = raw_orchestration_source(payload) - if source is None: - return None, "unresolved_exec_payload", None - skeleton, call_bodies = orchestration_source_skeleton(source) - if skeleton is None or call_bodies is None: - return None, "unresolved_exec_without_nested_tool", None - shape = orchestration_wrapper_shape(skeleton, len(call_bodies)) - if shape is None: - return None, "unresolved_exec_wrapper_shape", None - - payloads: list[dict[str, Any]] = [] - allowed_properties = { - "cmd", - "justification", - "login", - "max_output_tokens", - "prefix_rule", - "sandbox_permissions", - "shell", - "tty", - "workdir", - "yield_time_ms", - } - for call_body in call_bodies: - object_start = call_body.find("{") - if object_start < 0: - return None, "unresolved_exec_dynamic_arguments", None - object_value = matching_js_delimiter(call_body, object_start, "{", "}") - if object_value is None: - return None, "unresolved_exec_dynamic_arguments", None - object_body, object_end = object_value - if call_body[object_end:].strip().rstrip(","): - return None, "unresolved_exec_dynamic_arguments", None - properties, property_reason = parse_js_object_properties(object_body) - if properties is None: - return None, property_reason, None - if ( - not set(properties).issubset(allowed_properties) - or any( - not exact_safe_js_value(value) - for value in properties.values() - ) - ): - return None, "unresolved_exec_dynamic_arguments", None - command = exact_js_string(properties.get("cmd", "")) - if command is None: - return None, "unresolved_exec_dynamic_command", None - raw_workdir = properties.get("workdir") - workdir = ( - exact_js_string(raw_workdir) - if raw_workdir is not None - else None - ) - if raw_workdir is not None and workdir is None: - return None, "unresolved_exec_dynamic_workdir", None - payloads.append( - { - "arguments": { - "cmd": command, - "workdir": workdir or str(default_workdir), - } - } - ) - return payloads, None, shape - - -def orchestration_read_assessment( - tool_name: str, - payload: Mapping[str, Any], - tree_root: Path, - default_workdir: Path, -) -> ReadAssessment: - node_paths = extract_node_paths(payload, (tree_root,), default_workdir) - nested_payloads, reason, wrapper_shape = orchestration_command_payloads( - payload, - default_workdir, - ) - if nested_payloads is None: - return ( - rejected_assessment(reason or "unresolved_exec_payload") - if node_paths - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - components: list[ReadComponent] = [] - recovered_paths: set[str] = set() - nested_plans: list[ReadPlan] = [] - for nested_payload in nested_payloads: - assessment = shell_command_assessment( - "exec_command", - nested_payload, - tree_root, - default_workdir, - allow_diagnostic_plan=True, - ) - if assessment.status in {"rejected_unsafe", "unresolved_opaque"}: - return assessment - if assessment.plan is None: - return rejected_assessment( - "unresolved_exec_nested_output_attribution" - ) - nested_plans.append(assessment.plan) - components.extend(assessment.plan.components) - recovered_paths.update(assessment.plan.node_paths) - if not components: - return ( - rejected_assessment("unresolved_exec_without_content_reader") - if node_paths - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - ordered_paths = tuple(sorted(recovered_paths)) - mode = ( - "isolated" - if len(nested_plans) == 1 - and nested_plans[0].mode == "isolated" - and wrapper_shape == "sequential" - else "read_only_composite" - ) - return accepted_assessment( - ReadPlan( - node_paths=ordered_paths, - components=tuple(components), - command=f"{tool_name.rsplit('.', 1)[-1]} {mode} " - + " ".join(ordered_paths), - mode=mode, - auxiliary_output_possible=any( - plan.auxiliary_output_possible for plan in nested_plans - ), - auxiliary_literals=tuple( - literal - for plan in nested_plans - for literal in plan.auxiliary_literals - ), - output_requires_separation=any( - plan.output_requires_separation for plan in nested_plans - ), - ), - subplans=nested_plans, - ) - - -def markdown_read_plan( - tool_name: str, - payload: Mapping[str, Any], - tree_root: Path, - default_workdir: Path, -) -> ReadAssessment: - """Recognize isolated and provably read-only composite Markdown reads.""" - node_paths = extract_node_paths(payload, (tree_root,), default_workdir) - if node_paths and has_literal_non_tree_markdown(payload, tree_root): - return rejected_assessment("unsafe_literal_non_tree_path") - if tool_name in MUTATING_TOOLS: - return ( - rejected_assessment("unsafe_mutating_tool") - if node_paths - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - if tool_name in EXEC_ORCHESTRATION_TOOLS: - return orchestration_read_assessment( - tool_name, - payload, - tree_root, - default_workdir, - ) - if tool_name in DIRECT_READ_TOOLS: - if not node_paths: - return ReadAssessment(None, None, "not_a_tree_markdown_read") - component = ReadComponent( - reader=tool_name.rsplit(".", 1)[-1], - node_paths=tuple(node_paths), - ) - return accepted_assessment( - ReadPlan( - node_paths=tuple(node_paths), - components=(component,), - command=f"{component.reader} {' '.join(node_paths)}", - mode="isolated" if len(node_paths) == 1 else "read_only_composite", - ) - ) - if tool_name not in EXEC_COMMAND_TOOLS: - return ( - rejected_assessment("unresolved_tree_read_tool") - if node_paths - else ReadAssessment(None, None, "not_a_tree_markdown_read") - ) - return shell_command_assessment( - tool_name, - payload, - tree_root, - default_workdir, - ) - - -def content_class_hint(paths: Sequence[str]) -> str: - classes = { - "non-normal" if path == "AGENTS.md" or path.startswith(("members/", "raw-context/")) else "normal" - for path in paths - } - if len(classes) == 1: - return next(iter(classes)) - return "mixed" - - -def output_success(output: str) -> bool | None: - lowered = output.lower() - if re.search(r"(?:process exited with code|exit[_ ]code[\"']?\s*[:=])\s*[1-9][0-9]*", lowered): - return False - if re.search(r"(?:process exited with code|exit[_ ]code[\"']?\s*[:=])\s*0", lowered): - return True - try: - envelope = json.loads(output) - except (json.JSONDecodeError, TypeError): - envelope = None - if isinstance(envelope, Mapping): - status = envelope.get("status") - if ( - envelope.get("is_error") is True - or envelope.get("error") not in (None, False, "") - or (envelope.get("ok") is False and envelope.get("error")) - or ( - isinstance(status, str) - and status.lower() in {"error", "failed", "failure"} - ) - ): - return False - if re.search( - r"\A\s*(?:error|failed|permission denied|access denied|" - r"no such file(?: or directory)?|file not found)\b", - lowered, - ): - return False - return None - - -def content_output(output: str, tool_name: str) -> str: - if tool_name in EXEC_ORCHESTRATION_TOOLS: - return re.sub( - r"\A\s*Script completed successfully\s*(?:\r?\n)+" - r"Output:\s*(?:\r?\n)?", - "", - output, - count=1, - flags=re.IGNORECASE, - ) - if tool_name in EXEC_COMMAND_TOOLS: - return re.sub( - r"\A\s*Process exited with code 0\s*(?:\r?\n)+", - "", - output, - count=1, - flags=re.IGNORECASE, - ) - return output - - -def attributable_passage( - output: str, - plan: ReadPlan, -) -> tuple[str | None, str | None]: - if plan.output_requires_separation: - return None, "tree_read_auxiliary_output_unresolved" - passage = output - for literal in plan.auxiliary_literals: - if not literal: - continue - if passage.count(literal) != 1: - return None, "tree_read_static_output_attribution_unresolved" - passage = passage.replace(literal, "", 1) - if not passage.strip(): - return None, "tree_read_output_missing" - return passage.strip(), None - - -def is_root_managed_session(meta: Mapping[str, Any], workspace_roots: set[str]) -> bool: - cwd = meta.get("cwd") - if not isinstance(cwd, str) or str(Path(cwd).resolve()) not in workspace_roots: - return False - if meta.get("originator") != "first-tree" or meta.get("model_provider") != "openai": - return False - if meta.get("agent_path") or meta.get("parent_thread_id") or meta.get("forked_from_id"): - return False - if meta.get("thread_source") == "subagent" or isinstance(meta.get("source"), dict): - return False - return True - - -def canonical_runtime_provider(value: str) -> str: - if value not in RUNTIME_PROVIDER_VALUES: - raise AuditError( - f"Unsupported runtime provider {value!r}; expected one of " - + ", ".join(sorted(RUNTIME_PROVIDER_VALUES)) - + "." - ) - return "claude-code" if value == "claude-code-tui" else value - - -def resolve_runtime_provider(explicit: str | None) -> str: - runtime_value = os.environ.get("FIRST_TREE_PROVIDER") - if not runtime_value: - raise AuditError( - "FIRST_TREE_PROVIDER is required to select the invoking Agent's evidence adapter." - ) - runtime_canonical = canonical_runtime_provider(runtime_value) - if explicit: - canonical = canonical_runtime_provider(explicit) - if canonical != runtime_canonical: - raise AuditError( - "--runtime-provider must match FIRST_TREE_PROVIDER for the current Agent runtime." - ) - return runtime_canonical - - -def candidate_runtime_provider(value: Mapping[str, Any]) -> str: - """Read the v1 provider extension; 0.2.x artifacts were Codex-only.""" - raw = value.get("runtime_provider", "codex") - if not isinstance(raw, str): - raise AuditError("candidate.runtime_provider must be a string.") - canonical = canonical_runtime_provider(raw) - if canonical != raw: - raise AuditError("candidate.runtime_provider must use the canonical provider value.") - return canonical - - -def trace_root_default( - runtime_provider: str, - workspace_identity: WorkspaceIdentity, -) -> Path: - if runtime_provider == "codex": - codex_root = os.environ.get("CODEX_HOME") - return ( - Path(codex_root).expanduser() / "sessions" - if codex_root - else Path.home() / ".codex" / "sessions" - ) - if runtime_provider == "claude-code": - claude_root = os.environ.get("CLAUDE_CONFIG_DIR") - return ( - Path(claude_root).expanduser() / "projects" - if claude_root - else Path.home() / ".claude" / "projects" - ) - raise AuditError(f"No local evidence adapter exists for {runtime_provider}.") - - -def normalize_chat(value: Mapping[str, Any], window: Window) -> dict[str, Any]: - chat_id = value.get("chat_id") or value.get("source_id") - if not isinstance(chat_id, str) or re.fullmatch(UUID_PATTERN, chat_id) is None: - raise AuditError("Every Chat record must contain chat_id/source_id as a UUID.") - raw_messages = value.get("messages") or value.get("visible_messages") or [] - if not isinstance(raw_messages, list): - raise AuditError(f"Chat {chat_id} messages must be an array.") - messages, invalid_receipt = normalize_message_records( - (message for message in raw_messages if isinstance(message, dict)), - window=window, - ) - messages = [ - message - for message in messages - if isinstance(message.get("created_at"), str) and in_window(message["created_at"], window) - ] - messages.sort(key=lambda message: (message["created_at"], str(message.get("message_id") or ""))) - gaps = value.get("coverage_gaps") - coverage_gaps = [str(gap) for gap in gaps] if isinstance(gaps, list) else [] - if invalid_receipt: - coverage_gaps.append("context_decision_invalid") - source_agent_id = require_uuid(value.get("source_agent_id"), f"Chat {chat_id} source_agent_id") - expected_audit_id = audit_id(chat_id, source_agent_id) - supplied_audit_id = value.get("audit_id") - if supplied_audit_id is not None and supplied_audit_id != expected_audit_id: - raise AuditError(f"Chat {chat_id} audit_id does not match its Chat and audited-Agent UUIDs.") - authorization = validate_authorization( - value.get("authorization"), - f"Chat {chat_id} authorization", - ) - if "authorization_context" in value: - raise AuditError( - f"Chat {chat_id} must not contain authorization_context." - ) - return { - "audit_id": expected_audit_id, - "chat_id": chat_id, - "title": str(value.get("title") or value.get("topic") or chat_id), - "authorization": authorization, - "source_agent": require_string( - value.get("source_agent"), - f"Chat {chat_id} source_agent", - ), - "source_agent_id": source_agent_id, - "messages": messages, - "coverage_gaps": coverage_gaps, - } - - -def recent_trace_files( - trace_root: Path, - window: Window, - runtime_provider: str, -) -> list[Path]: - """Return traces that could contain an in-window call without reading their contents.""" - minimum_mtime = window.start.timestamp() if window.start is not None else None - files: list[Path] = [] - candidates = trace_root.rglob("*.jsonl") - for path in candidates: - try: - if path.is_symlink() or not path.is_file(): - continue - path_stat = path.stat() - if runtime_provider == "claude-code" and "subagents" in path.parts: - continue - resolved = path.resolve(strict=True) - resolved.relative_to(trace_root) - if minimum_mtime is None or path_stat.st_mtime >= minimum_mtime: - files.append(resolved) - except (OSError, ValueError): - continue - return sorted(files) - - -def opaque_identity(prefix: str, value: str) -> str: - digest = hashlib.sha256(value.encode("utf-8")).hexdigest()[:20] - return f"{prefix}-{digest}" - - -def trace_identity(path: Path, trace_root: Path) -> str: - try: - relative = path.relative_to(trace_root).as_posix() - except ValueError: - relative = path.name - return opaque_identity("trace", relative) - - -def tree_identity(identity: WorkspaceIdentity) -> str: - value = f"{identity.agent_id}\0{identity.bound_tree_root}" - return opaque_identity("tree", value) - - -def preflight_codex_trace( - path: Path, - trace_root: Path, - workspace_identity: WorkspaceIdentity, - authorized_audits: Mapping[tuple[str, str], str], -) -> tuple[TracePreflight | None, dict[str, set[str]]]: - """Map a trace to one authorized Chat without scanning its full contents.""" - all_audit_ids = set(authorized_audits.values()) - gaps: dict[str, set[str]] = {item: set() for item in all_audit_ids} - metadata: list[Mapping[str, Any]] = [] - context_chat_ids: set[str] = set() - relevant_malformed = False - malformed_metadata_might_match = False - last_canonical_ids: set[str] | None = None - last_canonical_line: int | None = None - bytes_seen = 0 - lines_seen = 0 - workspace_bytes = str(workspace_identity.workspace).encode("utf-8") - - try: - with path.open("rb") as handle: - while ( - bytes_seen < TRACE_PREFLIGHT_MAX_BYTES - and lines_seen < TRACE_PREFLIGHT_MAX_LINES - ): - remaining = TRACE_PREFLIGHT_MAX_BYTES - bytes_seen - raw_line = handle.readline(remaining + 1) - if not raw_line: - break - if len(raw_line) > remaining: - break - bytes_seen += len(raw_line) - lines_seen += 1 - if b'"session_meta"' not in raw_line and ( - b" tuple[str, bool]: - if len(text) <= limit: - return text, False - return text[:limit], True - - -def redact_local_roots( - text: str, - workspace: Path, - tree_root: Path, -) -> str: - result = text - replacements: list[tuple[str, str]] = [] - for path, replacement in ( - (tree_root, ""), - (workspace, ""), - ): - raw = path.as_posix() - variants = {raw} - # macOS resolves `/var` and `/tmp` through `/private`, while recorded - # shell output can preserve either lexical form. - if raw.startswith("/private/"): - variants.add(raw[len("/private") :]) - replacements.extend((variant, replacement) for variant in variants) - for raw, replacement in sorted( - replacements, - key=lambda item: len(item[0]), - reverse=True, - ): - result = result.replace(raw, replacement) - result = result.replace(raw.replace("/", "\\/"), replacement) - return result - - -def record_read_attempt( - attempt_counts: Counter[str], - gaps: set[str], - status: str, - reason: str | None = None, -) -> None: - attempt_counts[status] += 1 - if status in {"unresolved_opaque", "rejected_unsafe"}: - current_reason = reason or status - attempt_counts[f"reason:{current_reason}"] += 1 - gaps.add(current_reason) - - -def codex_trace_reads( - preflight: TracePreflight, - tree_root: Path, - tree_id: str, - window: Window, - max_passage_chars: int, -) -> tuple[list[dict[str, Any]], set[str], set[str], Counter[str]]: - """Read one trace only after its exact Chat-Agent preflight was accepted.""" - reads: list[dict[str, Any]] = [] - sessions = {preflight.trace_id} - gaps: set[str] = set() - attempt_counts: Counter[str] = Counter() - expected_chat_id = preflight.audit_id.split("@", 1)[0] - default_workdir = preflight.workspace - - call_id_counts: Counter[str] = Counter() - calls: dict[str, list[dict[str, Any]]] = {} - outputs: dict[ - str, - list[tuple[str, str | None, bool, tuple[str, ...]]], - ] = {} - try: - with preflight.path.open("r", encoding="utf-8") as handle: - for line in handle: - if not line.strip(): - continue - row = json.loads(line) - if not isinstance(row, dict) or row.get("type") != "response_item": - continue - payload = row.get("payload") - if not isinstance(payload, dict): - continue - payload_type = payload.get("type") - if payload_type == "message" and payload.get("role") == "user": - user_text = payload_text(payload.get("content")) - if CHAT_CONTEXT_PATTERN.search(user_text) is None: - continue - all_ids = set(chat_ids_from_text(user_text)) - if all_ids != {expected_chat_id}: - gaps.add("trace_chat_boundary_changed_after_preflight") - # Do not read any rows beyond an unexpected Chat - # boundary. Evidence already seen remains associated - # with the preflight-authorized Chat only. - break - continue - if payload_type in {"custom_tool_call", "function_call"}: - call_id = payload.get("call_id") - if not isinstance(call_id, str): - continue - call_id_counts[call_id] += 1 - tool_name = str(payload.get("name") or "") - is_continuation = ( - tool_name in SHELL_CONTINUATION_TOOLS | CELL_CONTINUATION_TOOLS - ) - read_plan: ReadPlan | None = None - assessment: ReadAssessment | None = None - if not is_continuation: - assessment = markdown_read_plan( - tool_name, - payload, - tree_root, - default_workdir, - ) - if assessment.status is None: - continue - read_plan = assessment.plan - calls.setdefault(call_id, []).append( - { - "timestamp": row.get("timestamp"), - "payload": payload, - "arguments": parse_tool_arguments(payload), - "read_plan": read_plan, - "assessment": assessment, - } - ) - continue - if payload_type in {"custom_tool_call_output", "function_call_output"}: - call_id = payload.get("call_id") - if not isinstance(call_id, str): - continue - raw_output_value = payload.get("output") - raw_parts = ( - [ - payload_text(item) - for item in raw_output_value - if payload_text(item) - ] - if isinstance(raw_output_value, list) - else [payload_text(raw_output_value)] - ) - output_parts: list[str] = [] - output_truncated = False - for raw_part in raw_parts: - part, part_truncated = clipped( - redact_local_roots( - raw_part, - preflight.workspace, - tree_root, - ), - max_passage_chars, - ) - output_parts.append(part) - output_truncated = output_truncated or part_truncated - output, aggregate_truncated = clipped( - "\n".join(output_parts), - max_passage_chars, - ) - output_truncated = output_truncated or aggregate_truncated - outputs.setdefault(call_id, []).append( - ( - output, - row.get("timestamp"), - output_truncated, - tuple(output_parts), - ) - ) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - gaps.add("codex_trace_malformed_or_partially_cleaned") - return reads, sessions, gaps, attempt_counts - - unique_calls = { - call_id: items[0] - for call_id, items in calls.items() - if len(items) == 1 and call_id_counts[call_id] == 1 - } - unique_outputs = { - call_id: items[0] - for call_id, items in outputs.items() - if len(items) == 1 - } - shell_sessions: dict[str, str] = {} - cell_sessions: dict[str, str] = {} - continuation_outputs: dict[ - str, - list[ - tuple[ - str, - str | None, - str | None, - bool, - tuple[str, ...], - ] - ], - ] = {} - continuation_failures: dict[str, str] = {} - - # Resolve parent session handles first, independently of continuation row - # order. A continuation can contribute evidence only when both its call and - # result are unique. - for call_id, call in unique_calls.items(): - payload = call["payload"] - tool_name = str(payload.get("name") or "") - output, output_completed_at, output_truncated, output_parts = unique_outputs.get( - call_id, - ("", None, False, ()), - ) - if tool_name in EXEC_COMMAND_TOOLS: - match = SHELL_SESSION_PATTERN.search(output) - if match: - shell_sessions[match.group(1)] = call_id - elif tool_name in EXEC_ORCHESTRATION_TOOLS: - match = CELL_SESSION_PATTERN.search(output) - if match: - cell_sessions[match.group(1)] = call_id - - for continuation_id, call_items in calls.items(): - continuation_tools = { - str(call["payload"].get("name") or "") - for call in call_items - } - if not continuation_tools.intersection( - SHELL_CONTINUATION_TOOLS | CELL_CONTINUATION_TOOLS - ): - continue - parent_ids: set[str] = set() - for call in call_items: - tool_name = str(call["payload"].get("name") or "") - if tool_name in SHELL_CONTINUATION_TOOLS: - parent = shell_sessions.get( - str(call["arguments"].get("session_id")) - ) - elif tool_name in CELL_CONTINUATION_TOOLS: - parent = cell_sessions.get( - str(call["arguments"].get("cell_id")) - ) - else: - parent = None - if parent is not None: - parent_ids.add(parent) - if not parent_ids: - continue - if len(call_items) != 1 or call_id_counts[continuation_id] != 1: - for parent in parent_ids: - continuation_failures.setdefault( - parent, - "tree_read_continuation_call_duplicate", - ) - continue - output_items = outputs.get(continuation_id, []) - if len(output_items) != 1: - reason = ( - "tree_read_continuation_output_missing" - if not output_items - else "tree_read_continuation_output_duplicate" - ) - for parent in parent_ids: - continuation_failures.setdefault(parent, reason) - continue - output, output_completed_at, output_truncated, output_parts = ( - output_items[0] - ) - call = call_items[0] - for parent in parent_ids: - continuation_outputs.setdefault(parent, []).append( - ( - output, - output_completed_at, - call.get("timestamp"), - output_truncated, - output_parts, - ) - ) - - for items in continuation_outputs.values(): - items.sort(key=lambda item: str(item[2] or "")) - - for call_id, call_items in calls.items(): - for call in call_items: - timestamp = call.get("timestamp") - if not isinstance(timestamp, str) or not in_window(timestamp, window): - continue - payload = call["payload"] - tool_name = str(payload.get("name") or "") - if tool_name in SHELL_CONTINUATION_TOOLS | CELL_CONTINUATION_TOOLS: - continue - assessment = call.get("assessment") - if not isinstance(assessment, ReadAssessment) or assessment.status is None: - continue - if len(call_items) != 1 or call_id_counts[call_id] != 1: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "codex_duplicate_call_id", - ) - continue - if assessment.status in {"unresolved_opaque", "rejected_unsafe"}: - record_read_attempt( - attempt_counts, - gaps, - assessment.status, - assessment.reason, - ) - continue - read_plan = call.get("read_plan") - if not isinstance(read_plan, ReadPlan): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_plan_missing", - ) - continue - output_items = outputs.get(call_id, []) - if not output_items: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_missing", - ) - continue - if len(output_items) != 1: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "codex_duplicate_tool_result", - ) - continue - ( - initial_output, - completed_at, - initial_truncated, - initial_parts, - ) = output_items[0] - if not isinstance(completed_at, str): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "codex_tool_result_timestamp_missing", - ) - continue - if not in_window(completed_at, window): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "codex_tool_result_outside_window", - ) - continue - if parse_datetime(completed_at) < parse_datetime(timestamp): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "codex_tool_result_out_of_order", - ) - continue - continuation_failure = continuation_failures.get(call_id) - continuations = continuation_outputs.get(call_id, []) - previous_completed_at = completed_at - for item in continuations: - continuation_completed_at = item[1] - continuation_started_at = item[2] - if ( - not isinstance(continuation_completed_at, str) - or not isinstance(continuation_started_at, str) - or not in_window(continuation_completed_at, window) - or not in_window(continuation_started_at, window) - or parse_datetime(continuation_started_at) - < parse_datetime(previous_completed_at) - or parse_datetime(continuation_completed_at) - < parse_datetime(continuation_started_at) - ): - continuation_failure = ( - "tree_read_continuation_incomplete_or_out_of_order" - ) - break - previous_completed_at = continuation_completed_at - if continuation_failure is not None: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - continuation_failure, - ) - continue - output = initial_output - output_parts = list(initial_parts) - output_was_truncated = initial_truncated - if continuations: - output = "\n".join( - [output, *(item[0] for item in continuations)] - ) - output_parts.extend( - part - for item in continuations - for part in item[4] - ) - completed_at = continuations[-1][1] - output_was_truncated = output_was_truncated or any( - item[3] for item in continuations - ) - initial_handle = ( - SHELL_SESSION_PATTERN.search(initial_output) - if tool_name in EXEC_COMMAND_TOOLS - else CELL_SESSION_PATTERN.search(initial_output) - if tool_name in EXEC_ORCHESTRATION_TOOLS - else None - ) - if initial_handle is not None: - last_output = continuations[-1][0] if continuations else "" - last_pending = ( - SHELL_SESSION_PATTERN.search(last_output) - if tool_name in EXEC_COMMAND_TOOLS - else CELL_SESSION_PATTERN.search(last_output) - ) - if not continuations or last_pending is not None: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_pending", - ) - continue - if not output.strip(): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_missing", - ) - continue - success = output_success(output) - if success is False: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_command_failed", - ) - continue - content_parts = [ - cleaned - for part in output_parts or [output] - if (cleaned := content_output(part, tool_name)).strip() - ] - if not content_parts: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_missing", - ) - continue - subplans = assessment.subplans or (read_plan,) - sliced = len(subplans) > 1 and len(content_parts) == len(subplans) - if sliced: - plan_outputs = list(zip(subplans, content_parts, strict=True)) - else: - if len(subplans) > 1: - gaps.add("tree_read_output_attribution_aggregate") - plan_outputs = [(read_plan, "\n".join(content_parts))] - call_reads: list[dict[str, Any]] = [] - attribution_failure: str | None = None - for read_index, (current_plan, passage_source) in enumerate( - plan_outputs - ): - node_paths = list(current_plan.node_paths) - passage_source, attribution_gap = attributable_passage( - passage_source, - current_plan, - ) - if passage_source is None: - attribution_failure = ( - attribution_gap - or "tree_read_output_attribution_unresolved" - ) - break - passage, passage_truncated = clipped( - passage_source, - max_passage_chars, - ) - passage_truncated = passage_truncated or output_was_truncated - if passage_truncated: - gaps.add("tree_read_passage_truncated") - read_id = hashlib.sha256( - f"{preflight.trace_id}:{call_id}:{read_index}".encode() - ).hexdigest()[:20] - call_reads.append( - { - "read_id": read_id, - "timestamp": timestamp, - "completed_at": completed_at, - "session_file": preflight.trace_id, - "call_id": call_id, - "nested_call_index": ( - read_index if sliced else None - ), - "tool_name": tool_name, - "runtime_provider": preflight.runtime_provider, - "reader_agent_id": preflight.agent_id, - "tree_identity": tree_id, - "node_paths": node_paths, - "read_components": [ - { - "reader": component.reader, - "node_paths": list(component.node_paths), - } - for component in current_plan.components - ], - "read_mode": current_plan.mode, - "output_attribution": ( - "exact" - if current_plan.mode == "isolated" - else "aggregate" - ), - "auxiliary_output_possible": ( - current_plan.auxiliary_output_possible - ), - "content_class_hint": content_class_hint(node_paths), - "command": current_plan.command, - "command_truncated": False, - "passage": passage, - "passage_truncated": passage_truncated, - "success": True, - } - ) - if attribution_failure is not None or not call_reads: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - attribution_failure - or "tree_read_output_attribution_unresolved", - ) - continue - reads.extend(call_reads) - record_read_attempt( - attempt_counts, - gaps, - assessment.status, - ) - return reads, sessions, gaps, attempt_counts - - -def provider_timestamp(value: Any) -> str | None: - if isinstance(value, str): - try: - return isoformat(parse_datetime(value)) - except (AuditError, ValueError): - return None - if isinstance(value, (int, float)) and not isinstance(value, bool): - seconds = value / 1000 if value > 10_000_000_000 else value - try: - return isoformat(datetime.fromtimestamp(seconds, tz=timezone.utc)) - except (OSError, OverflowError, ValueError): - return None - return None - - -def normalized_provider_tool( - runtime_provider: str, - tool_name: str, - arguments: Mapping[str, Any], - default_workdir: Path, -) -> tuple[str, dict[str, Any]]: - if runtime_provider == "claude-code": - if tool_name == "Read": - return "read_file", {"input": dict(arguments)} - if tool_name == "Bash": - command = arguments.get("command") - workdir = arguments.get("cwd") - return ( - "exec_command", - { - "arguments": { - "cmd": command, - "workdir": workdir if isinstance(workdir, str) else str(default_workdir), - } - }, - ) - return tool_name, {"input": dict(arguments)} - - -def provider_read_rows( - preflight: TracePreflight, - pairs: Sequence[Mapping[str, Any]], - tree_root: Path, - tree_id: str, - window: Window, - max_passage_chars: int, -) -> tuple[list[dict[str, Any]], set[str], set[str], Counter[str]]: - reads: list[dict[str, Any]] = [] - sessions = {preflight.trace_id} - gaps: set[str] = set() - attempt_counts: Counter[str] = Counter() - for pair in pairs: - started_at = provider_timestamp(pair.get("started_at")) - if started_at is None: - gaps.add( - f"{preflight.runtime_provider.replace('-', '_')}" - "_tool_timestamp_missing" - ) - continue - if not in_window(started_at, window): - continue - tool_name = pair.get("tool_name") - arguments = pair.get("arguments") - call_id = pair.get("call_id") - if ( - not isinstance(tool_name, str) - or not isinstance(arguments, Mapping) - or not isinstance(call_id, str) - ): - continue - normalized_name, payload = normalized_provider_tool( - preflight.runtime_provider, - tool_name, - arguments, - preflight.workspace, - ) - assessment = markdown_read_plan( - normalized_name, - payload, - tree_root, - preflight.workspace, - ) - if assessment.status is None: - continue - pairing_error = pair.get("pairing_error") - if isinstance(pairing_error, str): - reason = f"unresolved_{pairing_error}" - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - reason, - ) - gaps.add(pairing_error) - continue - completed_at = provider_timestamp(pair.get("completed_at")) - if completed_at is None: - gap = ( - f"{preflight.runtime_provider.replace('-', '_')}" - "_tool_timestamp_missing" - ) - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - f"unresolved_{gap}", - ) - gaps.add(gap) - continue - started_index = pair.get("started_index") - completed_index = pair.get("completed_index") - out_of_order = ( - isinstance(started_index, int) - and isinstance(completed_index, int) - and completed_index <= started_index - ) or parse_datetime(completed_at) < parse_datetime(started_at) - if out_of_order: - gap = ( - f"{preflight.runtime_provider.replace('-', '_')}" - "_tool_result_out_of_order" - ) - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - f"unresolved_{gap}", - ) - gaps.add(gap) - continue - if not in_window(completed_at, window): - gap = ( - f"{preflight.runtime_provider.replace('-', '_')}" - "_tool_result_outside_window" - ) - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - f"unresolved_{gap}", - ) - gaps.add(gap) - continue - if assessment.status in {"unresolved_opaque", "rejected_unsafe"}: - record_read_attempt( - attempt_counts, - gaps, - assessment.status, - assessment.reason, - ) - continue - if pair.get("success") is not True: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_command_failed", - ) - continue - raw_output = pair.get("output") - if not isinstance(raw_output, str) or not raw_output.strip(): - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_missing", - ) - continue - output, output_truncated = clipped( - redact_local_roots( - raw_output, - preflight.workspace, - tree_root, - ), - max_passage_chars, - ) - plan = assessment.plan - if plan is None: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_plan_missing", - ) - continue - content_parts = [ - cleaned - for raw_part in pair.get("output_parts", [output]) - if isinstance(raw_part, str) - and ( - cleaned := content_output( - redact_local_roots( - raw_part, - preflight.workspace, - tree_root, - ), - normalized_name, - ) - ).strip() - ] - if not content_parts: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - "tree_read_output_missing", - ) - continue - subplans = assessment.subplans or (plan,) - sliced = len(subplans) > 1 and len(content_parts) == len(subplans) - if sliced: - plan_outputs = list(zip(subplans, content_parts, strict=True)) - else: - if len(subplans) > 1: - gaps.add("tree_read_output_attribution_aggregate") - plan_outputs = [(plan, "\n".join(content_parts))] - call_reads: list[dict[str, Any]] = [] - attribution_failure: str | None = None - for read_index, (current_plan, passage_source) in enumerate(plan_outputs): - passage_source, attribution_gap = attributable_passage( - passage_source, - current_plan, - ) - if passage_source is None: - attribution_failure = ( - attribution_gap - or "tree_read_output_attribution_unresolved" - ) - break - passage, passage_truncated = clipped( - passage_source, - max_passage_chars, - ) - passage_truncated = passage_truncated or output_truncated - if passage_truncated: - gaps.add("tree_read_passage_truncated") - read_id = hashlib.sha256( - f"{preflight.trace_id}:{call_id}:{read_index}".encode() - ).hexdigest()[:20] - call_reads.append( - { - "read_id": read_id, - "timestamp": started_at, - "completed_at": completed_at, - "session_file": preflight.trace_id, - "call_id": call_id, - "nested_call_index": read_index if sliced else None, - "tool_name": normalized_name, - "runtime_provider": preflight.runtime_provider, - "reader_agent_id": preflight.agent_id, - "tree_identity": tree_id, - "node_paths": list(current_plan.node_paths), - "read_components": [ - { - "reader": component.reader, - "node_paths": list(component.node_paths), - } - for component in current_plan.components - ], - "read_mode": current_plan.mode, - "output_attribution": ( - "exact" - if current_plan.mode == "isolated" - else "aggregate" - ), - "auxiliary_output_possible": current_plan.auxiliary_output_possible, - "content_class_hint": content_class_hint(current_plan.node_paths), - "command": current_plan.command, - "command_truncated": False, - "passage": passage, - "passage_truncated": passage_truncated, - "success": True, - } - ) - if attribution_failure is not None or not call_reads: - record_read_attempt( - attempt_counts, - gaps, - "unresolved_opaque", - attribution_failure - or "tree_read_output_attribution_unresolved", - ) - continue - reads.extend(call_reads) - record_read_attempt( - attempt_counts, - gaps, - assessment.status, - ) - return reads, sessions, gaps, attempt_counts - - -def workspace_trace_cwd_matches( - value: Any, - workspace: Path, - chat_id: str | None = None, -) -> bool: - if not isinstance(value, str): - return False - try: - candidate = Path(value).expanduser().resolve(strict=False) - except OSError: - return False - if candidate == workspace: - return True - return chat_id is not None and candidate == workspace / chat_id - - -def claude_canonical_user_text(row: Mapping[str, Any]) -> str: - """Return only external-human text; never accept synthetic mirror rows.""" - if ( - row.get("type") != "user" - or row.get("isSidechain") is True - or row.get("isCompactSummary") is True - or row.get("isMeta") is True - ): - return "" - message = row.get("message") - if not isinstance(message, Mapping) or message.get("role") != "user": - return "" - content = message.get("content") - if isinstance(content, str): - return content - if not isinstance(content, list): - return "" - return "\n".join( - block["text"] - for block in content - if isinstance(block, dict) - and block.get("type") == "text" - and isinstance(block.get("text"), str) - ) - - -def preflight_claude_trace( - path: Path, - trace_root: Path, - workspace_identity: WorkspaceIdentity, - authorized_audits: Mapping[tuple[str, str], str], -) -> tuple[TracePreflight | None, dict[str, set[str]]]: - all_audit_ids = set(authorized_audits.values()) - gaps = {item: set() for item in all_audit_ids} - chat_ids: set[str] = set() - session_ids: set[str] = set() - cwd_values: set[str] = set() - malformed = False - bytes_seen = 0 - lines_seen = 0 - try: - with path.open("rb") as handle: - while ( - bytes_seen < TRACE_PREFLIGHT_MAX_BYTES - and lines_seen < TRACE_PREFLIGHT_MAX_LINES - ): - remaining = TRACE_PREFLIGHT_MAX_BYTES - bytes_seen - raw_line = handle.readline(remaining + 1) - if not raw_line: - break - if len(raw_line) > remaining: - break - bytes_seen += len(raw_line) - lines_seen += 1 - try: - row = json.loads(raw_line.decode("utf-8")) - except (UnicodeDecodeError, json.JSONDecodeError): - malformed = True - continue - if ( - not isinstance(row, dict) - or row.get("isSidechain") is True - or row.get("isCompactSummary") is True - or row.get("isMeta") is True - ): - continue - message = row.get("message") - if ( - row.get("type") not in {"user", "assistant"} - or not isinstance(message, Mapping) - ): - continue - cwd = row.get("cwd") - if isinstance(cwd, str): - cwd_values.add(cwd) - session_id = row.get("sessionId") - if isinstance(session_id, str): - session_ids.add(session_id) - text = claude_canonical_user_text(row) - if CHAT_CONTEXT_PATTERN.search(text) is not None: - found = set(chat_ids_from_text(text)) - if len(found) != 1: - malformed = True - chat_ids.update(found) - except OSError: - return None, gaps - - legacy = { - Path(cwd).name - for cwd in cwd_values - if Path(cwd).parent.resolve(strict=False) == workspace_identity.workspace - and re.fullmatch(UUID_PATTERN, Path(cwd).name) - } - workspace_related = any( - workspace_trace_cwd_matches(cwd, workspace_identity.workspace) - for cwd in cwd_values - ) or bool(legacy) - if not workspace_related: - return None, gaps - if not chat_ids: - chat_ids.update(legacy) - if malformed or len(chat_ids) != 1 or len(session_ids) != 1: - for item in all_audit_ids: - gaps[item].add("claude_trace_preflight_malformed_or_unmapped") - return None, gaps - chat_id = next(iter(chat_ids)) - if not cwd_values or any( - not workspace_trace_cwd_matches( - cwd, - workspace_identity.workspace, - chat_id, - ) - for cwd in cwd_values - ): - for item in all_audit_ids: - gaps[item].add("claude_trace_workspace_mismatch") - return None, gaps - current_audit_id = authorized_audits.get((chat_id, workspace_identity.agent_id)) - if current_audit_id is None: - return None, gaps - return ( - TracePreflight( - path=path, - trace_id=trace_identity(path, trace_root), - audit_id=current_audit_id, - agent_id=workspace_identity.agent_id, - workspace=workspace_identity.workspace, - runtime_provider="claude-code", - provider_session_id=next(iter(session_ids)), - ), - gaps, - ) - - -def claude_trace_pairs( - preflight: TracePreflight, -) -> tuple[list[dict[str, Any]], set[str]]: - calls: dict[str, list[dict[str, Any]]] = {} - results: dict[str, list[dict[str, Any]]] = {} - gaps: set[str] = set() - expected_chat_id = preflight.audit_id.split("@", 1)[0] - identity_changed = False - try: - with preflight.path.open("r", encoding="utf-8") as handle: - for row_index, line in enumerate(handle): - if not line.strip(): - continue - row = json.loads(line) - if ( - not isinstance(row, dict) - or row.get("isSidechain") is True - or row.get("isCompactSummary") is True - or row.get("isMeta") is True - ): - continue - message = row.get("message") - if ( - row.get("type") not in {"user", "assistant"} - or not isinstance(message, dict) - ): - continue - if row.get("sessionId") != preflight.provider_session_id: - gaps.add("claude_trace_session_changed") - identity_changed = True - break - if not workspace_trace_cwd_matches( - row.get("cwd"), - preflight.workspace, - expected_chat_id, - ): - gaps.add("claude_trace_workspace_changed") - identity_changed = True - break - if row.get("type") == "user" and message.get("role") == "user": - canonical_text = claude_canonical_user_text(row) - if CHAT_CONTEXT_PATTERN.search(canonical_text) is not None: - found = set(chat_ids_from_text(canonical_text)) - if found != {expected_chat_id}: - gaps.add("claude_trace_chat_boundary_changed") - identity_changed = True - break - content = message.get("content") - if not isinstance(content, list): - continue - if row.get("type") == "assistant": - for block in content: - if not isinstance(block, dict) or block.get("type") != "tool_use": - continue - call_id = block.get("id") - if not isinstance(call_id, str): - continue - call_rows = calls.setdefault(call_id, []) - call_rows.append( - { - "call_id": call_id, - "tool_name": block.get("name"), - "arguments": block.get("input"), - "started_at": row.get("timestamp"), - "started_index": row_index, - } - ) - elif row.get("type") == "user": - for block in content: - if not isinstance(block, dict) or block.get("type") != "tool_result": - continue - call_id = block.get("tool_use_id") - if not isinstance(call_id, str): - continue - result_rows = results.setdefault(call_id, []) - result_rows.append( - { - "completed_at": row.get("timestamp"), - "completed_index": row_index, - "output": payload_text(block.get("content")), - "success": block.get("is_error") is not True, - } - ) - except (OSError, UnicodeDecodeError, json.JSONDecodeError): - gaps.add("claude_trace_malformed_or_partially_cleaned") - if identity_changed: - return [], gaps - pairs: list[dict[str, Any]] = [] - for call_id, call_rows in calls.items(): - result_rows = results.get(call_id, []) - if len(call_rows) == 1 and len(result_rows) == 1: - pairs.append({**call_rows[0], **result_rows[0]}) - continue - if len(call_rows) > 1: - pairing_error = "claude_tool_call_duplicate" - elif not result_rows: - pairing_error = "claude_tool_result_missing" - else: - pairing_error = "claude_tool_result_duplicate" - pairs.extend( - {**call, "pairing_error": pairing_error} - for call in call_rows - ) - return pairs, gaps - - -def preflight_runtime_trace( - runtime_provider: str, - path: Path, - trace_root: Path, - workspace_identity: WorkspaceIdentity, - authorized_audits: Mapping[tuple[str, str], str], -) -> tuple[TracePreflight | None, dict[str, set[str]]]: - if runtime_provider == "codex": - return preflight_codex_trace( - path, - trace_root, - workspace_identity, - authorized_audits, - ) - if runtime_provider == "claude-code": - return preflight_claude_trace( - path, - trace_root, - workspace_identity, - authorized_audits, - ) - raise AuditError(f"No evidence preflight exists for {runtime_provider}.") - - -def runtime_trace_reads( - preflight: TracePreflight, - tree_root: Path, - tree_id: str, - window: Window, - max_passage_chars: int, -) -> tuple[list[dict[str, Any]], set[str], set[str], Counter[str]]: - if preflight.runtime_provider == "codex": - return codex_trace_reads( - preflight, - tree_root, - tree_id, - window, - max_passage_chars, - ) - if preflight.runtime_provider != "claude-code": - raise AuditError( - f"No evidence reader exists for {preflight.runtime_provider}." - ) - pairs, parse_gaps = claude_trace_pairs(preflight) - reads, sessions, gaps, counts = provider_read_rows( - preflight, - pairs, - tree_root, - tree_id, - window, - max_passage_chars, - ) - gaps.update(parse_gaps) - return reads, sessions, gaps, counts - - -def collect_evidence(args: argparse.Namespace) -> None: - workspace_identity = parse_agent_workspace(args.agent_workspace) - artifact_root = resolve_artifact_root(args.artifact_root, workspace_identity) - chats_path = artifact_path(artifact_root, args.chats, field="--chats", must_exist=True) - output_path = artifact_path(artifact_root, args.output, field="--output", must_exist=False) - require_distinct_paths({"--chats": chats_path, "--output": output_path}) - window = resolve_window(args.days, args.now) - chat_rows = [normalize_chat(row, window) for row in iter_jsonl(chats_path)] - if not chat_rows: - raise AuditError("No authorized Chat records were provided.") - if len({chat["audit_id"] for chat in chat_rows}) != len(chat_rows): - raise AuditError("The Chat export contains duplicate Chat and audited-Agent pairs.") - scoped_identities = { - (str(chat["source_agent"]), chat["source_agent_id"]) for chat in chat_rows - } - if len(scoped_identities) != 1: - raise AuditError("Chat export must contain one exact Agent name and UUID.") - authorizations = {chat["authorization"] for chat in chat_rows} - if len(authorizations) != 1: - raise AuditError("Chat export must not mix Agent-level and Chat-level authorization.") - chats = {chat["audit_id"]: chat for chat in chat_rows} - authorized_audits = { - (chat["chat_id"], chat["source_agent_id"]): chat["audit_id"] for chat in chat_rows - } - scoped_agent_name, scoped_agent_id = next(iter(scoped_identities)) - if ( - workspace_identity.agent_name != scoped_agent_name - or workspace_identity.agent_id != scoped_agent_id - ): - raise AuditError( - "--agent-workspace identity must match the one exact Agent name and UUID in --chats." - ) - - raw_tree_root = Path(args.tree_root).expanduser() - if not raw_tree_root.is_absolute() or raw_tree_root.is_symlink(): - raise AuditError("--tree-root must be an absolute, non-symbolic-link directory.") - try: - tree_root = raw_tree_root.resolve(strict=True) - except OSError as error: - raise AuditError(f"Could not resolve --tree-root: {error}") from error - if not tree_root.is_dir(): - raise AuditError("--tree-root is not a directory.") - if tree_root != workspace_identity.bound_tree_root: - raise AuditError( - "--tree-root must exactly match the Context Tree bound in workspace identity." - ) - current_tree_id = tree_identity(workspace_identity) - tree_source_snapshot = resolve_tree_source_snapshot(tree_root) - runtime_provider = resolve_runtime_provider(args.runtime_provider) - - if runtime_provider not in SUPPORTED_EVIDENCE_PROVIDERS: - for chat in chats.values(): - chat["coverage_gaps"].append( - f"{runtime_provider.replace('-', '_')}_historical_evidence_not_supported" - ) - trace_root = workspace_identity.workspace - trace_files: list[Path] = [] - else: - raw_trace_root = ( - Path(args.trace_root).expanduser() - if args.trace_root - else trace_root_default(runtime_provider, workspace_identity) - ) - if raw_trace_root.is_symlink(): - raise AuditError("--trace-root must not be a symbolic link.") - trace_root = raw_trace_root.resolve() - if not trace_root.is_dir(): - for chat in chats.values(): - chat["coverage_gaps"].append( - f"{runtime_provider.replace('-', '_')}_evidence_root_missing_or_cleaned" - ) - trace_files = [] - else: - trace_files = recent_trace_files( - trace_root, - window, - runtime_provider, - ) - - per_audit_reads: dict[str, list[dict[str, Any]]] = {item: [] for item in chats} - per_audit_sessions: dict[str, set[str]] = {item: set() for item in chats} - per_audit_gaps: dict[str, set[str]] = {item: set() for item in chats} - per_audit_attempt_counts: dict[str, Counter[str]] = { - item: Counter() for item in chats - } - - for trace_file in trace_files: - preflight, preflight_gaps = preflight_runtime_trace( - runtime_provider, - trace_file, - trace_root, - workspace_identity, - authorized_audits, - ) - for item in chats: - per_audit_gaps[item].update(preflight_gaps[item]) - if preflight is None: - continue - reads, sessions, gaps, attempt_counts = runtime_trace_reads( - preflight, - tree_root, - current_tree_id, - window, - args.max_passage_chars, - ) - per_audit_reads[preflight.audit_id].extend(reads) - per_audit_sessions[preflight.audit_id].update(sessions) - per_audit_gaps[preflight.audit_id].update(gaps) - per_audit_attempt_counts[preflight.audit_id].update(attempt_counts) - - output_rows: list[dict[str, Any]] = [] - for current_audit_id, chat in sorted(chats.items()): - chat_id = chat["chat_id"] - reads = sorted( - per_audit_reads[current_audit_id], key=lambda item: (item["timestamp"], item["read_id"]) - ) - messages = [] - for message in chat["messages"]: - sanitized = dict(message) - sanitized["content"] = redact_local_roots( - str(message.get("content") or ""), - workspace_identity.workspace, - tree_root, - ) - messages.append(sanitized) - visible_tree_mentions = [ - message - for message in messages - if TREE_MENTION_PATTERN.search(str(message.get("content") or "")) is not None - ] - choice_messages = [] - agent_messages = [ - message for message in messages if message.get("sender_id") == chat["source_agent_id"] - ] - for read in reads: - read["tree_source"] = read_tree_source( - read, - tree_root, - tree_source_snapshot, - ) - receipt_messages = [ - message for message in agent_messages if "decision_receipt" in message - ] - if reads: - earliest_read = parse_datetime(reads[0]["timestamp"]) - choice_messages = [ - message - for message in agent_messages - if isinstance(message.get("created_at"), str) - and parse_datetime(message["created_at"]) >= earliest_read - ] - elif visible_tree_mentions or receipt_messages: - choice_messages_by_id = { - message["message_id"]: message - for message in [ - *visible_tree_mentions, - *receipt_messages, - ] - if message.get("sender_id") == chat["source_agent_id"] - and isinstance(message.get("message_id"), str) - } - choice_messages = sorted( - choice_messages_by_id.values(), - key=lambda message: ( - str(message.get("created_at") or ""), - str(message.get("message_id") or ""), - ), - ) - candidate_status = ( - "candidate" - if reads or visible_tree_mentions or receipt_messages - else "outside_candidate_set" - ) - gaps = set(chat["coverage_gaps"]) | per_audit_gaps[current_audit_id] - attempt_counts = per_audit_attempt_counts[current_audit_id] - status_counts = { - status: attempt_counts[status] for status in READ_ATTEMPT_STATUSES - } - reason_counts = { - key.removeprefix("reason:"): count - for key, count in sorted(attempt_counts.items()) - if key.startswith("reason:") - } - if ( - runtime_provider in SUPPORTED_EVIDENCE_PROVIDERS - and not per_audit_sessions[current_audit_id] - ): - gaps.add( - f"no_mapped_{runtime_provider.replace('-', '_')}_evidence" - ) - elif ( - per_audit_sessions[current_audit_id] - and not reads - and (visible_tree_mentions or receipt_messages) - ): - gaps.add("no_successful_tree_content_read") - output_rows.append( - { - "schema_version": SCHEMA_VERSION, - "audit_id": current_audit_id, - "chat": { - "chat_id": chat_id, - "title": chat["title"], - "authorization": chat["authorization"], - "source_agent": chat["source_agent"], - "source_agent_id": chat["source_agent_id"], - "message_count": len(messages), - }, - "window": {"start": window_start_text(window), "end": isoformat(window.end)}, - "tree_identity": current_tree_id, - "tree_source_snapshot": tree_source_snapshot, - "runtime_provider": runtime_provider, - "candidate_status": candidate_status, - "mapped_trace_files": sorted(per_audit_sessions[current_audit_id]), - "collector_diagnostics": { - "in_window_tree_read_attempts": sum(status_counts.values()), - "attempt_status_counts": status_counts, - "attempt_reason_counts": reason_counts, - }, - "reads": reads, - "visible_messages": messages, - "visible_choice_candidates": choice_messages, - "visible_tree_mentions": visible_tree_mentions, - "coverage_gaps": sorted(gaps), - } - ) - write_jsonl(output_path, output_rows) - - -def string_id_list(value: Any, *, field: str) -> list[str]: - if not isinstance(value, list): - raise AuditError(f"{field} must be an array.") - items = [require_string(item, field) for item in value] - if len(set(items)) != len(items): - raise AuditError(f"{field} must not contain duplicate IDs.") - return items - - -def optional_text(value: Any, *, field: str) -> str | None: - if value is None: - return None - if not isinstance(value, str): - raise AuditError(f"{field} must be a string or null.") - return value.strip() or None - - -def require_exact_keys( - value: Mapping[str, Any], - allowed: set[str], - *, - field: str, -) -> None: - unexpected = sorted(set(value) - allowed) - if unexpected: - raise AuditError( - f"{field} contains unsupported field(s): {', '.join(unexpected)}." - ) - - -def strip_edge_decorations(value: str, *, preserve_at: bool) -> str: - result = value.strip() - while result: - removed = False - if ( - result - and (not preserve_at or result[0] != "@") - and unicodedata.category(result[0])[0] in {"P", "S"} - ): - result = result[1:].lstrip() - removed = True - if result and unicodedata.category(result[-1])[0] in {"P", "S"}: - result = result[:-1].rstrip() - removed = True - if not removed: - break - return result - - -def normalized_task_fragment(value: str) -> str: - normalized = re.sub(r"\s+", " ", value.strip().casefold()) - normalized = strip_edge_decorations(normalized, preserve_at=True) - while normalized.startswith("@"): - without_mention = re.sub( - r"^@[a-z0-9_-]{1,100}(?=[^a-z0-9_-]|$)\s*", - "", - normalized, - count=1, - ) - if without_mention == normalized: - break - normalized = strip_edge_decorations( - without_mention, - preserve_at=True, - ) - return strip_edge_decorations(normalized, preserve_at=False) - - -def strip_task_wrapper( - value: str, - wrapper: str, - *, - prefix: bool, -) -> str | None: - if value == wrapper: - return "" - if prefix: - if not value.startswith(wrapper): - return None - remainder = value[len(wrapper) :] - if wrapper.isascii() and not remainder.startswith(" "): - return None - return normalized_task_fragment(remainder) - if not value.endswith(wrapper): - return None - remainder = value[: -len(wrapper)] - if wrapper.isascii() and not remainder.endswith(" "): - return None - return normalized_task_fragment(remainder) - - -def is_weak_task_clause(value: str) -> bool: - pending = [normalized_task_fragment(value)] - seen: set[str] = set() - while pending: - current = pending.pop() - if current in seen: - continue - seen.add(current) - if ( - not current - or current in WEAK_TASK_OBJECTIVES - or any( - re.fullmatch(pattern, current) - for pattern in WEAK_TASK_ANCHORED_PATTERNS - ) - ): - return True - for prefix in WEAK_TASK_PREFIX_WRAPPERS: - stripped = strip_task_wrapper( - current, prefix, prefix=True - ) - if stripped is not None and stripped not in seen: - pending.append(stripped) - for suffix in WEAK_TASK_SUFFIX_WRAPPERS: - stripped = strip_task_wrapper( - current, suffix, prefix=False - ) - if stripped is not None and stripped not in seen: - pending.append(stripped) - return False - - -def is_weak_task_fragment(value: str) -> bool: - normalized = normalized_task_fragment(value) - if not normalized: - return True - clauses = [ - normalized_task_fragment(clause) - for clause in re.split(r"[,，;；:：/、.!?。！？]+", normalized) - if normalized_task_fragment(clause) - ] - return bool(clauses) and all( - is_weak_task_clause(clause) for clause in clauses - ) - - -def validate_task_sources(value: Mapping[str, Any], *, task_id: str) -> dict[str, Any]: - objective_source_ids = string_id_list( - value.get("objective_source_message_ids"), - field=f"task[{task_id}].objective_source_message_ids", - ) - outcome_source_ids = string_id_list( - value.get("outcome_source_message_ids"), - field=f"task[{task_id}].outcome_source_message_ids", - ) - if not objective_source_ids or not outcome_source_ids: - raise AuditError( - f"task[{task_id}] requires objective and outcome source message IDs." - ) - if set(objective_source_ids) & set(outcome_source_ids): - raise AuditError( - f"task[{task_id}] objective and outcome source messages must be separate." - ) - return { - "objective_source_message_ids": objective_source_ids, - "outcome_source_message_ids": outcome_source_ids, - "primary_deliverable": optional_text( - value.get("primary_deliverable"), - field=f"task[{task_id}].primary_deliverable", - ), - } - - -def positive_int(value: Any, *, field: str) -> int: - if not isinstance(value, int) or isinstance(value, bool) or value <= 0: - raise AuditError(f"{field} must be a positive integer.") - return value - - -def non_negative_int(value: Any, *, field: str) -> int: - if not isinstance(value, int) or isinstance(value, bool) or value < 0: - raise AuditError(f"{field} must be a non-negative integer.") - return value - - -def load_reviewed_baseline(path: Path) -> dict[str, Any]: - rows = list(iter_jsonl(path)) - if len(rows) != 1: - raise AuditError("--reviewed-baseline must contain exactly one JSONL row.") - row = rows[0] - if row.get("schema_version") != TASK_ANALYSIS_SCHEMA_VERSION: - raise AuditError( - "Reviewed baseline must use schema_version " - f"{TASK_ANALYSIS_SCHEMA_VERSION}." - ) - require_exact_keys( - row, - { - "schema_version", - "basis", - "reviewed_at", - "evidence_anchor", - "clear_tasks", - "effect_tasks", - "effects", - "effect_counts", - }, - field="reviewed_baseline", - ) - if row.get("basis") != "separately_reviewed_task_cases": - raise AuditError( - "Reviewed baseline basis must be separately_reviewed_task_cases." - ) - anchor = row.get("evidence_anchor") - if not isinstance(anchor, dict): - raise AuditError("Reviewed baseline requires evidence_anchor.") - require_exact_keys( - anchor, - {"artifact_id", "sha256"}, - field="reviewed_baseline.evidence_anchor", - ) - artifact_id = require_string( - anchor.get("artifact_id"), - "reviewed_baseline.evidence_anchor.artifact_id", - ) - if re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9._:-]{0,127}", artifact_id) is None: - raise AuditError( - "reviewed_baseline.evidence_anchor.artifact_id must be an opaque safe identifier." - ) - artifact_sha256 = require_string( - anchor.get("sha256"), - "reviewed_baseline.evidence_anchor.sha256", - ).lower() - if re.fullmatch(r"[0-9a-f]{64}", artifact_sha256) is None: - raise AuditError( - "reviewed_baseline.evidence_anchor.sha256 must be 64 lowercase hex characters." - ) - reviewed_at = isoformat( - parse_datetime( - require_string(row.get("reviewed_at"), "reviewed_baseline.reviewed_at"), - field="reviewed_baseline.reviewed_at", - ) - ) - clear_tasks = positive_int( - row.get("clear_tasks"), field="reviewed_baseline.clear_tasks" - ) - effect_tasks = non_negative_int( - row.get("effect_tasks"), field="reviewed_baseline.effect_tasks" - ) - if effect_tasks > clear_tasks: - raise AuditError( - "Reviewed baseline effect_tasks must not exceed clear_tasks." - ) - - effects = non_negative_int( - row.get("effects"), field="reviewed_baseline.effects" - ) - if effects < effect_tasks: - raise AuditError( - "Reviewed baseline effects must not be less than effect_tasks." - ) - effect_counts = row.get("effect_counts") - if not isinstance(effect_counts, dict) or set(effect_counts) != EFFECT_VALUES: - raise AuditError( - "Reviewed baseline effect_counts must contain exactly the four effect keys." - ) - normalized_effect_counts: dict[str, int] = {} - for effect in sorted(EFFECT_VALUES): - value = effect_counts[effect] - if not isinstance(value, int) or isinstance(value, bool) or value < 0: - raise AuditError( - f"reviewed_baseline.effect_counts.{effect} must be a non-negative integer." - ) - normalized_effect_counts[effect] = value - if sum(normalized_effect_counts.values()) != effects: - raise AuditError( - "Reviewed baseline effect_counts must conserve effects." - ) - return { - "basis": "separately_reviewed_task_cases", - "reviewed_at": reviewed_at, - "evidence_anchor": { - "artifact_id": artifact_id, - "sha256": artifact_sha256, - }, - "clear_tasks": clear_tasks, - "effect_tasks": effect_tasks, - "effects": effects, - "effect_counts": normalized_effect_counts, - } - - -def load_task_inventory( - path: Path, - *, - require_frozen: bool, -) -> tuple[list[dict[str, Any]], str | None]: - tasks: list[dict[str, Any]] = [] - task_ids: set[str] = set() - declared_digests: set[str] = set() - for row in iter_jsonl(path): - if row.get("schema_version") != TASK_INVENTORY_SCHEMA_VERSION: - raise AuditError( - "Every Task inventory row must use schema_version " - f"{TASK_INVENTORY_SCHEMA_VERSION}." - ) - declared_digest = row.get("inventory_sha256") - if require_frozen: - if ( - not isinstance(declared_digest, str) - or re.fullmatch(r"[0-9a-f]{64}", declared_digest) is None - ): - raise AuditError( - "Every frozen Task inventory row requires one lowercase " - "inventory_sha256." - ) - declared_digests.add(declared_digest) - elif declared_digest is not None: - raise AuditError( - "Task inventory drafts must not contain inventory_sha256; " - "freeze-tasks creates it." - ) - task_id = require_string(row.get("task_id"), "task.task_id") - if task_id in task_ids: - raise AuditError(f"Duplicate Task judgment {task_id}.") - task_ids.add(task_id) - status = require_string(row.get("status"), f"task[{task_id}].status") - if status not in TASK_STATUS_VALUES: - raise AuditError( - f"task[{task_id}].status must be one of: {', '.join(sorted(TASK_STATUS_VALUES))}." - ) - common_fields = { - "schema_version", - "task_id", - "status", - "objective", - "object_scope", - "outcome", - "started_at", - "ended_at", - "source_fragments", - } - allowed_fields = ( - common_fields - | {"exclusion_kind", "exclusion_reason"} - if status == "excluded" - else common_fields - | { - "objective_source_message_ids", - "outcome_source_message_ids", - "primary_deliverable", - } - ) - if require_frozen: - allowed_fields.add("inventory_sha256") - normalized: dict[str, Any] = { - "schema_version": TASK_INVENTORY_SCHEMA_VERSION, - } - normalized["task_id"] = task_id - normalized["status"] = status - normalized["objective"] = optional_text( - row.get("objective"), field=f"task[{task_id}].objective" - ) - normalized["object_scope"] = optional_text( - row.get("object_scope"), field=f"task[{task_id}].object_scope" - ) - normalized["outcome"] = optional_text( - row.get("outcome"), field=f"task[{task_id}].outcome" - ) - normalized["started_at"] = isoformat( - parse_datetime( - require_string(row.get("started_at"), f"task[{task_id}].started_at"), - field=f"task {task_id} started_at", - ) - ) - normalized["ended_at"] = isoformat( - parse_datetime( - require_string(row.get("ended_at"), f"task[{task_id}].ended_at"), - field=f"task {task_id} ended_at", - ) - ) - if parse_datetime(normalized["started_at"]) > parse_datetime(normalized["ended_at"]): - raise AuditError(f"task[{task_id}] starts after it ends.") - - fragments = row.get("source_fragments") - if not isinstance(fragments, list) or not fragments: - raise AuditError(f"task[{task_id}].source_fragments must be a non-empty array.") - normalized_fragments: list[dict[str, Any]] = [] - for index, fragment in enumerate(fragments): - field = f"task[{task_id}].source_fragments[{index}]" - if not isinstance(fragment, dict): - raise AuditError(f"{field} must be an object.") - require_exact_keys( - fragment, - {"audit_id", "message_ids", "linkage"}, - field=field, - ) - current_audit_id = require_string(fragment.get("audit_id"), f"{field}.audit_id") - message_ids = string_id_list( - fragment.get("message_ids"), field=f"{field}.message_ids" - ) - if not message_ids: - raise AuditError(f"{field}.message_ids must not be empty.") - projected_fragment: dict[str, Any] = { - "audit_id": current_audit_id, - "message_ids": message_ids, - } - linkage = fragment.get("linkage") - if linkage is not None: - if not isinstance(linkage, dict): - raise AuditError(f"{field}.linkage must be an object.") - require_exact_keys( - linkage, - {"kind", "key"}, - field=f"{field}.linkage", - ) - kind = require_string(linkage.get("kind"), f"{field}.linkage.kind") - if kind not in LINKAGE_VALUES: - raise AuditError( - f"{field}.linkage.kind must be one of: {', '.join(sorted(LINKAGE_VALUES))}." - ) - projected_fragment["linkage"] = { - "kind": kind, - "key": require_string(linkage.get("key"), f"{field}.linkage.key"), - } - normalized_fragments.append(projected_fragment) - normalized["source_fragments"] = normalized_fragments - - for removed_field in ( - "task_type", - "sampling_order", - "saturation_signals", - "exposure", - "support", - "episode", - "read", - "reads", - "read_ids", - "read_attribution", - "effect", - "effects", - "effect_type", - "effect_judgment", - "effect_reason", - "decision_receipt", - ): - if removed_field in row: - raise AuditError( - f"task[{task_id}].{removed_field} does not belong in the " - "pure Task inventory." - ) - require_exact_keys( - row, - allowed_fields, - field=f"task[{task_id}] pure Task inventory", - ) - - if status == "excluded": - if any( - field in row - for field in ( - "objective_source_message_ids", - "outcome_source_message_ids", - "primary_deliverable", - ) - ): - raise AuditError( - f"Excluded task[{task_id}] must not contain clear-Task sources " - "or a primary deliverable." - ) - exclusion_kind = require_string( - row.get("exclusion_kind"), f"task[{task_id}].exclusion_kind" - ) - if exclusion_kind not in TASK_EXCLUSION_VALUES: - raise AuditError( - f"task[{task_id}].exclusion_kind must be one of: " - f"{', '.join(sorted(TASK_EXCLUSION_VALUES))}." - ) - normalized["exclusion_kind"] = exclusion_kind - normalized["exclusion_reason"] = require_string( - row.get("exclusion_reason"), f"task[{task_id}].exclusion_reason" - ) - tasks.append(normalized) - continue - - if normalized["objective"] is None or normalized["outcome"] is None: - raise AuditError( - f"Clear task[{task_id}] requires objective and outcome." - ) - if "exclusion_kind" in row or "exclusion_reason" in row: - raise AuditError( - f"Clear task[{task_id}] must not contain exclusion fields." - ) - assert normalized["objective"] is not None - if is_weak_task_fragment(normalized["objective"]): - raise AuditError( - f"Clear task[{task_id}].objective is only a continuation, status " - "prompt, or context-dependent fragment; merge it into its parent " - "episode or exclude it." - ) - normalized.update(validate_task_sources(row, task_id=task_id)) - tasks.append(normalized) - if not tasks: - raise AuditError("No Task inventory rows were provided.") - digest = task_inventory_digest(tasks) - if require_frozen: - if len(declared_digests) != 1: - raise AuditError( - "Frozen Task inventory rows must share exactly one inventory_sha256." - ) - declared_digest = next(iter(declared_digests)) - if declared_digest != digest: - raise AuditError( - "Frozen Task inventory digest does not match its Task rows." - ) - return tasks, digest - return tasks, None - - -def sorted_task_inventory( - tasks: Sequence[Mapping[str, Any]], -) -> list[dict[str, Any]]: - return [ - dict(task) - for task in sorted( - tasks, - key=lambda item: ( - item["status"] != "clear", - item["started_at"], - item["task_id"], - ), - ) - ] - - -def task_inventory_digest(tasks: Sequence[Mapping[str, Any]]) -> str: - payload = "\n".join( - json.dumps( - task, - ensure_ascii=False, - sort_keys=True, - separators=(",", ":"), - ) - for task in sorted_task_inventory(tasks) - ) - return hashlib.sha256(f"{payload}\n".encode("utf-8")).hexdigest() - - -def frozen_task_inventory( - tasks: Sequence[Mapping[str, Any]], - digest: str, -) -> list[dict[str, Any]]: - return [ - {**task, "inventory_sha256": digest} - for task in sorted_task_inventory(tasks) - ] - - -def load_read_attributions( - path: Path, - *, - tasks: Sequence[Mapping[str, Any]], - inventory_sha256: str, -) -> dict[str, dict[str, Any]]: - clear_task_ids = { - task["task_id"] for task in tasks if task["status"] == "clear" - } - rows: dict[str, dict[str, Any]] = {} - for row in iter_jsonl(path): - if row.get("schema_version") != TASK_ANALYSIS_SCHEMA_VERSION: - raise AuditError( - "Every Read attribution must use schema_version " - f"{TASK_ANALYSIS_SCHEMA_VERSION}." - ) - if row.get("inventory_sha256") != inventory_sha256: - raise AuditError( - "Every Read attribution must reference the frozen Task inventory." - ) - require_exact_keys( - row, - { - "schema_version", - "inventory_sha256", - "task_id", - "status", - "read_ids", - "reason", - }, - field="read_attribution", - ) - task_id = require_string(row.get("task_id"), "read_attribution.task_id") - if task_id not in clear_task_ids: - raise AuditError( - f"Read attribution references unknown clear Task {task_id}." - ) - if task_id in rows: - raise AuditError(f"Duplicate Read attribution for Task {task_id}.") - status = require_string( - row.get("status"), f"read_attribution[{task_id}].status" - ) - if status not in READ_STATUS_VALUES: - raise AuditError( - f"read_attribution[{task_id}].status must be observed or unresolved." - ) - read_ids = string_id_list( - row.get("read_ids", []), - field=f"read_attribution[{task_id}].read_ids", - ) - reason = optional_text( - row.get("reason"), field=f"read_attribution[{task_id}].reason" - ) - if status == "observed" and not read_ids: - raise AuditError( - f"Observed Read attribution for Task {task_id} requires read_ids." - ) - if status == "observed" and reason is not None: - raise AuditError( - f"Observed Read attribution for Task {task_id} must not include a reason." - ) - if status == "unresolved" and reason is None: - raise AuditError( - f"Unresolved Read attribution for Task {task_id} requires a reason." - ) - if status == "unresolved" and read_ids: - raise AuditError( - f"Unresolved Read attribution for Task {task_id} must not contain read_ids." - ) - rows[task_id] = { - "status": status, - "read_ids": read_ids, - "reason": reason, - } - missing = clear_task_ids - set(rows) - if missing: - raise AuditError( - "Missing Read attribution for clear Task(s): " - + ", ".join(sorted(missing)) - + "." - ) - return rows - - -def load_effect_judgments( - path: Path, - *, - tasks: Sequence[Mapping[str, Any]], - reads_by_task: Mapping[str, Mapping[str, Any]], - inventory_sha256: str, -) -> dict[str, dict[str, Any]]: - clear_task_ids = { - task["task_id"] for task in tasks if task["status"] == "clear" - } - rows: dict[str, dict[str, Any]] = {} - for row in iter_jsonl(path): - if row.get("schema_version") != TASK_ANALYSIS_SCHEMA_VERSION: - raise AuditError( - "Every Effect judgment must use schema_version " - f"{TASK_ANALYSIS_SCHEMA_VERSION}." - ) - if row.get("inventory_sha256") != inventory_sha256: - raise AuditError( - "Every Effect judgment must reference the frozen Task inventory." - ) - require_exact_keys( - row, - { - "schema_version", - "inventory_sha256", - "task_id", - "effects", - "effect_reason", - }, - field="effect_judgment", - ) - task_id = require_string(row.get("task_id"), "effect_judgment.task_id") - if task_id not in clear_task_ids: - raise AuditError( - f"Effect judgment references unknown clear Task {task_id}." - ) - if task_id in rows: - raise AuditError(f"Duplicate Effect judgment for Task {task_id}.") - raw_effects = row.get("effects") - if not isinstance(raw_effects, list): - raise AuditError( - f"effect_judgment[{task_id}].effects must be an array." - ) - effects: list[dict[str, Any]] = [] - for index, effect in enumerate(raw_effects): - field = f"effect_judgment[{task_id}].effects[{index}]" - if not isinstance(effect, dict): - raise AuditError(f"{field} must be an object.") - for removed_field in ( - "original_judgment", - "rubric", - "support", - "confidence", - ): - if removed_field in effect: - raise AuditError( - f"{field}.{removed_field} belongs to a superseded model." - ) - require_exact_keys( - effect, - { - "type", - "read_ids", - "choice_message_ids", - "outcome_message_id", - "summary", - }, - field=field, - ) - effect_type = require_string(effect.get("type"), f"{field}.type") - if effect_type not in EFFECT_VALUES: - raise AuditError( - f"{field}.type must be one of: " - f"{', '.join(sorted(EFFECT_VALUES))}." - ) - effect_reads = string_id_list( - effect.get("read_ids"), field=f"{field}.read_ids" - ) - choice_ids = string_id_list( - effect.get("choice_message_ids"), - field=f"{field}.choice_message_ids", - ) - if not effect_reads or not choice_ids: - raise AuditError( - f"{field} requires read_ids and choice_message_ids." - ) - effects.append( - { - "type": effect_type, - "read_ids": effect_reads, - "choice_message_ids": choice_ids, - "outcome_message_id": require_string( - effect.get("outcome_message_id"), - f"{field}.outcome_message_id", - ), - "summary": require_string( - effect.get("summary"), f"{field}.summary" - ), - } - ) - effect_reason = optional_text( - row.get("effect_reason"), - field=f"effect_judgment[{task_id}].effect_reason", - ) - if effects and effect_reason is not None: - raise AuditError( - f"Task {task_id} with Effects must not include effect_reason." - ) - if not effects and effect_reason is None: - raise AuditError( - f"Task {task_id} without an Effect requires effect_reason." - ) - if reads_by_task[task_id]["status"] == "unresolved" and effects: - raise AuditError( - f"Unresolved Read attribution for Task {task_id} cannot have Effects." - ) - rows[task_id] = { - "effects": effects, - "effect_reason": effect_reason, - } - missing = clear_task_ids - set(rows) - if missing: - raise AuditError( - "Missing Effect judgment for clear Task(s): " - + ", ".join(sorted(missing)) - + "." - ) - return rows - - -def timestamp_in_task(value: Any, *, start: datetime, end: datetime, field: str) -> datetime: - timestamp = parse_datetime(require_string(value, field), field=field) - if not start <= timestamp <= end: - raise AuditError(f"{field} is outside the Task window.") - return timestamp - - -def validate_task_inventory_refs( - candidates: Sequence[Mapping[str, Any]], - tasks: Sequence[dict[str, Any]], -) -> dict[str, dict[str, Any]]: - candidates_by_id = {candidate["audit_id"]: candidate for candidate in candidates} - messages_by_audit: dict[str, dict[str, Mapping[str, Any]]] = {} - for candidate in candidates: - current_audit_id = candidate["audit_id"] - visible_messages = candidate.get("visible_messages") - if not isinstance(visible_messages, list): - raise AuditError( - f"Task source {current_audit_id} requires visible_messages." - ) - message_index = { - message.get("message_id"): message - for message in visible_messages - if isinstance(message, dict) and isinstance(message.get("message_id"), str) - } - messages_by_audit[current_audit_id] = message_index - - identity_source_owners: dict[str, str] = {} - contexts: dict[str, dict[str, Any]] = {} - for task in tasks: - task_id = task["task_id"] - start = parse_datetime(task["started_at"], field=f"task {task_id} started_at") - end = parse_datetime(task["ended_at"], field=f"task {task_id} ended_at") - source_audits: set[str] = set() - source_message_ids: set[str] = set() - source_messages: dict[str, Mapping[str, Any]] = {} - linkages: set[tuple[str, str]] = set() - for index, fragment in enumerate(task["source_fragments"]): - field = f"task[{task_id}].source_fragments[{index}]" - current_audit_id = fragment["audit_id"] - candidate = candidates_by_id.get(current_audit_id) - if candidate is None: - raise AuditError( - f"{field} references an unauthorized Chat-Agent audit {current_audit_id}." - ) - source_audits.add(current_audit_id) - window = candidate["window"] - acquisition_start = ( - parse_datetime(window["start"], field=f"{field} acquisition start") - if isinstance(window.get("start"), str) - else None - ) - acquisition_end = parse_datetime( - require_string(window.get("end"), f"{field} acquisition end"), - field=f"{field} acquisition end", - ) - if end > acquisition_end or (acquisition_start is not None and start < acquisition_start): - raise AuditError(f"task[{task_id}] is outside its authorized acquisition window.") - for message_id in fragment["message_ids"]: - if message_id in source_message_ids: - raise AuditError( - f"task[{task_id}] duplicates source message {message_id}." - ) - source_message_ids.add(message_id) - message = messages_by_audit[current_audit_id].get(message_id) - if message is None: - raise AuditError( - f"{field} references message {message_id} outside its authorized Chat." - ) - source_messages[message_id] = message - timestamp_in_task( - message.get("created_at"), - start=start, - end=end, - field=f"task {task_id} source message {message_id}", - ) - linkage = fragment.get("linkage") - if isinstance(linkage, dict): - linkages.add((linkage["kind"], linkage["key"])) - if len(source_audits) > 1: - if len(linkages) != 1 or any( - "linkage" not in fragment for fragment in task["source_fragments"] - ): - raise AuditError( - f"Cross-Chat task[{task_id}] requires one explicit shared linkage." - ) - elif linkages: - raise AuditError( - f"Single-Chat task[{task_id}] must not claim cross-Chat linkage." - ) - - if task["status"] == "excluded": - continue - objective_source_ids = task["objective_source_message_ids"] - outcome_source_ids = task["outcome_source_message_ids"] - identity_source_ids = { - *objective_source_ids, - *outcome_source_ids, - } - for message_id in identity_source_ids: - previous = identity_source_owners.setdefault(message_id, task_id) - if previous != task_id: - raise AuditError( - f"Task identity source {message_id} is copied across incompatible " - f"Tasks {previous} and {task_id}." - ) - if not identity_source_ids.issubset(source_message_ids): - unknown = ", ".join(sorted(identity_source_ids - source_message_ids)) - raise AuditError( - f"task[{task_id}] identity sources reference messages outside its source " - f"fragments: {unknown}." - ) - source_agent_ids = { - candidates_by_id[audit]["chat"]["source_agent_id"] - for audit in source_audits - } - if len(source_agent_ids) != 1: - raise AuditError( - f"task[{task_id}] source fragments must belong to one audited Agent." - ) - source_agent_id = next(iter(source_agent_ids)) - objective_source_messages = { - message_id: source_messages[message_id] - for message_id in objective_source_ids - } - concrete_objective_ids = [ - message_id - for message_id, message in objective_source_messages.items() - if isinstance(message.get("sender_id"), str) - and bool(message["sender_id"].strip()) - and isinstance(message.get("content"), str) - and not is_weak_task_fragment(message["content"]) - ] - if not concrete_objective_ids: - raise AuditError( - f"task[{task_id}] requires at least one concrete objective-source " - "message; weak continuations and context-only prompts cannot be " - "normalized into a clear objective." - ) - invalid_outcome_source_ids = [ - message_id - for message_id in outcome_source_ids - if source_messages[message_id].get("sender_id") != source_agent_id - or not isinstance( - source_messages[message_id].get("content"), str - ) - or not source_messages[message_id]["content"].strip() - ] - if invalid_outcome_source_ids: - raise AuditError( - f"task[{task_id}] requires every outcome source to be a " - "non-empty current-Agent message." - ) - objective_source_times = [ - parse_datetime( - source_messages[message_id]["created_at"], - field=f"task {task_id} objective source {message_id}", - ) - for message_id in objective_source_ids - ] - outcome_source_times = [ - parse_datetime( - source_messages[message_id]["created_at"], - field=f"task {task_id} outcome source {message_id}", - ) - for message_id in outcome_source_ids - ] - episode_started_at = min(objective_source_times) - episode_ended_at = max(outcome_source_times) - if min(outcome_source_times) <= max(objective_source_times): - raise AuditError( - f"task[{task_id}] outcome sources must strictly follow objective sources." - ) - if start != episode_started_at or end != episode_ended_at: - raise AuditError( - f"task[{task_id}] started_at and ended_at must equal its first " - "objective source and final outcome source." - ) - for message_id, message in source_messages.items(): - message_time = parse_datetime( - message["created_at"], - field=f"task {task_id} source message {message_id}", - ) - if not episode_started_at <= message_time <= episode_ended_at: - raise AuditError( - f"task[{task_id}] source message {message_id} is outside its " - "established episode." - ) - contexts[task_id] = { - "start": start, - "end": end, - "source_audits": source_audits, - "source_message_ids": source_message_ids, - "source_messages": source_messages, - "source_agent_id": source_agent_id, - } - return contexts - - -def candidate_reads( - candidates: Sequence[Mapping[str, Any]], -) -> dict[str, tuple[str, Mapping[str, Any]]]: - reads: dict[str, tuple[str, Mapping[str, Any]]] = {} - for candidate in candidates: - for read in candidate["reads"]: - read_id = read.get("read_id") - if isinstance(read_id, str): - if read_id in reads: - raise AuditError( - f"Read ID {read_id} is not globally unique." - ) - reads[read_id] = (candidate["audit_id"], read) - return reads - - -def validate_read_refs( - candidates: Sequence[Mapping[str, Any]], - tasks: Sequence[Mapping[str, Any]], - reads_by_task: Mapping[str, Mapping[str, Any]], - contexts: Mapping[str, Mapping[str, Any]], -) -> dict[str, tuple[str, Mapping[str, Any]]]: - reads = candidate_reads(candidates) - read_owners: dict[str, str] = {} - for task in tasks: - if task["status"] != "clear": - continue - task_id = task["task_id"] - context = contexts[task_id] - for read_id in reads_by_task[task_id]["read_ids"]: - item = reads.get(read_id) - if item is None: - raise AuditError( - f"Read attribution for Task {task_id} references unknown Read {read_id}." - ) - current_audit_id, read = item - if current_audit_id not in context["source_audits"]: - raise AuditError( - f"Task {task_id} Read {read_id} is outside its source Chat fragments." - ) - read_started_at = timestamp_in_task( - read.get("timestamp"), - start=context["start"], - end=context["end"], - field=f"task {task_id} read {read_id} timestamp", - ) - read_completed_at = timestamp_in_task( - read.get("completed_at"), - start=context["start"], - end=context["end"], - field=f"task {task_id} read {read_id} completion", - ) - if read_started_at >= read_completed_at: - raise AuditError( - f"Task {task_id} Read {read_id} must complete after it starts." - ) - previous = read_owners.setdefault(read_id, task_id) - if previous != task_id: - raise AuditError( - f"Read {read_id} is copied across incompatible Tasks " - f"{previous} and {task_id}." - ) - return reads - - -def candidate_choices( - candidates: Sequence[Mapping[str, Any]], -) -> dict[str, tuple[str, Mapping[str, Any]]]: - choices: dict[str, tuple[str, Mapping[str, Any]]] = {} - for candidate in candidates: - for choice in candidate["visible_choice_candidates"]: - message_id = choice.get("message_id") - if isinstance(message_id, str): - if message_id in choices: - raise AuditError( - f"Choice message ID {message_id} is not globally unique." - ) - choices[message_id] = (candidate["audit_id"], choice) - return choices - - -def validate_effect_refs( - candidates: Sequence[Mapping[str, Any]], - tasks: Sequence[Mapping[str, Any]], - reads_by_task: Mapping[str, Mapping[str, Any]], - effects_by_task: Mapping[str, Mapping[str, Any]], - contexts: Mapping[str, Mapping[str, Any]], - reads: Mapping[str, tuple[str, Mapping[str, Any]]], -) -> None: - choices = candidate_choices(candidates) - choice_owners: dict[str, tuple[str, int]] = {} - for task in tasks: - if task["status"] != "clear": - continue - task_id = task["task_id"] - context = contexts[task_id] - observed_reads = set(reads_by_task[task_id]["read_ids"]) - for effect_index, effect in enumerate( - effects_by_task[task_id]["effects"] - ): - effect_name = f"Effect {effect_index + 1} in Task {task_id}" - if not set(effect["read_ids"]).issubset(observed_reads): - raise AuditError( - f"{effect_name} references Reads outside its attributed Reads." - ) - read_times = [ - parse_datetime( - reads[read_id][1]["completed_at"], - field=f"task {task_id} read {read_id} completion", - ) - for read_id in effect["read_ids"] - ] - choice_times: list[datetime] = [] - for message_id in effect["choice_message_ids"]: - if message_id not in context["source_message_ids"]: - raise AuditError( - f"{effect_name} choice {message_id} is outside its " - "Task source fragments." - ) - item = choices.get(message_id) - if item is None: - raise AuditError( - f"{effect_name} references unknown same-Agent choice " - f"{message_id}." - ) - current_audit_id, choice = item - if current_audit_id not in context["source_audits"]: - raise AuditError( - f"{effect_name} choice {message_id} is outside its " - "source Chats." - ) - choice_time = timestamp_in_task( - choice.get("created_at"), - start=context["start"], - end=context["end"], - field=f"task {task_id} choice {message_id}", - ) - choice_times.append(choice_time) - owner = (task_id, effect_index) - previous = choice_owners.setdefault(message_id, owner) - if previous != owner: - raise AuditError( - f"Choice {message_id} is reused across independent Effects." - ) - outcome_message_id = effect["outcome_message_id"] - outcome_message = context["source_messages"].get(outcome_message_id) - if ( - outcome_message is None - or outcome_message.get("sender_id") - != context["source_agent_id"] - or not isinstance(outcome_message.get("content"), str) - or not outcome_message["content"].strip() - ): - raise AuditError( - f"{effect_name} requires a non-empty current-Agent outcome " - "message inside its Task." - ) - outcome_time = timestamp_in_task( - outcome_message.get("created_at"), - start=context["start"], - end=context["end"], - field=f"task {task_id} Effect outcome {outcome_message_id}", - ) - if max(read_times) >= min(choice_times): - raise AuditError( - f"{effect_name} requires every Read to complete before its " - "earliest choice." - ) - if outcome_time < max([*read_times, *choice_times]): - raise AuditError( - f"{effect_name} outcome precedes a cited Read completion or choice." - ) - - -def table_row(columns: Sequence[Any]) -> str: - return "| " + " | ".join(str(column).replace("|", "\\|") for column in columns) + " |" - - -def report_cell(value: Any) -> str: - if value is None: - return "—" - return re.sub(r"\s+", " ", str(value)).strip() or "—" - - -def independent_effect_id(effect: Mapping[str, Any]) -> str: - identity = { - "effect": effect["type"], - "read_ids": sorted(effect["read_ids"]), - "choice_message_ids": sorted(effect["choice_message_ids"]), - "outcome_message_id": effect["outcome_message_id"], - } - digest = hashlib.sha256( - json.dumps(identity, ensure_ascii=False, sort_keys=True).encode("utf-8") - ).hexdigest()[:20] - return f"effect-{digest}" - - -def build_task_evidence( - tasks: Sequence[Mapping[str, Any]], - *, - inventory_sha256: str, - reads_by_task: Mapping[str, Mapping[str, Any]], - effects_by_task: Mapping[str, Mapping[str, Any]], -) -> list[dict[str, Any]]: - evidence: list[dict[str, Any]] = [] - for task in sorted_task_inventory(tasks): - row = {**task, "inventory_sha256": inventory_sha256} - if task["status"] == "clear": - task_id = task["task_id"] - row["read"] = dict(reads_by_task[task_id]) - row["effects"] = [ - { - **effect, - "effect_id": independent_effect_id(effect), - } - for effect in effects_by_task[task_id]["effects"] - ] - row["effect_reason"] = effects_by_task[task_id]["effect_reason"] - evidence.append(row) - return evidence - - -def render_report( - candidates: Sequence[Mapping[str, Any]], - tasks: Sequence[Mapping[str, Any]], - generated_at: datetime, - reviewed_baseline: Mapping[str, Any] | None = None, -) -> str: - runtime_providers = {candidate_runtime_provider(row) for row in candidates} - if len(runtime_providers) != 1: - raise AuditError("One report cannot mix runtime evidence providers.") - runtime_provider = next(iter(runtime_providers)) - provider_note = ( - f"This audit uses the existing local `{runtime_provider}` evidence adapter." - if runtime_provider in SUPPORTED_EVIDENCE_PROVIDERS - else f"Historical `{runtime_provider}` evidence is not supported; affected Tasks remain unresolved." - ) - - clear_tasks = [task for task in tasks if task["status"] == "clear"] - excluded_tasks = [task for task in tasks if task["status"] == "excluded"] - observed_read_tasks = [ - task for task in clear_tasks if task["read"]["status"] == "observed" - ] - unresolved_read_tasks = [ - task for task in clear_tasks if task["read"]["status"] == "unresolved" - ] - effect_tasks = [task for task in clear_tasks if task["effects"]] - observed_without_effect_tasks = [ - task - for task in observed_read_tasks - if not task["effects"] - ] - effects = [ - effect - for task in clear_tasks - for effect in task["effects"] - ] - effect_counts = Counter(effect["type"] for effect in effects) - exclusion_counts = Counter( - task["exclusion_kind"] for task in excluded_tasks - ) - - if len(observed_read_tasks) + len(unresolved_read_tasks) != len(clear_tasks): - raise AuditError("Task read counts do not conserve clear Tasks.") - if len(effect_tasks) > len(observed_read_tasks): - raise AuditError("Effect Task count cannot exceed observed Read Task count.") - if len(effect_tasks) + len(observed_without_effect_tasks) != len( - observed_read_tasks - ): - raise AuditError( - "Effect and no-Effect counts do not conserve observed Read Tasks." - ) - if len(effects) < len(effect_tasks): - raise AuditError("Effect count cannot be less than Effect Task count.") - if sum(effect_counts.values()) != len(effects): - raise AuditError("Effect distribution does not conserve Effects.") - - mapped_audits = sum(1 for row in candidates if row["mapped_trace_files"]) - chat_message_counts: dict[str, int] = {} - mapped_chat_ids: set[str] = set() - for row in candidates: - chat_id = row["chat"]["chat_id"] - chat_message_counts[chat_id] = max( - chat_message_counts.get(chat_id, 0), - int(row["chat"]["message_count"]), - ) - if row["mapped_trace_files"]: - mapped_chat_ids.add(chat_id) - message_count = sum(chat_message_counts.values()) - gap_counts = Counter(gap for row in candidates for gap in row["coverage_gaps"]) - attempt_status_counts = Counter( - { - status: sum( - row["collector_diagnostics"]["attempt_status_counts"][status] - for row in candidates - ) - for status in READ_ATTEMPT_STATUSES - } - ) - attempt_total = sum( - row["collector_diagnostics"]["in_window_tree_read_attempts"] - for row in candidates - ) - attempt_reason_counts: Counter[str] = Counter() - for row in candidates: - attempt_reason_counts.update( - row["collector_diagnostics"]["attempt_reason_counts"] - ) - if sum(attempt_status_counts.values()) != attempt_total: - raise AuditError( - "Collector read-attempt status counts do not conserve the in-window total." - ) - bounded_starts = { - row["window"]["start"] - for row in candidates - if isinstance(row["window"].get("start"), str) - } - window_start = min(bounded_starts) if bounded_starts else "unbounded" - window_end = max(row["window"]["end"] for row in candidates) - - lines = [ - "# Context Tree Value Audit", - "", - f"Generated: {isoformat(generated_at)}", - f"Acquisition bound: {window_start} – {window_end}", - "", - "## Outcome", - "", - table_row(["Measure", "Count"]), - table_row(["---", "---:"]), - table_row(["Clear Tasks", len(clear_tasks)]), - table_row(["Excluded Tasks", len(excluded_tasks)]), - table_row(["Read observed Tasks", len(observed_read_tasks)]), - table_row(["Read unresolved Tasks", len(unresolved_read_tasks)]), - table_row(["Effect Tasks", len(effect_tasks)]), - table_row(["Effects", len(effects)]), - table_row( - [ - "Observed Read without Effect", - len(observed_without_effect_tasks), - ] - ), - "", - "This is a sampled evidence report, not causal proof, ROI, or a global effectiveness rate.", - "", - "An unresolved Read is unknown evidence coverage, not proof that the Tree was unused or had no value. Receipt absence is also unknown.", - "", - "Every available clear Task in the authorized acquisition bound is reported. There is no minimum Task quota or saturation gate; sample size limits the scope of the conclusion rather than whether the report may exist.", - "", - "## Effect Distribution", - "", - table_row(["Effect", "Effects"]), - table_row(["---", "---:"]), - ] - for effect_type in ("confirmed", "constrained", "redirected", "conflicted"): - lines.append(table_row([effect_type, effect_counts[effect_type]])) - - if excluded_tasks: - lines.extend( - [ - "", - "## Exclusion Distribution", - "", - table_row(["Exclusion kind", "Candidates"]), - table_row(["---", "---:"]), - ] - ) - for exclusion_kind in sorted(TASK_EXCLUSION_VALUES): - if exclusion_counts[exclusion_kind]: - lines.append( - table_row( - [exclusion_kind, exclusion_counts[exclusion_kind]] - ) - ) - - lines.extend( - [ - "", - "## Task Results", - "", - table_row( - [ - "Task ID", - "Objective", - "Object scope", - "Primary deliverable", - "Outcome", - "Read", - "Effect", - "Evidence summary", - ] - ), - table_row( - [ - "---", - "---", - "---", - "---", - "---", - "---", - "---", - "---", - ] - ), - ] - ) - for task in sorted( - clear_tasks, key=lambda item: (item["started_at"], item["task_id"]) - ): - task_effects = task["effects"] - effect_label = ( - ", ".join(effect["type"] for effect in task_effects) - if task_effects - else "not judged" - if task["read"]["status"] == "unresolved" - else "none" - ) - evidence_summary = ( - "; ".join(effect["summary"] for effect in task_effects) - if task_effects - else task["effect_reason"] - ) - lines.append( - table_row( - [ - f"`{task['task_id']}`", - report_cell(task["objective"]), - report_cell(task["object_scope"]), - report_cell(task["primary_deliverable"]), - report_cell(task["outcome"]), - task["read"]["status"], - effect_label, - evidence_summary, - ] - ) - ) - if not clear_tasks: - lines.append( - table_row(["None", "—", "—", "—", "—", "—", "—", "—"]) - ) - lines.append("") - - if tasks: - inventory_sha256 = tasks[0]["inventory_sha256"] - lines.extend( - [ - "## Frozen Task Inventory", - "", - f"Inventory digest: `sha256:{inventory_sha256}`.", - "", - ] - ) - if clear_tasks: - for index, task in enumerate( - sorted( - clear_tasks, - key=lambda item: (item["started_at"], item["task_id"]), - ), - start=1, - ): - lines.extend( - [ - f"### {index}. `{task['task_id']}`", - "", - "- Objective sources: " - + ", ".join( - f"`{message_id}`" - for message_id in task[ - "objective_source_message_ids" - ] - ), - "- Outcome sources: " - + ", ".join( - f"`{message_id}`" - for message_id in task[ - "outcome_source_message_ids" - ] - ), - "", - ] - ) - else: - lines.extend( - [ - "No clear Tasks were reconstructed; the digest still binds " - "the excluded-candidate inventory.", - "", - ] - ) - - if effects: - lines.extend(["## Effect Details", ""]) - effect_number = 0 - for task in sorted( - clear_tasks, - key=lambda item: (item["started_at"], item["task_id"]), - ): - for effect in task["effects"]: - effect_number += 1 - lines.extend( - [ - f"### {effect_number}. `{effect['effect_id']}`", - "", - f"- Task: `{task['task_id']}`", - f"- Type: `{effect['type']}`", - "- Reads: " - + ", ".join( - f"`{read_id}`" for read_id in effect["read_ids"] - ), - "- Choices: " - + ", ".join( - f"`{message_id}`" - for message_id in effect["choice_message_ids"] - ), - f"- Outcome message: `{effect['outcome_message_id']}`", - f"- Summary: {report_cell(effect['summary'])}", - "", - ] - ) - - if excluded_tasks: - lines.extend( - [ - "## Excluded Tasks", - "", - table_row(["Candidate", "Exclusion kind", "Observed scope", "Reason"]), - table_row(["---", "---", "---", "---"]), - ] - ) - for task in sorted( - excluded_tasks, key=lambda item: (item["started_at"], item["task_id"]) - ): - lines.append( - table_row( - [ - f"`{task['task_id']}`", - f"`{task['exclusion_kind']}`", - report_cell(task["object_scope"]), - report_cell(task["exclusion_reason"]), - ] - ) - ) - lines.append("") - - if reviewed_baseline is not None: - baseline_effects = reviewed_baseline["effect_counts"] - anchor = reviewed_baseline["evidence_anchor"] - lines.extend( - [ - "## Separately Reviewed Historical Baseline", - "", - "This hash-anchored baseline is shown separately and is not merged into the current rerun.", - "", - table_row(["Measure", "Count"]), - table_row(["---", "---:"]), - table_row(["Reviewed clear Tasks", reviewed_baseline["clear_tasks"]]), - table_row(["Reviewed effect Tasks", reviewed_baseline["effect_tasks"]]), - table_row(["Reviewed Effects", reviewed_baseline["effects"]]), - "", - table_row(["Effect", "Reviewed Effects"]), - table_row(["---", "---:"]), - ] - ) - for effect_type in ( - "confirmed", - "constrained", - "redirected", - "conflicted", - ): - lines.append(table_row([effect_type, baseline_effects[effect_type]])) - lines.extend( - [ - "", - f"Evidence anchor: `{anchor['artifact_id']}` / " - f"`sha256:{anchor['sha256']}`; reviewed at " - f"`{reviewed_baseline['reviewed_at']}`.", - "", - ] - ) - - lines.extend( - [ - "## Coverage", - "", - table_row(["Measure", "Count"]), - table_row(["---", "---:"]), - table_row(["Authorized Chats acquired", len(chat_message_counts)]), - table_row(["Authorized Chat-Agent audit units", len(candidates)]), - table_row(["Visible messages", message_count]), - table_row(["Runtime evidence provider", runtime_provider]), - table_row(["Chats mapped to local runtime evidence", len(mapped_chat_ids)]), - table_row(["Audit units mapped to local runtime evidence", mapped_audits]), - table_row(["In-window Tree-read attempts", attempt_total]), - table_row(["Frozen Task inventory rows", len(tasks)]), - "", - "### Tree-read grammar conservation", - "", - table_row(["Attempt classification", "Count"]), - table_row(["---", "---:"]), - *[ - table_row([status, attempt_status_counts[status]]) - for status in READ_ATTEMPT_STATUSES - ], - table_row(["Total", attempt_total]), - "", - "The four attempt classes conserve every in-window call whose payload referenced the bound Tree. Accepted classes describe command-shape recovery; unresolved and rejected attempts remain coverage gaps and never become a negative Read.", - "", - "Historical collection is best-effort. Missing reads and absent receipts do not establish that a Task did not use Context Tree.", - "", - "### Coverage gaps", - "", - ] - ) - if attempt_reason_counts: - lines.extend( - [ - "### Unresolved/rejected attempt reasons", - "", - table_row(["Reason", "Calls"]), - table_row(["---", "---:"]), - ] - ) - for reason, count in sorted(attempt_reason_counts.items()): - lines.append(table_row([reason, count])) - lines.append("") - if not gap_counts: - lines.append("- None recorded.") - else: - for gap, count in sorted(gap_counts.items()): - lines.append(f"- `{gap}`: {count} audit unit(s)") - - lines.extend( - [ - "", - "## Rule and Boundaries", - "", - "A clear Task is one continuous work episode in which the audited Agent accepted a concrete objective and produced an independently judgeable outcome. Scope and primary deliverable may clarify the boundary but are not mandatory. Short continuations, status prompts, context-dependent questions, and phases or corrections of one delivery stay inside their parent episode. Task reconstruction is frozen before Read attribution and Effect analysis.", - "", - "Read is `observed` only when the recovered Task-window evidence contains attributable Tree content; otherwise it is `unresolved` with a reason.", - "", - "Each Task may have zero or more Effects. Every Effect exists only when a real relevant normal-content Read precedes one distinct choice, the later same-Agent choice or outcome reasonably shows `confirmed`, `constrained`, `redirected`, or `conflicted` influence, and no more direct evidence fully explains the result. Otherwise the Effect list is empty with a short reason.", - "", - "A decision receipt may support this judgment but cannot create an Effect by itself.", - "", - f"{provider_note} The audit remains read-only and limited to one explicitly authorized Agent, one workspace, and one bound Tree.", - "", - ] - ) - return "\n".join(lines) - - -def validate_tree_source_snapshot( - value: Any, - *, - field: str, -) -> dict[str, Any]: - if value is None: - return { - "status": "unavailable", - "reason": "legacy_candidate_without_source_snapshot", - } - if not isinstance(value, Mapping): - raise AuditError(f"{field} must be an object.") - status = value.get("status") - if status == "unavailable": - return { - "status": status, - "reason": require_string(value.get("reason"), f"{field}.reason"), - } - if status != "local_default_branch": - raise AuditError(f"{field}.status is invalid.") - branch = require_string(value.get("branch"), f"{field}.branch") - commit = require_string(value.get("commit"), f"{field}.commit").lower() - if re.fullmatch(r"[0-9a-f]{40}", commit) is None: - raise AuditError(f"{field} has an invalid commit.") - return { - "status": status, - "branch": branch, - "commit": commit, - } - - -def validate_read_tree_source( - value: Any, - snapshot: Mapping[str, Any], - read: Mapping[str, Any], - *, - field: str, -) -> dict[str, Any]: - if not isinstance(value, Mapping): - raise AuditError(f"{field} must be an object.") - status = value.get("status") - if status == "unverified_source": - return {"status": status} - if status != "default_branch_match": - raise AuditError(f"{field}.status is invalid.") - if snapshot.get("status") != "local_default_branch": - raise AuditError( - f"{field} cannot claim a default-branch match without a recorded snapshot." - ) - branch = require_string(value.get("branch"), f"{field}.branch") - commit = require_string(value.get("commit"), f"{field}.commit").lower() - node_path = require_string(value.get("node_path"), f"{field}.node_path") - if ( - branch != snapshot.get("branch") - or commit != snapshot.get("commit") - or node_path not in read.get("node_paths", []) - ): - raise AuditError( - f"{field} does not match the candidate's collection-time Tree snapshot." - ) - return { - "status": status, - "branch": branch, - "commit": commit, - "node_path": node_path, - } - - -def validate_task_source_candidate( - value: Mapping[str, Any], - workspace_identity: WorkspaceIdentity, -) -> dict[str, Any]: - """Validate only authority and work messages needed for Stage 1.""" - if value.get("schema_version") != SCHEMA_VERSION: - raise AuditError( - f"Every candidate must use schema_version {SCHEMA_VERSION}." - ) - chat = value.get("chat") - window = value.get("window") - visible_messages = value.get("visible_messages") - if ( - not isinstance(chat, dict) - or not isinstance(window, dict) - or not isinstance(visible_messages, list) - ): - raise AuditError( - "Every Task-source candidate requires chat, window, and visible_messages." - ) - chat_id = require_uuid(chat.get("chat_id"), "candidate.chat.chat_id") - source_agent_id = require_uuid( - chat.get("source_agent_id"), - f"candidate[{chat_id}].chat.source_agent_id", - ) - source_agent = require_string( - chat.get("source_agent"), - f"candidate[{chat_id}].chat.source_agent", - ) - if ( - source_agent_id != workspace_identity.agent_id - or source_agent != workspace_identity.agent_name - ): - raise AuditError( - f"Candidate {chat_id} does not match the authorized Agent workspace identity." - ) - expected_audit_id = audit_id(chat_id, source_agent_id) - if value.get("audit_id") != expected_audit_id: - raise AuditError( - f"Candidate {chat_id} audit_id does not match its Chat and Agent UUIDs." - ) - authorization = validate_authorization( - chat.get("authorization"), - f"candidate[{expected_audit_id}].chat.authorization", - ) - if "authorization_context" in chat: - raise AuditError( - f"Candidate {expected_audit_id} must not contain authorization_context." - ) - message_count = chat.get("message_count") - if not isinstance(message_count, int) or isinstance(message_count, bool): - raise AuditError( - f"Candidate {expected_audit_id} must contain integer message_count." - ) - start_value = window.get("start") - parsed_window_start = None - if start_value is not None: - parsed_window_start = parse_datetime( - require_string( - start_value, - f"candidate[{expected_audit_id}].window.start", - ), - field=f"candidate {expected_audit_id} window start", - ) - end_value = require_string( - window.get("end"), - f"candidate[{expected_audit_id}].window.end", - ) - parsed_window_end = parse_datetime( - end_value, - field=f"candidate {expected_audit_id} window end", - ) - if ( - parsed_window_start is not None - and parsed_window_start > parsed_window_end - ): - raise AuditError( - f"Candidate {expected_audit_id} window starts after it ends." - ) - if message_count != len(visible_messages): - raise AuditError( - f"Candidate {expected_audit_id} message_count does not match visible_messages." - ) - seen_message_ids: set[str] = set() - projected_messages: list[dict[str, Any]] = [] - for message in visible_messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("message_id"), str) - or not isinstance(message.get("created_at"), str) - or not isinstance(message.get("sender_id"), str) - or not isinstance(message.get("content"), str) - ): - raise AuditError( - f"Candidate {expected_audit_id} contains an invalid visible message." - ) - message_id = message["message_id"] - if message_id in seen_message_ids: - raise AuditError( - f"Candidate {expected_audit_id} contains duplicate visible message " - f"{message_id}." - ) - seen_message_ids.add(message_id) - message_time = parse_datetime( - message["created_at"], - field=f"candidate {expected_audit_id} message {message_id}", - ) - if ( - parsed_window_start is not None - and message_time < parsed_window_start - ) or message_time > parsed_window_end: - raise AuditError( - f"Candidate {expected_audit_id} message {message_id} is outside " - "its acquisition window." - ) - projected_messages.append( - { - field: message.get(field) - for field in ( - "message_id", - "created_at", - "sender_id", - "sender_kind", - "content", - ) - } - ) - return { - "schema_version": SCHEMA_VERSION, - "audit_id": expected_audit_id, - "chat": { - "chat_id": chat_id, - "authorization": authorization, - "source_agent_id": source_agent_id, - }, - "window": { - "start": start_value, - "end": end_value, - }, - "visible_messages": projected_messages, - } - - -def validate_report_candidate( - value: Mapping[str, Any], - workspace_identity: WorkspaceIdentity, -) -> dict[str, Any]: - if value.get("schema_version") != SCHEMA_VERSION: - raise AuditError( - f"Every candidate must use schema_version {SCHEMA_VERSION}." - ) - runtime_provider = candidate_runtime_provider(value) - chat = value.get("chat") - if not isinstance(chat, dict): - raise AuditError("Every candidate must contain a chat object.") - chat_id = require_uuid(chat.get("chat_id"), "candidate.chat.chat_id") - source_agent_id = require_uuid( - chat.get("source_agent_id"), - f"candidate[{chat_id}].chat.source_agent_id", - ) - source_agent = require_string( - chat.get("source_agent"), - f"candidate[{chat_id}].chat.source_agent", - ) - if ( - source_agent_id != workspace_identity.agent_id - or source_agent != workspace_identity.agent_name - ): - raise AuditError( - f"Candidate {chat_id} does not match the authorized Agent workspace identity." - ) - expected_audit_id = audit_id(chat_id, source_agent_id) - if value.get("audit_id") != expected_audit_id: - raise AuditError( - f"Candidate {chat_id} audit_id does not match its Chat and Agent UUIDs." - ) - authorization = validate_authorization( - chat.get("authorization"), - f"candidate[{expected_audit_id}].chat.authorization", - ) - if "authorization_context" in chat: - raise AuditError( - f"Candidate {expected_audit_id} must not contain authorization_context." - ) - if not isinstance(chat.get("title"), str) or not isinstance( - chat.get("message_count"), int - ): - raise AuditError( - f"Candidate {expected_audit_id} must contain a title and integer message_count." - ) - candidate_status = value.get("candidate_status") - if candidate_status not in {"candidate", "outside_candidate_set"}: - raise AuditError( - f"Candidate {expected_audit_id} has an invalid candidate_status." - ) - expected_tree_id = tree_identity(workspace_identity) - if value.get("tree_identity") != expected_tree_id: - raise AuditError( - f"Candidate {expected_audit_id} does not match the workspace-bound Tree." - ) - tree_source_snapshot = validate_tree_source_snapshot( - value.get("tree_source_snapshot"), - field=f"candidate[{expected_audit_id}].tree_source_snapshot", - ) - window = value.get("window") - if not isinstance(window, dict): - raise AuditError(f"Candidate {expected_audit_id} must contain a window.") - parsed_window_start = None - if window.get("start") is not None: - parsed_window_start = parse_datetime( - require_string(window.get("start"), f"candidate[{expected_audit_id}].window.start"), - field=f"candidate {expected_audit_id} window start", - ) - parsed_window_end = parse_datetime( - require_string(window.get("end"), f"candidate[{expected_audit_id}].window.end"), - field=f"candidate {expected_audit_id} window end", - ) - if parsed_window_start is not None and parsed_window_start > parsed_window_end: - raise AuditError(f"Candidate {expected_audit_id} window starts after it ends.") - mapped_traces = value.get("mapped_trace_files") - collector_diagnostics = value.get("collector_diagnostics") - reads = value.get("reads") - visible_messages = value.get("visible_messages") - choices = value.get("visible_choice_candidates") - mentions = value.get("visible_tree_mentions") - coverage_gaps = value.get("coverage_gaps") - if not all( - isinstance(item, list) - for item in ( - mapped_traces, - reads, - visible_messages, - choices, - mentions, - coverage_gaps, - ) - ): - raise AuditError( - f"Candidate {expected_audit_id} evidence and coverage fields must be arrays." - ) - if not isinstance(collector_diagnostics, dict): - raise AuditError( - f"Candidate {expected_audit_id} must contain collector_diagnostics." - ) - attempt_total = collector_diagnostics.get("in_window_tree_read_attempts") - attempt_status_counts = collector_diagnostics.get("attempt_status_counts") - attempt_reason_counts = collector_diagnostics.get("attempt_reason_counts") - if ( - not isinstance(attempt_total, int) - or attempt_total < 0 - or not isinstance(attempt_status_counts, dict) - or set(attempt_status_counts) != set(READ_ATTEMPT_STATUSES) - or any( - not isinstance(attempt_status_counts[status], int) - or attempt_status_counts[status] < 0 - for status in READ_ATTEMPT_STATUSES - ) - or sum(attempt_status_counts.values()) != attempt_total - or not isinstance(attempt_reason_counts, dict) - or any( - not isinstance(reason, str) - or not reason - or not isinstance(count, int) - or count <= 0 - for reason, count in attempt_reason_counts.items() - ) - or sum(attempt_reason_counts.values()) - != ( - attempt_status_counts["unresolved_opaque"] - + attempt_status_counts["rejected_unsafe"] - ) - ): - raise AuditError( - f"Candidate {expected_audit_id} collector diagnostics do not conserve read attempts." - ) - if chat["message_count"] != len(visible_messages): - raise AuditError( - f"Candidate {expected_audit_id} message_count does not match visible_messages." - ) - if not all( - isinstance(item, str) and item.startswith("trace-") - for item in mapped_traces - ): - raise AuditError( - f"Candidate {expected_audit_id} has an invalid mapped trace identity." - ) - for read in reads: - if not isinstance(read, dict): - raise AuditError( - f"Candidate {expected_audit_id} contains a malformed read." - ) - if ( - read.get("reader_agent_id") != workspace_identity.agent_id - or read.get("tree_identity") != expected_tree_id - or read.get("runtime_provider", "codex") != runtime_provider - or not isinstance(read.get("read_id"), str) - or not isinstance(read.get("node_paths"), list) - or not isinstance(read.get("passage"), str) - or read.get("success") is not True - ): - raise AuditError( - f"Candidate {expected_audit_id} contains a read outside its Agent/Tree scope." - ) - read_mode = read.get("read_mode") - components = read.get("read_components") - if read_mode is not None and read_mode not in { - "isolated", - "read_only_composite", - }: - raise AuditError( - f"Candidate {expected_audit_id} contains an invalid read_mode." - ) - expected_attribution = ( - "exact" if read_mode in {None, "isolated"} else "aggregate" - ) - if read.get("output_attribution", expected_attribution) != expected_attribution: - raise AuditError( - f"Candidate {expected_audit_id} contains invalid output_attribution." - ) - if not isinstance(read.get("auxiliary_output_possible", False), bool): - raise AuditError( - f"Candidate {expected_audit_id} contains invalid auxiliary output metadata." - ) - nested_call_index = read.get("nested_call_index") - if nested_call_index is not None and ( - not isinstance(nested_call_index, int) - or nested_call_index < 0 - ): - raise AuditError( - f"Candidate {expected_audit_id} contains invalid nested_call_index." - ) - if components is not None: - if not isinstance(components, list) or not components: - raise AuditError( - f"Candidate {expected_audit_id} contains invalid read_components." - ) - component_paths: set[str] = set() - for component in components: - if ( - not isinstance(component, dict) - or not isinstance(component.get("reader"), str) - or not isinstance(component.get("node_paths"), list) - or not component["node_paths"] - or any( - not isinstance(path, str) - for path in component["node_paths"] - ) - ): - raise AuditError( - f"Candidate {expected_audit_id} contains a malformed read component." - ) - component_paths.update(component["node_paths"]) - if component_paths != set(read["node_paths"]): - raise AuditError( - f"Candidate {expected_audit_id} read_components do not conserve node_paths." - ) - visible_messages_by_id: dict[str, Mapping[str, Any]] = {} - for message in visible_messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("message_id"), str) - or not isinstance(message.get("created_at"), str) - ): - raise AuditError( - f"Candidate {expected_audit_id} contains an invalid visible message." - ) - message_id = message["message_id"] - if message_id in visible_messages_by_id: - raise AuditError( - f"Candidate {expected_audit_id} contains duplicate visible message {message_id}." - ) - visible_messages_by_id[message_id] = message - receipt = message.get("decision_receipt") - if receipt is not None and normalize_context_decision(receipt) != receipt: - raise AuditError( - f"Candidate {expected_audit_id} contains an invalid decision receipt." - ) - for read in reads: - if read.get("tree_source") is None: - read["tree_source"] = read_tree_source( - read, - workspace_identity.bound_tree_root, - tree_source_snapshot, - ) - else: - read["tree_source"] = validate_read_tree_source( - read["tree_source"], - tree_source_snapshot, - read, - field=( - f"candidate[{expected_audit_id}].read[{read['read_id']}]." - "tree_source" - ), - ) - choice_ids: set[str] = set() - for message in choices: - if ( - not isinstance(message, dict) - or message.get("sender_id") != workspace_identity.agent_id - or not isinstance(message.get("message_id"), str) - or not isinstance(message.get("created_at"), str) - ): - raise AuditError( - f"Candidate {expected_audit_id} contains an invalid visible choice." - ) - if message["message_id"] in choice_ids: - raise AuditError( - f"Candidate {expected_audit_id} contains duplicate visible choices." - ) - choice_ids.add(message["message_id"]) - source_message = visible_messages_by_id.get(message["message_id"]) - if source_message is None or any( - source_message.get(field) != message.get(field) - for field in ("created_at", "sender_id", "content") - ): - raise AuditError( - f"Candidate {expected_audit_id} choice is not an exact authorized visible message." - ) - return dict(value) - - -def build_task_source( - candidates: Sequence[Mapping[str, Any]], -) -> list[dict[str, Any]]: - return [ - { - "schema_version": SCHEMA_VERSION, - "audit_id": candidate["audit_id"], - "chat": { - "chat_id": candidate["chat"]["chat_id"], - "authorization": candidate["chat"]["authorization"], - "source_agent_id": candidate["chat"]["source_agent_id"], - }, - "window": dict(candidate["window"]), - "visible_messages": [ - { - field: message.get(field) - for field in ( - "message_id", - "created_at", - "sender_id", - "sender_kind", - "content", - ) - } - for message in candidate["visible_messages"] - ], - } - for candidate in sorted( - candidates, - key=lambda item: item["audit_id"], - ) - ] - - -def validate_task_source( - value: Mapping[str, Any], - workspace_identity: WorkspaceIdentity, -) -> dict[str, Any]: - if value.get("schema_version") != SCHEMA_VERSION: - raise AuditError( - f"Every Task source must use schema_version {SCHEMA_VERSION}." - ) - require_exact_keys( - value, - { - "schema_version", - "audit_id", - "chat", - "window", - "visible_messages", - }, - field="task_source", - ) - chat = value.get("chat") - window = value.get("window") - visible_messages = value.get("visible_messages") - if ( - not isinstance(chat, dict) - or not isinstance(window, dict) - or not isinstance(visible_messages, list) - ): - raise AuditError( - "Every Task source requires chat, window, and visible_messages." - ) - require_exact_keys( - chat, - {"chat_id", "authorization", "source_agent_id"}, - field="task_source.chat", - ) - require_exact_keys( - window, - {"start", "end"}, - field="task_source.window", - ) - chat_id = require_uuid(chat.get("chat_id"), "task_source.chat.chat_id") - source_agent_id = require_uuid( - chat.get("source_agent_id"), - f"task_source[{chat_id}].chat.source_agent_id", - ) - if source_agent_id != workspace_identity.agent_id: - raise AuditError( - f"Task source {chat_id} does not match the authorized Agent." - ) - expected_audit_id = audit_id(chat_id, source_agent_id) - if value.get("audit_id") != expected_audit_id: - raise AuditError( - f"Task source {chat_id} audit_id does not match its Chat and Agent." - ) - authorization = validate_authorization( - chat.get("authorization"), - f"task_source[{expected_audit_id}].chat.authorization", - ) - start_value = window.get("start") - if start_value is not None: - parse_datetime( - require_string( - start_value, - f"task_source[{expected_audit_id}].window.start", - ), - field=f"task source {expected_audit_id} window start", - ) - parse_datetime( - require_string( - window.get("end"), - f"task_source[{expected_audit_id}].window.end", - ), - field=f"task source {expected_audit_id} window end", - ) - seen_message_ids: set[str] = set() - projected_messages: list[dict[str, Any]] = [] - for message in visible_messages: - if ( - not isinstance(message, dict) - or not isinstance(message.get("message_id"), str) - or not isinstance(message.get("created_at"), str) - or not isinstance(message.get("sender_id"), str) - or not isinstance(message.get("content"), str) - ): - raise AuditError( - f"Task source {expected_audit_id} contains an invalid visible message." - ) - require_exact_keys( - message, - { - "message_id", - "created_at", - "sender_id", - "sender_kind", - "content", - }, - field=f"task_source[{expected_audit_id}].visible_message", - ) - message_id = message["message_id"] - if message_id in seen_message_ids: - raise AuditError( - f"Task source {expected_audit_id} contains duplicate message " - f"{message_id}." - ) - seen_message_ids.add(message_id) - parse_datetime( - message["created_at"], - field=f"task source {expected_audit_id} message {message_id}", - ) - projected_messages.append( - { - field: message.get(field) - for field in ( - "message_id", - "created_at", - "sender_id", - "sender_kind", - "content", - ) - } - ) - return { - "schema_version": SCHEMA_VERSION, - "audit_id": expected_audit_id, - "chat": { - "chat_id": chat_id, - "authorization": authorization, - "source_agent_id": source_agent_id, - }, - "window": { - "start": start_value, - "end": window["end"], - }, - "visible_messages": projected_messages, - } - - -def project_task_source(args: argparse.Namespace) -> None: - workspace_identity = parse_agent_workspace(args.agent_workspace) - artifact_root = resolve_artifact_root(args.artifact_root, workspace_identity) - candidates_path = artifact_path( - artifact_root, args.candidates, field="--candidates", must_exist=True - ) - output_path = artifact_path( - artifact_root, args.output, field="--output", must_exist=False - ) - require_distinct_paths( - { - "--candidates": candidates_path, - "--output": output_path, - } - ) - candidates = [ - validate_task_source_candidate(candidate, workspace_identity) - for candidate in iter_jsonl(candidates_path) - ] - if not candidates: - raise AuditError("No candidate records were provided.") - if len({candidate["audit_id"] for candidate in candidates}) != len(candidates): - raise AuditError("Candidate evidence contains duplicate Chat-Agent audit rows.") - write_jsonl(output_path, build_task_source(candidates)) - - -def freeze_task_inventory(args: argparse.Namespace) -> None: - workspace_identity = parse_agent_workspace(args.agent_workspace) - artifact_root = resolve_artifact_root(args.artifact_root, workspace_identity) - task_source_path = artifact_path( - artifact_root, - args.task_source, - field="--task-source", - must_exist=True, - ) - draft_path = artifact_path( - artifact_root, - args.task_inventory_draft, - field="--task-inventory-draft", - must_exist=True, - ) - output_path = artifact_path( - artifact_root, - args.task_inventory_output, - field="--task-inventory-output", - must_exist=False, - ) - require_distinct_paths( - { - "--task-source": task_source_path, - "--task-inventory-draft": draft_path, - "--task-inventory-output": output_path, - } - ) - task_sources = [ - validate_task_source(row, workspace_identity) - for row in iter_jsonl(task_source_path) - ] - if not task_sources: - raise AuditError("No Task source records were provided.") - if len({row["audit_id"] for row in task_sources}) != len(task_sources): - raise AuditError("Task source contains duplicate Chat-Agent audit rows.") - tasks, _ = load_task_inventory(draft_path, require_frozen=False) - validate_task_inventory_refs(task_sources, tasks) - digest = task_inventory_digest(tasks) - write_jsonl(output_path, frozen_task_inventory(tasks, digest)) - print(f"task_inventory_sha256={digest}") - - -def finalize_report(args: argparse.Namespace) -> None: - workspace_identity = parse_agent_workspace(args.agent_workspace) - artifact_root = resolve_artifact_root(args.artifact_root, workspace_identity) - candidates_path = artifact_path( - artifact_root, args.candidates, field="--candidates", must_exist=True - ) - task_inventory_path = artifact_path( - artifact_root, - args.task_inventory, - field="--task-inventory", - must_exist=True, - ) - read_attributions_path = artifact_path( - artifact_root, - args.read_attributions, - field="--read-attributions", - must_exist=True, - ) - effect_judgments_path = artifact_path( - artifact_root, - args.effect_judgments, - field="--effect-judgments", - must_exist=True, - ) - evidence_path = artifact_path( - artifact_root, args.evidence_output, field="--evidence-output", must_exist=False - ) - report_path = artifact_path( - artifact_root, args.report_output, field="--report-output", must_exist=False - ) - reviewed_baseline_path = ( - artifact_path( - artifact_root, - args.reviewed_baseline, - field="--reviewed-baseline", - must_exist=True, - ) - if args.reviewed_baseline - else None - ) - paths = { - "--candidates": candidates_path, - "--task-inventory": task_inventory_path, - "--read-attributions": read_attributions_path, - "--effect-judgments": effect_judgments_path, - "--evidence-output": evidence_path, - "--report-output": report_path, - } - if reviewed_baseline_path is not None: - paths["--reviewed-baseline"] = reviewed_baseline_path - require_distinct_paths(paths) - candidates = [ - validate_report_candidate(candidate, workspace_identity) - for candidate in iter_jsonl(candidates_path) - ] - if not candidates: - raise AuditError("No candidate records were provided.") - if len({candidate["audit_id"] for candidate in candidates}) != len(candidates): - raise AuditError("Candidate evidence contains duplicate Chat-Agent audit rows.") - tasks, inventory_sha256 = load_task_inventory( - task_inventory_path, - require_frozen=True, - ) - assert inventory_sha256 is not None - contexts = validate_task_inventory_refs(candidates, tasks) - reads_by_task = load_read_attributions( - read_attributions_path, - tasks=tasks, - inventory_sha256=inventory_sha256, - ) - reads = validate_read_refs( - candidates, - tasks, - reads_by_task, - contexts, - ) - effects_by_task = load_effect_judgments( - effect_judgments_path, - tasks=tasks, - reads_by_task=reads_by_task, - inventory_sha256=inventory_sha256, - ) - validate_effect_refs( - candidates, - tasks, - reads_by_task, - effects_by_task, - contexts, - reads, - ) - evidence = build_task_evidence( - tasks, - inventory_sha256=inventory_sha256, - reads_by_task=reads_by_task, - effects_by_task=effects_by_task, - ) - reviewed_baseline = ( - load_reviewed_baseline(reviewed_baseline_path) - if reviewed_baseline_path is not None - else None - ) - - generated_at = parse_datetime(args.generated_at, field="--generated-at") if args.generated_at else datetime.now(timezone.utc) - write_jsonl(evidence_path, evidence) - write_text( - report_path, - f"{render_report(candidates, evidence, generated_at, reviewed_baseline)}\n", - ) - - -def build_parser() -> argparse.ArgumentParser: - parser = argparse.ArgumentParser( - description=( - "Read-only Context Tree Value Audit collector with deterministic " - "Task freeze, Read attribution, Effect validation, and reporting." - ) - ) - subparsers = parser.add_subparsers(dest="command", required=True) - - export_parser = subparsers.add_parser("export-chats", help="Export only the Chats named by an authorization scope.") - export_parser.add_argument( - "--artifact-root", - required=True, - help="Exact triggering-agent artifact directory; every input and output must stay inside it.", - ) - export_parser.add_argument( - "--scope", - required=True, - help="Scope JSON for one explicitly authorized Agent or its exact authorized Chats.", - ) - export_parser.add_argument("--output", required=True, help="Destination Chat JSONL.") - export_parser.add_argument( - "--days", - type=int, - help="Optional acquisition lookback bound in days; every available Task in the bound is reported.", - ) - export_parser.add_argument("--now", help="Fixed RFC 3339 window end for reproducible runs.") - export_parser.add_argument( - "--agent-workspace", - required=True, - help="Exact AGENT_UUID=/absolute/workspace identity for the one scoped Agent.", - ) - export_parser.add_argument( - "--first-tree-bin", - help=( - "First Tree CLI executable (default: FIRST_TREE_BIN, then safe " - "discovery of first-tree or first-tree-staging)." - ), - ) - export_parser.set_defaults(handler=export_chats) - - collect_parser = subparsers.add_parser( - "collect", - help="Pair authorized Chat records with local runtime evidence.", - ) - collect_parser.add_argument( - "--artifact-root", - required=True, - help="Exact triggering-agent artifact directory; every input and output must stay inside it.", - ) - collect_parser.add_argument("--chats", required=True, help="Authorized Chat JSONL from export-chats.") - collect_parser.add_argument("--output", required=True, help="Destination candidate evidence JSONL.") - collect_parser.add_argument( - "--days", - type=int, - help="Optional acquisition lookback bound in days; every available Task in the bound is reported.", - ) - collect_parser.add_argument("--now", help="Fixed RFC 3339 window end for reproducible runs.") - collect_parser.add_argument( - "--trace-root", - help="Optional provider-native evidence root; defaults are selected by runtime.", - ) - collect_parser.add_argument( - "--runtime-provider", - choices=sorted(RUNTIME_PROVIDER_VALUES), - help=( - "Optional equality assertion for the required FIRST_TREE_PROVIDER " - "runtime identity." - ), - ) - collect_parser.add_argument( - "--agent-workspace", - required=True, - help="Exact AGENT_UUID=/absolute/workspace identity for the one scoped Agent.", - ) - collect_parser.add_argument( + add_feed_options(facts) + facts.add_argument( "--tree-root", - required=True, - help="Exact Context Tree root bound in the scoped workspace identity.", - ) - collect_parser.add_argument( - "--max-passage-chars", - type=int, - default=24000, - help="Maximum stored tool-output characters per read (default: 24000).", - ) - collect_parser.set_defaults(handler=collect_evidence) - - task_source_parser = subparsers.add_parser( - "task-source", - help="Project message-only evidence for pure Task reconstruction.", - ) - task_source_parser.add_argument( - "--artifact-root", - required=True, - help="Exact triggering-agent artifact directory.", - ) - task_source_parser.add_argument( - "--agent-workspace", - required=True, - help="Exact AGENT_UUID=/absolute/workspace identity.", - ) - task_source_parser.add_argument( - "--candidates", - required=True, - help="Candidate JSONL from collect.", - ) - task_source_parser.add_argument( - "--output", - required=True, - help="Destination message-only Task source JSONL.", - ) - task_source_parser.set_defaults(handler=project_task_source) - - freeze_parser = subparsers.add_parser( - "freeze-tasks", - help="Validate and freeze a pure Task inventory before Read analysis.", - ) - freeze_parser.add_argument( - "--artifact-root", - required=True, - help="Exact triggering-agent artifact directory.", - ) - freeze_parser.add_argument( - "--agent-workspace", - required=True, - help="Exact AGENT_UUID=/absolute/workspace identity.", - ) - freeze_parser.add_argument( - "--task-source", - required=True, - help="Message-only Task source JSONL.", - ) - freeze_parser.add_argument( - "--task-inventory-draft", - required=True, - help="Pure Task inventory draft JSONL.", - ) - freeze_parser.add_argument( - "--task-inventory-output", - required=True, - help="Destination frozen Task inventory JSONL.", - ) - freeze_parser.set_defaults(handler=freeze_task_inventory) - - report_parser = subparsers.add_parser( - "report", - help="Validate frozen Task, Read, and Effect artifacts and render the report.", - ) - report_parser.add_argument( - "--artifact-root", - required=True, - help="Exact triggering-agent artifact directory; every input and output must stay inside it.", - ) - report_parser.add_argument( - "--agent-workspace", - required=True, - help="Exact AGENT_UUID=/absolute/workspace identity for the one scoped Agent.", - ) - report_parser.add_argument("--candidates", required=True, help="Candidate JSONL from collect.") - report_parser.add_argument( - "--task-inventory", - required=True, - help="Frozen Task inventory JSONL from freeze-tasks.", - ) - report_parser.add_argument( - "--read-attributions", - required=True, - help="Read attribution JSONL tied to the frozen Task inventory.", - ) - report_parser.add_argument( - "--effect-judgments", - required=True, - help="Effect judgment JSONL tied to the frozen Task inventory.", - ) - report_parser.add_argument( - "--reviewed-baseline", - help=( - "Optional one-row JSONL aggregate for separately reviewed historical " - "positive Task cases; it is rendered separately from the current rerun." - ), + help="Bound Context Tree root; enables the no-observed-read node list and pins Tree identity.", + ) + facts.add_argument("--json", action="store_true", help="Emit structured facts instead of Markdown.") + facts.set_defaults(handler=command_facts) + + sample = sub.add_parser("sample", help="Draw a random read sample and assemble case material.") + add_feed_options(sample) + sample.add_argument("--tree-root", required=True, help="Bound Context Tree root.") + sample.add_argument("--size", type=int, default=40, help="Sample size (default 40).") + sample.add_argument("--seed", type=int, default=0, help="Sampling seed for reproducibility.") + sample.add_argument("--max-content-chars", type=int, default=8000, help="Per-node content cap.") + sample.set_defaults(handler=command_sample) + + report = sub.add_parser("report", help="Validate judgments and render the final report.") + add_feed_options(report) + report.add_argument( + "--tree-root", + help="Bound Context Tree root; enables the no-observed-read node list and pins Tree identity.", ) - report_parser.add_argument("--evidence-output", required=True, help="Destination final evidence JSONL.") - report_parser.add_argument("--report-output", required=True, help="Destination Markdown report.") - report_parser.add_argument("--generated-at", help="Fixed RFC 3339 report timestamp for reproducible runs.") - report_parser.set_defaults(handler=finalize_report) + report.add_argument("--sample", required=True, help="Sample JSON produced by `sample`.") + report.add_argument("--judgments", required=True, help="Judgment JSON array authored by the analyst.") + report.set_defaults(handler=command_report) return parser def main(argv: Sequence[str] | None = None) -> int: os.umask(0o077) - parser = build_parser() - args = parser.parse_args(argv) + args = build_parser().parse_args(argv) try: - if getattr(args, "max_passage_chars", 1) <= 0: - raise AuditError("--max-passage-chars must be greater than zero.") + if getattr(args, "size", 1) <= 0: + raise AuditError("--size must be greater than zero.") args.handler(args) except AuditError as error: print(f"error: {error}", file=sys.stderr) diff --git a/tests/test_context_tree_value_audit.py b/tests/test_context_tree_value_audit.py index f8e00ec..7d331b5 100644 --- a/tests/test_context_tree_value_audit.py +++ b/tests/test_context_tree_value_audit.py @@ -1,5503 +1,455 @@ +"""Deterministic tests for the Context Tree value audit. + +The audit's job is to be honest about a lossy feed, so most of what is worth +testing is what it refuses to claim: unobserved-node lists that ignore search +coverage, adoption presented as a rate, effects that skipped the adversarial +pass, and influence numbers surviving a run whose judgments were mostly refuted. +""" + from __future__ import annotations import json -import os -import re -import shutil -import stat -import subprocess import sys -import tempfile import unittest +from datetime import datetime, timezone from pathlib import Path -from typing import Any - - -ROOT = Path(__file__).resolve().parents[1] -SKILL_ROOT = ROOT / "skills" / "context-tree-value-audit" -CLAUDE_SKILL_ROOT = ( - ROOT / "projections" / "claude" / "context-tree-value-audit" -) -SCRIPT = SKILL_ROOT / "scripts" / "context_tree_value_audit.py" -AGENT_ID = "55555555-5555-4555-8555-555555555555" -OTHER_AGENT_ID = "99999999-9999-4999-8999-999999999999" -CHAT_ID = "11111111-1111-4111-8111-111111111111" -SECOND_CHAT_ID = "22222222-2222-4222-8222-222222222222" -UNAUTHORIZED_CHAT_ID = "88888888-8888-4888-8888-888888888888" -MESSAGE_ID = "33333333-3333-4333-8333-333333333333" -SECOND_MESSAGE_ID = "44444444-4444-4444-8444-444444444444" -SECOND_OBJECTIVE_MESSAGE_ID = "aaaaaaaa-aaaa-4aaa-8aaa-aaaaaaaaaaaa" -ACCEPTANCE_MESSAGE_ID = "66666666-6666-4666-8666-666666666666" -ORG_ID = "77777777-7777-4777-8777-777777777777" -NOW = "2026-07-24T00:00:00Z" - - -def write_json(path: Path, value: Any) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text(json.dumps(value), encoding="utf-8") - - -def write_jsonl(path: Path, rows: list[Any]) -> None: - path.parent.mkdir(parents=True, exist_ok=True) - path.write_text( - "".join(f"{json.dumps(row, sort_keys=True)}\n" for row in rows), - encoding="utf-8", - ) - path.chmod(0o600) - - -def read_jsonl(path: Path) -> list[dict[str, Any]]: - return [ - json.loads(line) - for line in path.read_text(encoding="utf-8").splitlines() - if line.strip() - ] - - -def run_cli( - *arguments: str, - runtime_agent_id: str | None = AGENT_ID, - runtime_agent_slug: str | None = "fixture-agent", - runtime_provider: str | None = "codex", - first_tree_json: str | None = None, - environment_overrides: dict[str, str | None] | None = None, -) -> subprocess.CompletedProcess[str]: - environment = os.environ.copy() - for key, value in ( - ("FIRST_TREE_AGENT_ID", runtime_agent_id), - ("FIRST_TREE_AGENT_SLUG", runtime_agent_slug), - ("FIRST_TREE_PROVIDER", runtime_provider), - ("FIRST_TREE_CHAT_ID", CHAT_ID), - ("FIRST_TREE_JSON", first_tree_json), - ): - if value is None: - environment.pop(key, None) - else: - environment[key] = value - for key, value in (environment_overrides or {}).items(): - if value is None: - environment.pop(key, None) - else: - environment[key] = value - return subprocess.run( - [sys.executable, str(SCRIPT), *arguments], - capture_output=True, - check=False, - text=True, - env=environment, - ) - - -def write_workspace_identity( - workspace: Path, - tree_root: Path, - agent_id: str = AGENT_ID, - display_name: str = "Fixture Agent", -) -> None: - workspace.mkdir(parents=True, exist_ok=True) - write_json( - workspace / ".first-tree-workspace" / "identity.json", - { - "agentId": agent_id, - "displayName": display_name, - "type": "agent", - "contextTreePath": str(tree_root), - }, - ) - - -def context_block(chat_id: str) -> str: - return ( - '\n' - f'{{"chatId":"{chat_id}"}}\n' - "" - ) - - -def session_meta(workspace: Path) -> dict[str, Any]: - return { - "timestamp": "2026-07-22T10:00:00Z", - "type": "session_meta", - "payload": { - "cwd": str(workspace), - "originator": "first-tree", - "model_provider": "openai", - "source": "vscode", - }, - } - - -def context_row(chat_id: str) -> dict[str, Any]: - return { - "timestamp": "2026-07-22T10:01:00Z", - "type": "response_item", - "payload": { - "type": "message", - "role": "user", - "content": [{"type": "input_text", "text": context_block(chat_id)}], - }, - } - - -def context_mirror_row(chat_id: str) -> dict[str, Any]: - return { - "timestamp": "2026-07-22T10:01:00.001Z", - "type": "event_msg", - "payload": { - "type": "user_message", - "message": context_block(chat_id), - "images": [], - "local_images": [], - "text_elements": [], - }, +from tempfile import TemporaryDirectory + +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "skills" / "context-tree-value-audit" / "scripts")) + +import context_tree_value_audit as audit # noqa: E402 + +REPO = "https://github.com/example/context-tree" + + +def event( + event_id: str, + path: str, + *, + action: str = "read", + kind: str = "file", + chat: str = "chat-1", + at: str = "2026-07-01T10:00:00Z", + commit: str | None = None, + source: str = "claude_read_tool", + repo: str = REPO, + branch: str = "main", +) -> dict[str, object]: + row: dict[str, object] = { + "id": event_id, + "chatId": chat, + "action": action, + "source": source, + "targetKind": kind, + "targetPath": path, + "treeRepoUrl": repo, + "treeBranch": branch, + "createdAt": at, } + if commit: + row["treeHeadCommit"] = commit + return row -class RepositoryContractTests(unittest.TestCase): - def test_skill_is_an_explicit_only_umbrella(self) -> None: - self.assertFalse((ROOT / "skills" / "context-tree-insights").exists()) - skill = (SKILL_ROOT / "SKILL.md").read_text(encoding="utf-8") - openai = (SKILL_ROOT / "agents" / "openai.yaml").read_text(encoding="utf-8") - reference = (SKILL_ROOT / "references" / "evidence-schema.md").read_text( - encoding="utf-8" - ) - task_reference = ( - SKILL_ROOT / "references" / "task-analysis-schema.md" - ).read_text(encoding="utf-8") - readme = (ROOT / "README.md").read_text(encoding="utf-8") - version = (SKILL_ROOT / "VERSION").read_text(encoding="utf-8").strip() - - self.assertIn("name: context-tree-value-audit", skill) - self.assertIn("$context-tree-value-audit", skill) - self.assertIn("manual and read-only", skill) - self.assertIn("Do not trigger from an ordinary task", skill) - self.assertIn("all authorized Chats are not an eligible", skill) - self.assertIn("accepted_read_only_composite", skill) - self.assertIn("There is no\nminimum Task quota", skill) - claude_skill = (CLAUDE_SKILL_ROOT / "SKILL.md").read_text( - encoding="utf-8" - ) - self.assertIn("disable-model-invocation: true", claude_skill) - self.assertNotIn("disable-model-invocation", skill) - self.assertIn("allow_implicit_invocation: false", openai) - self.assertIn("explicit_agent", reference) - self.assertIn("explicit_chat", reference) - self.assertIn("Task, Read, and Effect Schema", task_reference) - self.assertIn('"schema_version": 4', task_reference) - self.assertIn('"status": "observed"', task_reference) - self.assertIn('"effects": []', task_reference) - self.assertIn("no more direct user instruction", task_reference) - self.assertIn("in_window_tree_read_attempts", reference) - self.assertIn("unresolved_opaque", reference) - self.assertIn("explicit_agent", skill) - self.assertNotIn("original_judgment", task_reference) - self.assertNotIn("sampling_order", task_reference) - self.assertIn("sampled evidence report", skill) - self.assertEqual("0.5.0", version) - self.assertIn(".skill-quarantine/", readme) - self.assertIn("diff -qr", readme) - self.assertIn("rollback", readme) - self.assertNotIn( - "/Users/", "\n".join((skill, openai, reference, task_reference)) - ) +def normalized(rows: list[dict[str, object]]) -> list[audit.IoEvent]: + return [audit.normalize_event(row) for row in rows] - def test_private_schema_v4_artifacts_are_ignored_and_rejected(self) -> None: - private_names = { - "task-source.jsonl", - "task-inventory-draft.jsonl", - "task-inventory.jsonl", - "read-attributions.jsonl", - "effect-judgments.jsonl", - } - gitignore = (ROOT / ".gitignore").read_text(encoding="utf-8") - validator = (ROOT / "scripts" / "validate_skill.py").read_text( - encoding="utf-8" - ) - for name in private_names: - self.assertIn(f"/{name}", gitignore) - self.assertIn(f'"{name}"', validator) - def test_install_layout_supports_codex_and_claude_upgrade_and_rollback( - self, - ) -> None: - readme = (ROOT / "README.md").read_text(encoding="utf-8") - self.assertIn( - 'projections/claude/context-tree-value-audit', - readme, - ) - self.assertIn( - '../../.agents/skills/context-tree-insights', - readme, - ) - with tempfile.TemporaryDirectory(prefix="context-tree-value-audit-install-") as raw: - root = Path(raw) +class NormalizationTests(unittest.TestCase): + def test_rejects_unknown_action_and_target_kind(self) -> None: + with self.assertRaises(audit.AuditError): + audit.normalize_event(event("e1", "a.md", action="delete")) + with self.assertRaises(audit.AuditError): + audit.normalize_event(event("e1", "a.md", kind="socket")) - fresh = root / "fresh" - fresh_agents = fresh / ".agents" / "skills" - fresh_claude = fresh / ".claude" / "skills" - fresh_agents.mkdir(parents=True) - fresh_claude.mkdir(parents=True) - fresh_payload = fresh_agents / "context-tree-value-audit" - fresh_projection = fresh_claude / "context-tree-value-audit" - shutil.copytree(SKILL_ROOT, fresh_payload) - shutil.copytree(CLAUDE_SKILL_ROOT, fresh_projection) - self.assertTrue((fresh_payload / "SKILL.md").is_file()) - self.assertTrue((fresh_projection / "SKILL.md").is_file()) - self.assertEqual( - (fresh_payload / "SKILL.md").resolve(), - ( - fresh_projection - / "../../../.agents/skills/context-tree-value-audit/SKILL.md" - ).resolve(), - ) + def test_keeps_only_a_valid_head_commit(self) -> None: + good = audit.normalize_event(event("e1", "a.md", commit="a" * 40)) + bogus = audit.normalize_event(event("e2", "a.md", commit="nope")) + self.assertEqual(good.tree_head_commit, "a" * 40) + self.assertIsNone(bogus.tree_head_commit) - upgrade = root / "upgrade" - upgrade_agents = upgrade / ".agents" / "skills" - upgrade_claude = upgrade / ".claude" / "skills" - quarantine = upgrade / ".skill-quarantine" - upgrade_agents.mkdir(parents=True) - upgrade_claude.mkdir(parents=True) - quarantine.mkdir(parents=True) - old_payload = upgrade_agents / "context-tree-insights" - old_payload.mkdir() - (old_payload / "SKILL.md").write_text( - "---\nname: context-tree-insights\n---\n", - encoding="utf-8", - ) - old_link = upgrade_claude / "context-tree-insights" - old_link.symlink_to("../../.agents/skills/context-tree-insights") - retired_payload = quarantine / "context-tree-insights" - retired_link = quarantine / "context-tree-insights.claude-link" - old_link.rename(retired_link) - old_payload.rename(retired_payload) - new_payload = upgrade_agents / "context-tree-value-audit" - new_projection = upgrade_claude / "context-tree-value-audit" - shutil.copytree(SKILL_ROOT, new_payload) - shutil.copytree(CLAUDE_SKILL_ROOT, new_projection) + def test_non_normal_content_is_classified_out(self) -> None: + self.assertFalse(audit.normalize_event(event("e1", "AGENTS.md")).is_normal_content) + self.assertFalse(audit.normalize_event(event("e2", "members/alice/NODE.md")).is_normal_content) + self.assertFalse(audit.normalize_event(event("e3", "raw-context/dump.md")).is_normal_content) + self.assertTrue(audit.normalize_event(event("e4", "system/cli/NODE.md")).is_normal_content) - self.assertFalse(old_payload.exists()) - self.assertFalse(old_link.exists()) - self.assertTrue((new_payload / "SKILL.md").is_file()) - self.assertTrue((new_projection / "SKILL.md").is_file()) - self.assertEqual( - (new_payload / "SKILL.md").resolve(), - ( - new_projection - / "../../../.agents/skills/context-tree-value-audit/SKILL.md" - ).resolve(), - ) - new_projection.rename( - quarantine / "context-tree-value-audit.failed-claude" +class AggregationTests(unittest.TestCase): + def test_separates_reads_writes_and_search_roots(self) -> None: + agg = audit.aggregate( + normalized( + [ + event("r1", "system/a.md"), + event("r2", "system/a.md", chat="chat-2"), + event("r3", "system", kind="directory", source="claude_read_tool"), + event("w1", "system/b.md", action="write", source="claude_write_tool"), + ] ) - new_payload.rename(quarantine / "context-tree-value-audit.failed") - retired_payload.rename(old_payload) - retired_link.rename(old_link) - self.assertFalse(new_payload.exists()) - self.assertFalse(new_projection.exists()) - self.assertTrue((old_payload / "SKILL.md").is_file()) - self.assertTrue((old_link / "SKILL.md").is_file()) - - -class DeterministicPipelineTests(unittest.TestCase): - def setUp(self) -> None: - self.temporary = tempfile.TemporaryDirectory(prefix="context-tree-value-audit-") - self.root = Path(self.temporary.name) - self.workspace = self.root / "workspace" - self.artifacts = self.workspace / "artifacts" - self.artifacts.mkdir(parents=True) - self.tree_root = self.root / "tree" - self.tree_file = self.tree_root / "system" / "architecture.md" - self.tree_file.parent.mkdir(parents=True) - self.tree_file.write_text( - "# Architecture\n\n## Decision\n\nChat history is the authoritative state.\n", - encoding="utf-8", - ) - self.second_tree_file = self.tree_root / "team-practice" / "dogfooding.md" - self.second_tree_file.parent.mkdir(parents=True) - self.second_tree_file.write_text( - "# Dogfooding\n\n## Decision\n\nUse First Tree in daily work.\n", - encoding="utf-8", - ) - subprocess.run( - ["git", "init", "-b", "main", str(self.tree_root)], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "config", "user.email", "tests@example.com"], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "config", "user.name", "Tests"], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "add", "."], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "commit", "-m", "seed"], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - "git", - "-C", - str(self.tree_root), - "remote", - "add", - "origin", - "https://github.com/acme/tree.git", - ], - check=True, - capture_output=True, - text=True, - ) - self.tree_commit = subprocess.run( - ["git", "-C", str(self.tree_root), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - subprocess.run( - [ - "git", - "-C", - str(self.tree_root), - "update-ref", - "refs/remotes/origin/main", - self.tree_commit, - ], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - "git", - "-C", - str(self.tree_root), - "symbolic-ref", - "refs/remotes/origin/HEAD", - "refs/remotes/origin/main", - ], - check=True, - capture_output=True, - text=True, - ) - write_workspace_identity(self.workspace, self.tree_root) - self.trace_root = self.root / "sessions" - self.trace_root.mkdir() - - def tearDown(self) -> None: - self.temporary.cleanup() - - def make_fake_first_tree( - self, - message_metadata: dict[str, Any] | None = None, - chat_last_message_at: Any = "2026-07-22T10:05:00Z", - resolved_agent_id: str = AGENT_ID, - resolved_agent_name: str = "fixture-agent", - ) -> tuple[Path, Path]: - binary = self.root / "fake-first-tree" - log = self.root / "first-tree-commands.log" - source = f"""#!/usr/bin/env python3 -import json -import os -import sys -from pathlib import Path - -args = sys.argv[1:] -with Path({str(log)!r}).open("a", encoding="utf-8") as handle: - handle.write(" ".join(args) + "\\n") - -if "agent" in args and "list" in args and "--remote" in args: - print(" NAME TYPE RUNTIME ORG CLIENT") - print(" fixture-human human codex {ORG_ID} —") - print( - " {resolved_agent_name} agent codex {ORG_ID} fixture-client" - ) - raise SystemExit(0) -if "agent" in args and "list" in args: - if os.environ.get("FIRST_TREE_JSON") == "1": - raise SystemExit(0) - print( - " {resolved_agent_name} runtime: codex uuid: {resolved_agent_id}", - file=sys.stderr, - ) - raise SystemExit(0) -if "agent" in args: - print(json.dumps({{"ok": False, "error": "unexpected agent command"}})) - raise SystemExit(9) -if "chat" in args and "list" in args: - data = {{ - "items": [{{ - "id": {CHAT_ID!r}, - "topic": "Fixture Chat", - "organizationId": {ORG_ID!r}, - "lastMessageAt": {chat_last_message_at!r} - }}], - "nextCursor": None - }} -elif "chat" in args and "history" in args: - data = {{ - "items": [{{ - "id": {MESSAGE_ID!r}, - "createdAt": "2026-07-22T10:05:00Z", - "senderId": {AGENT_ID!r}, - "content": "The Context Tree constraint keeps one authoritative state source.", - "metadata": {message_metadata!r} - }}], - "nextCursor": None - }} -else: - print(json.dumps({{"ok": False, "error": "unexpected command"}})) - raise SystemExit(8) -print(json.dumps({{"ok": True, "data": data}})) -""" - binary.write_text(source, encoding="utf-8") - binary.chmod(0o755) - return binary, log - - def export_scope( - self, - scope: dict[str, Any], - output_name: str = "chats.jsonl", - message_metadata: dict[str, Any] | None = None, - *, - days: int | None = 7, - runtime_agent_id: str | None = AGENT_ID, - runtime_agent_slug: str | None = "fixture-agent", - resolved_agent_id: str = AGENT_ID, - first_tree_json: str | None = None, - chat_last_message_at: Any = "2026-07-22T10:05:00Z", - ) -> subprocess.CompletedProcess[str]: - scope_path = self.artifacts / "scope.json" - output_path = self.artifacts / output_name - write_json(scope_path, scope) - binary, _ = self.make_fake_first_tree( - message_metadata, - chat_last_message_at, - resolved_agent_id, - runtime_agent_slug - if isinstance(runtime_agent_slug, str) - else "fixture-agent", - ) - arguments = [ - "export-chats", - "--artifact-root", - str(self.artifacts), - "--scope", - str(scope_path), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--first-tree-bin", - str(binary), - "--now", - NOW, - "--output", - str(output_path), - ] - if days is not None: - arguments.extend(["--days", str(days)]) - return run_cli( - *arguments, - runtime_agent_id=runtime_agent_id, - runtime_agent_slug=runtime_agent_slug, - first_tree_json=first_tree_json, - ) - - def export_scope_with_runtime( - self, - scope: dict[str, Any], - *, - runtime_agent_id: str | None = AGENT_ID, - runtime_agent_slug: str | None = "fixture-agent", - resolved_agent_id: str = AGENT_ID, - first_tree_json: str | None = None, - ) -> subprocess.CompletedProcess[str]: - return self.export_scope( - scope, - "runtime-identity.jsonl", - runtime_agent_id=runtime_agent_id, - runtime_agent_slug=runtime_agent_slug, - resolved_agent_id=resolved_agent_id, - first_tree_json=first_tree_json, ) - - def test_scope_is_single_agent_and_uses_one_explicit_mode(self) -> None: - mixed = { - "schema_version": 1, - "agents": [ - { - "name": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_agent", - } - ], - "chats": [ - { - "chat_id": CHAT_ID, - "agent": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_chat", - } - ], - } - result = self.export_scope(mixed) - self.assertEqual(2, result.returncode) - self.assertIn("exactly one mode", result.stderr) - - multiple_agents = { - "schema_version": 1, - "agents": [], - "chats": [ - { - "chat_id": CHAT_ID, - "agent": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_chat", - }, - { - "chat_id": UNAUTHORIZED_CHAT_ID, - "agent": "other-agent", - "agent_id": OTHER_AGENT_ID, - "authorization": "explicit_chat", - }, - ], - } - result = self.export_scope(multiple_agents) - self.assertEqual(2, result.returncode) - self.assertIn("one exact Agent", result.stderr) - - def test_exact_chat_export_only_uses_local_agent_list_for_identity(self) -> None: - scope = { - "schema_version": 1, - "agents": [], - "chats": [ - { - "chat_id": CHAT_ID, - "agent": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_chat", - } - ], - } - result = self.export_scope(scope) - self.assertEqual(0, result.returncode, result.stderr) - rows = read_jsonl(self.artifacts / "chats.jsonl") - self.assertEqual(1, len(rows)) - self.assertEqual("explicit_chat", rows[0]["authorization"]) - self.assertEqual(CHAT_ID, rows[0]["chat_id"]) - - commands = (self.root / "first-tree-commands.log").read_text(encoding="utf-8") - self.assertIn("agent list", commands) - self.assertIn(f"chat history {CHAT_ID}", commands) - self.assertNotIn("chat list", commands) - self.assertEqual(0o700, stat.S_IMODE(self.artifacts.stat().st_mode)) - self.assertEqual( - 0o600, - stat.S_IMODE((self.artifacts / "chats.jsonl").stat().st_mode), + self.assertEqual(agg.node_reads["system/a.md"], 2) + self.assertEqual(agg.chats_with_read, {"chat-1", "chat-2"}) + self.assertEqual(agg.node_writes["system/b.md"], 1) + # A directory event names a search root; it must not credit a node. + self.assertNotIn("system", agg.node_reads) + self.assertEqual(agg.searched_dirs, {"system"}) + + def test_repo_level_event_covers_every_node(self) -> None: + agg = audit.aggregate(normalized([event("r1", "/", kind="repo")])) + self.assertTrue(audit.covered_by_search("anything/deep/node.md", agg.searched_dirs)) + + +class UnobservedNodeTests(unittest.TestCase): + def _tree(self, root: Path) -> None: + for relative in ("NODE.md", "system/NODE.md", "system/cli.md", "goal/NODE.md", "members/a/NODE.md"): + path = root / relative + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("# node\n", encoding="utf-8") + + def test_lists_only_nodes_with_no_observed_read_and_no_search_above_them(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + agg = audit.aggregate(normalized([event("r1", "goal/NODE.md")])) + never = audit.unobserved_nodes(root, agg) + self.assertIn("system/NODE.md", never) + self.assertIn("system/cli.md", never) + self.assertNotIn("goal/NODE.md", never) + # member content is not normal decision content + self.assertNotIn("members/a/NODE.md", never) + + def test_a_searched_directory_protects_every_node_beneath_it(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + self._tree(root) + agg = audit.aggregate(normalized([event("r1", "system", kind="directory")])) + never = audit.unobserved_nodes(root, agg) + # `Grep`/`Glob` record only the search root, so nothing under it can + # be reported as unobserved without risking a delete recommendation + # for a node the search actually surfaced. + self.assertNotIn("system/NODE.md", never) + self.assertNotIn("system/cli.md", never) + self.assertIn("goal/NODE.md", never) + + +class SamplingTests(unittest.TestCase): + def test_samples_only_normal_file_reads_and_is_reproducible(self) -> None: + events = normalized( + [event(f"r{index}", f"system/n{index}.md") for index in range(20)] + + [ + event("dir", "system", kind="directory"), + event("agents", "AGENTS.md"), + event("write", "system/n0.md", action="write"), + ] ) + reads = [item for item in events if item.action == "read"] + first = audit.sample_reads(reads, 5, seed=7) + second = audit.sample_reads(reads, 5, seed=7) + self.assertEqual([item.event_id for item in first], [item.event_id for item in second]) + self.assertEqual(len(first), 5) + for item in first: + self.assertEqual(item.target_kind, "file") + self.assertTrue(item.is_normal_content) - def test_explicit_agent_export_uses_the_simple_explicit_scope(self) -> None: - scope = { - "schema_version": 1, - "agents": [ - { - "name": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_agent", - } - ], - "chats": [], - } - result = self.export_scope(scope) - self.assertEqual(0, result.returncode, result.stderr) - rows = read_jsonl(self.artifacts / "chats.jsonl") - self.assertEqual("explicit_agent", rows[0]["authorization"]) - self.assertNotIn("authorization_context", rows[0]) - commands = (self.root / "first-tree-commands.log").read_text(encoding="utf-8") - self.assertIn("agent list", commands) - self.assertIn("chat list", commands) - self.assertIn("--agent=fixture-agent", commands) - self.assertNotIn("Fixture Agent", commands) - - def test_scope_rejects_extra_authorization_context(self) -> None: - scope = { - "schema_version": 1, - "agents": [ - { - "name": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_agent", - } - ], - "chats": [], - "authorization_context": { - "chat_id": CHAT_ID, - }, - } - rejected = self.export_scope(scope) - self.assertEqual(2, rejected.returncode) - self.assertIn("complete authorization model", rejected.stderr) - - def test_runtime_slug_and_uuid_bind_the_cli_selector(self) -> None: - scope = { - "schema_version": 1, - "agents": [ - { - "name": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_agent", - } - ], - "chats": [], - } + def test_returns_everything_when_the_sample_exceeds_the_population(self) -> None: + reads = normalized([event("r1", "system/a.md"), event("r2", "system/b.md")]) + self.assertEqual(len(audit.sample_reads(reads, 50, seed=1)), 2) - missing_slug = self.export_scope_with_runtime( - scope, - runtime_agent_slug=None, - ) - self.assertEqual(2, missing_slug.returncode) - self.assertIn("FIRST_TREE_AGENT_SLUG", missing_slug.stderr) - inherited_json_mode = self.export_scope_with_runtime( - scope, - first_tree_json="1", - ) - self.assertEqual( - 0, - inherited_json_mode.returncode, - inherited_json_mode.stderr, - ) +class JudgmentValidationTests(unittest.TestCase): + CASES = { + "r1": {"read_id": "r1", "target_path": "system/a.md", "read_at": "2026-07-01T10:00:00Z"}, + "r2": {"read_id": "r2", "target_path": "system/b.md", "read_at": "2026-07-01T11:00:00Z"}, + } - for valid_slug in ( - "fixture_agent", - "fixture-agent-", - "fixture-agent_", - "-fixture-agent", - "_fixture-agent", - "-", - "_", - "--json", - "--help", - "a" * 64, - "a" * 100, - ): - accepted = self.export_scope_with_runtime( + def test_accepts_a_null_effect_and_a_refuted_effect(self) -> None: + rows = audit.validate_judgments( + [ + {"read_id": "r1", "effect": None}, { - **scope, - "agents": [{**scope["agents"][0], "name": valid_slug}], + "read_id": "r2", + "effect": {"type": "redirected", "summary": "Changed the approach."}, + "refuted": True, + "refutation": "The human asked for it first.", }, - runtime_agent_slug=valid_slug, - ) - self.assertEqual(0, accepted.returncode, accepted.stderr) - - commands = (self.root / "first-tree-commands.log").read_text(encoding="utf-8") - self.assertIn("--agent=--json", commands) - self.assertIn("--agent=--help", commands) - self.assertNotIn("--agent --json", commands) - self.assertNotIn("--agent --help", commands) - - for invalid_slug_value in ( - "Fixture Agent", - "fixture.agent", - "a" * 101, - ): - invalid_slug = self.export_scope_with_runtime( - scope, - runtime_agent_slug=invalid_slug_value, - ) - self.assertEqual(2, invalid_slug.returncode) - self.assertIn("lowercase CLI selector", invalid_slug.stderr) - - wrong_uuid = self.export_scope_with_runtime( - scope, - runtime_agent_id=OTHER_AGENT_ID, - ) - self.assertEqual(2, wrong_uuid.returncode) - self.assertIn("does not match", wrong_uuid.stderr) - - mismatched_selector = self.export_scope_with_runtime( - scope, - resolved_agent_id=OTHER_AGENT_ID, - ) - self.assertEqual(2, mismatched_selector.returncode) - self.assertIn("does not resolve", mismatched_selector.stderr) - - display_name_scope = json.loads(json.dumps(scope)) - display_name_scope["agents"][0]["name"] = "Fixture Agent" - wrong_name = self.export_scope_with_runtime(display_name_scope) - self.assertEqual(2, wrong_name.returncode) - self.assertIn("identity must match", wrong_name.stderr) - - def test_chat_continuing_after_window_end_is_still_exported(self) -> None: - scope = { - "schema_version": 1, - "agents": [ - { - "name": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_agent", - } ], - "chats": [], - } - result = self.export_scope( - scope, - "continued-chat.jsonl", - chat_last_message_at="2026-07-25T10:05:00Z", - ) - self.assertEqual(0, result.returncode, result.stderr) - rows = read_jsonl(self.artifacts / "continued-chat.jsonl") - self.assertEqual([CHAT_ID], [row["chat_id"] for row in rows]) - commands = (self.root / "first-tree-commands.log").read_text( - encoding="utf-8" + self.CASES, ) - self.assertIn(f"chat history {CHAT_ID}", commands) + self.assertIsNone(rows[0]["effect"]) + self.assertTrue(rows[1]["refuted"]) + self.assertEqual(rows[1]["refutation"], "The human asked for it first.") - for malformed_summary in ({"unexpected": "object"}, 42): - malformed = self.export_scope( - scope, - "continued-chat.jsonl", - chat_last_message_at=malformed_summary, - ) - self.assertEqual(0, malformed.returncode, malformed.stderr) - rows = read_jsonl(self.artifacts / "continued-chat.jsonl") - self.assertEqual([CHAT_ID], [row["chat_id"] for row in rows]) - - def test_receipt_projection_is_minimal_and_absence_stays_unknown(self) -> None: - scope = { - "schema_version": 1, - "agents": [], - "chats": [ - { - "chat_id": CHAT_ID, - "agent": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_chat", - } - ], - } - valid_receipt = { - "contextDecision": { - "version": 1, - "effect": "constrained", - "summary": " Kept one state source. ", - "ignored": "do not persist", - "evidence": [ - { - "repoUrl": "https://github.com/example/tree", - "commit": "a" * 40, - "nodePath": "system/architecture.md", - "heading": "Decision", - "ignored": "do not persist", - } + def test_a_claimed_effect_must_go_through_the_adversarial_pass(self) -> None: + with self.assertRaisesRegex(audit.AuditError, "adversarial pass"): + audit.validate_judgments( + [ + {"read_id": "r1", "effect": {"type": "confirmed", "summary": "x"}}, + {"read_id": "r2", "effect": None}, ], - }, - "other": "private metadata", - } - valid = self.export_scope( - scope, - "valid-receipt.jsonl", - valid_receipt, - ) - self.assertEqual(0, valid.returncode, valid.stderr) - row = read_jsonl(self.artifacts / "valid-receipt.jsonl")[0] - receipt = row["messages"][0]["decision_receipt"] - self.assertEqual( - {"version", "effect", "summary", "evidence"}, - set(receipt), - ) - self.assertEqual( - {"repoUrl", "commit", "nodePath", "heading"}, - set(receipt["evidence"][0]), - ) - self.assertNotIn("context_decision_invalid", row["coverage_gaps"]) - receipt_chat = json.loads(json.dumps(row)) - receipt_chat["messages"][0]["content"] = "Kept one source." - write_jsonl(self.artifacts / "chats.jsonl", [receipt_chat]) - receipt_only = self.collect("receipt-only-candidates.jsonl") - self.assertEqual(0, receipt_only.returncode, receipt_only.stderr) - receipt_candidate = read_jsonl( - self.artifacts / "receipt-only-candidates.jsonl" - )[0] - self.assertEqual("candidate", receipt_candidate["candidate_status"]) - self.assertEqual( - [MESSAGE_ID], - [ - message["message_id"] - for message in receipt_candidate["visible_choice_candidates"] - ], - ) - - malformed = self.export_scope( - scope, - "malformed-receipt.jsonl", - {"contextDecision": {"version": 1, "effect": ["not", "valid"]}}, - ) - self.assertEqual(0, malformed.returncode, malformed.stderr) - malformed_row = read_jsonl(self.artifacts / "malformed-receipt.jsonl")[0] - self.assertNotIn("decision_receipt", malformed_row["messages"][0]) - self.assertIn( - "context_decision_invalid", malformed_row["coverage_gaps"] - ) - - for output_name, rejected_url in ( - ( - "credential-receipt.jsonl", - "https://secret-token@example.com/org/tree.git", - ), - ("local-receipt.jsonl", "/private/context-tree"), - ("non-repository-receipt.jsonl", "not-a-repository"), - ("short-loopback-receipt.jsonl", "https://127.1/org/tree"), - ( - "integer-loopback-receipt.jsonl", - "https://2130706433/org/tree", - ), - ("unspecified-ipv4-receipt.jsonl", "https://0.0.0.0/org/tree"), - ("unspecified-ipv6-receipt.jsonl", "https://[::]/org/tree"), - ( - "mapped-loopback-receipt.jsonl", - "https://[::ffff:127.0.0.1]/org/tree", - ), - ( - "hex-integer-loopback-receipt.jsonl", - "https://0x7f000001/org/tree", - ), - ( - "hex-scp-loopback-receipt.jsonl", - "git@0x7f000001:org/tree", - ), - ( - "hex-components-loopback-receipt.jsonl", - "https://0x7f.0x0.0x0.0x1/org/tree", - ), - ( - "encoded-loopback-receipt.jsonl", - "https://127%2e0%2e0%2e1/org/tree", - ), - ): - unsafe_receipt = json.loads(json.dumps(valid_receipt)) - unsafe_receipt["contextDecision"]["evidence"][0][ - "repoUrl" - ] = rejected_url - unsafe = self.export_scope( - scope, - output_name, - unsafe_receipt, - ) - self.assertEqual(0, unsafe.returncode, unsafe.stderr) - unsafe_text = (self.artifacts / output_name).read_text( - encoding="utf-8" - ) - self.assertNotIn(rejected_url, unsafe_text) - unsafe_row = read_jsonl(self.artifacts / output_name)[0] - self.assertNotIn("decision_receipt", unsafe_row["messages"][0]) - self.assertIn( - "context_decision_invalid", unsafe_row["coverage_gaps"] + self.CASES, ) - absent = self.export_scope(scope, "absent-receipt.jsonl", None) - self.assertEqual(0, absent.returncode, absent.stderr) - absent_row = read_jsonl(self.artifacts / "absent-receipt.jsonl")[0] - self.assertNotIn("decision_receipt", absent_row["messages"][0]) - self.assertNotIn("context_decision_invalid", absent_row["coverage_gaps"]) - - unbounded = self.export_scope( - scope, - "unbounded-window.jsonl", - None, - days=None, - ) - self.assertEqual(0, unbounded.returncode, unbounded.stderr) - self.assertIsNone( - read_jsonl(self.artifacts / "unbounded-window.jsonl")[0]["window"]["start"] - ) - - def write_chat_export(self) -> Path: - path = self.artifacts / "chats.jsonl" - write_jsonl( - path, - [ - { - "schema_version": 1, - "audit_id": f"{CHAT_ID}@{AGENT_ID}", - "chat_id": CHAT_ID, - "title": "Choose one state source", - "authorization": "explicit_chat", - "source_agent": "fixture-agent", - "source_agent_id": AGENT_ID, - "messages": [ - { - "message_id": ACCEPTANCE_MESSAGE_ID, - "created_at": "2026-07-22T10:01:00Z", - "sender_id": AGENT_ID, - "content": ( - "I will choose the authoritative state source " - "and document the decision." - ), - }, - { - "message_id": MESSAGE_ID, - "created_at": "2026-07-22T10:05:00Z", - "sender_id": AGENT_ID, - "content": ( - "The Context Tree requires one authoritative state " - "source, so I will not add a second table." - ), - "decision_receipt": { - "version": 1, - "effect": "constrained", - "summary": "Kept one authoritative state source.", - "evidence": [ - { - "repoUrl": "https://github.com/acme/tree", - "commit": self.tree_commit, - "nodePath": "system/architecture.md", - "heading": "Decision", - } - ], - }, - } - ], - "coverage_gaps": [], - } - ], - ) - return path - - def write_trace_fixtures(self) -> None: - write_jsonl( - self.trace_root / "authorized.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-authorized-read", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-authorized-read", - "output": ( - "Process exited with code 0\n" - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state." - ), - }, - }, - ], - ) - - unauthorized = self.trace_root / "unauthorized.jsonl" - write_jsonl( - unauthorized, - [session_meta(self.workspace), context_row(UNAUTHORIZED_CHAT_ID)], - ) - with unauthorized.open("a", encoding="utf-8") as handle: - handle.write("not-json private-unauthorized-sentinel\n") - - write_jsonl( - self.trace_root / "compound.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:03:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-compound-read", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file} && printf mixed-output-sentinel", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:03:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-compound-read", - "output": ( - "Process exited with code 0\n" - "Chat history is authoritative.\n" - "mixed-output-sentinel" - ), - }, - }, - ], - ) - - write_jsonl( - self.trace_root / "read-only-composite.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:03:10Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-read-only-composite", - "arguments": json.dumps( - { - "cmd": ( - f"cd {self.tree_root} && " - "test -f system/architecture.md && " - "printf '%s\\n' tree-node-content && " - "sed -n '1,80p' system/architecture.md && " - "sed -n '1,80p' team-practice/dogfooding.md" - ), - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:03:11Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-read-only-composite", - "output": ( - "Process exited with code 0\n" - "tree-node-content\n" - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state.\n" - "# Dogfooding\n\n## Decision\n\n" - "Use First Tree in daily work." - ), - }, - }, - ], - ) - - orchestration_source = ( - "var nested = await tools.exec_command({" - f"cmd: `sed -n '1,80p' {self.tree_file}; " - f"sed -n '1,80p' {self.second_tree_file}`, " - f'workdir: "{self.workspace}", ' - "yield_time_ms: 10000, max_output_tokens: 1000});\n" - "text(nested.output);" - ) - write_jsonl( - self.trace_root / "exec-orchestration.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:03:20Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call", - "name": "exec", - "call_id": "call-exec-orchestration", - "input": orchestration_source, - }, - }, - { - "timestamp": "2026-07-22T10:03:21Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call_output", - "call_id": "call-exec-orchestration", - "output": [ - { - "type": "input_text", - "text": "Script completed successfully\nOutput:\n", - }, - { - "type": "input_text", - "text": ( - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state.\n" - "# Dogfooding\n\n## Decision\n\n" - "Use First Tree in daily work." - ), - }, - ], - }, - }, - ], - ) - - other_tree = self.root / "other-tree" - other_file = other_tree / "system" / "architecture.md" - other_file.parent.mkdir(parents=True) - other_file.write_text("outside-bound-tree-sentinel", encoding="utf-8") - write_jsonl( - self.trace_root / "other-tree.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:04:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-other-tree-read", - "arguments": json.dumps( - { - "cmd": f"cat {other_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:04:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-other-tree-read", - "output": "Process exited with code 0\noutside-bound-tree-sentinel", - }, - }, - ], - ) - - def collect(self, output_name: str) -> subprocess.CompletedProcess[str]: - return run_cli( - "collect", - "--artifact-root", - str(self.artifacts), - "--chats", - str(self.artifacts / "chats.jsonl"), - "--trace-root", - str(self.trace_root), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--tree-root", - str(self.tree_root), - "--days", - "7", - "--now", - NOW, - "--output", - str(self.artifacts / output_name), - ) - - def collect_for_provider( - self, - provider: str, - trace_root: Path, - output_name: str, - ) -> subprocess.CompletedProcess[str]: - return run_cli( - "collect", - "--artifact-root", - str(self.artifacts), - "--chats", - str(self.artifacts / "chats.jsonl"), - "--trace-root", - str(trace_root), - "--runtime-provider", - provider, - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--tree-root", - str(self.tree_root), - "--days", - "7", - "--now", - NOW, - "--output", - str(self.artifacts / output_name), - runtime_provider=provider, - ) - - def task_judgment( - self, - candidate: dict[str, Any], - *, - task_id: str = "task-1", - message_id: str = MESSAGE_ID, - objective_message_id: str | None = None, - read_status: str = "observed", - read_ids: list[str] | None = None, - effect: dict[str, Any] | None | object = ..., - ) -> dict[str, Any]: - selected_reads = ( - read_ids - if read_ids is not None - else [candidate["reads"][0]["read_id"]] - if candidate["reads"] - else [] - ) - if effect is ...: - effect = ( - { - "type": "constrained", - "read_ids": selected_reads, - "choice_message_ids": [message_id], - "outcome_anchor": message_id, - "summary": "The Tree constraint prevented a second state table.", - } - if selected_reads and read_status == "observed" - else None + def test_refuted_requires_its_explanation(self) -> None: + with self.assertRaisesRegex(audit.AuditError, "refutation"): + audit.validate_judgments( + [ + {"read_id": "r1", "effect": {"type": "confirmed", "summary": "x"}, "refuted": True}, + {"read_id": "r2", "effect": None}, + ], + self.CASES, + ) + + def test_rejects_unknown_effect_types_and_missing_cases(self) -> None: + with self.assertRaises(audit.AuditError): + audit.validate_judgments( + [{"read_id": "r1", "effect": {"type": "helpful", "summary": "x"}, "refuted": False}, + {"read_id": "r2", "effect": None}], + self.CASES, + ) + with self.assertRaisesRegex(audit.AuditError, "missing"): + audit.validate_judgments([{"read_id": "r1", "effect": None}], self.CASES) + + def test_rejects_a_judgment_for_an_unsampled_read(self) -> None: + with self.assertRaises(audit.AuditError): + audit.validate_judgments([{"read_id": "not-sampled", "effect": None}], self.CASES) + + +class ReportTests(unittest.TestCase): + GENERATED = datetime(2026, 7, 31, tzinfo=timezone.utc) + + def _render(self, judgments: list[dict[str, object]] | None, unobserved: list[str] | None = None) -> str: + agg = audit.aggregate( + normalized([event("r1", "system/a.md"), event("w1", "system/b.md", action="write")]) + ) + return audit.render_report( + generated_at=self.GENERATED, + window_start=None, + window_end=None, + agg=agg, + unobserved=unobserved, + judgments=judgments, + sample_size=len(judgments or []), + ) + + def test_never_presents_adoption_as_a_rate_and_always_states_the_gaps(self) -> None: + text = self._render(None) + self.assertNotIn("%", text.split("## Known recording gaps")[0]) + self.assertIn("lower bound", text) + self.assertIn("pipeline", text) + self.assertIn("directory-level event", text) + self.assertIn("never proof that a node went unread", text) + + def test_withholds_influence_numbers_when_most_claims_were_refuted(self) -> None: + judgments = [ + {"read_id": f"r{index}", "target_path": "system/a.md", "read_at": "t", + "effect": {"type": "confirmed", "summary": "s"}, "refuted": index < 3} + for index in range(4) + ] + text = self._render(judgments) + self.assertIn("not reliable", text) + self.assertIn("withheld", text) + # The per-effect breakdown must not appear when the run is unreliable. + self.assertNotIn("| Effect | Upheld |", text) + + def test_shows_the_breakdown_when_claims_survived(self) -> None: + judgments = [ + {"read_id": "r1", "target_path": "system/a.md", "read_at": "t", + "effect": {"type": "redirected", "summary": "Dropped plan A."}, "refuted": False}, + {"read_id": "r2", "target_path": "system/b.md", "read_at": "t", "effect": None, "refuted": False}, + ] + text = self._render(judgments) + self.assertIn("| Effect | Upheld |", text) + self.assertIn("Dropped plan A.", text) + self.assertNotIn("withheld", text) + + def test_reports_an_empty_never_read_list_without_implying_failure(self) -> None: + text = self._render(None, unobserved=[]) + self.assertIn("Every normal node had an observed read or sat under a recorded search root", text) + + +class CliTests(unittest.TestCase): + def test_facts_json_runs_end_to_end_from_a_captured_feed(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + feed = root / "feed.json" + feed.write_text( + json.dumps({"items": [event("r1", "system/a.md"), event("w1", "system/b.md", action="write")]}), + encoding="utf-8", ) - visible_message_ids = { - message["message_id"] - for message in candidate.get("visible_messages", []) - if isinstance(message, dict) - and isinstance(message.get("message_id"), str) - } - objective_anchor_id = ( - objective_message_id - or ( - ACCEPTANCE_MESSAGE_ID - if ACCEPTANCE_MESSAGE_ID in visible_message_ids - and message_id != ACCEPTANCE_MESSAGE_ID - else message_id + out = root / "facts.json" + code = audit.main( + ["facts", "--events-file", str(feed), "--json", "--output", str(out), "--now", "2026-07-31T00:00:00Z"] ) - ) - source_message_ids = list( - dict.fromkeys([objective_anchor_id, message_id]) - ) - message_times = { - message["message_id"]: message["created_at"] - for message in candidate.get("visible_messages", []) - if isinstance(message, dict) - and isinstance(message.get("message_id"), str) - and isinstance(message.get("created_at"), str) - } - return { - "schema_version": 4, - "task_id": task_id, - "status": "clear", - "objective": "Choose one state source", - "object_scope": "state persistence", - "outcome": "Kept the existing authoritative state source.", - "started_at": message_times.get( - objective_anchor_id, "2026-07-22T10:01:00Z" - ), - "ended_at": message_times.get( - message_id, "2026-07-22T10:05:00Z" - ), - "source_fragments": [ - { - "audit_id": candidate["audit_id"], - "message_ids": source_message_ids, - } - ], - "episode": { - "ownership": { - "kind": "accepted", - "anchor_message_ids": [objective_anchor_id], - "reason": "The audited Agent visibly accepted the objective.", - }, - "objective_anchor_message_ids": [objective_anchor_id], - "outcome_anchor_message_ids": [message_id], - "continuation_message_ids": [], - "primary_deliverable": "A decision selecting one state source.", - "boundary_reason": ( - "One accepted objective produced one terminal decision." - ), - }, - "read": { - "status": read_status, - "read_ids": selected_reads, - "reason": ( - "Historical trace coverage cannot confirm a read." - if read_status == "unresolved" - else None - ), - }, - "effect": effect, - "effect_reason": ( - "No later choice reasonably shows Tree influence." - if effect is None - else None - ), - } + self.assertEqual(code, 0) + payload = json.loads(out.read_text(encoding="utf-8")) + self.assertEqual(payload["reads"], 1) + self.assertEqual(payload["writes"], 1) + self.assertEqual(payload["node_reads"], {"system/a.md": 1}) + self.assertTrue(payload["known_gaps"]) + self.assertEqual(oct(out.stat().st_mode & 0o777), "0o600") - def report( - self, - tasks: list[dict[str, Any]], - *, - candidates_name: str = "candidates.jsonl", - evidence_name: str = "evidence.jsonl", - report_name: str = "REPORT.md", - reviewed_baseline_name: str | None = None, - ) -> subprocess.CompletedProcess[str]: - candidates_path = self.artifacts / candidates_name - task_source_path = self.artifacts / "task-source.jsonl" - task_source_result = run_cli( - "task-source", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(candidates_path), - "--output", - str(task_source_path), - ) - if task_source_result.returncode != 0: - return task_source_result - - inventory_rows: list[dict[str, Any]] = [] - read_rows: list[dict[str, Any]] = [] - effect_rows: list[dict[str, Any]] = [] - for task in tasks: - inventory = json.loads(json.dumps(task)) - read = inventory.pop("read", None) - effect = inventory.pop("effect", None) - effects = inventory.pop("effects", None) - effect_reason = inventory.pop("effect_reason", None) - episode = inventory.pop("episode", None) - if isinstance(episode, dict): - inventory["objective_source_message_ids"] = episode.get( - "objective_anchor_message_ids" - ) - inventory["outcome_source_message_ids"] = episode.get( - "outcome_anchor_message_ids" - ) - inventory["primary_deliverable"] = episode.get( - "primary_deliverable" - ) - inventory_rows.append(inventory) - - if inventory.get("status") != "clear": - continue - task_id = inventory.get("task_id") - if isinstance(read, dict): - read_rows.append( - { - "schema_version": 4, - "task_id": task_id, - "status": read.get("status"), - "read_ids": read.get("read_ids"), - "reason": read.get("reason"), - } - ) - selected_effects = ( - effects - if effects is not None - else [effect] - if isinstance(effect, dict) - else [] - ) - projected_effects = [] - for selected_effect in selected_effects: - projected = dict(selected_effect) - if "outcome_anchor" in projected: - projected["outcome_message_id"] = projected.pop( - "outcome_anchor" - ) - projected_effects.append(projected) - effect_rows.append( - { - "schema_version": 4, - "task_id": task_id, - "effects": projected_effects, - "effect_reason": effect_reason, - } - ) + def test_a_malformed_feed_fails_closed(self) -> None: + with TemporaryDirectory() as tmp: + feed = Path(tmp) / "feed.json" + feed.write_text(json.dumps({"items": [{"id": "r1"}]}), encoding="utf-8") + self.assertEqual(audit.main(["facts", "--events-file", str(feed), "--json"]), 2) - inventory_draft_path = self.artifacts / "task-inventory-draft.jsonl" - inventory_path = self.artifacts / "task-inventory.jsonl" - write_jsonl(inventory_draft_path, inventory_rows) - freeze_result = run_cli( - "freeze-tasks", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--task-source", - str(task_source_path), - "--task-inventory-draft", - str(inventory_draft_path), - "--task-inventory-output", - str(inventory_path), - ) - if freeze_result.returncode != 0: - return freeze_result - frozen_rows = read_jsonl(inventory_path) - inventory_sha256 = frozen_rows[0]["inventory_sha256"] - for row in [*read_rows, *effect_rows]: - row["inventory_sha256"] = inventory_sha256 - read_path = self.artifacts / "read-attributions.jsonl" - effect_path = self.artifacts / "effect-judgments.jsonl" - write_jsonl(read_path, read_rows) - write_jsonl(effect_path, effect_rows) - arguments = [ - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(candidates_path), - "--task-inventory", - str(inventory_path), - "--read-attributions", - str(read_path), - "--effect-judgments", - str(effect_path), - "--evidence-output", - str(self.artifacts / evidence_name), - "--report-output", - str(self.artifacts / report_name), - "--generated-at", - NOW, - ] - if reviewed_baseline_name is not None: - arguments.extend( - [ - "--reviewed-baseline", - str(self.artifacts / reviewed_baseline_name), - ] - ) - return run_cli(*arguments) - def test_collect_prefilters_trace_and_recovers_read_only_composites(self) -> None: - self.write_chat_export() - self.write_trace_fixtures() +if __name__ == "__main__": + unittest.main() - first = self.collect("candidates-one.jsonl") - second = self.collect("candidates-two.jsonl") - self.assertEqual(0, first.returncode, first.stderr) - self.assertEqual(0, second.returncode, second.stderr) - self.assertEqual( - (self.artifacts / "candidates-one.jsonl").read_bytes(), - (self.artifacts / "candidates-two.jsonl").read_bytes(), - ) - rows = read_jsonl(self.artifacts / "candidates-one.jsonl") - self.assertEqual(1, len(rows)) - candidate = rows[0] - self.assertEqual("candidate", candidate["candidate_status"]) - self.assertTrue(candidate["tree_identity"].startswith("tree-")) - self.assertEqual(4, len(candidate["reads"])) - isolated_read, compound_read, composite_read, orchestration_read = candidate[ - "reads" +class TreeIdentityTests(unittest.TestCase): + def test_canonicalizes_equivalent_remote_spellings(self) -> None: + forms = [ + "https://github.com/Example/Context-Tree.git", + "https://github.com/example/context-tree", + "git@github.com:example/context-tree.git", + "ssh://git@github.com/example/context-tree", ] - self.assertEqual(["system/architecture.md"], isolated_read["node_paths"]) - self.assertEqual("isolated", isolated_read["read_mode"]) - self.assertEqual(candidate["tree_identity"], isolated_read["tree_identity"]) - self.assertEqual( - "default_branch_match", - isolated_read["tree_source"]["status"], - ) - self.assertIn("authoritative state", isolated_read["passage"]) - self.assertEqual( - [ - "system/architecture.md", - "team-practice/dogfooding.md", - ], - composite_read["node_paths"], - ) - self.assertEqual("read_only_composite", composite_read["read_mode"]) - self.assertEqual( - [ - { - "reader": "sed", - "node_paths": ["system/architecture.md"], - }, - { - "reader": "sed", - "node_paths": ["team-practice/dogfooding.md"], - }, - ], - composite_read["read_components"], - ) - self.assertIn("authoritative state", composite_read["passage"]) - self.assertIn("daily work", composite_read["passage"]) - self.assertEqual("read_only_composite", compound_read["read_mode"]) - self.assertEqual("read_only_composite", orchestration_read["read_mode"]) - self.assertEqual("exec", orchestration_read["tool_name"]) - self.assertEqual( - { - "accepted_exact": 1, - "accepted_read_only_composite": 3, - "unresolved_opaque": 0, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - 4, - candidate["collector_diagnostics"]["in_window_tree_read_attempts"], - ) - self.assertTrue(all(item.startswith("trace-") for item in candidate["mapped_trace_files"])) - self.assertEqual([], candidate["coverage_gaps"]) - self.assertEqual([MESSAGE_ID], [ - message["message_id"] for message in candidate["visible_choice_candidates"] - ]) - - serialized = json.dumps(candidate, sort_keys=True) - self.assertNotIn("private-unauthorized-sentinel", serialized) - self.assertNotIn("mixed-output-sentinel", serialized) - self.assertNotIn("tree-node-content", serialized) - self.assertNotIn("outside-bound-tree-sentinel", serialized) - self.assertNotIn(str(self.root), serialized) - self.assertEqual( - 0o600, - stat.S_IMODE((self.artifacts / "candidates-one.jsonl").stat().st_mode), - ) + self.assertEqual(len({audit.canonical_repo(form) for form in forms}), 1) - def test_codex_missing_and_duplicate_results_stay_unresolved(self) -> None: - self.write_chat_export() - command = { - "type": "function_call", - "name": "exec_command", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - } - output = ( - "Process exited with code 0\n" - + self.tree_file.read_text(encoding="utf-8") - ) - write_jsonl( - self.trace_root / "missing-result.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - **command, - "call_id": "call-missing-result", - }, - }, - ], - ) - write_jsonl( - self.trace_root / "duplicate-call.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - *[ - { - "timestamp": f"2026-07-22T10:03:0{index}Z", - "type": "response_item", - "payload": { - **command, - "call_id": "call-duplicate", - }, - } - for index in (0, 1) - ], - { - "timestamp": "2026-07-22T10:03:02Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-duplicate", - "output": output, - }, - }, - ], - ) - write_jsonl( - self.trace_root / "duplicate-result.jsonl", + def test_excludes_events_from_another_tree_rather_than_miscrediting_them(self) -> None: + events = normalized( [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:04:00Z", - "type": "response_item", - "payload": { - **command, - "call_id": "call-duplicate-result", - }, - }, - *[ - { - "timestamp": f"2026-07-22T10:04:0{index}Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-duplicate-result", - "output": output, - }, - } - for index in (1, 2) - ], - ], + event("mine", "system/a.md"), + event("theirs", "system/a.md", repo="https://github.com/other/tree"), + ] ) - - result = self.collect("codex-pairing-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "codex-pairing-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 4, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "codex_duplicate_call_id": 2, - "codex_duplicate_tool_result": 1, - "tree_read_output_missing": 1, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_codex_explicit_direct_read_error_stays_unresolved(self) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "direct-read-error.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "read_file", - "call_id": "call-direct-read-error", - "arguments": json.dumps( - {"path": str(self.tree_file)} - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-direct-read-error", - "output": "Error: permission denied", - }, - }, - { - "timestamp": "2026-07-22T10:02:02Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "read_file", - "call_id": "call-direct-read-json-error", - "arguments": json.dumps( - {"path": str(self.tree_file)} - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:03Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-direct-read-json-error", - "output": json.dumps( - {"error": "permission denied"} - ), - }, - }, - ], - ) - - result = self.collect("direct-read-error-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "direct-read-error-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 2, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - {"tree_read_command_failed": 2}, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_codex_read_call_id_shared_with_non_read_call_is_unresolved( - self, - ) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "ambiguous-call-id.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "read_file", - "call_id": "call-shared-with-non-read", - "arguments": json.dumps( - {"path": str(self.tree_file)} - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "web_search", - "call_id": "call-shared-with-non-read", - "arguments": json.dumps({"query": "unrelated"}), - }, - }, - { - "timestamp": "2026-07-22T10:02:02Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-shared-with-non-read", - "output": self.tree_file.read_text(encoding="utf-8"), - }, - }, - ], - ) - - result = self.collect("ambiguous-call-id-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "ambiguous-call-id-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 1, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - {"codex_duplicate_call_id": 1}, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_collects_claude_code_native_tool_results(self) -> None: - self.write_chat_export() - claude_config_root = self.root / "claude-config" - trace_root = claude_config_root / "projects" - write_jsonl( - trace_root / "project" / "session.jsonl", - [ - { - "type": "user", - "sessionId": "claude-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - }, - { - "type": "assistant", - "sessionId": "claude-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "claude-read", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - }, - { - "type": "user", - "sessionId": "claude-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:01Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "claude-read", - "content": self.tree_file.read_text(encoding="utf-8"), - } - ], - }, - }, - ], - ) - write_jsonl( - trace_root / "unrelated" / "session.jsonl", - [ - { - "type": "user", - "sessionId": "unrelated-session", - "cwd": str(self.root / "another-workspace"), - "timestamp": "2026-07-22T10:01:00Z", - "message": {"role": "user", "content": "unrelated"}, - } - ], - ) - - result = self.collect_for_provider( - "claude-code", - trace_root, - "claude-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl(self.artifacts / "claude-candidates.jsonl")[0] - self.assertEqual("claude-code", candidate["runtime_provider"]) - self.assertEqual(1, len(candidate["reads"])) - self.assertEqual( - "claude-code", - candidate["reads"][0]["runtime_provider"], - ) - self.assertEqual( - 1, - candidate["collector_diagnostics"]["attempt_status_counts"][ - "accepted_exact" - ], - ) - self.assertNotIn( - "claude_trace_preflight_malformed_or_unmapped", - candidate["coverage_gaps"], - ) - tui_result = self.collect_for_provider( - "claude-code-tui", - trace_root, - "claude-tui-candidates.jsonl", - ) - self.assertEqual(0, tui_result.returncode, tui_result.stderr) - self.assertEqual( - "claude-code", - read_jsonl(self.artifacts / "claude-tui-candidates.jsonl")[0][ - "runtime_provider" - ], - ) - default_root_result = run_cli( - "collect", - "--artifact-root", - str(self.artifacts), - "--chats", - str(self.artifacts / "chats.jsonl"), - "--runtime-provider", - "claude-code", - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--tree-root", - str(self.tree_root), - "--days", - "7", - "--now", - NOW, - "--output", - str(self.artifacts / "claude-custom-root-candidates.jsonl"), - runtime_provider="claude-code", - environment_overrides={ - "CLAUDE_CONFIG_DIR": str(claude_config_root), - }, - ) - self.assertEqual(0, default_root_result.returncode, default_root_result.stderr) - self.assertEqual( - 1, - len( - read_jsonl( - self.artifacts / "claude-custom-root-candidates.jsonl" - )[0]["reads"] - ), - ) - - def test_claude_failed_result_is_not_counted_as_accepted(self) -> None: - self.write_chat_export() - trace_root = self.root / "claude-failed" - write_jsonl( - trace_root / "project" / "session.jsonl", - [ - { - "type": "user", - "sessionId": "claude-failed-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - }, - { - "type": "assistant", - "sessionId": "claude-failed-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "claude-failed-read", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - }, - { - "type": "user", - "sessionId": "claude-failed-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:01Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "claude-failed-read", - "is_error": True, - "content": "read failed", - } - ], - }, - }, - ], - ) - result = self.collect_for_provider( - "claude-code", - trace_root, - "claude-failed-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "claude-failed-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - 0, - candidate["collector_diagnostics"]["attempt_status_counts"][ - "accepted_exact" - ], - ) - self.assertEqual( - 1, - candidate["collector_diagnostics"]["attempt_status_counts"][ - "unresolved_opaque" - ], - ) - self.assertEqual( - {"tree_read_command_failed": 1}, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_claude_tool_result_context_echo_is_not_identity(self) -> None: - self.write_chat_export() - trace_root = self.root / "claude-echo" - write_jsonl( - trace_root / "project" / "session.jsonl", - [ - { - "type": "user", - "sessionId": "claude-echo-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "echo", - "content": context_block(CHAT_ID), - } - ], - }, - } - ], - ) - - result = self.collect_for_provider( - "claude-code", - trace_root, - "claude-echo-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "claude-echo-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertIn( - "claude_trace_preflight_malformed_or_unmapped", - candidate["coverage_gaps"], - ) - - def test_claude_ignores_native_non_message_metadata_rows(self) -> None: - self.write_chat_export() - trace_root = self.root / "claude-native-metadata" - session_id = "claude-native-metadata-session" - metadata_rows = [ - { - "type": row_type, - "sessionId": session_id, - "timestamp": f"2026-07-22T10:01:0{index}Z", - } - for index, row_type in enumerate( - ("queue-operation", "last-prompt", "mode"), - start=1, - ) - ] - write_jsonl( - trace_root / "project" / "session.jsonl", - [ - { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - }, - *metadata_rows, - { - "type": "assistant", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "claude-metadata-read", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - }, - { - "type": "last-prompt", - "sessionId": session_id, - "timestamp": "2026-07-22T10:02:00.500Z", - }, - { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:01Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "claude-metadata-read", - "content": self.tree_file.read_text(encoding="utf-8"), - } - ], - }, - }, - ], - ) - - result = self.collect_for_provider( - "claude-code", - trace_root, - "claude-native-metadata-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "claude-native-metadata-candidates.jsonl" - )[0] - self.assertEqual(1, len(candidate["reads"])) - self.assertNotIn( - "claude_trace_workspace_changed", - candidate["coverage_gaps"], - ) - - def test_claude_compact_and_meta_context_echoes_are_not_identity(self) -> None: - self.write_chat_export() - for marker in ("isCompactSummary", "isMeta"): - with self.subTest(marker=marker, position="only"): - trace_root = self.root / f"claude-{marker}-only" - write_jsonl( - trace_root / "project" / "session.jsonl", - [ - { - "type": "user", - marker: True, - "sessionId": "claude-synthetic-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - } - ], - ) - result = self.collect_for_provider( - "claude-code", - trace_root, - f"claude-{marker}-only-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / f"claude-{marker}-only-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertIn( - "no_mapped_claude_code_evidence", - candidate["coverage_gaps"], - ) - - with self.subTest(marker=marker, position="after-preflight"): - trace_root = self.root / f"claude-{marker}-late" - session_id = "claude-late-session" - canonical = { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - } - filler = [ - { - "type": "assistant", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:01Z", - "message": {"role": "assistant", "content": []}, - } - for _ in range(511) - ] - synthetic = { - "type": "user", - marker: True, - "sessionId": session_id, - "timestamp": "2026-07-22T10:01:02Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(SECOND_CHAT_ID)}], - }, - } - call = { - "type": "assistant", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": f"claude-{marker}-read", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - } - tool_result = { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:01Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": f"claude-{marker}-read", - "content": self.tree_file.read_text(encoding="utf-8"), - } - ], - }, - } - write_jsonl( - trace_root / "project" / "session.jsonl", - [canonical, *filler, synthetic, call, tool_result], - ) - result = self.collect_for_provider( - "claude-code", - trace_root, - f"claude-{marker}-late-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / f"claude-{marker}-late-candidates.jsonl" - )[0] - self.assertEqual(1, len(candidate["reads"])) - self.assertNotIn( - "claude_trace_chat_boundary_changed", - candidate["coverage_gaps"], - ) - - def test_claude_full_scan_rejects_post_preflight_session_and_chat_drift( - self, - ) -> None: - self.write_chat_export() - initial = { - "type": "user", - "sessionId": "claude-session-a", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - } - filler = [ - { - "type": "assistant", - "sessionId": "claude-session-a", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:01Z", - "message": {"role": "assistant", "content": []}, - } - for _ in range(511) - ] - cases = { - "session": { - "type": "assistant", - "sessionId": "claude-session-b", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": {"role": "assistant", "content": []}, - }, - "chat": { - "type": "user", - "sessionId": "claude-session-a", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "user", - "content": [ - { - "type": "text", - "text": context_block(SECOND_CHAT_ID), - } - ], - }, - }, - } - expected = { - "session": "claude_trace_session_changed", - "chat": "claude_trace_chat_boundary_changed", - } - for name, drift in cases.items(): - with self.subTest(name=name): - trace_root = self.root / f"claude-{name}-drift" - write_jsonl( - trace_root / "project" / "session.jsonl", - [initial, *filler, drift], - ) - result = self.collect_for_provider( - "claude-code", - trace_root, - f"claude-{name}-drift-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / f"claude-{name}-drift-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertIn(expected[name], candidate["coverage_gaps"]) - - def test_runtimes_without_complete_native_evidence_stay_pending(self) -> None: - self.write_chat_export() - for provider in ("cursor", "kimi-code"): - with self.subTest(provider=provider): - result = self.collect_for_provider( - provider, - self.trace_root, - f"{provider}-unsupported-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / f"{provider}-unsupported-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertIn( - f"{provider.replace('-', '_')}_historical_evidence_not_supported", - candidate["coverage_gaps"], - ) - - def test_claude_pairing_fail_closes_duplicate_calls(self) -> None: - self.write_chat_export() - claude_root = self.root / "claude-duplicate" - claude_rows = [ - { - "type": "user", - "sessionId": "claude-duplicate-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - }, - *[ - { - "type": "assistant", - "sessionId": "claude-duplicate-session", - "cwd": str(self.workspace), - "timestamp": f"2026-07-22T10:02:0{index}Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "duplicate-read", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - } - for index in (0, 1) - ], - { - "type": "user", - "sessionId": "claude-duplicate-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:02Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "duplicate-read", - "content": self.tree_file.read_text(encoding="utf-8"), - } - ], - }, - }, - ] - write_jsonl(claude_root / "project" / "session.jsonl", claude_rows) - claude_result = self.collect_for_provider( - "claude-code", - claude_root, - "claude-duplicate-candidates.jsonl", - ) - self.assertEqual(0, claude_result.returncode, claude_result.stderr) - claude_candidate = read_jsonl( - self.artifacts / "claude-duplicate-candidates.jsonl" - )[0] - self.assertEqual([], claude_candidate["reads"]) - self.assertIn( - "claude_tool_call_duplicate", - claude_candidate["coverage_gaps"], - ) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 2, - "rejected_unsafe": 0, - }, - claude_candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - {"unresolved_claude_tool_call_duplicate": 2}, - claude_candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertEqual( - 2, - claude_candidate["collector_diagnostics"][ - "in_window_tree_read_attempts" - ], - ) - - def test_claude_unpaired_tree_calls_stay_in_attempt_denominator(self) -> None: - self.write_chat_export() - claude_root = self.root / "claude-unpaired" - context = { - "type": "user", - "sessionId": "claude-unpaired-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:01:00Z", - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - } - missing_call = { - "type": "assistant", - "sessionId": "claude-unpaired-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:00Z", - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": "missing-result", - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - } - write_jsonl( - claude_root / "missing" / "session.jsonl", - [context, missing_call], - ) - - duplicate_call = json.loads(json.dumps(missing_call)) - duplicate_call["message"]["content"][0]["id"] = "duplicate-result" - duplicate_result = { - "type": "user", - "sessionId": "claude-unpaired-session", - "cwd": str(self.workspace), - "timestamp": "2026-07-22T10:02:01Z", - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": "duplicate-result", - "content": self.tree_file.read_text(encoding="utf-8"), - }, - { - "type": "tool_result", - "tool_use_id": "duplicate-result", - "content": self.tree_file.read_text(encoding="utf-8"), - }, - ], - }, - } - write_jsonl( - claude_root / "duplicate" / "session.jsonl", - [context, duplicate_call, duplicate_result], - ) - - result = self.collect_for_provider( - "claude-code", - claude_root, - "claude-unpaired-candidates.jsonl", - ) - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "claude-unpaired-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 2, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unresolved_claude_tool_result_duplicate": 1, - "unresolved_claude_tool_result_missing": 1, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertEqual( - 2, - candidate["collector_diagnostics"]["in_window_tree_read_attempts"], - ) - self.assertIn("claude_tool_result_duplicate", candidate["coverage_gaps"]) - self.assertIn("claude_tool_result_missing", candidate["coverage_gaps"]) - - def test_claude_pairing_respects_the_acquisition_window(self) -> None: - self.write_chat_export() - claude_root = self.root / "claude-window" - - def context(timestamp: str, session_id: str) -> dict[str, Any]: - return { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": timestamp, - "message": { - "role": "user", - "content": [{"type": "text", "text": context_block(CHAT_ID)}], - }, - } - - def call(timestamp: str, session_id: str, call_id: str) -> dict[str, Any]: - return { - "type": "assistant", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": timestamp, - "message": { - "role": "assistant", - "content": [ - { - "type": "tool_use", - "id": call_id, - "name": "Read", - "input": {"file_path": str(self.tree_file)}, - } - ], - }, - } - - def result(timestamp: str, session_id: str, call_id: str) -> dict[str, Any]: - return { - "type": "user", - "sessionId": session_id, - "cwd": str(self.workspace), - "timestamp": timestamp, - "message": { - "role": "user", - "content": [ - { - "type": "tool_result", - "tool_use_id": call_id, - "content": self.tree_file.read_text(encoding="utf-8"), - } - ], - }, - } - - write_jsonl( - claude_root / "cross-end" / "session.jsonl", - [ - context("2026-07-23T23:58:00Z", "cross-end-session"), - call( - "2026-07-23T23:59:00Z", - "cross-end-session", - "cross-end-read", - ), - result( - "2026-07-24T00:01:00Z", - "cross-end-session", - "cross-end-read", - ), - ], - ) - write_jsonl( - claude_root / "old-missing" / "session.jsonl", - [ - context("2026-07-16T10:00:00Z", "old-missing-session"), - call( - "2026-07-16T10:01:00Z", - "old-missing-session", - "old-missing-read", - ), - ], - ) - old_duplicate_result = result( - "2026-07-16T10:02:00Z", - "old-duplicate-session", - "old-duplicate-read", - ) - write_jsonl( - claude_root / "old-duplicate" / "session.jsonl", - [ - context("2026-07-16T10:00:00Z", "old-duplicate-session"), - call( - "2026-07-16T10:01:00Z", - "old-duplicate-session", - "old-duplicate-read", - ), - old_duplicate_result, - json.loads(json.dumps(old_duplicate_result)), - ], - ) - - collect_result = self.collect_for_provider( - "claude-code", - claude_root, - "claude-window-candidates.jsonl", - ) - self.assertEqual(0, collect_result.returncode, collect_result.stderr) - candidate = read_jsonl( - self.artifacts / "claude-window-candidates.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 1, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - {"unresolved_claude_code_tool_result_outside_window": 1}, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertEqual( - 1, - candidate["collector_diagnostics"]["in_window_tree_read_attempts"], - ) - self.assertIn( - "claude_code_tool_result_outside_window", - candidate["coverage_gaps"], - ) - self.assertNotIn( - "claude_tool_result_missing", - candidate["coverage_gaps"], - ) - self.assertNotIn( - "claude_tool_result_duplicate", - candidate["coverage_gaps"], - ) - - def test_provider_mismatch_fails_before_trace_collection(self) -> None: - self.write_chat_export() - result = run_cli( - "collect", - "--artifact-root", - str(self.artifacts), - "--chats", - str(self.artifacts / "chats.jsonl"), - "--trace-root", - str(self.trace_root), - "--runtime-provider", - "claude-code", - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--tree-root", - str(self.tree_root), - "--output", - str(self.artifacts / "mismatch.jsonl"), - runtime_provider="cursor", - ) - self.assertNotEqual(0, result.returncode) - self.assertIn("must match FIRST_TREE_PROVIDER", result.stderr) - - def test_missing_runtime_provider_fails_before_trace_collection(self) -> None: - self.write_chat_export() - result = run_cli( - "collect", - "--artifact-root", - str(self.artifacts), - "--chats", - str(self.artifacts / "chats.jsonl"), - "--trace-root", - str(self.trace_root), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--tree-root", - str(self.tree_root), - "--output", - str(self.artifacts / "missing-provider.jsonl"), - runtime_provider=None, - ) - self.assertNotEqual(0, result.returncode) - self.assertIn("FIRST_TREE_PROVIDER is required", result.stderr) - - def test_preflight_accepts_same_id_mirror_and_ignores_noncanonical_echoes( - self, - ) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "production-envelope.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - context_mirror_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:01:01Z", - "type": "compacted", - "payload": { - "message": context_block(UNAUTHORIZED_CHAT_ID), - "replacement_history": [], - }, - }, - { - "timestamp": "2026-07-22T10:01:02Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-echo-only", - "output": context_block(UNAUTHORIZED_CHAT_ID), - }, - }, - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-production-read", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-production-read", - "output": ( - "Process exited with code 0\n" - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state." - ), - }, - }, - ], - ) - - result = self.collect("production-envelope-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "production-envelope-candidates.jsonl" - )[0] - self.assertEqual(1, len(candidate["mapped_trace_files"])) - self.assertEqual(1, len(candidate["reads"])) - self.assertNotIn( - "codex_trace_preflight_malformed_or_ambiguous", - candidate["coverage_gaps"], - ) - - def test_exec_command_continuations_complete_one_read_attempt(self) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "continued-read.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-started-read", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-started-read", - "output": "Script running with session ID 731", - }, - }, - { - "timestamp": "2026-07-22T10:02:02Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "write_stdin", - "call_id": "call-continued-read", - "arguments": json.dumps( - {"session_id": 731, "chars": ""} - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:03Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-continued-read", - "output": ( - "Process exited with code 0\n" - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state." - ), - }, - }, - ], - ) - - result = self.collect("continued-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl(self.artifacts / "continued-candidate.jsonl")[0] - self.assertEqual(1, len(candidate["reads"])) - self.assertIn("authoritative state", candidate["reads"][0]["passage"]) - self.assertNotIn("tree_read_output_pending", candidate["coverage_gaps"]) - self.assertEqual( - 1, - candidate["collector_diagnostics"]["attempt_status_counts"][ - "accepted_exact" - ], - ) - - def test_duplicate_continuation_output_keeps_parent_unresolved(self) -> None: - self.write_chat_export() - terminal_output = ( - "Process exited with code 0\n" - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state." - ) - write_jsonl( - self.trace_root / "duplicate-continuation.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-duplicate-continuation-parent", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-duplicate-continuation-parent", - "output": "Script running with session ID 733", - }, - }, - { - "timestamp": "2026-07-22T10:02:02Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "write_stdin", - "call_id": "call-duplicate-continuation", - "arguments": json.dumps( - {"session_id": 733, "chars": ""} - ), - }, - }, - *[ - { - "timestamp": f"2026-07-22T10:02:0{index}Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-duplicate-continuation", - "output": terminal_output, - }, - } - for index in (3, 4) - ], - ], - ) - - result = self.collect("duplicate-continuation-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "duplicate-continuation-candidate.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - {"tree_read_continuation_output_duplicate": 1}, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_continuation_after_acquisition_end_stays_pending(self) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "late-continuation.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-23T23:59:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-late-start", - "arguments": json.dumps( - { - "cmd": f"cat {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-23T23:59:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-late-start", - "output": "Script running with session ID 732", - }, - }, - { - "timestamp": "2026-07-24T00:01:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "write_stdin", - "call_id": "call-late-finish", - "arguments": json.dumps( - {"session_id": 732, "chars": ""} - ), - }, - }, - { - "timestamp": "2026-07-24T00:01:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-late-finish", - "output": ( - "Process exited with code 0\n" - "# Architecture\n\nlate-private-passage" - ), - }, - }, - ], - ) - - result = self.collect("late-continuation-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "late-continuation-candidate.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertIn( - "tree_read_continuation_incomplete_or_out_of_order", - candidate["coverage_gaps"], - ) - self.assertNotIn("late-private-passage", json.dumps(candidate)) - - def test_exec_orchestration_classifies_exact_and_promise_all(self) -> None: - self.write_chat_export() - exact_source = ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, workdir: `{self.workspace}`" - "});\ntext(result.output);" - ) - promise_source = ( - "const results = await Promise.all([" - "tools.exec_command({" - f"cmd: `cat {self.tree_file}`, workdir: `{self.workspace}`" - "})," - "tools.exec_command({" - f"cmd: `cat {self.second_tree_file}`, workdir: `{self.workspace}`" - "})]);\nresults.forEach((result) => text(result.output));" - ) - loop_source = ( - "const result = await tools.exec_command({" - "cmd: `for node in " - f"{self.tree_file} {self.second_tree_file}; " - "do sed -n '1,80p' \"$node\"; done`, " - f"workdir: `{self.workspace}`" - "});\ntext(result.output);" - ) - null_sink_source = ( - "const result = await tools.exec_command({" - f"cmd: `test -f {self.tree_file} >/dev/null 2>&1 && " - f"sed -n '1,80p' {self.tree_file}`, " - f"workdir: `{self.workspace}`" - "});\ntext(result.output);" - ) - diagnostic_source = ( - "const results = await Promise.all([" - "tools.exec_command({" - f"cmd: `pwd`, workdir: `{self.workspace}`" - "})," - "tools.exec_command({" - f"cmd: `cat {self.tree_file}`, workdir: `{self.workspace}`" - "})]);\nresults.forEach((result) => text(result.output));" - ) - for filename, call_id, timestamp, source, output in ( - ( - "exec-exact.jsonl", - "call-exec-exact", - "2026-07-22T10:02:00Z", - exact_source, - "# Architecture\n\nChat history is authoritative.", - ), - ( - "exec-promise.jsonl", - "call-exec-promise", - "2026-07-22T10:03:00Z", - promise_source, - [ - "# Architecture\n\nChat history is authoritative.\n" - , - "# Dogfooding\n\nUse First Tree daily.", - ], - ), - ( - "exec-loop.jsonl", - "call-exec-loop", - "2026-07-22T10:04:00Z", - loop_source, - ( - "# Architecture\n\nChat history is authoritative.\n" - "# Dogfooding\n\nUse First Tree daily." - ), - ), - ( - "exec-null-sink.jsonl", - "call-exec-null-sink", - "2026-07-22T10:05:00Z", - null_sink_source, - "# Architecture\n\nChat history is authoritative.", - ), - ( - "exec-diagnostic.jsonl", - "call-exec-diagnostic", - "2026-07-22T10:06:00Z", - diagnostic_source, - [ - str(self.workspace), - "# Architecture\n\nChat history is authoritative.", - ], - ), - ): - write_jsonl( - self.trace_root / filename, - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": timestamp, - "type": "response_item", - "payload": { - "type": "custom_tool_call", - "name": "functions.exec", - "call_id": call_id, - "input": source, - }, - }, - { - "timestamp": timestamp, - "type": "response_item", - "payload": { - "type": "custom_tool_call_output", - "call_id": call_id, - "output": [ - { - "type": "input_text", - "text": ( - "Script completed successfully\nOutput:\n" - ), - }, - *[ - {"type": "input_text", "text": item} - for item in ( - output - if isinstance(output, list) - else [output] - ) - ], - ], - }, - }, - ], - ) - - result = self.collect("exec-orchestration-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "exec-orchestration-candidate.jsonl" - )[0] - self.assertEqual(5, len(candidate["reads"])) - self.assertEqual( - [ - "isolated", - "isolated", - "isolated", - "read_only_composite", - "read_only_composite", - ], - [read["read_mode"] for read in candidate["reads"]], - ) - self.assertEqual( - ["exact", "exact", "exact", "aggregate", "aggregate"], - [read["output_attribution"] for read in candidate["reads"]], - ) - self.assertNotIn( - "Script completed successfully", - "\n".join(read["passage"] for read in candidate["reads"]), - ) - self.assertEqual( - { - "accepted_exact": 1, - "accepted_read_only_composite": 3, - "unresolved_opaque": 1, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertIn( - "tree_read_auxiliary_output_unresolved", - candidate["coverage_gaps"], - ) - - def test_literal_if_guards_preserve_reads_and_fail_closed(self) -> None: - self.write_chat_export() - commands = { - "test-guard": ( - f"if test -f {self.tree_file}; then " - f"sed -n '1,80p' {self.tree_file}; fi" - ), - "bracket-guard": ( - f"if [ -r {self.second_tree_file} ]; then " - f"cat {self.second_tree_file}; fi" - ), - "read-sequence": ( - f"cat {self.tree_file} && " - f"if [ -r {self.second_tree_file} ]; then " - f"sed -n '1,80p' {self.second_tree_file}; fi && " - f"cat {self.tree_file}" - ), - "mixed-diagnostic": ( - "pwd && " - f"cat {self.tree_file} && " - f"if [ -r {self.second_tree_file} ]; then " - f"cat {self.second_tree_file}; fi" - ), - "else-branch": ( - f"if test -f {self.tree_file}; then " - f"cat {self.tree_file}; else cat {self.second_tree_file}; fi" - ), - "dynamic-guard": ( - f'if test -f "$NODE"; then cat {self.tree_file}; fi' - ), - "mutating-body": ( - f"if test -f {self.tree_file}; then " - f"git -C {self.tree_root} pull; cat {self.tree_file}; fi" - ), - } - for index, (label, command) in enumerate(commands.items(), start=1): - call_id = f"call-if-{label}" - write_jsonl( - self.trace_root / f"if-{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T12:0{index}:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": call_id, - "arguments": json.dumps( - { - "cmd": command, - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": f"2026-07-22T12:0{index}:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": call_id, - "output": f"literal-if-output-{label}", - }, - }, - ], - ) - - result = self.collect("literal-if-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "literal-if-candidate.jsonl" - )[0] - self.assertEqual(3, len(candidate["reads"])) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 3, - "unresolved_opaque": 3, - "rejected_unsafe": 1, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unresolved_shell_conditional": 1, - "unresolved_unknown_program": 1, - "unsafe_git_mutation": 1, - "tree_read_auxiliary_output_unresolved": 1, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertIn( - "tree_read_auxiliary_output_unresolved", - candidate["coverage_gaps"], - ) - - def test_semicolonless_forwarding_rechecks_inner_literals( - self, - ) -> None: - self.write_chat_export() - sources = { - "literal-first": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, " - f"workdir: `{self.workspace}`" - "});\n" - "text(result.output)" - ), - "literal-second": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.second_tree_file}`, " - f"workdir: `{self.workspace}`" - "});\n" - "text(result.output)" - ), - "dynamic-command": ( - "const result = await tools.exec_command({" - f"cmd: prefix + `cat {self.tree_file}`, " - f"workdir: `{self.workspace}`" - "});\n" - "text(result.output)" - ), - "dynamic-workdir": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, workdir: selectedWorkdir" - "});\n" - "text(result.output)" - ), - "dynamic-options": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, " - f"workdir: `{self.workspace}`, yield_time_ms: selectedYield" - "});\n" - "text(result.output)" - ), - } - for index, (label, source) in enumerate(sources.items(), start=1): - call_id = f"call-semicolonless-{label}" - write_jsonl( - self.trace_root / f"semicolonless-{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T13:0{index}:00Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call", - "name": "functions.exec", - "call_id": call_id, - "input": source, - }, - }, - { - "timestamp": f"2026-07-22T13:0{index}:01Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call_output", - "call_id": call_id, - "output": [ - { - "type": "input_text", - "text": f"semicolonless-output-{label}", - } - ], - }, - }, - ], - ) - - result = self.collect("semicolonless-candidate.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "semicolonless-candidate.jsonl" - )[0] - self.assertEqual(2, len(candidate["reads"])) - self.assertEqual( - { - "accepted_exact": 2, - "accepted_read_only_composite": 0, - "unresolved_opaque": 3, - "rejected_unsafe": 0, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unresolved_exec_dynamic_arguments": 3, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - - def test_exec_orchestration_fail_closes_dynamic_and_unsafe_shapes(self) -> None: - self.write_chat_export() - shapes = ( - ( - "dynamic", - ( - f"const target = `{self.tree_file}`;\n" - "const result = await tools.exec_command({" - "cmd: `sed -n '1,80p' ${target}`});\n" - "text(result.output);" - ), - ), - ( - "unknown-program", - ( - "const result = await tools.exec_command({" - f"cmd: `awk '{{print}}' {self.tree_file}`" - "});\ntext(result.output);" - ), - ), - ( - "file-redirection", - ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file} > captured.txt`, " - f"workdir: `{self.workspace}`" - "});\ntext(result.output);" - ), - ), - ( - "git-pull", - ( - "const result = await tools.exec_command({" - f"cmd: `git -C {self.tree_root} pull && " - f"cat {self.tree_file}`" - "});\ntext(result.output);" - ), - ), - ( - "outside-tree", - ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file} /private/outside/example.md`" - "});\ntext(result.output);" - ), - ), - ) - for index, (label, source) in enumerate(shapes, start=1): - call_id = f"call-{label}" - write_jsonl( - self.trace_root / f"{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T10:0{index}:00Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call", - "name": "exec", - "call_id": call_id, - "input": source, - }, - }, - { - "timestamp": f"2026-07-22T10:0{index}:01Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call_output", - "call_id": call_id, - "output": f"unsafe-private-{label}", - }, - }, - ], - ) - - result = self.collect("exec-fail-closed.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl(self.artifacts / "exec-fail-closed.jsonl")[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 2, - "rejected_unsafe": 3, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unresolved_exec_wrapper_shape": 1, - "unresolved_unknown_program": 1, - "unsafe_shell_redirection": 1, - "unsafe_git_mutation": 1, - "unsafe_literal_non_tree_path": 1, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertNotIn("unsafe-private-", json.dumps(candidate, sort_keys=True)) - - def test_exec_orchestration_rejects_wrapper_side_effects_and_reordering( - self, - ) -> None: - self.write_chat_export() - call = ( - "tools.exec_command({" - f"cmd: `cat {self.tree_file}`, workdir: `{self.workspace}`" - "})" - ) - second_call = ( - "tools.exec_command({" - f"cmd: `cat {self.second_tree_file}`, workdir: `{self.workspace}`" - "})" - ) - sources = { - "mutated-output": ( - f"let result = await {call}; " - 'result.output = "forged"; text(result.output);' - ), - "aliased-text": ( - f"const emit = text; const result = await {call}; " - 'emit("forged"); text(result.output);' - ), - "promise-side-effect": ( - "const results = await Promise.all([" - f"{call}, {second_call}" - "]); results.forEach((result) => {" - 'result.output = "forged"; text(result.output); });' - ), - "reversed-forwarding": ( - f"const first = await {call}; " - f"const second = await {second_call}; " - "text(second.output); text(first.output);" - ), - "dead-branch": ( - f"const result = await {call}; " - 'if (false) { text("forged"); } text(result.output);' - ), - "property-expression": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, " - 'max_output_tokens: text("forged")' - "}); text(result.output);" - ), - "duplicate-command": ( - "const result = await tools.exec_command({" - f"cmd: `cat {self.tree_file}`, " - f"cmd: `cat {self.second_tree_file}`" - "}); text(result.output);" - ), - "unknown-nested-command": ( - "const results = await Promise.all([" - 'tools.exec_command({cmd: "date"}), ' - f"{call}" - "]); results.forEach((result) => text(result.output));" - ), - "unsafe-nested-command": ( - "const results = await Promise.all([" - 'tools.exec_command({cmd: "curl https://example.invalid"}), ' - f"{call}" - "]); results.forEach((result) => text(result.output));" - ), - } - for index, (label, source) in enumerate(sources.items(), start=1): - call_id = f"call-wrapper-{label}" - write_jsonl( - self.trace_root / f"wrapper-{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T11:0{index}:00Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call", - "name": "functions.exec", - "call_id": call_id, - "input": source, - }, - }, - { - "timestamp": f"2026-07-22T11:0{index}:01Z", - "type": "response_item", - "payload": { - "type": "custom_tool_call_output", - "call_id": call_id, - "output": f"forged-wrapper-output-{label}", - }, - }, - ], - ) - - result = self.collect("exec-wrapper-adversarial.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "exec-wrapper-adversarial.jsonl" - )[0] - self.assertEqual([], candidate["reads"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 0, - "unresolved_opaque": 8, - "rejected_unsafe": 1, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertNotIn( - "forged-wrapper-output", - json.dumps(candidate, sort_keys=True), - ) - - def test_git_rg_and_stderr_shapes_fail_closed(self) -> None: - self.write_chat_export() - root_node = self.tree_root / "NODE.md" - root_node.write_text("# Root\n", encoding="utf-8") - outside = self.root / "outside-secret.txt" - outside.write_text("outside", encoding="utf-8") - shapes = ( - ( - "git-output", - ( - f"git -C {self.tree_root} diff " - "--output=captured.diff && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-no-index", - ( - f"cd {self.tree_root} && " - f"git diff --no-index NODE.md {outside} && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-implicit-diff-helper", - ( - f"git -C {self.tree_root} diff && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-implicit-log-helper", - ( - f"git -C {self.tree_root} log -1 -p && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-implicit-show-helper", - ( - f"git -C {self.tree_root} show HEAD && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-implicit-status-hook", - ( - f"git -C {self.tree_root} status && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-global-exec-path", - ( - f"git --exec-path=/tmp diff && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-inline-config", - ( - "git -c diff.external=/tmp/helper diff && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-index-revision", - ( - f"git -C {self.tree_root} rev-parse --verify :tracked && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "git-external-repository-path", - ( - f"git -C {self.tree_root} rev-parse " - "--resolve-git-dir ~/.git && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "rg-pattern-only", - ( - f"cd {self.tree_root} && " - f"rg --no-config --no-ignore {self.tree_file}" - ), - "opaque", - ), - ( - "rg-file-read", - ( - f"cd {self.tree_root} && " - "rg --no-config --no-ignore " - "Decision system/architecture.md" - ), - "read", - ), - ( - "git-remote", - ( - f"git -C {self.tree_root} remote get-url origin && " - f"cat {self.tree_file}" - ), - "unsafe", - ), - ( - "stderr-null", - f"sed -n '1,80p' {self.tree_file} 2>/dev/null", - "read", - ), - ) - for index, (label, command, expected) in enumerate(shapes, start=1): - call_id = f"call-{label}" - output = ( - "# Architecture\n\n## Decision\n\n" - "Chat history is the authoritative state." - if expected == "read" - else "diagnostic-or-private-output" - ) - write_jsonl( - self.trace_root / f"{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T11:{10 + index:02d}:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": call_id, - "arguments": json.dumps( - { - "cmd": command, - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": f"2026-07-22T11:{10 + index:02d}:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": call_id, - "output": f"Process exited with code 0\n{output}", - }, - }, - ], - ) - - result = self.collect("git-rg-stderr.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl(self.artifacts / "git-rg-stderr.jsonl")[0] - self.assertEqual(2, len(candidate["reads"])) - self.assertEqual( - { - "accepted_exact": 1, - "accepted_read_only_composite": 1, - "unresolved_opaque": 1, - "rejected_unsafe": 11, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unsafe_git_configured_helper": 4, - "unsafe_git_option": 4, - "unsafe_git_unbound_configuration": 3, - "unresolved_tree_path_without_content_reader": 1, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertNotIn( - "tree_read_auxiliary_output_unresolved", - candidate["coverage_gaps"], - ) - self.assertNotIn( - "diagnostic-or-private-output", - json.dumps(candidate, sort_keys=True), - ) - - def test_tree_cli_and_rg_options_use_closed_read_only_grammars(self) -> None: - self.write_chat_export() - outside_ignore = self.root / "outside.ignore" - outside_ignore.write_text("private pattern", encoding="utf-8") - shapes = { - "tree-selector": ( - f"cd {self.tree_root} && " - "first-tree-staging tree tree -P '*.md' -L 2 --no-pull && " - f"cat {self.tree_file}" - ), - "tree-default-refresh": ( - f"cd {self.tree_root} && " - "first-tree-staging tree tree -P '*.md' -L 2 && " - f"cat {self.tree_file}" - ), - "tree-help": ( - "first-tree-staging tree tree --help && " - f"cat {self.tree_file}" - ), - "tree-mutating-namespace": ( - f"cd {self.tree_root} && " - "first-tree-staging chat send tree tree && " - f"cat {self.tree_file}" - ), - "tree-path-qualified": ( - f"/tmp/first-tree-staging tree tree && cat {self.tree_file}" - ), - "rg-file-equals": ( - f"cd {self.tree_root} && " - f"rg --file=/etc/passwd Decision {self.tree_file}" - ), - "rg-ignore-file": ( - f"cd {self.tree_root} && " - f"rg --ignore-file {outside_ignore} " - f"Decision {self.tree_file}" - ), - "rg-unknown-option": ( - f"cd {self.tree_root} && " - f"rg --mystery Decision {self.tree_file}" - ), - "rg-stdin": ( - f"cd {self.tree_root} && " - f"rg -- Decision - && cat {self.tree_file}" - ), - "rg-absolute-executable": ( - f"/tmp/rg Decision {self.tree_file}" - ), - "rg-relative-executable": ( - f"./rg Decision {self.tree_file}" - ), - "bat-bare": ( - f"bat {self.tree_file}" - ), - "bat-forced-paging": ( - f"bat --paging=always {self.tree_file}" - ), - "bat-closed-looking": ( - f"bat --no-config --paging=never {self.tree_file}" - ), - "find-bare": ( - f"find {self.tree_root} -type f && cat {self.tree_file}" - ), - "find-file-valued": ( - f"find -files0-from=/etc/passwd && cat {self.tree_file}" - ), - "ls-bare": ( - f"ls {self.tree_root} && cat {self.tree_file}" - ), - "ls-recursive-dereference": ( - f"ls -RL {self.tree_root} && cat {self.tree_file}" - ), - "head-open-pipeline-option": ( - f"cat {self.tree_file} | head --help" - ), - "tail-open-pipeline-option": ( - f"cat {self.tree_file} | tail --follow=name" - ), - "nl-open-pipeline-option": ( - f"cat {self.tree_file} | nl --help" - ), - "rg-implicit-config": ( - f"cd {self.tree_root} && " - f"rg Decision {self.tree_file}" - ), - "rg-implicit-ignore": ( - f"cd {self.tree_root} && " - f"rg --no-config Decision {self.tree_file}" - ), - "cat-path-qualified": ( - f"/usr/bin/cat {self.tree_file}" - ), - "rg-closed-options": ( - f"cd {self.tree_root} && " - "rg --no-config --no-ignore -n -g '*.md' " - f"Decision {self.tree_file}" - ), - } - for index, (label, command) in enumerate(shapes.items(), start=1): - call_id = f"call-closed-grammar-{label}" - write_jsonl( - self.trace_root / f"closed-grammar-{label}.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": f"2026-07-22T14:{index:02d}:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": call_id, - "arguments": json.dumps( - { - "cmd": command, - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": f"2026-07-22T14:{index:02d}:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": call_id, - "output": ( - "Process exited with code 0\n" - "# Architecture\n\n## Decision\n\n" - f"closed-grammar-output-{label}" - ), - }, - }, - ], - ) - - result = self.collect("closed-command-grammars.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "closed-command-grammars.jsonl" - )[0] - self.assertEqual(1, len(candidate["reads"])) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 1, - "unresolved_opaque": 5, - "rejected_unsafe": 19, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - { - "unsafe_first_tree_command": 2, - "unsafe_or_unresolved_rg": 4, - "unsafe_path_qualified_program": 4, - "unsafe_find_unbound_grammar": 2, - "unsafe_ls_unbound_grammar": 2, - "unsafe_program_bat": 3, - "unsafe_rg_option": 2, - "unresolved_unknown_program": 3, - "tree_read_auxiliary_output_unresolved": 2, - }, - candidate["collector_diagnostics"]["attempt_reason_counts"], - ) - self.assertEqual( - 25, - candidate["collector_diagnostics"]["in_window_tree_read_attempts"], - ) - serialized = json.dumps(candidate, sort_keys=True) - for label in ( - "tree-mutating-namespace", - "tree-default-refresh", - "rg-file-equals", - "rg-ignore-file", - "rg-unknown-option", - "rg-stdin", - "rg-implicit-config", - "rg-implicit-ignore", - "rg-absolute-executable", - "rg-relative-executable", - "bat-bare", - "bat-forced-paging", - "bat-closed-looking", - "find-bare", - "find-file-valued", - "ls-bare", - "ls-recursive-dereference", - "head-open-pipeline-option", - "tail-open-pipeline-option", - "nl-open-pipeline-option", - "cat-path-qualified", - "tree-path-qualified", - ): - self.assertNotIn(f"closed-grammar-output-{label}", serialized) - - def test_preflight_rejects_conflicting_mirror_and_mirror_only_trace( - self, - ) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "conflicting-mirror.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - context_mirror_row(UNAUTHORIZED_CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-conflicting-mirror", - "output": "conflicting-mirror-private-sentinel", - }, - }, - ], - ) - write_jsonl( - self.trace_root / "mirror-only.jsonl", - [ - session_meta(self.workspace), - context_mirror_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-mirror-only", - "output": "mirror-only-private-sentinel", - }, - }, - ], - ) - - result = self.collect("rejected-envelope-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "rejected-envelope-candidates.jsonl" - )[0] - self.assertEqual([], candidate["mapped_trace_files"]) - self.assertEqual([], candidate["reads"]) - self.assertIn( - "codex_trace_preflight_malformed_or_ambiguous", - candidate["coverage_gaps"], - ) - self.assertIn("codex_trace_preflight_unmapped", candidate["coverage_gaps"]) - serialized = json.dumps(candidate, sort_keys=True) - self.assertNotIn("conflicting-mirror-private-sentinel", serialized) - self.assertNotIn("mirror-only-private-sentinel", serialized) - - def test_preflight_rejects_two_canonical_chat_ids(self) -> None: - self.write_chat_export() - write_jsonl( - self.trace_root / "canonical-chat-drift.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - context_mirror_row(CHAT_ID), - context_row(SECOND_CHAT_ID), - context_mirror_row(SECOND_CHAT_ID), - ], - ) - - result = self.collect("canonical-chat-drift-candidates.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl( - self.artifacts / "canonical-chat-drift-candidates.jsonl" - )[0] - self.assertEqual([], candidate["mapped_trace_files"]) - self.assertIn( - "codex_trace_preflight_ambiguous_chat", - candidate["coverage_gaps"], - ) - - def test_report_is_task_first_deterministic_and_rejects_invalid_effects(self) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - task = self.task_judgment(candidate) - - first = self.report( - [task], - evidence_name="minimal-evidence-one.jsonl", - report_name="minimal-REPORT-one.md", - ) - second = self.report( - [task], - evidence_name="minimal-evidence-two.jsonl", - report_name="minimal-REPORT-two.md", - ) - self.assertEqual(0, first.returncode, first.stderr) - self.assertEqual(0, second.returncode, second.stderr) - self.assertEqual( - (self.artifacts / "minimal-evidence-one.jsonl").read_bytes(), - (self.artifacts / "minimal-evidence-two.jsonl").read_bytes(), - ) - self.assertEqual( - (self.artifacts / "minimal-REPORT-one.md").read_bytes(), - (self.artifacts / "minimal-REPORT-two.md").read_bytes(), - ) - markdown = (self.artifacts / "minimal-REPORT-one.md").read_text( - encoding="utf-8" - ) - self.assertIn("| Clear Tasks | 1 |", markdown) - self.assertIn("| Read observed Tasks | 1 |", markdown) - self.assertIn("| Effect Tasks | 1 |", markdown) - self.assertIn("| Observed Read without Effect | 0 |", markdown) - self.assertIn("| constrained | 1 |", markdown) - self.assertIn("There is no minimum Task quota", markdown) - self.assertIn("Tree-read grammar conservation", markdown) - evidence = read_jsonl(self.artifacts / "minimal-evidence-one.jsonl") - self.assertEqual("constrained", evidence[0]["effects"][0]["type"]) - self.assertIn("effect_id", evidence[0]["effects"][0]) - self.assertNotIn("derived_support", evidence[0]["effects"][0]) - - invalid = json.loads(json.dumps(task)) - invalid["effect"]["type"] = "informed" - invalid_result = self.report( - [invalid], - evidence_name="minimal-invalid-evidence.jsonl", - report_name="minimal-invalid-REPORT.md", - ) - self.assertEqual(2, invalid_result.returncode) - self.assertIn(".type must be one of", invalid_result.stderr) - - missing_anchor = json.loads(json.dumps(task)) - missing_anchor["effect"].pop("outcome_anchor") - missing_anchor_result = self.report( - [missing_anchor], - evidence_name="minimal-missing-anchor-evidence.jsonl", - report_name="minimal-missing-anchor-REPORT.md", - ) - self.assertEqual(2, missing_anchor_result.returncode) - self.assertIn("outcome_message_id", missing_anchor_result.stderr) - - def test_report_survives_tree_advance_and_rejects_v02_confidence_fields( - self, - ) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - task = self.task_judgment(candidate) - - self.tree_file.write_text( - "# Architecture\n\n## Decision\n\nChat history remains authoritative.\n", - encoding="utf-8", - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "add", "."], - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - ["git", "-C", str(self.tree_root), "commit", "-m", "advance"], - check=True, - capture_output=True, - text=True, - ) - advanced_commit = subprocess.run( - ["git", "-C", str(self.tree_root), "rev-parse", "HEAD"], - check=True, - capture_output=True, - text=True, - ).stdout.strip() - subprocess.run( - [ - "git", - "-C", - str(self.tree_root), - "update-ref", - "refs/remotes/origin/main", - advanced_commit, - ], - check=True, - capture_output=True, - text=True, - ) - - accepted = self.report( - [task], - evidence_name="source-neutral-evidence.jsonl", - report_name="source-neutral-REPORT.md", - ) - self.assertEqual(0, accepted.returncode, accepted.stderr) - - for field, value in ( - ("original_judgment", "probable"), - ("rubric", {"real_read": True}), - ("support", "limited"), - ): - legacy = json.loads(json.dumps(task)) - legacy["effect"][field] = value - result = self.report( - [legacy], - evidence_name=f"legacy-{field}-evidence.jsonl", - report_name=f"legacy-{field}-REPORT.md", - ) - self.assertEqual(2, result.returncode) - self.assertIn("superseded model", result.stderr) - - def test_report_supports_multiple_effects_and_enforces_inventory_digest( - self, - ) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - second_choice_id = "second-effect-choice" - second_choice = { - "message_id": second_choice_id, - "created_at": "2026-07-22T10:04:00Z", - "sender_id": AGENT_ID, - "sender_kind": "agent", - "content": "I will retain the current source and document its boundary.", - } - candidate["visible_messages"].append(second_choice) - candidate["visible_choice_candidates"].append( - { - field: second_choice[field] - for field in ("message_id", "created_at", "sender_id", "content") - } - ) - candidate["chat"]["message_count"] += 1 - write_jsonl(self.artifacts / "candidates.jsonl", [candidate]) - - task = self.task_judgment(candidate) - task["source_fragments"][0]["message_ids"].insert(-1, second_choice_id) - read_id = candidate["reads"][0]["read_id"] - task["effects"] = [ - { - "type": "confirmed", - "read_ids": [read_id], - "choice_message_ids": [second_choice_id], - "outcome_message_id": MESSAGE_ID, - "summary": "The Tree removed uncertainty about retaining the source.", - }, - { - "type": "constrained", - "read_ids": [read_id], - "choice_message_ids": [MESSAGE_ID], - "outcome_message_id": MESSAGE_ID, - "summary": "The Tree ruled out creating a second state table.", - }, - ] - task.pop("effect") - task["effect_reason"] = None - accepted = self.report( - [task], - evidence_name="multi-effect-evidence.jsonl", - report_name="multi-effect-REPORT.md", - ) - self.assertEqual(0, accepted.returncode, accepted.stderr) - evidence = read_jsonl(self.artifacts / "multi-effect-evidence.jsonl") - self.assertEqual(2, len(evidence[0]["effects"])) - self.assertEqual( - 2, len({effect["effect_id"] for effect in evidence[0]["effects"]}) - ) - report = (self.artifacts / "multi-effect-REPORT.md").read_text( - encoding="utf-8" - ) - self.assertIn("| Effect Tasks | 1 |", report) - self.assertIn("| Effects | 2 |", report) - self.assertIn("| confirmed | 1 |", report) - self.assertIn("| constrained | 1 |", report) - valid_read_rows = read_jsonl( - self.artifacts / "read-attributions.jsonl" - ) - valid_effect_rows = read_jsonl( - self.artifacts / "effect-judgments.jsonl" - ) - - same_time_candidate = json.loads(json.dumps(candidate)) - same_time_candidate["reads"][0]["completed_at"] = second_choice[ - "created_at" - ] - write_jsonl( - self.artifacts / "same-time-candidates.jsonl", - [same_time_candidate], - ) - same_time_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "same-time-candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "same-time-evidence.jsonl"), - "--report-output", - str(self.artifacts / "same-time-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, same_time_result.returncode) - self.assertIn( - "complete before its earliest choice", - same_time_result.stderr, - ) - - reversed_read_candidate = json.loads(json.dumps(candidate)) - reversed_read_candidate["reads"][0]["timestamp"] = ( - "2026-07-22T10:03:00Z" - ) - reversed_read_candidate["reads"][0]["completed_at"] = ( - "2026-07-22T10:02:00Z" - ) - write_jsonl( - self.artifacts / "reversed-read-candidates.jsonl", - [reversed_read_candidate], - ) - reversed_read_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "reversed-read-candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "reversed-read-evidence.jsonl"), - "--report-output", - str(self.artifacts / "reversed-read-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, reversed_read_result.returncode) - self.assertIn( - "must complete after it starts", - reversed_read_result.stderr, - ) - - zero_duration_read_candidate = json.loads(json.dumps(candidate)) - zero_duration_read_candidate["reads"][0]["completed_at"] = ( - zero_duration_read_candidate["reads"][0]["timestamp"] - ) - write_jsonl( - self.artifacts / "zero-duration-read-candidates.jsonl", - [zero_duration_read_candidate], - ) - zero_duration_read_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "zero-duration-read-candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "zero-duration-read-evidence.jsonl"), - "--report-output", - str(self.artifacts / "zero-duration-read-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, zero_duration_read_result.returncode) - self.assertIn( - "must complete after it starts", - zero_duration_read_result.stderr, - ) - - frozen_rows = read_jsonl(self.artifacts / "task-inventory.jsonl") - tampered_rows = json.loads(json.dumps(frozen_rows)) - tampered_rows[0]["objective"] = "A changed objective after freeze" - write_jsonl(self.artifacts / "task-inventory.jsonl", tampered_rows) - tampered_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "tampered-evidence.jsonl"), - "--report-output", - str(self.artifacts / "tampered-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, tampered_result.returncode) - self.assertIn("digest does not match", tampered_result.stderr) - write_jsonl(self.artifacts / "task-inventory.jsonl", frozen_rows) - - leaked_draft = json.loads(json.dumps(frozen_rows)) - for row in leaked_draft: - row.pop("inventory_sha256") - leaked_draft[0]["read"] = { - "status": "observed", - "read_ids": [read_id], - } - write_jsonl( - self.artifacts / "leaked-task-inventory-draft.jsonl", - leaked_draft, - ) - leaked_result = run_cli( - "freeze-tasks", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--task-source", - str(self.artifacts / "task-source.jsonl"), - "--task-inventory-draft", - str(self.artifacts / "leaked-task-inventory-draft.jsonl"), - "--task-inventory-output", - str(self.artifacts / "leaked-task-inventory.jsonl"), - ) - self.assertEqual(2, leaked_result.returncode) - self.assertIn("pure Task inventory", leaked_result.stderr) - - unknown_draft = json.loads(json.dumps(leaked_draft)) - unknown_draft[0].pop("read") - unknown_draft[0]["tree_passage"] = "derived content" - write_jsonl( - self.artifacts / "unknown-task-inventory-draft.jsonl", - unknown_draft, - ) - unknown_task_result = run_cli( - "freeze-tasks", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--task-source", - str(self.artifacts / "task-source.jsonl"), - "--task-inventory-draft", - str(self.artifacts / "unknown-task-inventory-draft.jsonl"), - "--task-inventory-output", - str(self.artifacts / "unknown-task-inventory.jsonl"), - ) - self.assertEqual(2, unknown_task_result.returncode) - self.assertIn("unsupported field", unknown_task_result.stderr) - - duplicate_choice = json.loads(json.dumps(task)) - duplicate_choice["effects"][1]["choice_message_ids"] = [ - second_choice_id - ] - rejected = self.report( - [duplicate_choice], - evidence_name="duplicate-effect-evidence.jsonl", - report_name="duplicate-effect-REPORT.md", - ) - self.assertEqual(2, rejected.returncode) - self.assertIn("reused across independent Effects", rejected.stderr) - write_jsonl( - self.artifacts / "read-attributions.jsonl", - valid_read_rows, - ) - write_jsonl( - self.artifacts / "effect-judgments.jsonl", - valid_effect_rows, - ) - - read_rows = json.loads(json.dumps(valid_read_rows)) - read_rows[0]["inventory_sha256"] = "0" * 64 - write_jsonl(self.artifacts / "read-attributions.jsonl", read_rows) - digest_rejected = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "digest-evidence.jsonl"), - "--report-output", - str(self.artifacts / "digest-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, digest_rejected.returncode) - self.assertIn("frozen Task inventory", digest_rejected.stderr) - write_jsonl( - self.artifacts / "read-attributions.jsonl", - valid_read_rows, - ) - - effect_rows = json.loads(json.dumps(valid_effect_rows)) - effect_rows[0]["inventory_sha256"] = "0" * 64 - write_jsonl(self.artifacts / "effect-judgments.jsonl", effect_rows) - effect_digest_rejected = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "effect-digest-evidence.jsonl"), - "--report-output", - str(self.artifacts / "effect-digest-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, effect_digest_rejected.returncode) - self.assertIn( - "frozen Task inventory", - effect_digest_rejected.stderr, - ) - write_jsonl( - self.artifacts / "effect-judgments.jsonl", - valid_effect_rows, - ) - - unknown_read_rows = json.loads(json.dumps(valid_read_rows)) - unknown_read_rows[0]["tree_passage"] = "derived content" - write_jsonl( - self.artifacts / "read-attributions.jsonl", - unknown_read_rows, - ) - unknown_read_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "unknown-read-evidence.jsonl"), - "--report-output", - str(self.artifacts / "unknown-read-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, unknown_read_result.returncode) - self.assertIn("unsupported field", unknown_read_result.stderr) - write_jsonl( - self.artifacts / "read-attributions.jsonl", - valid_read_rows, - ) - - unknown_effect_rows = json.loads(json.dumps(valid_effect_rows)) - unknown_effect_rows[0]["effect_claim"] = "derived content" - write_jsonl( - self.artifacts / "effect-judgments.jsonl", - unknown_effect_rows, - ) - unknown_effect_result = run_cli( - "report", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--task-inventory", - str(self.artifacts / "task-inventory.jsonl"), - "--read-attributions", - str(self.artifacts / "read-attributions.jsonl"), - "--effect-judgments", - str(self.artifacts / "effect-judgments.jsonl"), - "--evidence-output", - str(self.artifacts / "unknown-effect-evidence.jsonl"), - "--report-output", - str(self.artifacts / "unknown-effect-REPORT.md"), - "--generated-at", - NOW, - ) - self.assertEqual(2, unknown_effect_result.returncode) - self.assertIn("unsupported field", unknown_effect_result.stderr) - - def test_report_handles_excluded_task( - self, - ) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - excluded = { - "schema_version": 4, - "task_id": "excluded-1", - "status": "excluded", - "objective": None, - "object_scope": "unknown", - "outcome": None, - "started_at": "2026-07-22T10:00:00Z", - "ended_at": "2026-07-22T10:06:00Z", - "source_fragments": [ - { - "audit_id": candidate["audit_id"], - "message_ids": [MESSAGE_ID], - } - ], - "exclusion_kind": "missing_objective", - "exclusion_reason": "No clear objective and outcome boundary.", - } - result = self.report( - [excluded], - evidence_name="minimal-outside-evidence.jsonl", - report_name="minimal-outside-REPORT.md", - ) - self.assertEqual(0, result.returncode, result.stderr) - report = (self.artifacts / "minimal-outside-REPORT.md").read_text( - encoding="utf-8" - ) - self.assertIn("| Excluded Tasks | 1 |", report) - self.assertIn("## Excluded Tasks", report) - self.assertIn("No clear objective and outcome boundary.", report) - self.assertIn("## Frozen Task Inventory", report) - self.assertIn("Inventory digest:", report) - self.assertIn( - "digest still binds the excluded-candidate inventory", - report, - ) - - for legacy_field, legacy_value in ( - ("task_type", "solution_design"), - ("sampling_order", 1), - ("saturation_signals", []), - ): - legacy = json.loads(json.dumps(excluded)) - legacy[legacy_field] = legacy_value - rejected = self.report( - [legacy], - evidence_name=f"excluded-legacy-{legacy_field}-evidence.jsonl", - report_name=f"excluded-legacy-{legacy_field}-REPORT.md", - ) - self.assertEqual(2, rejected.returncode) - self.assertIn("pure Task inventory", rejected.stderr) - - def test_one_chat_splits_into_two_tasks_and_duplicate_read_is_rejected( - self, - ) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - candidate["visible_messages"].extend( - [ - { - "message_id": SECOND_OBJECTIVE_MESSAGE_ID, - "created_at": "2026-07-22T10:05:10Z", - "sender_id": AGENT_ID, - "sender_kind": None, - "content": "I will explain a separate result.", - }, - { - "message_id": SECOND_MESSAGE_ID, - "created_at": "2026-07-22T10:05:30Z", - "sender_id": AGENT_ID, - "sender_kind": None, - "content": "A separate task completed.", - }, - ] - ) - candidate["chat"]["message_count"] += 2 - write_jsonl(self.artifacts / "candidates.jsonl", [candidate]) - - first_task = self.task_judgment(candidate) - second_task = self.task_judgment( - candidate, - task_id="task-2", - message_id=SECOND_MESSAGE_ID, - objective_message_id=SECOND_OBJECTIVE_MESSAGE_ID, - read_status="unresolved", - read_ids=[], - effect=None, - ) - second_task["objective"] = "Explain a separate result" - second_task["object_scope"] = "independent outcome" - second_task["outcome"] = "Delivered the separate explanation." - - split = self.report( - [first_task, second_task], - evidence_name="minimal-split-evidence.jsonl", - report_name="minimal-split-REPORT.md", - ) - self.assertEqual(0, split.returncode, split.stderr) - report = (self.artifacts / "minimal-split-REPORT.md").read_text( - encoding="utf-8" - ) - self.assertIn("| Clear Tasks | 2 |", report) - self.assertIn("| Read unresolved Tasks | 1 |", report) - self.assertIn("| Observed Read without Effect | 0 |", report) - - unresolved_with_read = json.loads(json.dumps(second_task)) - unresolved_with_read["read"]["read_ids"] = [ - candidate["reads"][0]["read_id"] - ] - unresolved_result = self.report( - [unresolved_with_read], - evidence_name="minimal-unresolved-read-evidence.jsonl", - report_name="minimal-unresolved-read-REPORT.md", - ) - self.assertEqual(2, unresolved_result.returncode) - self.assertIn("must not contain read_ids", unresolved_result.stderr) - - duplicate = self.task_judgment( - candidate, - task_id="task-2", - message_id=SECOND_MESSAGE_ID, - objective_message_id=SECOND_OBJECTIVE_MESSAGE_ID, - read_ids=[candidate["reads"][0]["read_id"]], - effect=None, - ) - duplicate_result = self.report( - [first_task, duplicate], - evidence_name="minimal-duplicate-evidence.jsonl", - report_name="minimal-duplicate-REPORT.md", - ) - self.assertEqual(2, duplicate_result.returncode) - self.assertIn("outside the Task window", duplicate_result.stderr) - - def test_task_inventory_sources_and_weak_fragments_are_enforced( - self, - ) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - assignment_id = "assignment-message" - continuation_id = "continuation-message" - candidate["visible_messages"].extend( - [ - { - "message_id": assignment_id, - "created_at": "2026-07-22T10:00:00Z", - "sender_id": OTHER_AGENT_ID, - "sender_kind": "human", - "content": ( - "Choose the authoritative state source and deliver the " - "decision." - ), - }, - { - "message_id": continuation_id, - "created_at": "2026-07-22T10:04:00Z", - "sender_id": OTHER_AGENT_ID, - "sender_kind": "human", - "content": "Please continue.", - }, - ] - ) - candidate["chat"]["message_count"] += 2 - write_jsonl(self.artifacts / "candidates.jsonl", [candidate]) - - valid = self.task_judgment(candidate) - valid["started_at"] = "2026-07-22T10:00:00Z" - valid["source_fragments"][0]["message_ids"] = [ - assignment_id, - ACCEPTANCE_MESSAGE_ID, - continuation_id, - MESSAGE_ID, - ] - valid["episode"]["objective_anchor_message_ids"] = [assignment_id] - valid["episode"]["outcome_anchor_message_ids"] = [MESSAGE_ID] - accepted = self.report( - [valid], - evidence_name="inventory-valid-evidence.jsonl", - report_name="inventory-valid-REPORT.md", - ) - self.assertEqual(0, accepted.returncode, accepted.stderr) - report = (self.artifacts / "inventory-valid-REPORT.md").read_text( - encoding="utf-8" - ) - self.assertIn("## Frozen Task Inventory", report) - self.assertIn("Objective sources", report) - self.assertIn("Primary deliverable", report) - task_source = ( - self.artifacts / "task-source.jsonl" - ).read_text(encoding="utf-8") - self.assertNotIn("decision_receipt", task_source) - self.assertNotIn('"reads"', task_source) - - weak_variants = ( - "please continue", - "请继续", - "修一下吧", - "status please", - "please continue fixing it", - "please fix it", - "continue the work", - "proceed", - "go ahead", - "keep going", - "carry on", - "fix that", - "继续修一下", - "修这个", - "“please continue”", - "请继续（谢谢）", - "@agent-one @agent-two，请继续", - ) - for index, source_content in enumerate(weak_variants): - weak_candidate = json.loads(json.dumps(candidate)) - next( - message - for message in weak_candidate["visible_messages"] - if message["message_id"] == assignment_id - )["content"] = source_content - write_jsonl(self.artifacts / "candidates.jsonl", [weak_candidate]) - result = self.report( - [valid], - evidence_name=f"inventory-weak-{index}-evidence.jsonl", - report_name=f"inventory-weak-{index}-REPORT.md", - ) - self.assertEqual(2, result.returncode) - self.assertIn("concrete objective-source", result.stderr) - - concrete_variants = ( - "Please continue the state-source design and deliver the decision.", - "Proceed with the schema-v4 validator and deliver the PR.", - "请继续完成状态源方案并交付独立决定", - "@agent-one，请继续完成状态源方案并交付独立决定", - ) - for index, source_content in enumerate(concrete_variants): - concrete_candidate = json.loads(json.dumps(candidate)) - next( - message - for message in concrete_candidate["visible_messages"] - if message["message_id"] == assignment_id - )["content"] = source_content - write_jsonl( - self.artifacts / "candidates.jsonl", - [concrete_candidate], - ) - result = self.report( - [valid], - evidence_name=f"inventory-concrete-{index}-evidence.jsonl", - report_name=f"inventory-concrete-{index}-REPORT.md", - ) - self.assertEqual(0, result.returncode, result.stderr) - - write_jsonl(self.artifacts / "candidates.jsonl", [candidate]) - weak_objective = json.loads(json.dumps(valid)) - weak_objective["objective"] = "修一下吧" - weak_result = self.report( - [weak_objective], - evidence_name="inventory-weak-objective-evidence.jsonl", - report_name="inventory-weak-objective-REPORT.md", - ) - self.assertEqual(2, weak_result.returncode) - self.assertIn("only a continuation", weak_result.stderr) - - human_outcome_id = "human-outcome-message" - mixed_outcome_candidate = json.loads(json.dumps(candidate)) - mixed_outcome_candidate["visible_messages"].append( - { - "message_id": human_outcome_id, - "created_at": "2026-07-22T10:05:30Z", - "sender_id": OTHER_AGENT_ID, - "sender_kind": "human", - "content": "Thanks, this delivery is complete.", - } - ) - mixed_outcome_candidate["chat"]["message_count"] += 1 - write_jsonl( - self.artifacts / "candidates.jsonl", - [mixed_outcome_candidate], - ) - mixed_outcome = json.loads(json.dumps(valid)) - mixed_outcome["ended_at"] = "2026-07-22T10:05:30Z" - mixed_outcome["source_fragments"][0]["message_ids"].append( - human_outcome_id - ) - mixed_outcome["episode"]["outcome_anchor_message_ids"] = [ - human_outcome_id - ] - mixed_result = self.report( - [mixed_outcome], - evidence_name="inventory-human-outcome-evidence.jsonl", - report_name="inventory-human-outcome-REPORT.md", - ) - self.assertEqual(2, mixed_result.returncode) - self.assertIn("every outcome source", mixed_result.stderr) - - missing_sources = json.loads(json.dumps(valid)) - missing_sources.pop("episode") - missing_result = self.report( - [missing_sources], - evidence_name="inventory-missing-sources-evidence.jsonl", - report_name="inventory-missing-sources-REPORT.md", - ) - self.assertEqual(2, missing_result.returncode) - self.assertIn("objective_source_message_ids", missing_result.stderr) - - schema_v3 = json.loads(json.dumps(valid)) - schema_v3["schema_version"] = 3 - schema_result = self.report( - [schema_v3], - evidence_name="inventory-schema-v3-evidence.jsonl", - report_name="inventory-schema-v3-REPORT.md", - ) - self.assertEqual(2, schema_result.returncode) - self.assertIn("schema_version 4", schema_result.stderr) - - collapsed = self.task_judgment( - candidate, - message_id=ACCEPTANCE_MESSAGE_ID, - objective_message_id=ACCEPTANCE_MESSAGE_ID, - read_status="unresolved", - read_ids=[], - effect=None, - ) - collapsed_result = self.report( - [collapsed], - evidence_name="inventory-collapsed-evidence.jsonl", - report_name="inventory-collapsed-REPORT.md", - ) - self.assertEqual(2, collapsed_result.returncode) - self.assertIn("must be separate", collapsed_result.stderr) - - early_outcome_id = "early-outcome-message" - early_candidate = json.loads(json.dumps(candidate)) - early_candidate["visible_messages"].append( - { - "message_id": early_outcome_id, - "created_at": "2026-07-22T10:01:30Z", - "sender_id": AGENT_ID, - "sender_kind": "agent", - "content": "An early intermediate state was recorded.", - } - ) - early_candidate["chat"]["message_count"] += 1 - write_jsonl(self.artifacts / "candidates.jsonl", [early_candidate]) - early_effect = json.loads(json.dumps(valid)) - early_effect["source_fragments"][0]["message_ids"].insert( - -1, early_outcome_id - ) - early_effect["effect"]["outcome_anchor"] = early_outcome_id - early_result = self.report( - [early_effect], - evidence_name="effect-early-outcome-evidence.jsonl", - report_name="effect-early-outcome-REPORT.md", - ) - self.assertEqual(2, early_result.returncode) - self.assertIn("outcome precedes", early_result.stderr) - - def test_task_source_ignores_malformed_derived_evidence(self) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - - baseline_result = run_cli( - "task-source", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "candidates.jsonl"), - "--output", - str(self.artifacts / "task-source-baseline.jsonl"), - ) - self.assertEqual(0, baseline_result.returncode, baseline_result.stderr) - baseline = ( - self.artifacts / "task-source-baseline.jsonl" - ).read_bytes() - inventory = self.task_judgment(candidate) - - candidate["collector_diagnostics"] = "malformed" - candidate["tree_source_snapshot"] = {"status": "future-invalid"} - candidate["reads"] = [{"malformed": True}] - candidate["visible_choice_candidates"] = [{"malformed": True}] - candidate["visible_tree_mentions"] = "malformed" - candidate["visible_messages"][0]["decision_receipt"] = { - "future": "invalid" - } - write_jsonl( - self.artifacts / "derived-damage-candidates.jsonl", - [candidate], - ) - damaged_result = run_cli( - "task-source", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--candidates", - str(self.artifacts / "derived-damage-candidates.jsonl"), - "--output", - str(self.artifacts / "task-source-derived-damage.jsonl"), - ) - self.assertEqual(0, damaged_result.returncode, damaged_result.stderr) - self.assertEqual( - baseline, - (self.artifacts / "task-source-derived-damage.jsonl").read_bytes(), - ) - - task_source = read_jsonl( - self.artifacts / "task-source-derived-damage.jsonl" - ) - task_source[0]["reads"] = [] - write_jsonl( - self.artifacts / "task-source-with-derived-field.jsonl", - task_source, - ) - inventory.pop("read") - inventory.pop("effect") - inventory.pop("effect_reason") - episode = inventory.pop("episode") - inventory["objective_source_message_ids"] = episode[ - "objective_anchor_message_ids" - ] - inventory["outcome_source_message_ids"] = episode[ - "outcome_anchor_message_ids" - ] - inventory["primary_deliverable"] = episode["primary_deliverable"] - write_jsonl( - self.artifacts / "stage-one-inventory-draft.jsonl", - [inventory], - ) - rejected = run_cli( - "freeze-tasks", - "--artifact-root", - str(self.artifacts), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--task-source", - str(self.artifacts / "task-source-with-derived-field.jsonl"), - "--task-inventory-draft", - str(self.artifacts / "stage-one-inventory-draft.jsonl"), - "--task-inventory-output", - str(self.artifacts / "stage-one-inventory.jsonl"), - ) - self.assertEqual(2, rejected.returncode) - self.assertIn("unsupported field", rejected.stderr) - - def test_cross_chat_task_requires_explicit_linkage(self) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - first = read_jsonl(self.artifacts / "candidates.jsonl")[0] - second = json.loads(json.dumps(first)) - second["audit_id"] = f"{SECOND_CHAT_ID}@{AGENT_ID}" - second["chat"]["chat_id"] = SECOND_CHAT_ID - second["chat"]["title"] = "Follow-up Chat" - second["mapped_trace_files"] = [] - second["reads"] = [] - second["visible_messages"] = [ - { - "message_id": SECOND_MESSAGE_ID, - "created_at": "2026-07-22T10:05:30Z", - "sender_id": AGENT_ID, - "sender_kind": None, - "content": "Follow-up delivery for the same objective.", - } - ] - second["visible_choice_candidates"] = [] - second["visible_tree_mentions"] = [] - second["chat"]["message_count"] = 1 - write_jsonl(self.artifacts / "candidates.jsonl", [first, second]) - - task = self.task_judgment(first) - task["source_fragments"].append( - { - "audit_id": second["audit_id"], - "message_ids": [SECOND_MESSAGE_ID], - } - ) - task["episode"]["outcome_anchor_message_ids"].append( - SECOND_MESSAGE_ID - ) - task["ended_at"] = "2026-07-22T10:05:30Z" - no_linkage = self.report( - [task], - evidence_name="no-linkage-evidence.jsonl", - report_name="no-linkage-REPORT.md", - ) - self.assertEqual(2, no_linkage.returncode) - self.assertIn("requires one explicit shared linkage", no_linkage.stderr) - - linkage = { - "kind": "same_objective_delivery", - "key": "deliver-one-state-source", - } - for fragment in task["source_fragments"]: - fragment["linkage"] = linkage - linked = self.report( - [task], - evidence_name="linked-evidence.jsonl", - report_name="linked-REPORT.md", - ) - self.assertEqual(0, linked.returncode, linked.stderr) - - def test_every_available_task_reports_without_a_sampling_gate(self) -> None: - self.write_chat_export() - self.write_trace_fixtures() - collected = self.collect("candidates.jsonl") - self.assertEqual(0, collected.returncode, collected.stderr) - candidate = read_jsonl(self.artifacts / "candidates.jsonl")[0] - candidate["candidate_status"] = "outside_candidate_set" - candidate["mapped_trace_files"] = [] - candidate["reads"] = [] - candidate["visible_choice_candidates"] = [] - candidate["visible_tree_mentions"] = [] - candidate["visible_messages"] = [ - message - for index in range(1, 45) - for message in ( - { - "message_id": f"sample-objective-{index:03d}", - "created_at": "2026-07-22T10:04:00Z", - "sender_id": AGENT_ID, - "sender_kind": None, - "content": f"I will complete sample task {index}.", - }, - { - "message_id": f"sample-message-{index:03d}", - "created_at": "2026-07-22T10:05:00Z", - "sender_id": AGENT_ID, - "sender_kind": None, - "content": f"Sample task {index} completed.", - }, - ) - ] - candidate["chat"]["message_count"] = 88 - write_jsonl(self.artifacts / "candidates.jsonl", [candidate]) - tasks = [ - self.task_judgment( - candidate, - task_id=f"sample-task-{index:03d}", - message_id=f"sample-message-{index:03d}", - objective_message_id=f"sample-objective-{index:03d}", - read_status="unresolved", - read_ids=[], - effect=None, - ) - for index in range(1, 45) - ] - - for count in (1, 16, 44): - result = self.report( - tasks[:count], - evidence_name=f"sample-{count}-evidence.jsonl", - report_name=f"sample-{count}-REPORT.md", - ) - self.assertEqual(0, result.returncode, result.stderr) - report = (self.artifacts / f"sample-{count}-REPORT.md").read_text( - encoding="utf-8" - ) - self.assertIn(f"| Clear Tasks | {count} |", report) - self.assertIn(f"| Read unresolved Tasks | {count} |", report) - self.assertIn("| Observed Read without Effect | 0 |", report) - self.assertIn("There is no minimum Task quota", report) - self.assertNotIn("Effect saturation", report) - self.assertNotIn("Status: `saturated`", report) - self.assertNotIn("task type", report.lower()) - - write_jsonl( - self.artifacts / "reviewed-baseline.jsonl", - [ - { - "schema_version": 4, - "basis": "separately_reviewed_task_cases", - "reviewed_at": "2026-07-22T12:00:00Z", - "evidence_anchor": { - "artifact_id": "reviewed-task-cases-v2", - "sha256": "a" * 64, - }, - "clear_tasks": 44, - "effect_tasks": 16, - "effects": 16, - "effect_counts": { - "confirmed": 2, - "constrained": 8, - "redirected": 5, - "conflicted": 1, - }, - } - ], - ) - baseline = self.report( - tasks[:16], - evidence_name="minimal-baseline-evidence.jsonl", - report_name="minimal-baseline-REPORT.md", - reviewed_baseline_name="reviewed-baseline.jsonl", - ) - self.assertEqual(0, baseline.returncode, baseline.stderr) - baseline_report = ( - self.artifacts / "minimal-baseline-REPORT.md" - ).read_text(encoding="utf-8") - self.assertIn("## Separately Reviewed Historical Baseline", baseline_report) - self.assertIn("| Reviewed effect Tasks | 16 |", baseline_report) - self.assertNotIn("Derived support", baseline_report) - self.assertNotIn("support_counts", baseline_report) - - write_jsonl( - self.artifacts / "zero-effect-baseline.jsonl", - [ - { - "schema_version": 4, - "basis": "separately_reviewed_task_cases", - "reviewed_at": "2026-07-22T12:00:00Z", - "evidence_anchor": { - "artifact_id": "reviewed-zero-effect-cases", - "sha256": "b" * 64, - }, - "clear_tasks": 5, - "effect_tasks": 0, - "effects": 0, - "effect_counts": { - "confirmed": 0, - "constrained": 0, - "redirected": 0, - "conflicted": 0, - }, - } - ], - ) - zero_baseline = self.report( - tasks[:1], - evidence_name="zero-baseline-evidence.jsonl", - report_name="zero-baseline-REPORT.md", - reviewed_baseline_name="zero-effect-baseline.jsonl", - ) - self.assertEqual(0, zero_baseline.returncode, zero_baseline.stderr) - zero_report = ( - self.artifacts / "zero-baseline-REPORT.md" - ).read_text(encoding="utf-8") - self.assertIn("| Reviewed effect Tasks | 0 |", zero_report) - self.assertIn("| Reviewed Effects | 0 |", zero_report) - - for legacy_field, legacy_value in ( - ("task_type", "solution_design"), - ("sampling_order", 1), - ("saturation_signals", []), - ): - legacy = json.loads(json.dumps(tasks[0])) - legacy[legacy_field] = legacy_value - rejected = self.report( - [legacy], - evidence_name=f"legacy-{legacy_field}-evidence.jsonl", - report_name=f"legacy-{legacy_field}-REPORT.md", - ) - self.assertEqual(2, rejected.returncode) - self.assertIn("pure Task inventory", rejected.stderr) - - def test_symlinked_artifact_output_is_rejected(self) -> None: - self.write_chat_export() - outside = self.root / "outside.jsonl" - outside.write_text("preserve", encoding="utf-8") - linked = self.artifacts / "linked.jsonl" - linked.symlink_to(outside) - result = self.collect("linked.jsonl") - self.assertEqual(2, result.returncode) - self.assertIn("symbolic link", result.stderr) - self.assertEqual("preserve", outside.read_text(encoding="utf-8")) - - def test_artifact_root_must_be_strictly_inside_workspace_without_symlinks(self) -> None: - scope = { - "schema_version": 1, - "agents": [], - "chats": [ - { - "chat_id": CHAT_ID, - "agent": "fixture-agent", - "agent_id": AGENT_ID, - "authorization": "explicit_chat", - } - ], - } - scope_path = self.artifacts / "scope.json" - write_json(scope_path, scope) - binary, _ = self.make_fake_first_tree() - - def export_with_root(artifact_root: Path) -> subprocess.CompletedProcess[str]: - return run_cli( - "export-chats", - "--artifact-root", - str(artifact_root), - "--scope", - str(scope_path), - "--agent-workspace", - f"{AGENT_ID}={self.workspace}", - "--first-tree-bin", - str(binary), - "--now", - NOW, - "--output", - str(artifact_root / "chats.jsonl"), - ) - - outside = self.root / "outside-artifacts" - outside_result = export_with_root(outside) - self.assertEqual(2, outside_result.returncode) - self.assertIn("strict descendant", outside_result.stderr) - self.assertFalse(outside.exists()) - - workspace_result = export_with_root(self.workspace) - self.assertEqual(2, workspace_result.returncode) - self.assertIn("dedicated directory", workspace_result.stderr) - - real_parent = self.workspace / "real-artifacts" - real_parent.mkdir() - linked_parent = self.workspace / "linked-artifacts" - linked_parent.symlink_to(real_parent, target_is_directory=True) - linked_result = export_with_root(linked_parent / "audit") - self.assertEqual(2, linked_result.returncode) - self.assertIn("traverse a symbolic link", linked_result.stderr) - self.assertFalse((real_parent / "audit").exists()) - - def test_read_grammar_conserves_safe_unresolved_and_unsafe_attempts(self) -> None: - self.write_chat_export() - for filename, tool_name, arguments, sentinel in ( - ( - "stdin.jsonl", - "exec_command", - {"cmd": f"cat {self.tree_file} -", "workdir": str(self.workspace)}, - "stdin-mixed-sentinel", - ), - ( - "lookalike.jsonl", - "mcp__untrusted__read_file", - {"path": str(self.tree_file)}, - "lookalike-mixed-sentinel", - ), - ( - "semicolon.jsonl", - "exec_command", - { - "cmd": ( - f"cat {self.tree_file}; " - "printf semicolon-mixed-sentinel" - ), - "workdir": str(self.workspace), - }, - "semicolon-mixed-sentinel", - ), - ( - "pipe.jsonl", - "exec_command", - { - "cmd": f"cat {self.tree_file} | head", - "workdir": str(self.workspace), - }, - "pipe-mixed-sentinel", - ), - ( - "mutating-find.jsonl", - "exec_command", - { - "cmd": ( - f"cat {self.tree_file} && " - f"find {self.tree_root} -delete" - ), - "workdir": str(self.workspace), - }, - "mutating-find-sentinel", - ), - ( - "mutating-git.jsonl", - "exec_command", - { - "cmd": ( - f"cat {self.tree_file} && " - f"git -C {self.tree_root} branch -D temporary" - ), - "workdir": str(self.workspace), - }, - "mutating-git-sentinel", - ), - ): - call_id = f"call-{filename}" - write_jsonl( - self.trace_root / filename, - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-22T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": tool_name, - "call_id": call_id, - "arguments": json.dumps(arguments), - }, - }, - { - "timestamp": "2026-07-22T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": call_id, - "output": ( - "Process exited with code 0\n" - "# Architecture\n\n" - "Chat history is the authoritative state.\n" - + ( - sentinel - if filename == "semicolon.jsonl" - else "" - ) - ), - }, - }, - ], - ) - - write_jsonl( - self.trace_root / "outside-window-unsafe.jsonl", - [ - session_meta(self.workspace), - context_row(CHAT_ID), - { - "timestamp": "2026-07-16T10:02:00Z", - "type": "response_item", - "payload": { - "type": "function_call", - "name": "exec_command", - "call_id": "call-outside-window-unsafe", - "arguments": json.dumps( - { - "cmd": f"rm {self.tree_file}", - "workdir": str(self.workspace), - } - ), - }, - }, - { - "timestamp": "2026-07-16T10:02:01Z", - "type": "response_item", - "payload": { - "type": "function_call_output", - "call_id": "call-outside-window-unsafe", - "output": "Process exited with code 0\noutside-window-sentinel", - }, - }, - ], - ) - - result = self.collect("rejected-reads.jsonl") - self.assertEqual(0, result.returncode, result.stderr) - candidate = read_jsonl(self.artifacts / "rejected-reads.jsonl")[0] - self.assertEqual(2, len(candidate["reads"])) - self.assertTrue( - all(read["read_mode"] == "read_only_composite" for read in candidate["reads"]) - ) - self.assertIn("unresolved_stdin_tree_read", candidate["coverage_gaps"]) - self.assertIn("unresolved_tree_read_tool", candidate["coverage_gaps"]) - self.assertIn("unsafe_find_action", candidate["coverage_gaps"]) - self.assertIn("unsafe_git_mutation", candidate["coverage_gaps"]) - self.assertNotIn("unsafe_program_rm", candidate["coverage_gaps"]) - self.assertEqual( - { - "accepted_exact": 0, - "accepted_read_only_composite": 2, - "unresolved_opaque": 2, - "rejected_unsafe": 2, - }, - candidate["collector_diagnostics"]["attempt_status_counts"], - ) - self.assertEqual( - 6, - candidate["collector_diagnostics"]["in_window_tree_read_attempts"], - ) - serialized = json.dumps(candidate, sort_keys=True) - for sentinel in ("semicolon-mixed-sentinel", "pipe-mixed-sentinel"): - self.assertNotIn(sentinel, serialized) - for sentinel in ( - "stdin-mixed-sentinel", - "lookalike-mixed-sentinel", - "mutating-find-sentinel", - "mutating-git-sentinel", - "outside-window-sentinel", - ): - self.assertNotIn(sentinel, serialized) - - -if __name__ == "__main__": - unittest.main() + expected = audit.TreeIdentity(repo=audit.canonical_repo(REPO), branch="main") + selected, identity = audit.select_events_for_tree(events, expected) + self.assertEqual([item.event_id for item in selected], ["mine"]) + self.assertEqual(identity, expected) + + def test_a_changed_branch_binding_does_not_count_toward_the_current_tree(self) -> None: + events = normalized([event("old", "system/a.md", branch="legacy")]) + expected = audit.TreeIdentity(repo=audit.canonical_repo(REPO), branch="main") + selected, identity = audit.select_events_for_tree(events, expected) + # Excluded, not counted — and not an error either: zero observed IO for + # the current binding is itself a reportable result. + self.assertEqual(selected, []) + self.assertEqual(identity, expected) + + def test_a_mixed_feed_without_a_pinned_identity_fails_closed(self) -> None: + events = normalized( + [event("a", "x.md"), event("b", "x.md", repo="https://github.com/other/tree")] + ) + with self.assertRaisesRegex(audit.AuditError, "mixes Context Trees"): + audit.select_events_for_tree(events, None) + + +class ContentClaimTests(unittest.TestCase): + def test_head_commit_content_is_labelled_a_candidate_snapshot_with_its_caveat(self) -> None: + with TemporaryDirectory() as tmp: + root = Path(tmp) + (root / "a.md").write_text("current\n", encoding="utf-8") + ev = audit.normalize_event(event("e1", "a.md", commit="a" * 40)) + # No git repo here, so git show fails and it falls back honestly. + result = audit.node_content_at_read(root, ev, 100) + self.assertEqual(result["status"], "current_working_copy") + self.assertIn("not at read time", result["caveat"]) + + def test_report_states_the_uncommitted_read_limit(self) -> None: + agg = audit.aggregate(normalized([event("r1", "system/a.md")])) + text = audit.render_report( + generated_at=datetime(2026, 7, 31, tzinfo=timezone.utc), + window_start=None, + window_end=None, + agg=agg, + unobserved=None, + judgments=None, + sample_size=0, + ) + self.assertIn("candidate", text) + self.assertIn("uncommitted", text) + self.assertIn("git history", text) + + +class SampleConservationTests(unittest.TestCase): + def test_population_digest_changes_when_a_new_eligible_read_arrives(self) -> None: + before = audit.eligible_reads(normalized([event("r1", "a.md")])) + after = audit.eligible_reads(normalized([event("r1", "a.md"), event("r2", "b.md")])) + self.assertNotEqual(audit.population_digest(before), audit.population_digest(after)) + + def test_digest_ignores_ordering(self) -> None: + one = audit.eligible_reads(normalized([event("r1", "a.md"), event("r2", "b.md")])) + two = audit.eligible_reads(normalized([event("r2", "b.md"), event("r1", "a.md")])) + self.assertEqual(audit.population_digest(one), audit.population_digest(two)) + + +class WordingTests(unittest.TestCase): + def test_report_never_calls_a_node_never_read_or_proposes_deleting_it(self) -> None: + agg = audit.aggregate(normalized([event("r1", "system/a.md")])) + text = audit.render_report( + generated_at=datetime(2026, 7, 31, tzinfo=timezone.utc), + window_start=None, + window_end=None, + agg=agg, + unobserved=["system/unused.md"], + judgments=None, + sample_size=0, + ) + lowered = text.lower() + self.assertNotIn("never-read", lowered) + self.assertNotIn("candidate for removal", lowered) + self.assertIn("no observed read", lowered) + self.assertIn("evidence gap, not a finding", lowered) + self.assertIn("best-effort", lowered) + + +class OriginPortTests(unittest.TestCase): + def test_a_different_origin_port_is_a_different_tree(self) -> None: + # For Self-Managed GitLab the instance origin, port included, is the + # authority boundary. Folding ports would cross-credit instances. + self.assertNotEqual( + audit.canonical_repo("https://git.example:8443/org/tree"), + audit.canonical_repo("https://git.example:9443/org/tree"), + ) + + def test_ssh_url_with_a_port_is_not_missplit_as_scp(self) -> None: + self.assertEqual( + audit.canonical_repo("ssh://git@git.example:2222/org/tree.git"), + "git.example:2222/org/tree", + ) + + def test_scp_form_keeps_its_path_after_the_colon(self) -> None: + self.assertEqual(audit.canonical_repo("git@github.com:org/tree.git"), "github.com/org/tree") + + +class EmptyFeedTests(unittest.TestCase): + def test_no_io_in_the_window_is_a_result_not_a_failure(self) -> None: + expected = audit.TreeIdentity(repo=audit.canonical_repo(REPO), branch="main") + selected, identity = audit.select_events_for_tree([], expected) + self.assertEqual(selected, []) + self.assertEqual(identity, expected) + + def test_a_feed_holding_only_other_trees_reports_zero_for_the_target(self) -> None: + events = normalized([event("theirs", "a.md", repo="https://github.com/other/tree")]) + expected = audit.TreeIdentity(repo=audit.canonical_repo(REPO), branch="main") + selected, _ = audit.select_events_for_tree(events, expected) + self.assertEqual(selected, []) + + def test_an_empty_feed_without_a_pinned_tree_still_fails_closed(self) -> None: + with self.assertRaisesRegex(audit.AuditError, "cannot be identified"): + audit.select_events_for_tree([], None) + + +class SampleDrawIntegrityTests(unittest.TestCase): + """A matching population digest must not be enough to publish effects.""" + + def _feed(self) -> list[audit.IoEvent]: + return normalized([event(f"r{index}", f"system/n{index}.md") for index in range(10)]) + + def test_the_recorded_draw_is_reproducible_from_seed_and_size(self) -> None: + reads = self._feed() + first = [item.event_id for item in audit.sample_reads(reads, 3, seed=5)] + second = [item.event_id for item in audit.sample_reads(reads, 3, seed=5)] + self.assertEqual(first, second) + + def test_a_hand_picked_case_list_is_not_the_uniform_draw(self) -> None: + reads = self._feed() + drawn = [item.event_id for item in audit.sample_reads(reads, 3, seed=5)] + # The reviewer's attack: keep the population intact, swap in the reads + # that happen to be easiest to judge. + handpicked = [item.event_id for item in reads[:3]] + self.assertNotEqual(drawn, handpicked)