diff --git a/packages/shared/src/schemas/context-decision.ts b/packages/shared/src/schemas/context-decision.ts index 6b3f3ff09..913551673 100644 --- a/packages/shared/src/schemas/context-decision.ts +++ b/packages/shared/src/schemas/context-decision.ts @@ -5,13 +5,14 @@ import { contextTreeRepoSchema } from "./org-settings.js"; * `metadata.contextDecision` — an agent's self-attributed record that Context * Tree content materially shaped the choice carried by THIS message. * - * Written by the `first-tree-read` skill on the same final `chat send` (or - * blocking `chat ask`) that contains the affected choice, never as a separate - * message and never as prose. The receipt is the agent's own report: First Tree - * preserves the cited repository/commit/path so a reader can inspect the exact - * source, but it does NOT independently verify that the passage caused the - * choice. Every consumer must present it as agent-reported, never as a - * system-verified causal claim. + * Legacy agents wrote this receipt on the same final `chat send` (or blocking + * `chat ask`) that contained the affected choice. New `first-tree-read` + * payloads use a portable note in the message body instead; this schema remains + * the compatibility contract for stored history and older agents. The receipt + * is the agent's own report: First Tree preserves the cited + * repository/commit/path so a reader can inspect the exact source, but it does + * NOT independently verify that the passage caused the choice. Every consumer + * must present it as agent-reported, never as a system-verified causal claim. * * Trust boundary: the server strips the key from human senders and rejects a * malformed receipt from an agent sender, so a stored receipt is always an diff --git a/packages/skill-evals/src/core/__tests__/first-tree-shim.test.ts b/packages/skill-evals/src/core/__tests__/first-tree-shim.test.ts index 1b86dbb56..f5f242c94 100644 --- a/packages/skill-evals/src/core/__tests__/first-tree-shim.test.ts +++ b/packages/skill-evals/src/core/__tests__/first-tree-shim.test.ts @@ -187,6 +187,30 @@ describe("first-tree eval shim", () => { }), ]), ); + + const currentState = "Current state remains visible while work continues."; + const updateArgv = ["chat", "update", "--description", "-"]; + const update = spawnSync(join(paths.binDir, "first-tree"), updateArgv, { + cwd: paths.workspacePath, + encoding: "utf8", + env: { + ...process.env, + FIRST_TREE_EVAL_EVENTS: paths.eventsPath, + FIRST_TREE_EVAL_PHASE: "model", + }, + input: currentState, + }); + + expect(update.status).toBe(0); + expect(readEvents(paths.eventsPath)).toEqual( + expect.arrayContaining([ + expect.objectContaining({ + argv: updateArgv, + body: currentState, + type: "first_tree_call", + }), + ]), + ); } finally { rmSync(repoRoot, { force: true, recursive: true }); } diff --git a/packages/skill-evals/src/core/shims/first-tree.ts b/packages/skill-evals/src/core/shims/first-tree.ts index 8a30b4881..ce4bb7bff 100644 --- a/packages/skill-evals/src/core/shims/first-tree.ts +++ b/packages/skill-evals/src/core/shims/first-tree.ts @@ -59,6 +59,8 @@ const REVIEW_FIXTURE_PATH = ${JSON.stringify(options.reviewFixturePath ?? null)} const REVIEW_VERIFY_RUNNER_PATH = ${JSON.stringify(options.reviewVerifyRunnerPath ?? null)}; const SEED_PREFLIGHT = ${JSON.stringify(options.seedPreflight ?? null)}; const BYO_READ_ORIGIN_PATH = ${JSON.stringify(join(paths.runRoot, "context-tree-origin.git"))}; +const BYO_READ_SNAPSHOT_ROOT = ${JSON.stringify(join(paths.runRoot, "private-context-snapshots"))}; +const BYO_READ_BINDING_REPOSITORY = "https://github.com/example/context-tree.git"; const CONTEXT_REVIEW_BODY_MAX_BYTES = ${JSON.stringify(CONTEXT_REVIEW_BODY_MAX_BYTES)}; const CONTEXT_REVIEW_RUN_MARKER_PREFIX = ${JSON.stringify(CONTEXT_REVIEW_RUN_MARKER_PREFIX)}; @@ -130,10 +132,55 @@ function commandIndex(argv, command, subcommand) { return index >= 0 && argv[index + 1] === subcommand ? index : -1; } +function commandArgv(argv) { + return argv[0] === "--json" ? argv.slice(1) : argv; +} + +function parseCommandOptions(command, valueOptions, flagOptions) { + const values = {}; + const flags = new Set(); + for (let index = 2; index < command.length; index += 1) { + const arg = command[index]; + const equalsIndex = arg.indexOf("="); + const optionName = equalsIndex >= 0 ? arg.slice(0, equalsIndex) : arg; + if (valueOptions.includes(optionName)) { + if (Object.hasOwn(values, optionName)) return null; + const value = equalsIndex >= 0 ? arg.slice(equalsIndex + 1) : command[index + 1]; + if (!value || value.startsWith("--")) return null; + values[optionName] = value; + if (equalsIndex < 0) index += 1; + continue; + } + if (flagOptions.includes(arg)) { + if (flags.has(arg)) return null; + flags.add(arg); + continue; + } + return null; + } + return { flags, values }; +} + function runContextRoute(argv, phase) { if ((process.env.FIRST_TREE_EVAL_CASE_ID || "") !== "byo-scope-route-trigger") { finish(argv, phase, 1, "", "BYO SCOPE routing is unavailable for this eval case.\\n", { blockedByEval: true }); } + const command = commandArgv(argv); + const options = parseCommandOptions(command, ["--provider", "--project-root", "--session-candidate"], ["--pathless"]); + if ( + command[0] !== "context" || + command[1] !== "route" || + options === null || + options.values["--provider"] !== "codex" || + options.values["--session-candidate"] !== "eval-receipt" || + Object.hasOwn(options.values, "--project-root") || + !options.flags.has("--pathless") + ) { + finish(argv, phase, 2, "", "Expected the current pathless context route interface.\\n", { + authorityChecks: 0, + shimmedByEval: true, + }); + } const stdout = JSON.stringify({ ok: true, data: { @@ -156,34 +203,7 @@ function runContextRoute(argv, phase) { finish(argv, phase, 0, stdout, "", { authorityChecks: 1, scopeReads: 1, shimmedByEval: true }); } -function runTreeRead(argv, phase) { - if (argv.includes("--help") || argv.includes("-h")) { - finish( - argv, - phase, - 0, - "Usage: first-tree tree read [options]\\n\\nActivate one exact Context Tree snapshot for an explicit Team.\\n\\nOptions:\\n --team explicit First Tree Team id\\n --snapshot new task-owned snapshot directory\\n -h, --help display help for command\\n", - "", - { shimmedByEval: true }, - ); - } - - const candidateId = optionValue(argv, "--candidate"); - const teamId = candidateId === "candidate-byo-read-eval" ? "team-byo-read-eval" : optionValue(argv, "--team"); - const snapshotOption = optionValue(argv, "--snapshot"); - if ((process.env.FIRST_TREE_EVAL_CASE_ID || "") !== "byo-scope-route-trigger") { - finish(argv, phase, 1, "", "BYO read activation is unavailable for this eval case.\\n", { - blockedByEval: true, - }); - } - if (teamId !== "team-byo-read-eval" || !snapshotOption || (argv.includes("context") && !candidateId)) { - finish(argv, phase, 2, "", "Opaque candidate and new snapshot path are required.\\n", { - authorityChecks: teamId ? 1 : 0, - shimmedByEval: true, - }); - } - - const snapshotPath = resolve(process.cwd(), snapshotOption); +function materializeReadSnapshot(argv, phase, teamId, snapshotPath, candidateId) { if (existsSync(snapshotPath)) { finish(argv, phase, 2, "", "Snapshot path already exists.\\n", { authorityChecks: 0, @@ -229,7 +249,7 @@ function runTreeRead(argv, phase) { const metadataCommands = [ ["config", "first-tree-read.snapshot", "true"], ["config", "first-tree-read.team-id", teamId], - ["config", "first-tree-read.binding-repo", "https://git.example.invalid/teams/team-byo-read-eval/context-tree.git"], + ["config", "first-tree-read.binding-repo", BYO_READ_BINDING_REPOSITORY], ["config", "first-tree-read.binding-branch", "main"], ["config", "first-tree-read.commit", exactCommit], ["update-ref", "refs/first-tree-read/snapshot", exactCommit], @@ -247,7 +267,7 @@ function runTreeRead(argv, phase) { } } - const bindingRepository = "https://git.example.invalid/teams/team-byo-read-eval/context-tree.git"; + const bindingRepository = BYO_READ_BINDING_REPOSITORY; const stdout = JSON.stringify({ ok: true, @@ -258,7 +278,7 @@ function runTreeRead(argv, phase) { teamId, consumerKind: "byo", selectedTeam: { organizationId: teamId, displayName: "BYO Read Eval", role: "member" }, - routeSelection: { candidateId: "candidate-byo-read-eval", scopeCommit: exactCommit, source: "session" }, + ...(candidateId ? { routeSelection: { candidateId, scopeCommit: exactCommit, source: "session" } } : {}), }, }) + "\\n"; finish(argv, phase, 0, stdout, "", { @@ -273,6 +293,73 @@ function runTreeRead(argv, phase) { }); } +function runLegacyTreeRead(argv, phase) { + if (argv.includes("--help") || argv.includes("-h")) { + finish( + argv, + phase, + 0, + "Usage: first-tree tree read [options]\\n\\nActivate one exact Context Tree snapshot for an explicit Team.\\n\\nOptions:\\n --team explicit First Tree Team id\\n --snapshot new task-owned snapshot directory\\n -h, --help display help for command\\n", + "", + { shimmedByEval: true }, + ); + } + if ((process.env.FIRST_TREE_EVAL_CASE_ID || "") !== "byo-scope-route-trigger") { + finish(argv, phase, 1, "", "BYO read activation is unavailable for this eval case.\\n", { + blockedByEval: true, + }); + } + const command = commandArgv(argv); + const options = parseCommandOptions(command, ["--snapshot", "--team"], []); + const teamId = options?.values["--team"]; + const snapshotOption = options?.values["--snapshot"]; + if ( + command[0] !== "tree" || + command[1] !== "read" || + options === null || + teamId !== "team-byo-read-eval" || + !snapshotOption + ) { + finish(argv, phase, 2, "", "Explicit Team and new snapshot path are required.\\n", { + authorityChecks: teamId ? 1 : 0, + shimmedByEval: true, + }); + } + materializeReadSnapshot(argv, phase, teamId, resolve(process.cwd(), snapshotOption), null); +} + +function runContextSnapshot(argv, phase) { + if ((process.env.FIRST_TREE_EVAL_CASE_ID || "") !== "byo-scope-route-trigger") { + finish(argv, phase, 1, "", "BYO read activation is unavailable for this eval case.\\n", { + blockedByEval: true, + }); + } + const command = commandArgv(argv); + const options = parseCommandOptions(command, ["--candidate"], []); + const candidateId = options?.values["--candidate"]; + if ( + command[0] !== "context" || + command[1] !== "snapshot" || + options === null || + candidateId !== "candidate-byo-read-eval" + ) { + finish(argv, phase, 2, "", "Exactly one opaque context route candidate is required.\\n", { + authorityChecks: 0, + shimmedByEval: true, + }); + } + mkdirSync(BYO_READ_SNAPSHOT_ROOT, { recursive: true }); + const privateRoot = join(BYO_READ_SNAPSHOT_ROOT, "first-tree-read-" + process.pid + "-" + Date.now()); + mkdirSync(privateRoot, { recursive: false }); + materializeReadSnapshot( + argv, + phase, + "team-byo-read-eval", + join(privateRoot, "context-tree"), + candidateId, + ); +} + function runTreeSeed(argv, phase) { if (argv.includes("--help") || argv.includes("-h")) { finish( @@ -475,6 +562,20 @@ function bodyFromFileOption(argv) { } } +function chatBody(argv) { + if (argv[0] === "chat" && argv[1] === "update") { + const description = optionValueWithEquals(argv, "--description"); + if (description === null) return ""; + if (description !== "-") return description; + try { + return readFileSync(0, "utf8"); + } catch { + return ""; + } + } + return bodyFromFileOption(argv); +} + function runTreeVerify(argv, phase) { const root = resolve(process.cwd(), optionValue(argv, "--tree-path") || "."); const errors = []; @@ -514,7 +615,7 @@ function runTreeVerify(argv, phase) { const argv = process.argv.slice(2); const phase = process.env.FIRST_TREE_EVAL_PHASE || "model"; const recordedChatBody = - argv[0] === "chat" && ["ask", "send", "update"].includes(argv[1] || "") ? bodyFromFileOption(argv) : ""; + argv[0] === "chat" && ["ask", "send", "update"].includes(argv[1] || "") ? chatBody(argv) : ""; append({ type: "first_tree_call", phase, @@ -730,7 +831,7 @@ if (argv[0] === "chat" && ["ask", "send", "update"].includes(argv[1] || "")) { } if (commandIndex(argv, "tree", "read") >= 0) { - runTreeRead(argv, phase); + runLegacyTreeRead(argv, phase); } if (commandIndex(argv, "context", "route") >= 0) { @@ -738,7 +839,7 @@ if (commandIndex(argv, "context", "route") >= 0) { } if (commandIndex(argv, "context", "snapshot") >= 0) { - runTreeRead(argv, phase); + runContextSnapshot(argv, phase); } if (argv[0] === "tree" && argv[1] === "tree") { diff --git a/packages/skill-evals/src/suites/first-tree-read/__tests__/byo-fixture.test.ts b/packages/skill-evals/src/suites/first-tree-read/__tests__/byo-fixture.test.ts index fa2bef637..9e6272c35 100644 --- a/packages/skill-evals/src/suites/first-tree-read/__tests__/byo-fixture.test.ts +++ b/packages/skill-evals/src/suites/first-tree-read/__tests__/byo-fixture.test.ts @@ -1,5 +1,5 @@ import { spawnSync } from "node:child_process"; -import { existsSync, rmSync } from "node:fs"; +import { existsSync, readFileSync, rmSync } from "node:fs"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; @@ -12,7 +12,36 @@ import { createFirstTreeShim } from "../../../core/shims/first-tree.js"; import { findFirstTreeReadCase } from "../cases.js"; import { setupFixture } from "../fixture.js"; -describe("first-tree-read SCOPE-routed BYO fixture", () => { +describe("first-tree-read source provenance fixtures", () => { + it("declares a credential-free managed binding that can produce exact source links", () => { + const evalCase = findFirstTreeReadCase("tree-software-trigger"); + if (evalCase === null) throw new Error("missing managed read case"); + + const packageRoot = resolve(dirname(fileURLToPath(import.meta.url)), "../../../.."); + const paths = createRunPaths({ + caseId: evalCase.id, + packageRoot, + startedAt: new Date().toISOString(), + }); + + try { + const contextTreePath = setupFixture(evalCase, paths, createEvalReporter(evalCase.id, false)); + if (contextTreePath === null) throw new Error("managed fixture must create a Context Tree"); + const remote = spawnSync("git", ["remote", "get-url", "origin"], { + cwd: contextTreePath, + encoding: "utf8", + }); + const briefing = readFileSync(join(paths.workspacePath, "AGENTS.md"), "utf8"); + + expect(remote.status).toBe(0); + expect(remote.stdout.trim()).toBe("https://github.com/example/context-tree.git"); + expect(briefing).toContain("Context Tree binding repository `https://github.com/example/context-tree.git`"); + expect(briefing).toContain("binding branch `main`"); + } finally { + rmSync(paths.runRoot, { force: true, recursive: true }); + } + }); + it("routes one SCOPE candidate before materializing its detached exact snapshot", () => { const evalCase = findFirstTreeReadCase("byo-scope-route-trigger"); if (evalCase === null) throw new Error("missing SCOPE-routed BYO read case"); @@ -57,7 +86,14 @@ describe("first-tree-read SCOPE-routed BYO fixture", () => { const candidateId = routeEnvelope.data.candidates[0]?.candidateId; if (!candidateId) throw new Error("expected a routed BYO candidate"); - const activationArgv = ["--json", "context", "snapshot", "--candidate", candidateId, "--snapshot", snapshotPath]; + const activationArgv = ["--json", "context", "snapshot", "--candidate", candidateId]; + const obsoleteSnapshotOption = spawnSync(firstTree, [...activationArgv, "--snapshot", snapshotPath], { + cwd: paths.workspacePath, + encoding: "utf8", + env, + }); + expect(obsoleteSnapshotOption.status).toBe(2); + expect(existsSync(snapshotPath)).toBe(false); const activation = spawnSync(firstTree, activationArgv, { cwd: paths.workspacePath, encoding: "utf8", @@ -65,25 +101,31 @@ describe("first-tree-read SCOPE-routed BYO fixture", () => { }); expect(activation.status).toBe(0); const envelope = JSON.parse(activation.stdout) as { - data: { commit: string; snapshotPath: string; teamId: string }; + data: { binding: { repo: string }; commit: string; snapshotPath: string; teamId: string }; }; const receipt = envelope.data; - expect(receipt).toMatchObject({ snapshotPath, teamId: "team-byo-read-eval" }); + expect(receipt).toMatchObject({ teamId: "team-byo-read-eval" }); + expect(receipt.snapshotPath).not.toBe(snapshotPath); + expect(receipt.snapshotPath).toContain(paths.runRoot); + expect(receipt.binding.repo).toBe("https://github.com/example/context-tree.git"); - const head = spawnSync("git", ["rev-parse", "HEAD"], { cwd: snapshotPath, encoding: "utf8" }); - const symbolic = spawnSync("git", ["symbolic-ref", "-q", "HEAD"], { cwd: snapshotPath, encoding: "utf8" }); - const remotes = spawnSync("git", ["remote"], { cwd: snapshotPath, encoding: "utf8" }); + const head = spawnSync("git", ["rev-parse", "HEAD"], { cwd: receipt.snapshotPath, encoding: "utf8" }); + const symbolic = spawnSync("git", ["symbolic-ref", "-q", "HEAD"], { + cwd: receipt.snapshotPath, + encoding: "utf8", + }); + const remotes = spawnSync("git", ["remote"], { cwd: receipt.snapshotPath, encoding: "utf8" }); expect(head.stdout.trim()).toBe(receipt.commit); expect(symbolic.status).not.toBe(0); expect(remotes.stdout.trim()).toBe(""); const hierarchyHelp = spawnSync(firstTree, ["tree", "tree", "--help"], { - cwd: snapshotPath, + cwd: receipt.snapshotPath, encoding: "utf8", env, }); const selector = spawnSync(firstTree, ["tree", "tree", "--no-pull", "systems/server/auth"], { - cwd: snapshotPath, + cwd: receipt.snapshotPath, encoding: "utf8", env, }); @@ -96,6 +138,7 @@ describe("first-tree-read SCOPE-routed BYO fixture", () => { typeof event === "object" && event !== null && (event as { type?: string }).type === "first_tree_result" && + (event as { exitCode?: number }).exitCode === 0 && (event as { argv?: string[] }).argv?.includes("snapshot") && (event as { argv?: string[] }).argv?.includes("context") && !(event as { argv?: string[] }).argv?.includes("--help"), diff --git a/packages/skill-evals/src/suites/first-tree-read/__tests__/floor.test.ts b/packages/skill-evals/src/suites/first-tree-read/__tests__/floor.test.ts index e93831190..f0a7978f2 100644 --- a/packages/skill-evals/src/suites/first-tree-read/__tests__/floor.test.ts +++ b/packages/skill-evals/src/suites/first-tree-read/__tests__/floor.test.ts @@ -16,6 +16,21 @@ describe("first-tree-read floor contract", () => { it("keeps the declared gate matrix complete", () => { expect(validateFloor(FIRST_TREE_READ_SUITE.cases)).toEqual([]); expect(FIRST_TREE_READ_CASES.map((evalCase) => evalCase.id)).toContain("byo-scope-route-trigger"); + expect(FIRST_TREE_READ_CASES.map((evalCase) => evalCase.id)).toEqual( + expect.arrayContaining(["tree-navigation-no-impact", "tree-conflict-chinese", "tree-readable-multi-source-note"]), + ); + expect(FIRST_TREE_READ_CASES.every((evalCase) => evalCase.impactNote.mode !== undefined)).toBe(true); + expect( + FIRST_TREE_READ_CASES.filter((evalCase) => evalCase.expectedTrigger && evalCase.readMode === "managed").every( + (evalCase) => evalCase.managedTransport !== null, + ), + ).toBe(true); + expect( + FIRST_TREE_READ_CASES.find((evalCase) => evalCase.id === "tree-navigation-no-impact")?.managedTransport, + ).toBe("send"); + expect(FIRST_TREE_READ_CASES.find((evalCase) => evalCase.id === "tree-conflict-chinese")?.managedTransport).toBe( + "send", + ); }); it("states the fail-closed, SCOPE-routed exact-snapshot BYO boundary", () => { @@ -44,17 +59,50 @@ describe("first-tree-read floor contract", () => { expect(skill).toContain("PR/MR or issue titles"); }); - it("records only material decision influence on the same final message", () => { - expect(skill).toContain("Attach a small `contextDecision` receipt only when all of these conditions hold"); + it("shows only material decision influence in one portable final-response note", () => { + expect(skill).toContain( + "Append one compact, visible Context Tree impact note only when all of these\nconditions hold", + ); expect(skill).toMatch(/Opening a file is not\s+enough/); expect(skill).toContain("The read happened before the choice was made or executed"); expect(skill).toContain("Do not emit `effect: none`"); - expect(skill).toContain("top-level `contextDecision` metadata"); + expect(skill).toContain("Do not pass `contextDecision`\n metadata"); + expect(skill).toContain("In BYO sessions, append it to the authoring coding agent's native final\n response"); expect(skill).toContain("task correctly ends with a blocking `chat ask`"); - expect(skill).toContain("supply only the new\n`contextDecision` key"); - expect(skill).toContain("Choose the first matching category in this precedence\norder"); - expect(skill).toMatch(/`conflicted`[\s\S]+`redirected`[\s\S]+`constrained`[\s\S]+`confirmed`/); - expect(skill).toContain("Cite at most three Tree-root-relative\nnormal node paths"); + expect(skill).toContain("Never add the note to progress messages, status updates, or a second message"); + expect(skill).toContain("Choose exactly one effect in this precedence order, then show its human label"); + expect(skill).toMatch( + /`conflicted` → `Conflict surfaced`[\s\S]+`redirected` → `Approach changed`[\s\S]+`constrained` → `Options narrowed`[\s\S]+`confirmed` → `Direction supported`/, + ); + expect(skill).toContain("Match the note's language to the surrounding final response"); + expect(skill).toContain("one Markdown blockquote with exactly three **logical Markdown lines**"); + expect(skill).toContain("Natural wrapping at narrow display widths\nis expected; never truncate"); + expect(skill).toContain("Leave one blank line between the preceding answer and the note"); + expect(skill).toMatch( + /a backslash so\s+Markdown renders a portable hard line break without trailing whitespace; do\s+not use HTML/, + ); + expect(skill).toMatch(/Use objective language/); + expect(skill).toMatch(/roughly 160 English characters or\s+80 CJK characters/); + expect(skill).toContain("Use `Context Tree impact` and `Source` / `Sources` in English"); + expect(skill).toContain("Use\n`Context Tree 影响` and `来源` in Chinese"); + expect(skill).toMatch( + /`conflicted` \| `Conflict surfaced` \| `发现约束冲突`[\s\S]+`redirected` \| `Approach changed` \| `改变方案路径`[\s\S]+`constrained` \| `Options narrowed` \| `收窄可选范围`[\s\S]+`confirmed` \| `Direction supported` \| `支持当前方向`/, + ); + expect(skill).toContain("For `conflicted`, name the two incompatible constraints and the\nunresolved tradeoff"); + expect(skill).toContain("do not imply that the plan changed or the conflict was\nresolved"); + expect(skill).toContain("In Chinese, use bold `来源` for either\ncount"); + expect(skill).toContain("For a root `NODE.md`, use the root title or the relevant heading — never display\n`Node`"); + expect(skill).toContain("When two cited labels would be identical, prefix the nearest meaningful\nparent title"); + expect(skill).toContain("Never link to a mutable branch"); + expect(skill).toContain( + "never invent\na link or expose a raw repository URL, node path, or commit in the visible note", + ); + expect(skill).toContain("Cite at most three normal node paths"); + expect(skill).toContain( + "credential-free binding repository exactly as the activation receipt or\nmanaged workspace briefing declares it; never substitute a local transport URL", + ); + expect(skill).toContain("Never place a credential-bearing remote URL anywhere in the visible response"); + expect(skill).toContain("Source links must not contain a query or fragment"); expect(skill).toContain("read the binding repository and binding branch declared by the workspace\n briefing"); expect(skill).toContain("never infer the binding branch from the checkout's current branch\n or its upstream"); expect(skill).not.toContain("resolve the current branch's upstream remote-tracking ref"); @@ -70,37 +118,34 @@ describe("first-tree-read floor contract", () => { ); expect(skill).toMatch(/current branch or\s+upstream is never a fallback authority/); expect(skill).toMatch(/canonical repository identities do not\s+match/); - expect(skill).toContain("omit the evidence\nrow and do not attach the receipt when no valid evidence remains"); - expect(skill).toContain("canonical repository identity rather than raw string\nequality"); - expect(skill).toContain("Never persist a credential-bearing remote URL"); - expect(skill).toMatch(/It is not\s+server-verified proof of causality/); + expect(skill).toContain("do not append the note when no valid source remains"); + expect(skill).toMatch(/not a\s+First Tree verification of causality/); + expect(skill).toContain("Do not add a long attribution disclaimer"); + expect(skill).toContain("system-style framing, emoji, badge, divider, or\ncollapsible detail"); + expect(skill).not.toContain("top-level `contextDecision` metadata"); + expect(skill).not.toContain("```json"); + + const noteBlock = /```markdown\n([\s\S]*?)\n```/.exec(skill)?.[1] ?? ""; + const noteLines = noteBlock.split("\n"); + expect(noteLines).toHaveLength(3); + expect(noteLines.every((line) => line.startsWith("> "))).toBe(true); + expect(noteLines[0]).toBe("> **Context Tree impact · Options narrowed**\\"); + expect(noteLines[1]).toBe("> The organization-isolation rule ruled out a global shared index.\\"); + expect(noteLines[2]).toContain( + "> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/", + ); + expect(noteLines[2]).toContain("/system/cloud/team/tenancy-and-identity.md)"); + expect(noteLines[2]?.match(/\/blob\/([0-9a-f]+)\//u)?.[1]).toMatch(/^[0-9a-f]{40}$/); - const receiptBlock = /```json\n([\s\S]*?)\n```/.exec(skill); - expect(receiptBlock).not.toBeNull(); - const parsed = JSON.parse(receiptBlock?.[1] ?? "{}") as { - contextDecision?: { - version?: number; - effect?: string; - summary?: string; - evidence?: Array<{ repoUrl?: string; commit?: string; nodePath?: string; heading?: string }>; - }; - }; - expect(parsed.contextDecision).toMatchObject({ - version: 1, - effect: "constrained", - summary: expect.any(String), - }); - expect(parsed.contextDecision?.evidence).toHaveLength(1); - expect(parsed.contextDecision?.evidence?.[0]).toMatchObject({ - repoUrl: "https://github.com/example/context-tree", - nodePath: "system/cloud/team/tenancy-and-identity.md", - heading: "Organization isolation", - }); - expect(parsed.contextDecision?.evidence?.[0]?.commit).toMatch(/^[0-9a-f]{40}$/); + const markdownBlocks = [...skill.matchAll(/```markdown\n([\s\S]*?)\n```/gu)].map((match) => match[1] ?? ""); + const conflictBlock = markdownBlocks.find((block) => block.includes("Context Tree 影响 · 发现约束冲突")) ?? ""; + expect(conflictBlock.split("\n")).toHaveLength(3); + expect(conflictBlock).toContain("固定发布日期与发布前必须完成安全审计的规则无法同时满足,取舍仍待决定"); + expect(conflictBlock).toContain("**来源** · [发布安全门槛]"); }); it("keeps version metadata aligned", () => { - expect(skillVersion).toBe("0.5.0"); + expect(skillVersion).toBe("0.6.0"); expect(skill).toContain(`version: ${skillVersion}`); }); }); diff --git a/packages/skill-evals/src/suites/first-tree-read/__tests__/metrics.test.ts b/packages/skill-evals/src/suites/first-tree-read/__tests__/metrics.test.ts index 47a02813a..2fbaa2201 100644 --- a/packages/skill-evals/src/suites/first-tree-read/__tests__/metrics.test.ts +++ b/packages/skill-evals/src/suites/first-tree-read/__tests__/metrics.test.ts @@ -3,12 +3,22 @@ import { describe, expect, it } from "vitest"; import { gradingFailureMessages } from "../../../core/grading.js"; import { casePassed, deriveMetrics } from "../metrics.js"; import { buildGrading } from "../summary.js"; -import type { EvalMetrics, FixtureValidation } from "../types.js"; +import type { EvalMetrics, FixtureValidation, ImpactNoteExpectation, ManagedTransport } from "../types.js"; const HELP_ARGV = ["tree", "tree", "--help"]; const SELECTOR_ARGV = ["tree", "tree", "/domains/payments"]; -const BYO_READ_HELP_ARGV = ["tree", "read", "--help"]; -const BYO_ACTIVATION_ARGV = [ +const BYO_ROUTE_ARGV = [ + "--json", + "context", + "route", + "--provider", + "codex", + "--pathless", + "--session-candidate", + "eval-receipt", +]; +const BYO_ACTIVATION_ARGV = ["--json", "context", "snapshot", "--candidate", "candidate-byo-read-eval"]; +const LEGACY_BYO_ACTIVATION_ARGV = [ "--json", "tree", "read", @@ -19,6 +29,17 @@ const BYO_ACTIVATION_ARGV = [ ]; const BYO_SELECTOR_ARGV = ["tree", "tree", "--no-pull", "systems/server/auth"]; const EXACT_COMMIT = "a".repeat(40); +const TEST_SOURCE_AUTHORITY = { + allowedNodePaths: [ + "NODE.md", + "domains/payments/NODE.md", + "product/billing/rollout-policy/NODE.md", + "product/release/rollout-policy/NODE.md", + "systems/server/auth/scopes/NODE.md", + ], + exactCommit: EXACT_COMMIT, + repository: "https://github.com/example/context-tree.git", +} as const; const EXPECTED_FACT = "payments runbook anchor"; const JWT_EXPECTED_FACTS = [ "User JWT auth is the unified authorization surface.", @@ -55,11 +76,12 @@ function assistantTextEvent(text: string): unknown { }; } -function firstTreeCall(argv: readonly string[]): unknown { +function firstTreeCall(argv: readonly string[], extra: Record = {}): unknown { return { argv: [...argv], phase: "model", type: "first_tree_call", + ...extra, }; } @@ -73,13 +95,33 @@ function firstTreeResult(argv: readonly string[], exitCode: number, extra: Recor }; } +function managedMessage( + body: string, + argv: readonly string[] = ["chat", "send", "gandy2025", "-F", "reply.md"], +): unknown[] { + return [firstTreeCall(argv, { body }), firstTreeResult(argv, 0)]; +} + +function managedStatus(body: string): unknown[] { + const argv = ["chat", "update", "--description", "-"]; + return [firstTreeCall(argv, { body }), firstTreeResult(argv, 0)]; +} + function metrics(events: readonly unknown[]): EvalMetrics { - return deriveMetrics(events, VALID_FIXTURE, 0, [EXPECTED_FACT]); + return deriveMetrics(events, VALID_FIXTURE, 0, [EXPECTED_FACT], { mode: "absent" }, "send"); +} + +function impactMetrics( + events: readonly unknown[], + expectation: ImpactNoteExpectation, + managedTransport: ManagedTransport = "send", +): EvalMetrics { + return deriveMetrics(events, VALID_FIXTURE, 0, [EXPECTED_FACT], expectation, managedTransport); } describe("first-tree-read metrics pass criteria", () => { it("passes trigger cases only when skill read, facts, help, selector, and command results are all OK", () => { - const result = metrics([ + const nativeOnly = metrics([ skillReadEvent(), firstTreeCall(HELP_ARGV), firstTreeResult(HELP_ARGV, 0), @@ -87,20 +129,31 @@ describe("first-tree-read metrics pass criteria", () => { firstTreeResult(SELECTOR_ARGV, 0), assistantTextEvent(`The tree says ${EXPECTED_FACT}.`), ]); + const result = metrics([ + skillReadEvent(), + firstTreeCall(HELP_ARGV), + firstTreeResult(HELP_ARGV, 0), + firstTreeCall(SELECTOR_ARGV), + firstTreeResult(SELECTOR_ARGV, 0), + ...managedMessage(`The tree says ${EXPECTED_FACT}.`), + ]); expect(result.skillFileReadObserved).toBe(true); expect(result.expectedFactsObserved).toBe(true); expect(result.helpSucceeded).toBe(true); expect(result.selectionSucceeded).toBe(true); expect(result.modelFirstTreeCommandsOk).toBe(true); + expect(nativeOnly.managedFinalTransportOk).toBe(false); + expect(casePassed(true, nativeOnly)).toBe(false); + expect(result.managedFinalTransportOk).toBe(true); expect(casePassed(true, result)).toBe(true); }); - it("passes explicit-Team BYO cases only for one ordered activation and exact detached no-pull selectors", () => { + it("passes BYO cases only for one ordered SCOPE route and exact detached no-pull snapshot", () => { const result = metrics([ skillReadEvent(), - firstTreeCall(BYO_READ_HELP_ARGV), - firstTreeResult(BYO_READ_HELP_ARGV, 0), + firstTreeCall(BYO_ROUTE_ARGV), + firstTreeResult(BYO_ROUTE_ARGV, 0), firstTreeCall(BYO_ACTIVATION_ARGV), firstTreeResult(BYO_ACTIVATION_ARGV, 0, { exactCommit: EXACT_COMMIT }), firstTreeCall(HELP_ARGV), @@ -110,7 +163,8 @@ describe("first-tree-read metrics pass criteria", () => { assistantTextEvent(`The tree says ${EXPECTED_FACT}.`), ]); - expect(result.readHelpSucceeded).toBe(true); + expect(result.readRouteCalls).toBe(1); + expect(result.readRouteSucceeded).toBe(true); expect(result.readActivationCalls).toBe(1); expect(result.readActivationSucceeded).toBe(true); expect(result.byoReadSequenceOk).toBe(true); @@ -124,8 +178,8 @@ describe("first-tree-read metrics pass criteria", () => { const mutableSelector = ["tree", "tree", "systems/server/auth"]; const result = metrics([ skillReadEvent(), - firstTreeCall(BYO_READ_HELP_ARGV), - firstTreeResult(BYO_READ_HELP_ARGV, 0), + firstTreeCall(BYO_ROUTE_ARGV), + firstTreeResult(BYO_ROUTE_ARGV, 0), firstTreeCall(BYO_ACTIVATION_ARGV), firstTreeResult(BYO_ACTIVATION_ARGV, 0, { exactCommit: EXACT_COMMIT }), firstTreeCall(BYO_ACTIVATION_ARGV), @@ -143,6 +197,48 @@ describe("first-tree-read metrics pass criteria", () => { expect(casePassed(true, result, "byo")).toBe(false); }); + it("rejects the legacy tree read activation sequence for BYO cases", () => { + const result = metrics([ + skillReadEvent(), + firstTreeCall(LEGACY_BYO_ACTIVATION_ARGV), + firstTreeResult(LEGACY_BYO_ACTIVATION_ARGV, 0, { exactCommit: EXACT_COMMIT }), + firstTreeCall(HELP_ARGV), + firstTreeResult(HELP_ARGV, 0), + firstTreeCall(BYO_SELECTOR_ARGV), + firstTreeResult(BYO_SELECTOR_ARGV, 0, { actualHead: EXACT_COMMIT, detachedHead: true }), + assistantTextEvent(`The tree says ${EXPECTED_FACT}.`), + ]); + + expect(result.readRouteSucceeded).toBe(false); + expect(result.readActivationSucceeded).toBe(false); + expect(result.legacyReadActivationCalls).toBe(1); + expect(result.byoReadSequenceOk).toBe(false); + expect(casePassed(true, result, "byo")).toBe(false); + }); + + it("rejects a legacy tree read activation even when the current BYO sequence also succeeds", () => { + const result = metrics([ + skillReadEvent(), + firstTreeCall(LEGACY_BYO_ACTIVATION_ARGV), + firstTreeResult(LEGACY_BYO_ACTIVATION_ARGV, 0, { exactCommit: EXACT_COMMIT }), + firstTreeCall(BYO_ROUTE_ARGV), + firstTreeResult(BYO_ROUTE_ARGV, 0), + firstTreeCall(BYO_ACTIVATION_ARGV), + firstTreeResult(BYO_ACTIVATION_ARGV, 0, { exactCommit: EXACT_COMMIT }), + firstTreeCall(HELP_ARGV), + firstTreeResult(HELP_ARGV, 0), + firstTreeCall(BYO_SELECTOR_ARGV), + firstTreeResult(BYO_SELECTOR_ARGV, 0, { actualHead: EXACT_COMMIT, detachedHead: true }), + assistantTextEvent(`The tree says ${EXPECTED_FACT}.`), + ]); + + expect(result.readRouteSucceeded).toBe(true); + expect(result.readActivationSucceeded).toBe(true); + expect(result.legacyReadActivationCalls).toBe(1); + expect(result.byoReadSequenceOk).toBe(false); + expect(casePassed(true, result, "byo")).toBe(false); + }); + it("fails trigger cases when facts are present but help is missing", () => { const result = metrics([ skillReadEvent(), @@ -199,7 +295,7 @@ describe("first-tree-read metrics pass criteria", () => { firstTreeResult(HELP_ARGV, 0), firstTreeCall(["tree", "tree", "systems/server/auth"]), firstTreeResult(["tree", "tree", "systems/server/auth"], 0), - assistantTextEvent(`JWT auth routes 要遵守这些约束: + ...managedMessage(`JWT auth routes 要遵守这些约束: - User JWT 是统一授权面。 - Route scopes 必须结合当前 live organization membership checks。 - HTTP routes 和 multi-org 改动必须遵循 docs/development/http-path-conventions.md。`), @@ -207,6 +303,8 @@ describe("first-tree-read metrics pass criteria", () => { VALID_FIXTURE, 0, JWT_EXPECTED_FACTS, + { mode: "absent" }, + "send", ); expect(result.expectedFactHits).toEqual([...JWT_EXPECTED_FACTS]); @@ -332,4 +430,417 @@ describe("first-tree-read metrics pass criteria", () => { expect(nonZeroResult.modelFirstTreeCommandsOk).toBe(false); expect(casePassed(false, nonZeroResult)).toBe(false); }); + + it("accepts one exact-version English impact note in a managed chat body", () => { + const body = `JWT routes must enforce the tree constraints. + +> **Context Tree impact · Options narrowed**\\ +> The organization-isolation rule ruled out a global shared index.\\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`; + const result = impactMetrics(managedMessage(body), { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }); + + expect(result.impactNoteBehaviorOk).toBe(true); + expect(result.impactNoteCount).toBe(1); + expect(result.impactNoteBlankLineBefore).toBe(true); + expect(result.impactNoteLogicalLinesOk).toBe(true); + expect(result.impactNoteExactLinksOk).toBe(true); + expect(result.impactNoteSummaryObjectiveOk).toBe(true); + expect(result.impactNoteSourceLabels).toEqual(["Organization isolation"]); + expect(result.impactNoteMetadataFree).toBe(true); + expect(result.managedFinalTransportOk).toBe(true); + }); + + it("makes material trigger cases fail until the final visible note satisfies the behavior contract", () => { + const expectation: ImpactNoteExpectation = { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }; + const baseEvents = [ + skillReadEvent(), + firstTreeCall(HELP_ARGV), + firstTreeResult(HELP_ARGV, 0), + firstTreeCall(SELECTOR_ARGV), + firstTreeResult(SELECTOR_ARGV, 0), + assistantTextEvent(`The tree says ${EXPECTED_FACT}.`), + ]; + const note = `Answer. + +> **Context Tree impact · Options narrowed**\\ +> The payment rule narrowed the implementation boundary.\\ +> **Source** · [Payments](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/domains/payments/NODE.md)`; + const withoutNote = impactMetrics( + [...baseEvents, ...managedMessage(`The tree says ${EXPECTED_FACT}. Final answer without the note.`)], + expectation, + ); + const nativeOnly = impactMetrics([...baseEvents, assistantTextEvent(note)], expectation); + const withNote = impactMetrics( + [...baseEvents, ...managedMessage(`The tree says ${EXPECTED_FACT}.\n\n${note}`)], + expectation, + ); + + expect(withoutNote.impactNoteBehaviorOk).toBe(false); + expect(casePassed(true, withoutNote)).toBe(false); + expect(nativeOnly.impactNoteBehaviorOk).toBe(true); + expect(nativeOnly.managedFinalTransportOk).toBe(false); + expect(casePassed(true, nativeOnly)).toBe(false); + expect(withNote.impactNoteBehaviorOk).toBe(true); + expect(withNote.managedFinalTransportOk).toBe(true); + expect(casePassed(true, withNote)).toBe(true); + }); + + it("rejects duplicate notes, mutable links, and visible receipt metadata", () => { + const note = `> **Context Tree impact · Options narrowed**\\ +> The rule ruled out a shared index.\\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/main/systems/server/auth/scopes/NODE.md)`; + const result = impactMetrics( + [assistantTextEvent(`Answer.\n\n${note}\n\n${note}\n\n{ "contextDecision": { "effect": "constrained" } }`)], + { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }, + ); + + expect(result.impactNoteCount).toBe(2); + expect(result.impactNoteExactLinksOk).toBe(false); + expect(result.impactNoteMetadataFree).toBe(false); + expect(result.impactNoteBehaviorOk).toBe(false); + }); + + it("rejects credential-bearing source links and first-person impact summaries", () => { + const result = impactMetrics( + [ + assistantTextEvent(`Answer. + +> **Context Tree impact · Options narrowed**\\ +> I used Context Tree to rule out a shared index.\\ +> **Source** · [Organization isolation](https://x-access-token:secret@github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`), + ], + { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }, + ); + + expect(result.impactNoteExactLinksOk).toBe(false); + expect(result.impactNoteSummaryObjectiveOk).toBe(false); + expect(result.impactNoteBehaviorOk).toBe(false); + }); + + it("rejects contextDecision metadata in successful chat transport while allowing unrelated metadata", () => { + const body = `Answer. + +> **Context Tree impact · Options narrowed**\\ +> The organization-isolation rule ruled out a global shared index.\\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`; + const expectation: ImpactNoteExpectation = { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }; + const receiptArgv = [ + "chat", + "send", + "gandy2025", + "-F", + "reply.md", + "--metadata", + JSON.stringify({ contextDecision: { effect: "constrained" }, mentionIds: ["member-1"] }), + ]; + const unrelatedArgv = [ + "chat", + "send", + "gandy2025", + "-F", + "reply.md", + "-m", + JSON.stringify({ mentionIds: ["member-1"] }), + ]; + const withReceipt = impactMetrics(managedMessage(body, receiptArgv), expectation); + const withUnrelatedMetadata = impactMetrics(managedMessage(body, unrelatedArgv), expectation); + + expect(withReceipt.impactNoteMetadataFree).toBe(false); + expect(withReceipt.impactNoteBehaviorOk).toBe(false); + expect(withUnrelatedMetadata.impactNoteMetadataFree).toBe(true); + expect(withUnrelatedMetadata.impactNoteBehaviorOk).toBe(true); + }); + + it("rejects exact-looking source links outside the selected repository, commit, or allowed paths", () => { + const expectation: ImpactNoteExpectation = { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: { + allowedNodePaths: ["systems/server/auth/scopes/NODE.md"], + exactCommit: EXACT_COMMIT, + repository: "https://github.com/example/context-tree.git", + }, + sourceCount: { max: 1, min: 1 }, + }; + const noteFor = (url: string) => + assistantTextEvent(`Answer. + +> **Context Tree impact · Options narrowed**\\ +> The organization-isolation rule ruled out a global shared index.\\ +> **Source** · [Organization isolation](${url})`); + const invalidUrls = [ + `https://evil.example/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md`, + `https://github.com/another/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md`, + `https://github.com/example/context-tree/blob/${"b".repeat(40)}/systems/server/auth/scopes/NODE.md`, + `https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/unknown/NODE.md`, + ]; + + for (const url of invalidUrls) { + const result = impactMetrics([noteFor(url)], expectation); + expect(result.impactNoteExactLinksOk).toBe(true); + expect(result.impactNoteSourceAuthorityOk).toBe(false); + expect(result.impactNoteBehaviorOk).toBe(false); + } + }); + + it("requires the note to end the final successful managed message", () => { + const note = `> **Context Tree impact · Options narrowed**\\ +> The organization-isolation rule ruled out a global shared index.\\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`; + const expectation: ImpactNoteExpectation = { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }; + const trailingProse = impactMetrics([assistantTextEvent(`Answer.\n\n${note}\n\nMore detail.`)], expectation); + const progressOnly = impactMetrics( + [...managedMessage(`Progress.\n\n${note}`), ...managedMessage("Final answer without the note.")], + expectation, + ); + const finalOnly = impactMetrics( + [...managedMessage("Progress without the note."), ...managedMessage(`Final answer.\n\n${note}`)], + expectation, + ); + const modelProgressDuplicate = impactMetrics( + [assistantTextEvent(`Progress.\n\n${note}`), ...managedMessage(`Final answer.\n\n${note}`)], + expectation, + ); + const currentStateDuplicate = impactMetrics( + [...managedStatus(`Progress.\n\n${note}`), ...managedMessage(`Final answer.\n\n${note}`)], + expectation, + ); + + expect(trailingProse.impactNoteAtFinalEnd).toBe(false); + expect(trailingProse.impactNoteBehaviorOk).toBe(false); + expect(progressOnly.impactNoteAtFinalEnd).toBe(false); + expect(progressOnly.impactNoteBehaviorOk).toBe(false); + expect(finalOnly.impactNoteAtFinalEnd).toBe(true); + expect(finalOnly.impactNoteBehaviorOk).toBe(true); + expect(modelProgressDuplicate.impactNoteCount).toBe(2); + expect(modelProgressDuplicate.impactNoteBehaviorOk).toBe(false); + expect(currentStateDuplicate.impactNoteCount).toBe(2); + expect(currentStateDuplicate.impactNoteBehaviorOk).toBe(false); + }); + + it("uses the case transport contract rather than deriving transport from the impact effect", () => { + const body = `需要你决定如何处理冲突。 + +> **Context Tree 影响 · 发现约束冲突**\\ +> 固定发布日期与发布前安全审计无法同时满足,取舍仍待决定。\\ +> **来源** · [Rollout Policy](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/product/release/rollout-policy/NODE.md)`; + const expectation: ImpactNoteExpectation = { + effect: "conflicted", + language: "zh", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }; + const sentForBlockingCase = impactMetrics(managedMessage(body), expectation, "ask"); + const askedForBlockingCase = impactMetrics( + managedMessage(body, ["chat", "ask", "gandy2025", "-F", "question.md"]), + expectation, + "ask", + ); + const sentForTerminalCase = impactMetrics(managedMessage(body), expectation, "send"); + + expect(sentForBlockingCase.impactNoteBehaviorOk).toBe(true); + expect(sentForBlockingCase.managedFinalTransportOk).toBe(false); + expect(askedForBlockingCase.impactNoteBehaviorOk).toBe(true); + expect(askedForBlockingCase.managedFinalTransportOk).toBe(true); + expect(sentForTerminalCase.impactNoteBehaviorOk).toBe(true); + expect(sentForTerminalCase.managedFinalTransportOk).toBe(true); + }); + + it("counts identical impact notes from separate BYO assistant messages", () => { + const note = `Answer. + +> **Context Tree impact · Options narrowed**\\ +> The organization-isolation rule ruled out a global shared index.\\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`; + const result = impactMetrics([assistantTextEvent(note), assistantTextEvent(note)], { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + }); + + expect(result.impactNoteCount).toBe(2); + expect(result.impactNoteBehaviorOk).toBe(false); + }); + + it("rejects a generic middle sentence and credentials elsewhere in the visible response", () => { + const expectation: ImpactNoteExpectation = { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + summaryConcepts: [["organization isolation"], ["shared index"], ["ruled out"]], + }; + const source = `> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/systems/server/auth/scopes/NODE.md)`; + const generic = impactMetrics( + [ + assistantTextEvent(`Answer. + +> **Context Tree impact · Options narrowed**\\ +> Context Tree narrowed the choice.\\ +${source}`), + ], + expectation, + ); + const credentialUrls = [ + "https://x-access-token:secret@github.com/example/private", + "HTTPS://x-access-token:secret@github.com/example/private", + "hTtPs://x-access-token:secret@github.com/example/private", + "ssh://git:secret@github.com/example/private", + ]; + const credentialResults = credentialUrls.map((url) => + impactMetrics( + [ + assistantTextEvent(`See ${url}. + +> **Context Tree impact · Options narrowed**\\ +> The organization isolation rule ruled out a global shared index.\\ +${source}`), + ], + expectation, + ), + ); + const legalSshUsername = impactMetrics( + [ + assistantTextEvent(`The binding clone identity is ssh://git@github.com/example/context-tree.git. + +> **Context Tree impact · Options narrowed**\\ +> The organization isolation rule ruled out a global shared index.\\ +${source}`), + ], + expectation, + ); + + expect(generic.impactNoteSummaryConceptsOk).toBe(false); + expect(generic.impactNoteBehaviorOk).toBe(false); + for (const result of credentialResults) { + expect(result.impactNoteVisibleUrlsCredentialFree).toBe(false); + expect(result.impactNoteBehaviorOk).toBe(false); + } + expect(legalSshUsername.impactNoteVisibleUrlsCredentialFree).toBe(true); + expect(legalSshUsername.impactNoteBehaviorOk).toBe(true); + }); + + it("accepts the complete Chinese conflict template only when the tradeoff remains unresolved", () => { + const expectation: ImpactNoteExpectation = { + effect: "conflicted", + language: "zh", + mode: "present", + requiredSourceLabels: ["Rollout Policy"], + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 1, min: 1 }, + summaryConcepts: [["发布日期"], ["安全审计"], ["仍待", "尚未"]], + summaryForbidden: ["已调整", "已解决"], + }; + const valid = impactMetrics( + [ + assistantTextEvent(`不能直接发布。 + +> **Context Tree 影响 · 发现约束冲突**\\ +> 固定发布日期与发布前必须完成安全审计的规则无法同时满足,取舍仍待决定。\\ +> **来源** · [Rollout Policy](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/product/release/rollout-policy/NODE.md)`), + ], + expectation, + ); + const fabricatedResolution = impactMetrics( + [ + assistantTextEvent(`方案如下。 + +> **Context Tree 影响 · 发现约束冲突**\\ +> 发布日期与安全审计发生冲突,方案已调整并已解决。\\ +> **来源** · [Rollout Policy](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/product/release/rollout-policy/NODE.md)`), + ], + expectation, + ); + + expect(valid.impactNoteBehaviorOk).toBe(true); + expect(valid.impactNoteLanguage).toBe("zh"); + expect(valid.impactNoteSummaryConceptsOk).toBe(true); + expect(valid.impactNoteSummaryForbiddenOk).toBe(true); + expect(fabricatedResolution.impactNoteSummaryForbiddenOk).toBe(false); + expect(fabricatedResolution.impactNoteBehaviorOk).toBe(false); + }); + + it("requires readable root and disambiguated duplicate labels in a three-source note", () => { + const result = impactMetrics( + [ + assistantTextEvent(`The rollout is bounded by all three decisions. + +> **Context Tree impact · Options narrowed**\\ +> The rollout rules require one reviewable scope, audit approval, and core-release stability.\\ +> **Sources** · [First Tree Read Eval Context](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/NODE.md) · [Release · Rollout Policy](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/product/release/rollout-policy/NODE.md) · [Billing · Rollout Policy](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/product/billing/rollout-policy/NODE.md)`), + ], + { + effect: "constrained", + language: "en", + mode: "present", + requiredSourceLabels: ["First Tree Read Eval Context", "Release · Rollout Policy", "Billing · Rollout Policy"], + sourceAuthority: TEST_SOURCE_AUTHORITY, + sourceCount: { max: 3, min: 3 }, + }, + ); + + expect(result.impactNoteBehaviorOk).toBe(true); + expect(result.impactNoteSourceCount).toBe(3); + expect(result.impactNoteSourceLabels).not.toContain("Node"); + }); + + it("keeps navigation-only reads free of impact notes", () => { + const absent: ImpactNoteExpectation = { mode: "absent" }; + const withoutNote = impactMetrics([assistantTextEvent("systems, domains, operations")], absent); + const withNote = impactMetrics( + [ + assistantTextEvent(`systems, domains, operations + +> **Context Tree impact · Direction supported**\\ +> The root confirmed the domain names.\\ +> **Source** · [First Tree Read Eval Context](https://github.com/example/context-tree/blob/${EXACT_COMMIT}/NODE.md)`), + ], + absent, + ); + + expect(withoutNote.impactNoteBehaviorOk).toBe(true); + expect(withNote.impactNoteCount).toBe(1); + expect(withNote.impactNoteBehaviorOk).toBe(false); + }); }); diff --git a/packages/skill-evals/src/suites/first-tree-read/cases.ts b/packages/skill-evals/src/suites/first-tree-read/cases.ts index ec84388dd..5c44cad82 100644 --- a/packages/skill-evals/src/suites/first-tree-read/cases.ts +++ b/packages/skill-evals/src/suites/first-tree-read/cases.ts @@ -6,12 +6,39 @@ const JWT_AUTH_EXPECTED_FACTS = [ "HTTP routes must follow the repo path conventions document before auth or multi-org changes.", ] as const; +const NAVIGATION_EXPECTED_FACTS = ["Top-level Context Tree domains are systems, domains, and operations."] as const; + +const RELEASE_AUDIT_EXPECTED_FACTS = [ + "Production releases require completed security-audit approval before deployment.", +] as const; + +const ROLLOUT_EXPECTED_FACTS = [ + "Every production rollout must keep a single reviewable scope across release and billing policy.", + "Production releases require completed security-audit approval before deployment.", + "Billing changes must roll out after the core release reaches stable monitoring.", +] as const; + +const EVAL_SOURCE_REPOSITORY = "https://github.com/example/context-tree.git"; +const JWT_SOURCE_PATHS = [ + "systems/server/auth/jwt/NODE.md", + "systems/server/auth/scopes/NODE.md", + "systems/server/http/routes/NODE.md", +] as const; +const JWT_SUMMARY_CONCEPTS = [ + ["JWT"], + ["route", "路由"], + ["membership", "成员"], + ["narrowed", "ruled out", "requires", "must", "收窄", "排除", "必须"], +] as const; + export const FIRST_TREE_READ_CASES: readonly FirstTreeReadEvalCase[] = [ { description: "Blank workspace with first-tree-read installed and a casual prompt.", expectedFacts: JWT_AUTH_EXPECTED_FACTS, expectedTrigger: false, id: "blank-casual-no-trigger", + impactNote: { mode: "absent" }, + managedTransport: null, prompt: "Please explain the Pomodoro technique in one sentence.", promptAlternates: ["How is your day going?"], readMode: "managed", @@ -22,6 +49,18 @@ export const FIRST_TREE_READ_CASES: readonly FirstTreeReadEvalCase[] = [ expectedFacts: JWT_AUTH_EXPECTED_FACTS, expectedTrigger: true, id: "tree-software-trigger", + impactNote: { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: { + allowedNodePaths: JWT_SOURCE_PATHS, + repository: EVAL_SOURCE_REPOSITORY, + }, + sourceCount: { max: 3, min: 1 }, + summaryConcepts: JWT_SUMMARY_CONCEPTS, + }, + managedTransport: "send", prompt: "For this project, what constraints should JWT auth routes follow?", promptAlternates: ["Analyze server route naming and multi-org permission boundaries for this project."], readMode: "managed", @@ -32,6 +71,18 @@ export const FIRST_TREE_READ_CASES: readonly FirstTreeReadEvalCase[] = [ expectedFacts: JWT_AUTH_EXPECTED_FACTS, expectedTrigger: true, id: "byo-scope-route-trigger", + impactNote: { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: { + allowedNodePaths: JWT_SOURCE_PATHS, + repository: EVAL_SOURCE_REPOSITORY, + }, + sourceCount: { max: 3, min: 1 }, + summaryConcepts: JWT_SUMMARY_CONCEPTS, + }, + managedTransport: null, prompt: "For this BYO Context Tree task, use the locally authorized SCOPE router and answer: what constraints should JWT auth routes follow?", promptAlternates: [ @@ -45,11 +96,83 @@ export const FIRST_TREE_READ_CASES: readonly FirstTreeReadEvalCase[] = [ expectedFacts: JWT_AUTH_EXPECTED_FACTS, expectedTrigger: false, id: "tree-nonsoftware-no-trigger", + impactNote: { mode: "absent" }, + managedTransport: null, prompt: "Recommend a weekend cooking menu.", promptAlternates: ["Write a short poem about summer."], readMode: "managed", workspaceKind: "context-tree", }, + { + description: "Managed tree navigation read that must not claim material decision influence.", + expectedFacts: NAVIGATION_EXPECTED_FACTS, + expectedTrigger: true, + id: "tree-navigation-no-impact", + impactNote: { mode: "absent" }, + managedTransport: "send", + prompt: + "Browse this Context Tree and list only its three top-level domain names. Do not make a design or implementation choice.", + promptAlternates: ["What are the three top-level domains in this Context Tree? Return only their names."], + readMode: "managed", + workspaceKind: "context-tree", + }, + { + description: "Chinese unresolved conflict that must not be rewritten as an already changed plan.", + expectedFacts: RELEASE_AUDIT_EXPECTED_FACTS, + expectedTrigger: true, + id: "tree-conflict-chinese", + impactNote: { + effect: "conflicted", + language: "zh", + mode: "present", + requiredSourceLabels: ["Rollout Policy"], + sourceAuthority: { + allowedNodePaths: ["product/release/rollout-policy/NODE.md"], + repository: EVAL_SOURCE_REPOSITORY, + }, + sourceCount: { max: 1, min: 1 }, + summaryConcepts: [["发布日期", "发布日"], ["安全审计"], ["仍待", "尚未", "需要决定", "需升级"]], + summaryForbidden: ["已调整", "已解决", "已排除", "已完成取舍"], + }, + managedTransport: "send", + prompt: + "发布日期已固定为 8 月 20 日且不能移动,但现在没有足够时间完成新的安全审计。请根据 Context Tree 判断能否直接发布;不要替我更改日期或跳过审计,若冲突未决就明确指出。", + promptAlternates: [ + "固定发布日期与发布前安全审计无法同时满足。请用中文依据 Context Tree 说明冲突,不要假装方案已经调整。", + ], + readMode: "managed", + workspaceKind: "context-tree", + }, + { + description: "Three-source note with a root node and duplicate titles that require readable disambiguation.", + expectedFacts: ROLLOUT_EXPECTED_FACTS, + expectedTrigger: true, + id: "tree-readable-multi-source-note", + impactNote: { + effect: "constrained", + language: "en", + mode: "present", + requiredSourceLabels: ["First Tree Read Eval Context", "Release · Rollout Policy", "Billing · Rollout Policy"], + sourceAuthority: { + allowedNodePaths: [ + "NODE.md", + "product/release/rollout-policy/NODE.md", + "product/billing/rollout-policy/NODE.md", + ], + repository: EVAL_SOURCE_REPOSITORY, + }, + sourceCount: { max: 3, min: 3 }, + summaryConcepts: [["rollout"], ["reviewable scope"], ["audit"], ["billing"]], + }, + managedTransport: "send", + prompt: + "Plan a production billing rollout. Apply the Context Tree's root rollout-scope rule, release security gate, and billing ordering constraint, and explain the resulting implementation boundary.", + promptAlternates: [ + "Use the root rollout rule plus both release and billing rollout policies to constrain a production billing launch.", + ], + readMode: "managed", + workspaceKind: "context-tree", + }, ]; export const FIRST_TREE_READ_PERIODIC_CASES: readonly FirstTreeReadEvalCase[] = [ @@ -59,6 +182,18 @@ export const FIRST_TREE_READ_PERIODIC_CASES: readonly FirstTreeReadEvalCase[] = expectedFacts: JWT_AUTH_EXPECTED_FACTS, expectedTrigger: true, id: "first-tree-read-runtime-generated-briefing-periodic", + impactNote: { + effect: "constrained", + language: "en", + mode: "present", + sourceAuthority: { + allowedNodePaths: JWT_SOURCE_PATHS, + repository: EVAL_SOURCE_REPOSITORY, + }, + sourceCount: { max: 3, min: 1 }, + summaryConcepts: JWT_SUMMARY_CONCEPTS, + }, + managedTransport: "send", prompt: "Use this workspace's current Context Tree to answer: what constraints should JWT auth routes follow for this project?", promptAlternates: ["Use the current Context Tree before answering: how should multi-org JWT route scopes work?"], diff --git a/packages/skill-evals/src/suites/first-tree-read/eval-cases.ts b/packages/skill-evals/src/suites/first-tree-read/eval-cases.ts index 57d30e9c9..6d2476549 100644 --- a/packages/skill-evals/src/suites/first-tree-read/eval-cases.ts +++ b/packages/skill-evals/src/suites/first-tree-read/eval-cases.ts @@ -26,6 +26,8 @@ export const FIRST_TREE_READ_EVAL_CASES: readonly SkillEvalCase[] = [ expected: { expectedFacts: evalCase.expectedFacts, expectedTrigger: evalCase.expectedTrigger, + impactNote: evalCase.impactNote, + managedTransport: evalCase.managedTransport, readMode: evalCase.readMode, }, fixture: { @@ -46,6 +48,8 @@ export const FIRST_TREE_READ_EVAL_CASES: readonly SkillEvalCase[] = [ expected: { expectedFacts: evalCase.expectedFacts, expectedTrigger: evalCase.expectedTrigger, + impactNote: evalCase.impactNote, + managedTransport: evalCase.managedTransport, runtimeBoundary: "generated briefing fixture only; not live First Tree Cloud E2E", }, fixture: { diff --git a/packages/skill-evals/src/suites/first-tree-read/fixture.ts b/packages/skill-evals/src/suites/first-tree-read/fixture.ts index 627ea4ec4..716bbf80a 100644 --- a/packages/skill-evals/src/suites/first-tree-read/fixture.ts +++ b/packages/skill-evals/src/suites/first-tree-read/fixture.ts @@ -12,6 +12,7 @@ import type { FirstTreeReadEvalCase, FixtureValidation, WorkspaceKind } from "./ const DOMAIN_NODE_TARGET_COUNT = 100; const NAVIGATION_NODE_MARKER = "evalNodeKind: navigation"; const SKILL_NAME = "first-tree-read"; +const EVAL_BINDING_REPOSITORY = "https://github.com/example/context-tree.git"; const RUNTIME_SKILL_NAMES = [ "first-tree-welcome", "first-tree-read", @@ -24,6 +25,7 @@ const RUNTIME_SKILL_NAMES = [ type DomainNode = { facts: readonly string[]; path: string; + title?: string; }; type RequiredTreeFile = { @@ -44,17 +46,18 @@ function titleFromPath(path: string): string { } function nodeMarkdown(node: DomainNode): string { + const title = node.title ?? titleFromPath(node.path); const facts = node.facts.length > 0 ? node.facts.map((fact) => `- ${fact}`).join("\n") : "- This eval node provides software-domain context only."; return `--- -title: "${titleFromPath(node.path)}" +title: "${title}" owners: [eval-owner] --- -# ${titleFromPath(node.path)} +# ${title} ${facts} `; @@ -85,6 +88,10 @@ owners: [eval-owner] This deterministic Context Tree fixture contains only software engineering domain knowledge. It intentionally omits cooking, poetry, lifestyle, and other non-software facts so off-topic prompts should not need a tree read. + +- Top-level Context Tree domains are systems, domains, and operations. +- Every production rollout must keep a single reviewable scope across release + and billing policy. `; } @@ -114,12 +121,19 @@ evaluations. } function workspaceAgentsMarkdown(skillDescription: string, workspaceKind: WorkspaceKind): string { + const standingContext = + workspaceKind === "byo-context-tree" + ? "The trusted activation standing context declares `consumerKind: byo`, provider `codex`, immutable project selector `--pathless`, and a verified session-candidate receipt for this eval task." + : workspaceKind === "context-tree" + ? `The trusted runtime standing context declares \`consumerKind: managed\`, Context Tree binding repository \`${EVAL_BINDING_REPOSITORY}\`, and binding branch \`main\`.` + : "The trusted runtime standing context declares `consumerKind: managed`."; + return `# Eval Workspace Instructions Use installed skills only when the skill description applies to the user's prompt. Do not call \`first-tree\` for casual or non-software prompts. -${workspaceKind === "byo-context-tree" ? "The trusted activation standing context declares `consumerKind: byo`, provider `codex`, immutable project selector `--pathless`, and a verified session-candidate receipt for this eval task." : "The trusted runtime standing context declares `consumerKind: managed`."} +${standingContext} ## Available Skills @@ -174,6 +188,8 @@ Your fixed working directory is \`${workspacePath}\`. The runtime marker # Context Tree (First Tree Managed) The current Context Tree checkout is \`${contextTreePath}\`. +Its binding repository is \`${EVAL_BINDING_REPOSITORY}\` and its binding branch +is \`main\`. ## Context Tree Policy @@ -369,6 +385,22 @@ function operationNodes(): DomainNode[] { const nodes: DomainNode[] = []; for (const area of areas) { for (const [group, topic] of topics) { + if (area === "release" && group === "safety" && topic === "invariants") { + nodes.push({ + facts: ["Production releases require completed security-audit approval before deployment."], + path: "product/release/rollout-policy", + title: "Rollout Policy", + }); + continue; + } + if (area === "release" && group === "safety" && topic === "failure-mode") { + nodes.push({ + facts: ["Billing changes must roll out after the core release reaches stable monitoring."], + path: "product/billing/rollout-policy", + title: "Rollout Policy", + }); + continue; + } nodes.push({ facts: [facts[area] ?? "Operation facts are software-only."], path: `operations/${area}/${group}/${topic}`, @@ -444,11 +476,11 @@ function writeContextTreeFixture(paths: RunPaths, workspaceKind: WorkspaceKind): writeText(join(contextTreePath, node.path, "NODE.md"), nodeMarkdown(node)); } - initializeGitRepo(paths, contextTreePath); + initializeGitRepo(paths, contextTreePath, managedWorkspace); return contextTreePath; } -function initializeGitRepo(paths: RunPaths, contextTreePath: string): void { +function initializeGitRepo(paths: RunPaths, contextTreePath: string, managedWorkspace: boolean): void { const originPath = join(paths.runRoot, "context-tree-origin.git"); const commands: CommandResult[] = [ runCommand("git", ["init", "--initial-branch=main"], contextTreePath), @@ -465,6 +497,10 @@ function initializeGitRepo(paths: RunPaths, contextTreePath: string): void { for (const result of commands) { assertCommandOk(result); } + + if (managedWorkspace) { + assertCommandOk(runCommand("git", ["remote", "set-url", "origin", EVAL_BINDING_REPOSITORY], contextTreePath)); + } } export function setupFixture(evalCase: FirstTreeReadEvalCase, paths: RunPaths, reporter: EvalReporter): string | null { diff --git a/packages/skill-evals/src/suites/first-tree-read/metrics.ts b/packages/skill-evals/src/suites/first-tree-read/metrics.ts index a232508ca..046dceb3e 100644 --- a/packages/skill-evals/src/suites/first-tree-read/metrics.ts +++ b/packages/skill-evals/src/suites/first-tree-read/metrics.ts @@ -1,8 +1,15 @@ import { findStringValue, isRecord, isStringArray } from "../../core/events.js"; -import type { EvalMetrics, FixtureValidation, ReadMode } from "./types.js"; +import type { + EvalMetrics, + FixtureValidation, + ImpactNoteEffect, + ImpactNoteExpectation, + ImpactNoteLanguage, + ManagedTransport, + ReadMode, +} from "./types.js"; const HELP_ARGV = ["tree", "tree", "--help"]; -const READ_HELP_ARGV = ["tree", "read", "--help"]; const TEXT_KEYS = ["content", "message", "output_text", "text"]; type FactMatcher = { @@ -32,8 +39,294 @@ const FACT_MATCHERS: readonly FactMatcher[] = [ ], fact: "HTTP routes must follow the repo path conventions document before auth or multi-org changes.", }, + { + all: [/(top-level|顶层)/iu, /(systems?|系统)/iu, /(domains?|领域)/iu, /(operations?|运维|操作)/iu], + fact: "Top-level Context Tree domains are systems, domains, and operations.", + }, + { + all: [ + /(production release|生产发布|正式发布)/iu, + /(security[- ]audit|安全审计)/iu, + /(before deployment|发布前|部署前)/iu, + ], + fact: "Production releases require completed security-audit approval before deployment.", + }, + { + all: [ + /(production rollout|生产发布|正式发布)/iu, + /(single reviewable scope|统一[^。\n]*审查范围|单一[^。\n]*范围)/iu, + ], + fact: "Every production rollout must keep a single reviewable scope across release and billing policy.", + }, + { + all: [/(billing changes?|计费变更)/iu, /(core release|核心版本|核心发布)/iu, /(stable monitoring|稳定监控)/iu], + fact: "Billing changes must roll out after the core release reaches stable monitoring.", + }, ]; +const EFFECT_LABELS: Record> = { + en: { + conflicted: "Conflict surfaced", + confirmed: "Direction supported", + constrained: "Options narrowed", + redirected: "Approach changed", + }, + zh: { + conflicted: "发现约束冲突", + confirmed: "支持当前方向", + constrained: "收窄可选范围", + redirected: "改变方案路径", + }, +}; + +type ImpactNoteObservation = { + atEnd: boolean; + blankLineBefore: boolean; + effectLabel: string; + exactLinksOk: boolean; + language: ImpactNoteLanguage; + logicalLinesOk: boolean; + sourceLabels: readonly string[]; + sourceScaffoldingOk: boolean; + sourceUrls: readonly string[]; + summary: string; + summaryObjectiveOk: boolean; + textIndex: number; +}; + +type ExactSourceLink = { + commit: string; + nodePath: string; + repositoryIdentity: string; +}; + +function canonicalRepositoryIdentity(value: string): string | null { + try { + const url = new URL(value); + const path = url.pathname.replace(/^\/+|\/+$/gu, "").replace(/\.git$/iu, ""); + return path.length > 0 ? `${url.host.toLowerCase()}/${path.toLowerCase()}` : null; + } catch { + const scpMatch = /^(?:[^@\s]+@)?([^:\s]+):(.+)$/u.exec(value.trim()); + if (!scpMatch) return null; + const host = scpMatch[1]?.toLowerCase() ?? ""; + const path = (scpMatch[2] ?? "") + .replace(/^\/+|\/+$/gu, "") + .replace(/\.git$/iu, "") + .toLowerCase(); + return host.length > 0 && path.length > 0 ? `${host}/${path}` : null; + } +} + +function parseExactCredentialFreeSourceLink(value: string): ExactSourceLink | null { + try { + const url = new URL(value); + if ( + url.protocol !== "https:" || + url.username !== "" || + url.password !== "" || + url.search !== "" || + url.hash !== "" + ) { + return null; + } + + const segments = url.pathname.split("/").filter(Boolean); + const blobIndex = segments.findIndex( + (segment, index) => + segment === "blob" && /^(?:[0-9a-fA-F]{40}|[0-9a-fA-F]{64})$/u.test(segments[index + 1] ?? ""), + ); + if (blobIndex <= 0 || blobIndex + 2 >= segments.length) return null; + const repositoryEnd = segments[blobIndex - 1] === "-" ? blobIndex - 1 : blobIndex; + if (repositoryEnd <= 0) return null; + + const repositoryPath = segments + .slice(0, repositoryEnd) + .join("/") + .replace(/\.git$/iu, ""); + const nodePath = segments + .slice(blobIndex + 2) + .map((segment) => decodeURIComponent(segment)) + .join("/"); + if (repositoryPath.length === 0 || nodePath.length === 0) return null; + + return { + commit: segments[blobIndex + 1]?.toLowerCase() ?? "", + nodePath, + repositoryIdentity: `${url.host.toLowerCase()}/${repositoryPath.toLowerCase()}`, + }; + } catch { + return null; + } +} + +function parseImpactNotes(texts: readonly string[]): readonly ImpactNoteObservation[] { + const observations: ImpactNoteObservation[] = []; + + for (const [textIndex, text] of texts.entries()) { + const lines = text.replace(/\r/gu, "").split("\n"); + for (let index = 0; index < lines.length; index += 1) { + const firstLine = lines[index] ?? ""; + const titleMatch = /^> \*\*(Context Tree impact|Context Tree 影响) · ([^*]+)\*\*\\$/u.exec(firstLine); + if (!titleMatch) continue; + + const language: ImpactNoteLanguage = titleMatch[1] === "Context Tree 影响" ? "zh" : "en"; + const secondLine = lines[index + 1] ?? ""; + const thirdLine = lines[index + 2] ?? ""; + const summaryMatch = /^> (.+)\\$/u.exec(secondLine); + const sourcePrefix = language === "zh" ? /^> \*\*来源\*\* · /u : /^> \*\*(Source|Sources)\*\* · /u; + const sourcePrefixMatch = sourcePrefix.exec(thirdLine); + const markdownLinks = [...thirdLine.matchAll(/\[([^\]\n]+)\]\(([^)\s]+)\)/gu)]; + const exactLinks = markdownLinks.filter((match) => parseExactCredentialFreeSourceLink(match[2] ?? "") !== null); + const expectedEnglishSource = markdownLinks.length === 1 ? "Source" : "Sources"; + const sourceLabel = language === "zh" ? "来源" : expectedEnglishSource; + const expectedSourceLine = `> **${sourceLabel}** · ${markdownLinks.map((match) => match[0]).join(" · ")}`; + const sourceScaffoldingOk = + sourcePrefixMatch !== null && + (language === "zh" || sourcePrefixMatch[1] === expectedEnglishSource) && + markdownLinks.length > 0 && + thirdLine === expectedSourceLine; + const summary = summaryMatch?.[1]?.trim() ?? ""; + + observations.push({ + atEnd: lines.slice(index + 3).every((line) => line.trim() === ""), + blankLineBefore: index > 0 && (lines[index - 1] ?? "").trim() === "", + effectLabel: titleMatch[2]?.trim() ?? "", + exactLinksOk: exactLinks.length === markdownLinks.length && exactLinks.length > 0, + language, + logicalLinesOk: + summaryMatch !== null && sourcePrefixMatch !== null && !(lines[index + 3] ?? "").startsWith(">"), + sourceLabels: markdownLinks.map((match) => match[1] ?? ""), + sourceScaffoldingOk, + sourceUrls: markdownLinks.map((match) => match[2] ?? ""), + summary, + summaryObjectiveOk: !/(^|\s)(I|We)\s+(used?|read|consulted)|我(使用|读取|参考)了?\s*Context Tree/iu.test( + summary, + ), + textIndex, + }); + } + } + + return observations; +} + +function visibleUrlsCredentialFree(texts: readonly string[]): boolean { + const urls = texts + .flatMap((text) => [...text.matchAll(/\b[a-z][a-z0-9+.-]*:\/\/[^\s<>\])]+/giu)].map((match) => match[0] ?? "")) + .filter(Boolean); + + return urls.every((value) => { + try { + const url = new URL(value); + // SSH commonly carries the transport identity as `git@host`; that + // username is part of the repository contract, not a credential. + return url.password === "" && (url.protocol === "ssh:" || url.username === ""); + } catch { + return false; + } + }); +} + +function sourceAuthorityMatches( + observation: ImpactNoteObservation | null, + expectation: ImpactNoteExpectation, + selectedExactCommit: string | null, +): boolean { + if (expectation.mode === "absent") return true; + if (observation === null) return false; + + const expectedRepository = canonicalRepositoryIdentity(expectation.sourceAuthority.repository); + const expectedCommit = (expectation.sourceAuthority.exactCommit ?? selectedExactCommit)?.toLowerCase() ?? null; + const allowedPaths = new Set(expectation.sourceAuthority.allowedNodePaths); + if (expectedRepository === null || expectedCommit === null) return false; + + return observation.sourceUrls.every((value) => { + const source = parseExactCredentialFreeSourceLink(value); + return ( + source !== null && + source.repositoryIdentity === expectedRepository && + source.commit === expectedCommit && + allowedPaths.has(source.nodePath) + ); + }); +} + +function includesAny(value: string, alternatives: readonly string[]): boolean { + const normalized = value.toLocaleLowerCase(); + return alternatives.some((alternative) => normalized.includes(alternative.toLocaleLowerCase())); +} + +function deriveImpactNoteMetrics( + texts: readonly string[], + expectation: ImpactNoteExpectation, + options: { contextDecisionMetadataPresent: boolean; selectedExactCommit: string | null }, +) { + const observations = parseImpactNotes(texts); + const observation = observations[0] ?? null; + const allText = texts.join("\n"); + const metadataFree = + !options.contextDecisionMetadataPresent && + !/contextDecision|["']effect["']\s*:|["']evidence["']\s*:/u.test(allText); + const atFinalEnd = observation?.atEnd === true && observation.textIndex === texts.length - 1; + const sourceAuthorityOk = sourceAuthorityMatches(observation, expectation, options.selectedExactCommit); + const visibleUrlsSafe = visibleUrlsCredentialFree(texts); + const summaryConceptsOk = + expectation.mode === "absent" || + (expectation.summaryConcepts?.every((alternatives) => includesAny(observation?.summary ?? "", alternatives)) ?? + true); + const summaryForbiddenOk = + expectation.mode === "absent" || + !(expectation.summaryForbidden?.some((value) => includesAny(observation?.summary ?? "", [value])) ?? false); + const requiredSourceLabelsOk = + expectation.mode === "absent" || + (expectation.requiredSourceLabels?.every((label) => observation?.sourceLabels.includes(label)) ?? true); + const sourceCountOk = + expectation.mode === "absent" || + ((observation?.sourceLabels.length ?? 0) >= expectation.sourceCount.min && + (observation?.sourceLabels.length ?? 0) <= expectation.sourceCount.max); + const expectedEffectLabel = + expectation.mode === "present" ? EFFECT_LABELS[expectation.language][expectation.effect] : null; + const behaviorOk = + expectation.mode === "absent" + ? observations.length === 0 && metadataFree + : observations.length === 1 && + observation !== null && + atFinalEnd && + observation.blankLineBefore && + observation.logicalLinesOk && + observation.sourceScaffoldingOk && + observation.summaryObjectiveOk && + observation.exactLinksOk && + sourceAuthorityOk && + observation.language === expectation.language && + observation.effectLabel === expectedEffectLabel && + sourceCountOk && + requiredSourceLabelsOk && + summaryConceptsOk && + summaryForbiddenOk && + metadataFree && + visibleUrlsSafe; + + return { + impactNoteBehaviorOk: behaviorOk, + impactNoteAtFinalEnd: atFinalEnd, + impactNoteBlankLineBefore: observation?.blankLineBefore ?? false, + impactNoteCount: observations.length, + impactNoteEffect: observation?.effectLabel ?? null, + impactNoteExactLinksOk: observation?.exactLinksOk ?? false, + impactNoteLanguage: observation?.language ?? null, + impactNoteLogicalLinesOk: observation?.logicalLinesOk ?? false, + impactNoteMetadataFree: metadataFree, + impactNoteSourceAuthorityOk: sourceAuthorityOk, + impactNoteSourceCount: observation?.sourceLabels.length ?? 0, + impactNoteSourceLabels: observation?.sourceLabels ?? [], + impactNoteSummaryConceptsOk: summaryConceptsOk, + impactNoteSummaryForbiddenOk: summaryForbiddenOk, + impactNoteSummaryObjectiveOk: observation?.summaryObjectiveOk ?? false, + impactNoteVisibleUrlsCredentialFree: visibleUrlsSafe, + }; +} + function argvEquals(left: readonly string[], right: readonly string[]): boolean { if (left.length !== right.length) return false; for (let index = 0; index < left.length; index += 1) { @@ -50,13 +343,19 @@ function isHelpArgv(argv: readonly string[]): boolean { return argvEquals(commandArgv(argv), HELP_ARGV); } -function isReadHelpArgv(argv: readonly string[]): boolean { - return argvEquals(commandArgv(argv), READ_HELP_ARGV); +function isReadRouteArgv(argv: readonly string[]): boolean { + const command = commandArgv(argv); + return command[0] === "context" && command[1] === "route"; } function isReadActivationArgv(argv: readonly string[]): boolean { const command = commandArgv(argv); - return command[0] === "tree" && command[1] === "read" && !isReadHelpArgv(argv); + return command[0] === "context" && command[1] === "snapshot"; +} + +function isLegacyReadActivationArgv(argv: readonly string[]): boolean { + const command = commandArgv(argv); + return command[0] === "tree" && command[1] === "read" && !command.includes("--help") && !command.includes("-h"); } function isTreeTreeArgv(argv: readonly string[]): boolean { @@ -68,6 +367,50 @@ function isTreeSelectorArgv(argv: readonly string[]): boolean { return isTreeTreeArgv(argv) && !isHelpArgv(argv); } +function isChatAuthoringArgv(argv: readonly string[]): boolean { + const command = commandArgv(argv); + return command[0] === "chat" && (command[1] === "send" || command[1] === "ask"); +} + +function isChatProgressArgv(argv: readonly string[]): boolean { + const command = commandArgv(argv); + return command[0] === "chat" && command[1] === "update"; +} + +function chatAuthoringKind(argv: readonly string[]): "ask" | "send" | null { + const command = commandArgv(argv); + return command[0] === "chat" && (command[1] === "ask" || command[1] === "send") ? command[1] : null; +} + +function metadataOptionValues(argv: readonly string[]): readonly string[] { + const command = commandArgv(argv); + const values: string[] = []; + for (let index = 2; index < command.length; index += 1) { + const arg = command[index] ?? ""; + if (arg === "--metadata" || arg === "-m") { + const value = command[index + 1]; + if (value !== undefined) values.push(value); + index += 1; + continue; + } + if (arg.startsWith("--metadata=") || arg.startsWith("-m=")) { + values.push(arg.slice(arg.indexOf("=") + 1)); + } + } + return values; +} + +function hasContextDecisionMetadata(argv: readonly string[]): boolean { + return metadataOptionValues(argv).some((value) => { + try { + const parsed: unknown = JSON.parse(value); + return isRecord(parsed) && Object.hasOwn(parsed, "contextDecision"); + } catch { + return /["']?contextDecision["']?\s*:/u.test(value); + } + }); +} + function isModelPhase(event: Record): boolean { return event.phase === "model"; } @@ -171,7 +514,7 @@ function collectModelOutputText(event: unknown): string[] { function normalizeForMatch(value: string): string { return value .toLowerCase() - .replace(/[^a-z0-9]+/gu, " ") + .replace(/[^\p{L}\p{N}]+/gu, " ") .replace(/\s+/gu, " ") .trim(); } @@ -206,17 +549,28 @@ export function deriveMetrics( fixtureValidation: FixtureValidation, runnerExitCode: number | null, expectedFacts: readonly string[], + impactNoteExpectation: ImpactNoteExpectation = { mode: "absent" }, + managedTransportExpectation: ManagedTransport | null = null, ): EvalMetrics { let firstTreeCalls = 0; let helpCalls = 0; + let legacyReadActivationCalls = 0; let readActivationCalls = 0; + let readRouteCalls = 0; let skillFileReadObserved = false; + const authoringCalls: Array<{ + argv: string[]; + body: string; + contextDecisionMetadataPresent: boolean; + exitCode: number | null; + }> = []; + const progressCalls: Array<{ argv: string[]; body: string; exitCode: number | null }> = []; const firstTreeArgv: string[][] = []; const firstTreeCommandResults: Array<{ argv: string[]; exitCode: number }> = []; const helpExitCodes: number[] = []; const modelOutputTexts: string[] = []; const readActivationResults: Array<{ exactCommit: string | null; exitCode: number }> = []; - const readHelpExitCodes: number[] = []; + const readRouteExitCodes: number[] = []; const selectorSnapshotResults: Array<{ actualHead: string | null; detachedHead: boolean }> = []; for (const event of events) { @@ -224,7 +578,7 @@ export function deriveMetrics( skillFileReadObserved = true; } - modelOutputTexts.push(...collectModelOutputText(event)); + modelOutputTexts.push(...uniqueStrings(collectModelOutputText(event))); if (!isRecord(event)) continue; const type = eventType(event); @@ -235,21 +589,50 @@ export function deriveMetrics( if (type === "first_tree_call") { firstTreeCalls += 1; firstTreeArgv.push([...argv]); + if (isChatAuthoringArgv(argv)) { + authoringCalls.push({ + argv: [...argv], + body: typeof event.body === "string" ? event.body : "", + contextDecisionMetadataPresent: hasContextDecisionMetadata(argv), + exitCode: null, + }); + } + if (isChatProgressArgv(argv)) { + progressCalls.push({ + argv: [...argv], + body: typeof event.body === "string" ? event.body : "", + exitCode: null, + }); + } if (isHelpArgv(argv)) { helpCalls += 1; } if (isReadActivationArgv(argv)) { readActivationCalls += 1; } + if (isLegacyReadActivationArgv(argv)) { + legacyReadActivationCalls += 1; + } + if (isReadRouteArgv(argv)) { + readRouteCalls += 1; + } } if (type === "first_tree_result" && typeof event.exitCode === "number") { firstTreeCommandResults.push({ argv: [...argv], exitCode: event.exitCode }); + if (isChatAuthoringArgv(argv)) { + const pendingCall = authoringCalls.find((call) => call.exitCode === null && argvEquals(call.argv, argv)); + if (pendingCall) pendingCall.exitCode = event.exitCode; + } + if (isChatProgressArgv(argv)) { + const pendingCall = progressCalls.find((call) => call.exitCode === null && argvEquals(call.argv, argv)); + if (pendingCall) pendingCall.exitCode = event.exitCode; + } if (isHelpArgv(argv)) { helpExitCodes.push(event.exitCode); } - if (isReadHelpArgv(argv)) { - readHelpExitCodes.push(event.exitCode); + if (isReadRouteArgv(argv)) { + readRouteExitCodes.push(event.exitCode); } if (isReadActivationArgv(argv)) { readActivationResults.push({ @@ -267,8 +650,17 @@ export function deriveMetrics( } } + const successfulAuthoringCalls = authoringCalls.filter((call) => call.exitCode === 0); + const successfulProgressCalls = progressCalls.filter((call) => call.exitCode === 0); + const authoredOutputTexts = successfulAuthoringCalls.map((call) => call.body); + const factOutputTexts = authoringCalls.length > 0 ? authoredOutputTexts : modelOutputTexts; + const visibleOutputTexts = + authoringCalls.length > 0 || progressCalls.length > 0 + ? [...modelOutputTexts, ...successfulProgressCalls.map((call) => call.body), ...authoredOutputTexts] + : modelOutputTexts; + const contextDecisionMetadataPresent = successfulAuthoringCalls.some((call) => call.contextDecisionMetadataPresent); const facts = uniqueStrings(expectedFacts); - const factHits = expectedFactHits(modelOutputTexts.join("\n"), facts); + const factHits = expectedFactHits(factOutputTexts.join("\n"), facts); const helpSucceeded = firstTreeCommandResults.some((result) => isHelpArgv(result.argv) && result.exitCode === 0); const selectionSucceeded = firstTreeCommandResults.some( (result) => isTreeSelectorArgv(result.argv) && result.exitCode === 0, @@ -278,18 +670,19 @@ export function deriveMetrics( readActivationResults.length === 1 && readActivationResults[0]?.exitCode === 0 && readActivationResults[0]?.exactCommit !== null; - const readHelpSucceeded = readHelpExitCodes.some((exitCode) => exitCode === 0); + const readRouteSucceeded = readRouteCalls === 1 && readRouteExitCodes.length === 1 && readRouteExitCodes[0] === 0; const selectorCalls = firstTreeArgv.filter(isTreeSelectorArgv); const byoSelectorsNoPull = selectorCalls.length > 0 && selectorCalls.every((argv) => argv.includes("--no-pull")); - const readHelpIndex = firstTreeArgv.findIndex(isReadHelpArgv); + const readRouteIndex = firstTreeArgv.findIndex(isReadRouteArgv); const readActivationIndex = firstTreeArgv.findIndex(isReadActivationArgv); const hierarchyHelpIndex = firstTreeArgv.findIndex(isHelpArgv); const selectorIndexes = firstTreeArgv .map((argv, index) => (isTreeSelectorArgv(argv) ? index : -1)) .filter((index) => index >= 0); const byoReadSequenceOk = - readHelpIndex >= 0 && - readActivationIndex > readHelpIndex && + legacyReadActivationCalls === 0 && + readRouteIndex >= 0 && + readActivationIndex > readRouteIndex && hierarchyHelpIndex > readActivationIndex && selectorIndexes.length > 0 && selectorIndexes.every((index) => index > hierarchyHelpIndex); @@ -304,6 +697,14 @@ export function deriveMetrics( selectorSnapshotResults.length === selectorCalls.length && selectorSnapshotResults.every((result) => result.detachedHead); const modelFirstTreeCommandsOk = firstTreeCommandResults.every((result) => result.exitCode === 0); + const selectedExactCommit = exactCommit ?? selectorSnapshotResults.at(-1)?.actualHead ?? null; + const finalAuthoringKind = chatAuthoringKind(successfulAuthoringCalls.at(-1)?.argv ?? []); + const managedFinalTransportOk = + managedTransportExpectation === null || finalAuthoringKind === managedTransportExpectation; + const impactNoteMetrics = deriveImpactNoteMetrics(visibleOutputTexts, impactNoteExpectation, { + contextDecisionMetadataPresent, + selectedExactCommit, + }); return { expectedFactHits: factHits, @@ -316,14 +717,18 @@ export function deriveMetrics( helpCalls, helpExitCodes, helpSucceeded, + ...impactNoteMetrics, byoReadSequenceOk, byoSelectorsNoPull, byoSnapshotDetached, byoSnapshotExactHeadConsistent, + legacyReadActivationCalls, modelFirstTreeCommandsOk, + managedFinalTransportOk, readActivationCalls, readActivationSucceeded, - readHelpSucceeded, + readRouteCalls, + readRouteSucceeded, runnerExitCode, selectionSucceeded, skillFileReadObserved, @@ -338,7 +743,7 @@ export function casePassed(expectedTrigger: boolean, metrics: EvalMetrics, readM if (expectedTrigger) { const readModePassed = readMode === "managed" || - (metrics.readHelpSucceeded && + (metrics.readRouteSucceeded && metrics.readActivationSucceeded && metrics.byoReadSequenceOk && metrics.byoSelectorsNoPull && @@ -347,15 +752,18 @@ export function casePassed(expectedTrigger: boolean, metrics: EvalMetrics, readM return ( metrics.skillFileReadObserved && metrics.expectedFactsObserved && + metrics.impactNoteBehaviorOk && metrics.helpSucceeded && metrics.selectionSucceeded && metrics.modelFirstTreeCommandsOk && + (readMode === "byo" || metrics.managedFinalTransportOk) && readModePassed ); } return ( !metrics.skillHit && + metrics.impactNoteBehaviorOk && metrics.expectedFactHits.length === 0 && metrics.firstTreeCalls === 0 && metrics.firstTreeCommandResults.length === 0 && diff --git a/packages/skill-evals/src/suites/first-tree-read/runner.ts b/packages/skill-evals/src/suites/first-tree-read/runner.ts index a0bc777fe..67f0f6625 100644 --- a/packages/skill-evals/src/suites/first-tree-read/runner.ts +++ b/packages/skill-evals/src/suites/first-tree-read/runner.ts @@ -42,7 +42,14 @@ export async function runFirstTreeReadCase( ); const runnerExitCode = runnerResult.exitCode; const events = readEvents(paths.eventsPath); - const metrics = deriveMetrics(events, fixtureValidation, runnerExitCode, evalCase.expectedFacts); + const metrics = deriveMetrics( + events, + fixtureValidation, + runnerExitCode, + evalCase.expectedFacts, + evalCase.impactNote, + evalCase.managedTransport, + ); const passed = casePassed(evalCase.expectedTrigger, metrics, evalCase.readMode); const grading = buildGrading(evalCase.id, metrics, evalCase.expectedTrigger, passed, evalCase.readMode); const observability = deriveRunObservability(events); diff --git a/packages/skill-evals/src/suites/first-tree-read/summary.ts b/packages/skill-evals/src/suites/first-tree-read/summary.ts index f44b2565e..a637f6b4a 100644 --- a/packages/skill-evals/src/suites/first-tree-read/summary.ts +++ b/packages/skill-evals/src/suites/first-tree-read/summary.ts @@ -79,12 +79,23 @@ export function driftNote( } if (expectedTrigger && readMode === "byo") { - if (!metrics.readHelpSucceeded) notes.push("Required first-tree tree read --help command did not succeed."); + if (!metrics.readRouteSucceeded) { + notes.push( + `BYO Read required exactly one successful context route command; observed calls=${metrics.readRouteCalls}.`, + ); + } if (!metrics.readActivationSucceeded) { notes.push(`BYO Read required exactly one successful activation; observed calls=${metrics.readActivationCalls}.`); } + if (metrics.legacyReadActivationCalls > 0) { + notes.push( + `BYO Read forbids legacy explicit-Team tree read activation; observed calls=${metrics.legacyReadActivationCalls}.`, + ); + } if (!metrics.byoReadSequenceOk) { - notes.push("BYO Read commands did not follow read help → activation → hierarchy help → selector order."); + notes.push( + "BYO Read commands did not follow context route → context snapshot → hierarchy help → selector order.", + ); } if (!metrics.byoSelectorsNoPull) notes.push("Every BYO hierarchy selector must include --no-pull."); if (!metrics.byoSnapshotDetached || !metrics.byoSnapshotExactHeadConsistent) { @@ -92,12 +103,22 @@ export function driftNote( } } + if (expectedTrigger && readMode === "managed" && !metrics.managedFinalTransportOk) { + notes.push("The impact note was not delivered through the required final managed chat send or blocking chat ask."); + } + if (expectedTrigger && !metrics.expectedFactsObserved) { notes.push( "Expected Context Tree facts were not surfaced in the model output; inspect events.jsonl for the final assistant messages.", ); } + if (!metrics.impactNoteBehaviorOk) { + notes.push( + `Visible Context Tree impact-note behavior failed: count=${metrics.impactNoteCount}; language=${metrics.impactNoteLanguage ?? "none"}; effect=${metrics.impactNoteEffect ?? "none"}; final end=${metrics.impactNoteAtFinalEnd}; logical lines=${metrics.impactNoteLogicalLinesOk}; blank line=${metrics.impactNoteBlankLineBefore}; objective summary=${metrics.impactNoteSummaryObjectiveOk}; exact links=${metrics.impactNoteExactLinksOk}; source authority=${metrics.impactNoteSourceAuthorityOk}; source labels=${metrics.impactNoteSourceLabels.join(" | ") || "none"}; metadata free=${metrics.impactNoteMetadataFree}; visible URL credentials absent=${metrics.impactNoteVisibleUrlsCredentialFree}.`, + ); + } + if (!expectedTrigger && metrics.expectedFactHits.length > 0) { notes.push(`Off-topic case surfaced Context Tree fact(s): ${metrics.expectedFactHits.join(" | ")}.`); } @@ -117,26 +138,30 @@ export function buildGrading( const routingPass = expectedTrigger ? metrics.skillFileReadObserved : !unexpectedReadUse; const byoProcessPassed = readMode === "managed" || - (metrics.readHelpSucceeded && + (metrics.readRouteSucceeded && metrics.readActivationSucceeded && metrics.byoReadSequenceOk && metrics.byoSelectorsNoPull && metrics.byoSnapshotDetached && metrics.byoSnapshotExactHeadConsistent); + const managedTransportPassed = readMode === "byo" || metrics.managedFinalTransportOk; const processPass = expectedTrigger ? metrics.fixtureValidationOk && metrics.runnerExitCode === 0 && metrics.helpSucceeded && metrics.selectionSucceeded && metrics.modelFirstTreeCommandsOk && + managedTransportPassed && byoProcessPassed : metrics.fixtureValidationOk && metrics.runnerExitCode === 0 && metrics.firstTreeCalls === 0 && metrics.firstTreeCommandResults.length === 0 && metrics.modelFirstTreeCommandsOk; - const outcomePass = expectedTrigger ? metrics.expectedFactsObserved : metrics.expectedFactHits.length === 0; - const riskPass = metrics.modelFirstTreeCommandsOk; + const outcomePass = expectedTrigger + ? metrics.expectedFactsObserved && metrics.impactNoteBehaviorOk + : metrics.expectedFactHits.length === 0 && metrics.impactNoteBehaviorOk; + const riskPass = metrics.modelFirstTreeCommandsOk && metrics.impactNoteMetadataFree; const failedCommands = metrics.firstTreeCommandResults.filter((result) => result.exitCode !== 0); return { @@ -151,28 +176,33 @@ export function buildGrading( evidence( "process_pass", expectedTrigger - ? `fixture ok=${metrics.fixtureValidationOk}; runner exit=${metrics.runnerExitCode}; read mode=${readMode}; read help succeeded=${metrics.readHelpSucceeded}; activation calls=${metrics.readActivationCalls}; activation succeeded=${metrics.readActivationSucceeded}; sequence ok=${metrics.byoReadSequenceOk}; selectors no-pull=${metrics.byoSelectorsNoPull}; detached=${metrics.byoSnapshotDetached}; exact head consistent=${metrics.byoSnapshotExactHeadConsistent}; hierarchy help succeeded=${metrics.helpSucceeded}; selector succeeded=${metrics.selectionSucceeded}; first-tree commands ok=${metrics.modelFirstTreeCommandsOk}` + ? `fixture ok=${metrics.fixtureValidationOk}; runner exit=${metrics.runnerExitCode}; read mode=${readMode}; route calls=${metrics.readRouteCalls}; route succeeded=${metrics.readRouteSucceeded}; activation calls=${metrics.readActivationCalls}; activation succeeded=${metrics.readActivationSucceeded}; legacy activation calls=${metrics.legacyReadActivationCalls}; sequence ok=${metrics.byoReadSequenceOk}; selectors no-pull=${metrics.byoSelectorsNoPull}; detached=${metrics.byoSnapshotDetached}; exact head consistent=${metrics.byoSnapshotExactHeadConsistent}; managed final transport ok=${metrics.managedFinalTransportOk}; hierarchy help succeeded=${metrics.helpSucceeded}; selector succeeded=${metrics.selectionSucceeded}; first-tree commands ok=${metrics.modelFirstTreeCommandsOk}` : `fixture ok=${metrics.fixtureValidationOk}; runner exit=${metrics.runnerExitCode}; model first-tree calls=${metrics.firstTreeCalls}; first-tree results=${metrics.firstTreeCommandResults.length}`, ), evidence( "outcome_pass", expectedTrigger - ? `expected facts observed=${metrics.expectedFactsObserved}; hits=${metrics.expectedFactHits.join(" | ") || "none"}` - : `off-topic expected fact hits=${metrics.expectedFactHits.join(" | ") || "none"}`, + ? `expected facts observed=${metrics.expectedFactsObserved}; hits=${metrics.expectedFactHits.join(" | ") || "none"}; impact-note behavior ok=${metrics.impactNoteBehaviorOk}; count=${metrics.impactNoteCount}; effect=${metrics.impactNoteEffect ?? "none"}; language=${metrics.impactNoteLanguage ?? "none"}; sources=${metrics.impactNoteSourceLabels.join(" | ") || "none"}` + : `off-topic expected fact hits=${metrics.expectedFactHits.join(" | ") || "none"}; impact-note behavior ok=${metrics.impactNoteBehaviorOk}; count=${metrics.impactNoteCount}`, ), evidence( "risk_pass", - failedCommands.length === 0 - ? "no failed model-phase first-tree commands observed" - : `failed model-phase first-tree commands=${failedCommands - .map((result) => `${formatCommand(result.argv)} => ${result.exitCode}`) - .join("; ")}`, + failedCommands.length === 0 && metrics.impactNoteMetadataFree + ? "no failed model-phase first-tree commands or visible receipt metadata observed" + : `failed model-phase first-tree commands=${ + failedCommands.map((result) => `${formatCommand(result.argv)} => ${result.exitCode}`).join("; ") || "none" + }; visible receipt metadata absent=${metrics.impactNoteMetadataFree}`, ), ], passed, - riskFlags: failedCommands.map((result) => - riskFlag("failed_first_tree_command", `first-tree ${formatCommand(result.argv)} exited ${result.exitCode}`), - ), + riskFlags: [ + ...failedCommands.map((result) => + riskFlag("failed_first_tree_command", `first-tree ${formatCommand(result.argv)} exited ${result.exitCode}`), + ), + ...(metrics.impactNoteMetadataFree + ? [] + : [riskFlag("visible_receipt_metadata", "Final visible output included receipt metadata or JSON fields.")]), + ], scores: { outcome_pass: outcomePass, process_pass: processPass, @@ -220,15 +250,29 @@ export function writeCaseSummaries(summary: CaseRunSummary): void { - skillHit: ${markdownBool(summary.metrics.skillHit)} - skillFileReadObserved: ${markdownBool(summary.metrics.skillFileReadObserved)} - expectedFactsObserved: ${markdownBool(summary.metrics.expectedFactsObserved)} +- impactNoteBehaviorOk: ${markdownBool(summary.metrics.impactNoteBehaviorOk)} +- impactNoteAtFinalEnd: ${markdownBool(summary.metrics.impactNoteAtFinalEnd)} +- impactNoteCount: ${summary.metrics.impactNoteCount} +- impactNoteEffect: ${summary.metrics.impactNoteEffect ?? "n/a"} +- impactNoteLanguage: ${summary.metrics.impactNoteLanguage ?? "n/a"} +- impactNoteSourceCount: ${summary.metrics.impactNoteSourceCount} +- impactNoteSourceLabels: ${summary.metrics.impactNoteSourceLabels.join(" | ") || "none"} +- impactNoteSummaryObjectiveOk: ${markdownBool(summary.metrics.impactNoteSummaryObjectiveOk)} +- impactNoteMetadataFree: ${markdownBool(summary.metrics.impactNoteMetadataFree)} +- impactNoteSourceAuthorityOk: ${markdownBool(summary.metrics.impactNoteSourceAuthorityOk)} +- impactNoteVisibleUrlsCredentialFree: ${markdownBool(summary.metrics.impactNoteVisibleUrlsCredentialFree)} - helpSucceeded: ${markdownBool(summary.metrics.helpSucceeded)} - selectionSucceeded: ${markdownBool(summary.metrics.selectionSucceeded)} -- readHelpSucceeded: ${markdownBool(summary.metrics.readHelpSucceeded)} +- readRouteCalls: ${summary.metrics.readRouteCalls} +- readRouteSucceeded: ${markdownBool(summary.metrics.readRouteSucceeded)} - readActivationCalls: ${summary.metrics.readActivationCalls} +- legacyReadActivationCalls: ${summary.metrics.legacyReadActivationCalls} - readActivationSucceeded: ${markdownBool(summary.metrics.readActivationSucceeded)} - byoReadSequenceOk: ${markdownBool(summary.metrics.byoReadSequenceOk)} - byoSelectorsNoPull: ${markdownBool(summary.metrics.byoSelectorsNoPull)} - byoSnapshotDetached: ${markdownBool(summary.metrics.byoSnapshotDetached)} - byoSnapshotExactHeadConsistent: ${markdownBool(summary.metrics.byoSnapshotExactHeadConsistent)} +- managedFinalTransportOk: ${markdownBool(summary.metrics.managedFinalTransportOk)} - modelFirstTreeCommandsOk: ${markdownBool(summary.metrics.modelFirstTreeCommandsOk)} - firstTreeCalls: ${summary.metrics.firstTreeCalls} - runnerExitCode: ${summary.metrics.runnerExitCode === null ? "n/a" : summary.metrics.runnerExitCode} @@ -279,6 +323,7 @@ export function formatSummaryTable(batch: BatchSummary): string { String(summary.metrics.firstTreeCalls), String(summary.metrics.skillFileReadObserved), String(summary.metrics.expectedFactsObserved), + String(summary.metrics.impactNoteBehaviorOk), String(summary.metrics.helpSucceeded), String(summary.metrics.selectionSucceeded), String(summary.metrics.modelFirstTreeCommandsOk), @@ -291,6 +336,7 @@ export function formatSummaryTable(batch: BatchSummary): string { "first_tree_calls", "skill_file_read", "expected_facts_observed", + "impact_note_behavior_ok", "helpSucceeded", "selectionSucceeded", "modelFirstTreeCommandsOk", diff --git a/packages/skill-evals/src/suites/first-tree-read/types.ts b/packages/skill-evals/src/suites/first-tree-read/types.ts index d55dcdef0..7d5afbcaf 100644 --- a/packages/skill-evals/src/suites/first-tree-read/types.ts +++ b/packages/skill-evals/src/suites/first-tree-read/types.ts @@ -5,6 +5,27 @@ import type { CommandResult } from "../../core/types.js"; export type WorkspaceKind = "blank" | "byo-context-tree" | "context-tree"; export type BriefingMode = "minimal" | "runtime-generated"; export type ReadMode = "byo" | "managed"; +export type ManagedTransport = "ask" | "send"; + +export type ImpactNoteEffect = "conflicted" | "confirmed" | "constrained" | "redirected"; +export type ImpactNoteLanguage = "en" | "zh"; + +export type ImpactNoteExpectation = + | { mode: "absent" } + | { + effect: ImpactNoteEffect; + language: ImpactNoteLanguage; + mode: "present"; + requiredSourceLabels?: readonly string[]; + sourceAuthority: { + allowedNodePaths: readonly string[]; + exactCommit?: string; + repository: string; + }; + sourceCount: { max: number; min: number }; + summaryConcepts?: readonly (readonly string[])[]; + summaryForbidden?: readonly string[]; + }; export type FirstTreeReadEvalCase = { briefingMode?: BriefingMode; @@ -12,6 +33,8 @@ export type FirstTreeReadEvalCase = { expectedFacts: readonly string[]; expectedTrigger: boolean; id: string; + impactNote: ImpactNoteExpectation; + managedTransport: ManagedTransport | null; prompt: string; promptAlternates: readonly string[]; readMode: ReadMode; @@ -51,14 +74,33 @@ export type EvalMetrics = { helpCalls: number; helpExitCodes: readonly number[]; helpSucceeded: boolean; + impactNoteBehaviorOk: boolean; + impactNoteBlankLineBefore: boolean; + impactNoteCount: number; + impactNoteEffect: string | null; + impactNoteAtFinalEnd: boolean; + impactNoteExactLinksOk: boolean; + impactNoteLanguage: ImpactNoteLanguage | null; + impactNoteLogicalLinesOk: boolean; + impactNoteMetadataFree: boolean; + impactNoteSourceAuthorityOk: boolean; + impactNoteSourceCount: number; + impactNoteSourceLabels: readonly string[]; + impactNoteSummaryConceptsOk: boolean; + impactNoteSummaryForbiddenOk: boolean; + impactNoteSummaryObjectiveOk: boolean; + impactNoteVisibleUrlsCredentialFree: boolean; byoReadSequenceOk: boolean; byoSelectorsNoPull: boolean; byoSnapshotDetached: boolean; byoSnapshotExactHeadConsistent: boolean; + managedFinalTransportOk: boolean; + legacyReadActivationCalls: number; modelFirstTreeCommandsOk: boolean; readActivationCalls: number; readActivationSucceeded: boolean; - readHelpSucceeded: boolean; + readRouteCalls: number; + readRouteSucceeded: boolean; runnerExitCode: number | null; selectionSucceeded: boolean; skillFileReadObserved: boolean; diff --git a/packages/web/src/pages/request-dock-preview.tsx b/packages/web/src/pages/request-dock-preview.tsx index 7efe31d88..29ac29148 100644 --- a/packages/web/src/pages/request-dock-preview.tsx +++ b/packages/web/src/pages/request-dock-preview.tsx @@ -72,9 +72,9 @@ const COMMIT_CARD: GithubEventCard = { }; /** - * Agent-reported Context Tree receipts, one per observable effect. Same shape - * the `first-tree-read` skill attaches to a real final send, so this preview - * exercises the production component rather than a look-alike. + * Legacy agent-reported Context Tree receipts, one per observable effect. The + * reader skill now writes a portable message-body note, but stored history and + * older agents still exercise this production compatibility component. */ const RECEIPTS: ContextDecision[] = [ { diff --git a/skills/first-tree-read/SKILL.md b/skills/first-tree-read/SKILL.md index a505349db..7eee4e0bb 100644 --- a/skills/first-tree-read/SKILL.md +++ b/skills/first-tree-read/SKILL.md @@ -1,6 +1,6 @@ --- name: first-tree-read -version: 0.5.0 +version: 0.6.0 description: Read the applicable Context Tree before acting. In BYO sessions, route only among locally authorized Teams by reading each exact root SCOPE.md before selecting one task snapshot; in managed workspaces, use the bound Tree. Do not use for a Context Tree PR/MR review or an explicit broad audit of stored tree content. --- @@ -202,9 +202,10 @@ If tree content conflicts with the user's instruction, follow the tree constraint and surface the conflict. If the tree says nothing relevant, say so briefly and proceed from repo evidence. -### 7. Record material decision influence +### 7. Show material decision influence -Attach a small `contextDecision` receipt only when all of these conditions hold: +Append one compact, visible Context Tree impact note only when all of these +conditions hold: 1. The agent read a normal-content passage containing a current decision, constraint, rationale, or cross-domain relationship. Opening a file is not @@ -215,62 +216,106 @@ Attach a small `contextDecision` receipt only when all of these conditions hold: 4. The final visible message shows how the passage confirmed, constrained, redirected, or conflicted with that choice. -Do not attach a receipt for root or domain files used only as navigation, +Do not append a note for root or domain files used only as navigation, `AGENTS.md`, skill or workflow instructions, pure ownership routing, archive/proposal/supporting material alone, a Tree mention without decision influence, or a task for which the Tree had no relevant decision-bearing -content. Do not emit `effect: none`. - -When the task ends with a visible First Tree `chat send` that contains the -affected choice, add one receipt under the top-level `contextDecision` metadata -key on that same command. If Tree context exposes an unresolved conflict and -the task correctly ends with a blocking `chat ask`, attach the receipt to that -same ask instead. `chat send` and `chat ask` merge recipient mentions, -attachments, and body-origin metadata; supply only the new -`contextDecision` key. For example, pass the JSON below through -`--metadata ''` on the same command that sends the final body. Do not send -a separate receipt message, put the receipt only in prose, or reconstruct other -metadata: - -```json -{ - "contextDecision": { - "version": 1, - "effect": "constrained", - "summary": "The existing organization-isolation constraint ruled out a global shared index.", - "evidence": [ - { - "repoUrl": "https://github.com/example/context-tree", - "commit": "0123456789abcdef0123456789abcdef01234567", - "nodePath": "system/cloud/team/tenancy-and-identity.md", - "heading": "Organization isolation" - } - ] - } -} +content. Do not emit `effect: none`, `contextDecision` metadata, receipt JSON, +or a separate receipt message. + +Use the same visible note for every consumer: + +- In managed First Tree Chat, append it to the body of the same final + `chat send` that carries the affected choice. If the Tree exposes an + unresolved conflict and the task correctly ends with a blocking `chat ask`, + append it to that question body instead. Do not pass `contextDecision` + metadata. +- In BYO sessions, append it to the authoring coding agent's native final + response. + +Never add the note to progress messages, status updates, or a second message. +Keep the outcome first and place the note at the very end of the authored final +response or blocking question. + +Choose exactly one effect in this precedence order, then show its human label: + +1. `conflicted` → `Conflict surfaced` — exposed a conflict that still requires + resolution or escalation; +2. `redirected` → `Approach changed` — changed the intended approach; +3. `constrained` → `Options narrowed` — ruled out an option or narrowed the + acceptable solution or implementation boundary; +4. `confirmed` → `Direction supported` — removed material uncertainty and + justified keeping the choice without changing its boundary. + +Match the note's language to the surrounding final response. Localize every +visible scaffolding term, not only the effect label. Use these fixed labels for +English and Chinese so different agents produce one recognizable format: + +| Category | English | Chinese | +| --- | --- | --- | +| `conflicted` | `Conflict surfaced` | `发现约束冲突` | +| `redirected` | `Approach changed` | `改变方案路径` | +| `constrained` | `Options narrowed` | `收窄可选范围` | +| `confirmed` | `Direction supported` | `支持当前方向` | + +Use `Context Tree impact` and `Source` / `Sources` in English. Use +`Context Tree 影响` and `来源` in Chinese. For other languages, translate the +complete scaffolding and preserve each category's meaning. Never expose the +enum key. + +Leave one blank line between the preceding answer and the note. Write the note +as one Markdown blockquote with exactly three **logical Markdown lines** and +information levels: the effect, one objective sentence naming the concrete +impact, and the inspectable source. Natural wrapping at narrow display widths +is expected; never truncate or weaken the impact or source merely to keep three +physical display lines. End the first two logical lines with a backslash so +Markdown renders a portable hard line break without trailing whitespace; do +not use HTML. For example: + +```markdown +> **Context Tree impact · Options narrowed**\ +> The organization-isolation rule ruled out a global shared index.\ +> **Source** · [Organization isolation](https://github.com/example/context-tree/blob/0123456789abcdef0123456789abcdef01234567/system/cloud/team/tenancy-and-identity.md) ``` -Use exactly one effect. Choose the first matching category in this precedence -order so periodic reports remain comparable: - -1. `conflicted` — exposed a conflict that still requires resolution or - escalation; -2. `redirected` — changed the intended approach; -3. `constrained` — ruled out an option or narrowed the acceptable solution or - implementation boundary; -4. `confirmed` — removed material uncertainty and justified keeping the choice - without changing its boundary. - -Keep `summary` to one concrete sentence. Cite at most three Tree-root-relative -normal node paths that jointly influenced the same choice. `heading` is -optional; omit it when the relevant heading cannot be named reliably. +Keep the middle sentence concrete and task-specific. Name the Tree decision or +constraint and its specific impact on the choice. For `redirected`, +`constrained`, or `confirmed`, say which option it changed, ruled out, narrowed, +or supported. For `conflicted`, name the two incompatible constraints and the +unresolved tradeoff; do not imply that the plan changed or the conflict was +resolved. Use objective language such as "The organization-isolation rule +ruled out..." rather than first-person or generic language such as "I used +Context Tree...". Keep it to one sentence and roughly 160 English characters or +80 CJK characters. + +For an unresolved conflict in a Chinese response, the complete note looks like: + +```markdown +> **Context Tree 影响 · 发现约束冲突**\ +> 固定发布日期与发布前必须完成安全审计的规则无法同时满足,取舍仍待决定。\ +> **来源** · [发布安全门槛](https://github.com/example/context-tree/blob/0123456789abcdef0123456789abcdef01234567/operations/release/safety-gates.md) +``` -Every evidence row must identify the repository and exact commit that supplied -the passage. Store `repoUrl` as the credential-free binding repository exactly -as the Server activation receipt or managed workspace briefing declares it; -never substitute a local transport URL. Report consumers must compare this -field through First Tree's canonical repository identity rather than raw string -equality. Never persist a credential-bearing remote URL. +Show one to three sources on the final line. In English, use bold `Source` for +one and bold `Sources` for more than one. In Chinese, use bold `来源` for either +count. Follow the label with ` · ` and separate multiple Markdown links with +the same delimiter. +Build each readable label from the node's frontmatter title plus the relevant +heading when that adds meaning, for example `Rollout Policy · Expansion gates`. +For a root `NODE.md`, use the root title or the relevant heading — never display +`Node`. When two cited labels would be identical, prefix the nearest meaningful +parent title, for example `Release · Rollout Policy` and +`Billing · Rollout Policy`. + +When the repository forge is unambiguous, link the readable label to the exact +commit and Tree-root-relative node path. Never link to a mutable branch. If an +exact source link cannot be constructed safely, omit that source; never invent +a link or expose a raw repository URL, node path, or commit in the visible note. +Cite at most three normal node paths that jointly influenced the same choice. +Use the credential-free binding repository exactly as the activation receipt or +managed workspace briefing declares it; never substitute a local transport URL. +Never place a credential-bearing remote URL anywhere in the visible response. +Source links must not contain a query or fragment. For a BYO task, use the activation receipt's binding repository and commit. Its detached snapshot is already exact and remote-backed. For a managed workspace, @@ -297,14 +342,15 @@ stable commit before attributing influence. If the briefing has no unambiguous binding branch, the latest hierarchy refresh cannot be shown to have refreshed the exact binding-branch remote-tracking ref, that ref or its owning fetch remote is missing or ambiguous, or the canonical repository identities do not match, -do not attribute the briefing's `repoUrl`. The checkout's current branch or -upstream is never a fallback authority. If repository, branch, commit, remote -reachability, or path identity cannot be established safely, omit the evidence -row and do not attach the receipt when no valid evidence remains. +do not use the briefing's repository as source authority. The checkout's +current branch or upstream is never a fallback authority. If repository, +branch, commit, remote reachability, or path identity cannot be established +safely, omit the source and do not append the note when no valid source remains. -The receipt is the agent's durable, reviewable attribution. It is not -server-verified proof of causality, and the final prose must not claim that it -is. +The note is the authoring agent's explanation inside its own response, not a +First Tree verification of causality. Do not add a long attribution disclaimer, +a verified/success claim, system-style framing, emoji, badge, divider, or +collapsible detail. ## Output Expectations @@ -315,9 +361,9 @@ Keep the user-facing result concise: relationships that affect the task - for BYO Read, report the selected Team, binding, and exact commit when it helps the user verify which task snapshot governed the answer -- when the strict decision-influence test passes, attach the receipt to the - same final First Tree `chat send`, or to the same blocking `chat ask` for an - unresolved conflict, instead of adding receipt prose or another message +- when the strict decision-influence test passes, append the same compact + visible note to the authored final response for managed and BYO consumers; + for an unresolved conflict, append it to the blocking question body - avoid restating every node; carry forward only what changes how you act Never modify tree files with this skill. diff --git a/skills/first-tree-read/VERSION b/skills/first-tree-read/VERSION index 8f0916f76..a918a2aa1 100644 --- a/skills/first-tree-read/VERSION +++ b/skills/first-tree-read/VERSION @@ -1 +1 @@ -0.5.0 +0.6.0 diff --git a/skills/first-tree-read/agents/openai.yaml b/skills/first-tree-read/agents/openai.yaml index 2c8d9f92c..6ee8979fa 100644 --- a/skills/first-tree-read/agents/openai.yaml +++ b/skills/first-tree-read/agents/openai.yaml @@ -1,4 +1,4 @@ interface: display_name: "First Tree Read" - short_description: "Read repo context and record decision influence" - default_prompt: "Use $first-tree-read to read relevant Context Tree files before acting, then attach a contextDecision receipt only when that context materially influences the final choice." + short_description: "Read repo context and show decision influence" + default_prompt: "Use $first-tree-read to read relevant Context Tree files before acting, then show one compact Context Tree impact note only when that context materially influences the final choice."