diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59451db2..f79a6c6f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,6 +35,8 @@ jobs: cargo test --workspace --doc --locked - name: Tests run: cargo test --workspace --all-targets --locked + - name: Understanding study runner tests + run: python3 scripts/test-understanding-study.py msrv: name: MSRV 1.88 diff --git a/.gitignore b/.gitignore index 5daa1e3b..df5af011 100644 --- a/.gitignore +++ b/.gitignore @@ -24,6 +24,8 @@ # OS / editor cruft. .DS_Store Thumbs.db +__pycache__/ +*.py[cod] *.swp .idea/ .vscode/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 61dcecb0..526ed748 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -17,6 +17,14 @@ project uses version-gated milestones (see ROADMAP.md), not dates. and erase prototype rather than complete continuity. ### Added +- A dependency-free frozen runner and 20-probe bank for 0.4 Understanding + Alpha. It derives the complete balanced 24-pair allocation from the declared + seed, emits oracle-free public study packets, redacts forbidden host data, + enforces equal call budgets and identical Reveal payloads, handles ordered + reserves and one schema repair, computes the predeclared stratified paired + percentile bootstrap, and refuses to report incomplete cohorts. Fifteen + focused regressions run in CI and the release gate. No qualifying response or + study result is claimed. - Formula Jam discovery is shared across App, CLI, and MCP: curated recipe bank in core; MCP `plot_expression` supports manual expr, `recipe`, `seed`, `auto_step`, and `list_recipes` with structured discovery results; CLI `plot` diff --git a/README.md b/README.md index 03fb979d..9235fe63 100644 --- a/README.md +++ b/README.md @@ -172,11 +172,14 @@ and green CI. Human stranger sessions, accessibility panels, musician long-listening, and broad hardware certification are real gates for **0.8 / 1.0**, not stop-the-world work before 0.2. -**Current milestone:** 0.3 Tactile Alpha meets its agent-and-machine exit on -the five flagships, including the consented Watch Agent path and the declared -reference performance budget. The next incomplete milestone is 0.4 -Understanding Alpha. Soft-thin densify, bulk rooms, and Phase B glow stay background unless -measurement binds. Ordered criteria live in +**Current milestone:** 0.4 Understanding Alpha. The 0.3 Tactile Alpha +agent-and-machine exit is met on the five flagships, including the consented +Watch Agent path and the declared reference performance budget. The frozen +0.4 headless runner and held-out probe bank now exist and are CI-gated; no +qualifying cohort result is claimed. Returning-journal sovereignty, independent +pre-collection review, and the complete matched cohort remain open. Soft-thin +densify, bulk rooms, and Phase B glow stay background unless measurement binds. +Ordered criteria live in [`docs/ROADMAP.md`](docs/ROADMAP.md) (**Critical path right now**). The predeclared contract and evidence boundary for the next study live in [`docs/UNDERSTANDING_STUDY.md`](docs/UNDERSTANDING_STUDY.md); no qualifying diff --git a/VERIFY.md b/VERIFY.md index 92868058..8e54844b 100644 --- a/VERIFY.md +++ b/VERIFY.md @@ -17,6 +17,7 @@ contributors and the curious. fresh shell does not see `cargo`, add that to `PATH`. - Optional, for the local coverage gate: `cargo install cargo-llvm-cov`. - Optional, for the local supply-chain gate: `cargo install cargo-deny`. +- **Python 3.11 or newer**, for the frozen 0.4 study-runner regressions. - The Linux build needs the ALSA, xkbcommon, and libudev headers (the packages CI installs): `sudo apt-get install -y libasound2-dev libxkbcommon-dev libudev-dev`. @@ -27,7 +28,8 @@ Run the full gate and regenerate every artifact: - Windows: `scripts\verify.ps1` - macOS / Linux: `bash scripts/verify.sh` -It runs format, clippy and rustdoc with warnings denied, tests, locked build, +It runs format, clippy and rustdoc with warnings denied, Rust and 0.4 study +runner tests, locked build, coverage (if `cargo-llvm-cov` is present), supply-chain policy (if `cargo-deny` is present), the house-style guard, and the native installer safety self-test, then writes images and audio into `renders/`. @@ -42,6 +44,8 @@ RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --locked # macOS / L RUSTDOCFLAGS="-D warnings" cargo test --workspace --doc --locked # macOS / Linux cmd /d /c "set RUSTDOCFLAGS=-D warnings&& cargo doc --workspace --no-deps --locked && cargo test --workspace --doc --locked" # Windows cargo test --workspace --all-targets --locked +python scripts/test-understanding-study.py # Windows +python3 scripts/test-understanding-study.py # macOS / Linux cargo build --workspace --locked cargo +1.88.0 check --workspace --all-targets --locked cargo llvm-cov --workspace --fail-under-lines 80 --ignore-filename-regex '(crates[\\/](gpu|audio)[\\/]|faces[\\/]app[\\/]src[\\/]main\.rs)' @@ -54,9 +58,10 @@ powershell -NoProfile -ExecutionPolicy Bypass -File scripts/install.ps1 -SelfTes ``` -Expected right now: **format and clippy clean, 3,180 all-target test cases pass, -one screenshot diagnostic is ignored, 95.15% region coverage, and 95.33% line -coverage**. The `gpu` and `audio` crates plus the app event +Expected right now: **format and clippy clean, 3,180 all-target Rust test cases +and 15 frozen study-runner regressions pass, one screenshot diagnostic is +ignored, 95.15% region coverage, and 95.33% line coverage**. The `gpu` and +`audio` crates plus the app event loop are excluded from the coverage gate and have dev-machine integration evidence, see `docs/QUALITY.md`. Controller routing is pure-tested. Sessions with representative physical controller models remain open. diff --git a/docs/ENGINEERING.md b/docs/ENGINEERING.md index 25632c41..58a5cba6 100644 --- a/docs/ENGINEERING.md +++ b/docs/ENGINEERING.md @@ -188,8 +188,9 @@ git config core.hooksPath scripts/hooks It runs the house-style guard on every commit (instant, and it applies to docs as much as code), and the cargo gate (fmt, Clippy and rustdoc with warnings denied, plus doctests and the full test suite) only when the commit touches -Rust, `Cargo.*`, or a shader, so a docs-only commit stays fast. Coverage, the -locked build, and artifact regeneration stay in +Rust, `Cargo.*`, or a shader, so a docs-only commit stays fast. Changes to the +frozen 0.4 study runner, probe bank, or its test file run the focused Python +regressions. Coverage, the locked build, and artifact regeneration stay in `scripts/verify.sh` (the release gate); they are too slow for every commit. Emergency bypass is `git commit --no-verify`, after which you must run `scripts/verify.sh` before pushing. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 1e0f0bf6..eeea5923 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -44,15 +44,16 @@ stop-the-world work. | Priority | What | Why | | --- | --- | --- | -| **1. Build the frozen 0.4 study runner** | Deterministic allocation, held-out transfer probes, active control, scoring, redaction, and incomplete-cohort refusal from `UNDERSTANDING_STUDY.md` | The method must exist before qualifying data is collected, or the result can drift toward the desired outcome | +| **1. Audit and freeze the 0.4 collection** | Independently review the implemented runner and 20-probe bank, then track its generated 24-pair allocation before the first qualifying response | The runner now blocks method drift mechanically; pre-collection review must catch a flawed probe or oracle before results can influence a correction | | **2. Complete returning-journal sovereignty** | Add provenance-preserving correction, bounded structured export, two-process clean-clone acceptance, and managed-path residue evidence | Current read, record, and erase tools are only the prototype slice and do not yet satisfy the 0.4 return-session exit | | **3. Run and publish the 0.4 cohort** | Execute 20 matched pairs, publish every planned outcome and deviation, and obtain independent methodology and math review | A tracked runner without a complete, reviewable result is preparation, not milestone evidence | | **Standing gate** | Keep 0.2 and 0.3 proof plus public CI green | Regressions reopen completed milestones and invalidate new evidence | | **Deferred past 0.2** | Human stranger hallway, a11y sessions, musician long-listening, soft-thin densify, bulk rooms, Phase B glow | Humans before 1.0; densify/glow only if a sensory ceiling binds | The 0.3 agent-and-machine exit is met. The next incomplete milestone is 0.4 -understanding and retention. Its protocol and some mechanisms exist, but the -qualifying study result and returning-player exit evidence remain open. +understanding and retention. Its protocol, deterministic study runner, held-out +probe bank, and some mechanisms exist, but pre-collection independent review, +the qualifying study result, and returning-player exit evidence remain open. Detail below and in the version sections. - **Done:** the headless core (`Room` trait with `reveal()`, deterministic ASCII `Canvas`, seeded RNG, registry, `verb`, `render_poked`, and variation); the CLI face (`numinous`), the MCP face (`numinous-mcp`), and the windowed app; **354 catalog rooms** plus hidden content; 6 lever-driven sims; 11+ games; the full engineering harness (edition-2024 workspace, pinned toolchain, `-D warnings`, cargo-deny, house-style guard, an 80% line coverage gate, three-OS CI). Current local evidence: fmt, Clippy, 3,180 passing all-target test cases plus one ignored screenshot diagnostic, locked build, Windows release gate, 95.15% region coverage, and 95.33% line coverage all pass. @@ -777,10 +778,19 @@ read-only App viewer, with no private host or protocol data in the stream. sample, frozen outcomes, exclusions, pass rule, privacy boundary, and honest distinction between immediate transfer, within-context retention, and durable human learning. No qualifying result is claimed. -- **Incomplete:** implement the deterministic runner and held-out probe bank, - then run and publish the complete cohort with all outcomes and deviations. - Private working notes and scripted conclusions do not satisfy this evidence - gate. +- **Done (2026-07-27):** Implement the dependency-free headless study runner + and frozen 20-probe bank. The runner derives a balanced 24-pair allocation + from the declared seed, emits one oracle-free probe at a time, enforces equal + public tool-call budgets and identical Reveal payloads, permits one + schema-only repair, redacts forbidden host data, scores with independent math + oracles, runs the declared stratified paired percentile bootstrap, accounts + for ordered reserves, and refuses to report an incomplete cohort. Fifteen + focused regressions run in CI and the release gate. No qualifying response + has been collected. +- **Incomplete:** obtain independent pre-collection review, commit the exact + generated allocation before accepting a response, then run and publish the + complete cohort with all outcomes and deviations. Private working notes and + scripted conclusions do not satisfy this evidence gate. - **Done (cycle 152):** Add source provenance and an independent math-review checklist to every flagship Reveal. - **Done (architectural invariant):** Keep progression subordinate to autonomy: no streak loss, required grind, or diff --git a/docs/UNDERSTANDING_STUDY.md b/docs/UNDERSTANDING_STUDY.md index af23bf61..65d0fee8 100644 --- a/docs/UNDERSTANDING_STUDY.md +++ b/docs/UNDERSTANDING_STUDY.md @@ -1,8 +1,49 @@ # Understanding Alpha Study Protocol -Status: predeclared design, reviewed 2026-07-26. No qualifying cohort has -started and no result is claimed. Earlier private scripted notes are exploratory -only, are excluded from this study, and cannot satisfy the 0.4 evidence gate. +Status: predeclared design and implemented frozen runner, reviewed 2026-07-27. +No qualifying cohort has started and no result is claimed. Earlier private +scripted notes are exploratory only, are excluded from this study, and cannot +satisfy the 0.4 evidence gate. + +## Implemented runner boundary + +`scripts/understanding-study.py` and `scripts/understanding-probes.json` now +implement the first dependency in this protocol. The dependency-free runner +derives all 24 pairs from the declared allocation seed, balances condition +order and cyclic room order within each model family, and binds each manifest +to the canonical probe-bank hash. The bank contains 10 immediate and 10 late +probes, two per flagship per phase, plus one five-item distractor sequence. + +Public commands emit one condition, probe, repair schema, or distractor at a +time without the oracle. Analysis requires an exact runner-generated manifest, +explicit consent metadata, 20 public tool events per session, identical Reveal +payloads across each pair, ordered probe delivery, and all 20 qualifying pairs. +The sanitizer removes forbidden fields, absolute host paths, and named host +identifiers before tracked evidence is analyzed. Withdrawn records retain no +response content. The scorer uses independent mathematical oracles and the +declared 100,000-resample stratified paired percentile interval, implemented +with a stable SHA-256 counter stream and frozen linear percentile interpolation. + +The current canonical hashes are: + +- probe bank: `4ac647fdfc4559b26ab417ece0eb01a021fc53d0decda35d8d5c798fd000cbc0` +- generated allocation: `8a241287e91589d18e877f75d07d9fc03cb98dbd518e63b7cd26aa8922fd1a01` + +Before any qualifying response, independently review the probes and oracles, +generate the manifest into the tracked evidence directory, validate it with the +same runner revision, and commit it. A dry run stays gitignored: + +```text +python scripts/understanding-study.py allocate --output .agent/understanding-allocation.json +python scripts/understanding-study.py validate --manifest .agent/understanding-allocation.json +python scripts/test-understanding-study.py +``` + +The 15 focused regressions prove deterministic balance and hashes, oracle +coverage, oracle-free public packets, redaction, ordered reserve replacement, +single schema repair, equal call ordering, identical Reveal delivery, and hard +incomplete-cohort refusal. They do not constitute participant evidence or +independent methodology or math review. ## Decision and scope @@ -240,10 +281,10 @@ successful journal read does not satisfy this evidence contract. ## Implementation order and acceptance -1. Build the deterministic allocation, probe, scoring, redaction, and report - runner. Prove allocation balance, answer-key independence, no condition - leakage, deterministic scoring, complete failure accounting, and refusal to - report an incomplete cohort. +1. The deterministic allocation, probe, scoring, redaction, and report runner + is implemented and CI-gated. Independently review its probe content and + oracles, then commit its exact generated allocation before the first + qualifying response. 2. Complete the journal correction, structured export, and residue receipt, then prove the two-process return path with an isolated journal location. 3. Run the frozen cohort, publish all bounded evidence, obtain independent math @@ -260,15 +301,19 @@ does not belong in Numinous. ## Current sources - [Center for Open Science preregistration guidance](https://www.cos.io/initiatives/prereg), - reviewed 2026-07-26. It separates planned confirmatory work from exploratory + reviewed 2026-07-27. It separates planned confirmatory work from exploratory work and calls for methods detailed enough to replicate. - [Retrieval Practice Versus Elaborative Encoding, 2025 systematic and meta-analytic review](https://doi.org/10.1007/s10648-025-10076-6), reviewed 2026-07-26. It supports an active elaborative control, corrective feedback, and separate retention and transfer outcomes. - [NIST Privacy Framework](https://www.nist.gov/privacy-framework), reviewed - 2026-07-26. Version 1.0 remains final; version 1.1 is an initial public draft + 2026-07-27. Version 1.0 remains final; version 1.1 is an initial public draft as of this review. - [MCP 2025-11-25 security and trust principles](https://modelcontextprotocol.io/specification/2025-11-25), - reviewed 2026-07-26. The current specification requires explicit consent, + reviewed 2026-07-27. The current specification requires explicit consent, user control, and appropriate data protections. +- [SciPy bootstrap reference](https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.bootstrap.html), + reviewed 2026-07-27. It documents paired resampling, repeatable random input, + and two-sided percentile intervals. The runner implements the predeclared + stratified variant without adding SciPy as a runtime dependency. diff --git a/scripts/check.ps1 b/scripts/check.ps1 index e720befb..939c232c 100644 --- a/scripts/check.ps1 +++ b/scripts/check.ps1 @@ -28,5 +28,6 @@ Invoke-Step "docs" { } } Invoke-Step "test" { cargo test --workspace --all-targets --locked } +Invoke-Step "understanding study runner" { python scripts/test-understanding-study.py } Invoke-Step "house style" { powershell -NoProfile -ExecutionPolicy Bypass -File scripts/check-style.ps1 } Write-Host "All checks passed." diff --git a/scripts/check.sh b/scripts/check.sh index 89bcbf15..fd289e33 100644 --- a/scripts/check.sh +++ b/scripts/check.sh @@ -12,6 +12,8 @@ RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --locked RUSTDOCFLAGS="-D warnings" cargo test --workspace --doc --locked echo "== test ==" cargo test --workspace --all-targets --locked +echo "== understanding study runner ==" +python3 scripts/test-understanding-study.py echo "== house style ==" bash scripts/check-style.sh echo "All checks passed." diff --git a/scripts/hooks/pre-commit b/scripts/hooks/pre-commit index 48f0c6e3..4334a2ab 100755 --- a/scripts/hooks/pre-commit +++ b/scripts/hooks/pre-commit @@ -13,6 +13,8 @@ # full test suite) runs only when this commit actually touches Rust or the # build manifest, so a docs-only commit stays fast while a code commit # cannot land red. +# - The frozen 0.4 runner regressions run only when their runner, bank, or +# test file changes. # # Deliberately NOT here: coverage, the locked build, and artifact regeneration. # Those live in scripts/verify.sh (the release/CI gate); they are too slow to @@ -43,4 +45,18 @@ if git -c core.quotePath=false diff --cached --name-only --diff-filter=ACMRD -z cargo test --workspace --all-targets --locked fi +if git -c core.quotePath=false diff --cached --name-only --diff-filter=ACMRD -z \ + | grep -zqE '^scripts/(understanding-study\.py|understanding-probes\.json|test-understanding-study\.py)$'; then + if command -v python3 >/dev/null 2>&1; then + study_python=python3 + elif command -v python >/dev/null 2>&1; then + study_python=python + else + echo "pre-commit: Python 3.11 or newer is required for the 0.4 runner gate" >&2 + exit 1 + fi + echo "== understanding study runner ==" + "$study_python" scripts/test-understanding-study.py +fi + echo "pre-commit: all checks passed." diff --git a/scripts/test-understanding-study.py b/scripts/test-understanding-study.py new file mode 100644 index 00000000..979aa237 --- /dev/null +++ b/scripts/test-understanding-study.py @@ -0,0 +1,407 @@ +#!/usr/bin/env python3 +"""Regression tests for the frozen Understanding Alpha study runner.""" + +from __future__ import annotations + +import importlib.util +import json +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parent.parent +RUNNER = ROOT / "scripts" / "understanding-study.py" +sys.dont_write_bytecode = True + + +def load_runner(): + spec = importlib.util.spec_from_file_location("numinous_understanding_study", RUNNER) + if spec is None or spec.loader is None: + raise RuntimeError("could not load understanding-study.py") + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +study = load_runner() + + +def wrong_answer(probe: dict) -> int | float | str: + schema = probe["answerSchema"] + expected = study.oracle_answer(probe["oracle"]) + if schema["type"] == "number": + return float(expected) + 1.0 + return next(choice for choice in schema["enum"] if choice != expected) + + +def session_records( + bank: dict, + pair: dict, + session: dict, + correct: bool, +) -> list[dict]: + session_id = session["sessionId"] + condition = session["condition"] + roles = ( + ["encounter", "generation", "interaction", "reveal"] + if condition == study.CONDITIONS[0] + else ["reveal", "explanation", "interaction", "continuation"] + ) + records = [ + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "session", + "sessionId": session_id, + "consent": True, + "modelFamily": pair["modelFamily"], + "modelIdentifier": pair["modelFamily"], + "provider": "frozen-test-runtime", + "backendRevision": "revision-1", + "reasoningEffort": "high", + "settings": {"sampling": "platform-default"}, + "date": "2026-07-27", + "numinousCommit": "1" * 40, + "mcpProtocolRevision": "2025-11-25", + "operatingSystem": "test-os", + "runnerVersion": study.RUNNER_VERSION, + "condition": condition, + } + ] + for room in pair["roomOrder"]: + for sequence, role in enumerate(roles, start=1): + structured = {"reveal": "same public Reveal"} if role == "reveal" else {} + records.append( + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "tool", + "sessionId": session_id, + "room": room, + "sequence": sequence, + "role": role, + "tool": "study-fixture", + "arguments": {"room": room, "sequence": sequence}, + "structuredResult": structured, + "visibleText": f"public {role}", + } + ) + for phase in ("immediate", "late"): + if phase == "late": + for item in bank["distractorSequence"]["items"]: + records.append( + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "distractor_response", + "sessionId": session_id, + "itemId": item["id"], + "answer": "fixture", + } + ) + for probe in study.probe_sequence(bank, pair["roomOrder"], phase): + answer = study.oracle_answer(probe["oracle"]) if correct else wrong_answer(probe) + records.append( + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "response", + "sessionId": session_id, + "phase": phase, + "probeId": probe["id"], + "attempt": 1, + "answer": answer, + } + ) + records.append( + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "session_complete", + "sessionId": session_id, + } + ) + return records + + +def cohort_records( + bank: dict, + manifest: dict, + selected_orders: dict[str, range] | None = None, +) -> list[dict]: + selected_orders = selected_orders or { + "gpt-5.6-sol": range(1, 11), + "gpt-5.6-terra": range(1, 11), + } + records: list[dict] = [] + for pair in manifest["pairs"]: + if pair["order"] not in selected_orders[pair["modelFamily"]]: + continue + sessions_by_id = {session["sessionId"]: session for session in pair["sessions"]} + for session_id in pair["collectionOrder"]: + session = sessions_by_id[session_id] + records.extend( + session_records( + bank, + pair, + session, + correct=session["condition"] == study.CONDITIONS[0], + ) + ) + for index, record in enumerate(records): + record["_sourceIndex"] = index + return records + + +class ProbeBankTests(unittest.TestCase): + def setUp(self) -> None: + self.bank = study.load_bank() + self.manifest = study.build_allocation(self.bank) + + def test_frozen_hashes_and_balanced_allocation(self) -> None: + self.assertEqual( + study.content_sha256(self.bank), + "4ac647fdfc4559b26ab417ece0eb01a021fc53d0decda35d8d5c798fd000cbc0", + ) + self.assertEqual( + study.content_sha256(self.manifest), + "8a241287e91589d18e877f75d07d9fc03cb98dbd518e63b7cd26aa8922fd1a01", + ) + self.assertEqual(len(self.manifest["pairs"]), 24) + for model in study.MODEL_FAMILIES: + pairs = [pair for pair in self.manifest["pairs"] if pair["modelFamily"] == model] + self.assertEqual(sum(pair["allocationRole"] == "primary" for pair in pairs), 10) + self.assertEqual(sum(pair["allocationRole"] == "reserve" for pair in pairs), 2) + first_conditions = [ + next( + session["condition"] + for session in pair["sessions"] + if session["sessionId"] == pair["collectionOrder"][0] + ) + for pair in pairs + ] + self.assertEqual(first_conditions.count(study.CONDITIONS[0]), 6) + self.assertEqual(first_conditions.count(study.CONDITIONS[1]), 6) + room_counts = { + room: sum(pair["roomOrder"][0] == room for pair in pairs) + for room in study.ROOMS + } + self.assertLessEqual(max(room_counts.values()) - min(room_counts.values()), 1) + + def test_public_packets_never_expose_oracles(self) -> None: + packet = study.public_probe(self.bank["probes"][0], schema_only=False) + repair = study.public_probe(self.bank["probes"][0], schema_only=True) + self.assertNotIn("oracle", json.dumps(packet)) + self.assertNotIn("prompt", repair) + self.assertEqual(set(repair), {"schemaVersion", "probeId", "answerSchema"}) + + def test_manifest_mutation_is_rejected(self) -> None: + changed = json.loads(json.dumps(self.manifest)) + changed["pairs"][0]["roomOrder"].reverse() + with self.assertRaisesRegex(study.StudyError, "differs from the frozen"): + study.validate_manifest(changed, self.bank) + + def test_independent_oracles_cover_every_probe(self) -> None: + for probe in self.bank["probes"]: + expected = study.oracle_answer(probe["oracle"]) + valid, correct = study.score_answer(probe, expected) + self.assertTrue(valid, probe["id"]) + self.assertTrue(correct, probe["id"]) + + +class RedactionTests(unittest.TestCase): + def test_redaction_removes_private_fields_paths_and_host_values(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + raw = root / "raw.jsonl" + clean = root / "clean.jsonl" + raw.write_text( + json.dumps( + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "tool", + "systemPrompt": "private", + "token": "secret", + "visibleText": "C:\\Users\\PrivateName\\journal.txt PrivateName", + } + ) + + "\n", + encoding="utf-8", + ) + result = study.redact_jsonl(raw, clean, ("PrivateName",)) + self.assertEqual(result, "written") + record = json.loads(clean.read_text(encoding="utf-8")) + self.assertNotIn("systemPrompt", record) + self.assertNotIn("token", record) + self.assertIn("", record["visibleText"]) + self.assertIn("", record["visibleText"]) + study.assert_sanitized(record) + + def test_analysis_rejects_unsanitized_absolute_path(self) -> None: + with self.assertRaisesRegex(study.StudyError, "absolute host path"): + study.assert_sanitized({"visibleText": "/home/person/private.txt"}) + + +class CohortAnalysisTests(unittest.TestCase): + def setUp(self) -> None: + self.bank = study.load_bank() + self.manifest = study.build_allocation(self.bank) + + def test_complete_cohort_passes_frozen_analysis(self) -> None: + report = study.analyze_events( + self.manifest, + self.bank, + cohort_records(self.bank, self.manifest), + bootstrap_resamples=2_000, + ) + self.assertTrue(report["cohortComplete"]) + self.assertEqual(len(report["selectedPairs"]), 20) + self.assertEqual(report["primary"]["pairedMeanDifference"], 1.0) + self.assertEqual(report["primary"]["bootstrap"]["pooled95"], [1.0, 1.0]) + self.assertTrue(report["primary"]["predeclaredStatisticalGatePassed"]) + + def test_incomplete_cohort_refuses_to_report(self) -> None: + records = cohort_records(self.bank, self.manifest) + records = [ + record + for record in records + if not str(record.get("sessionId", "")).startswith("sol-p10-") + ] + for index, record in enumerate(records): + record["_sourceIndex"] = index + with self.assertRaisesRegex(study.StudyError, "incomplete cohort at sol-p10"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_ordered_reserve_replaces_withdrawn_pair(self) -> None: + records = cohort_records( + self.bank, + self.manifest, + selected_orders={ + "gpt-5.6-sol": range(2, 12), + "gpt-5.6-terra": range(1, 11), + }, + ) + records.insert( + 0, + { + "schemaVersion": study.EVENT_SCHEMA, + "type": "withdrawal", + "pairId": "sol-p01", + "_sourceIndex": 0, + }, + ) + for index, record in enumerate(records): + record["_sourceIndex"] = index + report = study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=200 + ) + self.assertEqual(report["withdrawals"], 1) + self.assertIn("sol-p11", report["selectedPairs"]) + self.assertNotIn("sol-p01", report["selectedPairs"]) + + def test_generation_condition_reveal_leak_is_rejected(self) -> None: + records = cohort_records(self.bank, self.manifest) + leaked = next( + record + for record in records + if record.get("sessionId") == "sol-p01-g" + and record.get("type") == "tool" + and record.get("sequence") == 1 + ) + leaked["structuredResult"] = {"reveal": "early"} + with self.assertRaisesRegex(study.StudyError, "leaked Reveal"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_pair_must_receive_the_same_reveal(self) -> None: + records = cohort_records(self.bank, self.manifest) + changed = next( + record + for record in records + if record.get("sessionId") == "sol-p01-c" + and record.get("type") == "tool" + and record.get("role") == "reveal" + ) + changed["visibleText"] = "different public Reveal" + with self.assertRaisesRegex(study.StudyError, "identical Reveal payloads"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_room_tool_calls_may_not_interleave(self) -> None: + records = cohort_records(self.bank, self.manifest) + indexes = [ + index + for index, record in enumerate(records) + if record.get("sessionId") == "sol-p01-g" and record.get("type") == "tool" + ] + records[indexes[1]], records[indexes[4]] = records[indexes[4]], records[indexes[1]] + for index, record in enumerate(records): + record["_sourceIndex"] = index + with self.assertRaisesRegex(study.StudyError, "interleaves or reorders"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_condition_collection_order_is_frozen(self) -> None: + records = cohort_records(self.bank, self.manifest) + first_pair = self.manifest["pairs"][0] + first_id, second_id = first_pair["collectionOrder"] + first_block = [record for record in records if record.get("sessionId") == first_id] + second_block = [record for record in records if record.get("sessionId") == second_id] + insertion = min(record["_sourceIndex"] for record in first_block + second_block) + records = [ + record + for record in records + if record.get("sessionId") not in (first_id, second_id) + ] + records[insertion:insertion] = second_block + first_block + for index, record in enumerate(records): + record["_sourceIndex"] = index + with self.assertRaisesRegex(study.StudyError, "condition collection order"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_complete_cohort_uses_one_numinous_commit(self) -> None: + records = cohort_records(self.bank, self.manifest) + for record in records: + if record.get("type") == "session" and str(record.get("sessionId")).startswith( + "sol-p02-" + ): + record["numinousCommit"] = "2" * 40 + with self.assertRaisesRegex(study.StudyError, "one Numinous commit"): + study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=100 + ) + + def test_one_schema_repair_is_scored(self) -> None: + records = cohort_records(self.bank, self.manifest) + target_index = next( + index + for index, record in enumerate(records) + if record.get("sessionId") == "sol-p01-g" + and record.get("type") == "response" + and record.get("phase") == "immediate" + ) + target = records[target_index] + probe = next(probe for probe in self.bank["probes"] if probe["id"] == target["probeId"]) + target["answer"] = {"invalid": True} + retry = dict(target) + retry["attempt"] = 2 + retry["answer"] = study.oracle_answer(probe["oracle"]) + records.insert(target_index + 1, retry) + for index, record in enumerate(records): + record["_sourceIndex"] = index + report = study.analyze_events( + self.manifest, self.bank, records, bootstrap_resamples=200 + ) + diagnostic = next( + item for item in report["sessionDiagnostics"] if item["sessionId"] == "sol-p01-g" + ) + self.assertEqual(diagnostic["schemaRepairs"], 1) + self.assertEqual(diagnostic["invalidAttempts"], 1) + self.assertEqual(diagnostic["immediateScore"], 1.0) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/scripts/understanding-probes.json b/scripts/understanding-probes.json new file mode 100644 index 00000000..aebd9b6e --- /dev/null +++ b/scripts/understanding-probes.json @@ -0,0 +1,176 @@ +{ + "schemaVersion": "numinous-understanding-probes-v1", + "protocolVersion": "0.4-v1", + "distractorSequence": { + "id": "numinous-understanding-distractor-v1", + "items": [ + { "id": "distractor-1", "prompt": "Return the word copper in uppercase." }, + { "id": "distractor-2", "prompt": "Sort cedar, birch, and alder alphabetically, separated by commas." }, + { "id": "distractor-3", "prompt": "Return the reverse of the string LANTERN." }, + { "id": "distractor-4", "prompt": "Count the letters in the word constellation." }, + { "id": "distractor-5", "prompt": "Return the second word in: quiet rivers remember." } + ] + }, + "probes": [ + { + "id": "times-immediate-1", + "phase": "immediate", + "room": "times-tables", + "prompt": "On a 12-point multiplication circle, point 7 is sent through multiplier 4. Which numbered point receives it?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "mod_product", "start": 7, "multiplier": 4, "modulus": 12 } + }, + { + "id": "times-immediate-2", + "phase": "immediate", + "room": "times-tables", + "prompt": "On a 16-point multiplication circle, point 9 is sent through multiplier 5. Which numbered point receives it?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "mod_product", "start": 9, "multiplier": 5, "modulus": 16 } + }, + { + "id": "pendulum-immediate-1", + "phase": "immediate", + "room": "double-pendulum", + "prompt": "Two deterministic double-pendulum simulations begin with exactly identical angles and velocities. With no outside perturbation, what is their separation after 800 identical integration steps?", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "identical_state_gap", "steps": 800 } + }, + { + "id": "pendulum-immediate-2", + "phase": "immediate", + "room": "double-pendulum", + "prompt": "Twin pair A grows from gap 0.002 to 0.16. Twin pair B grows from gap 0.002 to 0.04 over the same interval. Which pair shows the larger relative divergence? Answer A, B, or equal.", + "answerSchema": { "type": "string", "enum": ["A", "B", "equal"] }, + "oracle": { "kind": "relative_growth", "aInitial": 0.002, "aFinal": 0.16, "bInitial": 0.002, "bFinal": 0.04 } + }, + { + "id": "life-immediate-1", + "phase": "immediate", + "room": "game-of-life", + "prompt": "A live Game of Life cell has exactly two live neighbors. Is that center cell alive or dead in the next generation?", + "answerSchema": { "type": "string", "enum": ["alive", "dead"] }, + "oracle": { "kind": "life_center", "currentlyAlive": true, "liveNeighbors": 2 } + }, + { + "id": "life-immediate-2", + "phase": "immediate", + "room": "game-of-life", + "prompt": "A dead Game of Life cell has exactly three live neighbors. Is that center cell alive or dead in the next generation?", + "answerSchema": { "type": "string", "enum": ["alive", "dead"] }, + "oracle": { "kind": "life_center", "currentlyAlive": false, "liveNeighbors": 3 } + }, + { + "id": "galton-immediate-1", + "phase": "immediate", + "room": "galton-board", + "prompt": "A ball makes six left-or-right choices. How many distinct paths contain exactly two right choices?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "binomial_paths", "rows": 6, "rights": 2 } + }, + { + "id": "galton-immediate-2", + "phase": "immediate", + "room": "galton-board", + "prompt": "An unbiased Galton ball makes eight left-or-right choices. What is the expected number of right choices?", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "binomial_mean", "rows": 8, "rightProbability": 0.5 } + }, + { + "id": "formula-immediate-1", + "phase": "immediate", + "room": "formula-jam", + "prompt": "Classify y = sin(5x) by reflection through the origin. Answer odd, even, or neither.", + "answerSchema": { "type": "string", "enum": ["odd", "even", "neither"] }, + "oracle": { "kind": "trig_parity", "function": "sin", "frequency": 5, "verticalOffset": 0.0 } + }, + { + "id": "formula-immediate-2", + "phase": "immediate", + "room": "formula-jam", + "prompt": "For y = sin(6x), what is the fundamental period divided by pi? Give a number.", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "trig_period_over_pi", "frequency": 6 } + }, + { + "id": "times-late-1", + "phase": "late", + "room": "times-tables", + "prompt": "On a 14-point multiplication circle, point 11 is sent through multiplier 3. Which numbered point receives it?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "mod_product", "start": 11, "multiplier": 3, "modulus": 14 } + }, + { + "id": "times-late-2", + "phase": "late", + "room": "times-tables", + "prompt": "On a 15-point multiplication circle, point 7 is sent through multiplier 8. Which numbered point receives it?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "mod_product", "start": 7, "multiplier": 8, "modulus": 15 } + }, + { + "id": "pendulum-late-1", + "phase": "late", + "room": "double-pendulum", + "prompt": "Two deterministic double-pendulum simulations begin with exactly identical angles and velocities. With no outside perturbation, what is their separation after 1600 identical integration steps?", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "identical_state_gap", "steps": 1600 } + }, + { + "id": "pendulum-late-2", + "phase": "late", + "room": "double-pendulum", + "prompt": "Twin pair A grows from gap 0.004 to 0.08. Twin pair B grows from gap 0.001 to 0.08 over the same interval. Which pair shows the larger relative divergence? Answer A, B, or equal.", + "answerSchema": { "type": "string", "enum": ["A", "B", "equal"] }, + "oracle": { "kind": "relative_growth", "aInitial": 0.004, "aFinal": 0.08, "bInitial": 0.001, "bFinal": 0.08 } + }, + { + "id": "life-late-1", + "phase": "late", + "room": "game-of-life", + "prompt": "A live Game of Life cell has exactly one live neighbor. Is that center cell alive or dead in the next generation?", + "answerSchema": { "type": "string", "enum": ["alive", "dead"] }, + "oracle": { "kind": "life_center", "currentlyAlive": true, "liveNeighbors": 1 } + }, + { + "id": "life-late-2", + "phase": "late", + "room": "game-of-life", + "prompt": "A dead Game of Life cell has exactly four live neighbors. Is that center cell alive or dead in the next generation?", + "answerSchema": { "type": "string", "enum": ["alive", "dead"] }, + "oracle": { "kind": "life_center", "currentlyAlive": false, "liveNeighbors": 4 } + }, + { + "id": "galton-late-1", + "phase": "late", + "room": "galton-board", + "prompt": "A ball makes seven left-or-right choices. How many distinct paths contain exactly three right choices?", + "answerSchema": { "type": "number", "tolerance": 0.0 }, + "oracle": { "kind": "binomial_paths", "rows": 7, "rights": 3 } + }, + { + "id": "galton-late-2", + "phase": "late", + "room": "galton-board", + "prompt": "An unbiased Galton ball makes ten left-or-right choices. What is the expected number of right choices?", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "binomial_mean", "rows": 10, "rightProbability": 0.5 } + }, + { + "id": "formula-late-1", + "phase": "late", + "room": "formula-jam", + "prompt": "Classify y = cos(3x) by reflection through the origin. Answer odd, even, or neither.", + "answerSchema": { "type": "string", "enum": ["odd", "even", "neither"] }, + "oracle": { "kind": "trig_parity", "function": "cos", "frequency": 3, "verticalOffset": 0.0 } + }, + { + "id": "formula-late-2", + "phase": "late", + "room": "formula-jam", + "prompt": "For y = sin(8x), what is the fundamental period divided by pi? Give a number.", + "answerSchema": { "type": "number", "tolerance": 1e-12 }, + "oracle": { "kind": "trig_period_over_pi", "frequency": 8 } + } + ] +} diff --git a/scripts/understanding-study.py b/scripts/understanding-study.py new file mode 100644 index 00000000..397fc7f6 --- /dev/null +++ b/scripts/understanding-study.py @@ -0,0 +1,1532 @@ +#!/usr/bin/env python3 +"""Frozen allocation, probe delivery, redaction, and analysis for 0.4.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import math +import os +import re +import sys +import tempfile +from collections import defaultdict +from datetime import date +from pathlib import Path +from typing import Any, Iterable + +ROOT = Path(__file__).resolve().parent.parent +PROBE_BANK_PATH = ROOT / "scripts" / "understanding-probes.json" +RUNNER_VERSION = "numinous-understanding-runner-v1" +ALLOCATION_SEED = "numinous-understanding-alpha-v1" +BOOTSTRAP_SEED = "numinous-understanding-alpha-bootstrap-v1" +ALLOCATION_SCHEMA = "numinous-understanding-allocation-v1" +EVENT_SCHEMA = "numinous-understanding-events-v1" +REPORT_SCHEMA = "numinous-understanding-report-v1" +PROTOCOL_VERSION = "0.4-v1" +BOOTSTRAP_RESAMPLES = 100_000 +MODEL_FAMILIES = ("gpt-5.6-sol", "gpt-5.6-terra") +MODEL_ALIASES = {"gpt-5.6-sol": "sol", "gpt-5.6-terra": "terra"} +ROOMS = ( + "times-tables", + "double-pendulum", + "game-of-life", + "galton-board", + "formula-jam", +) +CONDITIONS = ("generation-before-reveal", "explanation-first") +TOOL_CALLS_PER_ROOM = 4 +MAX_JSONL_LINE_BYTES = 1_000_000 +ABSOLUTE_PATH = re.compile( + r"(?i)(?:\b[a-z]:\\[^\s\"']+|\\\\[^\\\s]+\\[^\s\"']+|" + r"(? bytes: + """Return the stable JSON representation used for content hashes.""" + return json.dumps( + value, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ).encode("utf-8") + + +def content_sha256(value: Any) -> str: + """Hash a JSON value after canonical serialization.""" + return hashlib.sha256(canonical_bytes(value)).hexdigest() + + +def keyed_digest(seed: str, label: str) -> bytes: + """Derive stable bytes without relying on a runtime random implementation.""" + return hashlib.sha256(f"{seed}\0{label}".encode("utf-8")).digest() + + +def stable_order(values: Iterable[Any], label: str) -> list[Any]: + """Order values by a seed-derived digest with a canonical tie breaker.""" + decorated = [ + ( + keyed_digest( + ALLOCATION_SEED, + f"{label}\0{index}\0{canonical_bytes(value).decode('utf-8')}", + ), + index, + value, + ) + for index, value in enumerate(values) + ] + return [value for _digest, _index, value in sorted(decorated)] + + +def load_json(path: Path) -> Any: + """Load a JSON document with a bounded, readable failure.""" + try: + return json.loads(path.read_text(encoding="utf-8")) + except OSError as error: + raise StudyError(f"could not read {path}: {error}") from error + except json.JSONDecodeError as error: + raise StudyError(f"invalid JSON in {path}: {error}") from error + + +def write_text_once(path: Path, text: str) -> str: + """Write atomically, refusing to replace evidence with different content.""" + if path.exists(): + try: + existing = path.read_text(encoding="utf-8") + except OSError as error: + raise StudyError(f"could not inspect existing {path}: {error}") from error + if existing == text: + return "unchanged" + raise StudyError(f"refusing to replace existing evidence file {path}") + path.parent.mkdir(parents=True, exist_ok=True) + handle = tempfile.NamedTemporaryFile( + mode="w", + encoding="utf-8", + newline="\n", + dir=path.parent, + prefix=f".{path.name}.", + suffix=".tmp", + delete=False, + ) + temporary = Path(handle.name) + try: + with handle: + handle.write(text) + handle.flush() + os.fsync(handle.fileno()) + os.replace(temporary, path) + except Exception: + temporary.unlink(missing_ok=True) + raise + return "written" + + +def write_json_once(path: Path, value: Any) -> str: + """Write indented JSON through the evidence-preserving writer.""" + text = json.dumps(value, ensure_ascii=False, sort_keys=True, indent=2, allow_nan=False) + return write_text_once(path, text + "\n") + + +def oracle_answer(oracle: dict[str, Any]) -> int | float | str: + """Compute an answer independently from the room implementation.""" + kind = oracle.get("kind") + if kind == "mod_product": + start = require_int(oracle, "start") + multiplier = require_int(oracle, "multiplier") + modulus = require_int(oracle, "modulus") + if modulus <= 0: + raise StudyError("mod_product modulus must be positive") + return (start * multiplier) % modulus + if kind == "identical_state_gap": + if require_int(oracle, "steps") <= 0: + raise StudyError("identical_state_gap steps must be positive") + return 0.0 + if kind == "relative_growth": + a_initial = require_positive_number(oracle, "aInitial") + b_initial = require_positive_number(oracle, "bInitial") + a_growth = require_number(oracle, "aFinal") / a_initial + b_growth = require_number(oracle, "bFinal") / b_initial + if math.isclose(a_growth, b_growth, rel_tol=0.0, abs_tol=1e-15): + return "equal" + return "A" if a_growth > b_growth else "B" + if kind == "life_center": + currently_alive = oracle.get("currentlyAlive") + if not isinstance(currently_alive, bool): + raise StudyError("life_center currentlyAlive must be boolean") + neighbors = require_int(oracle, "liveNeighbors") + if not 0 <= neighbors <= 8: + raise StudyError("life_center liveNeighbors must be in 0..8") + alive = neighbors == 3 or (currently_alive and neighbors == 2) + return "alive" if alive else "dead" + if kind == "binomial_paths": + rows = require_int(oracle, "rows") + rights = require_int(oracle, "rights") + if rows < 0 or not 0 <= rights <= rows: + raise StudyError("invalid binomial_paths parameters") + return math.comb(rows, rights) + if kind == "binomial_mean": + rows = require_int(oracle, "rows") + probability = require_number(oracle, "rightProbability") + if rows < 0 or not 0.0 <= probability <= 1.0: + raise StudyError("invalid binomial_mean parameters") + return rows * probability + if kind == "trig_parity": + function = oracle.get("function") + frequency = require_int(oracle, "frequency") + offset = require_number(oracle, "verticalOffset") + if frequency == 0 or offset != 0.0: + return "neither" + if function == "sin": + return "odd" + if function == "cos": + return "even" + raise StudyError(f"unsupported trig function {function!r}") + if kind == "trig_period_over_pi": + frequency = abs(require_int(oracle, "frequency")) + if frequency == 0: + raise StudyError("trig frequency must be nonzero") + return 2.0 / frequency + raise StudyError(f"unsupported oracle kind {kind!r}") + + +def require_int(value: dict[str, Any], key: str) -> int: + """Read an exact integer, rejecting booleans.""" + item = value.get(key) + if isinstance(item, bool) or not isinstance(item, int): + raise StudyError(f"{key} must be an integer") + return item + + +def require_number(value: dict[str, Any], key: str) -> float: + """Read a finite JSON number, rejecting booleans.""" + item = value.get(key) + if isinstance(item, bool) or not isinstance(item, (int, float)): + raise StudyError(f"{key} must be a number") + number = float(item) + if not math.isfinite(number): + raise StudyError(f"{key} must be finite") + return number + + +def require_positive_number(value: dict[str, Any], key: str) -> float: + """Read a finite, strictly positive number.""" + number = require_number(value, key) + if number <= 0.0: + raise StudyError(f"{key} must be positive") + return number + + +def normalize_answer(schema: dict[str, Any], answer: Any) -> tuple[bool, Any]: + """Validate an answer and return its canonical comparison value.""" + answer_type = schema.get("type") + if answer_type == "number": + if isinstance(answer, bool) or not isinstance(answer, (int, float)): + return False, None + number = float(answer) + if not math.isfinite(number): + return False, None + return True, number + if answer_type == "string": + if not isinstance(answer, str): + return False, None + choices = schema.get("enum") + if not isinstance(choices, list) or not choices: + raise StudyError("string answer schema requires a nonempty enum") + normalized = answer.strip().casefold() + for choice in choices: + if not isinstance(choice, str): + raise StudyError("string answer enum values must be strings") + if choice.casefold() == normalized: + return True, choice + return False, None + raise StudyError(f"unsupported answer schema type {answer_type!r}") + + +def score_answer(probe: dict[str, Any], answer: Any) -> tuple[bool, bool]: + """Return schema validity and objective correctness for one answer.""" + schema = probe["answerSchema"] + valid, normalized = normalize_answer(schema, answer) + if not valid: + return False, False + expected = oracle_answer(probe["oracle"]) + if schema["type"] == "number": + tolerance = require_number(schema, "tolerance") + if tolerance < 0.0: + raise StudyError("numeric tolerance must be nonnegative") + return True, abs(float(normalized) - float(expected)) <= tolerance + return True, normalized == expected + + +def validate_bank(bank: Any) -> dict[str, Any]: + """Validate the frozen inventory and every independent answer oracle.""" + if not isinstance(bank, dict): + raise StudyError("probe bank must be a JSON object") + if bank.get("schemaVersion") != "numinous-understanding-probes-v1": + raise StudyError("unsupported probe bank schema") + if bank.get("protocolVersion") != PROTOCOL_VERSION: + raise StudyError("probe bank protocol version does not match the runner") + distractor = bank.get("distractorSequence") + if not isinstance(distractor, dict) or not isinstance(distractor.get("id"), str): + raise StudyError("probe bank requires a named distractor sequence") + items = distractor.get("items") + if not isinstance(items, list) or len(items) != 5: + raise StudyError("distractor sequence must contain exactly five items") + distractor_ids: set[str] = set() + for item in items: + if not isinstance(item, dict): + raise StudyError("each distractor must be an object") + item_id = item.get("id") + prompt = item.get("prompt") + if not isinstance(item_id, str) or not item_id or item_id in distractor_ids: + raise StudyError("distractor ids must be unique nonempty strings") + if not isinstance(prompt, str) or not prompt.strip(): + raise StudyError("distractor prompts must be nonempty strings") + distractor_ids.add(item_id) + probes = bank.get("probes") + if not isinstance(probes, list): + raise StudyError("probe bank probes must be an array") + expected_counts = {(phase, room): 2 for phase in ("immediate", "late") for room in ROOMS} + counts: dict[tuple[str, str], int] = defaultdict(int) + probe_ids: set[str] = set() + for probe in probes: + if not isinstance(probe, dict): + raise StudyError("each probe must be an object") + probe_id = probe.get("id") + phase = probe.get("phase") + room = probe.get("room") + prompt = probe.get("prompt") + if not isinstance(probe_id, str) or not probe_id or probe_id in probe_ids: + raise StudyError("probe ids must be unique nonempty strings") + if (phase, room) not in expected_counts: + raise StudyError(f"probe {probe_id} has an invalid phase or room") + if not isinstance(prompt, str) or not prompt.strip(): + raise StudyError(f"probe {probe_id} has no prompt") + schema = probe.get("answerSchema") + oracle = probe.get("oracle") + if not isinstance(schema, dict) or not isinstance(oracle, dict): + raise StudyError(f"probe {probe_id} requires answerSchema and oracle objects") + expected = oracle_answer(oracle) + valid, _normalized = normalize_answer(schema, expected) + if not valid: + raise StudyError(f"probe {probe_id} oracle output violates its answer schema") + if schema.get("type") == "number" and require_number(schema, "tolerance") < 0.0: + raise StudyError(f"probe {probe_id} tolerance must be nonnegative") + counts[(phase, room)] += 1 + probe_ids.add(probe_id) + if counts != expected_counts: + raise StudyError(f"probe inventory mismatch: {dict(counts)}") + return bank + + +def load_bank() -> dict[str, Any]: + """Load and validate the tracked probe bank.""" + return validate_bank(load_json(PROBE_BANK_PATH)) + + +def build_allocation(bank: dict[str, Any]) -> dict[str, Any]: + """Build the complete 24-pair allocation from the literal protocol seed.""" + pairs: list[dict[str, Any]] = [] + for model in MODEL_FAMILIES: + rotations = stable_order([0, 0, 0, 1, 1, 1, 2, 2, 3, 3, 4, 4], f"{model}:rotations") + first_conditions = stable_order( + [CONDITIONS[0]] * 6 + [CONDITIONS[1]] * 6, + f"{model}:condition-order", + ) + for offset in range(12): + order = offset + 1 + pair_id = f"{MODEL_ALIASES[model]}-p{order:02d}" + first = first_conditions[offset] + second = CONDITIONS[1] if first == CONDITIONS[0] else CONDITIONS[0] + rotation = rotations[offset] + room_order = list(ROOMS[rotation:] + ROOMS[:rotation]) + sessions = [] + for condition in CONDITIONS: + suffix = "g" if condition == CONDITIONS[0] else "c" + sessions.append( + { + "sessionId": f"{pair_id}-{suffix}", + "condition": condition, + } + ) + pairs.append( + { + "pairId": pair_id, + "modelFamily": model, + "reasoningEffort": "high", + "order": order, + "allocationRole": "primary" if order <= 10 else "reserve", + "studySeed": keyed_digest(ALLOCATION_SEED, pair_id).hex(), + "roomOrder": room_order, + "collectionOrder": [ + next( + item["sessionId"] + for item in sessions + if item["condition"] == condition + ) + for condition in (first, second) + ], + "sessions": sessions, + } + ) + return { + "schemaVersion": ALLOCATION_SCHEMA, + "protocolVersion": PROTOCOL_VERSION, + "runnerVersion": RUNNER_VERSION, + "allocationSeed": ALLOCATION_SEED, + "probeBankSha256": content_sha256(bank), + "distractorSequenceId": bank["distractorSequence"]["id"], + "toolCallsPerRoom": TOOL_CALLS_PER_ROOM, + "models": [ + {"modelFamily": model, "reasoningEffort": "high", "qualifyingPairs": 10, "reserves": 2} + for model in MODEL_FAMILIES + ], + "pairs": pairs, + } + + +def validate_manifest(manifest: Any, bank: dict[str, Any]) -> dict[str, Any]: + """Require the byte-equivalent allocation generated by this revision.""" + expected = build_allocation(bank) + if canonical_bytes(manifest) != canonical_bytes(expected): + raise StudyError( + "allocation manifest differs from the frozen runner output; " + "use the exact tracked runner revision" + ) + return manifest + + +def manifest_indexes( + manifest: dict[str, Any], +) -> tuple[dict[str, dict[str, Any]], dict[str, tuple[dict[str, Any], dict[str, Any]]]]: + """Index pairs and sessions while rejecting duplicate identifiers.""" + pairs: dict[str, dict[str, Any]] = {} + sessions: dict[str, tuple[dict[str, Any], dict[str, Any]]] = {} + for pair in manifest["pairs"]: + pair_id = pair["pairId"] + if pair_id in pairs: + raise StudyError(f"duplicate pair id {pair_id}") + pairs[pair_id] = pair + for session in pair["sessions"]: + session_id = session["sessionId"] + if session_id in sessions: + raise StudyError(f"duplicate session id {session_id}") + sessions[session_id] = (pair, session) + return pairs, sessions + + +def probe_sequence( + bank: dict[str, Any], room_order: list[str], phase: str +) -> list[dict[str, Any]]: + """Return two probes per room in the pair's frozen cyclic order.""" + sequence: list[dict[str, Any]] = [] + for room in room_order: + sequence.extend( + probe + for probe in bank["probes"] + if probe["phase"] == phase and probe["room"] == room + ) + if len(sequence) != 10: + raise StudyError(f"expected 10 {phase} probes, found {len(sequence)}") + return sequence + + +def public_probe(probe: dict[str, Any], schema_only: bool) -> dict[str, Any]: + """Remove the oracle and optionally repeat only the repair schema.""" + packet = { + "schemaVersion": "numinous-understanding-public-probe-v1", + "probeId": probe["id"], + "answerSchema": probe["answerSchema"], + } + if not schema_only: + packet.update( + { + "phase": probe["phase"], + "room": probe["room"], + "prompt": probe["prompt"], + } + ) + return packet + + +def session_packet( + manifest: dict[str, Any], session_id: str +) -> dict[str, Any]: + """Describe one condition without exposing probes or answer keys.""" + _pairs, sessions = manifest_indexes(manifest) + if session_id not in sessions: + raise StudyError(f"unknown session id {session_id}") + pair, session = sessions[session_id] + condition = session["condition"] + if condition == CONDITIONS[0]: + roles = ["encounter", "generation", "interaction", "reveal"] + instruction = ( + "Encounter without Reveal, commit a prediction or construction, interact, " + "receive corrective feedback and Reveal, then self-explain." + ) + else: + roles = ["reveal", "explanation", "interaction", "continuation"] + instruction = ( + "Receive the same Reveal first, elaborate once, interact with the same budget " + "and corrective feedback, then continue without another generated answer." + ) + return { + "schemaVersion": "numinous-understanding-session-v1", + "sessionId": session_id, + "pairId": pair["pairId"], + "modelFamily": pair["modelFamily"], + "reasoningEffort": pair["reasoningEffort"], + "condition": condition, + "studySeed": pair["studySeed"], + "roomOrder": pair["roomOrder"], + "toolCallsPerRoom": manifest["toolCallsPerRoom"], + "toolRolesPerRoom": roles, + "instruction": instruction, + "dataBoundary": ( + "Record public tool names, arguments, structured results, and visible text only. " + "Do not record prompts, hidden reasoning, credentials, local paths, or unrelated state." + ), + } + + +def normalize_key(key: str) -> str: + """Normalize an object key for privacy screening.""" + return re.sub(r"[^a-z0-9]", "", key.casefold()) + + +def is_private_key(key: str) -> bool: + """Return whether a field can carry data forbidden by the protocol.""" + normalized = normalize_key(key) + if normalized == "reasoningeffort": + return False + return any(fragment in normalized for fragment in PRIVATE_KEY_FRAGMENTS) + + +def redact_value(value: Any, replacements: tuple[str, ...]) -> tuple[Any, int]: + """Remove forbidden keys and replace absolute roots in a JSON value.""" + if isinstance(value, dict): + clean: dict[str, Any] = {} + removed = 0 + for key, item in value.items(): + if not isinstance(key, str): + removed += 1 + continue + if is_private_key(key): + removed += 1 + continue + clean_item, nested = redact_value(item, replacements) + clean[key] = clean_item + removed += nested + return clean, removed + if isinstance(value, list): + clean_items = [] + removed = 0 + for item in value[:10_000]: + clean_item, nested = redact_value(item, replacements) + clean_items.append(clean_item) + removed += nested + removed += max(0, len(value) - len(clean_items)) + return clean_items, removed + if isinstance(value, str): + clean = ABSOLUTE_PATH.sub("", value) + count = 1 if clean != value else 0 + for replacement in replacements: + if replacement and replacement in clean: + clean = clean.replace(replacement, "") + count += 1 + if len(clean) > 65_536: + clean = clean[:65_536] + "" + count += 1 + return clean, count + return value, 0 + + +def assert_sanitized(value: Any, location: str = "event") -> None: + """Reject forbidden fields and absolute host paths before analysis.""" + if isinstance(value, dict): + for key, item in value.items(): + if not isinstance(key, str): + raise StudyError(f"{location} contains a non-string key") + if is_private_key(key): + raise StudyError(f"{location} contains forbidden field {key!r}") + assert_sanitized(item, f"{location}.{key}") + elif isinstance(value, list): + for index, item in enumerate(value): + assert_sanitized(item, f"{location}[{index}]") + elif isinstance(value, str) and ABSOLUTE_PATH.search(value): + raise StudyError(f"{location} contains an absolute host path") + + +def read_jsonl(path: Path) -> list[dict[str, Any]]: + """Read bounded JSONL records with stable source indexes.""" + records: list[dict[str, Any]] = [] + try: + with path.open("rb") as handle: + for line_number, raw in enumerate(handle, start=1): + if len(raw) > MAX_JSONL_LINE_BYTES: + raise StudyError(f"{path}:{line_number} exceeds the JSONL line limit") + if not raw.strip(): + continue + try: + record = json.loads(raw) + except json.JSONDecodeError as error: + raise StudyError(f"{path}:{line_number} is invalid JSON: {error}") from error + if not isinstance(record, dict): + raise StudyError(f"{path}:{line_number} must contain a JSON object") + record["_sourceIndex"] = len(records) + records.append(record) + except OSError as error: + raise StudyError(f"could not read {path}: {error}") from error + return records + + +def redact_jsonl(input_path: Path, output_path: Path, replacements: tuple[str, ...]) -> str: + """Produce a bounded ledger with prohibited fields removed.""" + output_lines: list[str] = [] + for record in read_jsonl(input_path): + record.pop("_sourceIndex", None) + clean, removed = redact_value(record, replacements) + if not isinstance(clean, dict): + raise StudyError("redaction produced a non-object record") + if removed: + clean["redactedFieldCount"] = removed + output_lines.append( + json.dumps( + clean, + ensure_ascii=False, + sort_keys=True, + separators=(",", ":"), + allow_nan=False, + ) + ) + return write_text_once(output_path, "\n".join(output_lines) + ("\n" if output_lines else "")) + + +def required_string(value: dict[str, Any], key: str) -> str: + """Read a nonempty string field.""" + item = value.get(key) + if not isinstance(item, str) or not item.strip(): + raise StudyError(f"{key} must be a nonempty string") + return item + + +def validate_session_header( + header: dict[str, Any], pair: dict[str, Any], session: dict[str, Any] +) -> None: + """Validate consent and the exact reproducibility metadata.""" + if header.get("schemaVersion") != EVENT_SCHEMA or header.get("type") != "session": + raise StudyError("session header has the wrong schema or type") + if header.get("consent") is not True: + raise StudyError(f"session {session['sessionId']} lacks explicit consent") + if header.get("sessionId") != session["sessionId"]: + raise StudyError("session header id mismatch") + if header.get("modelFamily") != pair["modelFamily"]: + raise StudyError(f"session {session['sessionId']} model family mismatch") + if header.get("reasoningEffort") != pair["reasoningEffort"]: + raise StudyError(f"session {session['sessionId']} reasoning effort mismatch") + if header.get("condition") != session["condition"]: + raise StudyError(f"session {session['sessionId']} condition mismatch") + for key in ( + "modelIdentifier", + "provider", + "backendRevision", + "date", + "mcpProtocolRevision", + "operatingSystem", + ): + required_string(header, key) + try: + date.fromisoformat(header["date"]) + except ValueError as error: + raise StudyError(f"session {session['sessionId']} date must be ISO 8601") from error + if header.get("runnerVersion") != RUNNER_VERSION: + raise StudyError(f"session {session['sessionId']} runner version mismatch") + commit = required_string(header, "numinousCommit") + if not COMMIT_SHA.fullmatch(commit): + raise StudyError(f"session {session['sessionId']} has an invalid commit SHA") + if not isinstance(header.get("settings"), dict): + raise StudyError(f"session {session['sessionId']} settings must be an object") + + +def contains_nonempty_key(value: Any, target: str) -> bool: + """Search structured public output for an early Reveal payload.""" + if isinstance(value, dict): + for key, item in value.items(): + if key.casefold() == target.casefold() and item not in (None, "", [], {}): + return True + if contains_nonempty_key(item, target): + return True + elif isinstance(value, list): + return any(contains_nonempty_key(item, target) for item in value) + return False + + +def validate_tool_events( + pair: dict[str, Any], session: dict[str, Any], events: list[dict[str, Any]] +) -> None: + """Enforce equal call budgets, room order, and Reveal ordering.""" + expected_roles = ( + ["encounter", "generation", "interaction", "reveal"] + if session["condition"] == CONDITIONS[0] + else ["reveal", "explanation", "interaction", "continuation"] + ) + if len(events) != len(ROOMS) * TOOL_CALLS_PER_ROOM: + raise StudyError( + f"session {session['sessionId']} must record exactly " + f"{len(ROOMS) * TOOL_CALLS_PER_ROOM} public tool calls" + ) + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for event in events: + room = event.get("room") + if room not in ROOMS: + raise StudyError(f"session {session['sessionId']} tool event has invalid room") + required_string(event, "tool") + if not isinstance(event.get("arguments"), dict): + raise StudyError("tool event arguments must be an object") + if event.get("structuredResult") is not None and not isinstance( + event.get("structuredResult"), dict + ): + raise StudyError("tool event structuredResult must be an object or null") + if not isinstance(event.get("visibleText"), str): + raise StudyError("tool event visibleText must be a string") + grouped[room].append(event) + observed_room_order = [ + room + for room, _first_index in sorted( + ( + (room, min(item["_sourceIndex"] for item in items)) + for room, items in grouped.items() + ), + key=lambda item: item[1], + ) + ] + if observed_room_order != pair["roomOrder"]: + raise StudyError(f"session {session['sessionId']} room order differs from allocation") + observed_calls = [ + (event["room"], event.get("sequence")) + for event in sorted(events, key=lambda item: item["_sourceIndex"]) + ] + expected_calls = [ + (room, sequence) + for room in pair["roomOrder"] + for sequence in range(1, TOOL_CALLS_PER_ROOM + 1) + ] + if observed_calls != expected_calls: + raise StudyError(f"session {session['sessionId']} interleaves or reorders room calls") + for room in pair["roomOrder"]: + room_events = sorted(grouped[room], key=lambda item: item["_sourceIndex"]) + sequences = [event.get("sequence") for event in room_events] + roles = [event.get("role") for event in room_events] + if sequences != list(range(1, TOOL_CALLS_PER_ROOM + 1)): + raise StudyError(f"session {session['sessionId']} {room} call sequence is invalid") + if roles != expected_roles: + raise StudyError(f"session {session['sessionId']} {room} role order is invalid") + reveal_event = next(event for event in room_events if event.get("role") == "reveal") + if not reveal_event["visibleText"] and not reveal_event["structuredResult"]: + raise StudyError(f"session {session['sessionId']} {room} has an empty Reveal") + if session["condition"] == CONDITIONS[0]: + for event in room_events[:-1]: + if contains_nonempty_key(event.get("structuredResult"), "reveal"): + raise StudyError( + f"session {session['sessionId']} {room} leaked Reveal before generation" + ) + + +def reveal_payloads(events: list[dict[str, Any]]) -> dict[str, bytes]: + """Return the public Reveal payload for pairwise equality checks.""" + payloads: dict[str, bytes] = {} + for event in events: + if event.get("type") != "tool" or event.get("role") != "reveal": + continue + room = event["room"] + if room in payloads: + raise StudyError(f"duplicate Reveal payload for {room}") + payloads[room] = canonical_bytes( + { + "structuredResult": event["structuredResult"], + "visibleText": event["visibleText"], + } + ) + if set(payloads) != set(ROOMS): + raise StudyError("session does not contain one Reveal payload per room") + return payloads + + +def ordered_response_score( + session_id: str, + expected_probes: list[dict[str, Any]], + distractor_items: list[dict[str, Any]], + response_events: list[dict[str, Any]], +) -> dict[str, Any]: + """Consume immediate, distractor, and late events in their frozen order.""" + cursor = 0 + scores: dict[str, int] = {} + invalid_attempts = 0 + repairs = 0 + refusals = 0 + + def current() -> dict[str, Any] | None: + return response_events[cursor] if cursor < len(response_events) else None + + for probe in expected_probes[:10]: + event = current() + if event is None or event.get("probeId") != probe["id"]: + raise StudyError(f"session {session_id} is missing ordered probe {probe['id']}") + if event.get("type") == "response_refusal": + scores[probe["id"]] = 0 + refusals += 1 + cursor += 1 + continue + if event.get("type") != "response" or event.get("attempt") != 1: + raise StudyError(f"session {session_id} has an invalid first response event") + valid, correct = score_answer(probe, event.get("answer")) + cursor += 1 + if valid: + scores[probe["id"]] = int(correct) + continue + invalid_attempts += 1 + retry = current() + if ( + retry is not None + and retry.get("type") == "response" + and retry.get("probeId") == probe["id"] + and retry.get("attempt") == 2 + ): + repairs += 1 + valid, correct = score_answer(probe, retry.get("answer")) + invalid_attempts += int(not valid) + scores[probe["id"]] = int(valid and correct) + cursor += 1 + else: + scores[probe["id"]] = 0 + + for item in distractor_items: + event = current() + if ( + event is None + or event.get("type") != "distractor_response" + or event.get("itemId") != item["id"] + ): + raise StudyError(f"session {session_id} is missing distractor {item['id']}") + answer = event.get("answer") + if isinstance(answer, (dict, list)) or answer is None: + raise StudyError(f"session {session_id} distractor answer must be scalar") + cursor += 1 + + for probe in expected_probes[10:]: + event = current() + if event is None or event.get("probeId") != probe["id"]: + raise StudyError(f"session {session_id} is missing ordered probe {probe['id']}") + if event.get("type") == "response_refusal": + scores[probe["id"]] = 0 + refusals += 1 + cursor += 1 + continue + if event.get("type") != "response" or event.get("attempt") != 1: + raise StudyError(f"session {session_id} has an invalid first response event") + valid, correct = score_answer(probe, event.get("answer")) + cursor += 1 + if valid: + scores[probe["id"]] = int(correct) + continue + invalid_attempts += 1 + retry = current() + if ( + retry is not None + and retry.get("type") == "response" + and retry.get("probeId") == probe["id"] + and retry.get("attempt") == 2 + ): + repairs += 1 + valid, correct = score_answer(probe, retry.get("answer")) + invalid_attempts += int(not valid) + scores[probe["id"]] = int(valid and correct) + cursor += 1 + else: + scores[probe["id"]] = 0 + if cursor != len(response_events): + raise StudyError(f"session {session_id} has extra or out-of-order probe events") + return { + "scores": scores, + "invalidAttempts": invalid_attempts, + "schemaRepairs": repairs, + "responseRefusals": refusals, + } + + +def validate_and_score_session( + bank: dict[str, Any], + pair: dict[str, Any], + session: dict[str, Any], + header: dict[str, Any], + completion: dict[str, Any], + events: list[dict[str, Any]], +) -> dict[str, Any]: + """Validate one isolated session and return objective phase scores.""" + validate_session_header(header, pair, session) + tool_events = [event for event in events if event.get("type") == "tool"] + probe_events = [ + event + for event in events + if event.get("type") in ("response", "response_refusal", "distractor_response") + ] + validate_tool_events(pair, session, tool_events) + if not probe_events: + raise StudyError(f"session {session['sessionId']} has no probe events") + if max(event["_sourceIndex"] for event in tool_events) >= min( + event["_sourceIndex"] for event in probe_events + ): + raise StudyError(f"session {session['sessionId']} probes began before encounters ended") + if header["_sourceIndex"] >= min(event["_sourceIndex"] for event in tool_events): + raise StudyError(f"session {session['sessionId']} activity precedes consent metadata") + if completion["_sourceIndex"] <= max(event["_sourceIndex"] for event in probe_events): + raise StudyError(f"session {session['sessionId']} completion is out of order") + expected = probe_sequence(bank, pair["roomOrder"], "immediate") + probe_sequence( + bank, pair["roomOrder"], "late" + ) + ordered = sorted(probe_events, key=lambda event: event["_sourceIndex"]) + result = ordered_response_score( + session["sessionId"], expected, bank["distractorSequence"]["items"], ordered + ) + probe_by_id = {probe["id"]: probe for probe in expected} + phase_scores: dict[str, float] = {} + room_scores: dict[str, dict[str, float]] = {} + for phase in ("immediate", "late"): + phase_items = [probe for probe in expected if probe["phase"] == phase] + phase_scores[phase] = sum(result["scores"][probe["id"]] for probe in phase_items) / 10.0 + room_scores[phase] = {} + for room in ROOMS: + ids = [ + probe_id + for probe_id, probe in probe_by_id.items() + if probe["phase"] == phase and probe["room"] == room + ] + room_scores[phase][room] = sum(result["scores"][probe_id] for probe_id in ids) / 2.0 + return { + "sessionId": session["sessionId"], + "condition": session["condition"], + "immediateScore": phase_scores["immediate"], + "lateScore": phase_scores["late"], + "roomScores": room_scores, + "invalidAttempts": result["invalidAttempts"], + "schemaRepairs": result["schemaRepairs"], + "responseRefusals": result["responseRefusals"], + } + + +class StableRng: + """Small SHA-256 counter stream used only for frozen bootstrap indexes.""" + + def __init__(self, seed: str) -> None: + self._seed = seed.encode("utf-8") + self._counter = 0 + self._buffer = bytearray() + + def _fill(self) -> None: + counter = self._counter.to_bytes(16, "big") + self._buffer.extend(hashlib.sha256(self._seed + b"\0" + counter).digest()) + self._counter += 1 + + def randbelow(self, upper: int) -> int: + """Return an unbiased integer in range(upper).""" + if upper <= 0: + raise StudyError("randbelow upper bound must be positive") + ceiling = 1 << 64 + limit = ceiling - (ceiling % upper) + while True: + while len(self._buffer) < 8: + self._fill() + raw = int.from_bytes(self._buffer[:8], "big") + del self._buffer[:8] + if raw < limit: + return raw % upper + + +def percentile(values: list[float], probability: float) -> float: + """Linear percentile over positions 0 through n - 1.""" + if not values or not 0.0 <= probability <= 1.0: + raise StudyError("invalid percentile input") + ordered = sorted(values) + position = (len(ordered) - 1) * probability + lower = math.floor(position) + upper = math.ceil(position) + if lower == upper: + return ordered[lower] + weight = position - lower + return ordered[lower] * (1.0 - weight) + ordered[upper] * weight + + +def stratified_bootstrap( + differences: dict[str, list[float]], resamples: int = BOOTSTRAP_RESAMPLES +) -> dict[str, Any]: + """Bootstrap 10 pairs within each family, then pool all 20 differences.""" + if set(differences) != set(MODEL_FAMILIES): + raise StudyError("bootstrap requires both frozen model families") + if any(len(differences[model]) != 10 for model in MODEL_FAMILIES): + raise StudyError("bootstrap requires exactly 10 pair differences per family") + if resamples <= 0: + raise StudyError("bootstrap resample count must be positive") + rng = StableRng(BOOTSTRAP_SEED) + pooled_distribution: list[float] = [] + family_distributions: dict[str, list[float]] = {model: [] for model in MODEL_FAMILIES} + for _sample in range(resamples): + family_means = [] + for model in MODEL_FAMILIES: + values = differences[model] + mean = sum(values[rng.randbelow(10)] for _draw in range(10)) / 10.0 + family_distributions[model].append(mean) + family_means.append(mean) + pooled_distribution.append(sum(family_means) / 2.0) + return { + "method": "two-sided percentile interval with linear interpolation", + "seed": BOOTSTRAP_SEED, + "resamples": resamples, + "pooled95": [ + percentile(pooled_distribution, 0.025), + percentile(pooled_distribution, 0.975), + ], + "family95": { + model: [ + percentile(family_distributions[model], 0.025), + percentile(family_distributions[model], 0.975), + ] + for model in MODEL_FAMILIES + }, + } + + +def paired_configuration_key(header: dict[str, Any]) -> bytes: + """Return the metadata that must match inside a pair.""" + return canonical_bytes( + { + key: header[key] + for key in ( + "modelFamily", + "modelIdentifier", + "provider", + "backendRevision", + "reasoningEffort", + "settings", + "numinousCommit", + "mcpProtocolRevision", + "operatingSystem", + "runnerVersion", + ) + } + ) + + +def analyze_events( + manifest: dict[str, Any], + bank: dict[str, Any], + records: list[dict[str, Any]], + bootstrap_resamples: int = BOOTSTRAP_RESAMPLES, +) -> dict[str, Any]: + """Validate a complete cohort and compute every predeclared objective result.""" + validate_manifest(manifest, bank) + pairs, sessions = manifest_indexes(manifest) + headers: dict[str, dict[str, Any]] = {} + completions: dict[str, dict[str, Any]] = {} + session_events: dict[str, list[dict[str, Any]]] = defaultdict(list) + pair_outcomes: dict[str, dict[str, Any]] = {} + pair_has_content: set[str] = set() + recruitment_refusals: dict[str, int] = defaultdict(int) + deviations: list[dict[str, Any]] = [] + allowed_types = { + "session", + "tool", + "response", + "response_refusal", + "distractor_response", + "session_complete", + "recruitment_refusal", + "withdrawal", + "infrastructure_failure", + "deviation", + } + for record in records: + assert_sanitized(record) + if record.get("schemaVersion") != EVENT_SCHEMA: + raise StudyError("every event must use the frozen event schema") + event_type = record.get("type") + if event_type not in allowed_types: + raise StudyError(f"unsupported event type {event_type!r}") + if event_type == "recruitment_refusal": + model = record.get("modelFamily") + if model not in MODEL_FAMILIES: + raise StudyError("recruitment refusal has an invalid model family") + allowed = { + "schemaVersion", + "type", + "modelFamily", + "_sourceIndex", + "redactedFieldCount", + } + if set(record) - allowed: + raise StudyError("recruitment refusals may contain only an aggregate model family") + recruitment_refusals[model] += 1 + continue + if event_type in ("withdrawal", "infrastructure_failure"): + pair_id = record.get("pairId") + if pair_id not in pairs or pair_id in pair_outcomes: + raise StudyError("pair outcome has an invalid or duplicate pair id") + if event_type == "infrastructure_failure": + if record.get("stage") != "before_exposure": + raise StudyError("infrastructure failures must occur before exposure") + required_string(record, "reasonCode") + allowed = { + "schemaVersion", + "type", + "pairId", + "stage", + "reasonCode", + "_sourceIndex", + "redactedFieldCount", + } + else: + allowed = { + "schemaVersion", + "type", + "pairId", + "_sourceIndex", + "redactedFieldCount", + } + if set(record) - allowed: + raise StudyError(f"{event_type} may not retain response content") + pair_outcomes[pair_id] = record + continue + if event_type == "deviation": + required_string(record, "code") + required_string(record, "description") + deviations.append( + {key: value for key, value in record.items() if key != "_sourceIndex"} + ) + continue + session_id = record.get("sessionId") + if session_id not in sessions: + raise StudyError(f"event has unknown session id {session_id!r}") + pair, _session = sessions[session_id] + pair_has_content.add(pair["pairId"]) + if event_type == "session": + if session_id in headers: + raise StudyError(f"duplicate session header {session_id}") + headers[session_id] = record + elif event_type == "session_complete": + if session_id in completions: + raise StudyError(f"duplicate session completion {session_id}") + allowed = { + "schemaVersion", + "type", + "sessionId", + "_sourceIndex", + "redactedFieldCount", + } + if set(record) - allowed: + raise StudyError("session completion may not contain response content") + completions[session_id] = record + else: + if event_type == "response_refusal": + allowed = { + "schemaVersion", + "type", + "sessionId", + "phase", + "probeId", + "_sourceIndex", + "redactedFieldCount", + } + if set(record) - allowed: + raise StudyError("response refusal may not contain response content") + session_events[session_id].append(record) + overlap = set(pair_outcomes) & pair_has_content + if overlap: + raise StudyError( + "withdrawn or failed pairs must retain no response content: " + + ", ".join(sorted(overlap)) + ) + + selected_pairs: list[dict[str, Any]] = [] + for model in MODEL_FAMILIES: + model_pairs = sorted( + (pair for pair in manifest["pairs"] if pair["modelFamily"] == model), + key=lambda pair: pair["order"], + ) + selected_for_model = 0 + selection_closed = False + for pair in model_pairs: + pair_id = pair["pairId"] + has_content = pair_id in pair_has_content + if selection_closed: + if has_content or pair_id in pair_outcomes: + raise StudyError(f"cohort continued after 10 complete pairs for {model}") + continue + if pair_id in pair_outcomes: + continue + session_ids = [session["sessionId"] for session in pair["sessions"]] + complete = all( + session_id in headers and session_id in completions for session_id in session_ids + ) + if not complete: + raise StudyError( + f"incomplete cohort at {pair_id}; no outcome report may be generated" + ) + selected_pairs.append(pair) + selected_for_model += 1 + if selected_for_model == 10: + selection_closed = True + if selected_for_model != 10: + raise StudyError(f"incomplete cohort for {model}; two reserves are exhausted") + + for model in MODEL_FAMILIES: + selected_for_model = [ + pair for pair in selected_pairs if pair["modelFamily"] == model + ] + last_order = max(pair["order"] for pair in selected_for_model) + consumed = sorted( + ( + pair + for pair in manifest["pairs"] + if pair["modelFamily"] == model and pair["order"] <= last_order + ), + key=lambda pair: pair["order"], + ) + first_indexes = [] + for pair in consumed: + pair_id = pair["pairId"] + if pair_id in pair_outcomes: + first_index = pair_outcomes[pair_id]["_sourceIndex"] + else: + first_index = min( + headers[session["sessionId"]]["_sourceIndex"] + for session in pair["sessions"] + ) + first_indexes.append((pair_id, first_index)) + observed = [pair_id for pair_id, _index in sorted(first_indexes, key=lambda item: item[1])] + expected = [pair["pairId"] for pair in consumed] + if observed != expected: + raise StudyError(f"{model} pairs were not collected in frozen order") + + selected_session_ids = [ + session["sessionId"] for pair in selected_pairs for session in pair["sessions"] + ] + cohort_commits = {headers[session_id]["numinousCommit"] for session_id in selected_session_ids} + if len(cohort_commits) != 1: + raise StudyError("all qualifying sessions must use one Numinous commit") + protocol_revisions = { + headers[session_id]["mcpProtocolRevision"] for session_id in selected_session_ids + } + if len(protocol_revisions) != 1: + raise StudyError("all qualifying sessions must use one MCP protocol revision") + cohort_commit = next(iter(cohort_commits)) + protocol_revision = next(iter(protocol_revisions)) + + session_scores: dict[str, dict[str, Any]] = {} + pair_results: list[dict[str, Any]] = [] + differences: dict[str, list[float]] = {model: [] for model in MODEL_FAMILIES} + room_differences: dict[str, list[float]] = {room: [] for room in ROOMS} + for pair in selected_pairs: + pair_headers = [headers[session["sessionId"]] for session in pair["sessions"]] + observed_collection_order = [ + session_id + for session_id, _index in sorted( + ( + (session["sessionId"], headers[session["sessionId"]]["_sourceIndex"]) + for session in pair["sessions"] + ), + key=lambda item: item[1], + ) + ] + if observed_collection_order != pair["collectionOrder"]: + raise StudyError(f"pair {pair['pairId']} condition collection order changed") + if paired_configuration_key(pair_headers[0]) != paired_configuration_key(pair_headers[1]): + raise StudyError(f"pair {pair['pairId']} does not use the same model configuration") + by_condition: dict[str, dict[str, Any]] = {} + for session in pair["sessions"]: + session_id = session["sessionId"] + score = validate_and_score_session( + bank, + pair, + session, + headers[session_id], + completions[session_id], + session_events[session_id], + ) + session_scores[session_id] = score + by_condition[session["condition"]] = score + generation = by_condition[CONDITIONS[0]] + control = by_condition[CONDITIONS[1]] + generation_reveals = reveal_payloads(session_events[generation["sessionId"]]) + control_reveals = reveal_payloads(session_events[control["sessionId"]]) + if generation_reveals != control_reveals: + raise StudyError(f"pair {pair['pairId']} did not receive identical Reveal payloads") + difference = generation["immediateScore"] - control["immediateScore"] + differences[pair["modelFamily"]].append(difference) + per_room = {} + for room in ROOMS: + room_difference = ( + generation["roomScores"]["immediate"][room] + - control["roomScores"]["immediate"][room] + ) + per_room[room] = room_difference + room_differences[room].append(room_difference) + pair_results.append( + { + "pairId": pair["pairId"], + "modelFamily": pair["modelFamily"], + "generationImmediate": generation["immediateScore"], + "controlImmediate": control["immediateScore"], + "pairedImmediateDifference": difference, + "generationLate": generation["lateScore"], + "controlLate": control["lateScore"], + "roomImmediateDifferences": per_room, + } + ) + bootstrap = stratified_bootstrap(differences, bootstrap_resamples) + family_means = { + model: sum(differences[model]) / len(differences[model]) for model in MODEL_FAMILIES + } + pooled_mean = sum(family_means.values()) / 2.0 + room_means = { + room: sum(room_differences[room]) / len(room_differences[room]) for room in ROOMS + } + criteria = { + "pairedMeanAtLeastTenPoints": pooled_mean >= 0.10, + "bootstrapLowerBoundAboveZero": bootstrap["pooled95"][0] > 0.0, + "eachModelNonnegative": all(value >= 0.0 for value in family_means.values()), + "fourOfFiveRoomsNonnegative": sum(value >= 0.0 for value in room_means.values()) >= 4, + "noRoomBelowNegativeTenPoints": all(value >= -0.10 for value in room_means.values()), + "completeFailureAndDeviationLedger": True, + } + return { + "schemaVersion": REPORT_SCHEMA, + "protocolVersion": PROTOCOL_VERSION, + "runnerVersion": RUNNER_VERSION, + "numinousCommit": cohort_commit, + "mcpProtocolRevision": protocol_revision, + "allocationSha256": content_sha256(manifest), + "probeBankSha256": content_sha256(bank), + "cohortComplete": True, + "selectedPairs": [pair["pairId"] for pair in selected_pairs], + "recruitmentRefusals": { + model: recruitment_refusals.get(model, 0) for model in MODEL_FAMILIES + }, + "withdrawals": sum( + event["type"] == "withdrawal" for event in pair_outcomes.values() + ), + "infrastructureFailures": sum( + event["type"] == "infrastructure_failure" for event in pair_outcomes.values() + ), + "deviations": deviations, + "primary": { + "pairedMeanDifference": pooled_mean, + "familyMeanDifferences": family_means, + "roomMeanDifferences": room_means, + "bootstrap": bootstrap, + "criteria": criteria, + "predeclaredStatisticalGatePassed": all(criteria.values()), + }, + "pairResults": pair_results, + "sessionDiagnostics": [session_scores[key] for key in sorted(session_scores)], + "evidenceBoundary": ( + "The computed statistical gate is not the 0.4 milestone by itself. Publication, " + "independent methodology and math review, and returning-journal acceptance " + "remain required." + ), + } + + +def load_manifest(path: Path, bank: dict[str, Any]) -> dict[str, Any]: + """Load and validate the exact allocation manifest.""" + manifest = load_json(path) + if not isinstance(manifest, dict): + raise StudyError("allocation manifest must be a JSON object") + return validate_manifest(manifest, bank) + + +def command_allocate(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = build_allocation(bank) + result = write_json_once(args.output, manifest) + print(f"{result} {args.output}") + print(f"allocation sha256 {content_sha256(manifest)}") + + +def command_validate(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = load_manifest(args.manifest, bank) + print(f"probe bank PASS {content_sha256(bank)}") + print(f"allocation PASS {content_sha256(manifest)}") + + +def command_session(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = load_manifest(args.manifest, bank) + print(json.dumps(session_packet(manifest, args.session_id), indent=2, sort_keys=True)) + + +def command_probe(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = load_manifest(args.manifest, bank) + _pairs, sessions = manifest_indexes(manifest) + if args.session_id not in sessions: + raise StudyError(f"unknown session id {args.session_id}") + pair, _session = sessions[args.session_id] + sequence = probe_sequence(bank, pair["roomOrder"], args.phase) + if not 1 <= args.index <= len(sequence): + raise StudyError(f"probe index must be in 1..{len(sequence)}") + print( + json.dumps( + public_probe(sequence[args.index - 1], args.schema_only), + indent=2, + sort_keys=True, + ) + ) + + +def command_distractor(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = load_manifest(args.manifest, bank) + _pairs, sessions = manifest_indexes(manifest) + if args.session_id not in sessions: + raise StudyError(f"unknown session id {args.session_id}") + items = bank["distractorSequence"]["items"] + if not 1 <= args.index <= len(items): + raise StudyError(f"distractor index must be in 1..{len(items)}") + item = items[args.index - 1] + print( + json.dumps( + { + "schemaVersion": "numinous-understanding-distractor-v1", + "sequenceId": bank["distractorSequence"]["id"], + "itemId": item["id"], + "prompt": item["prompt"], + }, + indent=2, + sort_keys=True, + ) + ) + + +def command_redact(args: argparse.Namespace) -> None: + result = redact_jsonl(args.input, args.output, tuple(args.replace)) + print(f"{result} {args.output}") + + +def command_analyze(args: argparse.Namespace) -> None: + bank = load_bank() + manifest = load_manifest(args.manifest, bank) + records = read_jsonl(args.responses) + report = analyze_events(manifest, bank, records) + result = write_json_once(args.output, report) + print(f"{result} {args.output}") + print( + "predeclared statistical gate " + + ("PASS" if report["primary"]["predeclaredStatisticalGatePassed"] else "FAIL") + ) + + +def build_parser() -> argparse.ArgumentParser: + """Build the command-line contract.""" + parser = argparse.ArgumentParser( + description=( + "Freeze and analyze the 0.4 Understanding Alpha study. The runner never calls a " + "model and refuses to report an incomplete cohort." + ) + ) + subparsers = parser.add_subparsers(dest="command", required=True) + + allocate = subparsers.add_parser("allocate", help="write the exact 24-pair manifest") + allocate.add_argument("--output", type=Path, required=True) + allocate.set_defaults(handler=command_allocate) + + validate = subparsers.add_parser("validate", help="validate a manifest and probe bank") + validate.add_argument("--manifest", type=Path, required=True) + validate.set_defaults(handler=command_validate) + + session = subparsers.add_parser("session", help="emit a public condition packet") + session.add_argument("--manifest", type=Path, required=True) + session.add_argument("--session-id", required=True) + session.set_defaults(handler=command_session) + + probe = subparsers.add_parser("probe", help="emit one held-out public probe") + probe.add_argument("--manifest", type=Path, required=True) + probe.add_argument("--session-id", required=True) + probe.add_argument("--phase", choices=("immediate", "late"), required=True) + probe.add_argument("--index", type=int, required=True) + probe.add_argument( + "--schema-only", + action="store_true", + help="repeat only the schema for the single permitted repair", + ) + probe.set_defaults(handler=command_probe) + + distractor = subparsers.add_parser("distractor", help="emit one frozen distractor") + distractor.add_argument("--manifest", type=Path, required=True) + distractor.add_argument("--session-id", required=True) + distractor.add_argument("--index", type=int, required=True) + distractor.set_defaults(handler=command_distractor) + + redact = subparsers.add_parser("redact", help="sanitize a raw JSONL event ledger") + redact.add_argument("--input", type=Path, required=True) + redact.add_argument("--output", type=Path, required=True) + redact.add_argument( + "--replace", + action="append", + default=[], + metavar="HOST_VALUE", + help="replace a known host identifier; repeat as needed", + ) + redact.set_defaults(handler=command_redact) + + analyze = subparsers.add_parser("analyze", help="score one complete frozen cohort") + analyze.add_argument("--manifest", type=Path, required=True) + analyze.add_argument("--responses", type=Path, required=True) + analyze.add_argument("--output", type=Path, required=True) + analyze.set_defaults(handler=command_analyze) + return parser + + +def main(argv: list[str] | None = None) -> int: + """Run one deterministic study command.""" + parser = build_parser() + args = parser.parse_args(argv) + try: + args.handler(args) + except StudyError as error: + print(f"error: {error}", file=sys.stderr) + return 2 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/verify.ps1 b/scripts/verify.ps1 index f216f095..d4ca2237 100644 --- a/scripts/verify.ps1 +++ b/scripts/verify.ps1 @@ -43,6 +43,7 @@ Step "documentation" { } } Step "tests" { cargo test --workspace --all-targets --locked } +Step "understanding study runner" { python scripts/test-understanding-study.py } Step "build" { cargo build --workspace --locked } if ($null -ne (Get-Command cargo-llvm-cov -ErrorAction SilentlyContinue)) { diff --git a/scripts/verify.sh b/scripts/verify.sh index 1458e616..6b0a2f61 100644 --- a/scripts/verify.sh +++ b/scripts/verify.sh @@ -18,6 +18,8 @@ RUSTDOCFLAGS="-D warnings" cargo doc --workspace --no-deps --locked RUSTDOCFLAGS="-D warnings" cargo test --workspace --doc --locked echo "== tests ==" cargo test --workspace --all-targets --locked +echo "== understanding study runner ==" +python3 scripts/test-understanding-study.py echo "== build ==" cargo build --workspace --locked