From 68768a7a8322c2e19f6e792934f3ec3802e8b325 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 13 Jun 2026 23:16:32 +0800 Subject: [PATCH 1/8] feat: add corpus phase4-status command for trigger monitoring Add read-only corpus phase4-status subcommand that reports all 8 Phase 4-3 trigger conditions with current values, thresholds, met/not-met status, affected theory candidates, and next actions. All metrics scoped correctly: Trigger 7 (runtime balance) counts only native strict sessions. Triggers 3/4 count tagged intervention lane sessions. Output format: table with current value, threshold, and status, followed by summary and per-trigger action guidance. Co-Authored-By: Claude Opus 4.7 --- causetrace/cli.py | 198 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 198 insertions(+) diff --git a/causetrace/cli.py b/causetrace/cli.py index dfa5f4b..2b67e97 100644 --- a/causetrace/cli.py +++ b/causetrace/cli.py @@ -296,6 +296,7 @@ def cli(argv: list[str] | None = None) -> None: p_cr_groups.add_argument("--label", default="task_type", help="Metadata label to group by") p_cr_health = p_cr_sub.add_parser("health", help="Show corpus milestone gaps and coverage") p_cr_health.add_argument("--output", "-o", help="Write report to file") + p_cr_phase4 = p_cr_sub.add_parser("phase4-status", help="Show Phase 4-3 trigger status for evidence refresh gating") p_cr_origins = p_cr_sub.add_parser("origins", help="Show corpus source-origin coverage for Phase 3C planning") p_cr_origins.add_argument("--output", "-o", help="Write report to file") p_cr_readiness = p_cr_sub.add_parser("readiness", help="Show phase-3 research readiness and blockers") @@ -1165,6 +1166,199 @@ def _print_lane_counts() -> None: print(f"{lane:45s} {lanes[lane]:8d} {lane_events[lane]:10d}") +def _print_phase4_trigger_status() -> None: + """Print Phase 4-3 trigger status for evidence refresh gating. + + Read-only. Reports all 8 triggers with current values, thresholds, + met/not-met status, affected candidates, and next actions. + """ + import json + from collections import Counter + + meta_dir = Path.home() / ".causetrace" / "metadata" + data_dir = Path.home() / ".causetrace" / "data" + + # Gather corpus metrics + total_meta = 0 + native_sessions = 0 + native_strict = 0 + sp_sessions = 0 + sp_runtimes: set[str] = set() + routed_sessions = 0 + controlled_sessions = 0 + failure_count = 0 + near_failure_count = 0 + safety_annotated = 0 + runtime_counts: Counter = Counter() # native strict only (Trigger 7) + native_strict_runtimes_with_5 = 0 + unlabeled = 0 + + for f in meta_dir.iterdir(): + if not f.name.endswith(".json") or f.name.endswith(".provenance.json"): + continue + total_meta += 1 + with open(f) as fh: + meta = json.load(fh) + ts = meta.get("task_source", "") + do = meta.get("data_origin", "") + rt = meta.get("runtime", "") + + # Lane classification + if ts in ("superpowers_workflow_intervention",): + sp_sessions += 1 + if rt: + sp_runtimes.add(rt) + elif ts == "routed_prompt_intervention": + routed_sessions += 1 + elif ts == "controlled_prompt_morphology": + controlled_sessions += 1 + elif ts == "real_work" or do in ("native", "real_work", "direct_prompt_native"): + native_sessions += 1 + # Check for strict native + tags = meta.get("causetrace_tags", []) + il = meta.get("intervention_lane", "") + is_strict = bool(not tags and not il) + if is_strict: + native_strict += 1 + if rt: + runtime_counts[rt] += 1 + if meta.get("success") is False: + failure_count += 1 + if meta.get("human_intervention") is True: + near_failure_count += 1 + if meta.get("causetrace_tags") or meta.get("intervention_evidence_source"): + safety_annotated += 1 + else: + unlabeled += 1 + + # Count runtimes with >=5 native strict sessions + for c in runtime_counts.values(): + if c >= 5: + native_strict_runtimes_with_5 += 1 + + # Count data sessions + data_sessions = sum(1 for f in data_dir.iterdir() if f.name.endswith(".jsonl")) + + trigger_results: list[dict] = [] + + # Trigger 1: Native strict growth + t1_current = native_strict + t1_threshold = 150 + t1_met = t1_current >= t1_threshold + trigger_results.append({ + "id": "1", "name": "Native strict growth", + "current": str(t1_current), "threshold": str(t1_threshold), + "met": t1_met, + "affected": "T-RM-001, T-RM-002, T-RM-003", + "action": "Re-run topology distribution against expanded native strict set." + }) + + # Trigger 2: Failure/near-failure threshold + t2_current = f"failure={failure_count}, near-failure={near_failure_count}" + t2_met = failure_count >= 10 and near_failure_count >= 10 + trigger_results.append({ + "id": "2", "name": "Failure/near-failure threshold", + "current": t2_current, "threshold": "failure>=10, near>=10", + "met": t2_met, + "affected": "T-FM-001, T-SC-004, T-SC-005", + "action": "Reopen Tier 2 failure/intervention validation." + }) + + # Trigger 3: Routed gate + t3_met = routed_sessions >= 5 + trigger_results.append({ + "id": "3", "name": "Routed-prompt gate", + "current": str(routed_sessions), "threshold": ">=5 tagged", + "met": t3_met, + "affected": "T-RP-001", + "action": "Open routed lane for basic characterization." + }) + + # Trigger 4: Controlled prompt expansion + t4_met = controlled_sessions >= 10 + trigger_results.append({ + "id": "4", "name": "Controlled prompt expansion", + "current": str(controlled_sessions), "threshold": ">=10 with variant tags", + "met": t4_met, + "affected": "T-PM-001", + "action": "Characterize per-variant topology." + }) + + # Trigger 5: SP lane growth + t5_current = f"{sp_sessions} sessions, {len(sp_runtimes)} runtimes" + t5_met = sp_sessions >= 15 and len(sp_runtimes) >= 2 + trigger_results.append({ + "id": "5", "name": "Superpowers lane growth", + "current": t5_current, "threshold": ">=15 sessions, >=2 runtimes", + "met": t5_met, + "affected": "T-WI-001, T-SC-003", + "action": "Re-run SP lane event density distribution." + }) + + # Trigger 6: Safety-control annotation + t6_met = safety_annotated >= 10 + trigger_results.append({ + "id": "6", "name": "Safety-control annotation", + "current": str(safety_annotated), "threshold": ">=10 annotated sessions", + "met": t6_met, + "affected": "T-SC-001 through T-SC-005", + "action": "First safety-control morphology baseline." + }) + + # Trigger 7: Runtime balance (native strict lane only) + dominant_pct = max(runtime_counts.values()) / max(sum(runtime_counts.values()), 1) * 100 if runtime_counts else 100 + t7_current = f"top runtime={dominant_pct:.0f}%, runtimes with >=5: {native_strict_runtimes_with_5}" + t7_met = dominant_pct < 60 and native_strict_runtimes_with_5 >= 4 + trigger_results.append({ + "id": "7", "name": "Runtime balance", + "current": t7_current, "threshold": "<60% single runtime, >=4 runtimes with >=5 sessions", + "met": t7_met, + "affected": "T-RM-001, T-RM-002, T-RM-003", + "action": "Test per-runtime topology distribution." + }) + + # Trigger 8: Metadata density + labeled = total_meta - unlabeled + pct_labeled = labeled / max(total_meta, 1) * 100 + t8_current = f"{pct_labeled:.1f}% labeled ({labeled}/{total_meta})" + t8_met = pct_labeled >= 40 + trigger_results.append({ + "id": "8", "name": "Metadata density", + "current": t8_current, "threshold": ">=40% labeled, >=80% lane coverage", + "met": t8_met, + "affected": "All (indirect)", + "action": "Re-run lane-count with reduced unlabeled population." + }) + + met_count = sum(1 for t in trigger_results if t["met"]) + + # Print report + print("Phase 4-3 Trigger Status") + print(f"Corpus: {total_meta} metadata sessions, {data_sessions} data sessions") + print(f"Phase 4: frozen (4-1/4-2 complete, 4-3 trigger-gated)") + print(f"Phase 5: not open") + print() + print(f"{'#':>3s} {'Trigger':40s} {'Current':>22s} {'Threshold':30s} {'Met':5s}") + print("-" * 107) + for t in trigger_results: + flag = " YES" if t["met"] else " no" + print(f"{t['id']:>3s} {t['name']:40s} {t['current']:>22s} {t['threshold']:30s} {flag:5s}") + print("-" * 107) + print(f"\nTriggers met: {met_count}/8") + if met_count == 0: + print("Phase 4-3 remains closed. No evidence refresh trigger has fired.") + else: + print("Phase 4-3 should reopen for affected candidates only.") + print() + print("Affected candidates per trigger:") + for t in trigger_results: + if t["met"]: + print(f" Trigger {t['id']}: {t['affected']}") + print(f" → {t['action']}") + print() + print("Next check: opportunistic — run after significant corpus growth.") + + def _print_gate_status() -> None: """Print Phase 3E parser detection gate readiness table.""" import json @@ -1252,6 +1446,10 @@ def _handle_corpus(store, args) -> None: print(report) return + if args.corpus_command == "phase4-status": + _print_phase4_trigger_status() + return + if args.corpus_command == "origins": report = generate_corpus_origin_report(store) if args.output: From fa315d6948ecacfa167fc4dca7e6de6e5b2c26fe Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 13 Jun 2026 23:21:15 +0800 Subject: [PATCH 2/8] feat: add corpus classify-unlabeled --dry-run command MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Conservative classification using only explicit metadata fields (data_origin, task_source, intervention_lane, causetrace_tags). High confidence: data_origin=native + task_source=real_work. Medium confidence: data_origin=native with no task_source. Dry-run only — no metadata writes. Tracks skip reasons for all unmatched sessions. Co-Authored-By: Claude Opus 4.7 --- causetrace/cli.py | 130 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) diff --git a/causetrace/cli.py b/causetrace/cli.py index 2b67e97..486e5c2 100644 --- a/causetrace/cli.py +++ b/causetrace/cli.py @@ -297,6 +297,11 @@ def cli(argv: list[str] | None = None) -> None: p_cr_health = p_cr_sub.add_parser("health", help="Show corpus milestone gaps and coverage") p_cr_health.add_argument("--output", "-o", help="Write report to file") p_cr_phase4 = p_cr_sub.add_parser("phase4-status", help="Show Phase 4-3 trigger status for evidence refresh gating") + p_cr_classify = p_cr_sub.add_parser("classify-unlabeled", help="Propose lane classification for unlabeled sessions (dry-run)") + p_cr_classify.add_argument("--dry-run", action="store_true", default=True, help="Proposal only, no metadata writes (default)") + p_cr_classify.add_argument("--limit", type=int, default=0, help="Limit to N sessions (0 = all)") + p_cr_classify.add_argument("--min-confidence", choices=["high", "medium"], default="high", + help="Minimum confidence threshold for proposals (default: high)") p_cr_origins = p_cr_sub.add_parser("origins", help="Show corpus source-origin coverage for Phase 3C planning") p_cr_origins.add_argument("--output", "-o", help="Write report to file") p_cr_readiness = p_cr_sub.add_parser("readiness", help="Show phase-3 research readiness and blockers") @@ -1359,6 +1364,127 @@ def _print_phase4_trigger_status() -> None: print("Next check: opportunistic — run after significant corpus growth.") +def _print_classify_unlabeled(limit: int = 0, min_confidence: str = "high") -> None: + """Propose lane classification for unlabeled metadata sessions. + + Dry-run only. No metadata writes. Only high-confidence explicit rules. + Does not infer intervention lanes. Does not use prompt length or style. + """ + import json + + meta_dir = Path.home() / ".causetrace" / "metadata" + + proposals: list[dict] = [] + skipped_reasons: dict[str, int] = {} + total_unlabeled = 0 + + for f in sorted(meta_dir.iterdir()): + if not f.name.endswith(".json") or f.name.endswith(".provenance.json"): + continue + with open(f) as fh: + meta = json.load(fh) + sid = f.stem + ts = meta.get("task_source", "") + do = meta.get("data_origin", "") + il = meta.get("intervention_lane", "") + tags = meta.get("causetrace_tags", []) + rt = meta.get("runtime", "") + + # Already classified via explicit lane assignment + if ts in ("routed_prompt_intervention", "superpowers_workflow_intervention", + "controlled_prompt_morphology"): + continue + if il: + continue + + total_unlabeled += 1 + + if limit and len(proposals) >= limit: + continue + + # Rule: external trajectory + if do == "external_trajectory": + proposals.append({ + "session_id": sid, "proposed_lane": "external_trajectory", + "evidence": f"data_origin={do}", "confidence": "high", + }) + continue + + # Rule: controlled benchmark + if do == "controlled_benchmark": + proposals.append({ + "session_id": sid, "proposed_lane": "controlled_prompt_morphology", + "evidence": f"data_origin={do}", "confidence": "high", + }) + continue + + # Rule: native direct prompt (high confidence) + # Requires: data_origin=native + task_source=real_work + no intervention markers + if do == "native" and ts == "real_work": + if tags or il: + skipped_reasons["has intervention markers despite native+real_work"] = \ + skipped_reasons.get("has intervention markers despite native+real_work", 0) + 1 + continue + proposals.append({ + "session_id": sid, "proposed_lane": "direct_prompt_native", + "evidence": f"data_origin={do}, task_source={ts}, no intervention markers", + "confidence": "high", + }) + continue + + # Medium confidence: data_origin=native with no task_source + if min_confidence == "medium" and do == "native" and not ts: + if tags or il: + skipped_reasons["native data_origin but has intervention markers"] = \ + skipped_reasons.get("native data_origin but has intervention markers", 0) + 1 + continue + proposals.append({ + "session_id": sid, "proposed_lane": "direct_prompt_native", + "evidence": f"data_origin={do}, no task_source, no intervention markers", + "confidence": "medium", + }) + continue + + # Count skip reasons + reason = f"no matching rule (do={do}, ts={ts or 'unset'}, rt={rt or 'unset'})" + skipped_reasons[reason] = skipped_reasons.get(reason, 0) + 1 + + # Print report + print(f"classify-unlabeled --dry-run (confidence >= {min_confidence})") + print(f" Total unlabeled: {total_unlabeled}") + print(f" Proposed: {len(proposals)}") + print(f" Skipped: {total_unlabeled - len(proposals)}") + print() + print(f"{'Confidence':12s} {'Proposed Lane':40s} {'Count':>6s}") + print("-" * 62) + from collections import Counter + conf_counter: Counter = Counter() + lane_counter: Counter = Counter() + for p in proposals: + conf_counter[p["confidence"]] += 1 + lane_counter[p["proposed_lane"]] += 1 + for conf in ["high", "medium"]: + for lane in sorted(lane_counter): + count = sum(1 for p in proposals if p["confidence"] == conf and p["proposed_lane"] == lane) + if count: + print(f"{conf:12s} {lane:40s} {count:>6d}") + print() + if skipped_reasons: + print("Top skip reasons:") + for reason, count in sorted(skipped_reasons.items(), key=lambda x: -x[1])[:5]: + print(f" [{count:>4d}] {reason[:80]}") + print() + if proposals: + print("Sample proposals:") + for p in proposals[:10]: + sid_short = p["session_id"][:40] + print(f" {sid_short:40s} → {p['proposed_lane']:35s} [{p['confidence']}]") + if len(proposals) > 10: + print(f" ... and {len(proposals) - 10} more") + print() + print("No metadata written. Use --apply-confirmed (future) to apply high-confidence proposals.") + + def _print_gate_status() -> None: """Print Phase 3E parser detection gate readiness table.""" import json @@ -1450,6 +1576,10 @@ def _handle_corpus(store, args) -> None: _print_phase4_trigger_status() return + if args.corpus_command == "classify-unlabeled": + _print_classify_unlabeled(args.limit, args.min_confidence) + return + if args.corpus_command == "origins": report = generate_corpus_origin_report(store) if args.output: From 0ea6df0fc3553ed7d14b4d7d046d2a0840632ec4 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 13 Jun 2026 23:21:42 +0800 Subject: [PATCH 3/8] docs: update corpus snapshot counts (1625 sessions, 132671 events) Co-Authored-By: Claude Opus 4.7 --- docs/research/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/research/README.md b/docs/research/README.md index 0a687e2..32c67d2 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -17,9 +17,9 @@ This directory groups the research tracks and branch studies that sit alongside ## Current Corpus Snapshot -- data sessions: `1517` +- data sessions: `1625` - metadata sessions: `992` -- events: `131,952` +- events: `132,671` - strict research-grade sessions: `157` - native strict sessions: `100` - agent field coverage: `100%` (inline) From 19193942ee6c790eae5dd20c906413fc02a85d9a Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 14 Jun 2026 10:00:32 +0800 Subject: [PATCH 4/8] feat: add --apply-confirmed to corpus classify-unlabeled Writes intervention_lane=direct_prompt_native for high-confidence proposals only (data_origin=native + task_source=real_work + no intervention markers). Medium-confidence proposals are never applied. Adds provenance: classified_from_explicit_metadata. Fixes native_strict counting in phase4-status to treat direct_prompt_native as the native baseline lane. Before: 12/992 classified (1.2%) After: 113/992 classified (11.4%) 879 unknown sessions remain unmodified. Co-Authored-By: Claude Opus 4.7 --- causetrace/cli.py | 91 ++++++++++++---- causetrace/metadata.py | 1 + tests/test_metadata_corpus_report.py | 149 ++++++++++++++++++++++++++- 3 files changed, 218 insertions(+), 23 deletions(-) diff --git a/causetrace/cli.py b/causetrace/cli.py index 486e5c2..1c8f9d6 100644 --- a/causetrace/cli.py +++ b/causetrace/cli.py @@ -297,8 +297,10 @@ def cli(argv: list[str] | None = None) -> None: p_cr_health = p_cr_sub.add_parser("health", help="Show corpus milestone gaps and coverage") p_cr_health.add_argument("--output", "-o", help="Write report to file") p_cr_phase4 = p_cr_sub.add_parser("phase4-status", help="Show Phase 4-3 trigger status for evidence refresh gating") - p_cr_classify = p_cr_sub.add_parser("classify-unlabeled", help="Propose lane classification for unlabeled sessions (dry-run)") + p_cr_classify = p_cr_sub.add_parser("classify-unlabeled", help="Propose lane classification for unlabeled sessions") p_cr_classify.add_argument("--dry-run", action="store_true", default=True, help="Proposal only, no metadata writes (default)") + p_cr_classify.add_argument("--apply-confirmed", action="store_true", default=False, + help="Apply high-confidence proposals to metadata sidecars") p_cr_classify.add_argument("--limit", type=int, default=0, help="Limit to N sessions (0 = all)") p_cr_classify.add_argument("--min-confidence", choices=["high", "medium"], default="high", help="Minimum confidence threshold for proposals (default: high)") @@ -1219,10 +1221,13 @@ def _print_phase4_trigger_status() -> None: controlled_sessions += 1 elif ts == "real_work" or do in ("native", "real_work", "direct_prompt_native"): native_sessions += 1 - # Check for strict native + # Check for strict native (direct_prompt_native is the native baseline lane) tags = meta.get("causetrace_tags", []) il = meta.get("intervention_lane", "") - is_strict = bool(not tags and not il) + is_intervention_lane = il in ("routed_prompt_intervention", + "superpowers_workflow_intervention", + "controlled_prompt_morphology") + is_strict = bool(not tags and not is_intervention_lane) if is_strict: native_strict += 1 if rt: @@ -1364,23 +1369,30 @@ def _print_phase4_trigger_status() -> None: print("Next check: opportunistic — run after significant corpus growth.") -def _print_classify_unlabeled(limit: int = 0, min_confidence: str = "high") -> None: +def _print_classify_unlabeled(limit: int = 0, min_confidence: str = "high", + apply_confirmed: bool = False) -> None: """Propose lane classification for unlabeled metadata sessions. - Dry-run only. No metadata writes. Only high-confidence explicit rules. - Does not infer intervention lanes. Does not use prompt length or style. + Dry-run by default. --apply-confirmed writes high-confidence proposals + to metadata sidecars. Does not infer intervention lanes. Does not use + prompt length, style, tool patterns, or runtime-only rules. """ import json - meta_dir = Path.home() / ".causetrace" / "metadata" + from causetrace.metadata import METADATA_DIR, merge_metadata, merge_metadata_provenance + + meta_dir = Path(METADATA_DIR) proposals: list[dict] = [] skipped_reasons: dict[str, int] = {} total_unlabeled = 0 + total_existing_lane = 0 + total_scanned = 0 for f in sorted(meta_dir.iterdir()): if not f.name.endswith(".json") or f.name.endswith(".provenance.json"): continue + total_scanned += 1 with open(f) as fh: meta = json.load(fh) sid = f.stem @@ -1393,8 +1405,10 @@ def _print_classify_unlabeled(limit: int = 0, min_confidence: str = "high") -> N # Already classified via explicit lane assignment if ts in ("routed_prompt_intervention", "superpowers_workflow_intervention", "controlled_prompt_morphology"): + total_existing_lane += 1 continue if il: + total_existing_lane += 1 continue total_unlabeled += 1 @@ -1449,40 +1463,73 @@ def _print_classify_unlabeled(limit: int = 0, min_confidence: str = "high") -> N reason = f"no matching rule (do={do}, ts={ts or 'unset'}, rt={rt or 'unset'})" skipped_reasons[reason] = skipped_reasons.get(reason, 0) + 1 + # Apply confirmed writes (high-confidence only) + applied_count = 0 + if apply_confirmed: + for p in proposals: + if p["confidence"] != "high": + continue + sid = p["session_id"] + merge_metadata(sid, {"intervention_lane": p["proposed_lane"]}) + merge_metadata_provenance(sid, { + "intervention_lane": "classified_from_explicit_metadata" + }) + applied_count += 1 + # Print report - print(f"classify-unlabeled --dry-run (confidence >= {min_confidence})") - print(f" Total unlabeled: {total_unlabeled}") - print(f" Proposed: {len(proposals)}") - print(f" Skipped: {total_unlabeled - len(proposals)}") + mode = "--apply-confirmed" if apply_confirmed else "--dry-run" + print(f"classify-unlabeled {mode} (confidence >= {min_confidence})") + print(f" Total scanned: {total_scanned}") + print(f" Total unlabeled (no intervention_lane): {total_unlabeled}") + print(f" Existing lane (already classified): {total_existing_lane}") + if apply_confirmed: + print(f" Applied (written to metadata): {applied_count}") + print(f" Proposed (not applied): {len(proposals) - applied_count if apply_confirmed else len(proposals)}") + print(f" Skipped (unknown/no rule): {total_unlabeled - len(proposals)}") print() print(f"{'Confidence':12s} {'Proposed Lane':40s} {'Count':>6s}") print("-" * 62) from collections import Counter - conf_counter: Counter = Counter() lane_counter: Counter = Counter() for p in proposals: - conf_counter[p["confidence"]] += 1 lane_counter[p["proposed_lane"]] += 1 for conf in ["high", "medium"]: for lane in sorted(lane_counter): count = sum(1 for p in proposals if p["confidence"] == conf and p["proposed_lane"] == lane) if count: - print(f"{conf:12s} {lane:40s} {count:>6d}") + applied_mark = " (applied)" if apply_confirmed and conf == "high" else "" + print(f"{conf:12s} {lane:40s} {count:>6d}{applied_mark}") print() - if skipped_reasons: - print("Top skip reasons:") - for reason, count in sorted(skipped_reasons.items(), key=lambda x: -x[1])[:5]: - print(f" [{count:>4d}] {reason[:80]}") - print() - if proposals: + + if not apply_confirmed and proposals: print("Sample proposals:") for p in proposals[:10]: sid_short = p["session_id"][:40] print(f" {sid_short:40s} → {p['proposed_lane']:35s} [{p['confidence']}]") if len(proposals) > 10: print(f" ... and {len(proposals) - 10} more") + print() + + if skipped_reasons: + print("Top skip reasons:") + for reason, count in sorted(skipped_reasons.items(), key=lambda x: -x[1])[:5]: + print(f" [{count:>4d}] {reason[:80]}") + print() + + if not apply_confirmed: + print("No metadata written. Use --apply-confirmed to apply high-confidence proposals.") + return + + # After-apply summary + unlabeled_after = total_unlabeled - applied_count + classified_after = total_existing_lane + applied_count + coverage_before = (total_existing_lane / total_scanned * 100) if total_scanned else 0 + coverage_after = (classified_after / total_scanned * 100) if total_scanned else 0 + print(f"Before: {total_existing_lane}/{total_scanned} classified ({coverage_before:.1f}%)") + print(f"After: {classified_after}/{total_scanned} classified ({coverage_after:.1f}%)") + print(f"Unlabeled remaining: {unlabeled_after}") print() - print("No metadata written. Use --apply-confirmed (future) to apply high-confidence proposals.") + print("Applied entries have provenance: intervention_lane=classified_from_explicit_metadata") def _print_gate_status() -> None: @@ -1577,7 +1624,7 @@ def _handle_corpus(store, args) -> None: return if args.corpus_command == "classify-unlabeled": - _print_classify_unlabeled(args.limit, args.min_confidence) + _print_classify_unlabeled(args.limit, args.min_confidence, args.apply_confirmed) return if args.corpus_command == "origins": diff --git a/causetrace/metadata.py b/causetrace/metadata.py index f054d0b..4f7cd84 100644 --- a/causetrace/metadata.py +++ b/causetrace/metadata.py @@ -24,6 +24,7 @@ "annotation", "materialized", "inferred_from_runtime_adapter", + "classified_from_explicit_metadata", "unknown", } diff --git a/tests/test_metadata_corpus_report.py b/tests/test_metadata_corpus_report.py index 9e221f4..ceeb3a3 100644 --- a/tests/test_metadata_corpus_report.py +++ b/tests/test_metadata_corpus_report.py @@ -12,7 +12,7 @@ from causetrace.annotation import save_annotation from causetrace.core import JSONStore, ToolEvent from causetrace.corpus import Phase3ReadinessRequirements, assess_phase3_readiness, benchmark_corpus, build_corpus_facts, compare_benchmark_manifests, export_dataset, list_corpus_records, materialize_corpus_metadata, snapshot_corpus, taxonomy_corpus, verify_benchmark_manifest, verify_snapshot -from causetrace.metadata import load_metadata, load_metadata_provenance, merge_metadata +from causetrace.metadata import load_metadata, load_metadata_provenance, merge_metadata, merge_metadata_provenance from causetrace.report import generate_corpus_health_report, generate_corpus_origin_report, generate_phase3_readiness_report from causetrace.corpus import summarize_corpus_health @@ -708,3 +708,150 @@ def test_corpus_verify_cli(monkeypatch, tmp_path): assert result.returncode == 0 assert "Snapshot:" in result.stdout assert "OK: True" in result.stdout + + +# ── classify-unlabeled tests ────────────────────────────────────────── + + +def test_classify_unlabeled_dry_run_no_write(monkeypatch, tmp_path): + """classify-unlabeled --dry-run must not write to metadata sidecars.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + # Write a session that qualifies for high-confidence classification + sid = "test-native-001" + (mdir / f"{sid}.json").write_text(json.dumps({ + "session_id": sid, + "data_origin": "native", + "task_source": "real_work", + "runtime": "anthropic", + })) + + _print_classify_unlabeled(min_confidence="high", apply_confirmed=False) + + # Metadata must NOT be modified + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert "intervention_lane" not in meta + + +def test_classify_unlabeled_apply_confirmed_writes_high_confidence(monkeypatch, tmp_path): + """--apply-confirmed writes intervention_lane for high-confidence proposals.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + sid = "test-native-002" + (mdir / f"{sid}.json").write_text(json.dumps({ + "session_id": sid, + "data_origin": "native", + "task_source": "real_work", + "runtime": "anthropic", + })) + + _print_classify_unlabeled(min_confidence="high", apply_confirmed=True) + + # Metadata must now have intervention_lane + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert meta.get("intervention_lane") == "direct_prompt_native" + + # Provenance must be written + prov = json.loads((mdir / f"{sid}.provenance.json").read_text()) + assert prov.get("intervention_lane") == "classified_from_explicit_metadata" + + +def test_classify_unlabeled_unknown_unchanged(monkeypatch, tmp_path): + """data_origin=unknown sessions must remain unmodified.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + sid = "test-unknown-001" + original = {"session_id": sid, "data_origin": "unknown", "runtime": "codex"} + (mdir / f"{sid}.json").write_text(json.dumps(original)) + + _print_classify_unlabeled(min_confidence="high", apply_confirmed=True) + + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert "intervention_lane" not in meta + assert meta.get("data_origin") == "unknown" + + +def test_classify_unlabeled_preserves_existing_lane(monkeypatch, tmp_path): + """Existing intervention_lane must not be overwritten.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + sid = "test-sp-001" + (mdir / f"{sid}.json").write_text(json.dumps({ + "session_id": sid, + "data_origin": "native", + "task_source": "superpowers_workflow_intervention", + "intervention_lane": "superpowers_workflow_intervention", + "runtime": "claude-code", + })) + + _print_classify_unlabeled(min_confidence="high", apply_confirmed=True) + + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert meta.get("intervention_lane") == "superpowers_workflow_intervention" + + +def test_classify_unlabeled_medium_not_applied(monkeypatch, tmp_path): + """Medium-confidence proposals are never applied even with --apply-confirmed.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + sid = "test-medium-001" + (mdir / f"{sid}.json").write_text(json.dumps({ + "session_id": sid, + "data_origin": "native", + "runtime": "anthropic", + })) + + _print_classify_unlabeled(min_confidence="medium", apply_confirmed=True) + + # Medium confidence => not applied + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert "intervention_lane" not in meta + + +def test_classify_unlabeled_high_confidence_not_applied_if_tags(monkeypatch, tmp_path): + """Sessions with causetrace_tags are skipped even if native+real_work.""" + import causetrace.metadata as metadata + from causetrace.cli import _print_classify_unlabeled + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + mdir = Path(metadata.METADATA_DIR) + mdir.mkdir(parents=True) + + sid = "test-tagged-001" + (mdir / f"{sid}.json").write_text(json.dumps({ + "session_id": sid, + "data_origin": "native", + "task_source": "real_work", + "causetrace_tags": ["superpowers-workflow"], + "runtime": "claude-code", + })) + + _print_classify_unlabeled(min_confidence="high", apply_confirmed=True) + + meta = json.loads((mdir / f"{sid}.json").read_text()) + assert "intervention_lane" not in meta From d6f2fad1aa9834f06ba07c05a75c557ba4af7f74 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 14 Jun 2026 10:07:07 +0800 Subject: [PATCH 5/8] docs: add metadata capture hardening baseline Documents current state after A2 classification pass: - 108/992 sessions with explicit intervention_lane (10.9%) - 101 direct_prompt_native via classified_from_explicit_metadata - 879 data_origin=unknown sessions cannot be auto-classified Defines upstream capture requirements (P0-P5), source-specific defaults, capture points, and explicit non-goals. Trigger 8 should improve through future capture quality, not aggressive retroactive inference. Co-Authored-By: Claude Opus 4.7 --- docs/research/README.md | 4 + .../metadata_capture_hardening_v0.2.5.md | 143 ++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 docs/research/corpus/metadata_capture_hardening_v0.2.5.md diff --git a/docs/research/README.md b/docs/research/README.md index 32c67d2..70a95c1 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -54,6 +54,10 @@ Phase 4 must not enter: - Phase 5 (evaluation / diagnostics) - Jailbreak reproduction or attack research +## Corpus Infrastructure + +- [Metadata Capture Hardening v0.2.5](corpus/metadata_capture_hardening_v0.2.5.md) — Upstream capture requirements, source-specific defaults, current baseline (108/992 labeled, 10.9%) + ## Cross-project Branch Studies - [Cross-project Prompt Morphology Study](branches/cross_project_prompt_morphology/README.md) diff --git a/docs/research/corpus/metadata_capture_hardening_v0.2.5.md b/docs/research/corpus/metadata_capture_hardening_v0.2.5.md new file mode 100644 index 0000000..29f4066 --- /dev/null +++ b/docs/research/corpus/metadata_capture_hardening_v0.2.5.md @@ -0,0 +1,143 @@ +# Metadata Capture Hardening v0.2.5 + +Improve future metadata capture quality so sessions enter the corpus with explicit `data_origin`, `task_source`, and `intervention_lane` — reducing reliance on retroactive classification. + +## Current Baseline (2026-06-14) + +| Metric | Value | +|--------|-------| +| Total metadata sessions | 992 | +| Sessions with explicit `intervention_lane` | 108 (10.9%) | +| `direct_prompt_native` | 101 (`classified_from_explicit_metadata`) | +| `superpowers_workflow_intervention` | 7 | +| Sessions classified via `task_source` only | 5 (SP: ts field, no il) | +| `data_origin` distribution | native=102, unknown=877, unset=10, controlled_benchmark=3 | +| `task_source` distribution | real_work=102, unset=815, demo=62, SP=8, controlled=3 | +| Phase 4-3 triggers | 0/8 | + +### How we got here + +1. Phase 3E-3 established explicit tag formats and enrichment auto-detection +2. A: `corpus classify-unlabeled --dry-run` identified 101 high-confidence native sessions +3. A2: `--apply-confirmed` wrote `intervention_lane=direct_prompt_native` with provenance `classified_from_explicit_metadata` +4. 879 sessions remain unlabeled — all have `data_origin=unknown` + +**Key finding:** The "safe auto-classification surface" is exhausted. Remaining unknown sessions cannot be automatically classified without heuristic inference that would risk mislabeling. + +## Upstream Capture Requirements + +For every session entering the corpus from this point, capture or annotate: + +### Required fields (priority order) + +| Priority | Field | Rationale | +|----------|-------|-----------| +| P0 | `data_origin` | Determines lane eligibility. Must be explicit. | +| P1 | `task_source` | Disambiguates real_work from demo/benchmark/external. | +| P2 | `intervention_lane` | Explicit lane assignment when known. | +| P3 | `causetrace_tags` | Machine-readable intervention evidence. | +| P4 | `task_type` | Structural classification (bug_fix, feature_add, etc.). | +| P5 | `success` | Session outcome. | + +### Validation rule + +A session MUST NOT be classified into a lane based on `runtime`, `model`, tool usage patterns, prompt length, or prompt style alone. Classification requires: + +- Explicit `data_origin` (native, controlled_benchmark, external_trajectory) + OR +- Explicit `intervention_lane` set by the capture source + OR +- Explicit `causetrace_tags` with recognized intervention markers + +## Source-Specific Defaults + +These defaults apply only when the capture source can confirm them. They are NOT inferred from session content. + +### Manual real project run + +```json +{ + "data_origin": "native", + "task_source": "real_work" +} +``` + +No `intervention_lane` needed — these sessions are the baseline. They will be classified as `direct_prompt_native` when both fields are confirmed. + +### Prompt morphology pilot + +```json +{ + "data_origin": "controlled_benchmark", + "task_source": "prompt_morphology_pilot", + "intervention_lane": "controlled_prompt_morphology" +} +``` + +### Prompt-routing-skill routed task + +```json +{ + "intervention_lane": "routed_prompt_intervention", + "causetrace_tags": ["prompt-routing", "routed-prompt", "causetrace-prompt-posture"] +} +``` + +### Superpowers workflow + +```json +{ + "intervention_lane": "superpowers_workflow_intervention", + "causetrace_tags": ["superpowers-workflow", "workflow-intervention"] +} +``` + +The `causetrace_tags` block must be emitted in the first response or plan output when superpowers workflow structure is applied. This is the auditable evidence for parser detection (Phase 3E-3 enrichment recognition plan, Phase 2). + +### External trajectory + +```json +{ + "data_origin": "external_trajectory" +} +``` + +`task_source` should document the external source (e.g., `external_log_import`, `third_party_trace`). + +## Capture Points + +Where metadata should be written, in priority order: + +1. **At enrichment time** (`causetrace enrich* --save`): When a session is enriched from an agent project, the enrichment handler writes `data_origin=native` and `task_source=real_work` if the source is a real project session (not a demo). + +2. **At hook time** (Claude Code hooks, OpenCode hooks): The hook bridge can emit `causetrace_tags` when workflow markers are detected. + +3. **At manual annotation time** (`causetrace annotate`): Used for post-hoc high-confidence annotation of individual sessions. + +4. **At metadata-set time** (`causetrace metadata-set`): Used for bulk or targeted field updates with provenance recording. + +## Non-Goals + +- Do NOT infer `data_origin` from runtime, model, or tool patterns. +- Do NOT classify `data_origin=unknown` sessions automatically. +- Do NOT derive `task_type` or `success` heuristically. +- Do NOT use `Skill` tool usage alone as intervention evidence. +- Do NOT expand topology taxonomy. +- Do NOT modify theory candidate grades. +- Do NOT enter Phase 4-3 unless a trigger is actually met. + +## Future Trigger Note + +Trigger 8 (metadata density: >=40% labeled, >=80% lane coverage) should improve primarily through: + +1. Future sessions entering the corpus with explicit metadata from capture points +2. Manual annotation of high-value sessions +3. Upstream agent runtime improvements that emit metadata natively + +It should NOT improve through aggressive retroactive inference on the existing 879 `data_origin=unknown` sessions. + +The current gap (108/992 labeled → need ~397 for 40%) is approximately 289 sessions. With correct capture defaults at enrichment time, new real-work sessions will enter with `data_origin=native` and `task_source=real_work`, accumulating toward the threshold naturally. + +## Operating Rule + +**Mislabeling is more dangerous than low coverage.** A session labeled `direct_prompt_native` that was actually an intervention session pollutes the baseline and can invalidate lane comparisons. A session left unlabeled is honest about its evidence gap. From 4a435097acc375be0ca182200504fafccc83f403 Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 14 Jun 2026 10:11:08 +0800 Subject: [PATCH 6/8] docs: add future run metadata checklist Operational checklists for each capture type: - direct native real work - routed prompt task - superpowers workflow - controlled prompt morphology pilot - external trajectory - demo/test run Includes post-capture verification commands and explicit non-rules (runtime-only, Skill-only, prompt-style inference are not valid). Co-Authored-By: Claude Opus 4.7 --- docs/research/README.md | 1 + .../future_run_metadata_checklist_v0.2.5.md | 145 ++++++++++++++++++ 2 files changed, 146 insertions(+) create mode 100644 docs/research/corpus/future_run_metadata_checklist_v0.2.5.md diff --git a/docs/research/README.md b/docs/research/README.md index 70a95c1..25dbb32 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -57,6 +57,7 @@ Phase 4 must not enter: ## Corpus Infrastructure - [Metadata Capture Hardening v0.2.5](corpus/metadata_capture_hardening_v0.2.5.md) — Upstream capture requirements, source-specific defaults, current baseline (108/992 labeled, 10.9%) +- [Future Run Metadata Checklist v0.2.5](corpus/future_run_metadata_checklist_v0.2.5.md) — Per-capture-type metadata declaration checklists, post-capture verification, non-rules ## Cross-project Branch Studies diff --git a/docs/research/corpus/future_run_metadata_checklist_v0.2.5.md b/docs/research/corpus/future_run_metadata_checklist_v0.2.5.md new file mode 100644 index 0000000..b5f5aa0 --- /dev/null +++ b/docs/research/corpus/future_run_metadata_checklist_v0.2.5.md @@ -0,0 +1,145 @@ +# Future Run Metadata Checklist v0.2.5 + +Before or immediately after each causetrace session capture, declare the session's metadata. Do not leave sessions to be classified later — retroactive surface is exhausted. + +## Rule + +Every session entering the corpus MUST carry: + +- `data_origin` +- `task_source` +- `intervention_lane` (when known) +- `causetrace_tags` (when applicable) + +Unknown historical sessions remain unknown. No inference. + +## Checklists by Capture Type + +### Direct Native Real Work + +Use when running a real task directly — no routing, no workflow plugin, no experiment. + +```bash +# Before or after the session, run: +causetrace metadata-set \ + --data-origin native \ + --task-source real_work \ + --intervention-lane direct_prompt_native +``` + +Check: +- [ ] Not a demo or test run +- [ ] No prompt-routing-skill involved +- [ ] No superpowers workflow structure applied +- [ ] No controlled experiment variables + +### Routed Prompt Task + +Use when `prompt-routing-skill` selected the prompt posture. + +```bash +causetrace metadata-set \ + --data-origin native \ + --task-source real_work \ + --intervention-lane routed_prompt_intervention +``` + +Check: +- [ ] `prompt-routing-skill` was explicitly invoked before the task +- [ ] `causetrace_tags` were emitted by the routing skill (verify in session content) +- [ ] Session is tagged `prompt-routing`, `routed-prompt`, `causetrace-prompt-posture` + +### Superpowers Workflow + +Use when superpowers workflow structure (Skill → Plan → Execute → Verify) was applied. + +```bash +causetrace metadata-set \ + --data-origin native \ + --task-source real_work \ + --intervention-lane superpowers_workflow_intervention +``` + +Check: +- [ ] `causetrace_tags` block was emitted in first response or plan output +- [ ] Tags include `superpowers-workflow`, `workflow-intervention` +- [ ] `intervention_evidence_level: strong` (from the emitted block) +- [ ] `workflow_label` is set in the emitted block + +### Controlled Prompt Morphology Pilot + +Use when running a controlled prompt comparison (A/B/C variants). + +```bash +causetrace metadata-set \ + --data-origin controlled_benchmark \ + --task-source prompt_morphology_pilot \ + --intervention-lane controlled_prompt_morphology +``` + +Check: +- [ ] Prompt variant tag is recorded (A, B, or C) +- [ ] Benchmark protocol is followed (same task, different prompt posture) +- [ ] Session is explicitly tagged as pilot/controlled + +### External Trajectory + +Use when importing external data (third-party logs, external traces). + +```bash +causetrace metadata-set \ + --data-origin external_trajectory \ + --task-source +``` + +Check: +- [ ] Source is documented in `task_source` +- [ ] External origin is explicit (not inferred from content) + +### Demo or Test Run + +Use when the session is a causetrace demo or a test. + +```bash +causetrace metadata-set \ + --data-origin native \ + --task-source demo +``` + +Check: +- [ ] Session is explicitly a demo, not real work +- [ ] Will NOT be counted in native strict lane (no real_work task_source) + +## Post-Capture Verification + +After declaring metadata, verify: + +```bash +# Confirm the session appears in the correct lane +causetrace corpus lane-count + +# Confirm intervention_lane is set +causetrace metadata-show | grep intervention_lane + +# Confirm provenance is recorded +causetrace metadata-show --provenance +``` + +## Non-Rules (Do Not Use) + +These are NOT valid classification signals: + +- `runtime=claude-code` → native (wrong: runtime does not determine lane) +- `used Skill tool` → superpowers (wrong: Skill alone is not evidence) +- `session ended with exit code 0` → success=true (wrong: clean exit ≠ success) +- `prompt looks structured` → routed (wrong: prompt style is not a marker) +- `data_origin=unknown + runtime known` → direct_prompt_native (wrong: origin is still unknown) + +## Historical Sessions + +Sessions captured before this checklist was in place (data_origin=unknown) remain unlabeled unless: + +- Explicit source evidence is found (e.g., the original task description, project context, or tool logs confirm the capture type) +- A human annotates the session with high confidence + +Do not bulk-classify old sessions. The 879 unlabeled sessions are evidence gaps, not technical debt. From 6711f0e06918c696995f45d63246c5708650b39c Mon Sep 17 00:00:00 2001 From: Your Name Date: Sun, 14 Jun 2026 22:17:56 +0800 Subject: [PATCH 7/8] Remove tracked files that match .gitignore patterns These were committed before the ignore rules were added. Now .gitignore will prevent them from being re-added. Co-Authored-By: Claude Opus 4.7 --- ...2026-05-14-codex-tailer-format-mismatch.md | 95 ------------------- ...026-05-14-hook-causality-silent-failure.md | 27 ------ ...2026-05-14-hook-ceiling-agent-reasoning.md | 88 ----------------- .../2026-05-28-corpus-gap-analysis.md | 69 -------------- docs/research-notes/README.md | 34 ------- .../2026-05-24-bash-dominant-transition.md | 85 ----------------- .../patterns/2026-05-24-deep-linear-claude.md | 58 ----------- .../patterns/2026-05-24-edit-read-locality.md | 60 ------------ ...roject_phase_runtime_reality_validation.md | 64 ------------- 9 files changed, 580 deletions(-) delete mode 100644 docs/research-notes/2026-05-14-codex-tailer-format-mismatch.md delete mode 100644 docs/research-notes/2026-05-14-hook-causality-silent-failure.md delete mode 100644 docs/research-notes/2026-05-14-hook-ceiling-agent-reasoning.md delete mode 100644 docs/research-notes/2026-05-28-corpus-gap-analysis.md delete mode 100644 docs/research-notes/README.md delete mode 100644 docs/research/patterns/2026-05-24-bash-dominant-transition.md delete mode 100644 docs/research/patterns/2026-05-24-deep-linear-claude.md delete mode 100644 docs/research/patterns/2026-05-24-edit-read-locality.md delete mode 100644 memory/project_phase_runtime_reality_validation.md diff --git a/docs/research-notes/2026-05-14-codex-tailer-format-mismatch.md b/docs/research-notes/2026-05-14-codex-tailer-format-mismatch.md deleted file mode 100644 index bb00abe..0000000 --- a/docs/research-notes/2026-05-14-codex-tailer-format-mismatch.md +++ /dev/null @@ -1,95 +0,0 @@ -# Codex CLI tailer format mismatch: `codex_tailer.py` is based on wrong assumptions - -## Problem - -`causetrace/hooks/codex_tailer.py` was written assuming Codex CLI session data -uses the same structure as generic OpenAI Chat Completions API traces: - -```python -# Assumed format (DOES NOT EXIST): -{"type": "action", "action": {"name": "Bash", "input": {...}}} -{"type": "observation", "observation": {"name": "Bash", "output": {...}}} -``` - -The actual Codex CLI session format is fundamentally different. - -## Actual format - -Codex CLI stores sessions as rollout JSONL files at: -``` -~/.codex/sessions/YYYY/MM/DD/rollout--.jsonl -``` - -Each line follows this structure (from `codex-rs/protocol/src/protocol.rs`): - -```rust -struct RolloutLine { - pub timestamp: String, - #[serde(flatten)] - pub item: RolloutItem, -} - -enum RolloutItem { - SessionMeta(SessionMetaLine), // type = "session_meta" - ResponseItem(ResponseItem), // type = "response_item" - EventMsg(EventMsg), // type = "event_msg" - TurnContext(TurnContextItem), // type = "turn_context" - Compacted(CompactedItem), // type = "compacted" -} -``` - -Serialized JSON: -```json -{"timestamp":"...","type":"session_meta","payload":{...}} -{"timestamp":"...","type":"response_item","payload":{"type":"message","role":"developer","content":[...]}} -{"timestamp":"...","type":"event_msg","payload":{"type":"agent_reasoning","text":"..."}} -{"timestamp":"...","type":"event_msg","payload":{"type":"mcp_tool_call_begin","call_id":"...","invocation":{...}}} -{"timestamp":"...","type":"event_msg","payload":{"type":"exec_command_begin","call_id":"...","command":[...]}} -{"timestamp":"...","type":"turn_context","payload":{"model":"gpt-5.5",...}} -``` - -## Key differences - -| Aspect | `codex_tailer.py` assumption | Actual format | -|---|---|---| -| Top-level types | `action`, `observation` | 5 types: `session_meta`, `response_item`, `event_msg`, `turn_context`, `compacted` | -| Tool calls | `action` entries | Paired `mcp_tool_call_begin/end` + `exec_command_begin/end` in event_msg | -| Reasoning | Not handled | `event_msg/agent_reasoning` | -| Output | `observation` entries | `mcp_tool_call_end.result.content` or `exec_command_end.stdout/stderr` | -| Role/content | Chat format: role+content | Responses API format with development_instructions | - -## What's available in the new codex_parser.py - -The new `causetrace/hooks/codex_parser.py` handles: - -- **`event_msg/agent_reasoning`** → `event_type="reasoning"`, `tool_name="Thinking"` -- **`event_msg/mcp_tool_call_begin/end`** → Paired tool calls with duration -- **`event_msg/exec_command_begin/end`** → `tool_name="Bash"` with command + output -- **`response_item`** → `tool_name="Response"` for assistant text outputs -- **`session_meta`** → model/provider extraction - -It also tracks in-flight call_ids to pair begin/end events and handles -orphan end events gracefully. - -## Status - -- `codex_tailer.py` — **KNOWN BROKEN**. Its format assumptions do not match - any known version of Codex CLI rollout data. It will produce empty results - or miss all events. -- `codex_parser.py` — **VALIDATED** 2026-05-14 against real session - `019e2553-ade5` (Codex v0.130.0 via DeepSeek proxy). Extracted 116 events - (114 tool_call + 2 reasoning) from 465-line rollout JSONL. Actual format uses - `response_item/function_call` + `response_item/function_call_output` paired by - `call_id`, plus `event_msg/agent_message` for reasoning. -- **Key format from real data** (not protocol.rs): - - `response_item` with `payload.type = "function_call"` → tool call events - - `response_item` with `payload.type = "function_call_output"` → paired output - - `event_msg` with `payload.type = "agent_message"` → reasoning - - `event_msg` with `payload.type = "token_count"` → token usage (skipped) - -## Status - -`causetrace enrich-codex` is now functional. `codex_tailer.py` remains broken -but is a separate code path (used by `causetrace codex`). - -See: https://github.com/openai/codex diff --git a/docs/research-notes/2026-05-14-hook-causality-silent-failure.md b/docs/research-notes/2026-05-14-hook-causality-silent-failure.md deleted file mode 100644 index 429586d..0000000 --- a/docs/research-notes/2026-05-14-hook-causality-silent-failure.md +++ /dev/null @@ -1,27 +0,0 @@ -# Hook causality silently failed for 61 sessions - -## Problem - -The Claude Code hook bridge had a bug where `parent_event_id` was never passed -to `record_call()`, resulting in every event being a root node. The system gave -no indication of failure — `validate` passes clean on a flat trace. - -## Why it was hard to catch - -- No integration test for the hook stdin/stdout flow -- Unit tests create chains via direct `TraceRecorder.record_call()` and never - exercise the actual hook bridge code path -- The schema considers null parent_event_id as valid (a root node), so there's - no structural error - -## Implication - -Runtime Integrity tools (validate, tests) are necessary but not sufficient. They -verify *structural* correctness but not *semantic* correctness. A trace can be -structurally valid (all invariants pass) but semantically broken (no causality). - -## What this suggests - -We need a way to distinguish "intentionally flat" from "broken causality." -Heuristic: a session with many events and zero parent links is suspicious. -But this heuristic could also be wrong for genuinely parallel agents. diff --git a/docs/research-notes/2026-05-14-hook-ceiling-agent-reasoning.md b/docs/research-notes/2026-05-14-hook-ceiling-agent-reasoning.md deleted file mode 100644 index 6e80dfd..0000000 --- a/docs/research-notes/2026-05-14-hook-ceiling-agent-reasoning.md +++ /dev/null @@ -1,88 +0,0 @@ -# The Hook Ceiling: why tool-level observation can't capture agent reasoning - -## The problem - -causetrace's hook/log layer sits at the tool boundary — it sees tool calls -and their results, but not the internal reasoning that produced them. This -is a fundamental limitation, not a gap that can be engineered away at the -current layer. - -## The stack - -``` -LLM (reasoning + decisions) ← invisible to causetrace - │ -Agent Runtime (orchestration) ← partially visible (some state) - │ -Tool boundary (hook/log layer) ← causetrace lives here - │ -OS / filesystem / shell -``` - -At the tool layer, we see *what* the agent did, never *why* it decided to do -it. The `caused_by` field exists to capture this, but it depends on the -runtime choosing to expose the reason. - -## Three approaches to deeper signals - -### 1. API intercept (highest fidelity, most invasive) - -Intercept the HTTP requests/responses between the agent and the LLM provider. -This reveals the full conversation: system prompt, user messages, assistant -reasoning, tool calls, tool results, and the model's internal thought process. - -**Feasibility for causetrace:** -- Works for self-hosted agents using OpenAI/Anthropic APIs directly -- NOT feasible for Claude Code (uses proprietary endpoint) -- NOT feasible for Copilot (uses Microsoft endpoint) -- Requires a local proxy or MitM setup -- Adds latency and complexity - -### 2. Agent internal state export (fragile, agent-specific) - -Some agents expose internal state: -- Claude Code checkpoints at `~/.claude/checkpoints/` -- Debug/verbose modes that dump plan structure or context -- Structured log formats with reasoning traces - -**Feasibility for causetrace:** -- Format is unstable and varies between agent versions -- No standardization across runtimes -- High maintenance burden for low reliability -- Violates runtime-neutrality principle - -### 3. Reverse inference from tool_io (current path) - -Infer intent from tool input/output patterns. For example: -- `Read(file=X)` + `Edit(file=X, pattern=Y)` → agent is fixing a bug in X -- `Grep(pattern=Y)` + multiple `Read` results → agent is investigating Y -- Repeated failing `Edit` + `Bash(test)` → agent is in a retry loop - -**Feasibility for causetrace:** -- Already works at a basic level (sequential chaining, fan-in detection) -- Always approximate — sees "what" not "why" -- Quality depends entirely on the richness of tool_input/tool_output -- This is the causetrace-native approach - -## Implication for causetrace - -The `caused_by` field in ToolEvent is the right place for reasoning signals, -but it can only be populated if the runtime provides the information at -hook time. causetrace should NOT attempt to: - -- Parse LLM responses retroactively (too fragile) -- Maintain agent-specific state extractors (too coupled) -- Guess intent from tool patterns (too speculative) - -## What causetrace SHOULD do - -1. Keep `caused_by` as a first-class field, ready for when runtimes provide - the signal -2. Track this limitation in pressure-log when it causes ambiguity -3. Document in schema docs that `caused_by` is a runtime-provided field, - not inferred - -## Related - -This connects to Pressure #002 (linear causality from missing turn -boundary) — both are runtime signal gaps, not schema gaps. diff --git a/docs/research-notes/2026-05-28-corpus-gap-analysis.md b/docs/research-notes/2026-05-28-corpus-gap-analysis.md deleted file mode 100644 index 7e7ead2..0000000 --- a/docs/research-notes/2026-05-28-corpus-gap-analysis.md +++ /dev/null @@ -1,69 +0,0 @@ -# Corpus gap analysis for 2026-05-28 - -This note captures the current state of the local corpus and the remaining conditions needed for the next topology milestone. - -## Snapshot - -- sessions: 509 -- events: 10,145 -- metadata sessions: 0 -- annotated sessions: 27 -- explicit runtime sessions: 0 -- heuristic runtime sessions: 50 -- task-type sessions: 27 -- source sessions: 27 - -## Milestones - -- Corpus scale: 509/1000 (remaining 491) -- Research-ready labeled corpus: 0/100 (remaining 100) -- Explicit runtime fingerprint set: 0/4 (remaining 4) -- Task taxonomy breadth: 7/4 (remaining 0) -- Fan-in exemplars: 1/10 (remaining 9) -- Branch-collapse exemplars: 1/10 (remaining 9) -- Multi-root exemplars: 2/10 (remaining 8) - -## Coverage - -- explicit runtime counts: - - none -- task type counts: - - exploration: 14 - - bug_fix: 5 - - unknown: 3 - - review: 2 - - debug_test: 1 - - feature_add: 1 - - migration: 1 -- topology counts: - - dominant_chain: 436 - - mixed: 72 - - multi_root_exploration: 1 - -## Structural Signals - -- long sessions (>=100 events): 14 -- branchy sessions: 95 -- frontier-wide sessions (max width >= 4): 14 -- retry-heavy sessions (retry density >= 0.2): 155 -- fan-in sessions: 1 -- branch-collapse sessions: 1 -- multi-root sessions (roots >= 5): 2 - -## What this means - -The corpus is no longer too small to study, but it is still too sparse in the dimensions that matter for topology research. - -- Explicit metadata coverage is effectively absent. -- Labeled task coverage is still narrow. -- The corpus is dominated by linear or near-linear behavior. -- High-signal structural exemplars are rare enough that taxonomy work would still overfit if treated as complete. - -## Next collection target - -Before treating milestone taxonomy or runtime fingerprinting as stable, the corpus needs: - -1. More explicit metadata sidecars. -2. More balanced task labels. -3. More fan-in, branch-collapse, and multi-root sessions. -4. Multiple runs from each runtime family under comparable task types. diff --git a/docs/research-notes/README.md b/docs/research-notes/README.md deleted file mode 100644 index 00d9aad..0000000 --- a/docs/research-notes/README.md +++ /dev/null @@ -1,34 +0,0 @@ -# Runtime Research Notes - -Observations about agent runtime behavior that inform schema evolution and causality model design. - -## Purpose - -These notes capture what the schema and code alone cannot: - -- Why a particular trace was surprising -- Why a causality pattern was unstable -- What runtime behavior couldn't be expressed in the current schema -- Why a replay failed or produced wrong semantics -- Correlations between trace patterns and agent behavior - -## Format - -Each note should be a single markdown file with a date prefix: - -``` -2026-05-14-parallel-causality-gap.md -2026-05-15-fan-in-ambiguity.md -``` - -## Topics worth tracking - -- **Causality ambiguity** — when parent_event_id can't express the real relationship -- **Schema pressure** — traces that strain the current field set -- **Replay failures** — why a trace didn't replay faithfully -- **Pattern mismatches** — expected causal patterns vs. observed ones -- **Agent-specific behavior** — quirks of particular runtimes - -Validated pattern observations belong in `docs/research/patterns/`. When an -analysis implementation changes a metric definition, annotate existing -observations rather than silently rewriting historical results. diff --git a/docs/research/patterns/2026-05-24-bash-dominant-transition.md b/docs/research/patterns/2026-05-24-bash-dominant-transition.md deleted file mode 100644 index 32f6645..0000000 --- a/docs/research/patterns/2026-05-24-bash-dominant-transition.md +++ /dev/null @@ -1,85 +0,0 @@ -# Pattern: bash-dominant-transition - -## Sessions - -| Session | Agent | Events | Bash→Bash | Bash % | -|---------|-------|--------|-----------|--------| -| 1a00157a | claude-code | 1499 | 648 | 57% (856/1499) | -| 270e9651 | claude-code | 1420 | 339 | 37% (528/1420) | -| codex_latest | codex-cli | 672 | 291 | 63% (420/672) | - -## Raw Metrics (Claude A) - -tool_freq: Bash×856, Read×408, Edit×178, Agent×22, TaskUpdate×7 -repeated_paths (>=2 occurrences): -- Bash → Bash: ×648 -- Bash → Bash → Bash: ×525 -- Bash → Bash → Bash → Bash: ×444 -- Bash → Bash → Bash → Bash → Bash: ×383 - -The n-gram frequency decays slowly: 648 → 525 → 444 → 383. This means Bash→Bash is not a 2-event flicker — it's sustained Bash chains of 5+ events. - -## Raw Metrics (Claude B) - -tool_freq: Bash×528, Thinking×303, Edit×203, Read×151, Response×114 -repeated_paths: -- Bash → Bash: ×339 -- Bash → Bash → Bash: ×185 -- Bash → Bash → Bash → Bash: ×129 -- Bash → Bash → Bash → Bash → Bash: ×88 - -Same pattern, but weaker decay (339 → 185 → 129 → 88). Bash chains are shorter here. This session also has Thinking→Response (×110) and Thinking→Bash (×88), suggesting reasoning steps interleaved with Bash that may break the chaining. - -## Raw Metrics (Codex CLI) - -tool_freq: Bash×420, write_stdin×151, Edit×70 -repeated_paths: -- Bash → Bash: ×291 -- Bash → Bash → Bash: ×217 -- Bash → Bash → Bash → Bash: ×171 -- Bash → Bash → Bash → Bash → Bash: ×136 - -Even stronger decay (291 → 217 → 171 → 136): Codex CLI has the longest sustained Bash chains. - -## Observation - -Bash→Bash is the single most common transition across all three sessions. It is not a short artifact — long chains of 5+ consecutive Bash events exist in every session. - -Possible explanations for what drives Bash→Bash: -1. **Shell batching**: a single logical "run tests" step produces multiple Bash events (install, compile, run, parse output) -2. **Iterative repair**: run → fail → run different command → fail → run again -3. **Verification chaining**: run test → run coverage → run lint → run format check -4. **Tool routing**: the agent uses Bash as a meta-tool for everything not covered by native tools - -## Open Questions - -- Does Bash→Bash correlate with failure states (retry loops)? -- Can we distinguish "productive Bash chain" from "oscillating Bash chain" without examining tool_output? -- Is Bash→Bash a Codex/DeepSeek-specific artifact (proxy loops)? -- Would first-class test/compile/lint semantics reduce Bash chaining? -- Is sustained Bash chaining a signature of "agent doesn't know when to stop trying"? -- Does the presence of Thinking events BETWEEN Bash events (Claude B: 129 Bash→Thinking) indicate different agent behavior than contiguous Bash→Bash (Claude A)? - -## Window Drift (Claude B: 270e9651) - -Count-based windows (size=300, overlap=100) over 1420 events: - -| Window | Events | Roots | Depth | Top Transition | -|--------|--------|-------|-------|----------------| -| W0 | 300 | 89 | 211 | Bash→Bash: 64 | -| W1 | 300 | 87 | 0 | Bash→Bash: 37 | -| W2 | 300 | 76 | 7 | Bash→Bash: 48 | -| W3 | 300 | 22 | 57 | Bash→Bash: 43 | -| W4 | 300 | 0 | 0 | Bash→Bash: 80 | -| W5 | 300 | 0 | 0 | Bash→Bash: 123 | -| W6 | 220 | 0 | 0 | Bash→Bash: 71 | - -Observation recorded from the original analysis run: roots decrease (89→0) -across the session as Bash→Bash concentration increases (64→123). Depth -regeneration in W3 (57) after zero-depth W1-W2 suggested a window-boundary -effect. - -Correction (2026-05-24): `analysis.py` now treats parent references outside a -loaded window as local-root boundaries. The zero-root/zero-depth values above -reflect the pre-fix implementation and should be recomputed before drawing -topology conclusions; the transition counts remain historical observations. diff --git a/docs/research/patterns/2026-05-24-deep-linear-claude.md b/docs/research/patterns/2026-05-24-deep-linear-claude.md deleted file mode 100644 index ef05b1f..0000000 --- a/docs/research/patterns/2026-05-24-deep-linear-claude.md +++ /dev/null @@ -1,58 +0,0 @@ -# Pattern: deep-linear-chain - -## Session - -session_id: 1a00157a-1359-4981-a11d-21f8164b2130 -agent: claude-code -event_count: 1499 -enriched: true (Claude Code project parser) -task: football system migration - -## Raw Metrics - -max_depth: 1273 -roots: 1 -leaf_nodes: 31 -avg_chain_length: 1274 -fan_out_max: 3 -fan_out_avg: 1.0 -link_ratio: 0.999 -time_span: 2883m (~48h) - -top_transitions: -- Bash -> Bash: 648 -- Bash -> Read: 169 -- Read -> Read: 167 -- Read -> Bash: 144 -- Read -> Edit: 81 -- Edit -> Edit: 77 -- Edit -> Read: 54 -- Edit -> Bash: 46 - -## Observation - -The session behaves like a single iterative repair chain rather than a branching planner graph. With 1 root and 1499 events, almost the entire session is one deep causal chain. The 31 leaf nodes indicate short side branches that terminate quickly while one dominant chain (depth=1273) absorbs the majority of execution. - -Bash→Bash (648) dominates all transitions — 43% of all events are Bash. This suggests the agent operates primarily through shell commands, with occasional reads and edits interspersed. - -Read→Edit (81) vs Edit→Read (54) ratio: slightly more read-before-edit than edit-then-re-read, but both are present and close. - -## Cross-session Comparison - -| Metric | Claude A (1499) | Claude B (1420) | Codex (672) | -|--------|----------------|----------------|-------------| -| roots | 1 | 193 | 2 | -| max_depth | 1273 | 749 | 641 | -| avg_chain | 1274 | 7.2 | 336 | -| fan-out max | 3 | 3 | 1 | -| Bash→Bash | 648 | 339 | 291 | - -Session B (270e9651) has 193 roots with avg_chain_length=7.2 — a fundamentally different topology. Unlike Session A's single deep chain, Session B is made of many short independent chains. This may reflect a different task type (multi-step exploration vs focused repair). - -## Open Questions - -- Is single-root deep-linear topology specific to continuous repair tasks (migration, debugging)? -- Does multi-root topology correspond to exploration/learning phases? -- Do all coding agents converge toward deep linear chains given enough time? -- Is the Bash→Bash dominance a function of the agent's tool-use policy or the task's actual needs? -- Does long-depth correlate with successful task completion, or with oscillation/stuck states? \ No newline at end of file diff --git a/docs/research/patterns/2026-05-24-edit-read-locality.md b/docs/research/patterns/2026-05-24-edit-read-locality.md deleted file mode 100644 index 8eb38d4..0000000 --- a/docs/research/patterns/2026-05-24-edit-read-locality.md +++ /dev/null @@ -1,60 +0,0 @@ -# Pattern: edit-read-locality - -## Session - -session_id: 1a00157a-1359-4981-a11d-21f8164b2130 -agent: claude-code -event_count: 1499 - -## Raw Metrics - -Read frequency: 408 (27%) -Edit frequency: 178 (12%) - -Transitions involving Edit/Read: -- Read -> Edit: 81 -- Edit -> Read: 54 -- Edit -> Bash: 46 -- Edit -> Edit: 77 -- Read -> Read: 167 -- Read -> Bash: 144 -- Bash -> Edit: 17 - -Edit→Read ratio: 54 out of 178 Edits (30%) are followed by a Read. - -Read→Edit ratio: 81 out of 408 Reads (20%) are followed by an Edit. - -Edit→Edit: 77 out of 178 Edits (43%) are followed by another Edit. - -## Cross-session Comparison (Claude B: 270e9651) - -Read frequency: 151 (11%) -Edit frequency: 203 (14%) - -Transitions: -- Read -> Read: 66 -- Read -> Thinking: 41 -- Edit -> Edit: 103 -- Edit -> Thinking: 60 -- Edit -> Bash: 30 -- Thinking -> Edit: 44 - -Distinctly different: Claude B interposes Thinking between Edit and Read. Edit→Thinking (60) is more common than Edit→Read (not in top 15). This suggests this agent reasons between mutations. - -## Observation - -The Read→Edit→Read cycle appears to be a fundamental agent operation: read context → make change → verify by re-reading. But Edit→Edit (77) is more common than Edit→Read (54), suggesting the agent often makes multiple edits before re-reading. - -Claude B's Edit→Edit (103) is even more dominant relative to its event count. 51% of edits are followed by another edit, compared to 43% in Claude A. - -The presence of Thinking between Edit and Read in Claude B (but not Claude A) raises a question: is Claude B's enriched trace capturing a different reasoning layer, or is the agent actually behaving differently? - -Codex CLI shows Edit→Bash (34) instead of Edit→Read — the Codex agent tests after editing rather than re-reading. - -## Open Questions - -- Is the Read→Edit→Read pattern the core "cognition locality" primitive for coding agents? -- Does Edit→Edit indicate productive multi-step editing or oscillation (reverting changes)? -- Is Edit followed by Thinking qualitatively different from Edit followed by Read? -- Do different runtimes have characteristic Edit transitions (Claude: Edit→Read, Codex: Edit→Bash)? -- Could Edit→Edit ratio serve as a proxy for "agent certainty" (low = confident single edit, high = uncertain trial/error)? \ No newline at end of file diff --git a/memory/project_phase_runtime_reality_validation.md b/memory/project_phase_runtime_reality_validation.md deleted file mode 100644 index ee58c7d..0000000 --- a/memory/project_phase_runtime_reality_validation.md +++ /dev/null @@ -1,64 +0,0 @@ ---- -name: Project Phase - Runtime Reality Validation -description: Strategic shift after v0.1 — from feature stacking to runtime analysis ontology via analysis.py -type: project ---- - -causetrace has entered Runtime Reality Validation phase (post-v0.1). The core question is no longer "does the code run" but "do the runtime abstractions match real agent behavior." - -## Strategic priorities (ordered): - -1. **Collect real traces** — 30min+ sessions, failure traces (retry loops, oscillation, abandoned branches), parallel execution traces. This is the single most important task. -2. **Build trace corpus** at `/examples/traces/` — organized by type (successful, failures, loops, retries, refactors, debugging), each with raw jsonl + tree/why output + observation notes. -3. **Runtime Pattern Discovery** — using `analysis.py` primitives (Layer 1 structural + Layer 2 pattern) to find repeated tool paths, transitions, and convergence patterns. Document findings in `/docs/research/patterns/`. -4. **Runtime Integrity** — tests (serialization roundtrip, DAG integrity, parent linking, replay consistency, malformed trace recovery), corruption handling, `causetrace validate` command. -5. **Observe users, not features** — no web UI, dashboards, AI features, embeddings, cloud sync. Watch what commands users actually run (tree vs timeline, why utility, graph complexity, stats output). -6. **Semantic stability** — schema_version field frozen, field names frozen, event_type frozen, parent semantics frozen. Schema migration cost will spike once real corpus exists. -7. **Runtime Research Notes** at `/docs/research-notes/` — capture strange traces, unstable causality patterns, inexpressible runtime behaviors, replay failures. These feed schema evolution and may become a paper/standard draft. - -## Key architectural addition: analysis.py - -Created `causetrace/analysis.py` as the Session Analysis Layer, sitting between core and CLI: - -```text -Trace Capture (core.py) - ↓ -Causal DAG (core.py: build_tree, trace_causal_chain) - ↓ -Session Analysis Primitives (analysis.py) ← NEW - ↓ -CLI / Compression / Intelligence -``` - -**Layer 1 — Structural (pure topology):** -- `compute_stats()` — event counts, depths, fan-out, link ratio, time span -- `find_roots()` — root events with downstream count and subtree depth -- `longest_path()` — longest root-to-leaf chain (critical path, topological) -- `fan_out_distribution()` — histogram of children per node -- `connected_components()` — weakly connected component sizes - -**Layer 2 — Pattern (structural only, no semantic naming):** -- `detect_repeated_paths()` — tool_name subsequences that repeat -- `detect_common_transitions()` — (tool_i → tool_j) transition counts -- `detect_fan_in_patterns()` — multi-parent convergence points -- `detect_branch_collapse()` — convergence of multiple root paths - -**CLI commands (thin renderers only):** -- `causetrace stats ` — structural session profile -- `causetrace roots ` — root events with downstream metrics -- `causetrace critical-path ` — deepest causal chain -- `causetrace patterns ` — repeated patterns + transitions - -**Design rule:** no semantic interpretation in analysis.py. Retry loops, verification chains, planning phases must NOT be named here — let them emerge from pattern observation across multiple traces and runtimes. - -**Boundary rule:** analysis is session-local. A `parent_event_id` absent from -the loaded event set is retained as provenance but does not become a graph -node; the child is treated as a local root. - -## Core principle - -causetrace's value is NOT ingestion volume. It's runtime semantics — explaining **why** an agent acted, not just logging what it did. This is the differentiator from all observability tools. - -**Why:** The user provided a deep strategic analysis — the project must validate its causality model against real agent behavior before adding features. - -**How to apply:** Every decision should be filtered through: "Does this help us understand agent behavior better, or just log more?" If the latter, defer. From 8a050d6524f3ba6801c055cd78041024953661df Mon Sep 17 00:00:00 2001 From: Your Name Date: Tue, 23 Jun 2026 14:35:54 +0800 Subject: [PATCH 8/8] Release v0.3.0 --- CHANGELOG.md | 7 + causetrace/__init__.py | 2 +- causetrace/bde/__init__.py | 19 + causetrace/bde/behavior_generator.py | 38 ++ causetrace/bde/failure_injection.py | 20 + causetrace/bde/multi_agent_simulator.py | 19 + causetrace/bde/task_distribution.py | 15 + causetrace/cli.py | 272 +++++++++++-- causetrace/core.py | 80 ++++ causetrace/crdd/__init__.py | 25 ++ causetrace/crdd/comparability.py | 41 ++ causetrace/crdd/comparability_score.py | 85 ++++ causetrace/crdd/constraints.py | 59 +++ causetrace/crdd/execution_queue.py | 69 ++++ causetrace/crdd/experiment_planner.py | 136 +++++++ causetrace/crdd/experimental_units.py | 78 ++++ causetrace/crdd/feedback.py | 366 ++++++++++++++++++ causetrace/crdd/gap_analyzer.py | 77 ++++ causetrace/crdd/scenario_mapper.py | 56 +++ causetrace/crdd/subset_builder.py | 247 ++++++++++++ causetrace/crdd/subset_registry.py | 51 +++ causetrace/hooks/opencode_parser.py | 20 +- causetrace/metadata.py | 9 + docs/project-principles.md | 14 + docs/research/README.md | 16 + docs/research/ai_behavior_science_os_v1.0.md | 111 ++++++ .../machine_state_pilot_v0.1.md | 99 +++++ docs/research/dataset_design/cerc_v0.3.md | 99 +++++ docs/research/dataset_design/crdd_v0.1.md | 158 ++++++++ docs/research/dataset_design/feedback_v0.4.md | 47 +++ .../subset_manifest_template.md | 115 ++++++ docs/research/phase4/README.md | 2 + .../phase4/phase4_3_trigger_conditions.md | 12 + .../runtime_morphology_theory_draft_v0.1.md | 2 + .../agentic_auditability_roadmap_v0.2.5.md | 324 ++++++++++++++++ .../data_grounded_method_discovery_v0.2.5.md | 157 ++++++++ pyproject.toml | 2 +- tests/test_invariants.py | 132 +++++++ tests/test_metadata_corpus_report.py | 333 ++++++++++++++++ tests/test_opencode_enrich.py | 39 ++ 40 files changed, 3408 insertions(+), 45 deletions(-) create mode 100644 causetrace/bde/__init__.py create mode 100644 causetrace/bde/behavior_generator.py create mode 100644 causetrace/bde/failure_injection.py create mode 100644 causetrace/bde/multi_agent_simulator.py create mode 100644 causetrace/bde/task_distribution.py create mode 100644 causetrace/crdd/__init__.py create mode 100644 causetrace/crdd/comparability.py create mode 100644 causetrace/crdd/comparability_score.py create mode 100644 causetrace/crdd/constraints.py create mode 100644 causetrace/crdd/execution_queue.py create mode 100644 causetrace/crdd/experiment_planner.py create mode 100644 causetrace/crdd/experimental_units.py create mode 100644 causetrace/crdd/feedback.py create mode 100644 causetrace/crdd/gap_analyzer.py create mode 100644 causetrace/crdd/scenario_mapper.py create mode 100644 causetrace/crdd/subset_builder.py create mode 100644 causetrace/crdd/subset_registry.py create mode 100644 docs/research/ai_behavior_science_os_v1.0.md create mode 100644 docs/research/branches/agentic_auditability/machine_state_pilot_v0.1.md create mode 100644 docs/research/dataset_design/cerc_v0.3.md create mode 100644 docs/research/dataset_design/crdd_v0.1.md create mode 100644 docs/research/dataset_design/feedback_v0.4.md create mode 100644 docs/research/dataset_design/subset_manifest_template.md create mode 100644 docs/research/roadmap/agentic_auditability_roadmap_v0.2.5.md create mode 100644 docs/research/roadmap/data_grounded_method_discovery_v0.2.5.md diff --git a/CHANGELOG.md b/CHANGELOG.md index c6c1f6b..97dc1c2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # Changelog +## 0.3.0 - 2026-06-23 + +- Add the AI Behavior Science OS v0.3 stack: descriptor-only BDE, read-only CRDD subset compilation, and external-only CERC experiment planning. +- Add feedback integration for external execution results with gap updates and experiment reprioritization. +- Add stable dedup/upsert import entry points and metadata extensions for behavior-distribution tracking. +- Update research documentation and regression coverage for the corpus design workflow. + ## 0.2.5 - 2026-05-29 - Harden corpus reporting with field-level metadata provenance and missing-field audits. diff --git a/causetrace/__init__.py b/causetrace/__init__.py index 438e1fc..72bcc12 100644 --- a/causetrace/__init__.py +++ b/causetrace/__init__.py @@ -5,7 +5,7 @@ from .hooks.opencode_parser import parse_session as enrich_opencode_session, list_sessions as list_opencode_sessions from .hooks.codex_parser import parse_session as enrich_codex_session, list_sessions as list_codex_sessions -__version__ = "0.2.5" +__version__ = "0.3.0" __all__ = [ "ToolEvent", "TraceRecorder", "JSONStore", "TimelineRenderer", "ReplayEngine", "build_tree", "infer_relations", "build_causal_graph", "cli", diff --git a/causetrace/bde/__init__.py b/causetrace/bde/__init__.py new file mode 100644 index 0000000..6bd7ff4 --- /dev/null +++ b/causetrace/bde/__init__.py @@ -0,0 +1,19 @@ +"""Behavior Design Engine interfaces. + +BDE is an opt-in scenario design layer. It defines behavior-generation metadata +only; it does not execute agents or modify runtime behavior. +""" + +from .behavior_generator import BehaviorGenerator, BehaviorScenario, PromptVariant +from .failure_injection import FailureInjection +from .multi_agent_simulator import MultiAgentSimulation +from .task_distribution import TaskDistribution + +__all__ = [ + "BehaviorGenerator", + "BehaviorScenario", + "FailureInjection", + "MultiAgentSimulation", + "PromptVariant", + "TaskDistribution", +] diff --git a/causetrace/bde/behavior_generator.py b/causetrace/bde/behavior_generator.py new file mode 100644 index 0000000..15ccb84 --- /dev/null +++ b/causetrace/bde/behavior_generator.py @@ -0,0 +1,38 @@ +"""Interfaces for controlled behavior scenario generation. + +This module is intentionally passive. It creates scenario descriptors that can +be logged as metadata, but it does not execute prompts or call agent runtimes. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any, Protocol + + +@dataclass(frozen=True) +class PromptVariant: + """A named prompt/task variant in an experiment design.""" + + variant_id: str + prompt: str + metadata: dict[str, Any] = field(default_factory=dict) + + +@dataclass(frozen=True) +class BehaviorScenario: + """Metadata-only description of a generated behavior scenario.""" + + scenario_id: str + task: str + behavior_distribution_tag: str | None = None + prompt_variants: tuple[PromptVariant, ...] = () + experiment_id: str | None = None + control_group_id: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) + + +class BehaviorGenerator(Protocol): + """Protocol for BDE scenario generators.""" + + def generate(self) -> list[BehaviorScenario]: + """Return scenario descriptors without executing them.""" diff --git a/causetrace/bde/failure_injection.py b/causetrace/bde/failure_injection.py new file mode 100644 index 0000000..6edf649 --- /dev/null +++ b/causetrace/bde/failure_injection.py @@ -0,0 +1,20 @@ +"""Failure-injection scenario descriptors. + +The descriptors here define experimental intent only. They must not mutate +runtime execution or agent configuration. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class FailureInjection: + """Metadata-only failure or near-failure design descriptor.""" + + injection_id: str + failure_mode: str + target_task_type: str | None = None + expected_observable_signal: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/causetrace/bde/multi_agent_simulator.py b/causetrace/bde/multi_agent_simulator.py new file mode 100644 index 0000000..211a5e1 --- /dev/null +++ b/causetrace/bde/multi_agent_simulator.py @@ -0,0 +1,19 @@ +"""Multi-agent simulation descriptors. + +This module does not simulate or launch agents. It only models planned runtime +roles so traces can later be grouped by experimental design. +""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class MultiAgentSimulation: + """Metadata-only descriptor for a planned multi-agent scenario.""" + + simulation_id: str + agent_roles: tuple[str, ...] + coordination_pattern: str | None = None + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/causetrace/bde/task_distribution.py b/causetrace/bde/task_distribution.py new file mode 100644 index 0000000..af85e39 --- /dev/null +++ b/causetrace/bde/task_distribution.py @@ -0,0 +1,15 @@ +"""Task distribution descriptors for behavior design experiments.""" +from __future__ import annotations + +from dataclasses import dataclass, field +from typing import Any + + +@dataclass(frozen=True) +class TaskDistribution: + """Metadata-only task distribution for a planned experiment.""" + + distribution_id: str + task_types: tuple[str, ...] + weights: dict[str, float] = field(default_factory=dict) + metadata: dict[str, Any] = field(default_factory=dict) diff --git a/causetrace/cli.py b/causetrace/cli.py index 1c8f9d6..ff77607 100644 --- a/causetrace/cli.py +++ b/causetrace/cli.py @@ -19,6 +19,15 @@ from .annotation import load_annotation, save_annotation, list_annotated, list_unannotated, TASK_TYPES, SOURCES from .causality import causal_quality_report from .corpus import benchmark_corpus, compare_benchmark_manifests, export_dataset, group_labeled_sessions, list_corpus_records, materialize_corpus_metadata, snapshot_corpus, taxonomy_corpus, verify_benchmark_manifest, verify_snapshot +from .crdd import ( + SUBSET_DEFINITIONS, + analyze_gaps, + compile_subsets, + ingest_feedback, + plan_experiments, + reprioritize_experiments, + update_gaps, +) from .hooks.claude_project_parser import parse_session as enrich_session, list_sessions as list_claude_sessions from .hooks.opencode_parser import parse_session as enrich_opencode_session, list_sessions as list_opencode_sessions from .hooks.codex_parser import parse_session as enrich_codex_session, list_sessions as list_codex_sessions @@ -38,7 +47,7 @@ from importlib.metadata import version as _import_version _CAUSETRACE_VERSION = _import_version("causetrace") except Exception: - _CAUSETRACE_VERSION = "0.1.3" + _CAUSETRACE_VERSION = "0.3.0" def _check_result(label: str, ok: bool, detail: str = "") -> tuple[bool, str, str]: @@ -167,21 +176,31 @@ def cli(argv: list[str] | None = None) -> None: p_oc = sub.add_parser("opencode", help="Scan OpenCode logs and show tool calls") p_oc.add_argument("--save", action="store_true", help="Save as a new causetrace session") + p_oc.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_oc.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_oc.add_argument("--files", type=int, default=3, help="Number of log files to scan (default: 3)") p_ai = sub.add_parser("aider", help="Run aider with causetrace tracing") p_ai.add_argument("aider_args", nargs="*", help="Arguments passed to aider") p_ai.add_argument("--save", action="store_true", help="Save session after completion") + p_ai.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_ai.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_co = sub.add_parser("continue", help="Scan Continue.dev logs") p_co.add_argument("--save", action="store_true", help="Save as a new causetrace session") + p_co.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_co.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_cx = sub.add_parser("codex", help="Scan OpenAI Codex CLI logs") p_cx.add_argument("--save", action="store_true", help="Save as a new causetrace session") + p_cx.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_cx.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_cx.add_argument("--sessions", type=int, default=3, help="Number of session dirs to scan (default: 3)") p_cp = sub.add_parser("copilot", help="Scan GitHub Copilot agent logs") p_cp.add_argument("--save", action="store_true", help="Save as a new causetrace session") + p_cp.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_cp.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_cp.add_argument("--max-dirs", type=int, default=3, help="Number of log dirs to scan (default: 3)") @@ -195,6 +214,8 @@ def cli(argv: list[str] | None = None) -> None: p_enrich = sub.add_parser("enrich", help="Enrich trace from Claude Code project session (extracts reasoning)") p_enrich.add_argument("session_id", help="Claude Code project session ID") p_enrich.add_argument("--save", action="store_true", help="Save enriched events as a causetrace session") + p_enrich.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_enrich.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_enrich.add_argument("--output", "-o", action="store_true", help="Show full timeline") sub.add_parser("enrich-opencode-sessions", help="List available OpenCode DB sessions") @@ -202,6 +223,8 @@ def cli(argv: list[str] | None = None) -> None: p_oc_enrich = sub.add_parser("enrich-opencode", help="Enrich trace from OpenCode DB session (extracts reasoning)") p_oc_enrich.add_argument("session_id", help="OpenCode session ID") p_oc_enrich.add_argument("--save", action="store_true", help="Save enriched events as a causetrace session") + p_oc_enrich.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_oc_enrich.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_oc_enrich.add_argument("--output", "-o", action="store_true", help="Show full timeline") sub.add_parser("enrich-codex-sessions", help="List available Codex CLI rollout sessions") @@ -209,6 +232,8 @@ def cli(argv: list[str] | None = None) -> None: p_cx_enrich = sub.add_parser("enrich-codex", help="Enrich trace from Codex CLI rollout session (extracts reasoning)") p_cx_enrich.add_argument("session_id", help="Codex session ID") p_cx_enrich.add_argument("--save", action="store_true", help="Save enriched events as a causetrace session") + p_cx_enrich.add_argument("--upsert", action="store_true", help="Save only events not already present") + p_cx_enrich.add_argument("--dry-run", action="store_true", help="Show upsert counts without writing") p_cx_enrich.add_argument("--output", "-o", action="store_true", help="Show full timeline") p_val = sub.add_parser("validate", help="Validate session integrity") @@ -310,6 +335,36 @@ def cli(argv: list[str] | None = None) -> None: p_cr_readiness.add_argument("--output", "-o", help="Write report to file") p_cr_materialize = p_cr_sub.add_parser("materialize", help="Materialize canonical metadata sidecars from annotations and runtime hints") p_cr_materialize.add_argument("--output", "-o", help="Write a summary report to file") + p_cr_compile = p_cr_sub.add_parser("compile-subsets", help="Compile CRDD comparable subset manifests") + p_cr_compile.add_argument("--subset", action="append", choices=sorted(SUBSET_DEFINITIONS), help="Subset to compile (repeatable; default: all)") + p_cr_compile.add_argument("--name", help="Manifest run name (default: timestamp)") + p_cr_compile.add_argument("--output-dir", help="Output directory (default: docs/research/dataset_design/manifests)") + p_cr_compile.add_argument("--dry-run", action="store_true", help="Build manifests without writing files") + p_cr_compile.add_argument("--json", action="store_true", help="Print full compile result as JSON") + p_cr_gaps = p_cr_sub.add_parser("analyze-gaps", help="Analyze CRDD subset coverage gaps") + p_cr_gaps.add_argument("--subset", action="append", choices=sorted(SUBSET_DEFINITIONS), help="Subset to analyze (repeatable; default: all)") + p_cr_gaps.add_argument("--output", "-o", help="Write gap report JSON to file") + p_cr_gaps.add_argument("--json", action="store_true", help="Print full gap report as JSON") + p_cr_plan = p_cr_sub.add_parser("plan-experiments", help="Plan external-only CERC experiment requirements") + p_cr_plan.add_argument("--target", choices=sorted(SUBSET_DEFINITIONS), default="failure_enriched", help="Target subset to plan for") + p_cr_plan.add_argument("--required-sessions", type=int, help="Override required missing session count") + p_cr_plan.add_argument("--name", help="Experiment plan run name / experiment_id") + p_cr_plan.add_argument("--output-dir", help="Output directory (default: docs/research/dataset_design/plans)") + p_cr_plan.add_argument("--dry-run", action="store_true", help="Build plan without writing files") + p_cr_plan.add_argument("--json", action="store_true", help="Print full plan result as JSON") + p_cr_feedback = p_cr_sub.add_parser("ingest-feedback", help="Ingest external execution feedback and normalize it") + p_cr_feedback.add_argument("input", help="Feedback payload JSON path") + p_cr_feedback.add_argument("--plan-dir", help="Experiment plan directory to link against") + p_cr_feedback.add_argument("--output-dir", help="Output directory (default: docs/research/dataset_design/feedback)") + p_cr_feedback.add_argument("--json", action="store_true", help="Print full feedback report as JSON") + p_cr_gaps_update = p_cr_sub.add_parser("update-gaps", help="Update gap projections from feedback") + p_cr_gaps_update.add_argument("input", help="Feedback report JSON path") + p_cr_gaps_update.add_argument("--output-dir", help="Output directory (default: docs/research/dataset_design/feedback)") + p_cr_gaps_update.add_argument("--json", action="store_true", help="Print gap update report as JSON") + p_cr_reprioritize = p_cr_sub.add_parser("reprioritize-experiments", help="Reprioritize future experiments from feedback") + p_cr_reprioritize.add_argument("input", help="Feedback report JSON path") + p_cr_reprioritize.add_argument("--output-dir", help="Output directory (default: docs/research/dataset_design/feedback)") + p_cr_reprioritize.add_argument("--json", action="store_true", help="Print reprioritized plan as JSON") p_cmp = sub.add_parser("compare", help="Compare two sessions side by side") p_cmp.add_argument("session_a", help="First session ID") @@ -418,10 +473,8 @@ def _load(sid: str | None): hdr = TimelineRenderer.session_header(events) print(f"OpenCode tool calls ({len(events)} events, {args.files} log files){hdr}\n") TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append("opencode_latest", ev) - print(f"\nSaved as session: opencode_latest ({len(events)} events)") + if args.save or args.upsert or args.dry_run: + _persist_imported_events(store, "opencode_latest", events, args) elif args.command == "aider": _handle_aider(store, args) @@ -492,10 +545,8 @@ def _load(sid: str | None): print() TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append(args.session_id, ev) - print(f"\nSaved as session: {args.session_id} ({len(events)} events)") + summary = _persist_imported_events(store, args.session_id, events, args) + if summary and summary["written"]: _auto_detect_intervention_tags(args.session_id) elif args.command == "enrich-opencode-sessions": @@ -528,10 +579,8 @@ def _load(sid: str | None): print() TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append(args.session_id, ev) - print(f"\nSaved as session: {args.session_id} ({len(events)} events)") + summary = _persist_imported_events(store, args.session_id, events, args) + if summary and summary["written"]: _auto_detect_intervention_tags(args.session_id) elif args.command == "enrich-codex-sessions": @@ -565,10 +614,8 @@ def _load(sid: str | None): print() TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append(args.session_id, ev) - print(f"\nSaved as session: {args.session_id} ({len(events)} events)") + summary = _persist_imported_events(store, args.session_id, events, args) + if summary and summary["written"]: _auto_detect_intervention_tags(args.session_id) elif args.command == "stats": @@ -827,6 +874,46 @@ def _detect_agent(events) -> str: return "unknown" +def _persist_imported_events( + store: JSONStore, + session_id: str, + events: list[ToolEvent], + args: argparse.Namespace, +) -> dict | None: + """Persist imported events through one CLI data path.""" + use_upsert = bool(getattr(args, "upsert", False) or getattr(args, "dry_run", False)) + dry_run = bool(getattr(args, "dry_run", False)) + should_save = bool(getattr(args, "save", False) or use_upsert) + if not should_save: + return None + + if use_upsert: + summary = store.append_missing(session_id, events, dry_run=dry_run) + verb = "Would upsert" if dry_run else "Upserted" + print( + f"\n{verb} session: {session_id} " + f"({summary['added']} added, {summary['skipped']} skipped, " + f"{summary['existing']} existing)" + ) + summary["written"] = not dry_run and summary["added"] > 0 + summary["mode"] = "upsert" + return summary + + for event in events: + store.append(session_id, event) + print(f"\nSaved as session: {session_id} ({len(events)} events)") + return { + "session_id": session_id, + "incoming": len(events), + "existing": None, + "added": len(events), + "skipped": 0, + "dry_run": False, + "written": bool(events), + "mode": "append", + } + + def _handle_aider(store: JSONStore, args: argparse.Namespace) -> None: """Handle `causetrace aider`.""" from .hooks.aider_bridge import run_with_tracing @@ -838,10 +925,7 @@ def _handle_aider(store: JSONStore, args: argparse.Namespace) -> None: return print(f"\n[causetrace] Session: {recorder.session_id} ({len(events)} events)") TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append(recorder.session_id, ev) - print(f"\nSaved as session: {recorder.session_id} ({len(events)} events)") + _persist_imported_events(store, recorder.session_id, events, args) def _handle_continue(store: JSONStore, args: argparse.Namespace) -> None: @@ -852,10 +936,7 @@ def _handle_continue(store: JSONStore, args: argparse.Namespace) -> None: return print(f"Continue.dev tool calls ({len(events)} events)\n") TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append("continue_latest", ev) - print(f"\nSaved as session: continue_latest ({len(events)} events)") + _persist_imported_events(store, "continue_latest", events, args) def _handle_codex(store: JSONStore, args: argparse.Namespace) -> None: @@ -871,10 +952,7 @@ def _handle_codex(store: JSONStore, args: argparse.Namespace) -> None: if report["score"] < 0.7: print(f"\n ⚠ Causal quality: {_quality_bar(report['score'])}") print(f" (Codex logs lack native causality — links are heuristic)") - if args.save: - for ev in events: - store.append("codex_latest", ev) - print(f"\nSaved as session: codex_latest ({len(events)} events)") + _persist_imported_events(store, "codex_latest", events, args) def _handle_copilot(store: JSONStore, args: argparse.Namespace) -> None: @@ -885,10 +963,7 @@ def _handle_copilot(store: JSONStore, args: argparse.Namespace) -> None: return print(f"Copilot tool calls ({len(events)} events, {args.max_dirs} log dirs)\n") TimelineRenderer.print_timeline(events) - if args.save: - for ev in events: - store.append("copilot_latest", ev) - print(f"\nSaved as session: copilot_latest ({len(events)} events)") + _persist_imported_events(store, "copilot_latest", events, args) def _print_stats(stats: dict) -> None: @@ -1670,6 +1745,137 @@ def _handle_corpus(store, args) -> None: print(report) return + if args.corpus_command == "compile-subsets": + result = compile_subsets( + store, + subset_ids=args.subset, + output_dir=args.output_dir, + name=args.name, + write=not args.dry_run, + ) + if args.json: + json.dump(result, sys.stdout, indent=2) + print() + return + action = "Compiled" if result["written"] else "Dry-run compiled" + print(f"{action} CRDD subsets from {result['source_session_count']} session(s)") + print(f"Output: {result['output_dir']}") + for manifest in result["manifests"]: + score = manifest["comparability"]["score"] + print( + f" {manifest['subset_id']}: " + f"{manifest['selected_count']} selected, " + f"{manifest['excluded_count']} excluded, " + f"score={score}" + ) + return + + if args.corpus_command == "analyze-gaps": + report = analyze_gaps(store, subset_ids=args.subset, output=args.output) + if args.json: + json.dump(report, sys.stdout, indent=2) + print() + return + if args.output: + print(f"Gap report written: {args.output}") + print(f"CERC gap report from {report['source_session_count']} session(s)") + for gap in report["subset_gaps"]: + print( + f" {gap['subset_id']}: " + f"{gap['current_sessions']}/{gap['target_sessions']} " + f"(missing {gap['missing_sessions']}, " + f"severity={gap['severity']}, " + f"score={gap['comparability_score']})" + ) + return + + if args.corpus_command == "plan-experiments": + result = plan_experiments( + store, + target_subset=args.target, + required_sessions=args.required_sessions, + output_dir=args.output_dir, + name=args.name, + write=not args.dry_run, + ) + if args.json: + json.dump(result, sys.stdout, indent=2) + print() + return + plan = result["plan"] + queue = plan["experiment_queue"] + gap = plan["gap"] + action = "Planned" if result["written"] else "Dry-run planned" + print(f"{action} CERC experiment: {queue['experiment_id']}") + print(f"Output: {result['output_dir']}") + print(f"Target subset: {queue['target_subset']}") + print(f"Current/target: {gap['current_sessions']}/{gap['target_sessions']}") + print(f"Required sessions: {queue['required_sessions']}") + print(f"Execution mode: {queue['execution_mode']}") + print(f"Must not execute: {queue['must_not_execute']}") + print(f"Evidence status: {queue['evidence_status']}") + print(f"Queue validation: {queue['validation']['ok']}") + return + + if args.corpus_command == "ingest-feedback": + report = ingest_feedback( + store, + input_path=args.input, + plan_dir=args.plan_dir, + output_dir=args.output_dir, + write=True, + ) + if args.json: + json.dump(report, sys.stdout, indent=2) + print() + return + print(f"Feedback report: {report['output_dir']}") + print(f" Experiment: {report['experiment_id']}") + print(f" Target subset: {report['target_subset']}") + print(f" Observed/resolved: {report['observed_count']}/{report['resolved_count']}") + print(f" Resolved ratio: {report['quality']['resolved_ratio']}") + print(f" External only: {report['constraints']['external_only']}") + return + + if args.corpus_command == "update-gaps": + feedback_report = json.loads(Path(args.input).read_text(encoding="utf-8")) + report = update_gaps( + store, + feedback_report=feedback_report, + output_dir=args.output_dir, + write=True, + ) + if args.json: + json.dump(report, sys.stdout, indent=2) + print() + return + print(f"Gap update: {report['output_dir']}") + print(f" Experiment: {report['experiment_id']}") + print(f" Target subset: {report['target_subset']}") + print(f" Remaining sessions: {report['remaining_sessions']}") + print(f" Status: {report['status']}") + print(f" Priority hint: {report['priority_hint']}") + return + + if args.corpus_command == "reprioritize-experiments": + feedback_report = json.loads(Path(args.input).read_text(encoding="utf-8")) + report = reprioritize_experiments( + store, + feedback_report=feedback_report, + output_dir=args.output_dir, + write=True, + ) + if args.json: + json.dump(report, sys.stdout, indent=2) + print() + return + print(f"Reprioritized plan: {report['output_dir']}") + print(f" Experiment: {report['experiment_id']}") + print(f" Target subset: {report['target_subset']}") + print(f" Remaining sessions: {report['feedback_summary']['remaining_sessions']}") + print(f" Top priority: {report['priorities'][0]['subset_id'] if report['priorities'] else 'none'}") + return + if args.corpus_command == "verify": result = verify_snapshot(args.snapshot_dir) print(f"Snapshot: {result['snapshot_dir']}") diff --git a/causetrace/core.py b/causetrace/core.py index 458af43..a805617 100644 --- a/causetrace/core.py +++ b/causetrace/core.py @@ -2,6 +2,7 @@ from __future__ import annotations import json +import hashlib import os import re import uuid @@ -126,6 +127,28 @@ def _safe_serialize(obj: Any) -> Any: return str(obj)[:2000] +def _stable_serialize(obj: Any) -> str: + try: + return json.dumps(obj, sort_keys=True, separators=(",", ":"), default=str) + except (TypeError, ValueError): + return str(obj) + + +def _event_fingerprint(ev: ToolEvent) -> str: + payload = { + "timestamp": ev.timestamp, + "event_type": ev.event_type, + "tool_name": ev.tool_name, + "agent": ev.agent, + "model": ev.model, + "provider": ev.provider, + "tool_input": _safe_serialize(ev.tool_input), + "tool_output": _safe_serialize(ev.tool_output), + } + raw = _stable_serialize(payload).encode("utf-8", "replace") + return hashlib.sha256(raw).hexdigest() + + def _parse_parents(ev: ToolEvent) -> List[str]: """Parse parent IDs from parent_event_id (may be comma-separated for multi-parent).""" if not ev.parent_event_id: @@ -133,6 +156,15 @@ def _parse_parents(ev: ToolEvent) -> List[str]: return [p.strip() for p in ev.parent_event_id.split(",") if p.strip()] +def _remap_parent_event_id(parent_event_id: Optional[str], id_map: Dict[str, str]) -> Optional[str]: + if not parent_event_id: + return parent_event_id + parents = [p.strip() for p in parent_event_id.split(",") if p.strip()] + if not parents: + return parent_event_id + return ",".join(id_map.get(parent, parent) for parent in parents) + + class CompressedRun: """A run of consecutive same-tool events compressed into a single node. @@ -326,6 +358,54 @@ def append(self, session_id: str, event: ToolEvent) -> None: with open(self._path(session_id), "a") as f: f.write(json.dumps(event.to_dict()) + "\n") + def append_missing( + self, + session_id: str, + events: List[ToolEvent], + *, + dry_run: bool = False, + ) -> dict: + """Append only events whose stable content fingerprint is not stored. + + Event IDs generated by post-hoc parsers are not stable. Deduplication + therefore ignores event_id and parent_event_id, then remaps parents of + newly appended events to the matching stored IDs when possible. + """ + existing_events = self.load(session_id) + fingerprint_to_id = { + _event_fingerprint(event): event.event_id + for event in existing_events + } + id_map: Dict[str, str] = {} + added = 0 + skipped = 0 + + for event in events: + fingerprint = _event_fingerprint(event) + existing_id = fingerprint_to_id.get(fingerprint) + if existing_id: + id_map[event.event_id] = existing_id + skipped += 1 + continue + + remapped_parent = _remap_parent_event_id(event.parent_event_id, id_map) + if not dry_run: + event.parent_event_id = remapped_parent + self.append(session_id, event) + + fingerprint_to_id[fingerprint] = event.event_id + id_map[event.event_id] = event.event_id + added += 1 + + return { + "session_id": session_id, + "incoming": len(events), + "existing": len(existing_events), + "added": added, + "skipped": skipped, + "dry_run": dry_run, + } + def load(self, session_id: str) -> List[ToolEvent]: path = self._path(session_id) if not path.exists(): diff --git a/causetrace/crdd/__init__.py b/causetrace/crdd/__init__.py new file mode 100644 index 0000000..b2ac396 --- /dev/null +++ b/causetrace/crdd/__init__.py @@ -0,0 +1,25 @@ +"""Causal Runtime Dataset Design tools. + +CRDD compiles stored traces into comparable subset manifests. It is read-only +over trace data and metadata sidecars. +""" + +from .comparability_score import compute_comparability_score +from .feedback import ingest_feedback, reprioritize_experiments, update_gaps +from .experiment_planner import plan_experiments +from .gap_analyzer import analyze_gaps +from .subset_builder import build_subset, compile_subsets +from .subset_registry import SUBSET_DEFINITIONS, get_subset_definition + +__all__ = [ + "SUBSET_DEFINITIONS", + "analyze_gaps", + "build_subset", + "compile_subsets", + "compute_comparability_score", + "ingest_feedback", + "get_subset_definition", + "plan_experiments", + "reprioritize_experiments", + "update_gaps", +] diff --git a/causetrace/crdd/comparability.py b/causetrace/crdd/comparability.py new file mode 100644 index 0000000..fe7f739 --- /dev/null +++ b/causetrace/crdd/comparability.py @@ -0,0 +1,41 @@ +"""CRDD comparability helpers.""" +from __future__ import annotations + +from collections import Counter +from typing import Any + +from .comparability_score import compute_comparability_score +from .experimental_units import ExperimentalUnit + + +def distribution(units: list[ExperimentalUnit], field: str) -> dict[str, int]: + """Count a comparison field across units.""" + counts: Counter[str] = Counter() + for unit in units: + if field == "topology": + value = unit.topology + elif field == "event_count": + value = str(unit.event_count) + else: + value = str(unit.metadata.get(field) or "") + counts[value or "unknown"] += 1 + return dict(sorted(counts.items())) + + +def summarize_comparability( + units: list[ExperimentalUnit], + *, + required_fields: tuple[str, ...] = (), +) -> dict[str, Any]: + """Return score plus distributions used to inspect subset comparability.""" + return { + "comparability": compute_comparability_score(units, required_fields=required_fields), + "distributions": { + "runtime": distribution(units, "runtime"), + "task_type": distribution(units, "task_type"), + "task_source": distribution(units, "task_source"), + "data_origin": distribution(units, "data_origin"), + "intervention_lane": distribution(units, "intervention_lane"), + "topology": distribution(units, "topology"), + }, + } diff --git a/causetrace/crdd/comparability_score.py b/causetrace/crdd/comparability_score.py new file mode 100644 index 0000000..ad80301 --- /dev/null +++ b/causetrace/crdd/comparability_score.py @@ -0,0 +1,85 @@ +"""Comparability scoring for CRDD subsets.""" +from __future__ import annotations + +from collections import Counter +from math import log +from typing import Any + +from .experimental_units import ExperimentalUnit + + +def _non_empty(value: Any) -> bool: + return value not in (None, "", [], {}) + + +def _normalized_entropy(values: list[str]) -> float: + counts = Counter(value for value in values if value) + if not counts: + return 0.0 + total = sum(counts.values()) + if len(counts) == 1: + return 0.0 + entropy = -sum((count / total) * log(count / total) for count in counts.values()) + return entropy / log(len(counts)) + + +def _saturating_density(count: int, total: int, target: float) -> float: + if total <= 0 or target <= 0: + return 0.0 + return min((count / total) / target, 1.0) + + +def compute_comparability_score( + units: list[ExperimentalUnit], + *, + required_fields: tuple[str, ...] = (), +) -> dict[str, Any]: + """Compute a bounded comparability score for a session set. + + The score is descriptive. It decides whether a subset is useful for + comparison; it does not validate or upgrade research claims. + """ + total = len(units) + if total == 0: + metrics = { + "metadata_completeness": 0.0, + "runtime_diversity": 0.0, + "task_variance": 0.0, + "failure_density": 0.0, + "intervention_density": 0.0, + "topology_variance": 0.0, + "reproducibility": 0.0, + } + return {"score": 0.0, "metrics": metrics} + + fields = required_fields or ("runtime", "task_type", "task_source", "success") + present = 0 + for unit in units: + for field in fields: + present += int(_non_empty(unit.metadata.get(field))) + metadata_completeness = present / (total * len(fields)) if fields else 1.0 + + failures = sum(1 for unit in units if unit.success is False) + interventions = sum( + 1 + for unit in units + if unit.human_intervention is True + or unit.intervention_lane not in ("", "direct_prompt_native") + or unit.task_source in { + "routed_prompt_intervention", + "superpowers_workflow_intervention", + "controlled_prompt_morphology", + } + ) + + metrics = { + "metadata_completeness": round(metadata_completeness, 4), + "runtime_diversity": round(_normalized_entropy([unit.runtime for unit in units]), 4), + "task_variance": round(_normalized_entropy([unit.task_type for unit in units]), 4), + "failure_density": round(_saturating_density(failures, total, 0.1), 4), + "intervention_density": round(_saturating_density(interventions, total, 0.1), 4), + "topology_variance": round(_normalized_entropy([unit.topology for unit in units]), 4), + "reproducibility": 1.0, + } + score = sum(metrics.values()) / len(metrics) + return {"score": round(score, 4), "metrics": metrics} diff --git a/causetrace/crdd/constraints.py b/causetrace/crdd/constraints.py new file mode 100644 index 0000000..7f9ced7 --- /dev/null +++ b/causetrace/crdd/constraints.py @@ -0,0 +1,59 @@ +"""Safety constraints for CERC experiment planning.""" +from __future__ import annotations + +from typing import Any + + +EXECUTION_MODE = "external_only" +EVIDENCE_STATUS = "planned_not_observed" +PHASE4_GRADE_EFFECT = "none" +PROHIBITED_EXECUTION_KEYS = { + "api_call", + "agent_command", + "bash", + "cmd", + "command", + "execute", + "shell", + "subprocess", +} + + +def _find_prohibited_keys(value: Any, *, path: str = "$") -> list[str]: + hits: list[str] = [] + if isinstance(value, dict): + for key, child in value.items(): + key_text = str(key) + child_path = f"{path}.{key_text}" + if key_text in PROHIBITED_EXECUTION_KEYS: + hits.append(child_path) + hits.extend(_find_prohibited_keys(child, path=child_path)) + elif isinstance(value, list): + for index, child in enumerate(value): + hits.extend(_find_prohibited_keys(child, path=f"{path}[{index}]")) + return hits + + +def validate_execution_queue(queue: dict[str, Any]) -> dict[str, Any]: + """Validate that a CERC queue is a plan, not an execution payload.""" + issues: list[str] = [] + if queue.get("execution_mode") != EXECUTION_MODE: + issues.append("execution_mode must be external_only") + if queue.get("must_not_execute") is not True: + issues.append("must_not_execute must be true") + if queue.get("evidence_status") != EVIDENCE_STATUS: + issues.append("evidence_status must be planned_not_observed") + if queue.get("observed_session_count") != 0: + issues.append("observed_session_count must be 0 for planned queues") + if queue.get("phase4_grade_effect") != PHASE4_GRADE_EFFECT: + issues.append("phase4_grade_effect must be none") + + for scenario in queue.get("bde_scenarios", []): + if scenario.get("descriptor_only") is not True: + issues.append(f"scenario {scenario.get('scenario_id', '')} must be descriptor_only") + + prohibited = _find_prohibited_keys(queue) + if prohibited: + issues.append("queue contains prohibited execution keys: " + ", ".join(sorted(prohibited))) + + return {"ok": not issues, "issues": issues} diff --git a/causetrace/crdd/execution_queue.py b/causetrace/crdd/execution_queue.py new file mode 100644 index 0000000..2b6d89c --- /dev/null +++ b/causetrace/crdd/execution_queue.py @@ -0,0 +1,69 @@ +"""Execution queue manifest generation for CERC. + +Queues are external-only requirements. They never contain runnable commands. +""" +from __future__ import annotations + +import hashlib +import json +from datetime import datetime +from typing import Any + +from .constraints import ( + EVIDENCE_STATUS, + EXECUTION_MODE, + PHASE4_GRADE_EFFECT, + validate_execution_queue, +) + + +def _queue_hash(queue: dict[str, Any]) -> str: + stable = { + "experiment_id": queue["experiment_id"], + "target_subset": queue["target_subset"], + "required_sessions": queue["required_sessions"], + "distribution_targets": queue["distribution_targets"], + "bde_scenarios": queue["bde_scenarios"], + "execution_mode": queue["execution_mode"], + "must_not_execute": queue["must_not_execute"], + "evidence_status": queue["evidence_status"], + "phase4_grade_effect": queue["phase4_grade_effect"], + } + payload = json.dumps(stable, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def build_execution_queue( + *, + experiment_id: str, + target_subset: str, + required_sessions: int, + distribution_targets: dict[str, dict[str, float]], + bde_scenarios: list[dict[str, Any]], +) -> dict[str, Any]: + """Build and validate an external-only experiment queue.""" + queue = { + "schema": "causetrace.cerc.execution_queue.v0.1", + "experiment_id": experiment_id, + "generated_at": datetime.now().isoformat(), + "target_subset": target_subset, + "required_sessions": required_sessions, + "distribution_targets": distribution_targets, + "bde_scenarios": bde_scenarios, + "execution_mode": EXECUTION_MODE, + "must_not_execute": True, + "evidence_status": EVIDENCE_STATUS, + "observed_session_count": 0, + "phase4_grade_effect": PHASE4_GRADE_EFFECT, + "may_trigger_future_sampling": True, + "descriptor_only": True, + "prohibited_actions": [ + "auto_agent_execution", + "runtime_mutation", + "phase4_grade_promotion", + "treating_plans_as_observed_evidence", + ], + } + queue["validation"] = validate_execution_queue(queue) + queue["queue_hash"] = _queue_hash(queue) + return queue diff --git a/causetrace/crdd/experiment_planner.py b/causetrace/crdd/experiment_planner.py new file mode 100644 index 0000000..950e83e --- /dev/null +++ b/causetrace/crdd/experiment_planner.py @@ -0,0 +1,136 @@ +"""CERC experiment requirement planner.""" +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from causetrace.core import JSONStore + +from .execution_queue import build_execution_queue +from .gap_analyzer import analyze_gaps +from .scenario_mapper import map_scenarios +from .subset_registry import get_subset_definition + + +DEFAULT_PLAN_OUTPUT_DIR = Path("docs/research/dataset_design/plans") + + +DEFAULT_DISTRIBUTION_TARGETS: dict[str, dict[str, dict[str, float]]] = { + "failure_enriched": { + "runtime": {"codex": 0.4, "claude": 0.4, "aider": 0.2}, + "task_type": {"bug_fix": 0.5, "refactor": 0.3, "review": 0.2}, + }, + "intervention_lane": { + "runtime": {"claude": 0.5, "codex": 0.3, "opencode": 0.2}, + "task_type": {"exploration": 0.4, "feature_add": 0.4, "review": 0.2}, + "intervention_lane": { + "superpowers_workflow_intervention": 0.5, + "routed_prompt_intervention": 0.25, + "controlled_prompt_morphology": 0.25, + }, + }, + "balanced_cross_runtime": { + "runtime": {"claude": 0.25, "codex": 0.25, "opencode": 0.25, "aider": 0.25}, + "task_type": {"bug_fix": 0.25, "feature_add": 0.25, "refactor": 0.25, "review": 0.25}, + }, + "strict_research_grade": { + "runtime": {"claude": 0.35, "codex": 0.35, "opencode": 0.2, "aider": 0.1}, + "task_type": {"bug_fix": 0.3, "feature_add": 0.3, "refactor": 0.2, "review": 0.2}, + }, +} + + +def _experiment_id(target_subset: str) -> str: + stamp = datetime.now().strftime("%Y%m%d-%H%M%S") + return f"exp_{target_subset}_{stamp}" + + +def _plan_markdown(plan: dict[str, Any]) -> str: + queue = plan["experiment_queue"] + gap = plan["gap"] + lines = [ + f"# CERC experiment plan: {queue['experiment_id']}", + "", + f"- target subset: `{queue['target_subset']}`", + f"- required sessions: `{queue['required_sessions']}`", + f"- current sessions: `{gap['current_sessions']}`", + f"- target sessions: `{gap['target_sessions']}`", + f"- evidence status: `{queue['evidence_status']}`", + f"- execution mode: `{queue['execution_mode']}`", + f"- must not execute: `{queue['must_not_execute']}`", + f"- Phase 4 grade effect: `{queue['phase4_grade_effect']}`", + "", + "## Distribution Targets", + "", + ] + for field, targets in queue["distribution_targets"].items(): + lines.append(f"### {field}") + for label, weight in targets.items(): + lines.append(f"- {label}: {weight}") + lines.append("") + lines.extend([ + "## BDE Scenarios", + "", + ]) + for scenario in queue["bde_scenarios"]: + lines.append(f"- `{scenario['scenario_id']}`: {scenario['purpose']}") + lines.extend([ + "", + "## Safety Boundary", + "", + "This plan is an external-only requirement queue. It is not observed evidence, does not run agents, and does not change Phase 4 evidence grades.", + ]) + return "\n".join(lines) + "\n" + + +def plan_experiments( + store: JSONStore, + *, + target_subset: str = "failure_enriched", + required_sessions: int | None = None, + output_dir: str | Path | None = None, + name: str | None = None, + write: bool = True, +) -> dict[str, Any]: + """Plan missing experimental work without executing runtimes.""" + get_subset_definition(target_subset) + gap_report = analyze_gaps(store, subset_ids=[target_subset]) + gap = gap_report["subset_gaps"][0] + required = required_sessions if required_sessions is not None else int(gap["missing_sessions"]) + experiment_id = name or _experiment_id(target_subset) + queue = build_execution_queue( + experiment_id=experiment_id, + target_subset=target_subset, + required_sessions=max(required, 0), + distribution_targets=DEFAULT_DISTRIBUTION_TARGETS.get(target_subset, {}), + bde_scenarios=map_scenarios(target_subset), + ) + plan = { + "schema": "causetrace.cerc.experiment_plan.v0.1", + "generated_at": datetime.now().isoformat(), + "target_subset": target_subset, + "gap": gap, + "gap_report": gap_report, + "experiment_queue": queue, + "claim_boundary": { + "planned_is_not_observed": True, + "phase4_grade_effect": "none", + "may_trigger_future_sampling": True, + }, + } + + root = Path(output_dir) if output_dir else DEFAULT_PLAN_OUTPUT_DIR + run_dir = root / experiment_id + if write: + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "gap_report.json").write_text(json.dumps(gap_report, indent=2, sort_keys=True), encoding="utf-8") + (run_dir / "experiment_queue.json").write_text(json.dumps(queue, indent=2, sort_keys=True), encoding="utf-8") + (run_dir / "experiment_plan.md").write_text(_plan_markdown(plan), encoding="utf-8") + + return { + "output_dir": str(run_dir), + "written": write, + "plan": plan, + } diff --git a/causetrace/crdd/experimental_units.py b/causetrace/crdd/experimental_units.py new file mode 100644 index 0000000..e0e6310 --- /dev/null +++ b/causetrace/crdd/experimental_units.py @@ -0,0 +1,78 @@ +"""Experimental-unit projection for corpus records.""" +from __future__ import annotations + +from dataclasses import dataclass +from typing import Any + + +@dataclass(frozen=True) +class ExperimentalUnit: + """A session projected into CRDD comparison dimensions.""" + + session_id: str + metadata: dict[str, Any] + metadata_provenance: dict[str, str] + stats: dict[str, Any] + topology: str + + @property + def runtime(self) -> str: + return str(self.metadata.get("runtime") or "") + + @property + def task_type(self) -> str: + return str(self.metadata.get("task_type") or "") + + @property + def task_source(self) -> str: + return str(self.metadata.get("task_source") or "") + + @property + def data_origin(self) -> str: + return str(self.metadata.get("data_origin") or "") + + @property + def intervention_lane(self) -> str: + return str(self.metadata.get("intervention_lane") or "") + + @property + def success(self) -> bool | None: + value = self.metadata.get("success") + return value if isinstance(value, bool) else None + + @property + def human_intervention(self) -> bool | None: + value = self.metadata.get("human_intervention") + return value if isinstance(value, bool) else None + + @property + def event_count(self) -> int: + return int(self.stats.get("event_count", 0) or 0) + + def has_fields(self, fields: tuple[str, ...]) -> bool: + return all(self.metadata.get(field) not in (None, "", [], {}) for field in fields) + + def to_manifest_record(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "runtime": self.runtime, + "task_type": self.task_type, + "task_source": self.task_source, + "data_origin": self.data_origin, + "intervention_lane": self.intervention_lane, + "success": self.success, + "human_intervention": self.human_intervention, + "topology": self.topology, + "event_count": self.event_count, + } + + +def record_to_unit(record: dict[str, Any]) -> ExperimentalUnit: + """Build an experimental unit from a corpus record.""" + return ExperimentalUnit( + session_id=str(record["session_id"]), + metadata=dict(record.get("metadata", {})), + metadata_provenance=dict(record.get("metadata_provenance", {})), + stats=dict(record.get("stats", {})), + topology=str(record.get("topology") or ""), + ) diff --git a/causetrace/crdd/feedback.py b/causetrace/crdd/feedback.py new file mode 100644 index 0000000..ef38ed2 --- /dev/null +++ b/causetrace/crdd/feedback.py @@ -0,0 +1,366 @@ +"""CERC feedback integration. + +This layer ingests external execution feedback and turns it into updated gap +reports and reprioritization manifests. It does not execute runtimes or mutate +trace data. +""" +from __future__ import annotations + +import json +from collections import Counter, defaultdict +from dataclasses import dataclass +from datetime import datetime +from pathlib import Path +from typing import Any, Iterable + +from causetrace.core import JSONStore +from causetrace.corpus import build_session_record + +from .experiment_planner import DEFAULT_DISTRIBUTION_TARGETS +from .gap_analyzer import analyze_gaps +from .subset_registry import SUBSET_DEFINITIONS + + +DEFAULT_FEEDBACK_OUTPUT_DIR = Path("docs/research/dataset_design/feedback") + + +@dataclass(frozen=True) +class FeedbackObservation: + """Normalized observation from an external execution feedback payload.""" + + session_id: str + runtime: str = "" + task_type: str = "" + task_source: str = "" + data_origin: str = "" + intervention_lane: str = "" + success: bool | None = None + human_intervention: bool | None = None + topology: str = "" + event_count: int = 0 + resolved: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "session_id": self.session_id, + "runtime": self.runtime, + "task_type": self.task_type, + "task_source": self.task_source, + "data_origin": self.data_origin, + "intervention_lane": self.intervention_lane, + "success": self.success, + "human_intervention": self.human_intervention, + "topology": self.topology, + "event_count": self.event_count, + "resolved": self.resolved, + } + + +def _load_payload(path: str | Path) -> dict[str, Any]: + payload_path = Path(path) + data = json.loads(payload_path.read_text(encoding="utf-8")) + if not isinstance(data, dict): + raise ValueError("feedback payload must be a JSON object") + return data + + +def _as_bool(value: Any) -> bool | None: + if value is None or isinstance(value, bool): + return value + if isinstance(value, str): + lowered = value.strip().lower() + if lowered in {"1", "true", "yes", "y"}: + return True + if lowered in {"0", "false", "no", "n"}: + return False + return None + + +def _normalize_observation(store: JSONStore, item: Any) -> FeedbackObservation: + if isinstance(item, str): + sid = item + try: + record = build_session_record(store, sid) + except Exception: + return FeedbackObservation(session_id=sid) + metadata = record.get("metadata", {}) + stats = record.get("stats", {}) + return FeedbackObservation( + session_id=sid, + runtime=str(metadata.get("runtime") or ""), + task_type=str(metadata.get("task_type") or ""), + task_source=str(metadata.get("task_source") or ""), + data_origin=str(metadata.get("data_origin") or ""), + intervention_lane=str(metadata.get("intervention_lane") or ""), + success=_as_bool(metadata.get("success")), + human_intervention=_as_bool(metadata.get("human_intervention")), + topology=str(record.get("topology") or ""), + event_count=int(stats.get("event_count", 0) or 0), + resolved=True, + ) + + if not isinstance(item, dict): + raise ValueError("observed_sessions must contain strings or objects") + + sid = str(item.get("session_id") or item.get("id") or item.get("observed_session_id") or "") + resolved = False + if sid: + try: + record = build_session_record(store, sid) + except Exception: + record = None + else: + metadata = record.get("metadata", {}) + stats = record.get("stats", {}) + resolved = True + return FeedbackObservation( + session_id=sid, + runtime=str(item.get("runtime") or metadata.get("runtime") or ""), + task_type=str(item.get("task_type") or metadata.get("task_type") or ""), + task_source=str(item.get("task_source") or metadata.get("task_source") or ""), + data_origin=str(item.get("data_origin") or metadata.get("data_origin") or ""), + intervention_lane=str(item.get("intervention_lane") or metadata.get("intervention_lane") or ""), + success=_as_bool(item.get("success")) if item.get("success") is not None else _as_bool(metadata.get("success")), + human_intervention=_as_bool(item.get("human_intervention")) + if item.get("human_intervention") is not None + else _as_bool(metadata.get("human_intervention")), + topology=str(item.get("topology") or record.get("topology") or ""), + event_count=int(item.get("event_count") or stats.get("event_count", 0) or 0), + resolved=resolved, + ) + + return FeedbackObservation( + session_id=sid or str(item.get("label") or item.get("name") or "unknown"), + runtime=str(item.get("runtime") or ""), + task_type=str(item.get("task_type") or ""), + task_source=str(item.get("task_source") or ""), + data_origin=str(item.get("data_origin") or ""), + intervention_lane=str(item.get("intervention_lane") or ""), + success=_as_bool(item.get("success")), + human_intervention=_as_bool(item.get("human_intervention")), + topology=str(item.get("topology") or ""), + event_count=int(item.get("event_count") or 0), + resolved=resolved, + ) + + +def _distribution(observations: Iterable[FeedbackObservation], field: str) -> dict[str, int]: + counts: Counter[str] = Counter() + for obs in observations: + value = getattr(obs, field) + if isinstance(value, bool): + key = str(value).lower() + else: + key = str(value or "unknown") + counts[key or "unknown"] += 1 + return dict(sorted(counts.items())) + + +def _required_target(subset_id: str) -> int: + defaults = { + "strict_research_grade": 150, + "balanced_cross_runtime": 20, + "failure_enriched": 50, + "intervention_lane": 15, + } + return defaults.get(subset_id, 0) + + +def _planned_target_sessions(payload: dict[str, Any], plan_queue: dict[str, Any], subset_id: str) -> int: + planned = plan_queue.get("required_sessions") + if isinstance(planned, int) and planned >= 0: + return planned + fallback = payload.get("required_sessions") + if isinstance(fallback, int) and fallback >= 0: + return fallback + return _required_target(subset_id) + + +def ingest_feedback( + store: JSONStore, + *, + input_path: str | Path, + plan_dir: str | Path | None = None, + output_dir: str | Path | None = None, + write: bool = True, +) -> dict[str, Any]: + """Ingest an external feedback payload into a normalized report.""" + payload = _load_payload(input_path) + observations_raw = payload.get("observed_sessions") or payload.get("observed_session_ids") or [] + if not isinstance(observations_raw, list): + raise ValueError("observed_sessions or observed_session_ids must be a list") + + observations = [_normalize_observation(store, item) for item in observations_raw] + resolved = sum(1 for obs in observations if obs.resolved) + unresolved_ids = [obs.session_id for obs in observations if not obs.resolved] + subset_id = str(payload.get("target_subset") or payload.get("subset_id") or "unknown") + experiment_id = str(payload.get("experiment_id") or payload.get("name") or "unknown") + plan_path = Path(plan_dir) if plan_dir else None + plan_queue = {} + if plan_path and (plan_path / "experiment_queue.json").exists(): + plan_queue = json.loads((plan_path / "experiment_queue.json").read_text(encoding="utf-8")) + + counts = { + "runtime": _distribution(observations, "runtime"), + "task_type": _distribution(observations, "task_type"), + "task_source": _distribution(observations, "task_source"), + "data_origin": _distribution(observations, "data_origin"), + "intervention_lane": _distribution(observations, "intervention_lane"), + "topology": _distribution(observations, "topology"), + } + + report: dict[str, Any] = { + "schema": "causetrace.cerc.feedback_report.v0.1", + "generated_at": datetime.now().isoformat(), + "experiment_id": experiment_id, + "target_subset": subset_id, + "plan_dir": str(plan_path) if plan_path else None, + "plan_queue": plan_queue, + "observed_count": len(observations), + "resolved_count": resolved, + "unresolved_session_ids": unresolved_ids, + "observed_distributions": counts, + "quality": { + "resolved_ratio": round(resolved / len(observations), 4) if observations else 0.0, + "has_failures": any(obs.success is False for obs in observations), + "has_interventions": any( + obs.human_intervention is True + or obs.intervention_lane not in ("", "direct_prompt_native") + or obs.task_source in { + "routed_prompt_intervention", + "superpowers_workflow_intervention", + "controlled_prompt_morphology", + } + for obs in observations + ), + }, + "observations": [obs.to_dict() for obs in observations], + "constraints": { + "external_only": True, + "planned_is_not_observed": True, + "may_affect_future_sampling": True, + }, + } + + target = _planned_target_sessions(payload, plan_queue, subset_id) + report["gap_projection"] = { + "target_sessions": target, + "observed_sessions": len(observations), + "remaining_sessions": max(target - len(observations), 0), + "progress_ratio": round(len(observations) / target, 4) if target else 0.0, + } + + if write: + root = Path(output_dir) if output_dir else DEFAULT_FEEDBACK_OUTPUT_DIR + run_dir = root / experiment_id + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "feedback_report.json").write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + report["output_dir"] = str(run_dir) + else: + report["output_dir"] = None + return report + + +def update_gaps( + store: JSONStore, + *, + feedback_report: dict[str, Any], + output_dir: str | Path | None = None, + write: bool = True, +) -> dict[str, Any]: + """Update gap projections from a normalized feedback report.""" + target_subset = str(feedback_report.get("target_subset") or "unknown") + current_gap = analyze_gaps(store, subset_ids=[target_subset]) if target_subset in SUBSET_DEFINITIONS else None + current_subset_gap = (current_gap or {}).get("subset_gaps", [{}])[0] + observed_count = int(feedback_report.get("observed_count", 0) or 0) + target = int(feedback_report.get("gap_projection", {}).get("target_sessions", 0) or 0) + remaining = max(target - observed_count, 0) + updated = { + "schema": "causetrace.cerc.gap_update.v0.1", + "generated_at": datetime.now().isoformat(), + "experiment_id": feedback_report.get("experiment_id"), + "target_subset": target_subset, + "current_gap": current_subset_gap, + "feedback_gap_projection": feedback_report.get("gap_projection", {}), + "observed_count": observed_count, + "remaining_sessions": remaining, + "status": "met" if remaining == 0 else "under_target", + "priority_hint": "reprioritize" if remaining > 0 else "hold", + "quality": feedback_report.get("quality", {}), + } + if write: + root = Path(output_dir) if output_dir else DEFAULT_FEEDBACK_OUTPUT_DIR + run_dir = root / str(feedback_report.get("experiment_id") or "unknown") + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "gap_update.json").write_text(json.dumps(updated, indent=2, sort_keys=True), encoding="utf-8") + updated["output_dir"] = str(run_dir) + else: + updated["output_dir"] = None + return updated + + +def reprioritize_experiments( + store: JSONStore, + *, + feedback_report: dict[str, Any], + output_dir: str | Path | None = None, + write: bool = True, +) -> dict[str, Any]: + """Reprioritize future experiment planning based on feedback.""" + current_gaps = analyze_gaps(store) + remaining_by_subset = { + gap["subset_id"]: int(gap["missing_sessions"]) + for gap in current_gaps["subset_gaps"] + } + feedback_target = str(feedback_report.get("target_subset") or "unknown") + feedback_remaining = int(feedback_report.get("gap_projection", {}).get("remaining_sessions", 0) or 0) + priorities: list[dict[str, Any]] = [] + for gap in current_gaps["subset_gaps"]: + subset_id = gap["subset_id"] + remaining = remaining_by_subset.get(subset_id, 0) + severity = gap["severity"] + severity_weight = {"high": 3, "medium": 2, "low": 1, "none": 0}.get(severity, 0) + feedback_boost = 1 if subset_id == feedback_target and feedback_remaining > 0 else 0 + priority_score = remaining + severity_weight + feedback_boost + priorities.append({ + "subset_id": subset_id, + "current_sessions": gap["current_sessions"], + "target_sessions": gap["target_sessions"], + "remaining_sessions": remaining, + "severity": severity, + "comparability_score": gap["comparability_score"], + "priority_score": priority_score, + "reason": ( + f"remaining={remaining}, severity={severity}, " + f"feedback_target={subset_id == feedback_target}" + ), + "default_distribution_targets": DEFAULT_DISTRIBUTION_TARGETS.get(subset_id, {}), + }) + + priorities.sort(key=lambda item: (-item["priority_score"], item["subset_id"])) + report = { + "schema": "causetrace.cerc.reprioritized_plan.v0.1", + "generated_at": datetime.now().isoformat(), + "experiment_id": feedback_report.get("experiment_id"), + "target_subset": feedback_target, + "priorities": priorities, + "feedback_summary": { + "observed_count": feedback_report.get("observed_count", 0), + "resolved_count": feedback_report.get("resolved_count", 0), + "remaining_sessions": feedback_remaining, + }, + "constraints": { + "no_execution": True, + "no_evidence_inflation": True, + "no_phase4_grade_promotion": True, + }, + } + if write: + root = Path(output_dir) if output_dir else DEFAULT_FEEDBACK_OUTPUT_DIR + run_dir = root / str(feedback_report.get("experiment_id") or "unknown") + run_dir.mkdir(parents=True, exist_ok=True) + (run_dir / "reprioritized_plan.json").write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + report["output_dir"] = str(run_dir) + else: + report["output_dir"] = None + return report diff --git a/causetrace/crdd/gap_analyzer.py b/causetrace/crdd/gap_analyzer.py new file mode 100644 index 0000000..d5b5169 --- /dev/null +++ b/causetrace/crdd/gap_analyzer.py @@ -0,0 +1,77 @@ +"""Corpus gap analysis for CERC experiment planning.""" +from __future__ import annotations + +import json +from datetime import datetime +from pathlib import Path +from typing import Any + +from causetrace.core import JSONStore + +from .subset_builder import compile_subsets +from .subset_registry import SUBSET_DEFINITIONS + + +DEFAULT_SUBSET_TARGETS = { + "strict_research_grade": 150, + "balanced_cross_runtime": 20, + "failure_enriched": 50, + "intervention_lane": 15, +} + + +def _severity(missing: int, target: int) -> str: + if missing <= 0: + return "none" + ratio = missing / target if target else 0 + if ratio >= 0.75: + return "high" + if ratio >= 0.35: + return "medium" + return "low" + + +def analyze_gaps( + store: JSONStore, + *, + subset_ids: list[str] | None = None, + targets: dict[str, int] | None = None, + output: str | Path | None = None, +) -> dict[str, Any]: + """Analyze CRDD subset gaps without mutating corpus data.""" + requested = subset_ids or list(SUBSET_DEFINITIONS) + target_map = dict(DEFAULT_SUBSET_TARGETS) + if targets: + target_map.update(targets) + + compiled = compile_subsets(store, subset_ids=requested, write=False) + subset_gaps: list[dict[str, Any]] = [] + for manifest in compiled["manifests"]: + subset_id = manifest["subset_id"] + current = int(manifest["selected_count"]) + target = int(target_map.get(subset_id, current)) + missing = max(target - current, 0) + subset_gaps.append({ + "subset_id": subset_id, + "current_sessions": current, + "target_sessions": target, + "missing_sessions": missing, + "status": "met" if missing == 0 else "under_target", + "severity": _severity(missing, target), + "comparability_score": manifest["comparability"]["score"], + "distributions": manifest["distributions"], + "bias_register": manifest["bias_register"], + "prohibited_claims": manifest["prohibited_claims"], + }) + + report = { + "schema": "causetrace.cerc.gap_report.v0.1", + "generated_at": datetime.now().isoformat(), + "source_session_count": compiled["source_session_count"], + "subset_gaps": subset_gaps, + } + if output: + path = Path(output) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(json.dumps(report, indent=2, sort_keys=True), encoding="utf-8") + return report diff --git a/causetrace/crdd/scenario_mapper.py b/causetrace/crdd/scenario_mapper.py new file mode 100644 index 0000000..1e0c109 --- /dev/null +++ b/causetrace/crdd/scenario_mapper.py @@ -0,0 +1,56 @@ +"""Map CRDD subset gaps to BDE scenario descriptors.""" +from __future__ import annotations + +from typing import Any + + +SCENARIO_MAP: dict[str, list[dict[str, Any]]] = { + "failure_enriched": [ + { + "scenario_id": "partial_context_v1", + "purpose": "increase near-failure coverage through incomplete context tasks", + "descriptor_only": True, + }, + { + "scenario_id": "ambiguous_requirements_v1", + "purpose": "surface clarification and recovery behavior", + "descriptor_only": True, + }, + { + "scenario_id": "tool_failure_observation_v1", + "purpose": "observe runtime response to failed tool feedback", + "descriptor_only": True, + }, + ], + "intervention_lane": [ + { + "scenario_id": "superpowers_workflow_prompt_v1", + "purpose": "expand explicit workflow-intervention traces", + "descriptor_only": True, + }, + { + "scenario_id": "human_checkpoint_v1", + "purpose": "collect human intervention boundary observations", + "descriptor_only": True, + }, + ], + "balanced_cross_runtime": [ + { + "scenario_id": "cross_runtime_matched_task_v1", + "purpose": "collect matched task traces across runtimes", + "descriptor_only": True, + }, + ], + "strict_research_grade": [ + { + "scenario_id": "metadata_complete_native_run_v1", + "purpose": "collect native runs with complete comparison metadata", + "descriptor_only": True, + }, + ], +} + + +def map_scenarios(target_subset: str) -> list[dict[str, Any]]: + """Return descriptor-only BDE scenarios for a target subset.""" + return [dict(item) for item in SCENARIO_MAP.get(target_subset, [])] diff --git a/causetrace/crdd/subset_builder.py b/causetrace/crdd/subset_builder.py new file mode 100644 index 0000000..4e8f3f9 --- /dev/null +++ b/causetrace/crdd/subset_builder.py @@ -0,0 +1,247 @@ +"""Read-only CRDD subset compiler.""" +from __future__ import annotations + +import hashlib +import json +from collections import Counter, defaultdict +from datetime import datetime +from pathlib import Path +from typing import Any + +from causetrace.core import JSONStore +from causetrace.corpus import list_corpus_records + +from .comparability import summarize_comparability +from .experimental_units import ExperimentalUnit, record_to_unit +from .subset_registry import SUBSET_DEFINITIONS, get_subset_definition + + +DEFAULT_SUBSET_OUTPUT_DIR = Path("docs/research/dataset_design/manifests") +INTERVENTION_SOURCES = { + "routed_prompt_intervention", + "superpowers_workflow_intervention", + "controlled_prompt_morphology", +} + + +def _manifest_hash(manifest: dict[str, Any]) -> str: + stable = { + "subset_id": manifest["subset_id"], + "session_ids": manifest["session_ids"], + "selected_count": manifest["selected_count"], + "excluded_count": manifest["excluded_count"], + "exclusion_counts": manifest["exclusion_counts"], + "comparability": manifest["comparability"], + "distributions": manifest["distributions"], + } + payload = json.dumps(stable, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + return hashlib.sha256(payload.encode("utf-8")).hexdigest() + + +def _exclusion(reason_counts: Counter[str], reason: str) -> bool: + reason_counts[reason] += 1 + return False + + +def _select_strict(unit: ExperimentalUnit, reason_counts: Counter[str]) -> bool: + definition = get_subset_definition("strict_research_grade") + if not unit.has_fields(definition.required_fields): + return _exclusion(reason_counts, "missing_required_metadata") + return True + + +def _select_failure(unit: ExperimentalUnit, reason_counts: Counter[str]) -> bool: + definition = get_subset_definition("failure_enriched") + if not unit.has_fields(definition.required_fields): + return _exclusion(reason_counts, "missing_required_metadata") + if unit.success is False or unit.human_intervention is True: + return True + return _exclusion(reason_counts, "not_failure_or_near_failure") + + +def _select_intervention(unit: ExperimentalUnit, reason_counts: Counter[str]) -> bool: + definition = get_subset_definition("intervention_lane") + if not unit.has_fields(definition.required_fields): + return _exclusion(reason_counts, "missing_required_metadata") + if unit.intervention_lane and unit.intervention_lane != "direct_prompt_native": + return True + if unit.task_source in INTERVENTION_SOURCES: + return True + if unit.human_intervention is True: + return True + return _exclusion(reason_counts, "not_intervention_lane") + + +def _balance_by_runtime(units: list[ExperimentalUnit], reason_counts: Counter[str]) -> list[ExperimentalUnit]: + definition = get_subset_definition("balanced_cross_runtime") + buckets: dict[str, list[ExperimentalUnit]] = defaultdict(list) + for unit in units: + if not unit.has_fields(definition.required_fields): + reason_counts["missing_required_metadata"] += 1 + continue + buckets[unit.runtime].append(unit) + + populated = {runtime: sorted(items, key=lambda unit: unit.session_id) for runtime, items in buckets.items() if runtime} + if len(populated) < 2: + reason_counts["insufficient_runtime_breadth"] += sum(len(items) for items in populated.values()) + return [] + + target = min(len(items) for items in populated.values()) + selected: list[ExperimentalUnit] = [] + selected_ids: set[str] = set() + for runtime in sorted(populated): + chosen = populated[runtime][:target] + selected.extend(chosen) + selected_ids.update(unit.session_id for unit in chosen) + reason_counts["runtime_cap_excluded"] += max(len(populated[runtime]) - target, 0) + + return sorted(selected, key=lambda unit: unit.session_id) + + +def _bias_register(units: list[ExperimentalUnit], total_units: int) -> dict[str, dict[str, Any]]: + total = len(units) + runtime_counts = Counter(unit.runtime or "unknown" for unit in units) + task_counts = Counter(unit.task_type or "unknown" for unit in units) + failures = sum(1 for unit in units if unit.success is False) + interventions = sum( + 1 + for unit in units + if unit.human_intervention is True + or unit.intervention_lane not in ("", "direct_prompt_native") + or unit.task_source in INTERVENTION_SOURCES + ) + unknown_required = sum( + 1 + for unit in units + if not unit.runtime or not unit.task_type or unit.success is None + ) + top_runtime_share = max(runtime_counts.values()) / total if total and runtime_counts else 0.0 + top_task_share = max(task_counts.values()) / total if total and task_counts else 0.0 + return { + "unlabeled_majority": { + "present": total < total_units, + "detail": f"{total}/{total_units} sessions selected", + }, + "failure_scarcity": { + "present": failures < 10, + "detail": f"{failures} failure sessions", + }, + "intervention_scarcity": { + "present": interventions < 10, + "detail": f"{interventions} intervention or near-intervention sessions", + }, + "runtime_imbalance": { + "present": top_runtime_share > 0.6 if total else False, + "detail": f"top runtime share {top_runtime_share:.2f}", + }, + "task_imbalance": { + "present": top_task_share > 0.6 if total else False, + "detail": f"top task share {top_task_share:.2f}", + }, + "success_label_scarcity": { + "present": unknown_required > 0, + "detail": f"{unknown_required} selected sessions missing runtime/task/success", + }, + "duration_absence": { + "present": any(unit.metadata.get("duration") in (None, "", [], {}) for unit in units), + "detail": "one or more selected sessions lacks duration", + }, + "post_hoc_parsing": { + "present": True, + "detail": "subset may include parser-derived causality depending on runtime", + }, + } + + +def build_subset( + records: list[dict[str, Any]], + subset_id: str, +) -> dict[str, Any]: + """Build one CRDD subset manifest from corpus records.""" + definition = get_subset_definition(subset_id) + units = [record_to_unit(record) for record in records] + reason_counts: Counter[str] = Counter() + + if subset_id == "strict_research_grade": + selected = [unit for unit in units if _select_strict(unit, reason_counts)] + elif subset_id == "failure_enriched": + selected = [unit for unit in units if _select_failure(unit, reason_counts)] + elif subset_id == "intervention_lane": + selected = [unit for unit in units if _select_intervention(unit, reason_counts)] + elif subset_id == "balanced_cross_runtime": + selected = _balance_by_runtime(units, reason_counts) + else: + raise ValueError(f"Unhandled CRDD subset: {subset_id}") + + selected = sorted(selected, key=lambda unit: unit.session_id) + selected_ids = {unit.session_id for unit in selected} + reason_counts["not_selected"] += len([unit for unit in units if unit.session_id not in selected_ids]) - sum(reason_counts.values()) + if reason_counts["not_selected"] <= 0: + reason_counts.pop("not_selected", None) + + comparability = summarize_comparability(selected, required_fields=definition.required_fields) + manifest = { + "schema": "causetrace.crdd.subset_manifest.v0.1", + "subset_id": subset_id, + "purpose": definition.purpose, + "generated_at": datetime.now().isoformat(), + "source": "corpus", + "source_session_count": len(records), + "selected_count": len(selected), + "excluded_count": max(len(records) - len(selected), 0), + "required_fields": list(definition.required_fields), + "prohibited_claims": list(definition.prohibited_claims), + "exclusion_counts": dict(sorted(reason_counts.items())), + "comparability": comparability["comparability"], + "distributions": comparability["distributions"], + "bias_register": _bias_register(selected, len(records)), + "session_ids": [unit.session_id for unit in selected], + "sessions": [unit.to_manifest_record() for unit in selected], + } + manifest["manifest_hash"] = _manifest_hash(manifest) + return manifest + + +def compile_subsets( + store: JSONStore, + *, + subset_ids: list[str] | None = None, + output_dir: str | Path | None = None, + name: str | None = None, + write: bool = True, +) -> dict[str, Any]: + """Compile CRDD subset manifests from the current corpus. + + This function is read-only over trace data and metadata. When ``write`` is + true it writes generated manifests only. + """ + selected_ids = subset_ids or list(SUBSET_DEFINITIONS) + records = list_corpus_records(store) + manifests = [build_subset(records, subset_id) for subset_id in selected_ids] + + run_name = name or datetime.now().strftime("%Y%m%d-%H%M%S") + root = Path(output_dir) if output_dir else DEFAULT_SUBSET_OUTPUT_DIR + run_dir = root / run_name + if write: + run_dir.mkdir(parents=True, exist_ok=True) + for manifest in manifests: + path = run_dir / f"{manifest['subset_id']}.json" + path.write_text(json.dumps(manifest, indent=2, sort_keys=True), encoding="utf-8") + index = { + "schema": "causetrace.crdd.compile_index.v0.1", + "generated_at": datetime.now().isoformat(), + "source_session_count": len(records), + "subset_ids": [manifest["subset_id"] for manifest in manifests], + "manifests": { + manifest["subset_id"]: f"{manifest['subset_id']}.json" + for manifest in manifests + }, + } + (run_dir / "index.json").write_text(json.dumps(index, indent=2, sort_keys=True), encoding="utf-8") + + return { + "output_dir": str(run_dir), + "source_session_count": len(records), + "manifests": manifests, + "written": write, + } diff --git a/causetrace/crdd/subset_registry.py b/causetrace/crdd/subset_registry.py new file mode 100644 index 0000000..d5e083d --- /dev/null +++ b/causetrace/crdd/subset_registry.py @@ -0,0 +1,51 @@ +"""Named CRDD subset definitions.""" +from __future__ import annotations + +from dataclasses import dataclass + + +@dataclass(frozen=True) +class SubsetDefinition: + """Selection contract for a comparable or experimental subset.""" + + subset_id: str + purpose: str + required_fields: tuple[str, ...] + prohibited_claims: tuple[str, ...] = () + + +SUBSET_DEFINITIONS: dict[str, SubsetDefinition] = { + "strict_research_grade": SubsetDefinition( + subset_id="strict_research_grade", + purpose="Baseline runtime morphology with declared core metadata.", + required_fields=("runtime", "task_type", "task_source", "success"), + prohibited_claims=("runtime-balanced conclusions", "failure dynamics"), + ), + "balanced_cross_runtime": SubsetDefinition( + subset_id="balanced_cross_runtime", + purpose="Runtime comparison with capped per-runtime counts.", + required_fields=("runtime", "task_type", "task_source", "success"), + prohibited_claims=("whole-corpus prevalence",), + ), + "failure_enriched": SubsetDefinition( + subset_id="failure_enriched", + purpose="Failure and near-failure boundary analysis.", + required_fields=("runtime", "task_type", "success"), + prohibited_claims=("overall success-rate estimation",), + ), + "intervention_lane": SubsetDefinition( + subset_id="intervention_lane", + purpose="Control-vs-intervention morphology study input.", + required_fields=("runtime", "task_type", "task_source"), + prohibited_claims=("native baseline conclusions",), + ), +} + + +def get_subset_definition(subset_id: str) -> SubsetDefinition: + """Return a subset definition or raise a clear error.""" + try: + return SUBSET_DEFINITIONS[subset_id] + except KeyError as exc: + known = ", ".join(sorted(SUBSET_DEFINITIONS)) + raise ValueError(f"Unknown CRDD subset: {subset_id}. Known: {known}") from exc diff --git a/causetrace/hooks/opencode_parser.py b/causetrace/hooks/opencode_parser.py index 134a1a5..c561d3f 100644 --- a/causetrace/hooks/opencode_parser.py +++ b/causetrace/hooks/opencode_parser.py @@ -100,7 +100,11 @@ def _get_parent_model( def _parse_part( - part_data: dict, msg_data: dict, model_info: Dict[str, str], index: int + part_data: dict, + msg_data: dict, + model_info: Dict[str, str], + index: int, + fallback_ms: Optional[int] = None, ) -> Optional[ToolEvent]: """Convert a part entry into a ToolEvent. Returns None for skipped types.""" ptype = part_data.get("type", "") @@ -110,7 +114,7 @@ def _parse_part( tool_name="Thinking", tool_input={"content": (part_data.get("text") or "")[:2000]}, event_type="reasoning", - timestamp=_extract_ts(part_data, "start"), + timestamp=_extract_ts(part_data, "start", fallback_ms=fallback_ms), model=model_info.get("model"), provider=model_info.get("provider"), agent=model_info.get("agent") or "opencode", @@ -124,7 +128,7 @@ def _parse_part( tool_name="Response", tool_input={"text": text[:500]}, event_type="reasoning", - timestamp=_extract_ts(part_data, "start"), + timestamp=_extract_ts(part_data, "start", fallback_ms=fallback_ms), model=model_info.get("model"), provider=model_info.get("provider"), agent=model_info.get("agent") or "opencode", @@ -141,7 +145,7 @@ def _parse_part( tool_input=tool_input if isinstance(tool_input, dict) else {"text": str(tool_input)[:2000]}, tool_output=tool_output, event_type="tool_call", - timestamp=_extract_ts(part_data, "start"), + timestamp=_extract_ts(part_data, "start", fallback_ms=fallback_ms), duration_ms=duration, model=model_info.get("model"), provider=model_info.get("provider"), @@ -154,7 +158,7 @@ def _parse_part( tool_name="Edit", tool_input={"files": files}, event_type="tool_call", - timestamp=_extract_ts(part_data, "start"), + timestamp=_extract_ts(part_data, "start", fallback_ms=fallback_ms), agent="opencode", ) @@ -293,14 +297,10 @@ def parse_session(session_id: str) -> List[ToolEvent]: else: parent_info = _get_parent_model(msg_data.get("parentID"), model_cache) - event = _parse_part(part_data, msg_data, parent_info, 0) + event = _parse_part(part_data, msg_data, parent_info, 0, fallback_ms=ts_ms) if event is None: continue - # Use DB time_created as fallback timestamp if part lacks embedded time - if event.timestamp is None and ts_ms: - event.timestamp = _extract_ts({"time": {"start": ts_ms}}, "start") - event.parent_event_id = last_event_id events.append(event) last_event_id = event.event_id diff --git a/causetrace/metadata.py b/causetrace/metadata.py index 4f7cd84..979cd9c 100644 --- a/causetrace/metadata.py +++ b/causetrace/metadata.py @@ -58,6 +58,10 @@ "causetrace_tags", "intervention_evidence_source", "intervention_evidence_level", + "behavior_distribution_tag", + "bde_generated", + "experiment_id", + "control_group_id", } @@ -79,6 +83,10 @@ class SessionMetadata: causetrace_tags: list[str] | None = None intervention_evidence_source: str | None = None intervention_evidence_level: str | None = None + behavior_distribution_tag: str | None = None + bde_generated: bool | None = None + experiment_id: str | None = None + control_group_id: str | None = None @classmethod def from_dict(cls, data: dict[str, Any]) -> "SessionMetadata": @@ -149,6 +157,7 @@ def validate_metadata(metadata: SessionMetadata | dict[str, Any]) -> SessionMeta meta.success = _coerce_bool(meta.success, "success") meta.human_intervention = _coerce_bool(meta.human_intervention, "human_intervention") + meta.bde_generated = _coerce_bool(meta.bde_generated, "bde_generated") meta.duration = _coerce_duration(meta.duration) return meta diff --git a/docs/project-principles.md b/docs/project-principles.md index d06cd58..e0d08ba 100644 --- a/docs/project-principles.md +++ b/docs/project-principles.md @@ -57,6 +57,20 @@ External research papers may inform hypotheses, terminology, and research questi Literature-driven ideas should first enter a literature note and the hypothesis registry, not the core implementation or the primary taxonomy. +## Data-Grounded Method Rule + +Do not assume a method improves performance, safety, or auditability. Record it, compare it, then decide. + +All future method claims must be grounded in recorded runtime traces. A method is not considered effective unless its impact on performance, morphology, and auditability is separately recorded. + +Final task success is not enough. A method may improve auditability while increasing overhead, reduce runtime events while weakening evidence, or improve task success while reducing human reviewability. + +## Comparability Rule + +Raw session volume is not research sample size. + +Cross-session claims require a named comparable corpus or experimental subset with disclosed inclusion rules, denominators, metadata tiers, and sampling bias. Metadata work serves comparability; it is not a goal by itself. + ## Critical External Research Absorption Rule External research should be critically absorbed. diff --git a/docs/research/README.md b/docs/research/README.md index 25dbb32..c0c77f4 100644 --- a/docs/research/README.md +++ b/docs/research/README.md @@ -2,6 +2,10 @@ This directory groups the research tracks and branch studies that sit alongside the main `causetrace` runtime-morphology work. +## System Definition + +- [AI Behavior Science OS v1.0](ai_behavior_science_os_v1.0.md): system-level architecture for generation, execution, observation, dataset design, and research validation. Current implementation starts with metadata-only BDE interfaces and read-only CRDD subset compilation. + ## Research Phase Status | Phase | Status | Summary | @@ -59,6 +63,18 @@ Phase 4 must not enter: - [Metadata Capture Hardening v0.2.5](corpus/metadata_capture_hardening_v0.2.5.md) — Upstream capture requirements, source-specific defaults, current baseline (108/992 labeled, 10.9%) - [Future Run Metadata Checklist v0.2.5](corpus/future_run_metadata_checklist_v0.2.5.md) — Per-capture-type metadata declaration checklists, post-capture verification, non-rules +## Dataset Design + +- [Causal Runtime Dataset Design v0.1](dataset_design/crdd_v0.1.md): defines the transition from raw trace volume to comparable experimental corpora. The central bottleneck is comparability, not ingestion volume. Future evidence refresh should build named subsets such as `strict_research_grade`, `balanced_cross_runtime`, `failure_enriched`, and `intervention_lane` before revisiting theory candidates. +- [Subset Manifest Template](dataset_design/subset_manifest_template.md): required structure for comparable and experimental subsets used in Phase 4 candidate revalidation. +- [Causal Experiment Requirement Compiler v0.3](dataset_design/cerc_v0.3.md): experiment planning layer that turns observed subset gaps into external-only execution queues. CERC plans do not execute agents, inflate evidence, or upgrade Phase 4 grades. +- [CERC Feedback Integration v0.4](dataset_design/feedback_v0.4.md): read-only feedback layer that ingests external execution results, updates gap projections, and reprioritizes future experiments without changing runtime authority. + +## Roadmap And Future Directions + +- [Agentic Auditability Roadmap](roadmap/agentic_auditability_roadmap_v0.2.5.md): `causetrace` may eventually extend from runtime morphology to auditability morphology, studying performance-auditability trade-offs, opaque communication boundaries, traceability of audit agents, and structured `machine_state` / `human_audit_summary` workflows. Long-term positioning: `causetrace` studies the observable runtime and auditability morphology of high-performance AI agents. This is a future direction only; Phase 4 remains frozen and Phase 5 is not open. +- [Data-Grounded Runtime Method Discovery](roadmap/data_grounded_method_discovery_v0.2.5.md): future method claims should come from recorded runtime traces and separate performance, morphology, and auditability effects. Methods are recorded, compared, and revised before being treated as effective. + ## Cross-project Branch Studies - [Cross-project Prompt Morphology Study](branches/cross_project_prompt_morphology/README.md) diff --git a/docs/research/ai_behavior_science_os_v1.0.md b/docs/research/ai_behavior_science_os_v1.0.md new file mode 100644 index 0000000..17907a5 --- /dev/null +++ b/docs/research/ai_behavior_science_os_v1.0.md @@ -0,0 +1,111 @@ +# AI Behavior Science OS v1.0 + +`causetrace` is evolving from a runtime tracing tool into a full-stack AI +behavior science operating system with explicit separation of generation, +execution, observation, dataset design, and research validation. + +## System Definition + +AI Behavior Science OS is a system that: + +1. generates controlled agent behavior scenarios through BDE, +2. executes them across agent runtimes, +3. captures runtime traces through `causetrace`, +4. structures traces into comparable datasets through CRDD, +5. derives and tests runtime morphology hypotheses. + +## Architecture + +```text + +------------------------------------+ + | Behavior Generation | + | (BDE) | + | - prompt shaping | + | - failure injection | + | - multi-agent simulation | + | - task distribution design | + +----------------+-------------------+ + | + v + +------------------------------------+ + | Execution Layer | + | Agent Runtime Pool | + | - Claude Code | + | - Codex CLI | + | - OpenCode | + | - Aider | + | - Continue.dev | + +----------------+-------------------+ + | + v + +------------------------------------+ + | Observability Layer | + | causetrace core | + | - event logging | + | - DAG / topology | + | - tool calls | + | - intervention tracking | + +----------------+-------------------+ + | + v + +------------------------------------+ + | Dataset Design Layer | + | CRDD | + | - comparable corpus | + | - subset definitions | + | - comparability scoring | + | - experimental units | + +----------------+-------------------+ + | + v + +------------------------------------+ + | Research Layer | + | Phase 4 / Hypothesis System | + | - runtime morphology theory | + | - evidence grading | + | - validation triggers | + +------------------------------------+ +``` + +## Control Loop + +```text +BDE -> Agent Runtime -> causetrace -> CRDD -> Hypothesis -> BDE +``` + +The loop is a research design loop, not an autonomous optimization loop. +No layer may silently change another layer's authority. + +## Layer Authority + +| Layer | Authority | Prohibition | +| --- | --- | --- | +| BDE | generate scenario descriptors | must not modify actual agent execution | +| Runtime pool | execute selected scenarios | must not define research claims | +| causetrace core | observe and persist traces | must not shape behavior | +| CRDD | structure comparable datasets | must not mutate corpus data | +| Research layer | grade evidence and hypotheses | must not hide denominators or lane scope | + +## Current Implementation Boundary + +The v1.0 implementation starts with passive interfaces: + +- `causetrace.bde`: metadata-only scenario descriptors. +- `causetrace.crdd`: read-only subset compilation and comparability scoring. +- `causetrace corpus compile-subsets`: generates subset manifests from existing + corpus records. +- `causetrace corpus analyze-gaps`: reports CERC subset coverage gaps. +- `causetrace corpus plan-experiments`: emits external-only experiment + requirement queues with `must_not_execute=true`. +- `causetrace corpus ingest-feedback`, `update-gaps`, and + `reprioritize-experiments`: normalize external feedback and reprioritize + future sampling without execution authority. + +The minimal experiment runner is not part of this boundary yet. It requires a +separate design review because it touches the execution layer. + +## Safety Constraint + +BDE must remain generate-only. `causetrace` remains a passive observer. CRDD +remains read-only over stored trace data and metadata. Any future experiment +runner must preserve these authority boundaries. diff --git a/docs/research/branches/agentic_auditability/machine_state_pilot_v0.1.md b/docs/research/branches/agentic_auditability/machine_state_pilot_v0.1.md new file mode 100644 index 0000000..600fbde --- /dev/null +++ b/docs/research/branches/agentic_auditability/machine_state_pilot_v0.1.md @@ -0,0 +1,99 @@ +# Machine-State Reasoning Pilot v0.1 + +## Status + +Single-run pilot complete. + +This note belongs to the future `agentic_auditability_branch`. It is not a Phase 4 evidence-grade update, does not open Phase 4-3, and does not open Phase 5. + +## Question + +Does `machine_state + human_audit_summary` improve runtime efficiency or audit structure compared with an ordinary natural-language prompt on the same small read-only inspection task? + +## Compared Sessions + +| Group | Prompt variant | Session | +| --- | --- | --- | +| A | ordinary natural-language prompt | `019ec9b8-f1ca-7ca0-ab17-25963df3a238` | +| C | `machine_state + human_audit_summary` | `019ec9b9-b85b-7b53-a1c9-2093695892d3` | + +Task: inspect `docs/research/roadmap/agentic_auditability_roadmap_v0.2.5.md` and answer whether it clearly distinguishes prompt-level compressed text, structured machine-state reasoning, and true latent/vector communication. + +## Observed Metrics + +| Metric | A: natural-language prompt | C: `machine_state + human_audit_summary` | Result | +| --- | ---: | ---: | --- | +| events | 8 | 9 | C higher | +| `exec_command` calls | 2 | 3 | C higher | +| topology | `dominant_chain` | `dominant_chain` | no topology change | +| max depth | 7 | 8 | C higher | +| time span | 18s | 26s | C higher | +| output tokens | 605 | 1107 | C higher | +| reasoning tokens | 132 | 240 | C higher | +| final answer words | 119 | 148 | C higher | +| validation | pass | pass | both valid | + +## Pilot Result + +On this small read-only inspection task, `machine_state + human_audit_summary` improved audit structure but did not improve runtime efficiency compared with a natural-language prompt. + +It increased event count, tool-call count, elapsed span, output tokens, and reasoning tokens. Both runs produced correct answers. The main observed benefit was clearer audit structure, not performance improvement. + +## Interpretation Boundary + +This pilot does not support the claim: + +```text +machine_state -> fewer tokens / faster execution / shorter runtime chain +``` + +It does support the narrower observation: + +```text +machine_state + human_audit_summary -> clearer audit output structure +``` + +The result should be interpreted as: + +```text +Auditability instrumentation may improve reviewability while increasing runtime overhead. +``` + +## Limitations + +- Single-run only. +- Small read-only inspection task. +- No accuracy difference: both variants answered correctly. +- Not generalizable to long-horizon, multi-file, handoff-heavy, safety-boundary, or constraint-heavy tasks. +- The task was too simple to test whether structured state reduces context drift or handoff loss. +- The result must not be used to promote a mainline theory candidate or modify Phase 4 evidence grades. + +## Revised Hypotheses + +H-MS-001: Auditability benefit + +```text +machine_state + human_audit_summary may improve audit structure and decision trace coverage. +``` + +H-MS-002: Overhead cost + +```text +machine_state + human_audit_summary may increase runtime overhead, especially on small or simple tasks. +``` + +H-MS-003: Task-dependent value + +```text +machine_state may be more useful for long-horizon, multi-step, handoff, safety-boundary, or constraint-heavy tasks than for small read-only inspection tasks. +``` + +## Mainline Impact + +- Phase 4 remains frozen. +- Phase 4-3 remains trigger-gated. +- Phase 5 remains closed. +- No schema fields are added. +- No topology taxonomy changes are made. +- No unknown sessions are classified. +- No performance improvement is claimed. diff --git a/docs/research/dataset_design/cerc_v0.3.md b/docs/research/dataset_design/cerc_v0.3.md new file mode 100644 index 0000000..9fc487e --- /dev/null +++ b/docs/research/dataset_design/cerc_v0.3.md @@ -0,0 +1,99 @@ +# Causal Experiment Requirement Compiler v0.3 + +CERC is the experiment planning layer above CRDD. It turns observed corpus gaps +into external-only experiment requirements. It does not execute agents, simulate +execution, or upgrade evidence grades. + +## Definition + +```text +Input: + observed runtime corpus + +Process: + gap analysis + subset imbalance detection + experimental requirement derivation + +Output: + structured experiment plans and execution queues + with execution_mode=external_only +``` + +## Commands + +```bash +causetrace corpus analyze-gaps +causetrace corpus plan-experiments --target failure_enriched +causetrace corpus ingest-feedback +causetrace corpus update-gaps +causetrace corpus reprioritize-experiments +``` + +`analyze-gaps` reports subset coverage against CRDD targets. `plan-experiments` +builds a plan directory with: + +```text +gap_report.json +experiment_queue.json +experiment_plan.md +``` + +## Safety Boundary + +Every CERC queue must preserve these fields: + +```json +{ + "execution_mode": "external_only", + "must_not_execute": true, + "evidence_status": "planned_not_observed", + "observed_session_count": 0, + "phase4_grade_effect": "none" +} +``` + +The queue must not contain shell commands, agent commands, API calls, or runtime +execution payloads. + +## Principles + +### No Execution Authority + +CERC never runs, launches, controls, or simulates agent runtimes. It only emits +requirements for external execution. + +### No Evidence Inflation + +Planned sessions are not observed sessions. A plan can identify a sampling gap, +but it cannot increase denominators or support claims until traces are actually +captured by `causetrace`. + +### No Theory Promotion + +CERC plans do not change Phase 4 evidence grades. They may prepare future +sampling work that could later satisfy Phase 4-3 trigger conditions. + +## Current Targets + +Default v0.3 targets: + +| Subset | Target sessions | Purpose | +| --- | ---: | --- | +| `strict_research_grade` | 150 | Baseline comparable corpus | +| `balanced_cross_runtime` | 20 | Runtime-balanced comparison input | +| `failure_enriched` | 50 | Failure and near-failure boundary study | +| `intervention_lane` | 15 | Control-vs-intervention study input | + +These are planning targets, not evidence thresholds. + +## Relationship To CRDD + +CRDD compiles existing comparable subsets. CERC compiles missing experimental +work. CRDD says what can be compared now; CERC says what must be collected next. + +## Feedback Integration + +Feedback integration consumes external execution results after collection. It +does not execute runtimes, synthesize commands, or change Phase 4 evidence +grading. It only updates gap projections and future priorities. diff --git a/docs/research/dataset_design/crdd_v0.1.md b/docs/research/dataset_design/crdd_v0.1.md new file mode 100644 index 0000000..c6241d9 --- /dev/null +++ b/docs/research/dataset_design/crdd_v0.1.md @@ -0,0 +1,158 @@ +# Causal Runtime Dataset Design v0.1 + +CRDD defines how `causetrace` turns raw runtime traces into comparable research +datasets. It treats `causetrace` as a runtime causality dataset, governance +system, and research harness, not only as an ingestion tool. + +## Position + +The project has moved from instrumentation-dominant work to data-dominant +research work. + +```text +Layer 1: ingestion system hooks, parsers, CLI import paths +Layer 2: causal graph system DAGs, topology metrics, corpus tools +Layer 3: research system phases, governance, hypotheses, evidence grades +``` + +The next research bottleneck is not trace volume. It is comparability: +most raw sessions cannot yet support cross-session conclusions because their +runtime, task, outcome, lane, and provenance are incomplete or mixed. + +## Core Rule + +Do not treat the raw corpus as the research sample. + +Every claim must name the dataset layer it uses: + +| Layer | Meaning | Valid use | +| --- | --- | --- | +| `raw_corpus` | All locally stored sessions that parse as traces | Ingestion health, coverage, source inventory | +| `observational_corpus` | Sessions with enough structure for descriptive analysis | Morphology exploration, candidate discovery | +| `comparable_corpus` | Sessions with sufficient metadata and lane separation | Cross-session comparison | +| `experimental_subset` | Balanced, enriched, or controlled subset with explicit inclusion rules | Candidate validation and revalidation | + +Counts from lower layers must not be reused as denominators for higher-layer +claims. + +## Comparability Dimensions + +A session becomes comparable only when the relevant comparison dimensions are +declared or explicitly marked unknown. + +| Dimension | Why it matters | +| --- | --- | +| `runtime` | Allows runtime-family comparison and balance checks | +| `task_type` | Prevents task mix from masquerading as topology effects | +| `success` | Enables outcome correlation and failure contrast | +| `duration` | Supports cost and density interpretation | +| `task_source` | Separates real work, demo, proxy, and controlled sources | +| `intervention_lane` | Separates native control behavior from workflow interventions | +| `data_origin` | Separates native, controlled benchmark, and external trajectories | +| `human_intervention` | Makes control and recovery dynamics observable | + +Missing metadata is not automatically disqualifying for descriptive work, but it +does disqualify a session from comparisons that depend on that dimension. + +## Metadata Tiers + +Metadata must keep provenance. Do not collapse inferred or manually completed +fields into factual capture fields. + +| Tier | Meaning | Examples | Use in claims | +| --- | --- | --- | --- | +| `observed` | Captured directly from runtime, hook, parser, or sidecar at collection time | runtime, model, provider, timestamps | Strongest evidence | +| `derived` | Computed from trace structure or deterministic parser rules | topology class, event density, inferred duration | Usable with method disclosure | +| `experimental` | Added by reviewer or controlled protocol after the run | task_type label, success label, intervention tag | Usable only with provenance and reviewer/protocol disclosure | + +Research reports must disclose the tier for fields that determine inclusion, +matching, or outcome classification. + +## Required Subsets + +CRDD requires named manifests for any theory refresh or cross-lane comparison. + +| Subset | Purpose | Minimum inclusion rule | +| --- | --- | --- | +| `strict_research_grade` | Baseline runtime morphology | Runtime, task source, task type, success, and provenance are explicit or trusted | +| `balanced_cross_runtime` | Runtime comparison | Matched or capped per-runtime sample counts with task mix disclosed | +| `failure_enriched` | Boundary and recovery analysis | Failure and near-failure sessions intentionally oversampled and labeled | +| `intervention_lane` | Control vs intervention study | Native and intervention lanes separated before comparison | +| `controlled_prompt_morphology` | Prompt posture effect study | Variant tags and protocol provenance present | +| `safety_control` | Safety-boundary morphology | Safety-control signal definitions and annotation provenance present | + +Subset manifests should include: + +- corpus snapshot date +- source query or selection rule +- inclusion and exclusion criteria +- denominator before and after filtering +- runtime and task distribution +- metadata tier requirements +- known sampling bias +- intended claims and prohibited claims + +## Sampling Bias Register + +Every analysis that leaves descriptive counting must include a bias register. + +Current known risks: + +| Risk | Effect | +| --- | --- | +| Unlabeled majority | Raw session counts overstate research sample size | +| Failure scarcity | Failure morphology and recovery dynamics are underpowered | +| Intervention scarcity | Control-vs-intervention effects may be invisible or unstable | +| Runtime imbalance | Apparent morphology defaults may be runtime-specific | +| Success-label scarcity | Outcome correlation can be dominated by labeled subset bias | +| Duration absence | Event density and cost interpretations are incomplete | +| Post-hoc parsing | Causality may be parser-dependent for some runtimes | + +Bias registers do not block analysis. They prevent overclaiming. + +## Continuous Sampling Loop + +Phase 4-3 remains a formal evidence refresh gate, but CRDD adds a continuous +sampling loop upstream of it. + +```text +raw corpus + -> classify candidate comparable sessions + -> build subset manifests + -> inspect balance and bias + -> enrich failure/intervention/control samples + -> re-run candidate checks in sandbox reports + -> open formal Phase 4-3 only when evidence thresholds are satisfied +``` + +This loop does not upgrade evidence grades by itself. It prepares the comparable +subsets needed for a defensible grade change. + +## Claim Discipline + +Use these claim scopes: + +| Scope | Allowed wording | +| --- | --- | +| Raw corpus | "The stored trace corpus contains..." | +| Observational corpus | "Among sessions with observable structure..." | +| Comparable corpus | "Among comparable sessions with declared metadata..." | +| Experimental subset | "Within the named subset and protocol..." | + +Avoid statements that imply all stored sessions are equivalent research samples. + +## Immediate Work Program + +1. Create subset manifests for `strict_research_grade`, `balanced_cross_runtime`, + `failure_enriched`, and `intervention_lane`. +2. Reclassify "metadata gap" work as "comparability gap" work. +3. Prioritize failure and near-failure acquisition over broad raw ingestion. +4. Preserve metadata tier provenance when filling labels. +5. Use Phase 4-3 triggers as formal promotion gates, not as the only sampling + activity. + +## Boundary + +CRDD does not open Phase 5. It does not introduce prediction, anomaly detection, +leaderboards, or automated diagnosis. It only defines how runtime traces become +valid research datasets for morphology analysis. diff --git a/docs/research/dataset_design/feedback_v0.4.md b/docs/research/dataset_design/feedback_v0.4.md new file mode 100644 index 0000000..f718754 --- /dev/null +++ b/docs/research/dataset_design/feedback_v0.4.md @@ -0,0 +1,47 @@ +# CERC Feedback Integration v0.4 + +CERC feedback integration turns external execution results into updated gap +projections and experiment priorities. It is a read-only planning layer above +CRDD and below any future execution system. + +## Definition + +```text +Input: + external execution feedback + plus optional experiment plan references + +Process: + normalize observations + compare observed outcomes with current gap projections + reprioritize future experimental sampling + +Output: + feedback reports, gap updates, and reprioritized plans +``` + +## Commands + +```bash +causetrace corpus ingest-feedback +causetrace corpus update-gaps +causetrace corpus reprioritize-experiments +``` + +## Safety Boundary + +Feedback integration does not execute runtimes, synthesize commands, or +upgrade Phase 4 evidence. It only normalizes observed results and updates the +next sampling plan. + +## Constraints + +- external execution only +- no evidence inflation +- no Phase 4 grade promotion +- no runtime control authority + +## Relationship To CERC + +CERC plans missing work. Feedback integration updates those plans after external +execution has already happened. diff --git a/docs/research/dataset_design/subset_manifest_template.md b/docs/research/dataset_design/subset_manifest_template.md new file mode 100644 index 0000000..d68b615 --- /dev/null +++ b/docs/research/dataset_design/subset_manifest_template.md @@ -0,0 +1,115 @@ +# Subset Manifest Template + +Use this template for any comparable or experimental subset used in morphology +analysis. A subset manifest is required before a subset can support Phase 4 +candidate revalidation. + +## Identity + +- **subset_id**: +- **subset_type**: +- **created_at**: +- **corpus_snapshot_date**: +- **owner/reviewer**: +- **status**: draft | active | superseded | retired + +## Purpose + +- **research purpose**: +- **intended claims**: +- **prohibited claims**: +- **related theory candidates**: + +## Source Corpus + +- **data sessions before filtering**: +- **metadata sessions before filtering**: +- **source command/query**: +- **source files or manifests**: + +## Inclusion Criteria + +| Criterion | Required value | Metadata tier | +| --- | --- | --- | +| runtime | | observed / derived / experimental | +| task_type | | observed / derived / experimental | +| success | | observed / derived / experimental | +| task_source | | observed / derived / experimental | +| intervention_lane | | observed / derived / experimental | +| data_origin | | observed / derived / experimental | +| human_intervention | | observed / derived / experimental | + +Additional structural criteria: + +- + +## Exclusion Criteria + +- + +## Resulting Denominators + +| Count | Value | +| --- | --- | +| sessions included | | +| events included | | +| sessions excluded | | +| missing required metadata | | +| rejected for lane ambiguity | | +| rejected for parse/validation issues | | + +## Distribution + +Runtime distribution: + +| Runtime | Sessions | Events | +| --- | ---: | ---: | +| | | | + +Task distribution: + +| Task type | Sessions | Events | +| --- | ---: | ---: | +| | | | + +Lane distribution: + +| Lane | Sessions | Events | +| --- | ---: | ---: | +| | | | + +Outcome distribution: + +| Outcome | Sessions | +| --- | ---: | +| success | | +| failure | | +| near_failure | | +| unknown | | + +## Bias Register + +| Bias risk | Present? | Mitigation | +| --- | --- | --- | +| unlabeled majority | | | +| failure scarcity | | | +| intervention scarcity | | | +| runtime imbalance | | | +| task imbalance | | | +| success-label scarcity | | | +| duration absence | | | +| post-hoc parsing | | | + +## Validation Checks + +- [ ] duplicate event scan completed +- [ ] broken parent references reviewed +- [ ] runtime/task/lane denominators disclosed +- [ ] metadata tiers disclosed +- [ ] excluded sessions counted +- [ ] bias register completed +- [ ] claims scoped to this subset only + +## Notes + +- diff --git a/docs/research/phase4/README.md b/docs/research/phase4/README.md index 349889f..2ede7a0 100644 --- a/docs/research/phase4/README.md +++ b/docs/research/phase4/README.md @@ -13,6 +13,8 @@ Phase 4 consolidates evidence-graded theory candidates from Phase 3D (hypothesis Convert the strongest evidence-backed findings from Phase 3D and Phase 3E into graded theory candidates. Each candidate must carry an evidence grade, a corpus snapshot, a lane scope, a denominator, runtime/task caveats, and a falsification condition. +Observed runtime morphology is an external behavioral projection. Future agent systems may use latent, compressed, or machine-readable communication that improves performance while reducing human auditability. Prompt-level compressed text and structured `machine_state` remain observable; true hidden-state, embedding, or KV-cache communication would be a different channel class. This is currently a theory-boundary caveat and does not change existing evidence grades. + ## What Phase 4 Is - Evidence-graded theory drafting diff --git a/docs/research/phase4/phase4_3_trigger_conditions.md b/docs/research/phase4/phase4_3_trigger_conditions.md index ba488b5..3f78460 100644 --- a/docs/research/phase4/phase4_3_trigger_conditions.md +++ b/docs/research/phase4/phase4_3_trigger_conditions.md @@ -2,6 +2,12 @@ Phase 4-3 is a trigger-based evidence refresh pass for Runtime Morphology Theory Draft v0.1. It will not be opened by calendar schedule. It opens only when one or more trigger conditions are met by corpus growth. +CRDD adds a continuous sampling layer before formal Phase 4-3. Sampling may build +balanced, failure-enriched, or intervention-specific subset manifests at any +time. Those manifests do not upgrade evidence grades by themselves; they prepare +comparable corpora so that a later trigger-based refresh can use defensible +denominators. + ## Status - Phase 4-1 (evidence grading): complete @@ -110,6 +116,12 @@ causetrace corpus gate-status Compare against trigger thresholds. If any trigger fires, open Phase 4-3 for the affected candidate set only. Do not re-grade unaffected candidates. +Before opening a formal refresh, also check whether the affected sessions belong +to a named comparable subset under +[`Causal Runtime Dataset Design v0.1`](../dataset_design/crdd_v0.1.md). If no +comparable subset exists, create the subset manifest first and treat the run as a +sampling pass, not an evidence-grade update. + ## Partial Phase 4-3 Phase 4-3 does not need to be a complete re-grade of all 12 candidates. A single trigger opens re-evaluation for its affected candidates only. Unaffected candidates retain their current grades. diff --git a/docs/research/phase4/runtime_morphology_theory_draft_v0.1.md b/docs/research/phase4/runtime_morphology_theory_draft_v0.1.md index cf3c7f0..dc9c015 100644 --- a/docs/research/phase4/runtime_morphology_theory_draft_v0.1.md +++ b/docs/research/phase4/runtime_morphology_theory_draft_v0.1.md @@ -159,6 +159,8 @@ These claims cannot be evaluated against the current corpus. They are deferred, ## 5. Theory Boundaries +Visible trace length and natural-language rationale density should not be treated as complete proxies for reasoning depth. Future agents may shift reasoning or coordination into compressed or opaque communication channels. Prompt-level compression and structured `machine_state` remain observable text artifacts, while true hidden-state, embedding, or KV-cache transfer would require separate instrumentation. `causetrace` therefore interprets topology as observable runtime morphology, not full internal cognition. + This draft explicitly does NOT support: | Exclusion | Rationale | diff --git a/docs/research/roadmap/agentic_auditability_roadmap_v0.2.5.md b/docs/research/roadmap/agentic_auditability_roadmap_v0.2.5.md new file mode 100644 index 0000000..a3b4f81 --- /dev/null +++ b/docs/research/roadmap/agentic_auditability_roadmap_v0.2.5.md @@ -0,0 +1,324 @@ +# Agentic Auditability Roadmap v0.2.5 + +**Status**: Long-term roadmap correction. Not an active implementation plan. + +`causetrace` currently studies observable runtime morphology: the external structure of agent execution as represented by events, tool calls, messages, file changes, interventions, and topology transitions. The long-term research direction should now be framed as: + +```text +Runtime Morphology +-> Auditability-aware Runtime Morphology +-> Data-grounded Method Discovery +-> Agentic Audit Infrastructure +``` + +This means future work may need to study not only what runtime structures are visible, but also whether those structures are human-auditable, machine-audited, partially observable, or opaque. This roadmap clarifies a future boundary only. It does not unfreeze Phase 4, open Phase 4-3, open Phase 5, change schema, or add audit automation. + +## Strategic Position + +`causetrace` should not position itself as a system for making agents more capable, faster, or more autonomous. Its long-term position is runtime auditability research infrastructure for high-performance AI agents. + +Working positioning statement: + +```text +causetrace studies the observable runtime and auditability morphology of high-performance AI agents. +``` + +The strategic premise is that mainstream agent development will often optimize first for capability, cost, latency, success rate, longer task chains, tool autonomy, and reduced human intervention. Auditability, explainability, and responsibility-chain reconstruction are likely to appear as downstream constraints rather than the primary optimization target. + +For `causetrace`, the core question is therefore not whether an agent is faster or stronger. The core question is whether stronger, faster, more autonomous agents still leave enough observable evidence for humans and audit systems to reconstruct what happened, why it happened, what channels were covered, and where responsibility or uncertainty remains. + +## Methodology + +The future auditability branch should use data-grounded runtime method discovery: + +```text +Capture -> Curate -> Compare -> Hypothesize -> Validate -> Revise +``` + +The rule is: + +```text +Do not assume a method improves performance, safety, or auditability. +Record it. +Compare it. +Then decide. +``` + +Each method claim should separately report performance, morphology, and auditability effects. The `machine_state + human_audit_summary` pilot illustrates why this matters: it improved audit structure on a small read-only task but increased runtime overhead. That result is useful precisely because it corrects the method boundary rather than confirming the original efficiency expectation. + +See [Data-Grounded Runtime Method Discovery](data_grounded_method_discovery_v0.2.5.md). + +## Conceptual Layers + +Auditability work should distinguish three different concepts that are easy to collapse. + +Layer A: Prompt-level compressed text + +Prompting an agent to "think in machine language," use symbols, or emit compact codes still operates at the visible text-token layer. It may reduce verbosity, but it does not give the model a true hidden-state, embedding, or KV-cache communication channel. For `causetrace`, this remains observable text morphology. + +Layer B: Structured machine-state reasoning + +Agents can maintain compact, structured state objects for task tracking, then expand only the audit-relevant parts into natural language. This is not latent reasoning, but it is immediately testable, auditable, and compatible with existing agent interfaces. + +Example: + +```yaml +msr_v1: + goal: improve_metadata_capture + scope: docs_only + constraints: + - no_schema_change + - phase4_frozen + - no_unknown_inference + evidence: + - 879_unknown_data_origin + - trigger8_10_9_percent + decision: future_capture_quality_only + risk: + - heuristic_pollution + next_action: update_runbook + stop_condition: git_diff_check_pass +``` + +Layer C: True latent or vector communication + +True machine-language or latent communication requires model or system support, such as direct hidden-state transfer, embedding exchange, shared KV-cache, a latent reasoning head, or multi-agent collaboration in latent space. This is not available through ordinary prompting alone and is outside the current `causetrace` mainline. + +## Why This Matters + +Future agent systems may use compressed, latent, high-density, non-human-readable, or agent-to-agent communication channels. These channels may improve performance, latency, coordination, and token efficiency. + +The trade-off is auditability. Natural-language traces may become incomplete external projections of the task process, while important coordination or reasoning moves occur in compact machine-readable channels. Human reviewers cannot manually inspect massive runtime traces at scale, so future AI audit will likely become highly automated. + +That creates a second-order audit problem: if audit agents review other agents, the audit agents themselves must be traceable. Their reasoning, evidence selection, coverage limits, and report generation need audit trails rather than becoming a new opaque authority layer. + +## Performance-Auditability Trade-Off + +The long-term research axis is: + +```text +Performance up vs. Auditability down +``` + +This is a hypothesis and research frame, not a validated law. It should guide future branch studies without changing current evidence grades. + +Possible performance optimizations and auditability costs: + +| Performance direction | Possible auditability cost | +| --- | --- | +| less visible reasoning | fewer decision justifications | +| shorter visible traces | missing intermediate states | +| latent or machine-readable communication | human-unreadable collaboration | +| automatic tool calls | larger external action surface | +| multi-agent collaboration | longer and less local responsibility chains | +| automatic repair or rollback | masked failure and recovery process | +| audit-agent summarization | second-order opacity | + +The resulting research question is: + +```text +When agent systems optimize for speed, cost, autonomy, or success rate, what evidence is lost, compressed, hidden, or shifted into channels that current trace capture does not cover? +``` + +The machine-state pilot reinforces this boundary in the opposite direction: auditability instrumentation is not free. It may improve reviewability while increasing runtime overhead. + +## What Causetrace Should Study In The Future + +Q1. Observable runtime projection: +Does the visible trace explain the task outcome, or are there unexplained topology transitions? + +Q2. Performance-auditability trade-off: +Do higher performance, lower event counts, or shorter visible traces correlate with reduced human-auditable evidence? + +Q3. Communication channel visibility: +Which communication channels are human-readable, machine-readable, partially observable, or opaque? + +Q4. Safety-control coverage: +Do safety gates, `need_review`, hard-stop, fallback, and `human_intervention` signals cover the actual communication and execution channels? + +Q5. Agentic audit traceability: +If an audit agent reviews another agent, is the audit agent's own reasoning, evidence selection, and report generation traceable? + +Q6. Audit evidence sufficiency: +Does the audit report include evidence references, coverage limits, uncertainty, and human-review entry points? + +## Structured Machine-State Reasoning + +A practical near-term experiment is structured machine-state reasoning: agents exchange a compact `machine_state` object while also emitting a concise `human_audit_summary`. + +The intended dual-channel shape is: + +```text +machine_state channel: compact state for agent continuity +audit_summary channel: natural-language summary for human review +``` + +The `machine_state` channel should track fields such as: + +- `goal` +- `scope` +- `constraints` +- `evidence` +- `decision` +- `risks` +- `next_action` +- `stop_condition` + +Every material `machine_state` update should be paired with a human-auditable summary explaining: + +- what decision was made +- what evidence supports it +- what remains uncertain +- what requires human review + +This preserves a useful compromise: higher-density agent state without turning the workflow into unreadable or unaudited communication. + +This roadmap explicitly rejects unreadable pseudo-machine language, random encodings, Base64-style obfuscation, private agent shorthand that bypasses review, or hidden reasoning used to avoid human oversight. + +## Future Concepts, Not Current Schema + +The following are future concepts only: + +- `auditability_morphology` +- `performance_auditability_tradeoff` +- `visible_runtime_projection` +- `opaque_communication_boundary` +- `audit_agent_trace` +- `audit_evidence_graph` +- `channel_visibility` +- `auditability_score` +- `decision_trace_coverage` +- `safety_gate_coverage` +- `human_review_queue` +- `audit_report_provenance` +- `structured_machine_state_reasoning` +- `human_audit_summary` + +These are not current schema fields and must not be added until supported by corpus evidence and a separate design review. + +## Audit Automation Hierarchy + +The expected future structure has four layers. + +Layer 1: Runtime capture + +- events +- tool calls +- messages +- file changes +- memory access +- permission use +- A2A communication where available + +Layer 2: Machine audit + +- automated screening +- policy checks +- causal graph analysis +- anomaly candidates +- safety-control coverage analysis +- audit agent summaries + +Layer 3: Evidence package + +- evidence references +- coverage report +- uncertainty report +- risk triage +- unresolved gaps +- human review queue + +Layer 4: Human oversight + +- high-risk sample review +- policy judgment +- exception approval +- liability and governance decisions + +The human role shifts from reading every trace to auditing the audit system, reviewing high-risk cases, and approving risk boundaries. + +## Non-Goals + +`causetrace` must not become: + +- a hidden-language decoder +- a latent-state interpreter +- a jailbreak or covert-channel research tool +- a universal safety classifier +- an automatic diagnosis engine +- a prediction system +- a generic observability SaaS +- an unverified audit-agent platform + +## Mainline Impact + +- Phase 4 remains frozen. +- Phase 4-3 remains trigger-gated. +- Phase 4-3 remains at 0/8 triggers met. +- New literature alone does not trigger Phase 4-3. +- Phase 5 remains closed. +- This roadmap only clarifies future direction. +- Any future move toward auditability morphology requires explicit corpus triggers or a separate branch. + +## Branch Proposal + +Branch name: + +```text +agentic_auditability_branch +``` + +Purpose: + +Study auditability morphology and agentic audit traceability without changing the `causetrace` mainline. + +Allowed scope: + +- literature notes +- external examples +- small controlled pilots +- auditability metrics prototypes +- audit report provenance experiments +- structured machine-state reasoning pilots with human audit summaries + +Disallowed scope: + +- mainline schema changes +- Phase 5 opening +- automatic safety judgment +- automatic unknown-session classification +- hidden language decoding +- latent hidden-state or KV-cache integration in the mainline +- unreadable pseudo-machine-language protocols + +### Possible Sub-Branch: machine_state_reasoning_branch + +Purpose: + +Study whether structured `machine_state` reduces token use, event count, retry density, or handoff cost while preserving or improving auditability through paired `human_audit_summary` records. + +Comparison groups: + +| Group | Description | +| --- | --- | +| A | ordinary natural-language prompt | +| B | structured `machine_state` | +| C | structured `machine_state` plus `human_audit_summary` | +| D | minimal prompt | + +Candidate metrics: + +- `event_count` +- `tool_call_count` +- `retry_density` +- `branch_collapse` +- `success` +- `human_review_time` +- `audit_summary_quality` +- `decision_trace_coverage` + +This sub-branch must remain experimental. It must not add schema fields, classify unknown sessions, open Phase 5, or infer that compact traces are deeper, safer, or more capable without corpus evidence. + +## Current Boundary + +This document is a roadmap correction from runtime-only morphology toward auditability-aware runtime research. It does not classify unknown sessions, promote any theory candidate, add topology classes, add schema fields, or implement audit agents. diff --git a/docs/research/roadmap/data_grounded_method_discovery_v0.2.5.md b/docs/research/roadmap/data_grounded_method_discovery_v0.2.5.md new file mode 100644 index 0000000..1f2d300 --- /dev/null +++ b/docs/research/roadmap/data_grounded_method_discovery_v0.2.5.md @@ -0,0 +1,157 @@ +# Data-Grounded Runtime Method Discovery v0.2.5 + +**Status**: Long-term research method. Not a new phase and not an implementation plan. + +`causetrace` should discover methods from recorded runtime behavior rather than starting with a fixed theory and forcing traces to confirm it. + +Core loop: + +```text +Capture -> Curate -> Compare -> Hypothesize -> Validate -> Revise +``` + +Expanded path: + +```text +real process records +-> structured corpus +-> reproducible analysis +-> observed patterns +-> falsifiable hypotheses +-> additional trace validation +-> revised method +``` + +This is the working rule: + +```text +Do not assume a method improves performance, safety, or auditability. +Record it. +Compare it. +Then decide. +``` + +## Principle + +All future method claims must be grounded in recorded runtime traces. A method is not considered effective unless its impact on performance, morphology, and auditability is separately recorded. + +Final task success is not enough. A method may improve auditability while increasing overhead, reduce tool calls while weakening evidence, or improve performance while reducing human reviewability. + +## Why This Matters + +Agent systems change quickly. Preset theories about prompting, workflow structure, machine-state reasoning, audit summaries, or agent handoff can become stale or misleading. + +The `machine_state + human_audit_summary` pilot is the current example. The initial expectation that structured state might improve efficiency was not supported on a small read-only task. The recorded runtime instead showed higher event count, more tool calls, longer elapsed span, and more output/reasoning tokens, while the visible benefit was clearer audit structure. + +That negative performance result improved the method: + +```text +machine_state is not a general performance optimization by default. +It is better treated as auditability instrumentation until larger traces show otherwise. +``` + +## Minimum Experiment Record + +Future prompt, workflow, machine-state, handoff, or auditability experiments should record at least: + +```yaml +experiment: + id: + date: + task: + project: + runtime: + model: + agent: + branch_or_lane: + +groups: + A: + prompt_type: + session_id: + B: + prompt_type: + session_id: + +metrics: + performance: + elapsed_time: + output_tokens: + reasoning_tokens: + tool_calls: + success: + morphology: + event_count: + topology: + max_depth: + retry_density: + branch_collapse: + auditability: + audit_summary_present: + evidence_references: + decision_trace_coverage: + human_review_needed: + unexplained_transitions: + +conclusion: + supported: + not_supported: + limitation: + next_test: +``` + +## Separate Result Axes + +Every method comparison should separate three result axes. + +Performance: + +- elapsed time +- output tokens +- reasoning tokens +- tool calls +- retry count or retry density +- success or failure + +Morphology: + +- event count +- topology label +- max depth +- branch or collapse behavior +- fan-in or multi-root behavior +- critical path length + +Auditability: + +- audit summary presence +- evidence references +- decision trace coverage +- human review entry points +- unresolved uncertainty +- unexplained topology transitions + +These axes must not be collapsed into a single good or bad outcome. + +## Method Claim Rules + +- Do not infer method effectiveness from final success alone. +- Do not promote a method because it is fashionable, simpler to explain, or supported by external literature only. +- Do not treat a negative result as failure if it clarifies method boundaries. +- Do not merge intervention-lane evidence into native baseline conclusions without lane disclosure. +- Do not classify unknown sessions to improve apparent coverage. +- Do not change schema or topology taxonomy based on a pilot. +- Do not open Phase 4-3 or Phase 5 based on method pilots alone. + +## Relationship To Roadmap + +The long-term route becomes: + +```text +Runtime Morphology +-> Auditability-aware Runtime Morphology +-> Data-grounded Method Discovery +-> Agentic Audit Infrastructure +``` + +`causetrace` should not directly prescribe which agent method to use. It should record real processes, compare methods across performance, morphology, and auditability, and let repeated runtime evidence shape later hypotheses and theory. diff --git a/pyproject.toml b/pyproject.toml index 08a7741..b62a09c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "causetrace" -version = "0.2.5" +version = "0.3.0" description = "AI coding agent observability and causal tracing for Claude Code, Codex CLI, and OpenCode" readme = "README.md" license = {text = "MIT"} diff --git a/tests/test_invariants.py b/tests/test_invariants.py index 6425d4b..9092503 100644 --- a/tests/test_invariants.py +++ b/tests/test_invariants.py @@ -184,6 +184,138 @@ def test_json_store_append_only_preserves_order(): assert names == ["Read", "Write", "Bash"], f"order broken: {names}" +def test_json_store_append_missing_skips_reparsed_events(): + with tempfile.TemporaryDirectory() as tmp: + store = JSONStore(store_dir=tmp) + session_id = "reparsed" + original = [ + ToolEvent( + "Read", + {"file_path": "a.py"}, + event_id="old_read", + timestamp="2026-01-01T00:00:00", + agent="codex", + ), + ToolEvent( + "Bash", + {"command": "pytest"}, + event_id="old_bash", + parent_event_id="old_read", + timestamp="2026-01-01T00:00:01", + agent="codex", + ), + ] + for event in original: + store.append(session_id, event) + + reparsed = [ + ToolEvent( + "Read", + {"file_path": "a.py"}, + event_id="new_read", + timestamp="2026-01-01T00:00:00", + agent="codex", + ), + ToolEvent( + "Bash", + {"command": "pytest"}, + event_id="new_bash", + parent_event_id="new_read", + timestamp="2026-01-01T00:00:01", + agent="codex", + ), + ] + + summary = store.append_missing(session_id, reparsed) + loaded = store.load(session_id) + + assert summary["added"] == 0 + assert summary["skipped"] == 2 + assert [event.event_id for event in loaded] == ["old_read", "old_bash"] + + +def test_json_store_append_missing_remaps_new_parent_refs(): + with tempfile.TemporaryDirectory() as tmp: + store = JSONStore(store_dir=tmp) + session_id = "tail" + store.append( + session_id, + ToolEvent( + "Read", + {"file_path": "a.py"}, + event_id="old_read", + timestamp="2026-01-01T00:00:00", + agent="codex", + ), + ) + store.append( + session_id, + ToolEvent( + "Bash", + {"command": "pytest"}, + event_id="old_bash", + parent_event_id="old_read", + timestamp="2026-01-01T00:00:01", + agent="codex", + ), + ) + + reparsed_with_tail = [ + ToolEvent( + "Read", + {"file_path": "a.py"}, + event_id="new_read", + timestamp="2026-01-01T00:00:00", + agent="codex", + ), + ToolEvent( + "Bash", + {"command": "pytest"}, + event_id="new_bash", + parent_event_id="new_read", + timestamp="2026-01-01T00:00:01", + agent="codex", + ), + ToolEvent( + "Response", + {"text": "done"}, + event_id="new_response", + parent_event_id="new_bash", + timestamp="2026-01-01T00:00:02", + event_type="reasoning", + agent="codex", + ), + ] + + summary = store.append_missing(session_id, reparsed_with_tail) + loaded = store.load(session_id) + + assert summary["added"] == 1 + assert summary["skipped"] == 2 + assert loaded[-1].event_id == "new_response" + assert loaded[-1].parent_event_id == "old_bash" + + +def test_json_store_append_missing_dry_run_does_not_write(): + with tempfile.TemporaryDirectory() as tmp: + store = JSONStore(store_dir=tmp) + incoming = [ + ToolEvent( + "Read", + {"file_path": "a.py"}, + event_id="read", + timestamp="2026-01-01T00:00:00", + agent="codex", + ) + ] + + summary = store.append_missing("dry", incoming, dry_run=True) + + assert summary["added"] == 1 + assert summary["dry_run"] is True + assert store.load("dry") == [] + + def test_json_store_multiple_sessions(): with tempfile.TemporaryDirectory() as tmp: store = JSONStore(store_dir=tmp) diff --git a/tests/test_metadata_corpus_report.py b/tests/test_metadata_corpus_report.py index ceeb3a3..6c99065 100644 --- a/tests/test_metadata_corpus_report.py +++ b/tests/test_metadata_corpus_report.py @@ -52,6 +52,30 @@ def test_metadata_merges_legacy_annotation(monkeypatch, tmp_path): assert load_metadata("s1").model == "gpt-5" +def test_metadata_accepts_behavior_distribution_fields(monkeypatch, tmp_path): + import causetrace.metadata as metadata + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + + empty = load_metadata("empty").to_dict() + assert empty == {} + + updated = merge_metadata("s1", { + "behavior_distribution_tag": "prompt_ablation_v1", + "bde_generated": "true", + "experiment_id": "exp_001", + "control_group_id": "ctrl_a", + }) + + assert updated.behavior_distribution_tag == "prompt_ablation_v1" + assert updated.bde_generated is True + assert updated.experiment_id == "exp_001" + assert updated.control_group_id == "ctrl_a" + + saved = load_metadata("s1") + assert saved.bde_generated is True + + def test_corpus_snapshot_and_export(monkeypatch, tmp_path): import causetrace.metadata as metadata @@ -710,6 +734,216 @@ def test_corpus_verify_cli(monkeypatch, tmp_path): assert "OK: True" in result.stdout +def test_crdd_compile_subsets_read_only_and_cli(monkeypatch, tmp_path): + import causetrace.metadata as metadata + from causetrace.crdd import compile_subsets + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + + store = JSONStore(store_dir=str(tmp_path / "data")) + _write_session(store, "native-success") + _write_session(store, "native-failure") + _write_session(store, "intervention") + _write_session(store, "codex-success") + + merge_metadata("native-success", { + "data_origin": "native", + "runtime": "claude", + "task_type": "feature_add", + "task_source": "real_work", + "success": True, + }) + merge_metadata("native-failure", { + "data_origin": "native", + "runtime": "claude", + "task_type": "bug_fix", + "task_source": "real_work", + "success": False, + }) + merge_metadata("intervention", { + "data_origin": "native", + "runtime": "claude", + "task_type": "exploration", + "task_source": "superpowers_workflow_intervention", + "intervention_lane": "superpowers_workflow_intervention", + "success": True, + "human_intervention": True, + }) + merge_metadata("codex-success", { + "data_origin": "native", + "runtime": "codex", + "task_type": "feature_add", + "task_source": "real_work", + "success": True, + }) + + before = sorted((tmp_path / "data").glob("*.jsonl")) + result = compile_subsets( + store, + subset_ids=["strict_research_grade", "failure_enriched", "intervention_lane", "balanced_cross_runtime"], + output_dir=tmp_path / "subsets", + name="daily", + ) + after = sorted((tmp_path / "data").glob("*.jsonl")) + + assert before == after + assert result["source_session_count"] == 4 + assert (tmp_path / "subsets" / "daily" / "index.json").exists() + + manifests = {manifest["subset_id"]: manifest for manifest in result["manifests"]} + assert manifests["strict_research_grade"]["selected_count"] == 4 + assert manifests["failure_enriched"]["session_ids"] == ["intervention", "native-failure"] + assert manifests["intervention_lane"]["session_ids"] == ["intervention"] + assert manifests["balanced_cross_runtime"]["selected_count"] == 2 + assert 0 <= manifests["strict_research_grade"]["comparability"]["score"] <= 1 + assert manifests["failure_enriched"]["bias_register"]["failure_scarcity"]["present"] is True + + cli_result = subprocess.run( + [ + sys.executable, + "-m", + "causetrace", + "corpus", + "compile-subsets", + "--subset", + "strict_research_grade", + "--dry-run", + ], + capture_output=True, + text=True, + env={**os.environ, "HOME": str(tmp_path)}, + ) + + assert cli_result.returncode == 0 + assert "Dry-run compiled CRDD subsets" in cli_result.stdout + assert "strict_research_grade" in cli_result.stdout + + +def test_cerc_gap_analysis_and_experiment_plan_are_external_only(monkeypatch, tmp_path): + import causetrace.metadata as metadata + from causetrace.crdd import analyze_gaps, plan_experiments + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + + store = JSONStore(store_dir=str(tmp_path / "data")) + _write_session(store, "failure") + _write_session(store, "success") + merge_metadata("failure", { + "runtime": "codex", + "task_type": "bug_fix", + "task_source": "real_work", + "success": False, + }) + merge_metadata("success", { + "runtime": "claude", + "task_type": "feature_add", + "task_source": "real_work", + "success": True, + }) + + gap_report = analyze_gaps(store, subset_ids=["failure_enriched"]) + assert gap_report["subset_gaps"][0]["subset_id"] == "failure_enriched" + assert gap_report["subset_gaps"][0]["current_sessions"] == 1 + assert gap_report["subset_gaps"][0]["missing_sessions"] == 49 + + before = sorted((tmp_path / "data").glob("*.jsonl")) + result = plan_experiments( + store, + target_subset="failure_enriched", + output_dir=tmp_path / "plans", + name="exp_failure_test", + ) + after = sorted((tmp_path / "data").glob("*.jsonl")) + queue = result["plan"]["experiment_queue"] + + assert before == after + assert result["written"] is True + assert (tmp_path / "plans" / "exp_failure_test" / "gap_report.json").exists() + assert (tmp_path / "plans" / "exp_failure_test" / "experiment_queue.json").exists() + assert (tmp_path / "plans" / "exp_failure_test" / "experiment_plan.md").exists() + assert queue["execution_mode"] == "external_only" + assert queue["must_not_execute"] is True + assert queue["evidence_status"] == "planned_not_observed" + assert queue["observed_session_count"] == 0 + assert queue["phase4_grade_effect"] == "none" + assert queue["validation"]["ok"] is True + assert all(scenario["descriptor_only"] is True for scenario in queue["bde_scenarios"]) + assert "command" not in json.dumps(queue) + + +def test_cerc_cli_analyze_and_plan_dry_run(tmp_path): + home = tmp_path / "home" + store = JSONStore(store_dir=str(home / ".causetrace" / "data")) + _write_session(store, "failure") + + env = {**os.environ, "HOME": str(home)} + metadata_dir = home / ".causetrace" / "metadata" + metadata_dir.mkdir(parents=True) + (metadata_dir / "failure.json").write_text(json.dumps({ + "session_id": "failure", + "runtime": "codex", + "task_type": "bug_fix", + "task_source": "real_work", + "success": False, + })) + + gaps = subprocess.run( + [sys.executable, "-m", "causetrace", "corpus", "analyze-gaps", "--subset", "failure_enriched"], + capture_output=True, + text=True, + env=env, + ) + assert gaps.returncode == 0 + assert "CERC gap report" in gaps.stdout + assert "failure_enriched" in gaps.stdout + + plan = subprocess.run( + [ + sys.executable, + "-m", + "causetrace", + "corpus", + "plan-experiments", + "--target", + "failure_enriched", + "--dry-run", + "--name", + "exp_cli_dry", + "--output-dir", + str(tmp_path / "cli-plans"), + ], + capture_output=True, + text=True, + env=env, + ) + assert plan.returncode == 0 + assert "Dry-run planned CERC experiment" in plan.stdout + assert "Execution mode: external_only" in plan.stdout + assert "Must not execute: True" in plan.stdout + assert not (tmp_path / "cli-plans" / "exp_cli_dry").exists() + + +def test_bde_interfaces_are_metadata_only(): + from causetrace.bde import BehaviorScenario, FailureInjection, MultiAgentSimulation, PromptVariant, TaskDistribution + + scenario = BehaviorScenario( + scenario_id="scn_1", + task="fix failing test", + behavior_distribution_tag="ablation", + prompt_variants=(PromptVariant("A", "minimal"), PromptVariant("B", "structured")), + experiment_id="exp_1", + control_group_id="control", + ) + distribution = TaskDistribution("dist_1", ("bug_fix", "feature_add"), {"bug_fix": 0.5}) + failure = FailureInjection("fail_1", "ambiguous_requirements", target_task_type="bug_fix") + simulation = MultiAgentSimulation("sim_1", ("planner", "executor")) + + assert scenario.prompt_variants[0].variant_id == "A" + assert distribution.task_types == ("bug_fix", "feature_add") + assert failure.failure_mode == "ambiguous_requirements" + assert simulation.agent_roles == ("planner", "executor") + + # ── classify-unlabeled tests ────────────────────────────────────────── @@ -855,3 +1089,102 @@ def test_classify_unlabeled_high_confidence_not_applied_if_tags(monkeypatch, tmp meta = json.loads((mdir / f"{sid}.json").read_text()) assert "intervention_lane" not in meta + + +def test_cerc_feedback_ingest_update_and_reprioritize(monkeypatch, tmp_path): + import causetrace.metadata as metadata + from causetrace.crdd import ingest_feedback, plan_experiments, reprioritize_experiments, update_gaps + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + store = JSONStore(store_dir=str(tmp_path / "data")) + _write_session(store, "s1") + _write_session(store, "s2") + merge_metadata("s1", {"runtime": "codex", "task_type": "bug_fix", "task_source": "real_work", "success": False}) + merge_metadata("s2", {"runtime": "claude", "task_type": "review", "task_source": "real_work", "success": True}) + + plan_result = plan_experiments( + store, + target_subset="failure_enriched", + required_sessions=7, + name="exp_feedback", + output_dir=tmp_path / "plans", + ) + plan_dir = Path(plan_result["output_dir"]) + payload_path = tmp_path / "feedback.json" + payload_path.write_text(json.dumps({ + "experiment_id": "exp_feedback", + "target_subset": "failure_enriched", + "observed_sessions": [ + "s1", + "s2", + {"label": "missing-session", "runtime": "opencode"}, + ], + })) + + report = ingest_feedback(store, input_path=payload_path, plan_dir=plan_dir, output_dir=tmp_path / "feedback") + assert report["constraints"]["external_only"] is True + assert report["observed_count"] == 3 + assert report["resolved_count"] == 2 + assert report["unresolved_session_ids"] == ["missing-session"] + assert report["plan_queue"]["experiment_id"] == "exp_feedback" + assert report["gap_projection"]["target_sessions"] == 7 + assert report["gap_projection"]["remaining_sessions"] == 4 + assert (Path(report["output_dir"]) / "feedback_report.json").exists() + + gap_update = update_gaps(store, feedback_report=report, output_dir=tmp_path / "feedback") + assert gap_update["status"] in {"met", "under_target"} + assert gap_update["priority_hint"] in {"reprioritize", "hold"} + assert (Path(gap_update["output_dir"]) / "gap_update.json").exists() + + prioritized = reprioritize_experiments(store, feedback_report=report, output_dir=tmp_path / "feedback") + assert prioritized["constraints"]["no_execution"] is True + assert prioritized["constraints"]["no_evidence_inflation"] is True + assert prioritized["priorities"] + assert (Path(prioritized["output_dir"]) / "reprioritized_plan.json").exists() + + +def test_cerc_feedback_cli_commands(monkeypatch, tmp_path): + import causetrace.metadata as metadata + from causetrace.crdd import plan_experiments + + monkeypatch.setattr(metadata, "METADATA_DIR", str(tmp_path / "metadata")) + store = JSONStore(store_dir=str(tmp_path / "data")) + _write_session(store, "s1") + merge_metadata("s1", {"runtime": "codex", "task_type": "bug_fix", "task_source": "real_work", "success": False}) + + plan_result = plan_experiments(store, target_subset="failure_enriched", name="exp_cli", output_dir=tmp_path / "plans") + plan_dir = Path(plan_result["output_dir"]) + payload_path = tmp_path / "feedback.json" + payload_path.write_text(json.dumps({ + "experiment_id": "exp_cli", + "target_subset": "failure_enriched", + "observed_sessions": ["s1"], + })) + + ingest_cmd = subprocess.run( + [sys.executable, "-m", "causetrace", "corpus", "ingest-feedback", str(payload_path), "--plan-dir", str(plan_dir), "--output-dir", str(tmp_path / "feedback")], + capture_output=True, + text=True, + env={**os.environ, "HOME": str(tmp_path)}, + ) + assert ingest_cmd.returncode == 0 + assert "External only: True" in ingest_cmd.stdout + + report_path = Path(tmp_path / "feedback" / "exp_cli" / "feedback_report.json") + update_cmd = subprocess.run( + [sys.executable, "-m", "causetrace", "corpus", "update-gaps", str(report_path), "--output-dir", str(tmp_path / "feedback")], + capture_output=True, + text=True, + env={**os.environ, "HOME": str(tmp_path)}, + ) + assert update_cmd.returncode == 0 + assert "Priority hint:" in update_cmd.stdout + + reprioritize_cmd = subprocess.run( + [sys.executable, "-m", "causetrace", "corpus", "reprioritize-experiments", str(report_path), "--output-dir", str(tmp_path / "feedback")], + capture_output=True, + text=True, + env={**os.environ, "HOME": str(tmp_path)}, + ) + assert reprioritize_cmd.returncode == 0 + assert "Top priority:" in reprioritize_cmd.stdout diff --git a/tests/test_opencode_enrich.py b/tests/test_opencode_enrich.py index 3484946..7159201 100644 --- a/tests/test_opencode_enrich.py +++ b/tests/test_opencode_enrich.py @@ -207,6 +207,45 @@ def test_parse_session_only_tools(): mod.DB_PATH = original +def test_parse_session_uses_db_time_as_stable_fallback(): + tmpdir = Path(tempfile.mkdtemp()) + db_path = tmpdir / "opencode.db" + + conn = sqlite3.connect(str(db_path)) + conn.execute("CREATE TABLE session (id TEXT PRIMARY KEY, slug TEXT, title TEXT, project_id TEXT, " + "time_created INTEGER, time_updated INTEGER)") + conn.execute("CREATE TABLE message (id TEXT PRIMARY KEY, session_id TEXT NOT NULL, " + "time_created INTEGER NOT NULL, time_updated INTEGER NOT NULL, data TEXT NOT NULL)") + conn.execute("CREATE TABLE part (id TEXT PRIMARY KEY, message_id TEXT NOT NULL, session_id TEXT NOT NULL, " + "time_created INTEGER NOT NULL, time_updated INTEGER NOT NULL, data TEXT NOT NULL)") + + sid = "stable_time" + conn.execute("INSERT INTO session VALUES (?, ?, ?, ?, ?, ?)", + (sid, "stable", "Stable Time", "p1", 1000, 2000)) + conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", + ("m0", sid, 1000, 1000, json.dumps({"role": "user"}))) + conn.execute("INSERT INTO message VALUES (?, ?, ?, ?, ?)", + ("m1", sid, 1778724854161, 1778724854161, + json.dumps({"role": "assistant", "parentID": "m0"}))) + conn.execute("INSERT INTO part VALUES (?, ?, ?, ?, ?, ?)", + ("p1", "m1", sid, 1778724854161, 1778724854161, + json.dumps({"type": "tool", "tool": "bash", "state": {"input": {"command": "ls"}}}))) + conn.commit() + conn.close() + + import causetrace.hooks.opencode_parser as mod + original = mod.DB_PATH + mod.DB_PATH = db_path + try: + first = parse_session(sid) + second = parse_session(sid) + assert len(first) == 1 + assert first[0].timestamp.startswith("2026-05-14") + assert first[0].timestamp == second[0].timestamp + finally: + mod.DB_PATH = original + + def test_parse_session_not_found(): """Non-existent session returns empty list.""" tmpdir = Path(tempfile.mkdtemp())