diff --git a/sia/context_manager.py b/sia/context_manager.py index 2c72ff8..f8b0fad 100644 --- a/sia/context_manager.py +++ b/sia/context_manager.py @@ -220,6 +220,7 @@ def add_generation(self, gen_num: int, gen_data: dict[str, Any]): - agent_path: str, path to target_agent.py - gen_dir: str, path to generation directory - improvement_path: Optional[str], path to improvement.md + - transfer_evidence_path: Optional[str], path to transfer_evidence.json - execution_type: str, 'Single' or 'Multi-trajectory' """ # Extract agent stats @@ -237,16 +238,30 @@ def add_generation(self, gen_num: int, gen_data: dict[str, Any]): # Extract metrics metrics = self._extract_metrics(gen_data["gen_dir"]) - # Extract insights from improvement.md (if exists) + # Prefer transfer_evidence.json when available, fallback to improvement.md + transfer_evidence = self._load_transfer_evidence(gen_data.get("transfer_evidence_path")) insights = [] - if gen_data.get("improvement_path") and os.path.exists(gen_data["improvement_path"]): + if ( + transfer_evidence is None + and gen_data.get("improvement_path") + and os.path.exists(gen_data["improvement_path"]) + ): insights = self._extract_insights(gen_data["improvement_path"]) # Generate LLM summary of changes and improvements llm_summary = self._generate_llm_summary(gen_num, gen_data, metrics) # Format entry - entry = self._format_generation_entry(gen_num, gen_data, agent_stats, deltas, metrics, insights, llm_summary) + entry = self._format_generation_entry( + gen_num=gen_num, + gen_data=gen_data, + stats=agent_stats, + deltas=deltas, + metrics=metrics, + insights=insights, + llm_summary=llm_summary, + transfer_evidence=transfer_evidence, + ) # Append to file with open(self.context_path, "a", encoding="utf-8") as f: @@ -264,6 +279,14 @@ def add_generation(self, gen_num: int, gen_data: dict[str, Any]): logger.info(f"Added Generation {gen_num} to context.md") + def _load_transfer_evidence(self, transfer_evidence_path: str | None) -> dict[str, Any] | None: + if transfer_evidence_path is None: + return None + evidence = _safe_load_json(transfer_evidence_path) + if not isinstance(evidence, dict): + return None + return evidence + def finalize(self): """Add summary statistics at the end of context.md""" if not self.generations: @@ -459,6 +482,7 @@ def _format_generation_entry( deltas: dict[str, float], metrics: dict[str, Any], insights: list[str], + transfer_evidence: dict[str, Any] | None = None, llm_summary: str | None = None, ) -> str: """Format markdown entry for a generation""" @@ -489,7 +513,42 @@ def _format_generation_entry( - File size: {stats["size"]:,} bytes ({delta_size_str}) - Lines: {stats["lines"]} ({delta_lines_str} lines) """ - if insights: + if transfer_evidence: + entry += "- Transfer evidence carryover:\n" + entry += f" * Reuse boundary: {transfer_evidence.get('claim_boundary', 'Reuse boundary follows the card.')}\n" + accepted_for_reuse = transfer_evidence.get("accepted_for_reuse") + if isinstance(accepted_for_reuse, bool): + entry += f" * Accepted for reuse: {'yes' if accepted_for_reuse else 'no'}\n" + reusable = transfer_evidence.get("reusable_changes", []) + if isinstance(reusable, list) and reusable: + reusable_label = ( + " * Reusable guidance:\n" + if accepted_for_reuse is not False + else " * Candidate changes rejected for reuse:\n" + ) + entry += reusable_label + for item in reusable[:3]: + if not isinstance(item, str): + continue + entry += f" * {item}\n" + residue = transfer_evidence.get("task_specific_residue", []) + if isinstance(residue, list) and residue: + entry += " * Residue / caution (not safe to reuse):\n" + for item in residue[:3]: + if not isinstance(item, str): + continue + entry += f" * {item}\n" + unsupported = transfer_evidence.get("unsupported_claims", []) + if isinstance(unsupported, list) and unsupported: + entry += " * Unsupported claim notes:\n" + for item in unsupported[:3]: + if not isinstance(item, str): + continue + entry += f" * {item}\n" + score_delta = transfer_evidence.get("score_delta") + if isinstance(score_delta, (int, float)): + entry += f" * Score change: {score_delta:+.4f}\n" + elif insights: entry += "- Key changes from improvement.md:\n" for insight in insights[:3]: # Truncate very long insights diff --git a/sia/layout.py b/sia/layout.py index 1efbc01..c875f18 100644 --- a/sia/layout.py +++ b/sia/layout.py @@ -30,6 +30,7 @@ class Names: TRAIN_STDOUT_LOG = "train_stdout.log" EVAL_LOG = "evaluation.log" RESULTS_JSON = "results.json" + TRANSFER_EVIDENCE_JSON = "transfer_evidence.json" CONTEXT_MD = "context.md" IMPROVEMENT_MD = "improvement.md" META_PROMPT = "meta_agent_prompt.txt" diff --git a/sia/orchestrator.py b/sia/orchestrator.py index 0411565..edf7538 100644 --- a/sia/orchestrator.py +++ b/sia/orchestrator.py @@ -46,22 +46,24 @@ import glob import json import os +import re import subprocess import time import traceback from datetime import datetime from pathlib import Path +from typing import Any from sia import __version__, cli from sia.agent_reference import ResolvedAgentReference, copy_reference_into, resolve_agent_reference from sia.config import Config -from sia.io_utils import file_size_ok, write_text +from sia.io_utils import file_size_ok, safe_load_json, write_text from sia.layout import BUNDLED_TASKS, Names, RunLayout, TaskLayout, resolve_task_dir, venv_python_path from sia.logging_setup import configure_logging, get_logger from sia.profiles import MetaAgentProfile, load_meta_agent_profile, load_target_agent_profile from sia.prompts import build_feedback_prompt, build_meta_prompt from sia.providers import Provider -from sia.results import FeedbackContext, TargetAgentResult +from sia.results import FeedbackContext, TargetAgentResult, TransferEvidenceCard from sia.run_setup import RunSetup, TaskFiles, install_requirements, load_task_files, setup_run_directory from sia.util import run_agent @@ -80,9 +82,241 @@ "setup_run_directory", ] +_TRANSFER_EVIDENCE_MAX_BULLETS = 5 +_TRANSFER_EVIDENCE_SCORE_KEYS = ("accuracy", "score", "f1", "reward", "loss") +_TRANSFER_EVIDENCE_LOWER_IS_BETTER_KEYS = {"loss"} +_RESIDUE_HINTS = ( + "task-specific", + "this task", + "this run", + "this dataset", + "this issue", + "in this gen", + "for this", + "specific to", + "hardcoded", +) + logger = get_logger(__name__) +def _truncate_transfer_list(values: list[Any]) -> list[str]: + cleaned: list[str] = [] + for value in values: + if len(cleaned) >= _TRANSFER_EVIDENCE_MAX_BULLETS: + break + if not isinstance(value, str): + continue + stripped = value.strip() + if stripped: + cleaned.append(stripped) + return cleaned + + +def _extract_improvement_bullets(improvement_content: str) -> list[str]: + bullet_pattern = r"^[-*]\s+(.+)$" + bullets = re.findall(bullet_pattern, improvement_content, re.MULTILINE) + + numbered_pattern = r"^\d+\.\s+(.+)$" + numbered = re.findall(numbered_pattern, improvement_content, re.MULTILINE) + + all_lines = bullets + numbered + return [line.strip() for line in all_lines if line.strip() and not line.strip().endswith(":")] + + +def _partition_bullets(bullets: list[str]) -> tuple[list[str], list[str]]: + reusable: list[str] = [] + residue: list[str] = [] + for line in bullets: + lower = line.lower() + if any(hint in lower for hint in _RESIDUE_HINTS): + residue.append(line) + else: + reusable.append(line) + return reusable, residue + + +def _read_score(results_data: dict[str, Any] | None, score_key: str | None = None) -> tuple[str | None, float | None]: + if not isinstance(results_data, dict): + return None, None + + keys = (score_key,) if score_key is not None else _TRANSFER_EVIDENCE_SCORE_KEYS + ordered_keys = [key for key in keys if isinstance(key, str)] + + for key in ordered_keys: + raw_value = results_data.get(key) + if not isinstance(raw_value, (int, float, str)): + continue + try: + parsed_value = raw_value.rstrip("%") if isinstance(raw_value, str) else raw_value + value = float(parsed_value) + return key, value + except (TypeError, ValueError): + continue + + return None, None + + +def _read_previous_score(current_gen: int, gen_dir: str, score_key: str | None) -> float | None: + previous_gen_num = current_gen - 1 + if previous_gen_num < 1: + return None + previous_dir = os.path.join(os.path.dirname(gen_dir), f"gen_{previous_gen_num}") + previous_results = safe_load_json(os.path.join(previous_dir, Names.RESULTS_JSON)) + previous_results_dict = previous_results if isinstance(previous_results, dict) else None + _key, value = _read_score(previous_results_dict, score_key=score_key) + return value + + +def _score_delta_supports_reuse(score_key: str | None, score_delta: float | None) -> bool: + if score_delta is None: + return True + if score_key in _TRANSFER_EVIDENCE_LOWER_IS_BETTER_KEYS: + return score_delta < 0 + return score_delta > 0 + + +def _build_transfer_evidence_card( + current_gen: int, + gen_dir: str, + improvement_path: str | None, + evaluation_result: dict, +) -> TransferEvidenceCard: + results_path = os.path.join(gen_dir, Names.RESULTS_JSON) + results_data = safe_load_json(results_path) + results_data_dict = results_data if isinstance(results_data, dict) else None + + evaluator_status = "missing" + if evaluation_result.get("status") == "success" and results_data_dict is not None: + evaluator_status = "passed" + elif evaluation_result.get("status") == "error": + evaluator_status = "error" + elif results_data_dict is not None: + evaluator_status = "failed" + + score_key, score_value = _read_score(results_data_dict) + prev_score = _read_previous_score(current_gen, gen_dir, score_key) + score_delta = score_value - prev_score if score_key and score_value is not None and prev_score is not None else None + + reusable_bullets: list[str] = [] + residue_bullets: list[str] = [] + if improvement_path and os.path.exists(improvement_path): + try: + raw_improvement = Path(improvement_path).read_text(encoding="utf-8") + bullets = _extract_improvement_bullets(raw_improvement) + reusable_bullets, residue_bullets = _partition_bullets(bullets) + except OSError as error: + logger.warning(f" ⚠ Could not read improvement.md for transfer evidence: {error}") + + default_claim_boundary = ( + "Treat residue as task-specific context, and apply only reusable bullets unless explicitly validated by " + "evaluation evidence." + ) + + unsupported_claims: list[str] = [] + if not score_key and not reusable_bullets: + unsupported_claims.append( + "No stable metric + bounded reusable signal was available this generation; avoid broad claims about transfer " + "quality." + ) + + accepted_for_reuse = ( + evaluator_status == "passed" and bool(reusable_bullets) and _score_delta_supports_reuse(score_key, score_delta) + ) + + return TransferEvidenceCard( + generation=current_gen, + accepted_for_reuse=accepted_for_reuse, + evaluator_status=evaluator_status, + score_delta=score_delta, + reusable_changes=_truncate_transfer_list(reusable_bullets), + task_specific_residue=_truncate_transfer_list(residue_bullets), + unsupported_claims=unsupported_claims, + claim_boundary=default_claim_boundary, + ) + + +def _write_transfer_evidence_card(gen_dir: str, card: TransferEvidenceCard) -> str | None: + transfer_path = os.path.join(gen_dir, Names.TRANSFER_EVIDENCE_JSON) + try: + write_text(transfer_path, json.dumps(card.as_dict(), indent=2)) + return transfer_path + except (OSError, TypeError) as error: + logger.warning(f" ✗ Failed to write transfer evidence to {transfer_path}: {error}") + return None + + +def _format_transfer_evidence_section(transfer_evidence_path: str | None) -> str: + if not transfer_evidence_path or not os.path.exists(transfer_evidence_path): + return ( + "**TRANSFER EVIDENCE**:\nNo transfer_evidence.json found. No reusable guidance boundary is available yet." + ) + + transfer_data = safe_load_json(transfer_evidence_path) + if not isinstance(transfer_data, dict): + return "**TRANSFER EVIDENCE**:\nMalformed transfer_evidence.json, reuse boundary is unavailable." + + accepted_for_reuse = transfer_data.get("accepted_for_reuse") + score_delta = transfer_data.get("score_delta") + evaluator_status = transfer_data.get("evaluator_status", "unknown") + generation = transfer_data.get("generation") + negative_probe_hits = transfer_data.get("negative_probe_hits") + + reusable_data = transfer_data.get("reusable_changes", []) + residue_data = transfer_data.get("task_specific_residue", []) + unsupported_data = transfer_data.get("unsupported_claims", []) + claim_boundary = transfer_data.get("claim_boundary") or "" + + reusable = _truncate_transfer_list(reusable_data) if isinstance(reusable_data, list) else [] + residue = _truncate_transfer_list(residue_data) if isinstance(residue_data, list) else [] + unsupported = _truncate_transfer_list(unsupported_data) if isinstance(unsupported_data, list) else [] + + lines = [f"**TRANSFER EVIDENCE**: evaluator={evaluator_status}"] + if isinstance(generation, int): + lines.append(f"- Generation: {generation}") + if isinstance(accepted_for_reuse, bool): + lines.append(f"- Accepted for reuse: {'yes' if accepted_for_reuse else 'no'}") + if isinstance(score_delta, (int, float)): + lines.append(f"- Score delta: {score_delta:+.4f}") + + if reusable: + reusable_label = ( + "- Accepted reusable changes:" + if accepted_for_reuse is not False + else "- Candidate changes not accepted for reuse:" + ) + lines.append(reusable_label) + lines.extend(f" * {item}" for item in reusable) + + if residue: + lines.append("- Task-specific residue to avoid carrying forward:") + lines.extend(f" * {item}" for item in residue) + + if unsupported: + lines.append("- Unsupported claim notes:") + lines.extend(f" * {item}" for item in unsupported) + + if isinstance(negative_probe_hits, int): + lines.append(f"- Negative probe hits: {negative_probe_hits}") + + if claim_boundary: + lines.append(f"- Claim boundary: {claim_boundary}") + + if not ( + isinstance(generation, int) + or isinstance(accepted_for_reuse, bool) + or isinstance(score_delta, (int, float)) + or reusable + or residue + or unsupported + or isinstance(negative_probe_hits, int) + or claim_boundary + ): + lines.append("- No usable transfer signal was detected.") + + return "\n".join(lines) + + # ======================== # HELPER FUNCTIONS # ======================== @@ -433,6 +667,7 @@ def _build_feedback_context( stdout_log_file: str, task_files: TaskFiles, config: Config | None = None, + transfer_evidence_path: str | None = None, ) -> tuple[str, str]: """Build execution status and section for feedback prompt. @@ -522,9 +757,13 @@ def _build_feedback_context( stdout_lines = target_agent_stdout.split("\n") last_10_lines = "\n".join(stdout_lines[-10:]) if len(stdout_lines) > 10 else target_agent_stdout + transfer_evidence_section = _format_transfer_evidence_section(transfer_evidence_path) + status_blocks = [block.strip() for block in (eval_results_section, transfer_evidence_section) if block] + status_text = "\n\n".join(status_blocks) + if target_agent_success: execution_status = f"""SUCCESS: Target agent completed execution successfully. -{eval_results_section} +{status_text} **Last 10 lines of output**: ``` @@ -535,7 +774,7 @@ def _build_feedback_context( """ else: execution_status = f"""FAILED: {target_agent_error_msg} -{eval_results_section} +{status_text} **Last 10 lines of output**: ``` @@ -692,7 +931,16 @@ def run_generation( # Run evaluation (if evaluate.py exists) logger.info("=" * 60) logger.info("Running evaluation (if available)...") - run_evaluation(gen_dir, dataset_dir, run_setup.venv_dir, config=env_config) + evaluation_result = run_evaluation(gen_dir, dataset_dir, run_setup.venv_dir, config=env_config) + transfer_evidence_path = _write_transfer_evidence_card( + gen_dir, + _build_transfer_evidence_card( + current_gen=current_gen, + gen_dir=gen_dir, + improvement_path=layout.improvement_md(current_gen), + evaluation_result=evaluation_result, + ), + ) logger.info("=" * 60) # Add generation to context @@ -706,6 +954,7 @@ def run_generation( "agent_path": target_agent_path, "gen_dir": gen_dir, "improvement_path": improvement_md_path if os.path.exists(improvement_md_path) else None, + "transfer_evidence_path": transfer_evidence_path, "execution_type": "Multi-trajectory" if os.path.isdir(layout.agent_execution_dir(current_gen)) else "Single", @@ -726,6 +975,7 @@ def run_generation( target_agent_stdout=target_agent_stdout, target_agent_stderr=target_agent_stderr, stdout_log_file=stdout_log_file, + transfer_evidence_path=transfer_evidence_path, task_files=task_files, config=env_config, ) diff --git a/sia/prompts.py b/sia/prompts.py index 4ca47a7..b38d352 100644 --- a/sia/prompts.py +++ b/sia/prompts.py @@ -765,6 +765,7 @@ def build_feedback_prompt( told it may add/edit a requirements.txt there. ``None`` keeps the historical text. """ context_md_path = os.path.join(run_dir, "context.md") + transfer_evidence_path = os.path.join(run_dir, f"gen_{current_gen}", "transfer_evidence.json") # Handle weights mode (RL-based tuning) if focus == "weights": @@ -776,13 +777,11 @@ def build_feedback_prompt( - Evolution history: {context_md_path} **BEFORE ANALYZING - READ THE FULL HISTORY**: -1. Read {context_md_path} to understand: - - What improvements were tried in each previous generation - - Training and performance trends across generations - - What worked and what didn't work -2. Review previous improvement.md files from earlier generations if helpful -3. Don't repeat failed approaches from earlier generations -4. Build upon successful RL patterns that improved performance +1. Read {context_md_path} for the full generation history +2. Read {transfer_evidence_path} and treat only reusable bullets as reusable guidance +3. Review previous improvement.md files from earlier generations if helpful +4. Don't repeat failed approaches from earlier generations +5. Build upon successful RL patterns that improved performance --- @@ -869,9 +868,10 @@ def build_feedback_prompt( - What improvements were tried in each previous generation - Performance trends across generations - What worked and what didn't work -2. Review previous improvement.md files from earlier generations if helpful -3. Don't repeat failed approaches from earlier generations -4. Build upon successful patterns that improved performance +2. Read {transfer_evidence_path} and treat residue as non-reusable context +3. Review previous improvement.md files from earlier generations if helpful +4. Don't repeat failed approaches from earlier generations +5. Build upon successful patterns that improved performance --- diff --git a/sia/results.py b/sia/results.py index c922ff3..461b6be 100644 --- a/sia/results.py +++ b/sia/results.py @@ -7,7 +7,8 @@ from __future__ import annotations -from dataclasses import dataclass +from dataclasses import dataclass, field +from typing import Any @dataclass @@ -32,3 +33,31 @@ class FeedbackContext: def as_tuple(self) -> tuple[str, str]: return (self.execution_status, self.execution_section) + + +@dataclass +class TransferEvidenceCard: + """Structured output produced after each generation for feedback context and context carryover.""" + + generation: int + accepted_for_reuse: bool + evaluator_status: str + score_delta: float | None + reusable_changes: list[str] = field(default_factory=list) + task_specific_residue: list[str] = field(default_factory=list) + unsupported_claims: list[str] = field(default_factory=list) + negative_probe_hits: int = 0 + claim_boundary: str = "No evidence supports task-agnostic transfer beyond the reusable bullets above." + + def as_dict(self) -> dict[str, Any]: + return { + "generation": self.generation, + "accepted_for_reuse": self.accepted_for_reuse, + "evaluator_status": self.evaluator_status, + "score_delta": self.score_delta, + "reusable_changes": self.reusable_changes, + "task_specific_residue": self.task_specific_residue, + "unsupported_claims": self.unsupported_claims, + "negative_probe_hits": self.negative_probe_hits, + "claim_boundary": self.claim_boundary, + } diff --git a/tests/golden/context.md b/tests/golden/context.md index e06ec8d..c4c6499 100644 --- a/tests/golden/context.md +++ b/tests/golden/context.md @@ -17,7 +17,7 @@ ### Target Agent Changes - Initial agent created by meta-agent -- File size: 21 bytes +- File size: 22 bytes - Lines of code: 1 ### Execution Summary @@ -39,12 +39,19 @@ ### Target Agent Changes - Modified by feedback agent -- File size: 69 bytes (+228.6%) +- File size: 77 bytes (+250.0%) - Lines: 8 (+7 lines) -- Key changes from improvement.md: - * Added structured error handling so the agent recovers from tool failures gracefully. - * Switched to a retry loop with exponential backoff for transient API errors. - * Improved logging to capture each tool call and its result for later analysis. +- Transfer evidence carryover: + * Reuse boundary: Treat residue as task-specific context. + * Accepted for reuse: yes + * Reusable guidance: + * Added structured error handling so the agent recovers from tool failures gracefully. + * Improved logging to capture each tool call and its result for later analysis. + * Residue / caution (not safe to reuse): + * The retry loop is task-specific to this evaluation harness. + * Unsupported claim notes: + * No benchmark-portable claim was validated in this run. + * Score change: +25.0000 ### Execution Summary - Execution status: ✓ SUCCESS @@ -72,6 +79,6 @@ - 50.00% → 75.00% (+25.00%) **Code Growth**: -- Initial: 1 lines (21 bytes) -- Final: 8 lines (69 bytes) -- Growth: 7 lines (+48 bytes) +- Initial: 1 lines (22 bytes) +- Final: 8 lines (77 bytes) +- Growth: 7 lines (+55 bytes) diff --git a/tests/golden/feedback_context_failure_single.txt b/tests/golden/feedback_context_failure_single.txt index 08e05ff..0e775b8 100644 --- a/tests/golden/feedback_context_failure_single.txt +++ b/tests/golden/feedback_context_failure_single.txt @@ -1,8 +1,9 @@ ===== EXECUTION STATUS ===== FAILED: Target agent failed with exit code 1 - **EVALUATION RESULTS**: No results.json found (evaluation may not have run or may have failed) +**TRANSFER EVIDENCE**: +No transfer_evidence.json found. No reusable guidance boundary is available yet. **Last 10 lines of output**: ``` diff --git a/tests/golden/feedback_context_success_multi.txt b/tests/golden/feedback_context_success_multi.txt index f3f1c23..44da1da 100644 --- a/tests/golden/feedback_context_success_multi.txt +++ b/tests/golden/feedback_context_success_multi.txt @@ -1,7 +1,5 @@ ===== EXECUTION STATUS ===== SUCCESS: Target agent completed execution successfully. - - **EVALUATION RESULTS**: ```json { @@ -9,6 +7,8 @@ SUCCESS: Target agent completed execution successfully. } ``` +**TRANSFER EVIDENCE**: +No transfer_evidence.json found. No reusable guidance boundary is available yet. **Last 10 lines of output**: ``` diff --git a/tests/golden/feedback_context_success_single.txt b/tests/golden/feedback_context_success_single.txt index d1c50f6..9a0c83c 100644 --- a/tests/golden/feedback_context_success_single.txt +++ b/tests/golden/feedback_context_success_single.txt @@ -1,7 +1,5 @@ ===== EXECUTION STATUS ===== SUCCESS: Target agent completed execution successfully. - - **EVALUATION RESULTS**: ```json { @@ -11,6 +9,19 @@ SUCCESS: Target agent completed execution successfully. } ``` +**TRANSFER EVIDENCE**: evaluator=passed +- Generation: 1 +- Accepted for reuse: yes +- Score delta: +0.1500 +- Accepted reusable changes: + * Use metric-guided rollout tuning for prompt templates. + * Avoid brittle assumptions about dataset-specific fields. +- Task-specific residue to avoid carrying forward: + * Task-specific retries were introduced in this run. +- Unsupported claim notes: + * No claim about benchmark portability was validated. +- Negative probe hits: 0 +- Claim boundary: Treat residue as task-specific context. **Last 10 lines of output**: ``` diff --git a/tests/golden/feedback_prompt.txt b/tests/golden/feedback_prompt.txt index 751d293..6fa5e1b 100644 --- a/tests/golden/feedback_prompt.txt +++ b/tests/golden/feedback_prompt.txt @@ -10,9 +10,10 @@ You are an expert AI Engineer analyzing agent scaffolds for iterative improvemen - What improvements were tried in each previous generation - Performance trends across generations - What worked and what didn't work -2. Review previous improvement.md files from earlier generations if helpful -3. Don't repeat failed approaches from earlier generations -4. Build upon successful patterns that improved performance +2. Read /RUN/run_1/gen_2/transfer_evidence.json and treat residue as non-reusable context +3. Review previous improvement.md files from earlier generations if helpful +4. Don't repeat failed approaches from earlier generations +5. Build upon successful patterns that improved performance --- diff --git a/tests/test_context_golden.py b/tests/test_context_golden.py index 16d8c59..4873a22 100644 --- a/tests/test_context_golden.py +++ b/tests/test_context_golden.py @@ -21,6 +21,20 @@ "- Switched to a retry loop with exponential backoff for transient API errors.\n" "- Improved logging to capture each tool call and its result for later analysis.\n" ) +TRANSFER_EVIDENCE = { + "generation": 2, + "accepted_for_reuse": True, + "evaluator_status": "passed", + "score_delta": 25.0, + "reusable_changes": [ + "Added structured error handling so the agent recovers from tool failures gracefully.", + "Improved logging to capture each tool call and its result for later analysis.", + ], + "task_specific_residue": ["The retry loop is task-specific to this evaluation harness."], + "unsupported_claims": ["No benchmark-portable claim was validated in this run."], + "negative_probe_hits": 0, + "claim_boundary": "Treat residue as task-specific context.", +} @patch("sia.context_manager.ContextManager._generate_llm_summary", return_value=None) @@ -36,6 +50,7 @@ def test_context_md_golden(_mock_llm, tmp_path): (gen2 / "improvement.md").write_text(IMPROVEMENT_MD) (gen1 / "results.json").write_text(json.dumps({"accuracy": 50.0, "correct": 99, "total": 198})) (gen2 / "results.json").write_text(json.dumps({"accuracy": 75.0, "correct": 148, "total": 198})) + (gen2 / "transfer_evidence.json").write_text(json.dumps(TRANSFER_EVIDENCE)) cm = ContextManager( str(run_dir), @@ -69,6 +84,7 @@ def test_context_md_golden(_mock_llm, tmp_path): "agent_path": str(gen2 / "target_agent.py"), "gen_dir": str(gen2), "improvement_path": str(gen2 / "improvement.md"), + "transfer_evidence_path": str(gen2 / "transfer_evidence.json"), "execution_type": "Single", }, ) diff --git a/tests/test_context_manager.py b/tests/test_context_manager.py index a7aaea3..b09ad4f 100644 --- a/tests/test_context_manager.py +++ b/tests/test_context_manager.py @@ -133,6 +133,17 @@ def test_multiple_generations_track_deltas(mock_llm, context_mgr, run_dir): (gen2 / "target_agent.py").write_text("print('improved')\nimport os\n") (gen2 / "results.json").write_text(json.dumps({"accuracy": 0.85})) (gen2 / "improvement.md").write_text("## Changes\n- Added better error handling\n- Improved prompt structure\n") + transfer_evidence = { + "generation": 2, + "accepted_for_reuse": True, + "evaluator_status": "passed", + "score_delta": 0.15, + "claim_boundary": "Treat residue as task-specific context.", + "reusable_changes": ["Added better error handling"], + "task_specific_residue": ["Hardcoded this task's timeout to avoid false positives."], + "negative_probe_hits": 0, + } + (gen2 / "transfer_evidence.json").write_text(json.dumps(transfer_evidence)) context_mgr.add_generation( gen_num=2, @@ -143,10 +154,11 @@ def test_multiple_generations_track_deltas(mock_llm, context_mgr, run_dir): "agent_path": str(gen2 / "target_agent.py"), "gen_dir": str(gen2), "improvement_path": str(gen2 / "improvement.md"), + "transfer_evidence_path": str(gen2 / "transfer_evidence.json"), "execution_type": "Single", }, ) content = (run_dir / "context.md").read_text() assert "Generation 2" in content - assert "Modified by feedback agent" in content + assert "Transfer evidence carryover" in content diff --git a/tests/test_feedback_context_golden.py b/tests/test_feedback_context_golden.py index e674ac2..3ea8077 100644 --- a/tests/test_feedback_context_golden.py +++ b/tests/test_feedback_context_golden.py @@ -3,6 +3,7 @@ """ import json +from pathlib import Path from golden_master import assert_golden, normalize_paths @@ -13,7 +14,29 @@ def _snapshot(gen_dir, stdout_log_file, status, section) -> str: text = "===== EXECUTION STATUS =====\n" + status + "\n===== EXECUTION SECTION =====\n" + section - return normalize_paths(text, {str(gen_dir): "", str(stdout_log_file): ""}) + return normalize_paths(text, {str(gen_dir): "", str(stdout_log_file): ""}).replace("\\", "/") + + +def _write_transfer_evidence(path: Path) -> None: + path.write_text( + json.dumps( + { + "generation": 1, + "accepted_for_reuse": True, + "evaluator_status": "passed", + "score_delta": 0.15, + "reusable_changes": [ + "Use metric-guided rollout tuning for prompt templates.", + "Avoid brittle assumptions about dataset-specific fields.", + ], + "task_specific_residue": ["Task-specific retries were introduced in this run."], + "unsupported_claims": ["No claim about benchmark portability was validated."], + "negative_probe_hits": 0, + "claim_boundary": "Treat residue as task-specific context.", + } + ), + encoding="utf-8", + ) def test_success_single_with_results(tmp_path): @@ -22,6 +45,8 @@ def test_success_single_with_results(tmp_path): (gen_dir / "agent_execution.json").write_text(json.dumps([{"role": "user", "content": "solve it"}])) (gen_dir / "results.json").write_text(json.dumps({"accuracy": 0.9, "correct": 9, "total": 10})) stdout_log = str(gen_dir / "target_agent_stdout.log") + transfer_evidence_path = gen_dir / "transfer_evidence.json" + _write_transfer_evidence(transfer_evidence_path) status, section = _build_feedback_context( current_gen=1, @@ -33,6 +58,7 @@ def test_success_single_with_results(tmp_path): target_agent_stderr="", stdout_log_file=stdout_log, task_files=TASK_FILES, + transfer_evidence_path=str(transfer_evidence_path), ) assert_golden("feedback_context_success_single.txt", _snapshot(gen_dir, stdout_log, status, section)) @@ -78,3 +104,87 @@ def test_success_multi_with_results(tmp_path): task_files=TASK_FILES, ) assert_golden("feedback_context_success_multi.txt", _snapshot(gen_dir, stdout_log, status, section)) + + +def test_malformed_transfer_evidence_does_not_break_context(tmp_path): + gen_dir = tmp_path / "gen_1" + gen_dir.mkdir() + (gen_dir / "agent_execution.json").write_text(json.dumps([{"role": "user", "content": "attempt"}])) + transfer_evidence_path = gen_dir / "transfer_evidence.json" + transfer_evidence_path.write_text( + json.dumps( + { + "generation": 1, + "accepted_for_reuse": False, + "evaluator_status": "missing", + "reusable_changes": [1, "valid reusable change"], + "task_specific_residue": [None, "Task-specific branch"], + "unsupported_claims": [2, "Missing evidence"], + "negative_probe_hits": 1, + "claim_boundary": "Stay conservative.", + } + ), + encoding="utf-8", + ) + stdout_log = str(gen_dir / "target_agent_stdout.log") + + status, section = _build_feedback_context( + current_gen=1, + gen_dir=str(gen_dir), + dataset_dir="/data/public", + target_agent_success=True, + target_agent_error_msg="", + target_agent_stdout="line1\nline2\n", + target_agent_stderr="", + stdout_log_file=stdout_log, + task_files=TASK_FILES, + transfer_evidence_path=str(transfer_evidence_path), + ) + + snapshot = _snapshot(gen_dir, stdout_log, status, section) + assert "**TRANSFER EVIDENCE**: evaluator=missing" in snapshot + assert "valid reusable change" in snapshot + assert "Task-specific branch" in snapshot + assert "Missing evidence" in snapshot + + +def test_negative_delta_transfer_evidence_is_not_rendered_as_reusable(tmp_path): + gen_dir = tmp_path / "gen_2" + gen_dir.mkdir() + (gen_dir / "agent_execution.json").write_text(json.dumps([{"role": "user", "content": "attempt"}])) + transfer_evidence_path = gen_dir / "transfer_evidence.json" + transfer_evidence_path.write_text( + json.dumps( + { + "generation": 2, + "accepted_for_reuse": False, + "evaluator_status": "passed", + "score_delta": -0.4, + "reusable_changes": ["Reusable-looking change"], + "task_specific_residue": ["Task-specific fallback"], + "unsupported_claims": [], + "negative_probe_hits": 0, + "claim_boundary": "Do not carry negative-delta changes forward.", + } + ), + encoding="utf-8", + ) + stdout_log = str(gen_dir / "target_agent_stdout.log") + + status, section = _build_feedback_context( + current_gen=2, + gen_dir=str(gen_dir), + dataset_dir="/data/public", + target_agent_success=True, + target_agent_error_msg="", + target_agent_stdout="line1\nline2\n", + target_agent_stderr="", + stdout_log_file=stdout_log, + task_files=TASK_FILES, + transfer_evidence_path=str(transfer_evidence_path), + ) + + snapshot = _snapshot(gen_dir, stdout_log, status, section) + assert "Accepted for reuse: no" in snapshot + assert "Candidate changes not accepted for reuse" in snapshot + assert "Accepted reusable changes" not in snapshot diff --git a/tests/test_orchestrator_helpers.py b/tests/test_orchestrator_helpers.py index 74b7738..163a1df 100644 --- a/tests/test_orchestrator_helpers.py +++ b/tests/test_orchestrator_helpers.py @@ -1,8 +1,12 @@ """Unit tests for orchestrator helper functions.""" import json +import math -from sia.orchestrator import load_agent_execution +from sia.orchestrator import ( + _build_transfer_evidence_card, + load_agent_execution, +) def test_load_single_trajectory(tmp_path): @@ -48,3 +52,137 @@ def test_load_empty_multi_trajectory_folder(tmp_path): data, is_multi = load_agent_execution(str(tmp_path)) assert is_multi assert "error" in data + + +def test_build_transfer_evidence_card_keeps_reusable_and_residue(tmp_path): + gen1 = tmp_path / "gen_1" + gen2 = tmp_path / "gen_2" + gen1.mkdir() + gen2.mkdir() + + (gen1 / "results.json").write_text(json.dumps({"accuracy": 0.7})) + (gen2 / "results.json").write_text(json.dumps({"accuracy": 0.85})) + (gen2 / "improvement.md").write_text( + "\n".join( + [ + "# Improvement Plan", + "- Added generic prompt retry flow.", + "- This task-specific branch added a hardcoded guard for sample 17.", + ] + ) + ) + + card = _build_transfer_evidence_card( + current_gen=2, + gen_dir=str(gen2), + improvement_path=str(gen2 / "improvement.md"), + evaluation_result={"status": "success"}, + ) + + assert card.generation == 2 + assert card.accepted_for_reuse is True + assert card.evaluator_status == "passed" + assert math.isclose(card.score_delta, 0.15, rel_tol=0, abs_tol=1e-12) + assert card.reusable_changes == ["Added generic prompt retry flow."] + assert card.task_specific_residue == ["This task-specific branch added a hardcoded guard for sample 17."] + + +def test_build_transfer_evidence_card_marks_missing_data(tmp_path): + gen1 = tmp_path / "gen_1" + gen1.mkdir() + (gen1 / "results.json").write_text(json.dumps({"correct": 9, "total": 10})) + + card = _build_transfer_evidence_card( + current_gen=1, + gen_dir=str(gen1), + improvement_path=None, + evaluation_result={"status": "warning"}, + ) + + assert card.accepted_for_reuse is False + assert card.evaluator_status == "failed" + assert card.unsupported_claims + + +def test_build_transfer_evidence_card_rejects_negative_score_delta(tmp_path): + gen1 = tmp_path / "gen_1" + gen2 = tmp_path / "gen_2" + gen1.mkdir() + gen2.mkdir() + + (gen1 / "results.json").write_text(json.dumps({"accuracy": 0.9})) + (gen2 / "results.json").write_text(json.dumps({"accuracy": 0.5})) + (gen2 / "improvement.md").write_text("- Added reusable planning scaffold.\n") + + card = _build_transfer_evidence_card( + current_gen=2, + gen_dir=str(gen2), + improvement_path=str(gen2 / "improvement.md"), + evaluation_result={"status": "success"}, + ) + + assert math.isclose(card.score_delta, -0.4, rel_tol=0, abs_tol=1e-12) + assert card.accepted_for_reuse is False + + +def test_build_transfer_evidence_card_accepts_lower_loss(tmp_path): + gen1 = tmp_path / "gen_1" + gen2 = tmp_path / "gen_2" + gen1.mkdir() + gen2.mkdir() + + (gen1 / "results.json").write_text(json.dumps({"loss": 0.9})) + (gen2 / "results.json").write_text(json.dumps({"loss": 0.5})) + (gen2 / "improvement.md").write_text("- Added reusable planning scaffold.\n") + + card = _build_transfer_evidence_card( + current_gen=2, + gen_dir=str(gen2), + improvement_path=str(gen2 / "improvement.md"), + evaluation_result={"status": "success"}, + ) + + assert math.isclose(card.score_delta, -0.4, rel_tol=0, abs_tol=1e-12) + assert card.accepted_for_reuse is True + + +def test_build_transfer_evidence_card_rejects_non_improving_percent_metric(tmp_path): + gen1 = tmp_path / "gen_1" + gen2 = tmp_path / "gen_2" + gen1.mkdir() + gen2.mkdir() + + (gen1 / "results.json").write_text(json.dumps({"accuracy": "90%"})) + (gen2 / "results.json").write_text(json.dumps({"accuracy": "50%"})) + (gen2 / "improvement.md").write_text("- Added reusable planning scaffold.\n") + + card = _build_transfer_evidence_card( + current_gen=2, + gen_dir=str(gen2), + improvement_path=str(gen2 / "improvement.md"), + evaluation_result={"status": "success"}, + ) + + assert math.isclose(card.score_delta, -40.0, rel_tol=0, abs_tol=1e-12) + assert card.accepted_for_reuse is False + + +def test_build_transfer_evidence_card_rejects_zero_delta_accuracy(tmp_path): + gen1 = tmp_path / "gen_1" + gen2 = tmp_path / "gen_2" + gen1.mkdir() + gen2.mkdir() + + (gen1 / "results.json").write_text(json.dumps({"accuracy": 0.9})) + (gen2 / "results.json").write_text(json.dumps({"accuracy": 0.9})) + (gen2 / "improvement.md").write_text("- Added reusable planning scaffold.\n") + + card = _build_transfer_evidence_card( + current_gen=2, + gen_dir=str(gen2), + improvement_path=str(gen2 / "improvement.md"), + evaluation_result={"status": "success"}, + ) + + assert math.isclose(card.score_delta, 0.0, rel_tol=0, abs_tol=1e-12) + assert card.accepted_for_reuse is False diff --git a/tests/test_prompts_snapshot.py b/tests/test_prompts_snapshot.py index 940f019..4d57411 100644 --- a/tests/test_prompts_snapshot.py +++ b/tests/test_prompts_snapshot.py @@ -64,4 +64,4 @@ def test_feedback_prompt_golden(): previous_gens="1", task_model="claude-haiku-4-5-20251001", ) - assert_golden("feedback_prompt.txt", prompt) + assert_golden("feedback_prompt.txt", prompt.replace("\\", "/"))