diff --git a/TODO.md b/TODO.md index 3ee0b915..09f3c11b 100644 --- a/TODO.md +++ b/TODO.md @@ -201,8 +201,9 @@ - Target: meaningful coverage on core logic paths, not just line count - [x] **Add regression test for config validation** - Malformed YAML, missing required sections, type mismatches -- [ ] **Add test for checkpoint save/restore** +- [x] **Add test for checkpoint save/restore** _(done 2026-07-30)_ - Interrupt mid-evolution, restore from checkpoint, verify state consistency + - 13 tests in `tests/unit/core/test_checkpoint_save_restore.py`: basic round-trip, metadata preservation, integrity hash verification, tamper detection, tamper-restore integration, nonexistent restore error, multi-checkpoint isolation, list/delete, target-directory clearing (protected dirs preserved), size reporting, system-state capture, and mid-evolution interruption recovery - [ ] **Add tests for safety-decision orchestration** _(found 2026-07-22 whole-repo review)_ — the code paths behind the checkpoint/ImprovementValidator/resilience-init bugs above have zero regression test coverage today, which is plausibly why they shipped unnoticed ### Medium-Priority Bugs Found in Whole-Repo Code Review (2026-07-22) @@ -305,9 +306,9 @@ |----------|-------|------|-------| | 🔴 P0 | 11 | 11 | Original 5 complete; all 6 critical bugs from 2026-07-22 whole-repo review fixed (PRs #74, #76-#79) | | 🟠 P1 | 24 | 14 | Original safety/integration items done; +12 high-priority bugs from 2026-07-22 review; signal-handler init fix; safety.yaml created | -| 🟡 P2 | 30 | 20 | Co-evolution loop gaps (8 items, 8 done) + existing P2 + 13 medium bugs from 2026-07-22 review + 4 latent collect->train bugs found closing the loop (1 fixed, 1 new HF-format gap resolved); provider_manager health-check await fix; workflow-agent private-API/event-loop fix | +| 🟡 P2 | 30 | 21 | Co-evolution loop gaps (8 items, 8 done) + existing P2 + 13 medium bugs from 2026-07-22 review + 4 latent collect->train bugs found closing the loop (1 fixed, 1 new HF-format gap resolved); provider_manager health-check await fix; workflow-agent private-API/event-loop fix; checkpoint save/restore test | | 🟢 P3 | 24 | 15 | Makefile, pre-commit, Docker, ADRs, ADR refresh, CHANGELOG complete; +11 hygiene items from 2026-07-22 review; Ollama provider retry/backoff fix; local_models TTL cache | -| **Total** | **89** | **60** | | +| **Total** | **89** | **61** | | > Update this table as you complete items. Recommended flow: P0 → P1 → P2 → P3. > diff --git a/tests/unit/core/test_checkpoint_save_restore.py b/tests/unit/core/test_checkpoint_save_restore.py new file mode 100644 index 00000000..980a25cc --- /dev/null +++ b/tests/unit/core/test_checkpoint_save_restore.py @@ -0,0 +1,234 @@ +"""Tests for checkpoint save and restore cycle. + +Verifies that saving a checkpoint and restoring it produces consistent +state — covering the round-trip through create → restore → verify. +""" + +from __future__ import annotations + +from pathlib import Path + +import pytest + +from evoseal.core.checkpoint_manager import CheckpointError, CheckpointManager + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture +def checkpoint_dir(tmp_path: Path) -> Path: + return tmp_path / "checkpoints" + + +@pytest.fixture +def target_dir(tmp_path: Path) -> Path: + return tmp_path / "restore_target" + + +@pytest.fixture +def manager(checkpoint_dir: Path) -> CheckpointManager: + return CheckpointManager(config={"checkpoint_directory": str(checkpoint_dir)}) + + +def _sample_version(version_id: str = "v1") -> dict: + """Return a minimal but valid version dict for checkpoint creation.""" + return { + "id": version_id, + "name": f"version-{version_id}", + "description": "test version", + "changes": { + "src/main.py": "print('hello')\n", + "src/utils.py": "def helper():\n return 42\n", + }, + "config": {"learning_rate": 0.001, "epochs": 3}, + "metrics": [{"name": "fitness", "value": 0.85, "timestamp": "2026-01-01T00:00:00Z"}], + "result": {"best_fitness": 0.85, "generations_completed": 5}, + "timestamp": "2026-01-01T00:00:00Z", + } + + +# --------------------------------------------------------------------------- +# Basic round-trip +# --------------------------------------------------------------------------- + + +class TestCheckpointSaveRestore: + """Core save/restore round-trip tests.""" + + def test_create_and_restore_basic(self, manager: CheckpointManager, target_dir: Path) -> None: + """Saving then restoring a checkpoint reproduces all files.""" + version = _sample_version("v1") + cp_path = manager.create_checkpoint("v1", version, capture_system_state=False) + + assert Path(cp_path).exists() + + result = manager.restore_checkpoint("v1", target_dir, verify_integrity=True) + + assert result["success"] is True + assert result["version_id"] == "v1" + assert (target_dir / "src" / "main.py").read_text() == "print('hello')\n" + assert (target_dir / "src" / "utils.py").read_text() == "def helper():\n return 42\n" + + def test_metadata_preserved_after_restore( + self, manager: CheckpointManager, target_dir: Path + ) -> None: + """Metadata (config, metrics, result) survives the round-trip.""" + version = _sample_version("v2") + manager.create_checkpoint("v2", version, capture_system_state=False) + + result = manager.restore_checkpoint("v2", target_dir, verify_integrity=True) + metadata = result["metadata"] + + assert metadata["version_id"] == "v2" + assert metadata["config_snapshot"]["learning_rate"] == 0.001 + assert metadata["metrics_count"] == 1 + assert metadata["has_results"] is True + + def test_integrity_hash_matches_after_save(self, manager: CheckpointManager) -> None: + """Integrity verification passes immediately after creation.""" + version = _sample_version("v3") + manager.create_checkpoint("v3", version, capture_system_state=False) + + assert manager.verify_checkpoint_integrity("v3") is True + + def test_integrity_hash_fails_when_file_tampered(self, manager: CheckpointManager) -> None: + """Integrity verification fails when a file is modified after checkpoint. + + NOTE: This test mutates a file directly inside the checkpoint directory, + coupling to the current plain-file storage layout. If the storage + backend changes (e.g. compressed/archived), this test will need updating. + """ + version = _sample_version("v4") + manager.create_checkpoint("v4", version, capture_system_state=False) + + # Tamper with a file inside the checkpoint + cp_path = Path(manager.get_checkpoint_path("v4")) + tampered = cp_path / "src" / "main.py" + tampered.write_text("EVIL CODE\n") + + assert manager.verify_checkpoint_integrity("v4") is False + + def test_restore_rejects_tampered_checkpoint( + self, manager: CheckpointManager, target_dir: Path + ) -> None: + """restore_checkpoint(verify_integrity=True) raises on a tampered checkpoint.""" + version = _sample_version("v4b") + manager.create_checkpoint("v4b", version, capture_system_state=False) + + # Tamper with a file inside the checkpoint + cp_path = Path(manager.get_checkpoint_path("v4b")) + (cp_path / "src" / "main.py").write_text("EVIL CODE\n") + + with pytest.raises(CheckpointError, match="Integrity"): + manager.restore_checkpoint("v4b", target_dir, verify_integrity=True) + + def test_restore_nonexistent_raises(self, manager: CheckpointManager, target_dir: Path) -> None: + """Restoring a version that was never checkpointed raises CheckpointError.""" + with pytest.raises(CheckpointError): + manager.restore_checkpoint("does_not_exist", target_dir) + + def test_multiple_checkpoints_restore_correct_one( + self, manager: CheckpointManager, tmp_path: Path + ) -> None: + """With multiple checkpoints, each restores its own data.""" + manager.create_checkpoint("a", _sample_version("a"), capture_system_state=False) + manager.create_checkpoint( + "b", + { + **_sample_version("b"), + "changes": {"file_b.txt": "content-b\n"}, + }, + capture_system_state=False, + ) + + target_a = tmp_path / "target_a" + target_b = tmp_path / "target_b" + manager.restore_checkpoint("a", target_a, verify_integrity=False) + manager.restore_checkpoint("b", target_b, verify_integrity=False) + + assert (target_a / "src" / "main.py").exists() + assert not (target_a / "file_b.txt").exists() + assert (target_b / "file_b.txt").read_text() == "content-b\n" + + def test_list_checkpoints_after_create(self, manager: CheckpointManager) -> None: + """list_checkpoints returns metadata for all created checkpoints.""" + for vid in ("x", "y"): + manager.create_checkpoint(vid, _sample_version(vid), capture_system_state=False) + + listed = manager.list_checkpoints() + ids = {cp["version_id"] for cp in listed} + assert ids == {"x", "y"} + + def test_delete_checkpoint(self, manager: CheckpointManager) -> None: + """Deleting a checkpoint removes it from disk and registry.""" + checkpoint_path = Path( + manager.create_checkpoint("del", _sample_version("del"), capture_system_state=False) + ) + assert manager.delete_checkpoint("del") is True + assert manager.get_checkpoint_path("del") is None + assert not checkpoint_path.exists() + + def test_restore_clears_target_except_protected( + self, manager: CheckpointManager, target_dir: Path + ) -> None: + """Restore clears the target directory but leaves .git/ alone.""" + target_dir.mkdir(parents=True) + (target_dir / "stale.txt").write_text("old\n") + git_dir = target_dir / ".git" + git_dir.mkdir() + (git_dir / "HEAD").write_text("ref: refs/heads/main\n") + + manager.create_checkpoint("c", _sample_version("c"), capture_system_state=False) + result = manager.restore_checkpoint("c", target_dir, verify_integrity=False) + + assert result["success"] is True + assert not (target_dir / "stale.txt").exists() + assert (target_dir / ".git" / "HEAD").exists() + + def test_checkpoint_size_reported(self, manager: CheckpointManager) -> None: + """get_checkpoint_size returns a plausible size for a known-content checkpoint.""" + version = _sample_version("sz") + manager.create_checkpoint("sz", version, capture_system_state=False) + size = manager.get_checkpoint_size("sz") + assert size is not None + + # Content files are deterministic: "print('hello')\n" (15 B) + + # "def helper():\n return 42\n" (28 B) = 43 B of content. + # Metadata.json adds a variable amount (~1-2 KB). Assert the size + # covers the known content and stays within a sane upper bound + # derived from actual content rather than an arbitrary constant. + expected_content_bytes = sum(len(v.encode()) for v in version["changes"].values()) + assert size >= expected_content_bytes + # Upper bound: content + allowance for metadata overhead (~24x observed). + # Uses a multiplier so the bound adapts if content grows. + assert size < expected_content_bytes * 30 + + def test_restore_with_system_state(self, manager: CheckpointManager, target_dir: Path) -> None: + """Restoring with capture_system_state=True returns system_state in result.""" + manager.create_checkpoint("ss", _sample_version("ss"), capture_system_state=True) + result = manager.restore_checkpoint("ss", target_dir, verify_integrity=True) + + assert result["system_state"] is not None + assert "system_info" in result["system_state"] + assert "capture_timestamp" in result["system_state"] + + def test_interrupt_and_restore_consistency( + self, manager: CheckpointManager, target_dir: Path + ) -> None: + """Simulate mid-evolution interruption: checkpoint, mutate, restore.""" + original = _sample_version("interrupt") + manager.create_checkpoint("interrupt", original, capture_system_state=False) + + # Simulate a partial evolution that corrupts the workspace + target_dir.mkdir(parents=True, exist_ok=True) + (target_dir / "src" / "main.py").parent.mkdir(parents=True, exist_ok=True) + (target_dir / "src" / "main.py").write_text("BROKEN PARTIAL EDIT\n") + (target_dir / "junk.txt").write_text("should be removed on restore\n") + + # Restore — should recover original state + result = manager.restore_checkpoint("interrupt", target_dir, verify_integrity=True) + assert result["success"] is True + assert (target_dir / "src" / "main.py").read_text() == "print('hello')\n" + assert not (target_dir / "junk.txt").exists()