diff --git a/TODO.md b/TODO.md index e2849cca..552ae83e 100644 --- a/TODO.md +++ b/TODO.md @@ -147,7 +147,7 @@ ### CLI Issues Found in Whole-Repo Code Review (2026-07-22) -- [ ] **`evoseal export` fabricates results instead of reporting real failures** _(exact file:line needs re-verification)_ +- [x] **`evoseal export` fabricates results instead of reporting real failures** _(done 2026-08-03)_ — `export_results`, `export_variant`, and `export_all` now query `ExperimentDatabase` instead of returning hardcoded sample data; graceful error when no database or experiment found - [ ] **Several `evoseal pipeline` subcommands are stubs**, not implemented behavior _(exact file:line needs re-verification)_ ### SEAL Subsystem Issues Found in Whole-Repo Code Review (2026-07-22) @@ -306,10 +306,10 @@ | Priority | Total | Done | Notes | |----------|-------|------|-------| | 🔴 P0 | 11 | 11 | Original 5 complete; all 6 critical bugs from 2026-07-22 whole-repo review fixed (PRs #74, #76-#79) | -| 🟠 P1 | 24 | 18 | Original safety/integration items done; +12 high-priority bugs from 2026-07-22 review (3 CI/CD pipeline fixes: workflow_run name mismatch, requirements/ path, security gate bypass); signal-handler init fix; safety.yaml created; monitoring dashboard auth+CORS fix | +| 🟠 P1 | 24 | 19 | Original safety/integration items done; +12 high-priority bugs from 2026-07-22 review (3 CI/CD pipeline fixes: workflow_run name mismatch, requirements/ path, security gate bypass); signal-handler init fix; safety.yaml created; monitoring dashboard auth+CORS fix; `evoseal export` now uses real data | | 🟡 P2 | 30 | 25 | Co-evolution loop gaps (8 items, 8 done) + existing P2 + 13 medium bugs from 2026-07-22 review + 4 latent collect->train bugs found closing the loop (1 fixed, 1 new HF-format gap resolved); provider_manager health-check await fix; workflow-agent private-API/event-loop fix; checkpoint save/restore test; trust_remote_code security fix; safety-decision orchestration tests; structured improvement units | | 🟢 P3 | 24 | 19 | Makefile, pre-commit, Docker, ADRs, ADR refresh, CHANGELOG complete; +11 hygiene items from 2026-07-22 review; Ollama provider retry/backoff fix; local_models TTL cache; workspace prompt file conventions; how-it-works tutorial; model_fine_tuner key validation; model_fine_tuner GPU availability check | -| **Total** | **89** | **73** | | +| **Total** | **89** | **74** | | > Update this table as you complete items. Recommended flow: P0 → P1 → P2 → P3. > diff --git a/evoseal/cli/commands/export.py b/evoseal/cli/commands/export.py index ae0fba6e..b39e4141 100644 --- a/evoseal/cli/commands/export.py +++ b/evoseal/cli/commands/export.py @@ -7,15 +7,26 @@ from __future__ import annotations import json +import os from datetime import datetime from pathlib import Path from typing import Annotated, Any import typer +from evoseal.core.experiment_database import ExperimentDatabase + # Initialize the Typer app app = typer.Typer(name="export", help="Export results/variants") +# Default database path: override with EVOSEAL_DB_PATH env var or --db-path flag. +# Falls back to .evoseal/experiments.db relative to CWD. +DEFAULT_DB_PATH = Path(os.environ.get("EVOSEAL_DB_PATH", ".evoseal/experiments.db")) + +# Artifact types considered "dependency" artifacts (e.g. requirements.txt). +# When include_dependencies=False, these are excluded from variant exports. +DEPENDENCY_ARTIFACT_TYPES = {"dependency", "requirements"} + # Supported export formats and their file extensions FORMAT_SUPPORT: dict[str, list[str]] = { "results": ["json", "csv"], @@ -54,6 +65,14 @@ def export_results( help=f"Output format: {', '.join(FORMAT_SUPPORT['results'])}.", ), ] = "json", + db_path: Annotated[ + Path, + typer.Option( + "--db-path", + help="Path to the experiment database. Overrides EVOSEAL_DB_PATH env var.", + dir_okay=False, + ), + ] = DEFAULT_DB_PATH, include_metrics: Annotated[ bool, typer.Option( @@ -76,57 +95,106 @@ def export_results( ) raise typer.Exit(1) - # TODO: Implement actual results export - results: dict[str, Any] = { - "run_id": run_id, - "timestamp": datetime.utcnow().isoformat(), - "status": "completed", - "metrics": ( - { - "fitness": 0.85, - "generations": 100, - "best_score": 0.92, + if not db_path.exists(): + typer.echo(f"Error: No experiment database found at {db_path}") + typer.echo("Run an evolution cycle first to generate data.") + raise typer.Exit(1) + + try: + db = ExperimentDatabase(db_path) + except Exception as e: + typer.echo(f"Error reading experiment database: {e}") + raise typer.Exit(1) from None + + try: + try: + experiment = db.get_experiment(run_id) + except Exception as e: + typer.echo(f"Error querying experiment database: {e}") + raise typer.Exit(1) from None + + if experiment is None: + typer.echo(f"Error: No experiment found with ID '{run_id}'") + typer.echo("Use 'evoseal status' to see available experiments.") + raise typer.Exit(1) + + results: dict[str, Any] = { + "run_id": experiment.id, + "name": experiment.name, + "description": experiment.description, + "status": experiment.status.value, + "created_at": experiment.created_at.isoformat(), + "started_at": experiment.started_at.isoformat() if experiment.started_at else None, + "completed_at": experiment.completed_at.isoformat() + if experiment.completed_at + else None, + } + + if experiment.result: + results["result"] = { + "best_fitness": experiment.result.best_fitness, + "generations_completed": experiment.result.generations_completed, + "total_evaluations": experiment.result.total_evaluations, + "convergence_iteration": experiment.result.convergence_iteration, + "execution_time": experiment.result.execution_time, + "error_message": experiment.result.error_message, } - if include_metrics - else {} - ), - "code": ("# Sample code\ndef main():\n print('Hello, World!')" if include_code else ""), - } - - output: str = "" - if format == "json": - output = json.dumps(results, indent=2) - elif format == "csv": - # Simple CSV output for metrics - import csv - import io - - output_io = io.StringIO() - writer = csv.writer(output_io) - writer.writerow(["Metric", "Value"]) - if include_metrics: - for k, v in results.get("metrics", {}).items(): - writer.writerow([k, v]) - output = output_io.getvalue() - else: # txt - output = f"Run ID: {results['run_id']}\n" - output += f"Status: {results['status']}\n" - output += f"Timestamp: {results['timestamp']}\n" - if include_metrics: - output += "\nMetrics:\n" - for k, v in results.get("metrics", {}).items(): - output += f" {k}: {v}\n" - if include_code and results.get("code"): - output += "\nCode:\n" - output += results["code"] - - if output_file: - output_file.parent.mkdir(parents=True, exist_ok=True) - with open(output_file, "w", encoding="utf-8") as f: - f.write(output) - typer.echo(f"Results exported to {output_file}") - else: - typer.echo(output) + + if include_metrics and experiment.metrics: + # NOTE: if duplicate metric names exist, last value wins. + results["metrics"] = {m.name: m.value for m in experiment.metrics} + + if include_code and experiment.artifacts: + results["artifacts"] = [ + { + "name": a.name, + "type": a.artifact_type, + "content": a.content, + "file_path": a.file_path, + } + for a in experiment.artifacts + if a.content + ] + + output: str = "" + if format == "json": + output = json.dumps(results, indent=2) + elif format == "csv": + # Simple CSV output for metrics + import csv + import io + + output_io = io.StringIO() + writer = csv.writer(output_io) + writer.writerow(["Metric", "Value"]) + if include_metrics: + for k, v in results.get("metrics", {}).items(): + writer.writerow([k, v]) + output = output_io.getvalue() + else: # txt + output = f"Run ID: {results['run_id']}\n" + output += f"Status: {results['status']}\n" + output += f"Created: {results['created_at']}\n" + if include_metrics: + output += "\nMetrics:\n" + for k, v in results.get("metrics", {}).items(): + output += f" {k}: {v}\n" + if include_code and results.get("artifacts"): + output += "\nArtifacts:\n" + for a in results["artifacts"]: + output += f" [{a['type']}] {a['name']}\n" + if a.get("content"): + output += f"{a['content']}\n" + + if output_file: + output_file.parent.mkdir(parents=True, exist_ok=True) + with open(output_file, "w", encoding="utf-8") as f: + f.write(output) + typer.echo(f"Results exported to {output_file}") + else: + typer.echo(output) + finally: + db.close() @app.command("variant") @@ -153,21 +221,101 @@ def export_variant( help="Include dependency information in the export.", ), ] = True, + db_path: Annotated[ + Path, + typer.Option( + "--db-path", + help="Path to the experiment database. Overrides EVOSEAL_DB_PATH env var.", + dir_okay=False, + ), + ] = DEFAULT_DB_PATH, ) -> None: """Export a specific code variant.""" - # TODO: Implement actual variant export - output_dir = output_dir / f"variant_{variant_id}" - output_dir.mkdir(parents=True, exist_ok=True) - - # Create sample files - (output_dir / "main.py").write_text( - "# Sample variant code\ndef main():\n print('Hello from variant!')" - ) + if not db_path.exists(): + typer.echo(f"Error: No experiment database found at {db_path}") + typer.echo("Run an evolution cycle first to generate data.") + raise typer.Exit(1) - if include_dependencies: - (output_dir / "requirements.txt").write_text("numpy>=1.20.0\npandas>=1.3.0") + try: + db = ExperimentDatabase(db_path) + except Exception as e: + typer.echo(f"Error reading experiment database: {e}") + raise typer.Exit(1) from None + + try: + try: + # NOTE: "variants" in the CLI map to experiments in the database. + # There is no separate Variant model; get_experiment is the correct lookup. + experiment = db.get_experiment(variant_id) + except Exception as e: + typer.echo(f"Error querying experiment database: {e}") + raise typer.Exit(1) from None + + if experiment is None: + typer.echo(f"Error: No experiment/variant found with ID '{variant_id}'") + raise typer.Exit(1) + + # Sanitize variant_id to prevent path traversal (e.g. "../../tmp/evil") + safe_variant_id = Path(variant_id).name + if not safe_variant_id or safe_variant_id in {".", ".."}: + typer.echo(f"Error: Invalid variant ID '{variant_id}'") + raise typer.Exit(1) + output_dir = output_dir / f"variant_{safe_variant_id}" + # Clean the directory before writing so re-exports with fewer/renamed + # artifacts don't leave stale files from a prior export. + import shutil + + if output_dir.exists(): + shutil.rmtree(output_dir) + output_dir.mkdir(parents=True, exist_ok=True) + + exported_files = 0 + written_names: set[str] = set() + # Reserve metadata.json so an artifact with that sanitized name + # doesn't get silently clobbered by the experiment-metadata write. + written_names.add("metadata.json") + if experiment.artifacts: + for artifact in experiment.artifacts: + # Skip dependency artifacts when the flag is off + if not include_dependencies and artifact.artifact_type in DEPENDENCY_ARTIFACT_TYPES: + continue + if artifact.content: + # Sanitize: use only the filename component to prevent path traversal + safe_name = Path(artifact.file_path or artifact.name).name + # Deduplicate: avoid silent overwrites when multiple artifacts + # collapse to the same basename + if safe_name in written_names: + stem = Path(safe_name).stem + suffix = Path(safe_name).suffix + counter = 2 + while f"{stem}_{counter}{suffix}" in written_names: + counter += 1 + safe_name = f"{stem}_{counter}{suffix}" + written_names.add(safe_name) + file_path = output_dir / safe_name + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(artifact.content, encoding="utf-8") + exported_files += 1 + + # Write experiment metadata + metadata = { + "id": experiment.id, + "name": experiment.name, + "status": experiment.status.value, + "best_fitness": experiment.result.best_fitness if experiment.result else None, + "generations_completed": experiment.result.generations_completed + if experiment.result + else 0, + } + (output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2), encoding="utf-8") + + # include_dependencies filtering is handled in the artifact loop above. - typer.echo(f"Variant {variant_id} exported to {output_dir}") + typer.echo( + f"Variant {variant_id} exported to {output_dir} ({exported_files} artifact files)" + ) + finally: + db.close() @app.command("all") @@ -205,6 +353,14 @@ def export_all( help=f"Output format: {', '.join(FORMAT_SUPPORT['all'])}.", ), ] = "json", + db_path: Annotated[ + Path, + typer.Option( + "--db-path", + help="Path to the experiment database. Overrides EVOSEAL_DB_PATH env var.", + dir_okay=False, + ), + ] = DEFAULT_DB_PATH, ) -> None: """Export all data from the EVOSEAL system. @@ -220,40 +376,74 @@ def export_all( ) raise typer.Exit(1) - # TODO: Implement actual export all - output_dir.mkdir(parents=True, exist_ok=True) - - # Create sample export structure - (output_dir / "config.yaml").write_text("# Configuration\nproject: evoseal\nversion: 0.1.0") - - results_dir = output_dir / "results" - results_dir.mkdir(exist_ok=True) - - # Sample results - results = [ - {"run_id": "run1", "fitness": 0.85, "generation": 100}, - {"run_id": "run2", "fitness": 0.92, "generation": 150}, - ] - - if format == "json": - with open(results_dir / "results.json", "w") as f: - json.dump({"results": results}, f, indent=2) - elif format == "yaml": - import yaml - - with open(results_dir / "results.yaml", "w") as f: - yaml.dump({"results": results}, f, default_flow_style=False) - elif format == "csv": - import csv - - with open(results_dir / "results.csv", "w", newline="") as f: - if results and isinstance(results, list) and len(results) > 0: - fieldnames = list(results[0].keys()) - writer = csv.DictWriter(f, fieldnames=fieldnames) - writer.writeheader() - writer.writerows(results) + if not db_path.exists(): + typer.echo(f"Error: No experiment database found at {db_path}") + typer.echo("Run an evolution cycle first to generate data.") + raise typer.Exit(1) - typer.echo(f"All data exported to {output_dir}") + try: + db = ExperimentDatabase(db_path) + except Exception as e: + typer.echo(f"Error reading experiment database: {e}") + raise typer.Exit(1) from None + + try: + try: + experiments = db.list_experiments() + except Exception as e: + typer.echo(f"Error querying experiment database: {e}") + raise typer.Exit(1) from None + + if not experiments: + typer.echo("No experiments found in the database.") + raise typer.Exit(1) + + output_dir.mkdir(parents=True, exist_ok=True) + results_dir = output_dir / "results" + results_dir.mkdir(exist_ok=True) + + # Build export records from real experiments + results = [] + for exp in experiments: + record: dict[str, Any] = { + "run_id": exp.id, + "name": exp.name, + "status": exp.status.value, + "created_at": exp.created_at.isoformat(), + } + if include_metrics and exp.result: + record["fitness"] = exp.result.best_fitness + record["generation"] = exp.result.generations_completed + record["execution_time"] = exp.result.execution_time + if include_code and exp.artifacts: + record["artifacts"] = ";".join(a.name for a in exp.artifacts) + results.append(record) + + if format == "json": + with open(results_dir / "results.json", "w") as f: + json.dump({"results": results, "count": len(results)}, f, indent=2) + elif format == "yaml": + import yaml + + with open(results_dir / "results.yaml", "w") as f: + yaml.dump({"results": results, "count": len(results)}, f, default_flow_style=False) + elif format == "csv": + import csv + + with open(results_dir / "results.csv", "w", newline="") as f: + if results: + # Compute fieldnames as union across all records to handle optional keys + all_keys: set[str] = set() + for r in results: + all_keys.update(r.keys()) + fieldnames = sorted(all_keys) + writer = csv.DictWriter(f, fieldnames=fieldnames, extrasaction="ignore") + writer.writeheader() + writer.writerows(results) + + typer.echo(f"Exported {len(results)} experiments to {output_dir}") + finally: + db.close() if __name__ == "__main__":