diff --git a/evolution/core/external_importers.py b/evolution/core/external_importers.py index 65fe0aa..24ddf84 100644 --- a/evolution/core/external_importers.py +++ b/evolution/core/external_importers.py @@ -155,25 +155,57 @@ def _is_relevant_to_skill(text: str, skill_name: str, skill_text: str) -> bool: class ClaudeCodeImporter: - """Import user prompts from Claude Code history.jsonl. + """Import sessions from Claude Code. - Claude Code stores a flat JSONL of user messages at ~/.claude/history.jsonl. - Each line has: display (user text), timestamp, project, sessionId. - Only user inputs are available — no assistant responses. + Claude Code stores data in two locations: + + 1. ``~/.claude/projects//.jsonl`` — full session + transcripts. Each line is one event (``user``, ``assistant``, + ``attachment``, ``permission-mode``, etc.). When present these yield + paired ``(task_input, assistant_response)`` examples comparable to the + Copilot/Hermes importers. + + 2. ``~/.claude/history.jsonl`` — flat log of user prompts only. Used as a + fallback when ``projects/`` is empty or missing (older Claude Code + installations, or fresh machines). + + The default behaviour is ``source="auto"``: prefer rich project transcripts + when available, fall back to ``history.jsonl`` otherwise. Pass + ``source="history"`` to force the legacy user-only path, or + ``source="projects"`` to read transcripts only. """ HISTORY_PATH = Path.home() / ".claude" / "history.jsonl" + PROJECTS_DIR = Path.home() / ".claude" / "projects" @staticmethod - def extract_messages(limit: int = 0) -> list[dict]: - """Read user messages from Claude Code history. + def extract_messages(limit: int = 0, source: str = "auto") -> list[dict]: + """Read messages from Claude Code session storage. Args: limit: Maximum messages to return (0 = no limit). + source: "auto" (default), "projects", or "history". Returns: - List of dicts with keys: source, task_input, project, session_id, timestamp. + List of dicts. Always include ``source``, ``task_input``, + ``project``, ``session_id``, ``timestamp``. Project transcripts + additionally include ``assistant_response``. """ + if source not in ("auto", "projects", "history"): + raise ValueError( + f"source must be 'auto', 'projects', or 'history' (got {source!r})" + ) + + if source in ("auto", "projects"): + messages = ClaudeCodeImporter._extract_from_projects(limit) + if messages or source == "projects": + return messages + + return ClaudeCodeImporter._extract_from_history(limit) + + @staticmethod + def _extract_from_history(limit: int = 0) -> list[dict]: + """Read user prompts from the flat ``history.jsonl`` log.""" if not ClaudeCodeImporter.HISTORY_PATH.exists(): return [] @@ -206,6 +238,26 @@ def extract_messages(limit: int = 0) -> list[dict]: return messages + @staticmethod + def _extract_from_projects(limit: int = 0) -> list[dict]: + """Read paired user/assistant turns from project session transcripts.""" + if not ClaudeCodeImporter.PROJECTS_DIR.exists(): + return [] + + session_files = sorted(ClaudeCodeImporter.PROJECTS_DIR.rglob("*.jsonl")) + if not session_files: + return [] + + messages: list[dict] = [] + for session_path in session_files: + project = session_path.parent.name + messages.extend(_parse_claude_code_session(session_path, project)) + if limit and len(messages) >= limit: + messages = messages[:limit] + break + + return messages + class CopilotImporter: """Import conversations from GitHub Copilot session events. @@ -270,6 +322,80 @@ def _read_copilot_workspace(workspace_path: Path) -> str: return "" +def _parse_claude_code_session(session_path: Path, project: str) -> list[dict]: + """Parse one Claude Code session JSONL into (user, assistant) pairs. + + Claude Code project transcripts interleave many record types. We keep only + real user prompts (``type == "user"`` with string content — array content + means a tool result, which we skip) and concatenate the text blocks of all + assistant turns that follow until the next user prompt. + + Records lacking ``type``, malformed JSON, or events containing detected + secrets are skipped. A session that yields no clean pairs returns an empty + list rather than raising. + """ + pairs: list[dict] = [] + current_user: Optional[str] = None + current_assistant_parts: list[str] = [] + + session_id = session_path.stem + + def flush() -> None: + if current_user and current_assistant_parts: + assistant = "\n".join(current_assistant_parts).strip() + if assistant and not _contains_secret(current_user) and not _contains_secret(assistant): + pairs.append({ + "source": "claude-code", + "task_input": current_user, + "assistant_response": assistant, + "project": project, + "session_id": session_id, + "timestamp": 0, + }) + + try: + with open(session_path) as f: + for line in f: + if not line.strip(): + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + continue + + event_type = event.get("type") + message = event.get("message") or {} + + if event_type == "user": + content = message.get("content") + # array content == tool_result, skip it + if not isinstance(content, str): + continue + text = content.strip() + if len(text) < 10: + continue + # close out the previous turn before starting a new one + flush() + current_user = text + current_assistant_parts = [] + + elif event_type == "assistant" and current_user is not None: + content = message.get("content") + if not isinstance(content, list): + continue + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + text = block.get("text", "").strip() + if text: + current_assistant_parts.append(text) + + flush() + except OSError as e: + console.print(f"[dim]Skipped {session_path.name}: {e}[/dim]") + + return pairs + + def _parse_copilot_events( events_path: Path, session_id: str, project: str, ) -> list[dict]: diff --git a/tests/core/test_external_importers.py b/tests/core/test_external_importers.py index adfacc0..7fc5fa9 100644 --- a/tests/core/test_external_importers.py +++ b/tests/core/test_external_importers.py @@ -264,7 +264,7 @@ def test_parses_history_jsonl(self, tmp_path): ) with patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): - messages = ClaudeCodeImporter.extract_messages() + messages = ClaudeCodeImporter.extract_messages(source="history") # 1 valid message (second too short, third has secret) assert len(messages) == 1 @@ -274,7 +274,7 @@ def test_parses_history_jsonl(self, tmp_path): def test_handles_missing_file(self, tmp_path): with patch.object(ClaudeCodeImporter, "HISTORY_PATH", tmp_path / "nonexistent.jsonl"): - messages = ClaudeCodeImporter.extract_messages() + messages = ClaudeCodeImporter.extract_messages(source="history") assert messages == [] def test_respects_limit(self, tmp_path): @@ -286,7 +286,7 @@ def test_respects_limit(self, tmp_path): history.write_text("\n".join(lines) + "\n") with patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): - messages = ClaudeCodeImporter.extract_messages(limit=5) + messages = ClaudeCodeImporter.extract_messages(limit=5, source="history") assert len(messages) == 5 @@ -299,7 +299,7 @@ def test_skips_malformed_json_lines(self, tmp_path): ) with patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): - messages = ClaudeCodeImporter.extract_messages() + messages = ClaudeCodeImporter.extract_messages(source="history") assert len(messages) == 1 @@ -312,10 +312,249 @@ def test_skips_empty_lines(self, tmp_path): ) with patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): - messages = ClaudeCodeImporter.extract_messages() + messages = ClaudeCodeImporter.extract_messages(source="history") assert len(messages) == 1 + def test_rejects_invalid_source(self): + with pytest.raises(ValueError): + ClaudeCodeImporter.extract_messages(source="bogus") + + +def _write_session(path: Path, events: list[dict]) -> None: + """Write a list of session events as JSONL.""" + path.write_text("\n".join(json.dumps(e) for e in events) + "\n") + + +def _user(text: str) -> dict: + return {"type": "user", "message": {"role": "user", "content": text}} + + +def _assistant_text(text: str) -> dict: + return { + "type": "assistant", + "message": {"role": "assistant", "content": [{"type": "text", "text": text}]}, + } + + +def _assistant_tool_use(name: str = "Bash") -> dict: + return { + "type": "assistant", + "message": { + "role": "assistant", + "content": [{"type": "tool_use", "name": name, "input": {}}], + }, + } + + +def _user_tool_result(text: str = "ok") -> dict: + return { + "type": "user", + "message": {"role": "user", "content": [{"type": "tool_result", "content": text}]}, + } + + +class TestClaudeCodeProjectsImporter: + """Parse paired user/assistant turns from ~/.claude/projects//.jsonl.""" + + def test_parses_paired_turns(self, tmp_path): + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + {"type": "permission-mode", "permissionMode": "default"}, + _user("explain how this codebase routes requests"), + _assistant_text("It uses Express middleware for routing."), + _user("now refactor it to use Fastify"), + _assistant_text("Here's the refactor plan."), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 2 + assert messages[0]["task_input"] == "explain how this codebase routes requests" + assert messages[0]["assistant_response"] == "It uses Express middleware for routing." + assert messages[0]["source"] == "claude-code" + assert messages[0]["project"] == "-Users-test" + assert messages[0]["session_id"] == "abc" + assert messages[1]["task_input"] == "now refactor it to use Fastify" + + def test_concatenates_multiple_assistant_text_blocks(self, tmp_path): + """Assistant turns may span tool calls; text blocks across them concat.""" + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + _user("run the tests and tell me what failed"), + _assistant_text("Running them now."), + _assistant_tool_use("Bash"), + _user_tool_result("FAILED: 2 tests"), + _assistant_text("Two integration tests failed in auth_test.py."), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 1 + assert "Running them now." in messages[0]["assistant_response"] + assert "Two integration tests failed" in messages[0]["assistant_response"] + + def test_skips_user_with_array_content(self, tmp_path): + """user records with array content are tool results, not real prompts.""" + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + _user_tool_result("dangling result without preceding prompt"), + _user("the actual question with enough length"), + _assistant_text("the actual answer"), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 1 + assert messages[0]["task_input"] == "the actual question with enough length" + + def test_drops_pair_with_secret_in_user_or_assistant(self, tmp_path): + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "leak.jsonl", [ + _user("here is sk-ant-api03-SECRETKEY123456789012345678 use it"), + _assistant_text("noted thanks"), + _user("now a clean question with enough length"), + _assistant_text("ANTHROPIC_API_KEY=sk-foo leaked in the response"), + _user("fully safe question with enough length here"), + _assistant_text("safe response"), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + # Only the third pair survives — secrets in either side reject the pair. + assert len(messages) == 1 + assert messages[0]["task_input"].startswith("fully safe question") + + def test_drops_short_user_prompts(self, tmp_path): + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + _user("hi"), + _assistant_text("hello"), + _user("now a substantive question with enough length"), + _assistant_text("a substantive answer"), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 1 + assert messages[0]["task_input"].startswith("now a substantive question") + + def test_drops_pair_without_assistant_text(self, tmp_path): + """A user prompt followed only by tool_use / no text yields no pair.""" + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + _user("a real user prompt with enough length"), + _assistant_tool_use("Bash"), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert messages == [] + + def test_skips_malformed_jsonl_lines(self, tmp_path): + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + (projects / "abc.jsonl").write_text( + "not json\n" + + json.dumps(_user("a real prompt with enough length to count")) + "\n" + + "{broken\n" + + json.dumps(_assistant_text("the response")) + "\n" + ) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 1 + + def test_walks_multiple_sessions_and_projects(self, tmp_path): + proj1 = tmp_path / "projects" / "-Users-a" + proj2 = tmp_path / "projects" / "-Users-b" + proj1.mkdir(parents=True) + proj2.mkdir(parents=True) + _write_session(proj1 / "s1.jsonl", [ + _user("question one with enough length"), + _assistant_text("answer one"), + ]) + _write_session(proj2 / "s2.jsonl", [ + _user("question two with enough length"), + _assistant_text("answer two"), + ]) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + + assert len(messages) == 2 + projects_seen = {m["project"] for m in messages} + assert projects_seen == {"-Users-a", "-Users-b"} + + def test_respects_limit(self, tmp_path): + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + events = [] + for i in range(10): + events.append(_user(f"question number {i} with enough length")) + events.append(_assistant_text(f"answer number {i}")) + _write_session(projects / "abc.jsonl", events) + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"): + messages = ClaudeCodeImporter.extract_messages(limit=3, source="projects") + + assert len(messages) == 3 + + def test_handles_missing_projects_dir(self, tmp_path): + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "nope"): + messages = ClaudeCodeImporter.extract_messages(source="projects") + assert messages == [] + + def test_auto_prefers_projects_over_history(self, tmp_path): + """When projects/ has data, history.jsonl is ignored.""" + projects = tmp_path / "projects" / "-Users-test" + projects.mkdir(parents=True) + _write_session(projects / "abc.jsonl", [ + _user("real transcript prompt with enough length"), + _assistant_text("real transcript response"), + ]) + history = tmp_path / "history.jsonl" + history.write_text(json.dumps( + {"display": "history-only message", "timestamp": 1, "project": "/x", "sessionId": "h"} + ) + "\n") + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", tmp_path / "projects"), \ + patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): + messages = ClaudeCodeImporter.extract_messages() # auto + + assert len(messages) == 1 + assert "real transcript prompt" in messages[0]["task_input"] + assert messages[0]["assistant_response"] == "real transcript response" + + def test_auto_falls_back_to_history(self, tmp_path): + """When projects/ is missing or empty, auto reads history.jsonl.""" + empty_projects = tmp_path / "no-projects" + history = tmp_path / "history.jsonl" + history.write_text(json.dumps( + {"display": "fallback message with enough length", "timestamp": 1, "project": "/x", "sessionId": "h"} + ) + "\n") + + with patch.object(ClaudeCodeImporter, "PROJECTS_DIR", empty_projects), \ + patch.object(ClaudeCodeImporter, "HISTORY_PATH", history): + messages = ClaudeCodeImporter.extract_messages() # auto + + assert len(messages) == 1 + assert messages[0]["task_input"] == "fallback message with enough length" + assert "assistant_response" not in messages[0] + # ── Copilot Importer ────────────────────────────────────────────────────────