diff --git a/fastapi_app/config/settings.py b/fastapi_app/config/settings.py index 1bdeead..9be9bd9 100644 --- a/fastapi_app/config/settings.py +++ b/fastapi_app/config/settings.py @@ -75,6 +75,48 @@ class AppSettings(BaseSettings): LOCAL_EMBEDDING_CUDA_VISIBLE_DEVICES: Optional[str] = None LOCAL_EMBEDDING_GPU_MEMORY_UTILIZATION: float = 0.3 + # ── GraphRAG ────────────────────────────────────────────────────────────── + # 查询侧默认用较快聊天模型；索引抽取亦走同一 default_chat_model（见 settings.yaml） + GRAPHRAG_LLM_MODEL: str = "deepseek-v3.2" + GRAPHRAG_EMBEDDING_MODEL: str = "text-embedding-3-small" + # True 且 USE_LOCAL_EMBEDDING=1 时，查询/索引写入的 embedding 指向本地 vLLM（须与向量维度一致；从 OpenAI 维度过来的库需重建索引） + GRAPHRAG_USE_LOCAL_EMBEDDING_RUNTIME: bool = True + GRAPHRAG_OUTPUT_DIR: str = "outputs/graphrag_kb" # workspace root, layout: {dir}/{email}/{nb_id}/ + GRAPHRAG_CMD: str = "" # graphrag CLI path; auto-detected from PATH if empty + GRAPHRAG_CHUNK_SIZE: int = 384 # chars per chunk; also written to settings.yaml chunks.size + GRAPHRAG_CHUNK_OVERLAP: int = 48 + # 写入 prompt，偏短输出可缩短 local_search 生成时间 + GRAPHRAG_RESPONSE_TYPE: str = "At most 4 bullet points; be concise." + GRAPHRAG_COMMUNITY_LEVEL: int = 1 # 低于 2 通常更快，社区上下文更少 + GRAPHRAG_LOCAL_SEARCH_CONTEXT_MAX_TOKENS: int = 12000 # 低于 24000 可加快检索上下文组装与生成 + GRAPHRAG_SUBGRAPH_PRUNE_ENABLED: bool = True # run LLM subgraph pruning after each query + GRAPHRAG_SUBGRAPH_PRUNE_MODEL: str = "deepseek-v3.2" # 仅单独裁剪路径使用 + GRAPHRAG_SUBGRAPH_PRUNE_MAX_EDGES_INPUT: int = 28 # 裁剪+Judge 合并路径：输入边数上限（越小越快） + GRAPHRAG_SUBGRAPH_PRUNE_MAX_TOKENS: int = 512 # 单独裁剪 LLM 输出上限（若仍启用旧路径） + # 裁剪与 Judge 合并为一次 LLM（graphrag_chat / graphrag_kb query） + GRAPHRAG_PRUNE_JUDGE_MODEL: Optional[str] = None # 为空则用 JUDGE_MODEL + GRAPHRAG_PRUNE_JUDGE_MAX_TOKENS: int = 768 # 合并调用输出上限（含 analysis + judge JSON） + GRAPHRAG_MAX_HIGHLIGHT_HINTS: int = 10 # max highlight_hints returned (0 = unlimited) + # 子图实体名 → Wikidata 搜索 → 在 GraphRAG query/chat 答案末尾附加简短参考（需出网） + GRAPHRAG_WIKIDATA_ENRICH_ENABLED: bool = True + GRAPHRAG_WIKIDATA_LANG: str = "zh" # wbsearchentities + 标签/描述优先语言 + GRAPHRAG_WIKIDATA_MAX_ENTITIES: int = 8 # 最多解析的不重复实体数 + # HTTPS 读超时（秒）；弱网可再加大或通过 HTTP 代理访问 wikidata.org + GRAPHRAG_WIKIDATA_TIMEOUT_SEC: float = 45.0 + GRAPHRAG_WIKIDATA_CONNECT_TIMEOUT_SEC: float = 10.0 + # 对 Read timeout / 连接错误额外重试次数（每次递增短暂退避） + GRAPHRAG_WIKIDATA_HTTP_RETRIES: int = 2 + GRAPHRAG_WIKIDATA_API_URL: str = "https://www.wikidata.org/w/api.php" + + # ── KGGen (optional triple extraction, disabled by default) ─────────────── + KGGEN_MODEL: str = "deepseek-v3.2" + KGGEN_PER_CHUNK: bool = True # True = per-chunk calls; False = full-text single call + KGGEN_LOG_CHUNK_INTERVAL: int = 10 # log every N chunks (0 = first/last only) + + # ── Judge (answer confidence scoring) ───────────────────────────────────── + JUDGE_MODEL: str = "deepseek-v3.2" # 单独 Judge；合并路径默认同此模型 + JUDGE_MAX_TOKENS: int = 256 # 弱化：更短 judge 输出 + class Config: env_file = ".env" env_file_encoding = "utf-8" diff --git a/fastapi_app/main.py b/fastapi_app/main.py index fdbd9b1..107b672 100644 --- a/fastapi_app/main.py +++ b/fastapi_app/main.py @@ -41,6 +41,7 @@ from fastapi.responses import FileResponse from fastapi_app.routers import auth, data_extract, files, kb, kb_embedding, paper2drawio, paper2ppt +from fastapi_app.routers import graphrag_kb from fastapi_app.middleware.api_key import APIKeyMiddleware from fastapi_app.middleware.logging import LoggingMiddleware from workflow_engine.utils import get_project_root @@ -428,6 +429,17 @@ async def _lifespan(app: FastAPI): os.environ["LOCAL_MINERU_API_URL"] = mineru_base_url os.environ["LOCAL_MINERU_MODEL"] = resolved_mineru_model + def _warmup_graphrag_imports() -> None: + try: + import graphrag.config.load_config # noqa: F401 + from graphrag import api as _graphrag_api # noqa: F401 + from graphrag.cli.query import _resolve_output_files # noqa: F401 + log.info("GraphRAG 相关 Python 包已预导入，可降低首次查询的 import 冷启动") + except ImportError as exc: + log.debug("GraphRAG 预导入跳过: %s", exc) + + _warmup_graphrag_imports() + yield for proc in managed_procs: if proc.poll() is None: @@ -476,6 +488,8 @@ def create_app() -> FastAPI: app.include_router(paper2drawio.router, prefix="/api/v1", tags=["Paper2Drawio"]) app.include_router(paper2ppt.router, prefix="/api/v1", tags=["Paper2PPT"]) app.include_router(auth.router, prefix="/api/v1", tags=["Auth"]) + # GraphRAG 知识库：/api/v1/graphrag-kb/{index,query,merge,chunk-snippet} → wa_graphrag_kb → wf_graphrag_kb + app.include_router(graphrag_kb.router, prefix="/api/v1", tags=["GraphRAG KB"]) # 静态文件：/outputs 下的文件（兼容 URL 中 %40 与 磁盘 @ 两种路径） project_root = get_project_root() diff --git a/fastapi_app/routers/graphrag_kb.py b/fastapi_app/routers/graphrag_kb.py new file mode 100644 index 0000000..39e93db --- /dev/null +++ b/fastapi_app/routers/graphrag_kb.py @@ -0,0 +1,729 @@ +"""GraphRAG 知识库 HTTP 路由（前缀在 ``main`` 中与 ``/api/v1`` 拼接）。 + +【端点与数据流】 + POST ``/graphrag-kb/index`` → ``wa_graphrag_kb.run_index`` → 建索引 → ``IndexResponse`` + POST ``/graphrag-kb/query`` → ``run_query`` → 检索 + Judge（+ 子图 CoT）→ ``QueryResponse`` + POST ``/graphrag-kb/merge`` → ``run_merge`` → 合并两 workspace → ``MergeResponse`` + POST ``/graphrag-kb/chunk-snippet`` → 按 ``chunk_id`` 从 ``input/*.txt`` 取块；可选 ``passage_for_llm`` 与 UI 文段对齐后做抽句高亮 + POST ``/graphrag-kb/context-refine`` → 首条 text_unit 原文 + ``reasoning_subgraph`` → LLM 清洗噪声并返回 ``cleaned_text`` + ``supporting_snippets`` + +【安全】 + ``_safe_workspace_dir`` 将路径解析到项目根目录下，防止目录穿越。 + +【说明】 + 请求体携带与其它路由一致的 LLM 凭证；前端不直连 ``workflow_engine``，仅调本路由。 +""" +from __future__ import annotations + +import json +import re +import asyncio +from pathlib import Path +from typing import Any, Dict, List, Optional + +from fastapi import APIRouter, HTTPException +from pydantic import BaseModel, Field + +from fastapi_app.config import settings +from fastapi_app.workflow_adapters.wa_graphrag_kb import run_index, run_query, run_merge, run_chat +from workflow_engine.logger import get_logger +from workflow_engine.utils import get_project_root + +log = get_logger(__name__) + + +# 匹配 GraphRAG input 文件中每个 chunk 段的起始行（后跟该段正文直至下一 chunk 或 EOF） +_CHUNK_HEAD = re.compile(r"\[chunk:([a-f0-9]+)\]\s*\n", re.IGNORECASE) + + +def _extract_chunk_block_from_input_text(text: str, chunk_id: str) -> str: + """在整份 ``input/.txt`` 文本中，定位 ``[chunk:目标id]`` 之后到下一 ``[chunk:`` 之前的正文。""" + want = chunk_id.strip().lower() + matches = list(_CHUNK_HEAD.finditer(text)) + for i, m in enumerate(matches): + if m.group(1).lower() != want: + continue + start = m.end() + end = matches[i + 1].start() if i + 1 < len(matches) else len(text) + return text[start:end].strip() + return "" + + +def _reanchor_graphrag_workspace_to_root(resolved_abs: Path, root: Path) -> Optional[Path]: + """If *resolved_abs* points at another clone (browser localStorage) but shares + ``outputs/graphrag_kb/`` with this repo, map to ``root / outputs/graphrag_kb/``. + + Returns a path under *root* only if that directory exists; otherwise ``None``. + """ + parts = resolved_abs.parts + for i in range(len(parts) - 1): + if parts[i] == "outputs" and parts[i + 1] == "graphrag_kb": + tail = Path(*parts[i:]) + candidate = (root / tail).resolve() + if not candidate.is_dir(): + return None + try: + candidate.relative_to(root) + return candidate + except ValueError: + return None + return None + + +def _safe_workspace_dir(raw: str) -> Path: + """将 *raw* 解析为项目根目录下的绝对路径；越界则抛 ``HTTPException(400)``。""" + root = get_project_root().resolve() + p = Path(raw.strip()) + if not p.is_absolute(): + p = (root / p).resolve() + else: + p = p.resolve() + try: + p.relative_to(root) + return p + except ValueError as exc: + alt = _reanchor_graphrag_workspace_to_root(p, root) + if alt is not None: + return alt + raise HTTPException(status_code=400, detail="workspace_dir must be under project root") from exc + +router = APIRouter(prefix="/graphrag-kb", tags=["GraphRAG KB"]) + +# --------------------------------------------------------------------------- +# Pydantic request/response models +# --------------------------------------------------------------------------- + +class _LLMBase(BaseModel): + api_url: str = Field(default_factory=lambda: settings.DEFAULT_LLM_API_URL) + api_key: str = "" + model: str = Field(default_factory=lambda: settings.GRAPHRAG_LLM_MODEL) + + +class IndexRequest(_LLMBase): + notebook_id: str + notebook_title: str = "" + email: str = "" + source_stems: Optional[List[str]] = None + workspace_dir: str = "" + force_reindex: bool = False + # Run MinerU on un-parsed PDFs before chunk extraction. + # Set to False if MinerU was already triggered via /kb/upload. + parse_pdfs: bool = True + # Default True: do not run KGGen (user-facing path is GraphRAG-only). + skip_kggen: bool = True + + +class IndexResponse(BaseModel): + workspace_dir: str + num_chunks: int + kg_entities: int + kg_relations: int + + +class QueryRequest(_LLMBase): + notebook_id: str + notebook_title: str = "" + email: str = "" + question: str + search_method: str = Field(default="local", pattern="^(local|global)$") + workspace_dir: str = "" + # None: use server GRAPHRAG_WIKIDATA_ENRICH_ENABLED; False: skip Wikidata appendix + wikidata_enrich: Optional[bool] = None + + +class QueryResponse(BaseModel): + answer: str + context_data: Dict[str, Any] = Field(default_factory=dict) + reasoning_subgraph: List[Dict[str, Any]] = Field(default_factory=list) + source_chunks: List[str] = Field(default_factory=list) + highlight_hints: List[Dict[str, Any]] = Field(default_factory=list) + judge_score: float = 0.0 + judge_rationale: str = "" + reasoning_subgraph_cot: str = "" + + +class MergeRequest(_LLMBase): + notebook_id: str = "" + notebook_title: str = "" + email: str = "" + workspace_dir_a: str + workspace_dir_b: str + dedupe: bool = False + + +class MergeResponse(BaseModel): + merged_workspace_dir: str + num_chunks: int + + +class ChatRequest(_LLMBase): + notebook_id: str + notebook_title: str = "" + email: str = "" + query: str + history: List[Dict[str, Any]] = Field(default_factory=list) + search_method: str = Field(default="auto", pattern="^(auto|local|global)$") + workspace_dir: str = "" + wikidata_enrich: Optional[bool] = None + defer_postprocess: bool = False + + +class ChatResponse(BaseModel): + answer: str + intent: Dict[str, Any] = Field(default_factory=dict) + rewritten_query: str = "" + context_data: Dict[str, Any] = Field(default_factory=dict) + reasoning_subgraph: List[Dict[str, Any]] = Field(default_factory=list) + reasoning_subgraph_cot: str = "" + source_chunks: List[str] = Field(default_factory=list) + highlight_hints: List[Dict[str, Any]] = Field(default_factory=list) + judge_score: float = 0.0 + judge_rationale: str = "" + postprocess_pending: bool = False + graphrag_raw_answer: str = "" + + +class ChatPostprocessRequest(_LLMBase): + query: str + answer: str = "" + reasoning_subgraph: List[Dict[str, Any]] = Field(default_factory=list) + wikidata_enrich: Optional[bool] = None + mode: str = Field(default="subgraph", pattern="^(all|subgraph|wikidata)$") + + +class ChatPostprocessResponse(BaseModel): + reasoning_subgraph: List[Dict[str, Any]] = Field(default_factory=list) + reasoning_subgraph_cot: str = "" + judge_score: float = 0.0 + judge_rationale: str = "" + wikidata_appendix: str = "" + subgraph_done: bool = False + wikidata_done: bool = False + done: bool = True + + +class ChunkSnippetRequest(BaseModel): + """Resolve *chunk_id* to raw text inside GraphRAG ``input/.txt`` markers.""" + + workspace_dir: str = Field(..., description="GraphRAG workspace root (contains chunk_meta.json + input/)") + chunk_id: str = Field(..., min_length=8, description="Hex chunk id from chunk_meta / query") + # LLM credentials forwarded from the frontend (same key/url used by query/index). + api_key: str = "" + api_url: str = "" + # Optional: pass reasoning_subgraph triples so the backend can ask an LLM to pick + # the exact sentence from the chunk that best expresses one of these relationships. + triples: Optional[List[Dict[str, Any]]] = None + # Optional: same passage as shown in UI (e.g. stripped text_units[0].text). When set, + # LLM extraction uses this instead of the raw ``input/*.txt`` block so highlights align with the box. + passage_for_llm: Optional[str] = Field( + default=None, + max_length=120_000, + description="Context passage for highlight LLM; must be substring-compatible with indexed chunk", + ) + + +class ChunkSnippetResponse(BaseModel): + text: str = "" + source_stem: str = "" + found: bool = False + # LLM-extracted verbatim sentence from the chunk that best matches the triples. + # Empty string if triples were not provided or LLM extraction failed. + highlighted_sentence: str = "" + + +class ContextRefineRequest(BaseModel): + """First retrieval text unit + reasoning subgraph → LLM cleans noise + picks supporting quotes.""" + + unit_text: str = Field(..., max_length=150_000, description="Raw text from context_data first Sources row") + subgraph: List[Dict[str, Any]] = Field( + default_factory=list, + description="reasoning_subgraph edges: source/target/relation", + ) + api_key: str = "" + api_url: str = "" + model: str = Field(default_factory=lambda: settings.GRAPHRAG_LLM_MODEL) + + +class ContextRefineResponse(BaseModel): + cleaned_text: str = "" + supporting_snippets: List[str] = Field(default_factory=list) + + +# --------------------------------------------------------------------------- +# Endpoints +# --------------------------------------------------------------------------- + + +def _extract_sentence_for_triples( + chunk_text: str, + triples: List[Dict[str, Any]], + *, + api_key: str = "", + api_url: str = "", +) -> str: + """Ask the configured LLM to pick the verbatim sentence from *chunk_text* that best + expresses one of the given triples. *chunk_text* may be the indexed ``input/*.txt`` + block or the UI passage (e.g. stripped ``text_units`` row) for alignment with highlights. + """ + if not chunk_text.strip() or not triples: + return "" + try: + from openai import OpenAI + except ImportError: + log.debug("[ChunkSnippet] openai not installed; skipping sentence extraction") + return "" + + triple_lines = "\n".join( + f" ({t.get('source', '?')}) --[{t.get('relation', '?')}]--> ({t.get('target', '?')})" + for t in triples[:20] + ) + system_prompt = ( + "You are a precise text extraction assistant. " + "Return ONLY the verbatim sentence or short phrase from the provided chunk " + "that best expresses one of the given relationships. " + "Do NOT paraphrase, add explanation, or include any other text." + ) + user_msg = ( + f"Knowledge graph relationships:\n{triple_lines}\n\n" + f"Chunk text:\n{chunk_text}\n\n" + "Extract the EXACT sentence or phrase from the chunk that best matches " + "one of the relationships above. Return only that text." + ) + try: + import os + resolved_key = api_key.strip() or os.getenv("DF_API_KEY", "") or "none" + api_base = (api_url.strip() or settings.DEFAULT_LLM_API_URL).rstrip("/") + client = OpenAI(api_key=resolved_key, base_url=api_base) + resp = client.chat.completions.create( + model=settings.GRAPHRAG_LLM_MODEL, + max_tokens=256, + temperature=0, + messages=[ + {"role": "system", "content": system_prompt}, + {"role": "user", "content": user_msg}, + ], + ) + sentence = (resp.choices[0].message.content or "").strip() + # Sanity check: LLM must return something that actually appears in the chunk + if sentence and sentence in chunk_text: + return sentence + log.debug("[ChunkSnippet] LLM sentence not found verbatim in chunk; discarding") + return "" + except Exception as exc: + log.warning("[ChunkSnippet] LLM extraction failed: %s", exc) + return "" + + +def _refine_context_unit_with_llm( + unit_text: str, + subgraph: List[Dict[str, Any]], + *, + api_key: str, + api_url: str, + model: str, +) -> tuple[str, List[str]]: + """Return (cleaned_text, supporting_snippets) from raw first-unit text + subgraph edges.""" + raw = (unit_text or "").strip() + if not raw: + return "", [] + if not subgraph: + return raw, [] + + edge_lines: List[str] = [] + for i, e in enumerate(subgraph[:80], start=1): + if not isinstance(e, dict): + continue + s = str(e.get("source") or "").strip() + t = str(e.get("target") or "").strip() + r = str(e.get("relation") or "").strip() + if not (s and t): + continue + edge_lines.append(f"{i}. ({s}) -[{r}]-> ({t})") + if not edge_lines: + return raw, [] + + system = ( + "You clean noisy document excerpts and select supporting quotes for a knowledge-graph subgraph.\n" + "Return ONLY valid JSON with keys: cleaned_text (string), supporting_snippets (array of strings).\n" + "Rules:\n" + "- cleaned_text: remove footers, URLs, page numbers, repeated headers, [chunk:...] / [Data:...] lines, " + "and other boilerplate. Preserve the substantive prose in reading order. Do not invent content.\n" + "- supporting_snippets: 1–6 short verbatim quotes from cleaned_text (exact substrings) that best " + "support the given subgraph edges (entities/relations). Each snippet should be one sentence or clause; " + "prefer distinct non-overlapping snippets.\n" + "- If nothing in the passage supports the subgraph, use an empty supporting_snippets array.\n" + "- Output JSON only, no markdown fences." + ) + user_msg = "raw_passage:\n" + raw[:120_000] + "\n\nsubgraph_edges:\n" + "\n".join(edge_lines) + + try: + from openai import OpenAI + import os + + resolved_key = api_key.strip() or os.getenv("DF_API_KEY", "") or "none" + api_base = (api_url.strip() or settings.DEFAULT_LLM_API_URL).rstrip("/") + client = OpenAI(api_key=resolved_key, base_url=api_base) + mdl = (model or "").strip() or settings.GRAPHRAG_LLM_MODEL + messages = [ + {"role": "system", "content": system}, + {"role": "user", "content": user_msg}, + ] + try: + comp = client.chat.completions.create( + model=mdl, + messages=messages, + temperature=0.1, + max_tokens=8192, + response_format={"type": "json_object"}, + ) + except Exception: + comp = client.chat.completions.create( + model=mdl, + messages=messages, + temperature=0.1, + max_tokens=8192, + ) + choice = (comp.choices[0].message.content or "").strip() + if choice.startswith("```"): + choice = re.sub(r"^```(?:json)?\s*", "", choice, flags=re.I) + choice = re.sub(r"\s*```\s*$", "", choice).strip() + try: + data = json.loads(choice) + except json.JSONDecodeError: + i, j = choice.find("{"), choice.rfind("}") + if i < 0 or j <= i: + return raw, [] + try: + data = json.loads(choice[i : j + 1]) + except json.JSONDecodeError: + return raw, [] + if not isinstance(data, dict): + return raw, [] + cleaned = str(data.get("cleaned_text") or "").strip() + snips_raw = data.get("supporting_snippets") + snips: List[str] = [] + if isinstance(snips_raw, list): + for x in snips_raw[:12]: + if isinstance(x, str) and x.strip(): + snips.append(x.strip()) + if not cleaned: + cleaned = raw + validated: List[str] = [] + for s in snips: + if s in cleaned: + validated.append(s) + continue + s2 = " ".join(s.split()) + if s2 in cleaned: + validated.append(s2) + return cleaned, validated[:6] + except Exception as exc: + log.warning("[ContextRefine] LLM refine failed: %s", exc) + return raw, [] + + +@router.post("/chat", response_model=ChatResponse, summary="GraphRAG conversational chat with intent detection") +async def chat_endpoint(req: ChatRequest): + """Multi-turn GraphRAG chat with intent detection, query rewriting, and answer synthesis.""" + try: + result = await run_chat( + notebook_id=req.notebook_id, + notebook_title=req.notebook_title, + email=req.email, + api_url=req.api_url, + api_key=req.api_key, + model=req.model, + query=req.query, + history=req.history, + search_method=req.search_method, + workspace_dir=req.workspace_dir, + wikidata_enrich=req.wikidata_enrich, + defer_postprocess=req.defer_postprocess, + ) + return ChatResponse( + answer=result.get("answer", ""), + intent=result.get("intent", {}), + rewritten_query=result.get("rewritten_query", ""), + context_data=result.get("context_data", {}), + reasoning_subgraph=result.get("reasoning_subgraph", []), + reasoning_subgraph_cot=result.get("reasoning_subgraph_cot", ""), + source_chunks=result.get("source_chunks", []), + highlight_hints=result.get("highlight_hints", []), + judge_score=float(result.get("judge_score", 0.0)), + judge_rationale=result.get("judge_rationale", ""), + postprocess_pending=bool(result.get("postprocess_pending", False)), + graphrag_raw_answer=result.get("graphrag_raw_answer", ""), + ) + except Exception as exc: + log.exception("[Router] /graphrag-kb/chat error: %s", exc) + raise HTTPException(status_code=500, detail=str(exc)) + + +@router.post("/chat-postprocess", response_model=ChatPostprocessResponse, summary="Postprocess chat metadata (prune/judge/wikidata)") +async def chat_postprocess_endpoint(req: ChatPostprocessRequest) -> ChatPostprocessResponse: + """Run prune+judge and Wikidata appendix after main answer has been shown.""" + from workflow_engine.toolkits.graphrag_ms_tool.judge import judge_confidence + from workflow_engine.toolkits.graphrag_ms_tool.prune_judge_combined import ( + prune_and_judge_combined_llm, + ) + from workflow_engine.toolkits.wikidata_subgraph_enrich import ( + format_wikidata_supplement_for_subgraph, + ) + + cfg = settings + edges = [e for e in (req.reasoning_subgraph or []) if isinstance(e, dict)] + if not edges: + return ChatPostprocessResponse(done=True) + + wd_flag = req.wikidata_enrich + wd_on = ( + bool(getattr(cfg, "GRAPHRAG_WIKIDATA_ENRICH_ENABLED", True)) + if wd_flag is None + else bool(wd_flag) + ) + + api_base = req.api_url.rstrip("/") + api_key = req.api_key + question = req.query + answer = req.answer + + async def _wikidata_task() -> str: + if not wd_on or req.mode == "subgraph": + return "" + return await asyncio.to_thread( + format_wikidata_supplement_for_subgraph, + edges, + lang=str(getattr(cfg, "GRAPHRAG_WIKIDATA_LANG", "zh") or "zh"), + max_entities=int(getattr(cfg, "GRAPHRAG_WIKIDATA_MAX_ENTITIES", 8) or 8), + connect_timeout=float( + getattr(cfg, "GRAPHRAG_WIKIDATA_CONNECT_TIMEOUT_SEC", 10.0) or 10.0 + ), + read_timeout=float(getattr(cfg, "GRAPHRAG_WIKIDATA_TIMEOUT_SEC", 45.0) or 45.0), + http_retries=int(getattr(cfg, "GRAPHRAG_WIKIDATA_HTTP_RETRIES", 2) or 2), + api_url=str( + getattr( + cfg, + "GRAPHRAG_WIKIDATA_API_URL", + "https://www.wikidata.org/w/api.php", + ) + or "https://www.wikidata.org/w/api.php" + ), + emit_failure_hint=True, + ) + + async def _judge_task() -> tuple[List[Dict[str, Any]], str, float, str]: + if req.mode == "wikidata": + return edges, "", 0.0, "" + if bool(getattr(cfg, "GRAPHRAG_SUBGRAPH_PRUNE_ENABLED", True)) and edges: + pj = await asyncio.to_thread( + prune_and_judge_combined_llm, + question, + answer, + edges, + api_base=api_base, + api_key=api_key, + max_edges_input=int(getattr(cfg, "GRAPHRAG_SUBGRAPH_PRUNE_MAX_EDGES_INPUT", 28) or 28), + max_tokens=int(getattr(cfg, "GRAPHRAG_PRUNE_JUDGE_MAX_TOKENS", 768) or 768), + ) + return pj.edges, pj.cot, float(pj.judge.score), str(pj.judge.rationale or "") + j = await asyncio.to_thread( + judge_confidence, + question, + answer, + edges, + api_base=api_base, + api_key=api_key, + ) + return edges, "", float(j.score), str(j.rationale or "") + + try: + judge_pack, wd_extra = await asyncio.gather(_judge_task(), _wikidata_task()) + out_edges, out_cot, out_score, out_rationale = judge_pack + subgraph_done = req.mode in ("all", "subgraph") + wikidata_done = req.mode in ("all", "wikidata") + return ChatPostprocessResponse( + reasoning_subgraph=out_edges, + reasoning_subgraph_cot=out_cot, + judge_score=out_score, + judge_rationale=out_rationale, + wikidata_appendix=wd_extra, + subgraph_done=subgraph_done, + wikidata_done=wikidata_done, + done=True, + ) + except Exception as exc: + log.warning("[Router] /graphrag-kb/chat-postprocess failed: %s", exc) + return ChatPostprocessResponse( + reasoning_subgraph=edges, + reasoning_subgraph_cot="", + judge_score=0.0, + judge_rationale=f"后处理失败：{exc}", + wikidata_appendix="", + subgraph_done=req.mode in ("all", "subgraph"), + wikidata_done=req.mode in ("all", "wikidata"), + done=True, + ) + + +@router.post("/chunk-snippet", response_model=ChunkSnippetResponse, summary="Extract [chunk:…] text from GraphRAG input") +async def chunk_snippet_endpoint(req: ChunkSnippetRequest) -> ChunkSnippetResponse: + """Used by the notebook reader to show the exact indexed chunk, not the full MinerU MD.""" + ws = _safe_workspace_dir(req.workspace_dir) + meta_path = ws / "chunk_meta.json" + if not meta_path.is_file(): + return ChunkSnippetResponse() + try: + meta = json.loads(meta_path.read_text(encoding="utf-8")) + except Exception: + return ChunkSnippetResponse() + cid = req.chunk_id.strip().lower() + entry = meta.get(req.chunk_id.strip()) or meta.get(cid) + if not isinstance(entry, dict): + return ChunkSnippetResponse() + stem = str(entry.get("source_stem") or "").strip() + if not stem: + return ChunkSnippetResponse() + txt_path = ws / "input" / f"{stem}.txt" + if not txt_path.is_file(): + return ChunkSnippetResponse(source_stem=stem, found=False) + try: + raw = txt_path.read_text(encoding="utf-8", errors="replace") + except Exception: + return ChunkSnippetResponse(source_stem=stem, found=False) + block = _extract_chunk_block_from_input_text(raw, cid) + if not block: + return ChunkSnippetResponse(source_stem=stem, found=False) + passage = (req.passage_for_llm or "").strip() + llm_context = passage if passage else block + highlighted_sentence = "" + if req.triples: + highlighted_sentence = _extract_sentence_for_triples( + llm_context, req.triples, api_key=req.api_key, api_url=req.api_url + ) + log.debug( + "[ChunkSnippet] chunk=%s hl_len=%d hl=%r", + req.chunk_id[:8], + len(highlighted_sentence), + highlighted_sentence[:80] if highlighted_sentence else "", + ) + return ChunkSnippetResponse(text=block, source_stem=stem, found=True, highlighted_sentence=highlighted_sentence) + + +@router.post("/context-refine", response_model=ContextRefineResponse, summary="Clean first unit + supporting snippets from subgraph") +async def context_refine_endpoint(req: ContextRefineRequest) -> ContextRefineResponse: + """Side panel: raw first text_unit + reasoning_subgraph → cleaned body + verbatim supporting quotes.""" + cleaned, snips = _refine_context_unit_with_llm( + req.unit_text, + req.subgraph, + api_key=req.api_key, + api_url=req.api_url, + model=req.model, + ) + return ContextRefineResponse(cleaned_text=cleaned, supporting_snippets=snips) + + +# --------------------------------------------------------------------------- +# Index / query / merge +# --------------------------------------------------------------------------- + +@router.post("/index", response_model=IndexResponse, summary="Build GraphRAG index from notebook sources") +async def index_endpoint(req: IndexRequest): + """Chunk notebook sources and run GraphRAG index (KGGen off by default). + + Requires that sources have already been imported into the notebook + (via the ``/kb`` upload endpoint) so that MinerU output exists. + """ + try: + result = await run_index( + notebook_id=req.notebook_id, + notebook_title=req.notebook_title, + email=req.email, + api_url=req.api_url, + api_key=req.api_key, + model=req.model, + source_stems=req.source_stems, + workspace_dir=req.workspace_dir, + force_reindex=req.force_reindex, + parse_pdfs=req.parse_pdfs, + skip_kggen=req.skip_kggen, + ) + return IndexResponse( + workspace_dir=result.get("workspace_dir", ""), + num_chunks=result.get("num_chunks", 0), + kg_entities=result.get("kg_entities", 0), + kg_relations=result.get("kg_relations", 0), + ) + except Exception as exc: + log.exception("[Router] /graphrag-kb/index error: %s", exc) + raise HTTPException(status_code=500, detail=str(exc)) + + +@router.post("/query", response_model=QueryResponse, summary="Query GraphRAG index with Judge scoring") +async def query_endpoint(req: QueryRequest): + """Run a local or global GraphRAG search and return a structured result. + + Returns: + - ``answer`` — model answer text + - ``context_data`` — serialised evidence tables (entities, relations, sources…) + - ``reasoning_subgraph`` — edge list induced from context_data + - ``source_chunks`` — chunk_ids that contributed to the answer + - ``highlight_hints`` — page/bbox hints for PDF highlighting + - ``judge_score`` — confidence score in [0.0, 1.0] + - ``judge_rationale`` — one-sentence judge explanation + - ``reasoning_subgraph_cot`` — LLM chain-of-thought for minimal subgraph (hop analysis) + """ + try: + result = await run_query( + notebook_id=req.notebook_id, + notebook_title=req.notebook_title, + email=req.email, + api_url=req.api_url, + api_key=req.api_key, + model=req.model, + question=req.question, + search_method=req.search_method, + workspace_dir=req.workspace_dir, + wikidata_enrich=req.wikidata_enrich, + ) + return QueryResponse( + answer=result.get("answer", ""), + context_data=result.get("context_data", {}), + reasoning_subgraph=result.get("reasoning_subgraph", []), + source_chunks=result.get("source_chunks", []), + highlight_hints=result.get("highlight_hints", []), + judge_score=float(result.get("judge_score", 0.0)), + judge_rationale=result.get("judge_rationale", ""), + reasoning_subgraph_cot=result.get("reasoning_subgraph_cot", ""), + ) + except Exception as exc: + log.exception("[Router] /graphrag-kb/query error: %s", exc) + raise HTTPException(status_code=500, detail=str(exc)) + + +@router.post("/merge", response_model=MergeResponse, summary="Merge two GraphRAG KG workspaces") +async def merge_endpoint(req: MergeRequest): + """Merge two GraphRAG workspaces using KGGen aggregate and re-index. + + Both ``workspace_dir_a`` and ``workspace_dir_b`` must be absolute paths to + valid, previously indexed workspaces. The merged workspace is written to + ``{workspace_dir_a}_merged/``. + """ + try: + result = await run_merge( + notebook_id=req.notebook_id, + notebook_title=req.notebook_title, + email=req.email, + api_url=req.api_url, + api_key=req.api_key, + model=req.model, + workspace_dir_a=req.workspace_dir_a, + workspace_dir_b=req.workspace_dir_b, + dedupe=req.dedupe, + ) + return MergeResponse( + merged_workspace_dir=result.get("merged_workspace_dir", ""), + num_chunks=result.get("num_chunks", 0), + ) + except Exception as exc: + log.exception("[Router] /graphrag-kb/merge error: %s", exc) + raise HTTPException(status_code=500, detail=str(exc)) diff --git a/fastapi_app/source_manager.py b/fastapi_app/source_manager.py index 5778163..2645cbc 100644 --- a/fastapi_app/source_manager.py +++ b/fastapi_app/source_manager.py @@ -7,16 +7,19 @@ - Generating unified markdown for every source type - Reading back markdown / MinerU data for feature cards - Fallback to legacy kb_data / kb_mineru paths +- Structured chunk extraction with chunk_id / page_index / order / bbox (for GraphRAG) """ from __future__ import annotations import asyncio +import hashlib +import json import re import shutil import time from dataclasses import dataclass, field from pathlib import Path -from typing import List, Optional, Tuple +from typing import Any, Dict, List, Optional, Tuple from workflow_engine.logger import get_logger from workflow_engine.utils import get_project_root @@ -237,6 +240,85 @@ def ensure_sam3_dir(self, source_stem: str) -> Path: sam3_dir.mkdir(parents=True, exist_ok=True) return sam3_dir + def get_chunks_with_meta( + self, + source_stem: str, + chunk_size: int = 512, # 默认值与 settings.GRAPHRAG_CHUNK_SIZE 一致 + chunk_overlap: int = 64, # 默认值与 settings.GRAPHRAG_CHUNK_OVERLAP 一致 + ) -> List[Dict[str, Any]]: + """Return structured chunks for a single source, used by GraphRAG indexing. + + Each dict has keys: chunk_id, text, page_index, order, bbox, source_stem. + chunk_id = SHA1("{stem}:{order}")[:16], embedded as [chunk:ID] in input/*.txt. + Priority: MinerU content_list.json (exact page+bbox) → MinerU MD (estimated page) + → unified MD (page_index=-1). + """ + chunks: List[Dict[str, Any]] = [] + + # 1) MinerU content_list.json — exact page + bbox per block + mineru_root = self.get_mineru_root(source_stem) + if mineru_root: + content_list_path = None + # rglob to handle varying MinerU output directory layouts + for candidate in mineru_root.parent.rglob("*_content_list.json"): + content_list_path = candidate + break + if content_list_path and content_list_path.exists(): + try: + raw_blocks = json.loads( + content_list_path.read_text(encoding="utf-8") + ) + order = 0 + for block in raw_blocks: + # MinerU uses "text" or "content" depending on version + text = (block.get("text") or block.get("content") or "").strip() + if not text: + continue # skip image / formula blocks + # MinerU uses "page_idx" or "page_index" depending on version + page_idx = int(block.get("page_idx", block.get("page_index", -1))) + bbox = block.get("bbox") # [x1,y1,x2,y2] normalized, may be None + # chunk_id = SHA1("{stem}:{order}")[:16], embedded as [chunk:ID] in input/*.txt + chunk_id = hashlib.sha1( + f"{source_stem}:{order}".encode() + ).hexdigest()[:16] + chunks.append( + { + "chunk_id": chunk_id, + "text": text, + "page_index": page_idx, + "order": order, + "bbox": bbox, + "source_stem": source_stem, + } + ) + order += 1 + if chunks: + return chunks + except Exception as e: + log.debug( + "[SourceManager] content_list.json parse failed for %s: %s", + source_stem, + e, + ) + + # 2) MinerU markdown fallback — sliding window, estimated page_index + mineru_md = self.get_mineru_md(source_stem) + if mineru_md.strip(): + chunks = self._split_text_to_chunks( + mineru_md, source_stem, chunk_size, chunk_overlap, estimate_pages=True + ) + if chunks: + return chunks + + # 3) Unified markdown fallback — no page info (Word/PPT/TXT) + md = self.get_markdown(source_stem) + if md.strip(): + return self._split_text_to_chunks( + md, source_stem, chunk_size, chunk_overlap, estimate_pages=False + ) + + return [] + def get_all_markdowns(self) -> List[Tuple[str, str]]: """Return [(stem, markdown_text), ...] for all sources.""" results: List[Tuple[str, str]] = [] @@ -393,3 +475,49 @@ def _find_in_sources(self, source_stem: str, subdir: str, pattern: str) -> str: except Exception: continue return "" + + @staticmethod + def _split_text_to_chunks( + text: str, + source_stem: str, + chunk_size: int, + chunk_overlap: int, + estimate_pages: bool, + ) -> List[Dict[str, Any]]: + """Sliding-window character chunking fallback when content_list is unavailable. + + estimate_pages=True roughly estimates page_index at ~2000 chars/page. + """ + chunks: List[Dict[str, Any]] = [] + text = text.strip() + if not text: + return chunks + + total_chars = len(text) + step = max(1, chunk_size - chunk_overlap) + order = 0 + pos = 0 + chars_per_page = 2000 # rough estimate: ~2000 chars per page + + while pos < total_chars: + end = min(pos + chunk_size, total_chars) + snippet = text[pos:end].strip() + if snippet: + page_idx = int(pos / chars_per_page) if estimate_pages else -1 + chunk_id = hashlib.sha1( + f"{source_stem}:{order}".encode() + ).hexdigest()[:16] + chunks.append( + { + "chunk_id": chunk_id, + "text": snippet, + "page_index": page_idx, + "order": order, + "bbox": None, + "source_stem": source_stem, + } + ) + order += 1 + pos += step + + return chunks diff --git a/fastapi_app/workflow_adapters/wa_graphrag_kb.py b/fastapi_app/workflow_adapters/wa_graphrag_kb.py new file mode 100644 index 0000000..846b71f --- /dev/null +++ b/fastapi_app/workflow_adapters/wa_graphrag_kb.py @@ -0,0 +1,195 @@ +"""GraphRAG KB 管线的工作流适配层。 + +【职责】 + 在 FastAPI 路由（Pydantic 请求体）与 ``wf_graphrag_kb``（``GraphRAGKBState`` 数据类）之间做转换， + 统一调用 ``run_workflow("graphrag_kb", state)``，再从 ``agent_results`` / ``temp_data.errors`` 取结果。 + +【数据流】 + ``run_index`` / ``run_query`` / ``run_merge`` → 组装 ``GraphRAGKBRequest.action`` → + ``GraphRAGKBState`` → LangGraph 执行 → 成功则返回对应 ``agent_results`` 字典；失败则 ``RuntimeError``（携带首条错误信息）。 + +【约定】 + 与 ``wa_paper2ppt.py`` 类似：``_workflow_outcome`` 兼容 LangGraph 返回 dataclass 或 dict。 +""" +from __future__ import annotations + +from typing import Any, Dict, List, Optional, Tuple + +from workflow_engine.logger import get_logger +from workflow_engine.workflow import run_workflow +from workflow_engine.workflow.wf_graphrag_kb import GraphRAGKBRequest, GraphRAGKBState +from workflow_engine.workflow.wf_graphrag_chat import GraphRAGChatRequest, GraphRAGChatState + +log = get_logger(__name__) + + +def _workflow_outcome(state: Any) -> Tuple[Dict[str, Any], Optional[list]]: + """统一解析工作流终态：得到 ``(agent_results, errors)``，兼容 dict 与 dataclass 两种返回形式。""" + if isinstance(state, dict): + td = state.get("temp_data") + td = td if isinstance(td, dict) else {} + errors = td.get("errors") + ar = state.get("agent_results") + ar = ar if isinstance(ar, dict) else {} + return ar, errors + td = getattr(state, "temp_data", None) + td = td if isinstance(td, dict) else {} + errors = td.get("errors") + ar = getattr(state, "agent_results", None) + ar = ar if isinstance(ar, dict) else {} + return ar, errors + + +# --------------------------------------------------------------------------- +# Public adapter functions (called by routers) +# --------------------------------------------------------------------------- + +async def run_index( + *, + notebook_id: str, + notebook_title: str, + email: str, + api_url: str, + api_key: str, + model: str, + source_stems: Optional[List[str]] = None, + workspace_dir: str = "", + force_reindex: bool = False, + parse_pdfs: bool = True, + skip_kggen: bool = True, +) -> Dict[str, Any]: + """Run indexing workflow; returns ``agent_results["index"]`` dict on success.""" + req = GraphRAGKBRequest( + action="index", + notebook_id=notebook_id, + notebook_title=notebook_title, + email=email, + chat_api_url=api_url, + api_key=api_key, + model=model, + source_stems=source_stems or [], + workspace_dir=workspace_dir, + force_reindex=force_reindex, + parse_pdfs=parse_pdfs, + skip_kggen=skip_kggen, + ) + state = GraphRAGKBState(request=req) + state = await run_workflow("graphrag_kb", state) + + agent_results, errors = _workflow_outcome(state) + if errors: + raise RuntimeError(f"Indexing failed: {errors[0]}") + + return agent_results.get("index", {}) + + +async def run_query( + *, + notebook_id: str, + notebook_title: str, + email: str, + api_url: str, + api_key: str, + model: str, + question: str, + search_method: str = "local", + workspace_dir: str = "", + wikidata_enrich: Optional[bool] = None, +) -> Dict[str, Any]: + """Run query workflow; returns ``agent_results["query"]`` dict on success.""" + req = GraphRAGKBRequest( + action="query", + notebook_id=notebook_id, + notebook_title=notebook_title, + email=email, + chat_api_url=api_url, + api_key=api_key, + model=model, + question=question, + search_method=search_method, + workspace_dir=workspace_dir, + wikidata_enrich=wikidata_enrich, + ) + state = GraphRAGKBState(request=req) + state = await run_workflow("graphrag_kb", state) + + agent_results, errors = _workflow_outcome(state) + if errors: + raise RuntimeError(f"Query failed: {errors[0]}") + + return agent_results.get("query", {}) + + +async def run_merge( + *, + notebook_id: str, + notebook_title: str, + email: str, + api_url: str, + api_key: str, + model: str, + workspace_dir_a: str, + workspace_dir_b: str, + dedupe: bool = False, +) -> Dict[str, Any]: + """Merge two GraphRAG workspaces and re-index; returns ``agent_results["merge"]``.""" + req = GraphRAGKBRequest( + action="merge", + notebook_id=notebook_id, + notebook_title=notebook_title, + email=email, + chat_api_url=api_url, + api_key=api_key, + model=model, + workspace_dir=workspace_dir_a, + workspace_dir_b=workspace_dir_b, + dedupe=dedupe, + ) + state = GraphRAGKBState(request=req) + state = await run_workflow("graphrag_kb", state) + + agent_results, errors = _workflow_outcome(state) + if errors: + raise RuntimeError(f"Merge failed: {errors[0]}") + + return agent_results.get("merge", {}) + + +async def run_chat( + *, + notebook_id: str, + notebook_title: str = "", + email: str = "", + api_url: str, + api_key: str, + model: str, + query: str, + history: List[Dict[str, str]], + search_method: str = "auto", + workspace_dir: str = "", + wikidata_enrich: Optional[bool] = None, + defer_postprocess: bool = False, +) -> Dict[str, Any]: + """Run GraphRAG conversational chat; returns ``agent_results["chat"]`` dict.""" + req = GraphRAGChatRequest( + notebook_id=notebook_id, + notebook_title=notebook_title, + email=email, + chat_api_url=api_url, + api_key=api_key, + model=model, + query=query, + history=history, + search_method=search_method, + workspace_dir=workspace_dir, + wikidata_enrich=wikidata_enrich, + defer_postprocess=defer_postprocess, + ) + state = GraphRAGChatState(request=req) + state = await run_workflow("graphrag_chat", state) + + agent_results, errors = _workflow_outcome(state) + if errors: + raise RuntimeError(f"GraphRAG chat failed: {errors[0]}") + + return agent_results.get("chat", {}) diff --git a/frontend_en/package-lock.json b/frontend_en/package-lock.json index 873bc08..df09701 100644 --- a/frontend_en/package-lock.json +++ b/frontend_en/package-lock.json @@ -19,6 +19,7 @@ "react-dom": "^18.2.0", "react-markdown": "^9.1.0", "react-pdf": "^10.3.0", + "rehype-raw": "^7.0.0", "tailwind-merge": "^2.0.0", "zustand": "^4.4.7" }, @@ -2632,6 +2633,17 @@ "integrity": "sha512-f/ZeWvW/BCXbhGEf1Ujp29EASo/lk1FDnETgNKwJrsVvGZhUWCZyg3xLJjAsxfOmt8KjswHmI5EwCQcPMpOYhQ==", "license": "EPL-2.0" }, + "node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/esbuild": { "version": "0.21.5", "resolved": "https://registry.npmjs.org/esbuild/-/esbuild-0.21.5.tgz", @@ -2849,6 +2861,71 @@ "node": ">= 0.4" } }, + "node_modules/hast-util-from-parse5": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/hast-util-from-parse5/-/hast-util-from-parse5-8.0.3.tgz", + "integrity": "sha512-3kxEVkEKt0zvcZ3hCRYI8rqrgwtlIOFMWkbclACvjlDw8Li9S2hk/d51OI0nr/gIpdMHNepwgOKqZ/sy0Clpyg==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "devlop": "^1.0.0", + "hastscript": "^9.0.0", + "property-information": "^7.0.0", + "vfile": "^6.0.0", + "vfile-location": "^5.0.0", + "web-namespaces": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-from-parse5/node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" + }, + "node_modules/hast-util-parse-selector": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/hast-util-parse-selector/-/hast-util-parse-selector-4.0.0.tgz", + "integrity": "sha512-wkQCkSYoOGCRKERFWcxMVMOcYE2K1AaNLU8DXS9arxnLOUEWbOXKXiJUNzEpqZ3JOKpnha3jkFrumEjVliDe7A==", + "dependencies": { + "@types/hast": "^3.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/hast-util-raw/-/hast-util-raw-9.1.0.tgz", + "integrity": "sha512-Y8/SBAHkZGoNkpzqqfCldijcuUKh7/su31kEBp67cFY09Wy0mTRgtsLYsiIxMJxlu0f6AA5SUTbDR8K0rxnbUw==", + "dependencies": { + "@types/hast": "^3.0.0", + "@types/unist": "^3.0.0", + "@ungap/structured-clone": "^1.0.0", + "hast-util-from-parse5": "^8.0.0", + "hast-util-to-parse5": "^8.0.0", + "html-void-elements": "^3.0.0", + "mdast-util-to-hast": "^13.0.0", + "parse5": "^7.0.0", + "unist-util-position": "^5.0.0", + "unist-util-visit": "^5.0.0", + "vfile": "^6.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/hast-util-raw/node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" + }, "node_modules/hast-util-to-jsx-runtime": { "version": "2.3.6", "resolved": "https://registry.npmjs.org/hast-util-to-jsx-runtime/-/hast-util-to-jsx-runtime-2.3.6.tgz", @@ -2882,6 +2959,24 @@ "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==", "license": "MIT" }, + "node_modules/hast-util-to-parse5": { + "version": "8.0.1", + "resolved": "https://registry.npmjs.org/hast-util-to-parse5/-/hast-util-to-parse5-8.0.1.tgz", + "integrity": "sha512-MlWT6Pjt4CG9lFCjiz4BH7l9wmrMkfkJYCxFwKQic8+RTZgWPuWxwAfjJElsXkex7DJjfSJsQIt931ilUgmwdA==", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "devlop": "^1.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0", + "web-namespaces": "^2.0.0", + "zwitch": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/hast-util-whitespace": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/hast-util-whitespace/-/hast-util-whitespace-3.0.0.tgz", @@ -2895,6 +2990,22 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/hastscript": { + "version": "9.0.1", + "resolved": "https://registry.npmjs.org/hastscript/-/hastscript-9.0.1.tgz", + "integrity": "sha512-g7df9rMFX/SPi34tyGCyUBREQoKkapwdY/T04Qn9TDWfHhAYt4/I0gMVirzK5wEzeUqIjEB+LXC/ypb7Aqno5w==", + "dependencies": { + "@types/hast": "^3.0.0", + "comma-separated-tokens": "^2.0.0", + "hast-util-parse-selector": "^4.0.0", + "property-information": "^7.0.0", + "space-separated-tokens": "^2.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/html-url-attributes": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/html-url-attributes/-/html-url-attributes-3.0.1.tgz", @@ -2905,6 +3016,15 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/html-void-elements": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/html-void-elements/-/html-void-elements-3.0.0.tgz", + "integrity": "sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/iceberg-js": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/iceberg-js/-/iceberg-js-0.8.1.tgz", @@ -5773,6 +5893,17 @@ "url": "https://github.com/sponsors/wooorm" } }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, "node_modules/path-parse": { "version": "1.0.7", "resolved": "https://registry.npmjs.org/path-parse/-/path-parse-1.0.7.tgz", @@ -6149,6 +6280,20 @@ "node": ">=8.10.0" } }, + "node_modules/rehype-raw": { + "version": "7.0.0", + "resolved": "https://registry.npmjs.org/rehype-raw/-/rehype-raw-7.0.0.tgz", + "integrity": "sha512-/aE8hCfKlQeA8LmyeyQvQF3eBiLRGNlfBJEvWH7ivp9sBqs7TNqBL5X3v157rM4IFETqDnIOO+z5M/biZbo9Ww==", + "dependencies": { + "@types/hast": "^3.0.0", + "hast-util-raw": "^9.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, "node_modules/remark-parse": { "version": "11.0.0", "resolved": "https://registry.npmjs.org/remark-parse/-/remark-parse-11.0.0.tgz", @@ -7361,6 +7506,24 @@ "url": "https://opencollective.com/unified" } }, + "node_modules/vfile-location": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/vfile-location/-/vfile-location-5.0.3.tgz", + "integrity": "sha512-5yXvWDEgqeiYiBe1lbxYF7UMAIm/IcopxMHrMQDq3nvKcjPKIhZklUKL+AE7J7uApI4kwe2snsK+eI6UTj9EHg==", + "dependencies": { + "@types/unist": "^3.0.0", + "vfile": "^6.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/unified" + } + }, + "node_modules/vfile-location/node_modules/@types/unist": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/unist/-/unist-3.0.3.tgz", + "integrity": "sha512-ko/gIFJRv177XgZsZcBwnqJN5x/Gien8qNOn0D5bQU/zAzVf9Zt3BlcUiLqhV9y4ARk0GbT3tnUiPNgnTXzc/Q==" + }, "node_modules/vfile-message": { "version": "4.0.3", "resolved": "https://registry.npmjs.org/vfile-message/-/vfile-message-4.0.3.tgz", @@ -7469,6 +7632,15 @@ "loose-envify": "^1.0.0" } }, + "node_modules/web-namespaces": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/web-namespaces/-/web-namespaces-2.0.1.tgz", + "integrity": "sha512-bKr1DkiNa2krS7qxNtdrtHAmzuYGFQLiQ13TsorsdT6ULTkPLKuu5+GsFpDlg6JFjUTwX2DyhMPG2be8uPrqsQ==", + "funding": { + "type": "github", + "url": "https://github.com/sponsors/wooorm" + } + }, "node_modules/web-worker": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/web-worker/-/web-worker-1.5.0.tgz", diff --git a/frontend_en/package.json b/frontend_en/package.json index b6d9db4..d2b2fac 100644 --- a/frontend_en/package.json +++ b/frontend_en/package.json @@ -20,6 +20,7 @@ "react-dom": "^18.2.0", "react-markdown": "^9.1.0", "react-pdf": "^10.3.0", + "rehype-raw": "^7.0.0", "tailwind-merge": "^2.0.0", "zustand": "^4.4.7" }, diff --git a/frontend_en/src/components/graphrag-kb/GraphRAGKbPanel.tsx b/frontend_en/src/components/graphrag-kb/GraphRAGKbPanel.tsx new file mode 100644 index 0000000..9de171a --- /dev/null +++ b/frontend_en/src/components/graphrag-kb/GraphRAGKbPanel.tsx @@ -0,0 +1,2 @@ +export { GraphRAGKbPanel } from './GraphRAGKbPanelChatAligned'; +export type { GraphRAGKbPanelProps } from './GraphRAGKbPanelChatAligned'; diff --git a/frontend_en/src/components/graphrag-kb/GraphRAGKbPanelChatAligned.tsx b/frontend_en/src/components/graphrag-kb/GraphRAGKbPanelChatAligned.tsx new file mode 100644 index 0000000..a0ef997 --- /dev/null +++ b/frontend_en/src/components/graphrag-kb/GraphRAGKbPanelChatAligned.tsx @@ -0,0 +1,820 @@ +/** + * English GraphRAG KB panel aligned with frontend_zh flow: + * - index + * - chat with deferred postprocess + * - split postprocess: subgraph/judge and Wikidata + * - merge + */ +import React, { useCallback, useEffect, useMemo, useRef, useState } from 'react'; +import ReactMarkdown from 'react-markdown'; +import { Loader2, Copy, ChevronDown, ChevronRight, Network, Send } from 'lucide-react'; +import { getApiSettings } from '../../services/apiSettingsService'; +import { + indexGraphragKb, + mergeGraphragKb, + chatGraphragKb, + chatGraphragKbPostprocess, + defaultGraphragModel, + refineGraphragContextRefine, +} from '../../services/graphragKbService'; +import type { ChatMessage, ChatResponse, GraphragWorkspacePersist } from '../../types/graphragKb'; +import { MermaidPreview } from '../knowledge-base/tools/MermaidPreview'; +import { injectMultipleGraphragHighlightsInMarkdown } from '../../utils/graphragMarkdownHighlight'; +import { + extractChunkIdFromText, + stripGraphragContextNoise, +} from '../../utils/stripGraphragContextNoise'; + +function getWorkspaceStorageKey(userId: string, notebookId: string) { + return `graphrag_workspace_${userId}_${notebookId}`; +} + +function sanitizeMermaidLabel(s: string, max = 48): string { + return s.replace(/["[\]#]/g, ' ').slice(0, max).trim() || '?'; +} + +export function reasoningSubgraphToMermaid(edges: Array>, maxEdges = 36): string | null { + if (!edges.length) return null; + const slice = edges.slice(0, maxEdges); + const idFor = (() => { + const m = new Map(); + let n = 0; + return (raw: string) => { + const k = raw || `_${n}`; + if (!m.has(k)) m.set(k, `N${n++}`); + return m.get(k)!; + }; + })(); + const lines: string[] = [ + '%%{init: {"flowchart": {"htmlLabels": true, "wrappingWidth": 250}} }%%', + 'graph TD', + ]; + for (let i = 0; i < slice.length; i++) { + const e = slice[i]; + const src = String(e.source ?? e.src ?? e.from ?? e.head ?? `s${i}`); + const tgt = String(e.target ?? e.tgt ?? e.to ?? e.tail ?? `t${i}`); + const rel = String(e.relation ?? e.relationship ?? e.label ?? e.predicate ?? ''); + const sid = idFor(src); + const tid = idFor(tgt); + const sl = sanitizeMermaidLabel(src, 40); + const tl = sanitizeMermaidLabel(tgt, 40); + const rl = sanitizeMermaidLabel(rel, 60); + lines.push(` ${sid}["${sl}"] -->|"${rl}"| ${tid}["${tl}"]`); + } + return lines.join('\n'); +} + +interface ContextChunk { + chunkId: string; + text: string; + nTokens?: number; + sourceStem?: string; +} + +function extractTopChunk( + contextData: Record, + highlightHints: Array>, +): ContextChunk | null { + const textUnits = + (contextData['sources'] as Array> | undefined) ?? + (contextData['text_units'] as Array> | undefined); + if (!textUnits || !Array.isArray(textUnits) || textUnits.length === 0) return null; + + const first = textUnits[0]; + const rawText = String(first['text'] ?? first['content'] ?? ''); + if (!rawText.trim()) return null; + + const docIds = first['document_ids']; + const sourceStemFromUnit = Array.isArray(docIds) && docIds.length > 0 ? String(docIds[0]) : ''; + const sourceStemFromHint = highlightHints.length > 0 ? String(highlightHints[0]['source_stem'] ?? '') : ''; + + const embedded = extractChunkIdFromText(rawText); + const idField = String(first['id'] ?? first['chunk_id'] ?? '').trim(); + const chunkId = embedded || idField; + + return { + chunkId, + text: rawText, + nTokens: first['n_tokens'] != null ? Number(first['n_tokens']) : undefined, + sourceStem: sourceStemFromUnit || sourceStemFromHint || undefined, + }; +} + +const STR = { + zh: { + headerTitle: 'GraphRAG 知识库', + headerSub: '分块（MinerU）+ GraphRAG 建索引与检索', + apiWarn: '请先在设置中配置 API URL 与 API Key', + noNotebook: '缺少笔记本 ID', + indexBtn: '构建索引', + indexing: '索引构建中…', + indexOk: '索引构建完成', + forceReindex: '强制重建', + parsePdfs: '解析 PDF（MinerU）', + summary: '上次索引摘要', + chunks: '分块数', + workspace: '工作区目录', + copy: '复制', + copied: '已复制', + modelLabel: 'LLM 模型名', + copyFailed: '复制失败', + mergeTitle: '合并工作区', + mergeA: 'workspace_dir A', + mergeB: 'workspace_dir B', + dedupe: '去重合并', + mergeBtn: '合并并重建索引', + merging: '合并中…', + mergeOk: '合并完成', + chatPlaceholder: '向知识库提问…', + send: '发送', + searchMethodLabel: '检索策略', + wikidataEnrich: 'Wikidata 参考（附在答案后）', + clearChat: '清空对话', + emptyReady: '索引已就绪，开始提问吧', + emptyNoIndex: '请先完成索引构建', + contextRefTitle: '上下文参考', + subgraph: '推理子图', + subgraphRaw: '推理全图（未裁剪版）', + noSubgraph: '无子图数据', + subgraphCot: '最小子图推理（CoT / 跳数）', + mermaidTitle: '子图（Mermaid）', + judge: 'Judge 分数', + postprocessSubgraphPending: '正在裁剪子图…', + postprocessWikidataPending: '正在补充 Wikidata 参考…', + }, + en: { + headerTitle: 'GraphRAG Knowledge Base', + headerSub: 'Chunking (MinerU) + GraphRAG index & query', + apiWarn: 'Configure API URL and API Key in Settings first', + noNotebook: 'Notebook ID is missing', + indexBtn: 'Build index', + indexing: 'Indexing…', + indexOk: 'Index completed', + forceReindex: 'Force reindex', + parsePdfs: 'Parse PDFs (MinerU)', + summary: 'Last index summary', + chunks: 'Chunks', + workspace: 'Workspace directory', + copy: 'Copy', + copied: 'Copied', + modelLabel: 'LLM model', + copyFailed: 'Copy failed', + mergeTitle: 'Merge workspaces', + mergeA: 'workspace_dir A', + mergeB: 'workspace_dir B', + dedupe: 'Deduplicate when merging', + mergeBtn: 'Merge and re-index', + merging: 'Merging…', + mergeOk: 'Merge completed', + chatPlaceholder: 'Ask the knowledge base…', + send: 'Send', + searchMethodLabel: 'Search method', + wikidataEnrich: 'Wikidata supplement (after answer)', + clearChat: 'Clear chat', + emptyReady: 'Index ready. Start asking questions.', + emptyNoIndex: 'Build the index first.', + contextRefTitle: 'Context Reference', + subgraph: 'Reasoning subgraph', + subgraphRaw: 'Full reasoning graph (unpruned)', + noSubgraph: 'No subgraph', + subgraphCot: 'Minimal subgraph reasoning (CoT / hops)', + mermaidTitle: 'Subgraph (Mermaid)', + judge: 'Judge score', + postprocessSubgraphPending: 'Pruning subgraph…', + postprocessWikidataPending: 'Enriching Wikidata supplement…', + }, +} as const; + +function ContextRefHtml({ + topChunk, + subgraph, + userId, + colorIdx, + locale, +}: { + topChunk: ContextChunk; + subgraph: Array>; + userId: string | null; + colorIdx: number; + locale: 'zh' | 'en'; +}) { + const basePlain = useMemo(() => stripGraphragContextNoise(topChunk.text), [topChunk.text]); + const [html, setHtml] = useState(basePlain); + const [loading, setLoading] = useState(false); + + useEffect(() => { + setHtml(basePlain); + if (!subgraph?.length) { + setLoading(false); + return; + } + let cancelled = false; + setLoading(true); + (async () => { + try { + const st = getApiSettings(userId); + const out = await refineGraphragContextRefine( + topChunk.text, + subgraph, + st?.apiKey?.trim() || '', + st?.apiUrl?.trim() || '', + defaultGraphragModel(), + ); + if (cancelled) return; + const body = (out.cleaned_text || '').trim() || basePlain; + const snips = (out.supporting_snippets || []).map((s) => s.trim()).filter(Boolean); + if (snips.length) { + setHtml(injectMultipleGraphragHighlightsInMarkdown(body, snips, { baseColorIndex: colorIdx })); + } else { + setHtml(body); + } + } catch { + if (!cancelled) setHtml(basePlain); + } finally { + if (!cancelled) setLoading(false); + } + })(); + return () => { + cancelled = true; + }; + }, [topChunk.chunkId, subgraph, basePlain, userId, colorIdx, topChunk.text]); + + return ( +
+ {loading ? ( +
+ {locale === 'zh' ? '正在清洗正文并选取支撑句…' : 'Cleaning passage and selecting evidence…'} +
+ ) : null} +
+
+ ); +} + +interface AssistantMetaProps { + meta: NonNullable; + locale: 'zh' | 'en'; + L: typeof STR['zh']; + userId: string | null; + subgraphPending: boolean; +} + +function AssistantMeta({ meta, locale, L, userId, subgraphPending }: AssistantMetaProps) { + const topChunk = useMemo(() => { + if (!meta.context_data) return null; + return extractTopChunk( + meta.context_data as Record, + (meta.highlight_hints ?? []) as Array>, + ); + }, [meta.context_data, meta.highlight_hints]); + + const mermaidCode = useMemo(() => { + if (!meta.reasoning_subgraph?.length) return null; + return reasoningSubgraphToMermaid(meta.reasoning_subgraph as Array>); + }, [meta.reasoning_subgraph]); + + const subgraphRows = (meta.reasoning_subgraph ?? []) as Array>; + const judgePct = Math.round(Math.max(0, Math.min(1, meta.judge_score ?? 0)) * 100); + const [subgraphOpen, setSubgraphOpen] = useState(false); + + return ( +
+ {meta.intent?.use_graphrag === false && ( +
+ {locale === 'zh' ? '直接回答（无检索）' : 'Direct answer (no retrieval)'} +
+ )} + {meta.intent?.use_graphrag === true && meta.rewritten_query && ( +
+ {locale === 'zh' ? '检索问题：' : 'Retrieval query: '} + {meta.rewritten_query} +
+ )} + + {topChunk && ( +
+
{L.contextRefTitle}
+ {topChunk.sourceStem && ( +
+ {topChunk.sourceStem} + {topChunk.nTokens != null && ( + {topChunk.nTokens} tokens + )} +
+ )} + +
+ )} + + {mermaidCode && ( +
+ + {subgraphOpen && ( + <> +
+ +
+ {meta.reasoning_subgraph_cot ? ( +
+ {L.subgraphCot} +
+ {meta.reasoning_subgraph_cot} +
+
+ ) : null} + + )} +
+ )} + + {judgePct > 0 && ( +
+
{L.judge}: {judgePct}%
+ {meta.judge_rationale ? ( +
{meta.judge_rationale}
+ ) : null} +
+ )} +
+ ); +} + +export type GraphragOpenSourcePayload = { + sourceStem: string; + pageIndex: number; + chunkId?: string; + workspaceDir?: string; +}; + +export interface GraphRAGKbPanelProps { + notebook: { id?: string; title?: string; name?: string }; + userId: string | null; + email: string; + locale?: 'zh' | 'en'; + showToast: (message: string, type?: 'success' | 'error' | 'warning') => void; + onOpenGraphragSource?: (payload: GraphragOpenSourcePayload) => void | Promise; +} + +export function GraphRAGKbPanel({ + notebook, + userId, + email, + locale = 'en', + showToast, +}: GraphRAGKbPanelProps) { + const L = STR[locale]; + const notebookId = notebook?.id || ''; + const notebookTitle = notebook?.title || notebook?.name || ''; + + const [persist, setPersist] = useState(null); + const [forceReindex, setForceReindex] = useState(false); + const [parsePdfs, setParsePdfs] = useState(true); + const [indexLoading, setIndexLoading] = useState(false); + const [modelName, setModelName] = useState(defaultGraphragModel()); + + const [messages, setMessages] = useState([]); + const [inputValue, setInputValue] = useState(''); + const [chatLoading, setChatLoading] = useState(false); + const [searchMethod, setSearchMethod] = useState<'auto' | 'local' | 'global'>('auto'); + const [wikidataEnrich, setWikidataEnrich] = useState(true); + const messagesEndRef = useRef(null); + + useEffect(() => { + try { + const v = localStorage.getItem('graphrag_wikidata_enrich'); + if (v !== null) setWikidataEnrich(v === '1' || v === 'true'); + } catch { + // keep default + } + }, []); + + const [mergeA, setMergeA] = useState(''); + const [mergeB, setMergeB] = useState(''); + const [mergeDedupe, setMergeDedupe] = useState(false); + const [mergeLoading, setMergeLoading] = useState(false); + + const storageKey = useMemo(() => { + const uid = userId || 'global'; + if (!notebookId) return null; + return getWorkspaceStorageKey(uid, notebookId); + }, [userId, notebookId]); + + const loadPersist = useCallback(() => { + if (!storageKey) { setPersist(null); return; } + try { + const raw = localStorage.getItem(storageKey); + if (!raw) { setPersist(null); return; } + const p = JSON.parse(raw) as GraphragWorkspacePersist; + if (p?.workspace_dir) setPersist(p); else setPersist(null); + } catch { setPersist(null); } + }, [storageKey]); + + useEffect(() => { loadPersist(); }, [loadPersist]); + useEffect(() => { if (persist?.workspace_dir) setMergeA((a) => (a ? a : persist.workspace_dir)); }, [persist?.workspace_dir]); + + const llmBody = useCallback(() => { + const settings = getApiSettings(userId); + const api_url = settings?.apiUrl?.trim() || ''; + const api_key = settings?.apiKey?.trim() || ''; + const model = modelName.trim() || defaultGraphragModel(); + return { api_url, api_key, model }; + }, [userId, modelName]); + + const copyText = async (text: string, okMsg?: string) => { + try { await navigator.clipboard.writeText(text); showToast(okMsg || L.copied, 'success'); } + catch { showToast(L.copyFailed, 'error'); } + }; + + useEffect(() => { + messagesEndRef.current?.scrollIntoView({ behavior: 'smooth' }); + }, [messages, chatLoading]); + + const handleIndex = async () => { + if (!notebookId) { showToast(L.noNotebook, 'warning'); return; } + const { api_url, api_key, model } = llmBody(); + if (!api_url || !api_key) { showToast(L.apiWarn, 'warning'); return; } + setIndexLoading(true); + try { + const res = await indexGraphragKb({ + notebook_id: notebookId, notebook_title: notebookTitle, email: email || '', + api_url, api_key, model, + source_stems: null, workspace_dir: persist?.workspace_dir || '', + force_reindex: forceReindex, parse_pdfs: parsePdfs, skip_kggen: true, + }); + const next: GraphragWorkspacePersist = { workspace_dir: res.workspace_dir, updatedAt: Date.now(), num_chunks: res.num_chunks }; + if (storageKey) localStorage.setItem(storageKey, JSON.stringify(next)); + setPersist(next); + showToast(L.indexOk, 'success'); + } catch (e: unknown) { + showToast(e instanceof Error ? e.message : String(e), 'error'); + } finally { setIndexLoading(false); } + }; + + const handleChat = async () => { + const userInput = inputValue.trim(); + if (!userInput || chatLoading) return; + if (!persist?.workspace_dir) { + showToast(locale === 'zh' ? '请先完成索引构建' : 'Build the index first', 'warning'); + return; + } + const { api_url, api_key, model } = llmBody(); + if (!api_url || !api_key) { showToast(L.apiWarn, 'warning'); return; } + + const history = messages.map((m) => ({ + role: m.role, + content: m.content, + ...(m.meta ? { meta: { ...m.meta } } : {}), + })); + const userMsg: ChatMessage = { id: `u_${Date.now()}`, role: 'user', content: userInput }; + setMessages((prev) => [...prev, userMsg]); + setInputValue(''); + setChatLoading(true); + + try { + const resp: ChatResponse = await chatGraphragKb({ + notebook_id: notebookId, notebook_title: notebookTitle, email: email || '', + query: userInput, history, search_method: searchMethod, + workspace_dir: persist.workspace_dir, api_url, api_key, model, + wikidata_enrich: wikidataEnrich, + defer_postprocess: true, + }); + const assistantId = `a_${Date.now()}`; + const assistantMsg: ChatMessage = { + id: assistantId, role: 'assistant', content: resp.answer, + meta: { + intent: resp.intent, + rewritten_query: resp.rewritten_query, + graphrag_raw_answer: resp.graphrag_raw_answer || '', + context_data: resp.context_data, + reasoning_subgraph: resp.reasoning_subgraph, + reasoning_subgraph_cot: resp.reasoning_subgraph_cot, + judge_score: resp.judge_score, + judge_rationale: resp.judge_rationale, + source_chunks: resp.source_chunks, + highlight_hints: resp.highlight_hints, + }, + postprocessPending: !!resp.postprocess_pending, + postprocessSubgraphPending: !!resp.postprocess_pending, + postprocessWikidataPending: !!resp.postprocess_pending && wikidataEnrich, + }; + setMessages((prev) => [...prev, assistantMsg]); + + if (resp.postprocess_pending) { + chatGraphragKbPostprocess({ + query: resp.rewritten_query || userInput, + answer: resp.graphrag_raw_answer || '', + reasoning_subgraph: (resp.reasoning_subgraph || []) as Array>, + api_url, + api_key, + model, + wikidata_enrich: wikidataEnrich, + mode: 'subgraph', + }) + .then((pp) => { + setMessages((prev) => prev.map((m) => { + if (m.id !== assistantId || m.role !== 'assistant') return m; + return { + ...m, + postprocessSubgraphPending: false, + postprocessPending: !!m.postprocessWikidataPending, + meta: { + ...(m.meta || {}), + reasoning_subgraph: pp.reasoning_subgraph, + reasoning_subgraph_cot: pp.reasoning_subgraph_cot, + judge_score: pp.judge_score, + judge_rationale: pp.judge_rationale, + }, + }; + })); + }) + .catch((e) => { + setMessages((prev) => prev.map((m) => { + if (m.id !== assistantId || m.role !== 'assistant') return m; + return { + ...m, + postprocessSubgraphPending: false, + postprocessPending: !!m.postprocessWikidataPending, + meta: { + ...(m.meta || {}), + judge_rationale: String(e), + }, + }; + })); + }); + + if (wikidataEnrich) { + chatGraphragKbPostprocess({ + query: resp.rewritten_query || userInput, + answer: resp.graphrag_raw_answer || '', + reasoning_subgraph: (resp.reasoning_subgraph || []) as Array>, + api_url, + api_key, + model, + wikidata_enrich: true, + mode: 'wikidata', + }) + .then((pp) => { + setMessages((prev) => prev.map((m) => { + if (m.id !== assistantId || m.role !== 'assistant') return m; + const appendix = (pp.wikidata_appendix || '').trim(); + const base = m.content || ''; + const nextContent = appendix ? `${base}\n\n${appendix}` : base; + return { + ...m, + content: nextContent, + postprocessWikidataPending: false, + postprocessPending: !!m.postprocessSubgraphPending, + }; + })); + }) + .catch(() => { + setMessages((prev) => prev.map((m) => { + if (m.id !== assistantId || m.role !== 'assistant') return m; + return { + ...m, + postprocessWikidataPending: false, + postprocessPending: !!m.postprocessSubgraphPending, + }; + })); + }); + } + } + } catch (err) { + showToast(String(err), 'error'); + setMessages((prev) => prev.slice(0, -1)); + setInputValue(userInput); + } finally { setChatLoading(false); } + }; + + const handleMerge = async () => { + if (!notebookId) { showToast(L.noNotebook, 'warning'); return; } + const a = mergeA.trim(); const b = mergeB.trim(); + if (!a || !b) { showToast(locale === 'zh' ? '请填写两个 workspace 路径' : 'Enter both workspace paths', 'warning'); return; } + const { api_url, api_key, model } = llmBody(); + if (!api_url || !api_key) { showToast(L.apiWarn, 'warning'); return; } + setMergeLoading(true); + try { + const res = await mergeGraphragKb({ + notebook_id: notebookId, notebook_title: notebookTitle, email: email || '', + api_url, api_key, model, workspace_dir_a: a, workspace_dir_b: b, dedupe: mergeDedupe, + }); + const next: GraphragWorkspacePersist = { workspace_dir: res.merged_workspace_dir, updatedAt: Date.now(), num_chunks: res.num_chunks }; + if (storageKey) localStorage.setItem(storageKey, JSON.stringify(next)); + setPersist(next); + setMergeA(res.merged_workspace_dir); + showToast(L.mergeOk, 'success'); + } catch (e: unknown) { + showToast(e instanceof Error ? e.message : String(e), 'error'); + } finally { setMergeLoading(false); } + }; + + return ( +
+
+ +
+
{L.headerTitle}
+
{L.headerSub}
+
+
+ +
+
+

{L.indexBtn}

+
+ + +
+
+ + setModelName(e.target.value)} + className="w-full max-w-md px-3 py-2 border border-ios-gray-200 rounded-lg text-sm" + placeholder={defaultGraphragModel()} + /> +
+ + + {persist && ( +
+
{L.summary}
+
+ {L.chunks}: {persist.num_chunks ?? '—'} +
+
+ {L.workspace}: + {persist.workspace_dir} + +
+
+ )} +
+ +
+
+ {L.searchMethodLabel} + + + +
+ +
+ {messages.length === 0 && ( +
+ {persist?.workspace_dir ? L.emptyReady : L.emptyNoIndex} +
+ )} + {messages.map((msg) => ( +
+
+ {msg.role === 'user' ? ( +
{msg.content}
+ ) : ( +
+
+ {msg.content || '—'} +
+ {msg.meta && ( + + )} + {msg.postprocessSubgraphPending ? ( +
+ + {L.postprocessSubgraphPending} +
+ ) : null} + {msg.postprocessWikidataPending ? ( +
+ + {L.postprocessWikidataPending} +
+ ) : null} +
+ )} +
+
+ ))} + {chatLoading && ( +
+
+ +
+
+ )} +
+
+ +
+