Merge branch 'feat/pdf-reader' of https://github.com/open-sciencelab/GraphGen into feat/pdf-reader

Author: ChenZiHong-Gavin

graphgen/generate.py

@@ -72,7 +72,8 @@ def main():
 
     graph_gen.search(search_config=config["search"])
 
-    graph_gen.quiz_and_judge(quiz_and_judge_config=config["quiz_and_judge"])
+    if config.get("quiz_and_judge", {}).get("enabled"):
+        graph_gen.quiz_and_judge(quiz_and_judge_config=config["quiz_and_judge"])
 
     # TODO: add data filtering step here in the future
     # graph_gen.filter(filter_config=config["filter"])

graphgen/graphgen.py

@@ -105,7 +105,7 @@ async def insert(self, read_config: Dict, split_config: Dict):
                 "content": doc["content"]
             }
             for doc in data
-            if doc["type"] == "text"
+            if doc.get("type", "text") == "text"
         }
         _add_doc_keys = await self.full_docs_storage.filter_keys(list(new_docs.keys()))
         new_docs = {k: v for k, v in new_docs.items() if k in _add_doc_keys}

tests/integration_tests/models/reader/test_mineru_parser.py

@@ -12,7 +12,6 @@ def test_check_bin():
 def test_parse_pdf():
     """Parse a real PDF and verify basic structure."""
     repo_root = Path(__file__).resolve().parents[4]
-    os.chdir(repo_root)
 
     sample_pdf = os.path.join(repo_root, "resources", "input_examples", "pdf_demo.pdf")
     parser = MinerUParser()