not stucked by failed downloading

dblp_crawler/__main__.py

@@ -41,7 +41,14 @@ def filter_publications_at_crawler(publications, year: int, keywords: Keywords):
 
 
 async def bfs_to_end(graph, limit: int = 0):
-    while max(*(await graph.bfs_once())) > 0 and (limit != 0):
+    while limit != 0:
+        remain_none, total_author_count, all_fail = await graph.bfs_once()
+        if all_fail:
+            logger.info("Downloading all failed, exit")
+            return
+        if (remain_none + total_author_count) <= 0:
+            logger.info("No more need downloaded, exit")
+            return
         logger.info("Still running......")
         limit -= 1
 

dblp_crawler/graph.py

@@ -57,8 +57,9 @@ def filter_publications_at_output(self, publications: Iterable[Publication]) ->
     async def download_person(self, pid: str) -> None:
         data = await download_person(pid)
         if data is None:
-            return
+            return False
         self.persons[pid] = DBLPPerson(data)
+        return True
 
     async def bfs_once(self) -> tuple[int, int]:
         if not self.journals_inited:
@@ -101,15 +102,18 @@ async def bfs_once(self) -> tuple[int, int]:
             total_author_count += author_count
             total_publication_count += publication_count
         logger.info("%d authors from %d publications is fetching" % (total_author_count, total_publication_count))
-        await asyncio.gather(*tasks)
+        success = await asyncio.gather(*tasks)
+        all_fail = len(success) <= 0 or True not in success
         logger.info("%d authors added from %d publications" % (total_author_count, total_publication_count))
+        if all_fail:
+            logger.warn("%d download_person all failed in this loop" % all_fail)
         remain_none = 0
         for person in self.persons.values():
             if person is None:
                 remain_none += 1
         logger.info("There are %d authors need init in next loop" % remain_none)
         logger.info("There are %d authors need publications fetching in next loop" % total_author_count)
-        return remain_none, total_author_count
+        return remain_none, total_author_count, all_fail
 
     @abc.abstractmethod
     def summarize_person(self, a: str, person: Optional[DBLPPerson]) -> None:  # 构建summary

setup.py

@@ -17,7 +17,7 @@
 
 setup(
     name='dblp_crawler',
-    version='1.8.8',
+    version='1.8.9',
     author='yindaheng98',
     author_email='[email protected]',
     url='https://github.com/yindaheng98/dblp-crawler',