JFetcher v4.2.0

功能变动：

- 优化数据库连接池配置
- 升级到 Ruff v0.11.0，并应用最新静态检查规则
- 调整部署配置
- 移除未被实际使用的数据库连接池准备代码
- 更新依赖库

错误修复：

- 修复采集简书资产排行榜数据工作流断点续采异常的问题

Author: FHU-yezi

docker-compose.yml

@@ -6,7 +6,7 @@ networks:
 
 services:
   main:
-    image: jfetcher:4.1.0
+    image: jfetcher:4.2.0
     container_name: jfetcher
     build: .
     volumes:
@@ -20,7 +20,7 @@ services:
     deploy:
       resources:
         limits:
-          memory: 1G
+          memory: 768M
       restart_policy:
         condition: on-failure
         delay: 5s

flows/beijiaoyi/fetch_ftn_market_orders_data.py

@@ -18,7 +18,7 @@ def get_fetch_time() -> datetime:
     time = datetime.now().replace(second=0, microsecond=0)
 
     # 12:01 -> timedelta = -1 minute
-    if time.minute % 10 < 5:  # noqa: PLR2004
+    if time.minute % 10 < 5:
         delta = timedelta(minutes=time.minute % 10)
         return time - delta
     # 11:57 -> timedelta = +3 minute

flows/jianshu/fetch_core_user_assets_data.py

@@ -26,7 +26,7 @@ def get_fetch_time() -> datetime:
     time = datetime.now().replace(second=0, microsecond=0)
 
     # 12:13 -> 12:00
-    if time.minute < 30:  # noqa: PLR2004
+    if time.minute < 30:
         return time.replace(minute=0)
     # 12:49 -> 13:00
     else:  # noqa: RET505

flows/jianshu/fetch_user_assets_ranking_data.py

@@ -62,12 +62,10 @@ async def iter_user_assets_ranking(
     start_ranking: int,
     total_count: int,
 ) -> AsyncGenerator[RecordData]:
-    current_count = 0
     async for item in UserAssetsRanking(start_ranking=start_ranking).iter_records():
         yield item
-        current_count += 1
 
-        if current_count == total_count:
+        if item.ranking == total_count:
             return
 
 
@@ -130,7 +128,7 @@ async def save_user_assets_ranking_record_data(
 
     try:
         assets_info: AssetsInfoData = await get_user_assets_info(item)
-    except Exception:
+    except ResourceUnavailableError:
         assets_info = None  # type: ignore
     else:
         if assets_info.ftn_amount is None and assets_info.assets_amount is None:

flows/jpep/fetch_ftn_market_orders_data.py

@@ -16,7 +16,7 @@ def get_fetch_time() -> datetime:
     time = datetime.now().replace(second=0, microsecond=0)
 
     # 12:01 -> timedelta = -1 minute
-    if time.minute % 10 < 5:  # noqa: PLR2004
+    if time.minute % 10 < 5:
         delta = timedelta(minutes=time.minute % 10)
         return time - delta
     # 11:57 -> timedelta = +3 minute

models/__init__.py

@@ -20,7 +20,7 @@ class FtnMarketRecord(Table, frozen=True):
     completed_trades_count: NonNegativeInt
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         fetch_time: datetime,

models/beijiaoyi/ftn_market_record.py

@@ -20,7 +20,7 @@ class FtnMarketSummaryRecord(Table, frozen=True):
     remaining_amount: int
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         fetch_time: datetime,

models/beijiaoyi/ftn_market_summary_record.py

@@ -19,7 +19,7 @@ class ArticleEarningRankingRecord(Table, frozen=True):
     voter_earning: PositiveFloat
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         date: date,

models/jianshu/article_earning_ranking_record.py

@@ -24,7 +24,7 @@ class User(Table, frozen=True):
     membership_expire_time: datetime | None
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         slug: str,

models/jianshu/user.py

@@ -18,7 +18,7 @@ class UserAssetsRankingRecord(Table, frozen=True):
     assets: NonNegativeFloat | None
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         date: date,

models/jianshu/user_assets_ranking_record.py

@@ -22,7 +22,7 @@ class UserEarningRankingRecord(Table, frozen=True):
     voting_earning: NonNegativeFloat
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         date: date,

models/jianshu/user_earning_ranking_record.py

@@ -19,7 +19,7 @@ class FtnMarketRecord(Table, frozen=True):
     completed_trades_count: NonNegativeInt
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         fetch_time: datetime,

models/jpep/ftn_market_record.py

@@ -20,7 +20,7 @@ class FtnMarketSummaryRecord(Table, frozen=True):
     remaining_amount: int
 
     @classmethod
-    async def create(  # noqa: PLR0913
+    async def create(
         cls,
         *,
         fetch_time: datetime,

models/jpep/ftn_market_summary_record.py

@@ -1,6 +1,6 @@
 [project]
 name = "jfetcher"
-version = "4.1.0"
+version = "4.2.0"
 description = "简书数据采集工具"
 readme = "README.md"
 license = { file = "LICENSE" }
@@ -13,7 +13,7 @@ dependencies = [
 ]
 
 [tool.uv]
-dev-dependencies = ["pyright>=1.1.0", "ruff>=0.9.0"]
+dev-dependencies = ["pyright>=1.1.0", "ruff>=0.11.0"]
 
 [tool.ruff]
 target-version = "py39"
@@ -23,16 +23,19 @@ lint.select = [
     "ARG",
     "ASYNC",
     "B",
+    "BLE",
     "C4",
     "E",
     "F",
     "FA",
     "FBT",
+    "FLY",
     "FURB",
     "I",
     "ISC",
     "N",
     "PERF",
+    "PGH",
     "PIE",
     "PL",
     "PT",
@@ -43,21 +46,19 @@ lint.select = [
     "RUF",
     "S",
     "SIM",
-    "SLF",
-    "T20",
-    "TID",
     "TRY",
     "UP",
     "W",
 ]
 lint.ignore = [
-    "A005",
     "PERF203",
+    "PGH003",
+    "PLR0913",
+    "PLR2004",
     "RUF001",
     "RUF002",
     "RUF003",
     "S311",
     "TRY003",
 ]
-
-lint.flake8-builtins.builtins-ignorelist = ["id", "range", "type"]
+lint.flake8-builtins.builtins-ignorelist = ["id", "range", "type"]

pyproject.toml

@@ -1,12 +1,12 @@
 aiosqlite==0.21.0
 alembic==1.15.1
 annotated-types==0.7.0
-anyio==4.8.0
+anyio==4.9.0
 apprise==1.9.2
 asgi-lifespan==2.1.0
 async-timeout==5.0.1 ; python_full_version < '3.11'
 asyncpg==0.30.0
-attrs==25.2.0
+attrs==25.3.0
 cachetools==5.5.2
 certifi==2025.1.31
 cffi==1.17.1 ; platform_python_implementation != 'PyPy'
@@ -24,7 +24,7 @@ fastapi==0.115.11
 fsspec==2025.3.0
 graphviz==0.20.3
 greenlet==3.1.1
-griffe==1.6.0
+griffe==1.6.2
 h11==0.14.0
 h2==4.2.0
 hpack==4.1.0
@@ -33,7 +33,7 @@ httpx==0.28.1
 humanize==4.12.1
 hyperframe==6.1.0
 idna==3.10
-importlib-metadata==8.5.0
+importlib-metadata==8.6.1
 jinja2==3.1.6
 jinja2-humanize-extension==0.4.0
 jkit==3.0.0b5
@@ -49,28 +49,28 @@ mdurl==0.1.2
 msgspec==0.19.0
 nodeenv==1.9.1
 oauthlib==3.2.2
-opentelemetry-api==1.30.0
+opentelemetry-api==1.31.1
 orjson==3.10.15
 packaging==24.2
 pathspec==0.12.1
 pendulum==3.0.0
-prefect==3.2.12
+prefect==3.2.14
 prometheus-client==0.21.1
-psycopg==3.2.5
-psycopg-binary==3.2.5 ; implementation_name != 'pypy'
+psycopg==3.2.6
+psycopg-binary==3.2.6 ; implementation_name != 'pypy'
 pycparser==2.22 ; platform_python_implementation != 'PyPy'
 pydantic==2.10.6
 pydantic-core==2.27.2
 pydantic-extra-types==2.10.3
 pydantic-settings==2.8.1
 pygments==2.19.1
-pyright==1.1.396
+pyright==1.1.397
 python-dateutil==2.9.0.post0
 python-dotenv==1.0.1
 python-slugify==8.0.4
 python-socks==2.7.1
 pytz==2025.1
-pywin32==309 ; sys_platform == 'win32'
+pywin32==310 ; sys_platform == 'win32'
 pyyaml==6.0.2
 readchar==4.2.1
 referencing==0.36.2
@@ -82,7 +82,7 @@ rich==13.9.4
 rpds-py==0.23.1
 ruamel-yaml==0.18.10
 ruamel-yaml-clib==0.2.12 ; python_full_version < '3.13' and platform_python_implementation == 'CPython'
-ruff==0.9.10
+ruff==0.11.2
 shellingham==1.5.4
 six==1.17.0
 sniffio==1.3.1
@@ -99,7 +99,7 @@ tzdata==2025.1
 tzlocal==5.3.1
 ujson==5.10.0
 urllib3==2.3.0
-uv==0.6.6
+uv==0.6.9
 uvicorn==0.34.0
 websockets==15.0.1
 wrapt==1.17.2

requirements-dev.txt

@@ -1,12 +1,12 @@
 aiosqlite==0.21.0
 alembic==1.15.1
 annotated-types==0.7.0
-anyio==4.8.0
+anyio==4.9.0
 apprise==1.9.2
 asgi-lifespan==2.1.0
 async-timeout==5.0.1 ; python_full_version < '3.11'
 asyncpg==0.30.0
-attrs==25.2.0
+attrs==25.3.0
 cachetools==5.5.2
 certifi==2025.1.31
 cffi==1.17.1 ; platform_python_implementation != 'PyPy'
@@ -24,7 +24,7 @@ fastapi==0.115.11
 fsspec==2025.3.0
 graphviz==0.20.3
 greenlet==3.1.1
-griffe==1.6.0
+griffe==1.6.2
 h11==0.14.0
 h2==4.2.0
 hpack==4.1.0
@@ -33,7 +33,7 @@ httpx==0.28.1
 humanize==4.12.1
 hyperframe==6.1.0
 idna==3.10
-importlib-metadata==8.5.0
+importlib-metadata==8.6.1
 jinja2==3.1.6
 jinja2-humanize-extension==0.4.0
 jkit==3.0.0b5
@@ -48,15 +48,15 @@ markupsafe==3.0.2
 mdurl==0.1.2
 msgspec==0.19.0
 oauthlib==3.2.2
-opentelemetry-api==1.30.0
+opentelemetry-api==1.31.1
 orjson==3.10.15
 packaging==24.2
 pathspec==0.12.1
 pendulum==3.0.0
-prefect==3.2.12
+prefect==3.2.14
 prometheus-client==0.21.1
-psycopg==3.2.5
-psycopg-binary==3.2.5 ; implementation_name != 'pypy'
+psycopg==3.2.6
+psycopg-binary==3.2.6 ; implementation_name != 'pypy'
 pycparser==2.22 ; platform_python_implementation != 'PyPy'
 pydantic==2.10.6
 pydantic-core==2.27.2
@@ -68,7 +68,7 @@ python-dotenv==1.0.1
 python-slugify==8.0.4
 python-socks==2.7.1
 pytz==2025.1
-pywin32==309 ; sys_platform == 'win32'
+pywin32==310 ; sys_platform == 'win32'
 pyyaml==6.0.2
 readchar==4.2.1
 referencing==0.36.2
@@ -96,7 +96,7 @@ tzdata==2025.1
 tzlocal==5.3.1
 ujson==5.10.0
 urllib3==2.3.0
-uv==0.6.6
+uv==0.6.9
 uvicorn==0.34.0
 websockets==15.0.1
 wrapt==1.17.2

requirements.txt

@@ -6,19 +6,19 @@
 
 jianshu_pool = Pool(
     CONFIG.jianshu_postgres.connection_string,
-    min_size=1,
+    min_size=2,
     max_size=4,
     app_name="JFetcher",
 )
 jpep_pool = Pool(
     CONFIG.jpep_postgres.connection_string,
-    min_size=1,
-    max_size=4,
+    min_size=2,
+    max_size=2,
     app_name="JFetcher",
 )
 beijiaoyi_pool = Pool(
     CONFIG.beijiaoyi_postgres.connection_string,
-    min_size=1,
-    max_size=4,
+    min_size=2,
+    max_size=2,
     app_name="JFetcher",
 )