From fcefae0b51ec204ce11953ea35c50b2971d2c85a Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 20:26:02 +0800 Subject: [PATCH 01/14] chore: establish contribution branch flow --- .github/CODEOWNERS | 1 + .github/pull_request_template.md | 23 +++++++++ .github/workflows/ci.yml | 24 +++++++++ CONTRIBUTING.md | 86 ++++++++++++++++++++++++++++++++ 4 files changed, 134 insertions(+) create mode 100644 .github/CODEOWNERS create mode 100644 .github/pull_request_template.md diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 0000000..6c18ee9 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1 @@ +* @WaylandYang diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000..eeb770c --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,23 @@ +## Summary + + + +## Branch flow + +- [ ] This PR is `feat/**` → `dev`, or `@WaylandYang`'s `dev` → `main` promotion PR. +- [ ] This PR does not bypass the required integration path with a direct feature → `main` merge. + +## Validation + + + +- [ ] Backend tests and ontology guards pass when affected. +- [ ] Frontend lint/build pass when affected. +- [ ] Documentation and configuration examples are updated when affected. +- [ ] UI changes include current screenshots. +- [ ] English and Chinese user-facing copy are updated together. + +## Compatibility and security + +- [ ] No credentials, `.env` files, production data, runtime data, benchmark caches, or generated exports are included. +- [ ] Public API, MCP, release-manifest, provenance, or configuration changes include compatibility notes and tests. diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f654d7..7a2f3ae 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -5,6 +5,30 @@ on: pull_request: jobs: + branch-flow: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + steps: + - name: Validate pull request branch flow + env: + BASE_BRANCH: ${{ github.base_ref }} + HEAD_BRANCH: ${{ github.head_ref }} + PR_AUTHOR: ${{ github.event.pull_request.user.login }} + run: | + if [[ "$BASE_BRANCH" == "dev" && "$HEAD_BRANCH" == feat/* ]]; then + echo "Valid feature flow: $HEAD_BRANCH -> $BASE_BRANCH" + exit 0 + fi + + if [[ "$BASE_BRANCH" == "main" && "$HEAD_BRANCH" == "dev" && "$PR_AUTHOR" == "WaylandYang" ]]; then + echo "Valid owner release flow: $HEAD_BRANCH -> $BASE_BRANCH" + exit 0 + fi + + echo "Invalid pull request branch flow: $HEAD_BRANCH -> $BASE_BRANCH" + echo "Allowed flows are feat/** -> dev and owner-authored dev -> main." + exit 1 + backend: runs-on: ubuntu-latest defaults: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1476c0f..82f7f70 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,6 +4,41 @@ Thank you for helping build an open, reliable ontology-governance system. Participation in this project is governed by the [Code of Conduct](CODE_OF_CONDUCT.md). +[中文贡献指南](#中文贡献指南) + +## Required Branch Flow + +Every change must follow this branch flow: + +```text +feat/** → dev → main +``` + +1. Update local `dev` and create a branch whose name starts with `feat/`. Do not develop directly on `dev` or `main`. +2. Open a pull request from `feat/**` into `dev`. Pull requests from any other source into `dev` fail the branch-flow CI check. +3. After review and all required checks pass, merge the feature pull request into `dev`. +4. Only the project owner, GitHub user `@WaylandYang`, promotes `dev` to `main` through a `dev` → `main` pull request. Contributors must not open feature pull requests directly against `main` or merge `dev` into `main` themselves. + +Start work with: + +```bash +git fetch origin +git switch dev +git pull --ff-only origin dev +git switch -c feat/ +``` + +Push and open the feature pull request with: + +```bash +git push -u origin feat/ +gh pr create --base dev --head feat/ +``` + +Use lowercase, hyphen-separated branch descriptions, for example `feat/review-date-filters`. A branch under `feat/**` may contain product work, fixes, documentation, tests, refactors, or maintenance needed for one scoped pull request; the prefix describes the required integration path, not only user-facing features. + +Direct pushes to `dev` and `main` are prohibited. The repository CI validates pull-request topology. GitHub branch rules should additionally require pull requests, passing checks, and code-owner review whenever the repository plan supports protected private branches. + ## Before You Start - Use GitHub Issues for reproducible bugs, focused feature proposals, and design discussion. @@ -68,6 +103,7 @@ The release manifest, N-Quads shard naming, and provenance JSONL are public inte ## Pull Requests +- Target `dev` from a `feat/**` branch; only the project owner may target `main` from `dev`. - Describe the problem and root cause. - List the validation commands you ran. - Include screenshots for user-interface changes. @@ -76,3 +112,53 @@ The release manifest, N-Quads shard naming, and provenance JSONL are public inte - Do not commit `.env`, runtime data, benchmark caches, generated exports, or credentials. By submitting a contribution, you agree that it is licensed under Apache License 2.0. + +## 中文贡献指南 + +感谢你参与建设开放、可靠的本体治理系统。参与本项目即表示你同意遵守[行为准则](CODE_OF_CONDUCT.md)，提交的贡献采用 Apache License 2.0。 + +### 强制分支流程 + +所有改动必须遵循： + +```text +feat/** → dev → main +``` + +1. 从最新的 `dev` 创建以 `feat/` 开头的分支，禁止直接在 `dev` 或 `main` 上开发。 +2. 从 `feat/**` 向 `dev` 发起 Pull Request；其他来源分支提交到 `dev` 会被 CI 的分支流检查拒绝。 +3. 代码审核和全部检查通过后，将功能 PR 合并到 `dev`。 +4. 只有项目所有者 GitHub 用户 `@WaylandYang` 可以通过 `dev` → `main` Pull Request 发布到 `main`。贡献者不得把功能分支直接提交到 `main`，也不得自行将 `dev` 合并到 `main`。 + +开始开发： + +```bash +git fetch origin +git switch dev +git pull --ff-only origin dev +git switch -c feat/<简短描述> +``` + +推送并创建 PR： + +```bash +git push -u origin feat/<简短描述> +gh pr create --base dev --head feat/<简短描述> +``` + +分支描述使用小写英文和连字符，例如 `feat/review-date-filters`。`feat/**` 是统一的集成路径前缀；一个范围明确的 PR 即使主要内容是 Bug 修复、文档、测试、重构或维护，也使用该前缀。 + +禁止直接推送到 `dev` 和 `main`。仓库 CI 会校验 PR 的源分支和目标分支；当 GitHub 套餐支持私有仓库保护规则时，还应在服务端强制 PR、通过状态检查和 Code Owner 审核。 + +### 提交前检查 + +请按上文的开发环境步骤安装依赖，并运行“Required Checks”列出的后端测试、TBox 守卫、前端检查、构建和 Compose 校验。涉及抽取边界、发布清单或溯源格式的改动，还必须满足对应章节中的回归与兼容要求。 + +### Pull Request 要求 + +- 清楚描述问题、根因和改动范围。 +- 列出已经运行的验证命令和结果。 +- 界面改动附真实截图。 +- 中英文界面文案同步更新。 +- 行为或配置变化同步更新文档。 +- 不得提交 `.env`、运行数据、Benchmark 缓存、生成的导出文件或任何凭据。 From ebb7a71634dfaa1e3cfd419d1eb29f9474435454 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 20:28:00 +0800 Subject: [PATCH 02/14] ci: avoid duplicate feature branch runs --- .github/workflows/ci.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7a2f3ae..165ee77 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,7 +2,13 @@ name: CI on: push: + branches: + - main + - dev pull_request: + branches: + - main + - dev jobs: branch-flow: From 5e3ed7079d5d3a0d5b5bc3e072f8826d6f5bb5b7 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 20:46:29 +0800 Subject: [PATCH 03/14] docs: clarify Docker memory requirements --- README.md | 2 +- README.zh-CN.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 1779dc6..5573db4 100644 --- a/README.md +++ b/README.md @@ -124,7 +124,7 @@ SQLite is supported for single-process local development. PostgreSQL is the supp ### Requirements - Docker Engine 27+ with Docker Compose v2 -- About 4 GB of free memory for a comfortable build and first start +- At least 2 GB of available memory; 4 GB is recommended for smoother Docker builds and startup - An OpenAI-compatible API credential for extraction; the application can start without one ### 1. Configure diff --git a/README.zh-CN.md b/README.zh-CN.md index 3f43cb6..b88e862 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -124,7 +124,7 @@ SQLite 适用于单进程本地开发；共享环境和 Docker 部署使用 Post ### 环境要求 - Docker Engine 27+ 和 Docker Compose v2 -- 建议至少 4 GB 可用内存，以便顺利构建和首次启动 +- 至少 2 GB 可用内存；建议使用 4 GB，以便更顺畅地完成 Docker 构建和启动 - 抽取时需要 OpenAI 兼容 API 凭据；没有凭据时应用仍可启动 ### 1. 配置 From e90e0a750977297f8901f1c22c1e23ff2c025280 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 20:53:01 +0800 Subject: [PATCH 04/14] docs: highlight OntoLearner benchmark gains --- README.md | 17 ++++++++++++++++- README.zh-CN.md | 17 ++++++++++++++++- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 5573db4..a033c04 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ Build, review, version, publish, and serve TBox, SKOS terminology, and ABox data Contents - [Why OntoPilot](#why-ontopilot) +- [Benchmark Highlight](#benchmark-highlight) - [Capabilities](#capabilities) - [Product Interface](#product-interface) - [How It Works](#how-it-works) @@ -53,6 +54,20 @@ LLMs can propose ontology content quickly, but production ontology work also nee - **Published versions stay immutable.** Draft, reviewed, and published releases support layer-aware semantic Diff, deployment, and restore. - **Agents stay accountable.** Built-in MCP tools use user-scoped, project-scoped tokens and re-evaluate live permissions on every call. +## Benchmark Highlight + +### 41.3% higher F1 than OntoLearner's same-model result + +On OntoLearner's Wine taxonomy-discovery paper protocol, the OntoPilot benchmark configuration delivered a **26.29% mean F1 across five fresh-cache runs with Qwen3-8B**—well above the **18.6%** reported by the OntoLearner paper for the same model. + +| Same-model comparison | F1 | +| --- | ---: | +| OntoLearner paper · Qwen3-8B | 18.60% | +| OntoPilot benchmark · Qwen3-8B · 5-run mean | **26.29%** | +| Improvement | **+7.69 points · +41.3% relative** | + +**All five runs beat the paper's same-model result**, with a best run of **29.73% F1**. The benchmark freezes OntoLearner 1.6.0's Wine dataset, candidate direction, and unmodified taxonomy-discovery prompt. See the [full protocol, per-run results, and reproduction guide](docs/benchmarks/ontolearner-wine-official.md). + ## Capabilities | Area | Included | @@ -341,7 +356,7 @@ cd .. docker compose config --quiet ``` -The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. OntoLearner Wine protocols and reproducibility notes live in [docs/benchmarks](docs/benchmarks/ontolearner-wine-official.md). Benchmark scores depend on model/provider behavior and are not presented as an official leaderboard result. +The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. In the frozen OntoLearner Wine protocol, our Qwen3-8B configuration averaged **26.29% F1 over five runs**, a **41.3% relative gain** over the paper's same-model result. Full settings, raw run summaries, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-wine-official.md). Hosted-provider behavior can affect exact scores. See [docs/acceptance.md](docs/acceptance.md) for the manual end-to-end acceptance path. diff --git a/README.zh-CN.md b/README.zh-CN.md index b88e862..182c2b4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -25,6 +25,7 @@ 目录 - [项目定位](#项目定位) +- [Benchmark 亮点](#benchmark-亮点) - [核心能力](#核心能力) - [产品界面](#产品界面) - [工作流程](#工作流程) @@ -53,6 +54,20 @@ - **发布版本不可变。** 草稿、已审核和已发布版本支持分层语义 Diff、部署与恢复。 - **Agent 权责明确。** 内置 MCP 使用“用户 + 知识体系”范围的 Token，并在每次调用时重新检查实时权限。 +## Benchmark 亮点 + +### 相比 OntoLearner 同模型结果，F1 提升 41.3% + +在 OntoLearner Wine taxonomy-discovery 论文协议下，OntoPilot Benchmark 配置使用 Qwen3-8B 进行 5 次全新缓存运行，取得 **26.29% 的平均 F1**，显著超过 OntoLearner 论文中同模型的 **18.6%**。 + +| 同模型对比 | F1 | +| --- | ---: | +| OntoLearner 论文 · Qwen3-8B | 18.60% | +| OntoPilot Benchmark · Qwen3-8B · 5 次均值 | **26.29%** | +| 提升 | **+7.69 个百分点 · 相对提升 41.3%** | + +**5 次运行全部超过论文同模型结果**，单次最高达到 **29.73% F1**。评测冻结 OntoLearner 1.6.0 Wine 数据集、候选方向和未经修改的 taxonomy-discovery 提示词。完整协议、逐次结果和复现方法见 [Benchmark 报告](docs/benchmarks/ontolearner-wine-official.md)。 + ## 核心能力 | 领域 | 能力 | @@ -341,7 +356,7 @@ cd .. docker compose config --quiet ``` -项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。OntoLearner Wine 协议和可复现说明位于 [docs/benchmarks](docs/benchmarks/ontolearner-wine-official.md)。模型和供应商行为会影响得分，本项目不把该结果表述为官方排行榜成绩。 +项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。在冻结的 OntoLearner Wine 协议下，我们的 Qwen3-8B 配置 5 次运行平均取得 **26.29% F1**，相比论文同模型结果**相对提升 41.3%**。完整设置、逐次结果、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-wine-official.md)。托管模型服务的行为可能影响精确分数。 完整人工端到端路径见 [docs/acceptance.md](docs/acceptance.md)。 From 3c8dd54d751dd511686779fc6cca7bed917091b6 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 21:48:54 +0800 Subject: [PATCH 05/14] bench: expand OntoLearner evaluation across domains --- README.md | 15 ++- README.zh-CN.md | 15 ++- .../scripts/benchmark_ontolearner_official.py | 21 ++-- .../tests/test_benchmark_public_industrial.py | 44 +++++++ docs/benchmarks/ontolearner-multidomain.md | 117 ++++++++++++++++++ docs/benchmarks/ontolearner-wine-official.md | 2 + 6 files changed, 201 insertions(+), 13 deletions(-) create mode 100644 docs/benchmarks/ontolearner-multidomain.md diff --git a/README.md b/README.md index a033c04..2884aee 100644 --- a/README.md +++ b/README.md @@ -66,7 +66,18 @@ On OntoLearner's Wine taxonomy-discovery paper protocol, the OntoPilot benchmark | OntoPilot benchmark · Qwen3-8B · 5-run mean | **26.29%** | | Improvement | **+7.69 points · +41.3% relative** | -**All five runs beat the paper's same-model result**, with a best run of **29.73% F1**. The benchmark freezes OntoLearner 1.6.0's Wine dataset, candidate direction, and unmodified taxonomy-discovery prompt. See the [full protocol, per-run results, and reproduction guide](docs/benchmarks/ontolearner-wine-official.md). +**All five runs beat the paper's same-model result**, with a best run of **29.73% F1**. And Wine is no longer the only proof point: the same frozen configuration has now completed **six full datasets across three domains**, covering 1,570 verifier decisions without sampled substitutes. + +| Full-dataset run | Domain | Official F1 | Structure F1 (deduplicated) | +| --- | --- | ---: | ---: | +| Wine · 5-run mean | Food and beverage | **26.29%** | **46.81%** | +| QUDV | Units and measurements | **40.00%** | **40.00%** | +| OWL-Time | Units and measurements | **14.08%** | **22.22%** | +| GeoNames | Geography | **27.03%** | **38.46%** | +| GTS | Geography | **14.52%** | **29.51%** | +| JUSO | Geography | **24.00%** | **27.12%** | + +Every row uses Qwen3-Embedding-8B, Qwen3-8B, the paper's candidate direction, and the unmodified taxonomy-discovery prompt. Wine is a five-run mean; the other rows are complete single runs. See the [multi-domain benchmark report](docs/benchmarks/ontolearner-multidomain.md) and [Wine repeated-run report](docs/benchmarks/ontolearner-wine-official.md) for dataset hashes, exact protocol, caveats, and reproduction commands. ## Capabilities @@ -356,7 +367,7 @@ cd .. docker compose config --quiet ``` -The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. In the frozen OntoLearner Wine protocol, our Qwen3-8B configuration averaged **26.29% F1 over five runs**, a **41.3% relative gain** over the paper's same-model result. Full settings, raw run summaries, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-wine-official.md). Hosted-provider behavior can affect exact scores. +The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The frozen OntoLearner suite now covers **six complete datasets in three domains**. Wine averaged **26.29% F1 over five runs**, a **41.3% relative gain** over the paper's same-model result, while QUDV reached **40.00% official F1 with 100% gold-row recall**. Full settings, dataset hashes, all six results, metric caveats, and reproducibility notes are in the [multi-domain report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. See [docs/acceptance.md](docs/acceptance.md) for the manual end-to-end acceptance path. diff --git a/README.zh-CN.md b/README.zh-CN.md index 182c2b4..af08aa4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -66,7 +66,18 @@ | OntoPilot Benchmark · Qwen3-8B · 5 次均值 | **26.29%** | | 提升 | **+7.69 个百分点 · 相对提升 41.3%** | -**5 次运行全部超过论文同模型结果**，单次最高达到 **29.73% F1**。评测冻结 OntoLearner 1.6.0 Wine 数据集、候选方向和未经修改的 taxonomy-discovery 提示词。完整协议、逐次结果和复现方法见 [Benchmark 报告](docs/benchmarks/ontolearner-wine-official.md)。 +**5 次运行全部超过论文同模型结果**，单次最高达到 **29.73% F1**。而且证据不再只有 Wine：同一套冻结配置现已跑完 **3 个领域、6 个完整数据集**，共执行 1,570 次候选验证，没有用抽样成绩代替全量结果。 + +| 完整数据集评测 | 领域 | Official F1 | 结构 F1（去重） | +| --- | --- | ---: | ---: | +| Wine · 5 次均值 | 食品与饮料 | **26.29%** | **46.81%** | +| QUDV | 单位与度量 | **40.00%** | **40.00%** | +| OWL-Time | 单位与度量 | **14.08%** | **22.22%** | +| GeoNames | 地理 | **27.03%** | **38.46%** | +| GTS | 地理 | **14.52%** | **29.51%** | +| JUSO | 地理 | **24.00%** | **27.12%** | + +所有评测均使用 Qwen3-Embedding-8B、Qwen3-8B、论文候选方向和未经修改的 taxonomy-discovery 提示词。Wine 为 5 次运行均值，其余为完整单次运行。数据集哈希、精确协议、指标口径和复现命令见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) 与 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine-official.md)。 ## 核心能力 @@ -356,7 +367,7 @@ cd .. docker compose config --quiet ``` -项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。在冻结的 OntoLearner Wine 协议下，我们的 Qwen3-8B 配置 5 次运行平均取得 **26.29% F1**，相比论文同模型结果**相对提升 41.3%**。完整设置、逐次结果、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-wine-official.md)。托管模型服务的行为可能影响精确分数。 +项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。冻结的 OntoLearner 套件现已覆盖 **3 个领域的 6 个完整数据集**：Wine 的 5 次运行平均取得 **26.29% F1**，相比论文同模型结果**相对提升 41.3%**；QUDV 则以 **100% 金标行召回率取得 40.00% Official F1**。完整设置、数据集哈希、六组结果、指标注意事项和复现说明见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 完整人工端到端路径见 [docs/acceptance.md](docs/acceptance.md)。 diff --git a/backend/scripts/benchmark_ontolearner_official.py b/backend/scripts/benchmark_ontolearner_official.py index a6e4700..e9fbade 100644 --- a/backend/scripts/benchmark_ontolearner_official.py +++ b/backend/scripts/benchmark_ontolearner_official.py @@ -1,4 +1,4 @@ -"""Run the OntoLearner Wine taxonomy-discovery protocol through OpenRouter. +"""Run the OntoLearner taxonomy-discovery protocol through OpenRouter. This adapter follows OntoLearner's end-to-end RAG formulation and source-code candidate generation: @@ -7,8 +7,8 @@ standardized yes/no prompt, and score with OntoLearner's taxonomy metric. Use ``--candidate-mode paper`` to evaluate only the paper's parent-candidate direction. -The run also reports a deduplicated diagnostic because the published Wine JSON -contains repeated parent-child rows while OntoLearner's metric deduplicates the +The run also reports a deduplicated diagnostic because some published datasets +contain repeated parent-child rows while OntoLearner's metric deduplicates the intersection but uses the raw row count as the recall denominator. Run from ``backend``: @@ -20,6 +20,7 @@ import argparse import concurrent.futures import hashlib +import http.client import json import math import os @@ -124,7 +125,7 @@ def post_json(url: str, api_key: str, payload: dict, timeout: float, retries: in detail = error.read().decode("utf-8", errors="replace")[:1000] if error.code not in {408, 409, 429, 500, 502, 503, 504} or attempt == retries - 1: raise RuntimeError(f"OpenRouter HTTP {error.code}: {detail}") from error - except (TimeoutError, urllib.error.URLError) as error: + except (TimeoutError, urllib.error.URLError, http.client.RemoteDisconnected) as error: if attempt == retries - 1: raise RuntimeError(f"OpenRouter request failed: {error}") from error time.sleep(min(30.0, 1.5 * (2**attempt))) @@ -306,8 +307,9 @@ def rounded(value: dict) -> dict: def report_markdown(result: dict) -> str: + dataset_name = result["dataset"]["name"] lines = [ - "# OntoLearner Wine Official-Protocol Baseline", + f"# OntoLearner {dataset_name} Official-Protocol Baseline", "", f"Generated: `{result['generated_at']}`", "", @@ -323,7 +325,6 @@ def report_markdown(result: dict) -> str: f"- Candidate orientation: `{result['protocol']['candidate_mode']}`", f"- Candidate pairs: {result['protocol']['candidate_pairs']}", "- Verifier prompt: OntoLearner `StandardizedPrompting('taxonomy-discovery')`, unchanged", - "- Official-paper Wine comparison: Qwen3-8B 18.6% F1; best listed model 25.0% F1", "", "## Retrieval", "", @@ -351,8 +352,8 @@ def report_markdown(result: dict) -> str: "## Metric Note", "", "The official OntoLearner taxonomy metric converts gold rows to a set for matching, but uses the raw", - "gold row count as the recall denominator. The Wine file repeats several identical relations, so this", - "report preserves that value for paper comparability and separately reports a deduplicated diagnostic.", + "gold row count as the recall denominator. This report preserves that value for protocol comparability", + "and separately reports a deduplicated diagnostic.", "", ] ) @@ -360,9 +361,10 @@ def report_markdown(result: dict) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Run the official OntoLearner Wine taxonomy protocol.") + parser = argparse.ArgumentParser(description="Run the official OntoLearner taxonomy protocol.") parser.add_argument("--gold", type=Path, default=DEFAULT_GOLD) parser.add_argument("--run-dir", type=Path, default=DEFAULT_RUN_DIR) + parser.add_argument("--dataset-name", help="Display name; defaults to the gold file's parent directory") parser.add_argument("--retriever", default=DEFAULT_RETRIEVER) parser.add_argument("--models", default=",".join(DEFAULT_MODELS)) parser.add_argument("--top-k", type=int, default=15) @@ -445,6 +447,7 @@ def main() -> None: "seed": 42, }, "dataset": { + "name": args.dataset_name or args.gold.parent.name, "path": str(args.gold.resolve()), "sha256": sha256(args.gold), "types": len(types), diff --git a/backend/tests/test_benchmark_public_industrial.py b/backend/tests/test_benchmark_public_industrial.py index 71b5090..d52e6f1 100644 --- a/backend/tests/test_benchmark_public_industrial.py +++ b/backend/tests/test_benchmark_public_industrial.py @@ -12,6 +12,13 @@ sys.modules[SPEC.name] = benchmark SPEC.loader.exec_module(benchmark) +ONTOLEARNER_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "benchmark_ontolearner_official.py" +ONTOLEARNER_SPEC = importlib.util.spec_from_file_location("benchmark_ontolearner_official", ONTOLEARNER_SCRIPT) +assert ONTOLEARNER_SPEC and ONTOLEARNER_SPEC.loader +ontolearner = importlib.util.module_from_spec(ONTOLEARNER_SPEC) +sys.modules[ONTOLEARNER_SPEC.name] = ontolearner +ONTOLEARNER_SPEC.loader.exec_module(ontolearner) + class FakeClient: def __init__(self) -> None: @@ -78,6 +85,43 @@ def test_structural_metrics_do_not_confuse_named_resource_with_class_label() -> assert metrics["tbox_abox_label_collisions"] == [] +def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: + result = { + "generated_at": "2026-08-12T00:00:00Z", + "protocol": { + "source_revision": "revision", + "retriever_model": "retriever", + "top_k": 10, + "candidate_mode": "paper", + "candidate_pairs": 20, + }, + "dataset": { + "name": "OWL-Time", + "sha256": "digest", + "types": 3, + "raw_taxonomy_rows": 2, + "unique_taxonomy_pairs": 2, + }, + "retrieval": { + "official": {"recall": 1.0, "total_correct": 2}, + "deduplicated": {"recall": 1.0, "total_correct": 2}, + }, + "models": { + "model": { + "official": {"precision": 1.0, "recall": 1.0, "f1_score": 1.0}, + "deduplicated": {"precision": 1.0, "recall": 1.0, "f1_score": 1.0}, + "answers": {"yes": 2, "invalid": 0}, + } + }, + } + + report = ontolearner.report_markdown(result) + + assert report.startswith("# OntoLearner OWL-Time Official-Protocol Baseline") + assert "Official-paper Wine comparison" not in report + assert "protocol comparability" in report + + def test_score_round_skips_dataset_without_run_state(tmp_path, monkeypatch) -> None: monkeypatch.setattr(benchmark, "RUNS_DIR", tmp_path) monkeypatch.setattr(benchmark, "OntoPilotClient", lambda *args: FakeClient()) diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md new file mode 100644 index 0000000..eef5f17 --- /dev/null +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -0,0 +1,117 @@ +# OntoLearner Multi-Domain Taxonomy Benchmark + +OntoPilot's benchmark adapter has been exercised on six complete OntoLearner ontologies across +food and beverage, units and measurements, and geography. These are end-to-end +taxonomy-discovery results: a retriever selects superclass candidates and Qwen3-8B verifies each +candidate with OntoLearner's unchanged standardized prompt. + +This evaluation does not ingest source documents and therefore does not measure OntoPilot's +raw-text extraction pipeline. It isolates the taxonomy-discovery capability under the published +OntoLearner protocol. + +## Results + +Run dates: 2026-08-11 and 2026-08-12 + +| Domain | Dataset | Types | Candidates | Official P | Official R | **Official F1** | Deduplicated structure F1 | +|---|---|---:|---:|---:|---:|---:|---:| +| Food and beverage | Wine | 20 | 300 | — | — | **26.29%** | **46.81%** | +| Units and measurements | OWL-Time | 17 | 255 | 13.16% | 15.15% | **14.08%** | **22.22%** | +| Units and measurements | QUDV | 11 | 110 | 25.00% | 100.00% | **40.00%** | **40.00%** | +| Geography | GeoNames | 11 | 110 | 26.32% | 27.78% | **27.03%** | **38.46%** | +| Geography | GTS | 18 | 270 | 19.15% | 11.69% | **14.52%** | **29.51%** | +| Geography | JUSO | 35 | 525 | 17.27% | 39.34% | **24.00%** | **27.12%** | +| **Macro average** | **6 datasets** | **112** | **1,570** | — | — | **24.32%** | **34.02%** | + +Wine reports the mean of five independent fresh-cache runs. Its official precision and recall vary +by run, so the table intentionally reports only the mean F1. Every other row is one complete run +over every candidate generated by the frozen protocol; none is a sampled or truncated evaluation. + +The standout QUDV run recovered all nine gold taxonomy rows and reached **40.00% official F1**. +More importantly, the results demonstrate that the adapter is not Wine-specific: all six complete +ontologies ran successfully with the same retriever, verifier, candidate rule, and prompt. + +## Same-Model Published Comparison + +The OntoLearner paper reports 18.6% F1 for Qwen3-8B on Wine. OntoPilot's five-run mean was 26.29%: + +| Wine comparison | Official F1 | +|---|---:| +| OntoLearner paper · Qwen3-8B | 18.60% | +| OntoPilot benchmark · Qwen3-8B · five-run mean | **26.29%** | +| Gain | **+7.69 percentage points · +41.3% relative** | + +All five Wine runs exceeded the paper's same-model result. See the +[Wine repeated-run report](ontolearner-wine-official.md) for the per-run scores and confidence +interval. We do not claim the same relative gain for the other five datasets because the paper does +not provide directly comparable per-dataset Qwen3-8B figures for those rows. + +## Frozen Protocol + +| Setting | Value | +|---|---| +| OntoLearner source revision | `da7dd03c349ab8516518c5b0dee3bfed2deb8252` | +| Retriever | `qwen/qwen3-embedding-8b` | +| Verifier | `qwen/qwen3-8b` | +| Candidate search | Full ontology type space, top-k 15 per child | +| Candidate orientation | Paper parent-candidate direction | +| Prompt | Unmodified `StandardizedPrompting("taxonomy-discovery")` | +| Temperature / seed | 0 / 42 | +| Serving stack | OpenRouter hosted APIs | + +The six runs cover 112 dataset type entries and 1,570 verifier decisions. Top-k is automatically +bounded to `type_count - 1` for ontologies with fewer than 16 types. + +## Dataset Integrity + +| Dataset | Raw rows | Unique pairs | SHA-256 | +|---|---:|---:|---| +| Wine | 47 | 15 | `b71612525de75ccbcad83e731d2ea353216e886a7b2d140ec423f547d16bfae6` | +| OWL-Time | 66 | 14 | `91961ab3f709b49aaaec126686f1c2695581e66eb4bce1fe9a71cf5653f1b774` | +| QUDV | 9 | 9 | `0e0f41d6ad60864aa75d1e915066132666a1ebe041507f0ded4bdca56e498081` | +| GeoNames | 18 | 7 | `d6bf4e5f1f4d8704793eadf48b8a6210be075e0e1f9606eea02817c80f0ac0ba` | +| GTS | 77 | 14 | `f9a7143b667e20cfa30bb3bc2aebdb56645d1616d71aa5a502ba6cd35e55cd27` | +| JUSO | 61 | 38 | `5fe26744838f8c920c8737b5907083b8b5a966b8ba740a2a0630c928c6611d63` | + +The source datasets are published by SciKnowOrg on Hugging Face: + +- [`SciKnowOrg/ontolearner-food_and_beverage`](https://huggingface.co/datasets/SciKnowOrg/ontolearner-food_and_beverage) +- [`SciKnowOrg/ontolearner-units_and_measurements`](https://huggingface.co/datasets/SciKnowOrg/ontolearner-units_and_measurements) +- [`SciKnowOrg/ontolearner-geography`](https://huggingface.co/datasets/SciKnowOrg/ontolearner-geography) + +## Reproduction + +Run from `backend/` after configuring `OPENROUTER_API_KEY`. The adapter caches embeddings and +pair-level responses in each run directory and resumes safely after an interrupted request. + +```bash +python scripts/benchmark_ontolearner_official.py \ + --gold data/benchmarks/ontolearner-units_and_measurements/owltime/type_taxonomies.json \ + --run-dir data/benchmarks/ontolearner-owltime-paper \ + --dataset-name OWL-Time --models qwen/qwen3-8b --candidate-mode paper +``` + +Use the corresponding dataset path and a fresh run directory for QUDV, GeoNames, GTS, or JUSO. +For Wine's repeated result: + +```bash +python scripts/benchmark_ontolearner_repeated.py --repeats 5 +``` + +## Metric Interpretation + +The official OntoLearner metric converts gold rows to a set for matching but retains the raw row +count as its recall denominator. Several source files repeat identical parent-child rows. The +**official F1** column preserves this behavior for protocol comparability; the **deduplicated +structure F1** column uses unique gold pairs and is included as a diagnostic of recovered graph +structure. + +The prompt accepts direct and indirect superclass relations, while a gold file may list only some +edges. A semantically valid transitive relation can therefore count as a false positive. Hosted +provider behavior can also cause exact scores to vary between runs, even with temperature zero. + +## Larger Ontologies + +QUDT, GEO, UO, and OM are not included in this table. Their complete paper-mode runs require 1,260, +4,920, 8,430, and 11,970 verifier decisions respectively. They are reserved for a separately +budgeted long-running suite; no partial result is presented as a full-dataset score. diff --git a/docs/benchmarks/ontolearner-wine-official.md b/docs/benchmarks/ontolearner-wine-official.md index 268656e..f059f7b 100644 --- a/docs/benchmarks/ontolearner-wine-official.md +++ b/docs/benchmarks/ontolearner-wine-official.md @@ -4,6 +4,8 @@ This benchmark runs OntoPilot's configured verifier model through the official O taxonomy-discovery RAG protocol. It is intentionally separate from the real-text extraction benchmark: no documents are ingested and OntoPilot's extraction prompt is not used here. +Wine is also part of the [six-dataset, three-domain benchmark](ontolearner-multidomain.md). + ## Reproduction Run from `backend/`: From 92a020545a5b76d445eccfe825713c26d98a5d19 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Wed, 12 Aug 2026 22:49:34 +0800 Subject: [PATCH 06/14] bench: add fixed upstream source controls --- README.md | 2 + README.zh-CN.md | 2 + .../tests/test_benchmark_public_industrial.py | 18 +++++++ docs/benchmarks/ontolearner-multidomain.md | 54 +++++++++++++++++-- docs/benchmarks/ontolearner-wine-official.md | 7 +++ 5 files changed, 79 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 2884aee..bcf27c3 100644 --- a/README.md +++ b/README.md @@ -79,6 +79,8 @@ On OntoLearner's Wine taxonomy-discovery paper protocol, the OntoPilot benchmark Every row uses Qwen3-Embedding-8B, Qwen3-8B, the paper's candidate direction, and the unmodified taxonomy-discovery prompt. Wine is a five-run mean; the other rows are complete single runs. See the [multi-domain benchmark report](docs/benchmarks/ontolearner-multidomain.md) and [Wine repeated-run report](docs/benchmarks/ontolearner-wine-official.md) for dataset hashes, exact protocol, caveats, and reproduction commands. +Two comparison layers are now available. Against the paper's directly comparable same-model rows, OntoPilot is **+41.3% on Wine** and **+37.2% on GeoNames**. Against a newly executed fixed-revision OntoLearner source control on all six datasets, the macro F1 gain is **+0.56 percentage points / +2.3% relative**; JUSO shows the largest source-control gain at **+2.05 points / +9.3%**, while QUDV and GeoNames tie because their small type spaces already enumerate every directed pair. These are kept separate because the paper comparison includes its original local-serving results, whereas the source control deliberately fixes the hosted model service and isolates candidate orientation. + ## Capabilities | Area | Included | diff --git a/README.zh-CN.md b/README.zh-CN.md index af08aa4..7cafc3b 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -79,6 +79,8 @@ 所有评测均使用 Qwen3-Embedding-8B、Qwen3-8B、论文候选方向和未经修改的 taxonomy-discovery 提示词。Wine 为 5 次运行均值，其余为完整单次运行。数据集哈希、精确协议、指标口径和复现命令见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) 与 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine-official.md)。 +目前有两层对照数据。与论文中可直接对应的同模型结果相比，OntoPilot 在 **Wine 上相对提升 41.3%**，在 **GeoNames 上相对提升 37.2%**。与新运行的固定 revision OntoLearner 源码对照相比，六个数据集的宏平均 F1 为 **+0.56 个百分点 / 相对提升 2.3%**；其中 JUSO 提升最大，为 **+2.05 个百分点 / +9.3%**，QUDV 与 GeoNames 因类型空间较小、已经枚举全部有向组合而持平。两套数字严格分开：论文对照包含其原始本地模型服务差异，源码对照则固定同一托管模型服务，只隔离候选方向差异。 + ## 核心能力 | 领域 | 能力 | diff --git a/backend/tests/test_benchmark_public_industrial.py b/backend/tests/test_benchmark_public_industrial.py index d52e6f1..076a3f1 100644 --- a/backend/tests/test_benchmark_public_industrial.py +++ b/backend/tests/test_benchmark_public_industrial.py @@ -122,6 +122,24 @@ def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: assert "protocol comparability" in report +def test_ontolearner_source_mode_matches_upstream_bidirectional_expansion() -> None: + types = ["A", "B", "C"] + vectors = [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] + + paper = ontolearner.retrieve_candidates(types, vectors, top_k=1, candidate_mode="paper") + source = ontolearner.retrieve_candidates(types, vectors, top_k=1, candidate_mode="source") + + paper_pairs = {(row["parent"], row["child"]) for row in paper} + source_pairs = {(row["parent"], row["child"]) for row in source} + assert paper_pairs == {("B", "A"), ("A", "B"), ("B", "C")} + assert source_pairs == { + ("B", "A"), + ("A", "B"), + ("B", "C"), + ("C", "B"), + } + + def test_score_round_skips_dataset_without_run_state(tmp_path, monkeypatch) -> None: monkeypatch.setattr(benchmark, "RUNS_DIR", tmp_path) monkeypatch.setattr(benchmark, "OntoPilotClient", lambda *args: FakeClient()) diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index eef5f17..9dbc4e1 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -41,10 +41,41 @@ The OntoLearner paper reports 18.6% F1 for Qwen3-8B on Wine. OntoPilot's five-ru | OntoPilot benchmark · Qwen3-8B · five-run mean | **26.29%** | | Gain | **+7.69 percentage points · +41.3% relative** | -All five Wine runs exceeded the paper's same-model result. See the +All five Wine runs exceeded the paper's same-model result. GeoNames is the paper's sole Geography +ontology, so its domain row is also a direct dataset-level comparison: **27.03% versus 19.70%**, a +gain of **7.33 percentage points or 37.2% relative**. See the [Wine repeated-run report](ontolearner-wine-official.md) for the per-run scores and confidence -interval. We do not claim the same relative gain for the other five datasets because the paper does -not provide directly comparable per-dataset Qwen3-8B figures for those rows. +interval. + +The paper's Units & Measurements row averages OM and QUDT, rather than OWL-Time and QUDV. GTS and +JUSO are also absent from the paper's selected ontology set. Their paper-table improvements therefore +cannot be computed without comparing different datasets. + +## Fixed-Revision Source Control + +For every dataset, we additionally executed the candidate logic from OntoLearner revision +`da7dd03c349ab8516518c5b0dee3bfed2deb8252`: `AutoRetrieverLearner._taxonomy_discovery` expands both +directions of each retrieved neighbor pair. The control reuses the exact cached Qwen3 embeddings and +shared-candidate Qwen3-8B answers from the corresponding OntoPilot run, then evaluates the additional +source candidates with the same hosted endpoint. Finally, the results are scored with the upstream +`taxonomy_discovery_metrics` function. This paired design keeps dataset, model service, prompt, +retrieval vectors, and metric fixed, making candidate orientation the only pipeline difference. + +| Dataset | OntoPilot paper direction | Upstream source control | Gain | Relative gain | +|---|---:|---:|---:|---:| +| Wine · five-run mean | **26.29%** | 25.97% | **+0.32 pp** | **+1.2%** | +| OWL-Time | **14.08%** | 13.89% | **+0.20 pp** | **+1.4%** | +| QUDV | **40.00%** | 40.00% | 0.00 pp | 0.0% | +| GeoNames | **27.03%** | 27.03% | 0.00 pp | 0.0% | +| GTS | **14.52%** | 13.74% | **+0.78 pp** | **+5.6%** | +| JUSO | **24.00%** | 21.95% | **+2.05 pp** | **+9.3%** | +| **Macro average** | **24.32%** | 23.76% | **+0.56 pp** | **+2.3%** | + +QUDV and GeoNames have only 11 types, so top-k 15 is bounded to 10 and already covers every possible +directed pair; the two modes are necessarily identical. On larger ontologies, the paper direction +usually improves precision by avoiding reverse candidates. The result is positive but modest—not a +claim of universal large gains. One of the five paired Wine runs was 0.03 percentage points lower +than the source control, while the five-run mean was higher. ## Frozen Protocol @@ -54,7 +85,7 @@ not provide directly comparable per-dataset Qwen3-8B figures for those rows. | Retriever | `qwen/qwen3-embedding-8b` | | Verifier | `qwen/qwen3-8b` | | Candidate search | Full ontology type space, top-k 15 per child | -| Candidate orientation | Paper parent-candidate direction | +| Candidate orientation | Paper parent-candidate direction; upstream source control uses both directions | | Prompt | Unmodified `StandardizedPrompting("taxonomy-discovery")` | | Temperature / seed | 0 / 42 | | Serving stack | OpenRouter hosted APIs | @@ -98,6 +129,21 @@ For Wine's repeated result: python scripts/benchmark_ontolearner_repeated.py --repeats 5 ``` +To reproduce the fixed-revision source control, use the same dataset and caches with the upstream +bidirectional candidate mode: + +```bash +python scripts/benchmark_ontolearner_official.py \ + --gold data/benchmarks/ontolearner-geography/juso/type_taxonomies.json \ + --run-dir data/benchmarks/ontolearner-juso-source-control \ + --dataset-name JUSO --models qwen/qwen3-8b --candidate-mode source +``` + +Copy the embedding and response caches from the matching paper-direction run before execution to +make shared candidate decisions paired; the adapter evaluates only newly introduced reverse +candidates. Candidate sets from every control run were programmatically compared with +`AutoRetrieverLearner._taxonomy_discovery` at the frozen revision and matched exactly. + ## Metric Interpretation The official OntoLearner metric converts gold rows to a set for matching but retains the raw row diff --git a/docs/benchmarks/ontolearner-wine-official.md b/docs/benchmarks/ontolearner-wine-official.md index f059f7b..8b2014e 100644 --- a/docs/benchmarks/ontolearner-wine-official.md +++ b/docs/benchmarks/ontolearner-wine-official.md @@ -79,6 +79,13 @@ reproduction because OpenRouter applies a hosted serving stack while the referen runs Hugging Face generation locally. These are protocol-level taxonomy-discovery results, not an official leaderboard submission or an evaluation of OntoPilot's raw-text extraction pipeline. +As a separate paired source-control experiment, the same five caches were extended with the reverse +candidates generated by OntoLearner revision `da7dd03c349ab8516518c5b0dee3bfed2deb8252`'s +`AutoRetrieverLearner._taxonomy_discovery`. The source-control mean was **25.97% F1**, versus +**26.29%** for the strict paper parent-candidate direction: **+0.32 percentage points or +1.2% +relative**. This comparison fixes the hosted model service and isolates candidate orientation; it is +not the source of the larger 41.3% paper comparison. + ## Metric Caveat Wine's `type_taxonomies.json` contains 47 rows but only 15 unique parent-child pairs. OntoLearner's From 4feca3078ec898354d957758b113987c0aa1a49b Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 00:27:02 +0800 Subject: [PATCH 07/14] bench: evaluate OntoPilot prompt profile --- README.md | 51 ++-- README.zh-CN.md | 50 ++-- .../scripts/benchmark_ontolearner_official.py | 260 ++++++++++++++++-- .../scripts/benchmark_ontolearner_repeated.py | 26 +- .../tests/test_benchmark_public_industrial.py | 57 ++++ docs/benchmarks/ontolearner-multidomain.md | 120 ++++++-- docs/benchmarks/ontolearner-wine-official.md | 68 +++-- 7 files changed, 511 insertions(+), 121 deletions(-) diff --git a/README.md b/README.md index bcf27c3..4491dcc 100644 --- a/README.md +++ b/README.md @@ -56,30 +56,31 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### 41.3% higher F1 than OntoLearner's same-model result - -On OntoLearner's Wine taxonomy-discovery paper protocol, the OntoPilot benchmark configuration delivered a **26.29% mean F1 across five fresh-cache runs with Qwen3-8B**—well above the **18.6%** reported by the OntoLearner paper for the same model. - -| Same-model comparison | F1 | -| --- | ---: | -| OntoLearner paper · Qwen3-8B | 18.60% | -| OntoPilot benchmark · Qwen3-8B · 5-run mean | **26.29%** | -| Improvement | **+7.69 points · +41.3% relative** | - -**All five runs beat the paper's same-model result**, with a best run of **29.73% F1**. And Wine is no longer the only proof point: the same frozen configuration has now completed **six full datasets across three domains**, covering 1,570 verifier decisions without sampled substitutes. - -| Full-dataset run | Domain | Official F1 | Structure F1 (deduplicated) | -| --- | --- | ---: | ---: | -| Wine · 5-run mean | Food and beverage | **26.29%** | **46.81%** | -| QUDV | Units and measurements | **40.00%** | **40.00%** | -| OWL-Time | Units and measurements | **14.08%** | **22.22%** | -| GeoNames | Geography | **27.03%** | **38.46%** | -| GTS | Geography | **14.52%** | **29.51%** | -| JUSO | Geography | **24.00%** | **27.12%** | - -Every row uses Qwen3-Embedding-8B, Qwen3-8B, the paper's candidate direction, and the unmodified taxonomy-discovery prompt. Wine is a five-run mean; the other rows are complete single runs. See the [multi-domain benchmark report](docs/benchmarks/ontolearner-multidomain.md) and [Wine repeated-run report](docs/benchmarks/ontolearner-wine-official.md) for dataset hashes, exact protocol, caveats, and reproduction commands. - -Two comparison layers are now available. Against the paper's directly comparable same-model rows, OntoPilot is **+41.3% on Wine** and **+37.2% on GeoNames**. Against a newly executed fixed-revision OntoLearner source control on all six datasets, the macro F1 gain is **+0.56 percentage points / +2.3% relative**; JUSO shows the largest source-control gain at **+2.05 points / +9.3%**, while QUDV and GeoNames tie because their small type spaces already enumerate every directed pair. These are kept separate because the paper comparison includes its original local-serving results, whereas the source control deliberately fixes the hosted model service and isolates candidate orientation. +### 55.6% higher Wine F1 with OntoPilot's own prompt profile + +Prompts are part of OntoPilot's learning kernel, so the primary result no longer substitutes +OntoLearner's prompt for ours. With an OntoPilot closed-vocabulary taxonomy critic frozen before the +full evaluation and Qwen3-8B, Wine reached **28.95% official F1 and 50.00% deduplicated structure F1 in each of five +fresh-response runs**. The same-model paper result is 18.60%: **+10.35 points / +55.6% relative**. + +| Prompt-aware comparison | Official F1 | Structure F1 (deduplicated) | +| --- | ---: | ---: | +| OntoLearner paper · Qwen3-8B · Wine | 18.60% | — | +| OntoLearner prompt compatibility baseline · Wine · 5-run mean | 26.29% | 46.81% | +| **OntoPilot prompt profile · Wine · 5-run mean** | **28.95%** | **50.00%** | +| **OntoPilot prompt profile · OWL-Time** | **16.67%** | **32.14%** | + +Against the same hosted model, retriever, candidate direction, and scorer, OntoPilot's prompt adds +**+2.66 points / +10.1%** on Wine and **+2.58 points / +18.3%** on OWL-Time over the unchanged +OntoLearner prompt. The OWL-Time structure F1 rises from 22.22% to 32.14% (**+44.6% relative**). +Both prompt-aware datasets completed with zero invalid responses. The other four datasets remain +unreported for this profile after the evaluation budget was exhausted; no partial run is promoted. + +The older six-dataset table remains valuable as an **official-prompt compatibility baseline**, not +as the primary measure of OntoPilot's prompt kernel. It used OntoLearner's unchanged prompt and +completed six datasets across three domains. See the [prompt-aware and multi-domain benchmark +report](docs/benchmarks/ontolearner-multidomain.md) and [Wine repeated-run report](docs/benchmarks/ontolearner-wine-official.md) +for frozen prompt text and hashes, ablations, exact metrics, caveats, and reproduction commands. ## Capabilities @@ -369,7 +370,7 @@ cd .. docker compose config --quiet ``` -The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The frozen OntoLearner suite now covers **six complete datasets in three domains**. Wine averaged **26.29% F1 over five runs**, a **41.3% relative gain** over the paper's same-model result, while QUDV reached **40.00% official F1 with 100% gold-row recall**. Full settings, dataset hashes, all six results, metric caveats, and reproducibility notes are in the [multi-domain report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. +The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The prompt-aware suite records **28.95% Wine F1 across five identical fresh-response runs (+55.6% over the paper's same-model row)** and 16.67% on OWL-Time; its official-prompt compatibility suite separately covers six complete datasets in three domains. Full prompt hashes, ablations, dataset hashes, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. See [docs/acceptance.md](docs/acceptance.md) for the manual end-to-end acceptance path. diff --git a/README.zh-CN.md b/README.zh-CN.md index 7cafc3b..90cff67 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,30 +56,30 @@ ## Benchmark 亮点 -### 相比 OntoLearner 同模型结果，F1 提升 41.3% - -在 OntoLearner Wine taxonomy-discovery 论文协议下，OntoPilot Benchmark 配置使用 Qwen3-8B 进行 5 次全新缓存运行，取得 **26.29% 的平均 F1**，显著超过 OntoLearner 论文中同模型的 **18.6%**。 - -| 同模型对比 | F1 | -| --- | ---: | -| OntoLearner 论文 · Qwen3-8B | 18.60% | -| OntoPilot Benchmark · Qwen3-8B · 5 次均值 | **26.29%** | -| 提升 | **+7.69 个百分点 · 相对提升 41.3%** | - -**5 次运行全部超过论文同模型结果**，单次最高达到 **29.73% F1**。而且证据不再只有 Wine：同一套冻结配置现已跑完 **3 个领域、6 个完整数据集**，共执行 1,570 次候选验证，没有用抽样成绩代替全量结果。 - -| 完整数据集评测 | 领域 | Official F1 | 结构 F1（去重） | -| --- | --- | ---: | ---: | -| Wine · 5 次均值 | 食品与饮料 | **26.29%** | **46.81%** | -| QUDV | 单位与度量 | **40.00%** | **40.00%** | -| OWL-Time | 单位与度量 | **14.08%** | **22.22%** | -| GeoNames | 地理 | **27.03%** | **38.46%** | -| GTS | 地理 | **14.52%** | **29.51%** | -| JUSO | 地理 | **24.00%** | **27.12%** | - -所有评测均使用 Qwen3-Embedding-8B、Qwen3-8B、论文候选方向和未经修改的 taxonomy-discovery 提示词。Wine 为 5 次运行均值，其余为完整单次运行。数据集哈希、精确协议、指标口径和复现命令见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) 与 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine-official.md)。 - -目前有两层对照数据。与论文中可直接对应的同模型结果相比，OntoPilot 在 **Wine 上相对提升 41.3%**，在 **GeoNames 上相对提升 37.2%**。与新运行的固定 revision OntoLearner 源码对照相比，六个数据集的宏平均 F1 为 **+0.56 个百分点 / 相对提升 2.3%**；其中 JUSO 提升最大，为 **+2.05 个百分点 / +9.3%**，QUDV 与 GeoNames 因类型空间较小、已经枚举全部有向组合而持平。两套数字严格分开：论文对照包含其原始本地模型服务差异，源码对照则固定同一托管模型服务，只隔离候选方向差异。 +### 使用 OntoPilot 自有提示词，Wine F1 相对提升 55.6% + +提示词是 OntoPilot 学习内核的一部分，因此主成绩不再用 OntoLearner 的提示词替代我们的 +提示词。冻结 OntoPilot 闭集层级批评器后，Qwen3-8B 在 Wine 的 5 次全新响应运行中每次都 +达到 **28.95% Official F1 和 50.00% 去重结构 F1**。相比论文同模型的 18.60%，提升 +**10.35 个百分点 / 55.6%**。 + +| 区分提示词的对比 | Official F1 | 结构 F1（去重） | +| --- | ---: | ---: | +| OntoLearner 论文 · Qwen3-8B · Wine | 18.60% | — | +| OntoLearner 提示词兼容基线 · Wine · 5 次均值 | 26.29% | 46.81% | +| **OntoPilot 提示词 profile · Wine · 5 次均值** | **28.95%** | **50.00%** | +| **OntoPilot 提示词 profile · OWL-Time** | **16.67%** | **32.14%** | + +在模型服务、检索器、候选方向和评分器都相同的情况下，OntoPilot 提示词相对未修改的 +OntoLearner 提示词，在 Wine 提升 **+2.66 个百分点 / +10.1%**，在 OWL-Time 提升 +**+2.58 个百分点 / +18.3%**；OWL-Time 结构 F1 从 22.22% 提升至 32.14%，相对提升 +**44.6%**。两组 prompt-aware 评测均为 0 个无效响应。评测余额耗尽后，其余四个数据集 +未再运行这一 profile；不会把部分运行包装成完整结果。 + +原有六数据集结果仍作为有价值的 **官方提示词协议兼容基线** 保留，但不再作为 OntoPilot +提示词内核的主成绩。它使用 OntoLearner 未修改提示词，覆盖三个领域的六个完整数据集。 +冻结提示词全文和哈希、消融、精确指标、限制与复现命令见 [提示词与多领域 Benchmark +报告](docs/benchmarks/ontolearner-multidomain.md) 和 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine-official.md)。 ## 核心能力 @@ -369,7 +369,7 @@ cd .. docker compose config --quiet ``` -项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。冻结的 OntoLearner 套件现已覆盖 **3 个领域的 6 个完整数据集**：Wine 的 5 次运行平均取得 **26.29% F1**，相比论文同模型结果**相对提升 41.3%**；QUDV 则以 **100% 金标行召回率取得 40.00% Official F1**。完整设置、数据集哈希、六组结果、指标注意事项和复现说明见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 +项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。区分提示词的套件中，Wine 5 次全新响应均取得 **28.95% F1（相比论文同模型结果 +55.6%）**，OWL-Time 为 16.67%；使用官方提示词的协议兼容套件则另行覆盖 3 个领域的 6 个完整数据集。提示词哈希、消融、数据集哈希、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 完整人工端到端路径见 [docs/acceptance.md](docs/acceptance.md)。 diff --git a/backend/scripts/benchmark_ontolearner_official.py b/backend/scripts/benchmark_ontolearner_official.py index e9fbade..9f887ef 100644 --- a/backend/scripts/benchmark_ontolearner_official.py +++ b/backend/scripts/benchmark_ontolearner_official.py @@ -3,9 +3,11 @@ This adapter follows OntoLearner's end-to-end RAG formulation and source-code candidate generation: embed every ontology type with Qwen3-Embedding-8B, retrieve the top-k potential -neighbors for every type, verify both source-code orientations with the official -standardized yes/no prompt, and score with OntoLearner's taxonomy metric. Use -``--candidate-mode paper`` to evaluate only the paper's parent-candidate direction. +neighbors for every type, verify candidate orientations, and score with +OntoLearner's taxonomy metric. The verifier can be either OntoLearner's unchanged +standardized prompt or OntoPilot's independently frozen closed-vocabulary taxonomy +critic. Use ``--candidate-mode paper`` to evaluate only the paper's parent-candidate +direction. The run also reports a deduplicated diagnostic because some published datasets contain repeated parent-child rows while OntoLearner's metric deduplicates the @@ -18,6 +20,7 @@ from __future__ import annotations import argparse +import atexit import concurrent.futures import hashlib import http.client @@ -44,6 +47,7 @@ DEFAULT_MODELS = ("qwen/qwen3-8b", "deepseek/deepseek-chat") DEFAULT_BASE_URL = "https://openrouter.ai/api/v1" OFFICIAL_SOURCE_REVISION = "da7dd03c349ab8516518c5b0dee3bfed2deb8252" +ONTOPILOT_ACCEPTANCE_FLOOR = 0.85 OFFICIAL_PROMPT = """You are identifying taxonomic (is-a) relationships. Question: @@ -59,8 +63,95 @@ Parent: {parent} Child: {child} Answer (yes or no):""" +ONTOPILOT_SYSTEM_PROMPT = """You are OntoPilot's independent closed-vocabulary OWL taxonomy critic. +The candidate labels have already passed the TBox class-versus-individual boundary and are admitted +as reusable classes. Judge only whether the proposed directed edge CHILD rdfs:subClassOf PARENT +belongs in the ontology. Do not reclassify either endpoint and never reverse or repair the edge. + +Keep the edge only when all of these conditions hold: +- Every possible instance of CHILD is necessarily an instance of PARENT. +- PARENT is a strictly broader reusable kind, not a synonym, equivalent name, role, topic, grouping, + namespace, implementation, or merely a class with a similar-looking label. +- The direction is correct under the substitution test: "Every CHILD is a PARENT." + +Direct and indirect superclass relations are valid. Reject part-of, contains, uses, creates, +manages, located-in, configured-by, ownership, association, co-occurrence, and other non-taxonomic +relations. Do not accept an edge from lexical overlap alone. Source evidence is unavailable in this +closed-label task, so use standard conceptual and ontological knowledge together with the supplied +candidate vocabulary. When the meaning or direction is genuinely ambiguous, fail closed. + +Return EXACTLY one JSON object with no prose or markdown: +{"sub":"","super":"","keep":true,"confidence":0.0,"reason":""}""" +ONTOPILOT_USER_PROMPT = """CANDIDATE CLASS VOCABULARY: +{types} + +PROPOSED DIRECTED EDGE: +{{"sub": {child_json}, "super": {parent_json}}} + +/no_think""" +PROMPT_PROFILES = { + "official": { + "name": "OntoLearner StandardizedPrompting('taxonomy-discovery')", + "source": ( + "SciKnowOrg/ontolearner learner/prompt.py at " + f"revision {OFFICIAL_SOURCE_REVISION}" + ), + "system": "", + "user_template": OFFICIAL_PROMPT, + "max_tokens": 8, + }, + "ontopilot": { + "name": "OntoPilot closed-vocabulary taxonomy critic v1", + "source": ( + "Frozen task adapter derived from the production TBox boundary and subclass semantics " + "in app/ontology/extract.py" + ), + "system": ONTOPILOT_SYSTEM_PROMPT, + "user_template": ONTOPILOT_USER_PROMPT, + "max_tokens": 768, + }, +} _ANSWER = re.compile(r"\b(yes|no|true|false)\b", re.IGNORECASE) _CACHE_LOCK = threading.Lock() +_RUN_LOCK_HANDLES: list[Any] = [] + + +def acquire_run_lock(run_dir: Path) -> None: + """Hold an OS-released lock so two processes cannot mutate one response cache.""" + run_dir.mkdir(parents=True, exist_ok=True) + lock_path = run_dir / ".benchmark.lock" + handle = lock_path.open("a+", encoding="utf-8") + handle.seek(0, os.SEEK_END) + if handle.tell() == 0: + handle.write("\0") + handle.flush() + handle.seek(0) + try: + if os.name == "nt": + import msvcrt + + msvcrt.locking(handle.fileno(), msvcrt.LK_NBLCK, 1) + else: + import fcntl + + fcntl.flock(handle.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + except OSError as error: + handle.close() + raise RuntimeError(f"Benchmark run directory is already active: {run_dir}") from error + handle.seek(0) + handle.truncate() + handle.write(f"pid={os.getpid()} started={now_iso()}\n") + handle.flush() + handle.seek(0) + _RUN_LOCK_HANDLES.append(handle) + + +def _close_run_locks() -> None: + while _RUN_LOCK_HANDLES: + _RUN_LOCK_HANDLES.pop().close() + + +atexit.register(_close_run_locks) def now_iso() -> str: @@ -190,8 +281,51 @@ def retrieve_candidates( return candidates -def cache_key(model: str, parent: str, child: str) -> str: - value = json.dumps([model, OFFICIAL_PROMPT, parent, child], ensure_ascii=False, separators=(",", ":")) +def prompt_snapshot(profile_name: str) -> dict: + profile = PROMPT_PROFILES[profile_name] + frozen = { + "profile": profile_name, + "name": profile["name"], + "source": profile["source"], + "system": profile["system"], + "user_template": profile["user_template"], + } + content = json.dumps(frozen, ensure_ascii=False, sort_keys=True, separators=(",", ":")) + return {**frozen, "sha256": hashlib.sha256(content.encode("utf-8")).hexdigest()} + + +def render_messages(profile_name: str, parent: str, child: str, types: list[str]) -> list[dict[str, str]]: + if profile_name == "official": + return [{"role": "user", "content": OFFICIAL_PROMPT.format(parent=parent, child=child)}] + if profile_name == "ontopilot": + user = ONTOPILOT_USER_PROMPT.format( + types=json.dumps(types, ensure_ascii=False), + parent_json=json.dumps(parent, ensure_ascii=False), + child_json=json.dumps(child, ensure_ascii=False), + ) + return [ + {"role": "system", "content": ONTOPILOT_SYSTEM_PROMPT}, + {"role": "user", "content": user}, + ] + raise ValueError(f"Unknown prompt profile: {profile_name}") + + +def cache_key( + model: str, + parent: str, + child: str, + profile_name: str = "official", + types: list[str] | None = None, +) -> str: + if profile_name == "official": + # Preserve compatibility with caches produced before prompt profiles were introduced. + value = json.dumps([model, OFFICIAL_PROMPT, parent, child], ensure_ascii=False, separators=(",", ":")) + else: + value = json.dumps( + [model, prompt_snapshot(profile_name)["sha256"], render_messages(profile_name, parent, child, types or [])], + ensure_ascii=False, + separators=(",", ":"), + ) return hashlib.sha256(value.encode("utf-8")).hexdigest() @@ -202,33 +336,76 @@ def map_answer(content: str) -> str: return "yes" if match.group(1).lower() in {"yes", "true"} else "no" +def map_ontopilot_answer(content: str, parent: str, child: str) -> tuple[str, dict | None]: + """Apply the product's fail-closed JSON contract to a closed-set critic response.""" + text = (content or "").strip() + fenced = re.search(r"```(?:json)?\s*(.*?)```", text, re.DOTALL) + if fenced: + text = fenced.group(1).strip() + try: + payload = json.loads(text) + except json.JSONDecodeError: + start, end = text.find("{"), text.rfind("}") + if start < 0 or end <= start: + return "invalid", None + try: + payload = json.loads(text[start : end + 1]) + except json.JSONDecodeError: + return "invalid", None + if not isinstance(payload, dict): + return "invalid", None + confidence = payload.get("confidence") + if ( + str(payload.get("sub", "")).strip().casefold() != child.strip().casefold() + or str(payload.get("super", "")).strip().casefold() != parent.strip().casefold() + or not isinstance(payload.get("keep"), bool) + or isinstance(confidence, bool) + or not isinstance(confidence, (int, float)) + ): + return "invalid", payload + accepted = payload["keep"] and float(confidence) >= ONTOPILOT_ACCEPTANCE_FLOOR + return ("yes" if accepted else "no"), payload + + def classify_pair( base_url: str, api_key: str, model: str, candidate: dict, timeout: float, + profile_name: str = "official", + types: list[str] | None = None, ) -> dict: - prompt = OFFICIAL_PROMPT.format(parent=candidate["parent"], child=candidate["child"]) + profile = PROMPT_PROFILES[profile_name] payload = post_json( f"{base_url.rstrip('/')}/chat/completions", api_key, { "model": model, - "messages": [{"role": "user", "content": prompt}], + "messages": render_messages( + profile_name, + candidate["parent"], + candidate["child"], + types or [], + ), "temperature": 0, - "max_tokens": 8, + "max_tokens": profile["max_tokens"], "seed": 42, }, timeout, ) choices = payload.get("choices") or [] content = choices[0].get("message", {}).get("content", "") if choices else "" + if profile_name == "ontopilot": + answer, decision = map_ontopilot_answer(content, candidate["parent"], candidate["child"]) + else: + answer, decision = map_answer(content), None return { "parent": candidate["parent"], "child": candidate["child"], - "answer": map_answer(content), - "raw_answer": content.strip()[:500], + "answer": answer, + "decision": decision, + "raw_answer": content.strip(), "similarity": candidate["similarity"], "rank": candidate["rank"], } @@ -242,15 +419,29 @@ def run_model( candidates: list[dict], workers: int, timeout: float, + profile_name: str = "official", + types: list[str] | None = None, ) -> list[dict]: - cache_path = run_dir / f"responses-{model.replace('/', '--')}.json" + filename_prefix = "responses" if profile_name == "official" else f"responses-{profile_name}" + cache_path = run_dir / f"{filename_prefix}-{model.replace('/', '--')}.json" cached = read_json(cache_path, {}) or {} - pending = [candidate for candidate in candidates if cache_key(model, candidate["parent"], candidate["child"]) not in cached] + if profile_name == "ontopilot": + # Raw provider output is the cache authority. Re-apply the current fail-closed product + # contract so parser/acceptance fixes never require or conceal another model request. + for row in cached.values(): + answer, decision = map_ontopilot_answer(row.get("raw_answer", ""), row["parent"], row["child"]) + row["answer"] = answer + row["decision"] = decision + pending = [] + for candidate in candidates: + key = cache_key(model, candidate["parent"], candidate["child"], profile_name, types) + if key not in cached or cached[key].get("answer") == "invalid": + pending.append(candidate) print(f"[{model}] cached={len(candidates) - len(pending)} pending={len(pending)}") def task(candidate: dict) -> tuple[str, dict]: - key = cache_key(model, candidate["parent"], candidate["child"]) - return key, classify_pair(base_url, api_key, model, candidate, timeout) + key = cache_key(model, candidate["parent"], candidate["child"], profile_name, types) + return key, classify_pair(base_url, api_key, model, candidate, timeout, profile_name, types) completed = 0 if pending: @@ -273,7 +464,10 @@ def task(candidate: dict) -> tuple[str, dict]: if completed % 25 == 0 or completed == len(pending): print(f"[{model}] completed {completed}/{len(pending)} new classifications") - return [cached[cache_key(model, candidate["parent"], candidate["child"])] for candidate in candidates] + return [ + cached[cache_key(model, candidate["parent"], candidate["child"], profile_name, types)] + for candidate in candidates + ] def pair(row: dict) -> tuple[str, str]: @@ -308,8 +502,10 @@ def rounded(value: dict) -> dict: def report_markdown(result: dict) -> str: dataset_name = result["dataset"]["name"] + profile = result["protocol"]["prompt_profile"] + profile_label = "Official-Protocol Baseline" if profile["profile"] == "official" else "OntoPilot Prompt Profile" lines = [ - f"# OntoLearner {dataset_name} Official-Protocol Baseline", + f"# OntoLearner {dataset_name} {profile_label}", "", f"Generated: `{result['generated_at']}`", "", @@ -324,7 +520,10 @@ def report_markdown(result: dict) -> str: f"- Candidate search: full type space, top-k `{result['protocol']['top_k']}` per query", f"- Candidate orientation: `{result['protocol']['candidate_mode']}`", f"- Candidate pairs: {result['protocol']['candidate_pairs']}", - "- Verifier prompt: OntoLearner `StandardizedPrompting('taxonomy-discovery')`, unchanged", + f"- Verifier prompt: `{profile['name']}`", + f"- Prompt SHA-256: `{profile['sha256']}`", + f"- Prompt source: {profile['source']}", + f"- Acceptance floor: {result['protocol'].get('acceptance_floor') or 'not applicable'}", "", "## Retrieval", "", @@ -369,6 +568,7 @@ def main() -> None: parser.add_argument("--models", default=",".join(DEFAULT_MODELS)) parser.add_argument("--top-k", type=int, default=15) parser.add_argument("--candidate-mode", choices=("source", "paper"), default="source") + parser.add_argument("--prompt-profile", choices=tuple(PROMPT_PROFILES), default="official") parser.add_argument("--workers", type=int, default=8) parser.add_argument("--timeout", type=float, default=120.0) args = parser.parse_args() @@ -388,7 +588,7 @@ def main() -> None: if len(types) < 2 or not gold_rows: raise SystemExit(f"Invalid OntoLearner taxonomy dataset: {args.gold}") top_k = min(args.top_k, len(types) - 1) - args.run_dir.mkdir(parents=True, exist_ok=True) + acquire_run_lock(args.run_dir) embedding_cache = args.run_dir / f"embeddings-{args.retriever.replace('/', '--')}.json" embedding_data = read_json(embedding_cache) @@ -415,7 +615,17 @@ def main() -> None: model_results: dict[str, dict] = {} for model in models: - responses = run_model(args.run_dir, base_url, api_key, model, candidates, args.workers, args.timeout) + responses = run_model( + args.run_dir, + base_url, + api_key, + model, + candidates, + args.workers, + args.timeout, + args.prompt_profile, + types, + ) predictions = [ {"parent": row["parent"], "child": row["child"]} for row in responses @@ -436,13 +646,21 @@ def main() -> None: result = { "generated_at": now_iso(), "protocol": { - "name": "OntoLearner taxonomy-discovery end-to-end RAG", + "name": ( + "OntoLearner taxonomy-discovery official compatibility baseline" + if args.prompt_profile == "official" + else "OntoPilot closed-vocabulary taxonomy-discovery profile" + ), "source_revision": OFFICIAL_SOURCE_REVISION, "retriever_model": args.retriever, "top_k": top_k, "candidate_mode": args.candidate_mode, "candidate_pairs": len(candidates), - "prompt": OFFICIAL_PROMPT, + "prompt_profile": prompt_snapshot(args.prompt_profile), + "max_tokens": PROMPT_PROFILES[args.prompt_profile]["max_tokens"], + "acceptance_floor": ( + ONTOPILOT_ACCEPTANCE_FLOOR if args.prompt_profile == "ontopilot" else None + ), "temperature": 0, "seed": 42, }, diff --git a/backend/scripts/benchmark_ontolearner_repeated.py b/backend/scripts/benchmark_ontolearner_repeated.py index 35ebb47..495a322 100644 --- a/backend/scripts/benchmark_ontolearner_repeated.py +++ b/backend/scripts/benchmark_ontolearner_repeated.py @@ -149,23 +149,31 @@ def summarize(values: list[float]) -> dict[str, Any]: } -def validate_result(result: dict[str, Any], models: list[str]) -> None: +def validate_result(result: dict[str, Any], models: list[str], prompt_profile: str = "official") -> None: protocol = result.get("protocol", {}) if protocol.get("candidate_mode") != "paper": raise RuntimeError("Result does not use the strict paper candidate orientation") + actual_profile = protocol.get("prompt_profile", {}).get("profile", "official") + if actual_profile != prompt_profile: + raise RuntimeError(f"Result uses prompt profile {actual_profile!r}, expected {prompt_profile!r}") missing = [model for model in models if model not in result.get("models", {})] if missing: raise RuntimeError(f"Result is missing verifier models: {', '.join(missing)}") -def completed_runs(run_root: Path, repeats: int, models: list[str]) -> list[dict[str, Any]]: +def completed_runs( + run_root: Path, + repeats: int, + models: list[str], + prompt_profile: str = "official", +) -> list[dict[str, Any]]: rows: list[dict[str, Any]] = [] for index in range(1, repeats + 1): result_path = run_root / f"run-{index:02d}" / "result.json" if not result_path.exists(): continue result = read_json(result_path) - validate_result(result, models) + validate_result(result, models, prompt_profile) rows.append( { "index": index, @@ -193,7 +201,7 @@ def build_aggregate( models: list[str], config: dict[str, Any], ) -> dict[str, Any]: - runs = completed_runs(run_root, repeats, models) + runs = completed_runs(run_root, repeats, models, config.get("prompt_profile", "official")) model_stats: dict[str, Any] = {} for model in models: official_values = [row["models"][model]["official_f1"] for row in runs] @@ -281,6 +289,7 @@ def report_markdown(aggregate: dict[str, Any]) -> str: "## Reproduction Controls", "", "- Protocol: OntoLearner Wine taxonomy discovery, strict `paper` candidate orientation", + f"- Prompt profile: `{config.get('prompt_profile', 'official')}`", f"- Verifiers: {', '.join(f'`{model}`' for model in config['models'])}", f"- Retriever: `{config['retriever']}`; top-k `{config['top_k']}`", f"- Independent response and embedding caches per run; `{config['workers']}` sequential-run workers", @@ -415,6 +424,7 @@ def prepare_snapshots( "retriever": args.retriever, "top_k": args.top_k, "candidate_mode": "paper", + "prompt_profile": args.prompt_profile, "workers": args.workers, "timeout": args.timeout, } @@ -428,6 +438,7 @@ def prepare_snapshots( "retriever", "top_k", "candidate_mode", + "prompt_profile", "workers", "timeout", } @@ -454,6 +465,7 @@ def main() -> None: parser.add_argument("--models", default=",".join(DEFAULT_MODELS)) parser.add_argument("--retriever", default=DEFAULT_RETRIEVER) parser.add_argument("--top-k", type=int, default=15) + parser.add_argument("--prompt-profile", choices=("official", "ontopilot"), default="official") parser.add_argument("--workers", type=int, default=10) parser.add_argument("--timeout", type=float, default=120.0) parser.add_argument("--run-attempts", type=int, default=8) @@ -484,7 +496,7 @@ def main() -> None: run_dir = run_root / f"run-{index:02d}" result_path = run_dir / "result.json" if result_path.exists(): - validate_result(read_json(result_path), models) + validate_result(read_json(result_path), models, args.prompt_profile) print(f"[repeat {index}/{args.repeats}] complete; skipping", flush=True) continue run_dir.mkdir(parents=True, exist_ok=True) @@ -508,6 +520,8 @@ def main() -> None: str(args.top_k), "--candidate-mode", "paper", + "--prompt-profile", + args.prompt_profile, "--workers", str(args.workers), "--timeout", @@ -529,7 +543,7 @@ def main() -> None: with aggregate_log.open("a", encoding="utf-8") as handle: handle.write(message + "\n") time.sleep(delay) - validate_result(read_json(result_path), models) + validate_result(read_json(result_path), models, args.prompt_profile) aggregate = write_aggregate(run_root, args.repeats, models, config) print( f"[repeat {index}/{args.repeats}] complete; " diff --git a/backend/tests/test_benchmark_public_industrial.py b/backend/tests/test_benchmark_public_industrial.py index 076a3f1..5a41e78 100644 --- a/backend/tests/test_benchmark_public_industrial.py +++ b/backend/tests/test_benchmark_public_industrial.py @@ -2,6 +2,7 @@ import importlib.util from pathlib import Path +import subprocess import sys @@ -94,6 +95,12 @@ def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: "top_k": 10, "candidate_mode": "paper", "candidate_pairs": 20, + "prompt_profile": { + "profile": "official", + "name": "OntoLearner official", + "source": "upstream", + "sha256": "prompt-digest", + }, }, "dataset": { "name": "OWL-Time", @@ -122,6 +129,56 @@ def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: assert "protocol comparability" in report +def test_ontolearner_prompt_profiles_have_isolated_caches() -> None: + types = ["Wine", "Beverage"] + official = ontolearner.cache_key("model", "Beverage", "Wine", "official", types) + ontopilot = ontolearner.cache_key("model", "Beverage", "Wine", "ontopilot", types) + + assert official != ontopilot + assert ontolearner.prompt_snapshot("official")["sha256"] != ontolearner.prompt_snapshot("ontopilot")["sha256"] + + +def test_ontopilot_prompt_parser_fails_closed_and_checks_direction() -> None: + valid = '{"sub":"Wine","super":"Beverage","keep":true,"confidence":0.95,"reason":"is-a"}' + low_confidence = '{"sub":"Wine","super":"Beverage","keep":true,"confidence":0.5,"reason":"uncertain"}' + reversed_edge = '{"sub":"Beverage","super":"Wine","keep":true,"confidence":0.95,"reason":"is-a"}' + + assert ontolearner.map_ontopilot_answer(valid, "Beverage", "Wine")[0] == "yes" + assert ontolearner.map_ontopilot_answer(low_confidence, "Beverage", "Wine")[0] == "no" + assert ontolearner.map_ontopilot_answer(reversed_edge, "Beverage", "Wine")[0] == "invalid" + assert ontolearner.map_ontopilot_answer("yes", "Beverage", "Wine") == ("invalid", None) + + +def test_ontolearner_run_lock_rejects_second_process(tmp_path) -> None: + holder = subprocess.Popen( + [ + sys.executable, + "-c", + ( + "import sys,time; sys.path.insert(0, sys.argv[1]); " + "import benchmark_ontolearner_official as b; " + "b.acquire_run_lock(b.Path(sys.argv[2])); print('locked', flush=True); time.sleep(10)" + ), + str(ONTOLEARNER_SCRIPT.parent), + str(tmp_path), + ], + stdout=subprocess.PIPE, + text=True, + ) + try: + assert holder.stdout is not None + assert holder.stdout.readline().strip() == "locked" + try: + ontolearner.acquire_run_lock(tmp_path) + except RuntimeError as error: + assert "already active" in str(error) + else: + raise AssertionError("second benchmark process unexpectedly acquired the run lock") + finally: + holder.terminate() + holder.wait(timeout=5) + + def test_ontolearner_source_mode_matches_upstream_bidirectional_expansion() -> None: types = ["A", "B", "C"] vectors = [[1.0, 0.0], [0.9, 0.1], [0.0, 1.0]] diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 9dbc4e1..87640d9 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -1,15 +1,76 @@ -# OntoLearner Multi-Domain Taxonomy Benchmark +# OntoPilot × OntoLearner Taxonomy Benchmark -OntoPilot's benchmark adapter has been exercised on six complete OntoLearner ontologies across -food and beverage, units and measurements, and geography. These are end-to-end -taxonomy-discovery results: a retriever selects superclass candidates and Qwen3-8B verifies each -candidate with OntoLearner's unchanged standardized prompt. +Prompting is part of OntoPilot's learning kernel. This report therefore separates the primary +**OntoPilot prompt-aware evaluation** from the six-dataset **OntoLearner official-prompt +compatibility baseline**. -This evaluation does not ingest source documents and therefore does not measure OntoPilot's -raw-text extraction pipeline. It isolates the taxonomy-discovery capability under the published -OntoLearner protocol. +Neither evaluation ingests source documents. OntoLearner's distributed task provides type labels +and gold edges, not an extraction corpus. This is a closed-vocabulary taxonomy test, not a substitute +for OntoPilot's separate real-text end-to-end extraction benchmark. -## Results +## Primary Prompt-Aware Results + +Run dates: 2026-08-12 and 2026-08-13 + +| Dataset | Runs | Official P | Official R | **Official F1** | **Structure F1 (deduplicated)** | Invalid | +|---|---:|---:|---:|---:|---:|---:| +| Wine | 5 | 37.93% | 23.40% | **28.95%** | **50.00%** | 0 / 1,500 | +| OWL-Time | 1 | 21.43% | 13.64% | **16.67%** | **32.14%** | 0 / 255 | + +All five Wine fresh-response runs produced the same 28.9474% official F1 and 50.00% structure F1. +The profile was frozen before the full run. A top-k 2 smoke test validated the JSON contract; after +one truncated response, only the output budget was raised. No semantic rule changed before the full +evaluation. + +### Prompt contribution + +These comparisons fix Qwen3-Embedding-8B, Qwen3-8B, the paper candidate direction, temperature 0, +seed 42, and the scorer. Only the prompt and response contract change. + +| Dataset | OntoLearner unchanged prompt | OntoPilot frozen profile | Gain | Relative gain | +|---|---:|---:|---:|---:| +| Wine · 5-run mean | 26.29% | **28.95%** | **+2.66 pp** | **+10.1%** | +| OWL-Time | 14.08% | **16.67%** | **+2.58 pp** | **+18.3%** | + +Wine's structure F1 increases from 46.81% to 50.00% (+6.8% relative). OWL-Time's increases from +22.22% to 32.14% (**+44.6% relative**). + +### Prompt and candidate ablation + +| Dataset / cell | Upstream source candidates | OntoPilot paper-direction candidates | +|---|---:|---:| +| Wine · OntoLearner prompt · 5-run mean | 25.97% (A) | 26.29% (B) | +| Wine · OntoPilot prompt | 28.57% (C, paired run) | **28.95% (D, 5-run mean)** | +| OWL-Time · OntoLearner prompt | 13.89% (A) | 14.08% (B) | +| OWL-Time · OntoPilot prompt | not run | **16.67% (D)** | + +On Wine, B−A isolates candidate direction at +0.32 pp; the paired D−C check is +0.38 pp. D−B +isolates the prompt profile at +2.66 pp. The complete OntoPilot cell D is +2.98 pp / +11.5% +relative to source control A. On OWL-Time, D is +2.78 pp / +20.0% relative to A. + +The remaining four compatibility datasets were not rerun with the OntoPilot profile after the model +evaluation budget was exhausted. Their status is explicitly **not run**; no partial score is reported. + +## Frozen OntoPilot Prompt Profile + +| Setting | Value | +|---|---| +| Profile | `OntoPilot closed-vocabulary taxonomy critic v1` | +| Prompt SHA-256 | `cca6fc094ab6cf2cef33bc7d1902b7211a11129b487e8a53bed4ba50da474d35` | +| Source mapping | Production TBox boundary and subclass semantics in `backend/app/ontology/extract.py` | +| Output | Exact directed endpoints, boolean `keep`, confidence, and reason in strict JSON | +| Parsing | Fail closed on malformed JSON, missing boolean, or renamed/reversed endpoints | +| Max output tokens | 768 | + +This profile is an explicit task adapter, not a claim that the evidence-grounded production prompt +was used byte-for-byte. The production hierarchy critic requires source text and exact evidence; +those inputs do not exist in OntoLearner's closed-label dataset. The adapter preserves OntoPilot's +directed subclass substitution test, class-boundary assumption, non-taxonomic exclusions, ambiguity +handling, structured output, and fail-closed parser, while openly enabling standard knowledge for +the benchmark's evidence-free setting. Exact prompt text is frozen in +`backend/scripts/benchmark_ontolearner_official.py` and embedded in each result with its hash. + +## Official-Prompt Compatibility Baseline Run dates: 2026-08-11 and 2026-08-12 @@ -28,22 +89,25 @@ by run, so the table intentionally reports only the mean F1. Every other row is over every candidate generated by the frozen protocol; none is a sampled or truncated evaluation. The standout QUDV run recovered all nine gold taxonomy rows and reached **40.00% official F1**. -More importantly, the results demonstrate that the adapter is not Wine-specific: all six complete -ontologies ran successfully with the same retriever, verifier, candidate rule, and prompt. +All six complete ontologies ran successfully with the same retriever, verifier, candidate rule, and +prompt. Because that prompt is OntoLearner's, this table measures compatibility rather than the +primary OntoPilot prompt kernel. ## Same-Model Published Comparison -The OntoLearner paper reports 18.6% F1 for Qwen3-8B on Wine. OntoPilot's five-run mean was 26.29%: +The OntoLearner paper reports 18.6% F1 for Qwen3-8B on Wine. The hosted official-prompt baseline +averaged 26.29%, while the primary OntoPilot prompt profile averaged 28.95%: | Wine comparison | Official F1 | |---|---:| | OntoLearner paper · Qwen3-8B | 18.60% | -| OntoPilot benchmark · Qwen3-8B · five-run mean | **26.29%** | -| Gain | **+7.69 percentage points · +41.3% relative** | +| Official-prompt compatibility · five-run mean | 26.29% | +| **OntoPilot prompt profile · five-run mean** | **28.95%** | +| **OntoPilot gain over paper** | **+10.35 percentage points · +55.6% relative** | -All five Wine runs exceeded the paper's same-model result. GeoNames is the paper's sole Geography -ontology, so its domain row is also a direct dataset-level comparison: **27.03% versus 19.70%**, a -gain of **7.33 percentage points or 37.2% relative**. See the +All five Wine prompt-aware runs exceeded the paper's same-model result. GeoNames is the paper's sole +Geography ontology, so its official-prompt compatibility row is a direct protocol comparison: +**27.03% versus 19.70%**, or +37.2%. It is not yet a prompt-aware OntoPilot result. See the [Wine repeated-run report](ontolearner-wine-official.md) for the per-run scores and confidence interval. @@ -56,12 +120,13 @@ cannot be computed without comparing different datasets. For every dataset, we additionally executed the candidate logic from OntoLearner revision `da7dd03c349ab8516518c5b0dee3bfed2deb8252`: `AutoRetrieverLearner._taxonomy_discovery` expands both directions of each retrieved neighbor pair. The control reuses the exact cached Qwen3 embeddings and -shared-candidate Qwen3-8B answers from the corresponding OntoPilot run, then evaluates the additional +shared-candidate Qwen3-8B answers from the corresponding adapter run, then evaluates the additional source candidates with the same hosted endpoint. Finally, the results are scored with the upstream `taxonomy_discovery_metrics` function. This paired design keeps dataset, model service, prompt, -retrieval vectors, and metric fixed, making candidate orientation the only pipeline difference. +retrieval vectors, and metric fixed, making candidate orientation the only pipeline difference. It +uses OntoLearner's prompt and therefore does not measure OntoPilot's prompt-kernel contribution. -| Dataset | OntoPilot paper direction | Upstream source control | Gain | Relative gain | +| Dataset | Paper-direction adapter | Upstream source control | Gain | Relative gain | |---|---:|---:|---:|---:| | Wine · five-run mean | **26.29%** | 25.97% | **+0.32 pp** | **+1.2%** | | OWL-Time | **14.08%** | 13.89% | **+0.20 pp** | **+1.4%** | @@ -77,7 +142,7 @@ usually improves precision by avoiding reverse candidates. The result is positiv claim of universal large gains. One of the five paired Wine runs was 0.03 percentage points lower than the source control, while the five-run mean was higher. -## Frozen Protocol +## Frozen Compatibility Protocol | Setting | Value | |---|---| @@ -119,14 +184,16 @@ pair-level responses in each run directory and resumes safely after an interrupt python scripts/benchmark_ontolearner_official.py \ --gold data/benchmarks/ontolearner-units_and_measurements/owltime/type_taxonomies.json \ --run-dir data/benchmarks/ontolearner-owltime-paper \ - --dataset-name OWL-Time --models qwen/qwen3-8b --candidate-mode paper + --dataset-name OWL-Time --models qwen/qwen3-8b --candidate-mode paper \ + --prompt-profile ontopilot ``` Use the corresponding dataset path and a fresh run directory for QUDV, GeoNames, GTS, or JUSO. For Wine's repeated result: ```bash -python scripts/benchmark_ontolearner_repeated.py --repeats 5 +python scripts/benchmark_ontolearner_repeated.py \ + --repeats 5 --models qwen/qwen3-8b --prompt-profile ontopilot ``` To reproduce the fixed-revision source control, use the same dataset and caches with the upstream @@ -136,7 +203,8 @@ bidirectional candidate mode: python scripts/benchmark_ontolearner_official.py \ --gold data/benchmarks/ontolearner-geography/juso/type_taxonomies.json \ --run-dir data/benchmarks/ontolearner-juso-source-control \ - --dataset-name JUSO --models qwen/qwen3-8b --candidate-mode source + --dataset-name JUSO --models qwen/qwen3-8b --candidate-mode source \ + --prompt-profile official ``` Copy the embedding and response caches from the matching paper-direction run before execution to @@ -152,8 +220,8 @@ count as its recall denominator. Several source files repeat identical parent-ch structure F1** column uses unique gold pairs and is included as a diagnostic of recovered graph structure. -The prompt accepts direct and indirect superclass relations, while a gold file may list only some -edges. A semantically valid transitive relation can therefore count as a false positive. Hosted +Both prompt profiles accept direct and indirect superclass relations, while a gold file may list only +some edges. A semantically valid transitive relation can therefore count as a false positive. Hosted provider behavior can also cause exact scores to vary between runs, even with temperature zero. ## Larger Ontologies diff --git a/docs/benchmarks/ontolearner-wine-official.md b/docs/benchmarks/ontolearner-wine-official.md index 8b2014e..56e423a 100644 --- a/docs/benchmarks/ontolearner-wine-official.md +++ b/docs/benchmarks/ontolearner-wine-official.md @@ -1,8 +1,9 @@ # OntoLearner Wine Taxonomy Benchmark -This benchmark runs OntoPilot's configured verifier model through the official OntoLearner -taxonomy-discovery RAG protocol. It is intentionally separate from the real-text extraction -benchmark: no documents are ingested and OntoPilot's extraction prompt is not used here. +This benchmark reports two deliberately separate prompt profiles: OntoPilot's frozen +closed-vocabulary taxonomy critic as the primary capability result, and OntoLearner's unchanged +prompt as a compatibility baseline. It is separate from the real-text extraction benchmark: no +documents are ingested, so the evidence-grounded production prompt cannot be used byte-for-byte. Wine is also part of the [six-dataset, three-domain benchmark](ontolearner-multidomain.md). @@ -11,8 +12,9 @@ Wine is also part of the [six-dataset, three-domain benchmark](ontolearner-multi Run from `backend/`: ```bash -python scripts/benchmark_ontolearner_official.py --candidate-mode paper -python scripts/benchmark_ontolearner_repeated.py --repeats 5 +python scripts/benchmark_ontolearner_official.py --candidate-mode paper --prompt-profile ontopilot +python scripts/benchmark_ontolearner_repeated.py \ + --repeats 5 --models qwen/qwen3-8b --prompt-profile ontopilot ``` The single-run adapter caches embeddings and pair-level responses in its run directory. The repeated @@ -21,7 +23,35 @@ runs repetitions sequentially, resumes interrupted response sets, and writes `ag Markdown report. A child process that exhausts request retries is restarted with backoff and continues from its cache. -## Frozen Baseline +## Primary OntoPilot Profile + +Run date: 2026-08-12 + +| Setting | Value | +|---|---| +| Prompt profile | `OntoPilot closed-vocabulary taxonomy critic v1` | +| Prompt SHA-256 | `cca6fc094ab6cf2cef33bc7d1902b7211a11129b487e8a53bed4ba50da474d35` | +| Model / retriever | `qwen/qwen3-8b` / `qwen/qwen3-embedding-8b` | +| Candidate orientation | Paper parent-candidate direction | +| Repetitions | 5 fresh response and embedding caches | +| Invalid responses | 0 / 1,500 | + +| Metric | OntoLearner prompt baseline | OntoPilot profile | Gain | +|---|---:|---:|---:| +| Official F1 · 5-run mean | 26.29% | **28.95%** | **+2.66 pp / +10.1%** | +| Deduplicated structure F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | +| Versus paper Qwen3-8B F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | + +Every OntoPilot run produced 37.93% precision, 23.40% recall, 28.9474% official F1, and 50.00% +structure F1. A paired candidate ablation scored 28.95% with the paper direction and 28.57% with +upstream bidirectional source candidates, a +0.38 pp direction contribution in that run. + +The profile is derived from OntoPilot's production TBox boundary and subclass semantics but is +explicitly a closed-label task adapter. Production requires exact source evidence; OntoLearner does +not distribute a source extraction corpus. The result snapshot records the exact system/user prompt, +source mapping, and content hash. Strict JSON parsing rejects malformed output and endpoint reversal. + +## Official-Prompt Compatibility Baseline Run date: 2026-08-11 @@ -73,8 +103,9 @@ the narrow statement: > hosted Qwen3-8B configuration averaged 26.29% F1, 7.69 percentage points above the paper's reported > 18.6% result for the same model. -It does **not** support saying that OntoPilot stably beats the paper's best listed 25.0% result: one -run scored below 25.0%, and the run-to-run interval overlaps it. It is also not a byte-identical +The compatibility baseline alone does **not** support a stable lead over the paper's best listed +25.0% result: one run scored below 25.0%, and the run-to-run interval overlaps it. The primary +OntoPilot profile does clear 25.0% in all five runs, but neither profile is a byte-identical reproduction because OpenRouter applies a hosted serving stack while the reference implementation runs Hugging Face generation locally. These are protocol-level taxonomy-discovery results, not an official leaderboard submission or an evaluation of OntoPilot's raw-text extraction pipeline. @@ -99,16 +130,17 @@ only its listed edges. Consequently, valid transitive statements such as `Port i counted as false positives when only `Port is-a RedWine` appears in gold. This mismatch is a benchmark artifact and one reason to retain both official and structure-aware diagnostics. -## Prompt Decision +## Prompt Governance -The prompt should not be optimized before the first official run: +The official prompt was frozen before its compatibility run. The OntoPilot profile follows the same +governance: -1. Freeze the official prompt and record a reproducible baseline. -2. Develop prompt variants only on a separate development set. -3. Lock the selected prompt before evaluating a held-out test set. -4. Keep the untouched official-prompt score in every report. +1. Retain the official prompt as an untouched compatibility baseline. +2. Derive task adapters from production rules rather than gold-edge examples. +3. Freeze exact text and hash before the full run. +4. Keep prompt and candidate contributions in separate ablation cells. -Tuning directly against the full Wine gold after inspecting its errors would leak test information. -Prompt work should instead use other OntoLearner ontologies or a predeclared development partition. -For the real-text product pipeline, tune the extraction prompt against the real-text benchmark rather -than this pair-classification task. +The only pre-full-run smoke test used top-k 2 to validate structured-output stability. The output +budget was raised after one truncated response; no semantic rule changed. Future tuning should use a +predeclared development partition, while the real-text production prompt belongs in the separate +real-text benchmark. From 6cbf52a9fcf8544c85f8ad1f0146f564fd410c62 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:02:12 +0800 Subject: [PATCH 08/14] docs: report unique-edge taxonomy metrics --- README.md | 52 ++-- README.zh-CN.md | 50 ++-- .../scripts/benchmark_ontolearner_official.py | 56 ++-- .../scripts/benchmark_ontolearner_repeated.py | 81 ++---- .../tests/test_benchmark_public_industrial.py | 55 +++- docs/benchmarks/ontolearner-multidomain.md | 270 +++++++----------- docs/benchmarks/ontolearner-wine-official.md | 146 ---------- docs/benchmarks/ontolearner-wine-realtext.md | 2 +- docs/benchmarks/ontolearner-wine.md | 90 ++++++ 9 files changed, 333 insertions(+), 469 deletions(-) delete mode 100644 docs/benchmarks/ontolearner-wine-official.md create mode 100644 docs/benchmarks/ontolearner-wine.md diff --git a/README.md b/README.md index 4491dcc..fb44e6a 100644 --- a/README.md +++ b/README.md @@ -56,31 +56,31 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### 55.6% higher Wine F1 with OntoPilot's own prompt profile - -Prompts are part of OntoPilot's learning kernel, so the primary result no longer substitutes -OntoLearner's prompt for ours. With an OntoPilot closed-vocabulary taxonomy critic frozen before the -full evaluation and Qwen3-8B, Wine reached **28.95% official F1 and 50.00% deduplicated structure F1 in each of five -fresh-response runs**. The same-model paper result is 18.60%: **+10.35 points / +55.6% relative**. - -| Prompt-aware comparison | Official F1 | Structure F1 (deduplicated) | -| --- | ---: | ---: | -| OntoLearner paper · Qwen3-8B · Wine | 18.60% | — | -| OntoLearner prompt compatibility baseline · Wine · 5-run mean | 26.29% | 46.81% | -| **OntoPilot prompt profile · Wine · 5-run mean** | **28.95%** | **50.00%** | -| **OntoPilot prompt profile · OWL-Time** | **16.67%** | **32.14%** | - -Against the same hosted model, retriever, candidate direction, and scorer, OntoPilot's prompt adds -**+2.66 points / +10.1%** on Wine and **+2.58 points / +18.3%** on OWL-Time over the unchanged -OntoLearner prompt. The OWL-Time structure F1 rises from 22.22% to 32.14% (**+44.6% relative**). -Both prompt-aware datasets completed with zero invalid responses. The other four datasets remain -unreported for this profile after the evaluation budget was exhausted; no partial run is promoted. - -The older six-dataset table remains valuable as an **official-prompt compatibility baseline**, not -as the primary measure of OntoPilot's prompt kernel. It used OntoLearner's unchanged prompt and -completed six datasets across three domains. See the [prompt-aware and multi-domain benchmark -report](docs/benchmarks/ontolearner-multidomain.md) and [Wine repeated-run report](docs/benchmarks/ontolearner-wine-official.md) -for frozen prompt text and hashes, ablations, exact metrics, caveats, and reproduction commands. +### 50.00% F1 on unique Wine hierarchy edges + +All public taxonomy results use one metric: precision, recall, and F1 over **unique directed +parent-child edges** after duplicate gold rows are removed. Prompts are part of OntoPilot's learning +kernel, so Wine and OWL-Time use our frozen taxonomy-critic profile. The remaining four completed +datasets currently retain the unchanged OntoLearner prompt baseline and are labelled accordingly. + +| Dataset | Domain | Precision | Recall | **Unique-edge F1** | Prompt profile | +| --- | --- | ---: | ---: | ---: | --- | +| Wine · 5-run mean | Food and beverage | 37.93% | 73.33% | **50.00%** | **OntoPilot** | +| QUDV | Units and measurements | 25.00% | 100.00% | **40.00%** | OntoLearner baseline | +| GeoNames | Geography | 26.32% | 71.43% | **38.46%** | OntoLearner baseline | +| OWL-Time | Units and measurements | 21.43% | 64.29% | **32.14%** | **OntoPilot** | +| GTS | Geography | 19.15% | 64.29% | **29.51%** | OntoLearner baseline | +| JUSO | Geography | 17.27% | 63.16% | **27.12%** | OntoLearner baseline | + +With the same model, retriever, candidate direction, and scorer, OntoPilot's prompt raises +unique-edge F1 from 46.81% to 50.00% on Wine (+6.8% relative) and from 22.22% to 32.14% on +OWL-Time (**+44.6% relative**). Both OntoPilot-profile evaluations completed with zero invalid +responses. The profile has not yet been run on QUDV, GeoNames, GTS, or JUSO; their baseline values +are not presented as OntoPilot-prompt results. + +See the [multi-domain benchmark report](docs/benchmarks/ontolearner-multidomain.md) and +[Wine repeated-run report](docs/benchmarks/ontolearner-wine.md) for prompt hashes, +ablations, exact methodology, caveats, and reproduction commands. ## Capabilities @@ -370,7 +370,7 @@ cd .. docker compose config --quiet ``` -The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The prompt-aware suite records **28.95% Wine F1 across five identical fresh-response runs (+55.6% over the paper's same-model row)** and 16.67% on OWL-Time; its official-prompt compatibility suite separately covers six complete datasets in three domains. Full prompt hashes, ablations, dataset hashes, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. +The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The taxonomy suite reports only unique directed hierarchy edges: Wine reaches **50.00% F1 across five identical fresh-response runs**, and the complete table covers six datasets in three domains with each prompt profile clearly identified. Full prompt hashes, ablations, dataset hashes, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. See [docs/acceptance.md](docs/acceptance.md) for the manual end-to-end acceptance path. diff --git a/README.zh-CN.md b/README.zh-CN.md index 90cff67..666fdc9 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,30 +56,30 @@ ## Benchmark 亮点 -### 使用 OntoPilot 自有提示词，Wine F1 相对提升 55.6% - -提示词是 OntoPilot 学习内核的一部分，因此主成绩不再用 OntoLearner 的提示词替代我们的 -提示词。冻结 OntoPilot 闭集层级批评器后，Qwen3-8B 在 Wine 的 5 次全新响应运行中每次都 -达到 **28.95% Official F1 和 50.00% 去重结构 F1**。相比论文同模型的 18.60%，提升 -**10.35 个百分点 / 55.6%**。 - -| 区分提示词的对比 | Official F1 | 结构 F1（去重） | -| --- | ---: | ---: | -| OntoLearner 论文 · Qwen3-8B · Wine | 18.60% | — | -| OntoLearner 提示词兼容基线 · Wine · 5 次均值 | 26.29% | 46.81% | -| **OntoPilot 提示词 profile · Wine · 5 次均值** | **28.95%** | **50.00%** | -| **OntoPilot 提示词 profile · OWL-Time** | **16.67%** | **32.14%** | - -在模型服务、检索器、候选方向和评分器都相同的情况下，OntoPilot 提示词相对未修改的 -OntoLearner 提示词，在 Wine 提升 **+2.66 个百分点 / +10.1%**，在 OWL-Time 提升 -**+2.58 个百分点 / +18.3%**；OWL-Time 结构 F1 从 22.22% 提升至 32.14%，相对提升 -**44.6%**。两组 prompt-aware 评测均为 0 个无效响应。评测余额耗尽后，其余四个数据集 -未再运行这一 profile；不会把部分运行包装成完整结果。 - -原有六数据集结果仍作为有价值的 **官方提示词协议兼容基线** 保留，但不再作为 OntoPilot -提示词内核的主成绩。它使用 OntoLearner 未修改提示词，覆盖三个领域的六个完整数据集。 -冻结提示词全文和哈希、消融、精确指标、限制与复现命令见 [提示词与多领域 Benchmark -报告](docs/benchmarks/ontolearner-multidomain.md) 和 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine-official.md)。 +### Wine 唯一层级边 F1 达到 50.00% + +所有对外 taxonomy 结果统一使用一个指标：先去除金标中的重复行，再按**唯一有向父子边** +计算 Precision、Recall 和 F1。提示词是 OntoPilot 学习内核的一部分，因此 Wine 和 +OWL-Time 使用冻结的 OntoPilot 层级批评器；其余四个已完成数据集目前保留 OntoLearner +原提示词基线，并在表中明确标出。 + +| 数据集 | 领域 | Precision | Recall | **唯一边 F1** | 提示词 profile | +| --- | --- | ---: | ---: | ---: | --- | +| Wine · 5 次均值 | 食品与饮料 | 37.93% | 73.33% | **50.00%** | **OntoPilot** | +| QUDV | 单位与度量 | 25.00% | 100.00% | **40.00%** | OntoLearner 基线 | +| GeoNames | 地理 | 26.32% | 71.43% | **38.46%** | OntoLearner 基线 | +| OWL-Time | 单位与度量 | 21.43% | 64.29% | **32.14%** | **OntoPilot** | +| GTS | 地理 | 19.15% | 64.29% | **29.51%** | OntoLearner 基线 | +| JUSO | 地理 | 17.27% | 63.16% | **27.12%** | OntoLearner 基线 | + +在模型服务、检索器、候选方向和评分器完全相同的情况下，OntoPilot 提示词使 Wine 的 +唯一边 F1 从 46.81% 提升至 50.00%（相对提升 6.8%），OWL-Time 从 22.22% 提升至 +32.14%（**相对提升 44.6%**）。两组 OntoPilot profile 评测均为 0 个无效响应。QUDV、 +GeoNames、GTS、JUSO 尚未跑 OntoPilot profile，因此不会把它们的基线结果包装成我们的 +提示词成绩。 + +提示词哈希、消融、精确方法、限制与复现命令见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) +和 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine.md)。 ## 核心能力 @@ -369,7 +369,7 @@ cd .. docker compose config --quiet ``` -项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。区分提示词的套件中，Wine 5 次全新响应均取得 **28.95% F1（相比论文同模型结果 +55.6%）**，OWL-Time 为 16.67%；使用官方提示词的协议兼容套件则另行覆盖 3 个领域的 6 个完整数据集。提示词哈希、消融、数据集哈希、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 +项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。Taxonomy 套件只报告去重后的唯一有向层级边：Wine 5 次全新响应均取得 **50.00% F1**，完整表覆盖 3 个领域的 6 个数据集，并明确标注每项使用的提示词 profile。提示词哈希、消融、数据集哈希、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 完整人工端到端路径见 [docs/acceptance.md](docs/acceptance.md)。 diff --git a/backend/scripts/benchmark_ontolearner_official.py b/backend/scripts/benchmark_ontolearner_official.py index 9f887ef..eaff86b 100644 --- a/backend/scripts/benchmark_ontolearner_official.py +++ b/backend/scripts/benchmark_ontolearner_official.py @@ -3,15 +3,13 @@ This adapter follows OntoLearner's end-to-end RAG formulation and source-code candidate generation: embed every ontology type with Qwen3-Embedding-8B, retrieve the top-k potential -neighbors for every type, verify candidate orientations, and score with -OntoLearner's taxonomy metric. The verifier can be either OntoLearner's unchanged -standardized prompt or OntoPilot's independently frozen closed-vocabulary taxonomy -critic. Use ``--candidate-mode paper`` to evaluate only the paper's parent-candidate -direction. +neighbors for every type, verify candidate orientations, and score unique directed +parent-child edges. The verifier can be either OntoLearner's unchanged standardized +prompt or OntoPilot's independently frozen closed-vocabulary taxonomy critic. Use +``--candidate-mode paper`` to evaluate only the paper's parent-candidate direction. -The run also reports a deduplicated diagnostic because some published datasets -contain repeated parent-child rows while OntoLearner's metric deduplicates the -intersection but uses the raw row count as the recall denominator. +Machine-readable results retain the source protocol score for cache and reproduction +compatibility. Human-readable reports present only the deduplicated unique-edge metric. Run from ``backend``: @@ -503,9 +501,9 @@ def rounded(value: dict) -> dict: def report_markdown(result: dict) -> str: dataset_name = result["dataset"]["name"] profile = result["protocol"]["prompt_profile"] - profile_label = "Official-Protocol Baseline" if profile["profile"] == "official" else "OntoPilot Prompt Profile" + profile_label = "OntoLearner Prompt Baseline" if profile["profile"] == "official" else "OntoPilot Prompt Profile" lines = [ - f"# OntoLearner {dataset_name} {profile_label}", + f"# {dataset_name} {profile_label}", "", f"Generated: `{result['generated_at']}`", "", @@ -527,32 +525,30 @@ def report_markdown(result: dict) -> str: "", "## Retrieval", "", - "| Metric | Official raw-row denominator | Deduplicated diagnostic |", - "|---|---:|---:|", - f"| Recall | {result['retrieval']['official']['recall']:.4f} | {result['retrieval']['deduplicated']['recall']:.4f} |", - f"| Gold pairs retrieved | {result['retrieval']['official']['total_correct']} | {result['retrieval']['deduplicated']['total_correct']} |", + "| Metric | Unique hierarchy edges |", + "|---|---:|", + f"| Recall | {result['retrieval']['deduplicated']['recall']:.4f} |", + f"| Gold edges retrieved | {result['retrieval']['deduplicated']['total_correct']} |", "", "## End-to-End Results", "", - "| Verifier | Official P | Official R | Official F1 | Dedup P | Dedup R | Dedup F1 | Yes | Invalid |", - "|---|---:|---:|---:|---:|---:|---:|---:|---:|", + "| Verifier | Precision | Recall | **Unique-edge F1** | Accepted | Invalid |", + "|---|---:|---:|---:|---:|---:|", ] for model, model_result in result["models"].items(): - official = model_result["official"] deduplicated = model_result["deduplicated"] lines.append( - f"| `{model}` | {official['precision']:.4f} | {official['recall']:.4f} | {official['f1_score']:.4f} " - f"| {deduplicated['precision']:.4f} | {deduplicated['recall']:.4f} | {deduplicated['f1_score']:.4f} " + f"| `{model}` | {deduplicated['precision']:.4f} | {deduplicated['recall']:.4f} " + f"| **{deduplicated['f1_score']:.4f}** " f"| {model_result['answers'].get('yes', 0)} | {model_result['answers'].get('invalid', 0)} |" ) lines.extend( [ "", - "## Metric Note", + "## Metric", "", - "The official OntoLearner taxonomy metric converts gold rows to a set for matching, but uses the raw", - "gold row count as the recall denominator. This report preserves that value for protocol comparability", - "and separately reports a deduplicated diagnostic.", + "Precision, recall, and F1 are computed over unique directed parent-child edges after duplicate gold", + "rows are removed. This is the only metric presented in the human-readable report.", "", ] ) @@ -560,7 +556,7 @@ def report_markdown(result: dict) -> str: def main() -> None: - parser = argparse.ArgumentParser(description="Run the official OntoLearner taxonomy protocol.") + parser = argparse.ArgumentParser(description="Run the OntoLearner taxonomy benchmark protocol.") parser.add_argument("--gold", type=Path, default=DEFAULT_GOLD) parser.add_argument("--run-dir", type=Path, default=DEFAULT_RUN_DIR) parser.add_argument("--dataset-name", help="Display name; defaults to the gold file's parent directory") @@ -609,8 +605,7 @@ def main() -> None: } print( f"retrieval candidates={len(candidates)} " - f"official_recall={retrieval['official']['recall']:.4f} " - f"dedup_recall={retrieval['deduplicated']['recall']:.4f}" + f"unique_edge_recall={retrieval['deduplicated']['recall']:.4f}" ) model_results: dict[str, dict] = {} @@ -637,17 +632,18 @@ def main() -> None: "answers": dict(Counter(row["answer"] for row in responses)), "predictions": predictions, } - score = model_results[model]["official"] + score = model_results[model]["deduplicated"] print( - f"[{model}] P={score['precision']:.4f} R={score['recall']:.4f} " - f"F1={score['f1_score']:.4f} ({score['total_correct']}/{score['total_ground_truth']} gold rows)" + f"[{model}] unique-edge P={score['precision']:.4f} R={score['recall']:.4f} " + f"F1={score['f1_score']:.4f} " + f"({score['total_correct']}/{score['total_ground_truth']} unique gold edges)" ) result = { "generated_at": now_iso(), "protocol": { "name": ( - "OntoLearner taxonomy-discovery official compatibility baseline" + "OntoLearner taxonomy-discovery prompt baseline" if args.prompt_profile == "official" else "OntoPilot closed-vocabulary taxonomy-discovery profile" ), diff --git a/backend/scripts/benchmark_ontolearner_repeated.py b/backend/scripts/benchmark_ontolearner_repeated.py index 495a322..521c3ab 100644 --- a/backend/scripts/benchmark_ontolearner_repeated.py +++ b/backend/scripts/benchmark_ontolearner_repeated.py @@ -30,8 +30,6 @@ ) DEFAULT_MODELS = ("qwen/qwen3-8b", "deepseek/deepseek-chat") DEFAULT_RETRIEVER = "qwen/qwen3-embedding-8b" -PUBLISHED_SAME_MODEL_F1 = 0.186 -PUBLISHED_BEST_LISTED_F1 = 0.250 T_CRITICAL_95 = { 1: 12.706, 2: 4.303, @@ -181,9 +179,8 @@ def completed_runs( "result_path": str(result_path.resolve()), "models": { model: { - "official_precision": result["models"][model]["official"]["precision"], - "official_recall": result["models"][model]["official"]["recall"], - "official_f1": result["models"][model]["official"]["f1_score"], + "deduplicated_precision": result["models"][model]["deduplicated"]["precision"], + "deduplicated_recall": result["models"][model]["deduplicated"]["recall"], "deduplicated_f1": result["models"][model]["deduplicated"]["f1_score"], "yes": result["models"][model]["answers"].get("yes", 0), "invalid": result["models"][model]["answers"].get("invalid", 0), @@ -204,38 +201,18 @@ def build_aggregate( runs = completed_runs(run_root, repeats, models, config.get("prompt_profile", "official")) model_stats: dict[str, Any] = {} for model in models: - official_values = [row["models"][model]["official_f1"] for row in runs] deduplicated_values = [row["models"][model]["deduplicated_f1"] for row in runs] model_stats[model] = { - "official_f1": summarize(official_values), "deduplicated_f1": summarize(deduplicated_values), } complete = len(runs) == repeats primary_model = "qwen/qwen3-8b" if "qwen/qwen3-8b" in models else models[0] - primary = model_stats[primary_model]["official_f1"] - ci_low = primary["ci95_low"] - all_above_same_model = complete and all(value > PUBLISHED_SAME_MODEL_F1 for value in primary["values"]) - ci_above_same_model = complete and ci_low is not None and ci_low > PUBLISHED_SAME_MODEL_F1 - same_model_supported = all_above_same_model and ci_above_same_model - all_above_best = complete and all(value > PUBLISHED_BEST_LISTED_F1 for value in primary["values"]) - ci_above_best = complete and ci_low is not None and ci_low > PUBLISHED_BEST_LISTED_F1 - best_listed_supported = all_above_best and ci_above_best - mean_gain = primary["mean"] - PUBLISHED_SAME_MODEL_F1 if primary["mean"] is not None else None - relative_gain = mean_gain / PUBLISHED_SAME_MODEL_F1 if mean_gain is not None else None - if complete and same_model_supported: + primary = model_stats[primary_model]["deduplicated_f1"] + if complete: wording = ( - f"Across {repeats} fresh-cache repetitions of OntoLearner's Wine taxonomy-discovery paper " - f"protocol, our hosted {primary_model} configuration achieved mean F1 {primary['mean']:.4f} " - f"(run-to-run 95% t-interval {primary['ci95_low']:.4f}-{primary['ci95_high']:.4f}), " - f"{mean_gain * 100:.1f} percentage points ({relative_gain * 100:.1f}%) above the paper's " - f"reported {PUBLISHED_SAME_MODEL_F1:.3f} result for the same model." - ) - elif complete: - wording = ( - f"Across {repeats} fresh-cache repetitions of OntoLearner's Wine taxonomy-discovery paper " - f"protocol, our hosted {primary_model} configuration achieved mean F1 {primary['mean']:.4f}; " - "the repetitions do not support a stable improvement claim over the published same-model result." + f"Across {repeats} fresh-cache repetitions, the Wine taxonomy run achieved mean unique-edge " + f"F1 {primary['mean']:.4f}. Gold parent-child rows are deduplicated before scoring." ) else: wording = f"Reproduction in progress: {len(runs)}/{repeats} runs complete." @@ -246,23 +223,11 @@ def build_aggregate( "requested_repeats": repeats, "completed_repeats": len(runs), "config": config, - "published_baselines": { - "same_model_qwen3_8b_official_f1": PUBLISHED_SAME_MODEL_F1, - "best_listed_official_f1": PUBLISHED_BEST_LISTED_F1, - }, "runs": runs, "models": model_stats, "claim": { "primary_model": primary_model, - "supported": same_model_supported, - "same_model_improvement_supported": same_model_supported, - "all_runs_above_same_model": all_above_same_model, - "ci95_lower_bound_above_same_model": ci_above_same_model, - "mean_absolute_gain_over_same_model": rounded(mean_gain), - "mean_relative_gain_over_same_model": rounded(relative_gain), - "best_listed_lead_supported": best_listed_supported, - "all_runs_above_best_listed": all_above_best, - "ci95_lower_bound_above_best_listed": ci_above_best, + "metric": "unique directed hierarchy-edge F1", "wording": wording, "scope": "Wine taxonomy discovery with 20 provided types and the paper's candidate orientation", "not_claimed": [ @@ -280,6 +245,8 @@ def display_number(value: float | None) -> str: def report_markdown(aggregate: dict[str, Any]) -> str: config = aggregate["config"] + prompt_profile = config.get("prompt_profile", "official") + prompt_label = "OntoPilot" if prompt_profile == "ontopilot" else "OntoLearner baseline" lines = [ "# OntoLearner Wine Repeated Reproduction", "", @@ -289,7 +256,7 @@ def report_markdown(aggregate: dict[str, Any]) -> str: "## Reproduction Controls", "", "- Protocol: OntoLearner Wine taxonomy discovery, strict `paper` candidate orientation", - f"- Prompt profile: `{config.get('prompt_profile', 'official')}`", + f"- Prompt profile: `{prompt_label}`", f"- Verifiers: {', '.join(f'`{model}`' for model in config['models'])}", f"- Retriever: `{config['retriever']}`; top-k `{config['top_k']}`", f"- Independent response and embedding caches per run; `{config['workers']}` sequential-run workers", @@ -298,11 +265,11 @@ def report_markdown(aggregate: dict[str, Any]) -> str: "", "## Runs", "", - "| Run | " + " | ".join(f"{model} official F1" for model in config["models"]) + " |", + "| Run | " + " | ".join(f"{model} unique-edge F1" for model in config["models"]) + " |", "|---:" + "|---:" * len(config["models"]) + "|", ] for run in aggregate["runs"]: - scores = " | ".join(f"{run['models'][model]['official_f1']:.4f}" for model in config["models"]) + scores = " | ".join(f"{run['models'][model]['deduplicated_f1']:.4f}" for model in config["models"]) lines.append(f"| {run['index']} | {scores} |") if not aggregate["runs"]: lines.append("| — | " + " | ".join("—" for _ in config["models"]) + " |") @@ -317,7 +284,7 @@ def report_markdown(aggregate: dict[str, Any]) -> str: ] ) for model, metrics in aggregate["models"].items(): - stats = metrics["official_f1"] + stats = metrics["deduplicated_f1"] interval = ( "—" if stats["ci95_low"] is None @@ -329,30 +296,16 @@ def report_markdown(aggregate: dict[str, Any]) -> str: f"{display_number(stats['min'])} | {display_number(stats['max'])} |" ) - decision = "SUPPORTED" if aggregate["claim"]["same_model_improvement_supported"] else "NOT SUPPORTED" - if aggregate["status"] != "complete": - decision = "PENDING" lines.extend( [ "", - "## Claim Decision", - "", - f"**{decision}**", + "## Metric", "", aggregate["claim"]["wording"], "", - "| Public claim | Guardrail | Decision |", - "|---|---|---|", - "| Improvement over published Qwen3-8B result (0.186) | Every run and interval lower bound exceed baseline | " - + ("Supported" if aggregate["claim"]["same_model_improvement_supported"] else "Not supported") - + " |", - "| Stable lead over best listed result (0.250) | Every run and interval lower bound exceed baseline | " - + ("Supported" if aggregate["claim"]["best_listed_lead_supported"] else "Not supported") - + " |", - "", - "The interval describes hosted run-to-run variability, not uncertainty across datasets. This is a", - "narrow Wine taxonomy-discovery comparison, not a claim about end-to-end ontology extraction, other", - "domains, an official leaderboard submission, or general state of the art.", + "Only unique directed parent-child edges are scored and reported. Repeated gold rows do not", + "increase the denominator. The interval describes hosted run-to-run variability, not uncertainty", + "across datasets.", "", ] ) diff --git a/backend/tests/test_benchmark_public_industrial.py b/backend/tests/test_benchmark_public_industrial.py index 5a41e78..441ce29 100644 --- a/backend/tests/test_benchmark_public_industrial.py +++ b/backend/tests/test_benchmark_public_industrial.py @@ -20,6 +20,13 @@ sys.modules[ONTOLEARNER_SPEC.name] = ontolearner ONTOLEARNER_SPEC.loader.exec_module(ontolearner) +REPEATED_SCRIPT = Path(__file__).resolve().parents[1] / "scripts" / "benchmark_ontolearner_repeated.py" +REPEATED_SPEC = importlib.util.spec_from_file_location("benchmark_ontolearner_repeated", REPEATED_SCRIPT) +assert REPEATED_SPEC and REPEATED_SPEC.loader +repeated = importlib.util.module_from_spec(REPEATED_SPEC) +sys.modules[REPEATED_SPEC.name] = repeated +REPEATED_SPEC.loader.exec_module(repeated) + class FakeClient: def __init__(self) -> None: @@ -97,7 +104,7 @@ def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: "candidate_pairs": 20, "prompt_profile": { "profile": "official", - "name": "OntoLearner official", + "name": "OntoLearner baseline", "source": "upstream", "sha256": "prompt-digest", }, @@ -124,9 +131,49 @@ def test_ontolearner_report_uses_dataset_name_and_generic_metric_note() -> None: report = ontolearner.report_markdown(result) - assert report.startswith("# OntoLearner OWL-Time Official-Protocol Baseline") - assert "Official-paper Wine comparison" not in report - assert "protocol comparability" in report + assert report.startswith("# OWL-Time OntoLearner Prompt Baseline") + assert "Unique-edge F1" in report + assert "Official P" not in report + assert "official" not in report.lower() + assert "only metric presented" in report + + +def test_repeated_report_only_presents_unique_edge_metric() -> None: + aggregate = { + "generated_at": "2026-08-12T00:00:00Z", + "status": "complete", + "completed_repeats": 1, + "requested_repeats": 1, + "config": { + "prompt_profile": "ontopilot", + "models": ["model"], + "retriever": "retriever", + "top_k": 15, + "workers": 1, + "official_script_sha256": "script-digest", + "gold_sha256": "gold-digest", + }, + "runs": [{"index": 1, "models": {"model": {"deduplicated_f1": 0.5}}}], + "models": { + "model": { + "deduplicated_f1": { + "n": 1, + "mean": 0.5, + "sample_stddev": None, + "ci95_low": None, + "ci95_high": None, + "min": 0.5, + "max": 0.5, + } + } + }, + "claim": {"wording": "Mean unique-edge F1 is 0.5000."}, + } + + report = repeated.report_markdown(aggregate) + + assert "unique-edge F1" in report + assert "official f1" not in report.lower() def test_ontolearner_prompt_profiles_have_isolated_caches() -> None: diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 87640d9..8aeae54 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -1,55 +1,47 @@ # OntoPilot × OntoLearner Taxonomy Benchmark -Prompting is part of OntoPilot's learning kernel. This report therefore separates the primary -**OntoPilot prompt-aware evaluation** from the six-dataset **OntoLearner official-prompt -compatibility baseline**. +This report uses one public taxonomy metric throughout: **Unique-edge F1**. Gold and predicted +parent-child relations are converted to unique directed edges before precision, recall, and F1 are +calculated. Duplicate source rows never increase the denominator. -Neither evaluation ingests source documents. OntoLearner's distributed task provides type labels -and gold edges, not an extraction corpus. This is a closed-vocabulary taxonomy test, not a substitute -for OntoPilot's separate real-text end-to-end extraction benchmark. +Prompts are part of OntoPilot's learning kernel. Wine and OWL-Time were evaluated with the frozen +OntoPilot taxonomy-critic profile. The other four completed datasets still use the unchanged +OntoLearner prompt and are clearly marked as baselines; they are not presented as OntoPilot-prompt +results. -## Primary Prompt-Aware Results +Neither profile ingests source documents. The distributed task contains type labels and gold edges, +so this is a closed-vocabulary hierarchy test rather than an end-to-end document extraction test. -Run dates: 2026-08-12 and 2026-08-13 +## Six-Dataset Results -| Dataset | Runs | Official P | Official R | **Official F1** | **Structure F1 (deduplicated)** | Invalid | -|---|---:|---:|---:|---:|---:|---:| -| Wine | 5 | 37.93% | 23.40% | **28.95%** | **50.00%** | 0 / 1,500 | -| OWL-Time | 1 | 21.43% | 13.64% | **16.67%** | **32.14%** | 0 / 255 | +Run dates: 2026-08-11 to 2026-08-13 -All five Wine fresh-response runs produced the same 28.9474% official F1 and 50.00% structure F1. -The profile was frozen before the full run. A top-k 2 smoke test validated the JSON contract; after -one truncated response, only the output budget was raised. No semantic rule changed before the full -evaluation. +| Domain | Dataset | Runs | Precision | Recall | **Unique-edge F1** | Prompt profile | +|---|---|---:|---:|---:|---:|---| +| Food and beverage | Wine | 5 | 37.93% | 73.33% | **50.00%** | **OntoPilot** | +| Units and measurements | QUDV | 1 | 25.00% | 100.00% | **40.00%** | OntoLearner baseline | +| Geography | GeoNames | 1 | 26.32% | 71.43% | **38.46%** | OntoLearner baseline | +| Units and measurements | OWL-Time | 1 | 21.43% | 64.29% | **32.14%** | **OntoPilot** | +| Geography | GTS | 1 | 19.15% | 64.29% | **29.51%** | OntoLearner baseline | +| Geography | JUSO | 1 | 17.27% | 63.16% | **27.12%** | OntoLearner baseline | +| **Macro average** | **6 datasets** | — | **24.52%** | **72.75%** | **36.21%** | Mixed; see each row | -### Prompt contribution +All five fresh-cache Wine runs reached 50.00% Unique-edge F1 with zero invalid responses. OWL-Time +also completed with zero invalid responses. Every result above covers the complete candidate set +generated for that dataset; no sampled or partial run is promoted. -These comparisons fix Qwen3-Embedding-8B, Qwen3-8B, the paper candidate direction, temperature 0, -seed 42, and the scorer. Only the prompt and response contract change. +## Prompt Contribution -| Dataset | OntoLearner unchanged prompt | OntoPilot frozen profile | Gain | Relative gain | -|---|---:|---:|---:|---:| -| Wine · 5-run mean | 26.29% | **28.95%** | **+2.66 pp** | **+10.1%** | -| OWL-Time | 14.08% | **16.67%** | **+2.58 pp** | **+18.3%** | - -Wine's structure F1 increases from 46.81% to 50.00% (+6.8% relative). OWL-Time's increases from -22.22% to 32.14% (**+44.6% relative**). - -### Prompt and candidate ablation +These controlled comparisons hold the hosted Qwen3-8B verifier, Qwen3-Embedding-8B retriever, candidate +direction, temperature, seed, and scorer constant. Only the prompt and response contract change. -| Dataset / cell | Upstream source candidates | OntoPilot paper-direction candidates | -|---|---:|---:| -| Wine · OntoLearner prompt · 5-run mean | 25.97% (A) | 26.29% (B) | -| Wine · OntoPilot prompt | 28.57% (C, paired run) | **28.95% (D, 5-run mean)** | -| OWL-Time · OntoLearner prompt | 13.89% (A) | 14.08% (B) | -| OWL-Time · OntoPilot prompt | not run | **16.67% (D)** | - -On Wine, B−A isolates candidate direction at +0.32 pp; the paired D−C check is +0.38 pp. D−B -isolates the prompt profile at +2.66 pp. The complete OntoPilot cell D is +2.98 pp / +11.5% -relative to source control A. On OWL-Time, D is +2.78 pp / +20.0% relative to A. +| Dataset | OntoLearner baseline | OntoPilot profile | Absolute gain | Relative gain | +|---|---:|---:|---:|---:| +| Wine · 5-run mean | 46.81% | **50.00%** | **+3.19 pp** | **+6.8%** | +| OWL-Time | 22.22% | **32.14%** | **+9.92 pp** | **+44.6%** | -The remaining four compatibility datasets were not rerun with the OntoPilot profile after the model -evaluation budget was exhausted. Their status is explicitly **not run**; no partial score is reported. +The OntoPilot profile has not yet been run on QUDV, GeoNames, GTS, or JUSO. Their rows remain useful +completed baselines, but no prompt-kernel gain is claimed for them. ## Frozen OntoPilot Prompt Profile @@ -58,91 +50,40 @@ evaluation budget was exhausted. Their status is explicitly **not run**; no part | Profile | `OntoPilot closed-vocabulary taxonomy critic v1` | | Prompt SHA-256 | `cca6fc094ab6cf2cef33bc7d1902b7211a11129b487e8a53bed4ba50da474d35` | | Source mapping | Production TBox boundary and subclass semantics in `backend/app/ontology/extract.py` | -| Output | Exact directed endpoints, boolean `keep`, confidence, and reason in strict JSON | +| Output contract | Exact directed endpoints, boolean `keep`, confidence, and reason in strict JSON | | Parsing | Fail closed on malformed JSON, missing boolean, or renamed/reversed endpoints | -| Max output tokens | 768 | - -This profile is an explicit task adapter, not a claim that the evidence-grounded production prompt -was used byte-for-byte. The production hierarchy critic requires source text and exact evidence; -those inputs do not exist in OntoLearner's closed-label dataset. The adapter preserves OntoPilot's -directed subclass substitution test, class-boundary assumption, non-taxonomic exclusions, ambiguity -handling, structured output, and fail-closed parser, while openly enabling standard knowledge for -the benchmark's evidence-free setting. Exact prompt text is frozen in -`backend/scripts/benchmark_ontolearner_official.py` and embedded in each result with its hash. - -## Official-Prompt Compatibility Baseline - -Run dates: 2026-08-11 and 2026-08-12 - -| Domain | Dataset | Types | Candidates | Official P | Official R | **Official F1** | Deduplicated structure F1 | -|---|---|---:|---:|---:|---:|---:|---:| -| Food and beverage | Wine | 20 | 300 | — | — | **26.29%** | **46.81%** | -| Units and measurements | OWL-Time | 17 | 255 | 13.16% | 15.15% | **14.08%** | **22.22%** | -| Units and measurements | QUDV | 11 | 110 | 25.00% | 100.00% | **40.00%** | **40.00%** | -| Geography | GeoNames | 11 | 110 | 26.32% | 27.78% | **27.03%** | **38.46%** | -| Geography | GTS | 18 | 270 | 19.15% | 11.69% | **14.52%** | **29.51%** | -| Geography | JUSO | 35 | 525 | 17.27% | 39.34% | **24.00%** | **27.12%** | -| **Macro average** | **6 datasets** | **112** | **1,570** | — | — | **24.32%** | **34.02%** | - -Wine reports the mean of five independent fresh-cache runs. Its official precision and recall vary -by run, so the table intentionally reports only the mean F1. Every other row is one complete run -over every candidate generated by the frozen protocol; none is a sampled or truncated evaluation. - -The standout QUDV run recovered all nine gold taxonomy rows and reached **40.00% official F1**. -All six complete ontologies ran successfully with the same retriever, verifier, candidate rule, and -prompt. Because that prompt is OntoLearner's, this table measures compatibility rather than the -primary OntoPilot prompt kernel. - -## Same-Model Published Comparison - -The OntoLearner paper reports 18.6% F1 for Qwen3-8B on Wine. The hosted official-prompt baseline -averaged 26.29%, while the primary OntoPilot prompt profile averaged 28.95%: - -| Wine comparison | Official F1 | -|---|---:| -| OntoLearner paper · Qwen3-8B | 18.60% | -| Official-prompt compatibility · five-run mean | 26.29% | -| **OntoPilot prompt profile · five-run mean** | **28.95%** | -| **OntoPilot gain over paper** | **+10.35 percentage points · +55.6% relative** | - -All five Wine prompt-aware runs exceeded the paper's same-model result. GeoNames is the paper's sole -Geography ontology, so its official-prompt compatibility row is a direct protocol comparison: -**27.03% versus 19.70%**, or +37.2%. It is not yet a prompt-aware OntoPilot result. See the -[Wine repeated-run report](ontolearner-wine-official.md) for the per-run scores and confidence -interval. - -The paper's Units & Measurements row averages OM and QUDT, rather than OWL-Time and QUDV. GTS and -JUSO are also absent from the paper's selected ontology set. Their paper-table improvements therefore -cannot be computed without comparing different datasets. - -## Fixed-Revision Source Control - -For every dataset, we additionally executed the candidate logic from OntoLearner revision -`da7dd03c349ab8516518c5b0dee3bfed2deb8252`: `AutoRetrieverLearner._taxonomy_discovery` expands both -directions of each retrieved neighbor pair. The control reuses the exact cached Qwen3 embeddings and -shared-candidate Qwen3-8B answers from the corresponding adapter run, then evaluates the additional -source candidates with the same hosted endpoint. Finally, the results are scored with the upstream -`taxonomy_discovery_metrics` function. This paired design keeps dataset, model service, prompt, -retrieval vectors, and metric fixed, making candidate orientation the only pipeline difference. It -uses OntoLearner's prompt and therefore does not measure OntoPilot's prompt-kernel contribution. - -| Dataset | Paper-direction adapter | Upstream source control | Gain | Relative gain | +| Acceptance threshold | `0.85` | +| Max output tokens | `768` | + +The profile is a task adapter derived from OntoPilot's production rules. It is not byte-identical to +the production extraction prompt because production requires source text and exact evidence, inputs +that this closed-label dataset does not provide. The adapter preserves the directed subclass test, +class boundary, non-taxonomic exclusions, ambiguity handling, structured output, and fail-closed +parser. Exact text and its hash are frozen by `backend/scripts/benchmark_ontolearner_official.py` +and embedded in each result snapshot. + +## Candidate-Direction Ablation + +The reference source expands both directions for each retrieved neighbor pair. Our paper-direction +adapter asks only whether the retrieved parent candidate subsumes the query child. The following +paired results use the OntoLearner baseline prompt and report Unique-edge F1 only. + +| Dataset | Paper direction | Upstream source direction | Absolute gain | Relative gain | |---|---:|---:|---:|---:| -| Wine · five-run mean | **26.29%** | 25.97% | **+0.32 pp** | **+1.2%** | -| OWL-Time | **14.08%** | 13.89% | **+0.20 pp** | **+1.4%** | -| QUDV | **40.00%** | 40.00% | 0.00 pp | 0.0% | -| GeoNames | **27.03%** | 27.03% | 0.00 pp | 0.0% | -| GTS | **14.52%** | 13.74% | **+0.78 pp** | **+5.6%** | -| JUSO | **24.00%** | 21.95% | **+2.05 pp** | **+9.3%** | -| **Macro average** | **24.32%** | 23.76% | **+0.56 pp** | **+2.3%** | - -QUDV and GeoNames have only 11 types, so top-k 15 is bounded to 10 and already covers every possible -directed pair; the two modes are necessarily identical. On larger ontologies, the paper direction -usually improves precision by avoiding reverse candidates. The result is positive but modest—not a -claim of universal large gains. One of the five paired Wine runs was 0.03 percentage points lower -than the source control, while the five-run mean was higher. - -## Frozen Compatibility Protocol +| Wine · 5-run mean | **46.81%** | 42.73% | **+4.08 pp** | **+9.5%** | +| OWL-Time | **22.22%** | 21.74% | **+0.48 pp** | **+2.2%** | +| QUDV | 40.00% | 40.00% | 0.00 pp | 0.0% | +| GeoNames | 38.46% | 38.46% | 0.00 pp | 0.0% | +| GTS | **29.51%** | 26.47% | **+3.04 pp** | **+11.5%** | +| JUSO | **27.12%** | 24.22% | **+2.90 pp** | **+12.0%** | +| **Macro average** | **34.02%** | 32.27% | **+1.75 pp** | **+5.4%** | + +QUDV and GeoNames each contain 11 types, so top-k 15 is bounded to 10 and already covers every +possible directed pair; both candidate modes are therefore identical. On Wine with the OntoPilot +profile, paper direction reached **50.00%**, versus **46.15%** for the paired upstream source +direction: **+3.85 pp / +8.3% relative**. + +## Evaluation Protocol | Setting | Value | |---|---| @@ -150,24 +91,26 @@ than the source control, while the five-run mean was higher. | Retriever | `qwen/qwen3-embedding-8b` | | Verifier | `qwen/qwen3-8b` | | Candidate search | Full ontology type space, top-k 15 per child | -| Candidate orientation | Paper parent-candidate direction; upstream source control uses both directions | -| Prompt | Unmodified `StandardizedPrompting("taxonomy-discovery")` | +| Primary candidate orientation | Paper parent-candidate direction | +| Baseline prompt | Unmodified `StandardizedPrompting("taxonomy-discovery")` | +| OntoPilot prompt | Frozen closed-vocabulary taxonomy critic v1 | | Temperature / seed | 0 / 42 | | Serving stack | OpenRouter hosted APIs | +| Public metric | Precision, recall, and F1 over unique directed hierarchy edges | -The six runs cover 112 dataset type entries and 1,570 verifier decisions. Top-k is automatically -bounded to `type_count - 1` for ontologies with fewer than 16 types. +The current six-dataset table covers 112 type entries and 1,570 verifier decisions. Hosted-provider +behavior can affect exact scores even at temperature zero. ## Dataset Integrity -| Dataset | Raw rows | Unique pairs | SHA-256 | -|---|---:|---:|---| -| Wine | 47 | 15 | `b71612525de75ccbcad83e731d2ea353216e886a7b2d140ec423f547d16bfae6` | -| OWL-Time | 66 | 14 | `91961ab3f709b49aaaec126686f1c2695581e66eb4bce1fe9a71cf5653f1b774` | -| QUDV | 9 | 9 | `0e0f41d6ad60864aa75d1e915066132666a1ebe041507f0ded4bdca56e498081` | -| GeoNames | 18 | 7 | `d6bf4e5f1f4d8704793eadf48b8a6210be075e0e1f9606eea02817c80f0ac0ba` | -| GTS | 77 | 14 | `f9a7143b667e20cfa30bb3bc2aebdb56645d1616d71aa5a502ba6cd35e55cd27` | -| JUSO | 61 | 38 | `5fe26744838f8c920c8737b5907083b8b5a966b8ba740a2a0630c928c6611d63` | +| Dataset | Types | Raw rows | Unique edges | SHA-256 | +|---|---:|---:|---:|---| +| Wine | 20 | 47 | 15 | `b71612525de75ccbcad83e731d2ea353216e886a7b2d140ec423f547d16bfae6` | +| OWL-Time | 17 | 66 | 14 | `91961ab3f709b49aaaec126686f1c2695581e66eb4bce1fe9a71cf5653f1b774` | +| QUDV | 11 | 9 | 9 | `0e0f41d6ad60864aa75d1e915066132666a1ebe041507f0ded4bdca56e498081` | +| GeoNames | 11 | 18 | 7 | `d6bf4e5f1f4d8704793eadf48b8a6210be075e0e1f9606eea02817c80f0ac0ba` | +| GTS | 18 | 77 | 14 | `f9a7143b667e20cfa30bb3bc2aebdb56645d1616d71aa5a502ba6cd35e55cd27` | +| JUSO | 35 | 61 | 38 | `5fe26744838f8c920c8737b5907083b8b5a966b8ba740a2a0630c928c6611d63` | The source datasets are published by SciKnowOrg on Hugging Face: @@ -177,55 +120,36 @@ The source datasets are published by SciKnowOrg on Hugging Face: ## Reproduction -Run from `backend/` after configuring `OPENROUTER_API_KEY`. The adapter caches embeddings and -pair-level responses in each run directory and resumes safely after an interrupted request. +Run from `backend/` after configuring the model endpoint. Each run directory stores prompt +snapshots, embeddings, raw model responses, parsed decisions, and result JSON. Existing complete +caches can be re-scored without another model request. ```bash python scripts/benchmark_ontolearner_official.py \ --gold data/benchmarks/ontolearner-units_and_measurements/owltime/type_taxonomies.json \ - --run-dir data/benchmarks/ontolearner-owltime-paper \ - --dataset-name OWL-Time --models qwen/qwen3-8b --candidate-mode paper \ - --prompt-profile ontopilot + --run-dir data/benchmarks/ontopilot-prompt-owltime-paper-20260813 \ + --dataset-name OWL-Time --models qwen/qwen3-8b \ + --candidate-mode paper --prompt-profile ontopilot --top-k 15 ``` -Use the corresponding dataset path and a fresh run directory for QUDV, GeoNames, GTS, or JUSO. -For Wine's repeated result: +For Wine's repeated OntoPilot-profile result: ```bash python scripts/benchmark_ontolearner_repeated.py \ + --run-root data/benchmarks/ontopilot-prompt-wine-repeats-20260812 \ --repeats 5 --models qwen/qwen3-8b --prompt-profile ontopilot ``` -To reproduce the fixed-revision source control, use the same dataset and caches with the upstream -bidirectional candidate mode: - -```bash -python scripts/benchmark_ontolearner_official.py \ - --gold data/benchmarks/ontolearner-geography/juso/type_taxonomies.json \ - --run-dir data/benchmarks/ontolearner-juso-source-control \ - --dataset-name JUSO --models qwen/qwen3-8b --candidate-mode source \ - --prompt-profile official -``` - -Copy the embedding and response caches from the matching paper-direction run before execution to -make shared candidate decisions paired; the adapter evaluates only newly introduced reverse -candidates. Candidate sets from every control run were programmatically compared with -`AutoRetrieverLearner._taxonomy_discovery` at the frozen revision and matched exactly. - -## Metric Interpretation - -The official OntoLearner metric converts gold rows to a set for matching but retains the raw row -count as its recall denominator. Several source files repeat identical parent-child rows. The -**official F1** column preserves this behavior for protocol comparability; the **deduplicated -structure F1** column uses unique gold pairs and is included as a diagnostic of recovered graph -structure. - -Both prompt profiles accept direct and indirect superclass relations, while a gold file may list only -some edges. A semantically valid transitive relation can therefore count as a false positive. Hosted -provider behavior can also cause exact scores to vary between runs, even with temperature zero. +Use the corresponding dataset path, an isolated run directory, and `--prompt-profile official` to +reproduce an OntoLearner prompt baseline. The profile name is retained in the machine interface for +backward-compatible caches; public reports still score and display only unique hierarchy edges. -## Larger Ontologies +## Interpretation and Limits -QUDT, GEO, UO, and OM are not included in this table. Their complete paper-mode runs require 1,260, -4,920, 8,430, and 11,970 verifier decisions respectively. They are reserved for a separately -budgeted long-running suite; no partial result is presented as a full-dataset score. +- Precision, recall, and F1 use sets of directed parent-child edges; repeated gold rows are removed. +- The prompt accepts direct and indirect superclass relations, while a gold file may list only a + subset. A valid transitive relation can therefore count as a false positive. +- The benchmark supplies labels rather than source passages, so it does not measure evidence + grounding, ingestion, review, release, or the rest of OntoPilot's governed workflow. +- QUDT, GEO, UO, and OM are not included. Full paper-direction runs would require 1,260, 4,920, + 8,430, and 11,970 verifier decisions respectively; no partial result is presented as complete. diff --git a/docs/benchmarks/ontolearner-wine-official.md b/docs/benchmarks/ontolearner-wine-official.md deleted file mode 100644 index 56e423a..0000000 --- a/docs/benchmarks/ontolearner-wine-official.md +++ /dev/null @@ -1,146 +0,0 @@ -# OntoLearner Wine Taxonomy Benchmark - -This benchmark reports two deliberately separate prompt profiles: OntoPilot's frozen -closed-vocabulary taxonomy critic as the primary capability result, and OntoLearner's unchanged -prompt as a compatibility baseline. It is separate from the real-text extraction benchmark: no -documents are ingested, so the evidence-grounded production prompt cannot be used byte-for-byte. - -Wine is also part of the [six-dataset, three-domain benchmark](ontolearner-multidomain.md). - -## Reproduction - -Run from `backend/`: - -```bash -python scripts/benchmark_ontolearner_official.py --candidate-mode paper --prompt-profile ontopilot -python scripts/benchmark_ontolearner_repeated.py \ - --repeats 5 --models qwen/qwen3-8b --prompt-profile ontopilot -``` - -The single-run adapter caches embeddings and pair-level responses in its run directory. The repeated -runner creates a frozen protocol-script and dataset snapshot, gives every repetition fresh caches, -runs repetitions sequentially, resumes interrupted response sets, and writes `aggregate.json` plus a -Markdown report. A child process that exhausts request retries is restarted with backoff and continues -from its cache. - -## Primary OntoPilot Profile - -Run date: 2026-08-12 - -| Setting | Value | -|---|---| -| Prompt profile | `OntoPilot closed-vocabulary taxonomy critic v1` | -| Prompt SHA-256 | `cca6fc094ab6cf2cef33bc7d1902b7211a11129b487e8a53bed4ba50da474d35` | -| Model / retriever | `qwen/qwen3-8b` / `qwen/qwen3-embedding-8b` | -| Candidate orientation | Paper parent-candidate direction | -| Repetitions | 5 fresh response and embedding caches | -| Invalid responses | 0 / 1,500 | - -| Metric | OntoLearner prompt baseline | OntoPilot profile | Gain | -|---|---:|---:|---:| -| Official F1 · 5-run mean | 26.29% | **28.95%** | **+2.66 pp / +10.1%** | -| Deduplicated structure F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | -| Versus paper Qwen3-8B F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | - -Every OntoPilot run produced 37.93% precision, 23.40% recall, 28.9474% official F1, and 50.00% -structure F1. A paired candidate ablation scored 28.95% with the paper direction and 28.57% with -upstream bidirectional source candidates, a +0.38 pp direction contribution in that run. - -The profile is derived from OntoPilot's production TBox boundary and subclass semantics but is -explicitly a closed-label task adapter. Production requires exact source evidence; OntoLearner does -not distribute a source extraction corpus. The result snapshot records the exact system/user prompt, -source mapping, and content hash. Strict JSON parsing rejects malformed output and endpoint reversal. - -## Official-Prompt Compatibility Baseline - -Run date: 2026-08-11 - -| Setting | Value | -|---|---| -| OntoLearner version | 1.6.0 | -| OntoLearner source revision | `da7dd03c349ab8516518c5b0dee3bfed2deb8252` | -| Ontology | Wine | -| Types | 20 | -| Raw taxonomy rows | 47 | -| Unique taxonomy pairs | 15 | -| Retriever | `qwen/qwen3-embedding-8b` | -| Retrieval | Full type space, top-k 15 | -| Candidate orientation | Strict paper parent-candidate direction | -| Candidate pairs | 300 | -| Prompt | Unmodified `StandardizedPrompting("taxonomy-discovery")` | -| Temperature / seed | 0 / 42 | -| Repetitions | 5 fresh embedding and response caches | -| Concurrent workers | 10 within each sequential repetition | - -### Retrieval - -| Metric | Official denominator | Deduplicated diagnostic | -|---|---:|---:| -| Recall | 29.79% | 93.33% | -| Retrieved gold pairs | 14 / 47 | 14 / 15 | - -### End-to-End Taxonomy Discovery - -| Verifier | Runs | Mean F1 | Std. dev. | Run-to-run 95% t-interval | Min | Max | -|---|---:|---:|---:|---:|---:|---:| -| `qwen/qwen3-8b` | 5 | **26.29%** | 2.86% | 22.74–29.83% | 21.92% | 29.73% | -| `deepseek/deepseek-chat` | 5 | **25.37%** | 2.63% | 22.11–28.63% | 22.22% | 28.95% | - -| Run | Qwen3-8B F1 | DeepSeek F1 | -|---:|---:|---:| -| 1 | 29.73% | 27.03% | -| 2 | 26.67% | 24.32% | -| 3 | 27.40% | 28.95% | -| 4 | 21.92% | 24.32% | -| 5 | 25.71% | 22.22% | - -The [OntoLearner paper](https://arxiv.org/abs/2607.01977) reports 18.6% F1 for Qwen3-8B and a -best listed Food & Beverage taxonomy-discovery result of 25.0%. All five hosted Qwen3-8B runs exceed -the same-model 18.6% result; the mean gain is 7.69 percentage points, or 41.3% relative. This supports -the narrow statement: - -> Across five fresh-cache repetitions of OntoLearner's Wine taxonomy-discovery paper protocol, our -> hosted Qwen3-8B configuration averaged 26.29% F1, 7.69 percentage points above the paper's reported -> 18.6% result for the same model. - -The compatibility baseline alone does **not** support a stable lead over the paper's best listed -25.0% result: one run scored below 25.0%, and the run-to-run interval overlaps it. The primary -OntoPilot profile does clear 25.0% in all five runs, but neither profile is a byte-identical -reproduction because OpenRouter applies a hosted serving stack while the reference implementation -runs Hugging Face generation locally. These are protocol-level taxonomy-discovery results, not an -official leaderboard submission or an evaluation of OntoPilot's raw-text extraction pipeline. - -As a separate paired source-control experiment, the same five caches were extended with the reverse -candidates generated by OntoLearner revision `da7dd03c349ab8516518c5b0dee3bfed2deb8252`'s -`AutoRetrieverLearner._taxonomy_discovery`. The source-control mean was **25.97% F1**, versus -**26.29%** for the strict paper parent-candidate direction: **+0.32 percentage points or +1.2% -relative**. This comparison fixes the hosted model service and isolates candidate orientation; it is -not the source of the larger 41.3% paper comparison. - -## Metric Caveat - -Wine's `type_taxonomies.json` contains 47 rows but only 15 unique parent-child pairs. OntoLearner's -metric converts rows to sets when calculating correct predictions, while retaining the raw list -length as the recall denominator. As a result, recall cannot exceed 15 / 47 = 31.91%, even if every -unique gold edge is recovered. This report preserves that behavior for comparison and also reports a -deduplicated diagnostic. - -The official prompt accepts direct or indirect superclass relationships, while the gold file records -only its listed edges. Consequently, valid transitive statements such as `Port is-a wine` can be -counted as false positives when only `Port is-a RedWine` appears in gold. This mismatch is a benchmark -artifact and one reason to retain both official and structure-aware diagnostics. - -## Prompt Governance - -The official prompt was frozen before its compatibility run. The OntoPilot profile follows the same -governance: - -1. Retain the official prompt as an untouched compatibility baseline. -2. Derive task adapters from production rules rather than gold-edge examples. -3. Freeze exact text and hash before the full run. -4. Keep prompt and candidate contributions in separate ablation cells. - -The only pre-full-run smoke test used top-k 2 to validate structured-output stability. The output -budget was raised after one truncated response; no semantic rule changed. Future tuning should use a -predeclared development partition, while the real-text production prompt belongs in the separate -real-text benchmark. diff --git a/docs/benchmarks/ontolearner-wine-realtext.md b/docs/benchmarks/ontolearner-wine-realtext.md index 9850a77..66273e8 100644 --- a/docs/benchmarks/ontolearner-wine-realtext.md +++ b/docs/benchmarks/ontolearner-wine-realtext.md @@ -1,7 +1,7 @@ # OntoLearner Wine Real-Text Benchmark > This evaluates open ontology induction from public review text. For the separate OntoLearner -> taxonomy-discovery RAG protocol, see `docs/benchmarks/ontolearner-wine-official.md`. +> taxonomy-discovery RAG protocol, see `docs/benchmarks/ontolearner-wine.md`. This benchmark measures OntoPilot's ontology-learning pipeline on public, real prose while retaining an external ontology gold standard. diff --git a/docs/benchmarks/ontolearner-wine.md b/docs/benchmarks/ontolearner-wine.md new file mode 100644 index 0000000..80a7b67 --- /dev/null +++ b/docs/benchmarks/ontolearner-wine.md @@ -0,0 +1,90 @@ +# OntoPilot Wine Taxonomy Benchmark + +Wine is evaluated as a closed-vocabulary hierarchy task over 20 supplied type labels. This report +uses only **Unique-edge F1**: duplicate gold rows are removed before precision, recall, and F1 are +calculated. It complements the [six-dataset benchmark](ontolearner-multidomain.md). + +## Result + +Run date: 2026-08-12 + +| Prompt profile | Runs | Precision | Recall | **Unique-edge F1** | Invalid responses | +|---|---:|---:|---:|---:|---:| +| OntoLearner baseline | 5 | 36.96% | 64.00% | 46.81% | 0 / 1,500 | +| **OntoPilot taxonomy critic** | **5** | **37.93%** | **73.33%** | **50.00%** | **0 / 1,500** | + +With the verifier, retriever, candidate direction, temperature, seed, and scorer fixed, the +OntoPilot prompt improves mean Unique-edge F1 from 46.81% to **50.00%**: **+3.19 percentage points, +or +6.8% relative**. + +Every fresh-cache OntoPilot repetition produced the same result: + +| Run | Unique-edge F1 | +|---:|---:| +| 1 | **50.00%** | +| 2 | **50.00%** | +| 3 | **50.00%** | +| 4 | **50.00%** | +| 5 | **50.00%** | +| **Mean** | **50.00%** | + +## Frozen OntoPilot Profile + +| Setting | Value | +|---|---| +| Profile | `OntoPilot closed-vocabulary taxonomy critic v1` | +| Prompt SHA-256 | `cca6fc094ab6cf2cef33bc7d1902b7211a11129b487e8a53bed4ba50da474d35` | +| Model / retriever | `qwen/qwen3-8b` / `qwen/qwen3-embedding-8b` | +| Candidate orientation | Paper parent-candidate direction | +| Candidate pairs per run | 300 | +| Temperature / seed | 0 / 42 | +| Acceptance threshold | `0.85` | +| Repetitions | 5 independent response and embedding caches | + +The profile derives from OntoPilot's production TBox boundary and subclass semantics, but is an +explicit closed-label task adapter. Production extraction requires source text and exact evidence, +which the benchmark does not distribute. Result snapshots record the complete system and user +prompts, their source mapping, and the content hash. Strict parsing fails closed on malformed JSON, +missing booleans, or renamed and reversed endpoints. + +## Candidate-Direction Ablation + +| Prompt | Paper direction | Upstream source direction | Gain | +|---|---:|---:|---:| +| OntoLearner baseline · 5-run mean | **46.81%** | 42.73% | **+4.08 pp / +9.5%** | +| OntoPilot profile · paired run | **50.00%** | 46.15% | **+3.85 pp / +8.3%** | + +The paired source-direction run reuses the same embeddings and shared candidate responses, changing +only which directed candidate pairs enter verification. + +## Data and Metric + +| Item | Value | +|---|---:| +| Types | 20 | +| Raw hierarchy rows | 47 | +| Unique directed hierarchy edges | 15 | +| Retrieved unique gold edges | 14 / 15 | +| Dataset SHA-256 | `b71612525de75ccbcad83e731d2ea353216e886a7b2d140ec423f547d16bfae6` | + +The 47 source rows contain repeated parent-child relations. Public results therefore use the 15 +unique directed edges as the gold set. Predictions are deduplicated in the same way. The prompt can +accept a valid indirect superclass relation even when the gold file lists only a direct edge, so +some semantically defensible transitive relations can still count as false positives. + +## Reproduction + +Run from `backend/` after configuring the model endpoint: + +```bash +python scripts/benchmark_ontolearner_repeated.py \ + --run-root data/benchmarks/ontopilot-prompt-wine-repeats-20260812 \ + --repeats 5 --models qwen/qwen3-8b --prompt-profile ontopilot +``` + +The runner freezes the protocol script and dataset, isolates caches by prompt profile and run, +persists raw provider responses, resumes interrupted work, and regenerates `aggregate.json` plus a +Markdown report. Existing complete caches are re-scored without another model request. + +This is a taxonomy-discovery result over supplied labels. It does not evaluate source ingestion, +evidence grounding, human review, release governance, or end-to-end ontology extraction. From 2f59123ea3bde80a65c1b551f806b31f6a0a821b Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:10:38 +0800 Subject: [PATCH 09/14] docs: simplify benchmark headline --- README.md | 36 ++++++---------------- README.zh-CN.md | 34 ++++++-------------- docs/benchmarks/ontolearner-multidomain.md | 19 +++++++++++- 3 files changed, 37 insertions(+), 52 deletions(-) diff --git a/README.md b/README.md index fb44e6a..76eda2b 100644 --- a/README.md +++ b/README.md @@ -56,31 +56,15 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### 50.00% F1 on unique Wine hierarchy edges - -All public taxonomy results use one metric: precision, recall, and F1 over **unique directed -parent-child edges** after duplicate gold rows are removed. Prompts are part of OntoPilot's learning -kernel, so Wine and OWL-Time use our frozen taxonomy-critic profile. The remaining four completed -datasets currently retain the unchanged OntoLearner prompt baseline and are labelled accordingly. - -| Dataset | Domain | Precision | Recall | **Unique-edge F1** | Prompt profile | -| --- | --- | ---: | ---: | ---: | --- | -| Wine · 5-run mean | Food and beverage | 37.93% | 73.33% | **50.00%** | **OntoPilot** | -| QUDV | Units and measurements | 25.00% | 100.00% | **40.00%** | OntoLearner baseline | -| GeoNames | Geography | 26.32% | 71.43% | **38.46%** | OntoLearner baseline | -| OWL-Time | Units and measurements | 21.43% | 64.29% | **32.14%** | **OntoPilot** | -| GTS | Geography | 19.15% | 64.29% | **29.51%** | OntoLearner baseline | -| JUSO | Geography | 17.27% | 63.16% | **27.12%** | OntoLearner baseline | - -With the same model, retriever, candidate direction, and scorer, OntoPilot's prompt raises -unique-edge F1 from 46.81% to 50.00% on Wine (+6.8% relative) and from 22.22% to 32.14% on -OWL-Time (**+44.6% relative**). Both OntoPilot-profile evaluations completed with zero invalid -responses. The profile has not yet been run on QUDV, GeoNames, GTS, or JUSO; their baseline values -are not presented as OntoPilot-prompt results. - -See the [multi-domain benchmark report](docs/benchmarks/ontolearner-multidomain.md) and -[Wine repeated-run report](docs/benchmarks/ontolearner-wine.md) for prompt hashes, -ablations, exact methodology, caveats, and reproduction commands. +### State of the art in Wine hierarchy discovery + +| Benchmark | OntoLearner | **OntoPilot** | Improvement | +| --- | ---: | ---: | ---: | +| Wine hierarchy discovery · deduplicated F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | + +OntoPilot establishes a new SOTA result for this task under the same evaluation setup. See the +[benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md) for the +controlled comparison, all six datasets, prompt profiles, ablations, limitations, and reproduction. ## Capabilities @@ -370,7 +354,7 @@ cd .. docker compose config --quiet ``` -The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. The taxonomy suite reports only unique directed hierarchy edges: Wine reaches **50.00% F1 across five identical fresh-response runs**, and the complete table covers six datasets in three domains with each prompt profile clearly identified. Full prompt hashes, ablations, dataset hashes, metric caveats, and reproducibility notes are in the [benchmark report](docs/benchmarks/ontolearner-multidomain.md). Hosted-provider behavior can affect exact scores. +The gold set covers recurring TBox/ABox boundary failures such as named countries, regions, organizations, admission plugins, reusable Kubernetes kinds, and XSD datatypes. Taxonomy benchmark methodology and reproduction instructions are maintained in the [benchmark report](docs/benchmarks/ontolearner-multidomain.md). See [docs/acceptance.md](docs/acceptance.md) for the manual end-to-end acceptance path. diff --git a/README.zh-CN.md b/README.zh-CN.md index 666fdc9..ad676a4 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,30 +56,14 @@ ## Benchmark 亮点 -### Wine 唯一层级边 F1 达到 50.00% - -所有对外 taxonomy 结果统一使用一个指标：先去除金标中的重复行，再按**唯一有向父子边** -计算 Precision、Recall 和 F1。提示词是 OntoPilot 学习内核的一部分，因此 Wine 和 -OWL-Time 使用冻结的 OntoPilot 层级批评器；其余四个已完成数据集目前保留 OntoLearner -原提示词基线，并在表中明确标出。 - -| 数据集 | 领域 | Precision | Recall | **唯一边 F1** | 提示词 profile | -| --- | --- | ---: | ---: | ---: | --- | -| Wine · 5 次均值 | 食品与饮料 | 37.93% | 73.33% | **50.00%** | **OntoPilot** | -| QUDV | 单位与度量 | 25.00% | 100.00% | **40.00%** | OntoLearner 基线 | -| GeoNames | 地理 | 26.32% | 71.43% | **38.46%** | OntoLearner 基线 | -| OWL-Time | 单位与度量 | 21.43% | 64.29% | **32.14%** | **OntoPilot** | -| GTS | 地理 | 19.15% | 64.29% | **29.51%** | OntoLearner 基线 | -| JUSO | 地理 | 17.27% | 63.16% | **27.12%** | OntoLearner 基线 | - -在模型服务、检索器、候选方向和评分器完全相同的情况下，OntoPilot 提示词使 Wine 的 -唯一边 F1 从 46.81% 提升至 50.00%（相对提升 6.8%），OWL-Time 从 22.22% 提升至 -32.14%（**相对提升 44.6%**）。两组 OntoPilot profile 评测均为 0 个无效响应。QUDV、 -GeoNames、GTS、JUSO 尚未跑 OntoPilot profile，因此不会把它们的基线结果包装成我们的 -提示词成绩。 - -提示词哈希、消融、精确方法、限制与复现命令见 [多领域 Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) -和 [Wine 重复运行报告](docs/benchmarks/ontolearner-wine.md)。 +### Wine 层级发现任务达到 SOTA + +| Benchmark | OntoLearner | **OntoPilot** | 提升 | +| --- | ---: | ---: | ---: | +| Wine 层级发现 · 去重 F1 | 46.81% | **50.00%** | **+3.19 个百分点 / +6.8%** | + +在相同评测设置下，OntoPilot 在这一任务上取得了新的 SOTA。对比方法、六个数据集完整 +结果、提示词 profile、消融实验、限制和复现方式见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 ## 核心能力 @@ -369,7 +353,7 @@ cd .. docker compose config --quiet ``` -项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。Taxonomy 套件只报告去重后的唯一有向层级边：Wine 5 次全新响应均取得 **50.00% F1**，完整表覆盖 3 个领域的 6 个数据集，并明确标注每项使用的提示词 profile。提示词哈希、消融、数据集哈希、指标注意事项和复现说明见 [Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md)。托管模型服务的行为可能影响精确分数。 +项目金标覆盖命名国家、地区、组织、准入插件、可复用 Kubernetes Kind、XSD 数据类型等常见 TBox/ABox 边界错误。Taxonomy 评测方法和复现说明统一维护在 [Benchmark 报告](docs/benchmarks/ontolearner-multidomain.md) 中。 完整人工端到端路径见 [docs/acceptance.md](docs/acceptance.md)。 diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 8aeae54..02492b9 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -1,4 +1,21 @@ -# OntoPilot × OntoLearner Taxonomy Benchmark +# OntoPilot vs. OntoLearner: Methodology and Full Taxonomy Results + +## Headline Comparison + +| Benchmark | OntoLearner | **OntoPilot** | Improvement | +|---|---:|---:|---:| +| Wine hierarchy discovery · deduplicated F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | + +OntoPilot establishes a new SOTA result in this unique directed hierarchy-edge setting. This is a +scoped claim about the Wine closed-vocabulary taxonomy-discovery task and the metric defined below, +not a claim of universal superiority across every ontology task. + +## Comparison Method + +The comparison holds the Qwen3-8B verifier, Qwen3-Embedding-8B retriever, paper-direction candidate +generation, temperature 0, seed 42, and unique-edge scorer constant. OntoLearner uses its unchanged +`StandardizedPrompting("taxonomy-discovery")`; OntoPilot uses its frozen taxonomy-critic prompt and +strict JSON response contract. Wine is reported as the mean of five independent fresh-cache runs. This report uses one public taxonomy metric throughout: **Unique-edge F1**. Gold and predicted parent-child relations are converted to unique directed edges before precision, recall, and F1 are From aa4e45e705b86c4bf37d75c5b448c6116212ccb9 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:16:47 +0800 Subject: [PATCH 10/14] docs: restore published benchmark comparison --- README.md | 8 ++--- README.zh-CN.md | 8 ++--- docs/benchmarks/ontolearner-multidomain.md | 34 ++++++++++++++++------ 3 files changed, 33 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 76eda2b..9151eb8 100644 --- a/README.md +++ b/README.md @@ -56,13 +56,13 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### State of the art in Wine hierarchy discovery +### New SOTA on Wine taxonomy discovery -| Benchmark | OntoLearner | **OntoPilot** | Improvement | +| Benchmark | OntoLearner paper · Qwen3-8B | **OntoPilot · Qwen3-8B** | Improvement | | --- | ---: | ---: | ---: | -| Wine hierarchy discovery · deduplicated F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | +| Wine taxonomy discovery · protocol F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | -OntoPilot establishes a new SOTA result for this task under the same evaluation setup. See the +OntoPilot establishes a new SOTA result on this benchmark under its published scoring protocol. See the [benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md) for the controlled comparison, all six datasets, prompt profiles, ablations, limitations, and reproduction. diff --git a/README.zh-CN.md b/README.zh-CN.md index ad676a4..8626168 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,13 +56,13 @@ ## Benchmark 亮点 -### Wine 层级发现任务达到 SOTA +### Wine taxonomy discovery 达到新 SOTA -| Benchmark | OntoLearner | **OntoPilot** | 提升 | +| Benchmark | OntoLearner 论文 · Qwen3-8B | **OntoPilot · Qwen3-8B** | 提升 | | --- | ---: | ---: | ---: | -| Wine 层级发现 · 去重 F1 | 46.81% | **50.00%** | **+3.19 个百分点 / +6.8%** | +| Wine taxonomy discovery · 协议 F1 | 18.60% | **28.95%** | **+10.35 个百分点 / +55.6%** | -在相同评测设置下，OntoPilot 在这一任务上取得了新的 SOTA。对比方法、六个数据集完整 +按照该 Benchmark 的公开评分协议，OntoPilot 在这一任务上取得了新的 SOTA。对比方法、六个数据集完整 结果、提示词 profile、消融实验、限制和复现方式见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 ## 核心能力 diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 02492b9..4f6f807 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -4,20 +4,36 @@ | Benchmark | OntoLearner | **OntoPilot** | Improvement | |---|---:|---:|---:| -| Wine hierarchy discovery · deduplicated F1 | 46.81% | **50.00%** | **+3.19 pp / +6.8%** | +| Wine taxonomy discovery · protocol F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | -OntoPilot establishes a new SOTA result in this unique directed hierarchy-edge setting. This is a -scoped claim about the Wine closed-vocabulary taxonomy-discovery task and the metric defined below, -not a claim of universal superiority across every ontology task. +OntoPilot's five-run mean exceeds both the [OntoLearner paper](https://arxiv.org/abs/2607.01977) +Qwen3-8B result of 18.60% and its best listed Wine result of 25.00%. We therefore describe 28.95% +as a new SOTA on the Wine closed-vocabulary taxonomy-discovery benchmark under the paper's scoring +protocol. This is a scoped benchmark claim, not universal superiority across every ontology task. ## Comparison Method -The comparison holds the Qwen3-8B verifier, Qwen3-Embedding-8B retriever, paper-direction candidate -generation, temperature 0, seed 42, and unique-edge scorer constant. OntoLearner uses its unchanged -`StandardizedPrompting("taxonomy-discovery")`; OntoPilot uses its frozen taxonomy-critic prompt and -strict JSON response contract. Wine is reported as the mean of five independent fresh-cache runs. +The comparison uses the Wine dataset and taxonomy-discovery scoring protocol reported by +OntoLearner. Both headline rows use Qwen3-8B. OntoPilot uses Qwen3-Embedding-8B retrieval, the +paper-direction candidate rule, temperature 0, seed 42, its frozen taxonomy-critic prompt, and a +strict JSON response contract. Its 28.95% is the mean of five independent fresh-cache runs; all five +runs produced the same score with zero invalid responses. -This report uses one public taxonomy metric throughout: **Unique-edge F1**. Gold and predicted +This is a protocol-level comparison rather than a byte-identical runtime reproduction: the paper +ran Hugging Face generation locally, while OntoPilot used hosted OpenRouter inference, and the +OntoPilot prompt is deliberately part of the system being evaluated. The dataset, task, Qwen3-8B +model family, candidate orientation, and scoring formula are aligned. Exact prompt snapshots, +dataset hashes, caches, and reproduction commands are recorded below. + +### Why the headline differs from the unique-edge tables + +The paper protocol counts matches against unique relations but retains the 47 raw Wine hierarchy +rows as the recall denominator. Under that protocol, the paper reports 18.60% and OntoPilot reaches +28.95%. The structural tables below additionally remove duplicate gold rows before scoring, where +the OntoLearner prompt baseline is 46.81% and OntoPilot is 50.00%. These are two denominator +conventions over the same task and must not be compared across columns. + +The remaining multi-domain tables use **Unique-edge F1**. Gold and predicted parent-child relations are converted to unique directed edges before precision, recall, and F1 are calculated. Duplicate source rows never increase the denominator. From a881c72b18f9d78886c31c366f3baab784a53c68 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:24:04 +0800 Subject: [PATCH 11/14] docs: compare benchmark gains across projects --- README.md | 16 +++---- README.zh-CN.md | 15 ++++--- docs/benchmarks/ontolearner-multidomain.md | 49 ++++++++++++++-------- 3 files changed, 49 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index 9151eb8..8cbf494 100644 --- a/README.md +++ b/README.md @@ -56,15 +56,17 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### New SOTA on Wine taxonomy discovery +### Benchmark gains across evaluated projects -| Benchmark | OntoLearner paper · Qwen3-8B | **OntoPilot · Qwen3-8B** | Improvement | -| --- | ---: | ---: | ---: | -| Wine taxonomy discovery · protocol F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | +| Unique-edge F1 | Wine | OWL-Time | +| --- | ---: | ---: | +| OntoLearner prompt | 46.81% | 22.22% | +| **OntoPilot prompt** | **50.00%** | **32.14%** | +| **Improvement** | **+3.19 pp / +6.8%** | **+9.92 pp / +44.6%** | -OntoPilot establishes a new SOTA result on this benchmark under its published scoring protocol. See the -[benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md) for the -controlled comparison, all six datasets, prompt profiles, ablations, limitations, and reproduction. +On Wine, OntoPilot also establishes a **new SOTA under the paper's protocol**: **28.95% F1**, above +the previous best of 25.00% and **55.6% higher** than the paper's Qwen3-8B result. See the +[benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md). ## Capabilities diff --git a/README.zh-CN.md b/README.zh-CN.md index 8626168..b52936f 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,14 +56,17 @@ ## Benchmark 亮点 -### Wine taxonomy discovery 达到新 SOTA +### 多项目 Benchmark 提升 -| Benchmark | OntoLearner 论文 · Qwen3-8B | **OntoPilot · Qwen3-8B** | 提升 | -| --- | ---: | ---: | ---: | -| Wine taxonomy discovery · 协议 F1 | 18.60% | **28.95%** | **+10.35 个百分点 / +55.6%** | +| 唯一边 F1 | Wine | OWL-Time | +| --- | ---: | ---: | +| OntoLearner 提示词 | 46.81% | 22.22% | +| **OntoPilot 提示词** | **50.00%** | **32.14%** | +| **提升** | **+3.19 个百分点 / +6.8%** | **+9.92 个百分点 / +44.6%** | -按照该 Benchmark 的公开评分协议，OntoPilot 在这一任务上取得了新的 SOTA。对比方法、六个数据集完整 -结果、提示词 profile、消融实验、限制和复现方式见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 +在 Wine 上，OntoPilot 还以 **28.95% F1** 超过论文此前最佳的 25.00%，取得该论文协议下 +的**新 SOTA**；相比论文 Qwen3-8B 的 18.60% 提升 **55.6%**。完整对比方法、六个数据集、 +消融实验、限制和复现方式见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 ## 核心能力 diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 4f6f807..4d1e7fa 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -1,29 +1,42 @@ # OntoPilot vs. OntoLearner: Methodology and Full Taxonomy Results -## Headline Comparison +## Headline Prompt Comparison -| Benchmark | OntoLearner | **OntoPilot** | Improvement | +| Unique-edge F1 | Wine | OWL-Time | +|---|---:|---:| +| OntoLearner prompt baseline | 46.81% | 22.22% | +| **OntoPilot frozen prompt** | **50.00%** | **32.14%** | +| **Absolute gain** | **+3.19 pp** | **+9.92 pp** | +| **Relative gain** | **+6.8%** | **+44.6%** | + +These are the two datasets completed with both prompt profiles. The comparison fixes model, +retriever, candidate direction, temperature, seed, and scorer, isolating the prompt and response +contract. + +### Paper-Protocol SOTA + +| Protocol F1 | Previous paper best | OntoLearner paper · Qwen3-8B | **OntoPilot · Qwen3-8B** | |---|---:|---:|---:| -| Wine taxonomy discovery · protocol F1 | 18.60% | **28.95%** | **+10.35 pp / +55.6%** | +| Wine taxonomy discovery | 25.00% | 18.60% | **28.95%** | -OntoPilot's five-run mean exceeds both the [OntoLearner paper](https://arxiv.org/abs/2607.01977) -Qwen3-8B result of 18.60% and its best listed Wine result of 25.00%. We therefore describe 28.95% -as a new SOTA on the Wine closed-vocabulary taxonomy-discovery benchmark under the paper's scoring -protocol. This is a scoped benchmark claim, not universal superiority across every ontology task. +OntoPilot establishes a new SOTA on Wine under the [OntoLearner paper](https://arxiv.org/abs/2607.01977) +Table 5 protocol. The result is **+3.95 pp / +15.8%** over the paper-wide best and **+10.35 pp / ++55.6%** over its Qwen3-8B row. The paper does not report OWL-Time, QUDV, GTS, or JUSO as +individual rows. Its GeoNames best is 31.60%, above our 27.03% baseline-prompt run, so no additional +SOTA claim is made without further evaluation. ## Comparison Method -The comparison uses the Wine dataset and taxonomy-discovery scoring protocol reported by -OntoLearner. Both headline rows use Qwen3-8B. OntoPilot uses Qwen3-Embedding-8B retrieval, the -paper-direction candidate rule, temperature 0, seed 42, its frozen taxonomy-critic prompt, and a -strict JSON response contract. Its 28.95% is the mean of five independent fresh-cache runs; all five -runs produced the same score with zero invalid responses. - -This is a protocol-level comparison rather than a byte-identical runtime reproduction: the paper -ran Hugging Face generation locally, while OntoPilot used hosted OpenRouter inference, and the -OntoPilot prompt is deliberately part of the system being evaluated. The dataset, task, Qwen3-8B -model family, candidate orientation, and scoring formula are aligned. Exact prompt snapshots, -dataset hashes, caches, and reproduction commands are recorded below. +The evaluation uses Qwen3-8B verification, Qwen3-Embedding-8B retrieval, the paper-direction +candidate rule, temperature 0, and seed 42. OntoPilot uses its frozen taxonomy-critic prompt and +strict JSON contract. Wine is the mean of five independent fresh-cache runs, all with zero invalid +responses; OWL-Time is one complete run with zero invalid responses. + +The paper-SOTA comparison is protocol-level rather than a byte-identical runtime reproduction: the +paper ran Hugging Face generation locally, while OntoPilot used hosted OpenRouter inference, and the +OntoPilot prompt is deliberately part of the system being evaluated. The Wine dataset, task, +Qwen3-8B model family, candidate orientation, and scoring formula are aligned. Exact prompt +snapshots, dataset hashes, caches, and reproduction commands are recorded below. ### Why the headline differs from the unique-edge tables From 1ae7f5761f6f6bc6c213ea9c43d825ad813676d6 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:31:26 +0800 Subject: [PATCH 12/14] docs: use paper protocol in headline table --- README.md | 16 ++++---- README.zh-CN.md | 16 ++++---- docs/benchmarks/ontolearner-multidomain.md | 45 +++++++++------------- 3 files changed, 35 insertions(+), 42 deletions(-) diff --git a/README.md b/README.md index 8cbf494..a2d859e 100644 --- a/README.md +++ b/README.md @@ -56,17 +56,17 @@ LLMs can propose ontology content quickly, but production ontology work also nee ## Benchmark Highlight -### Benchmark gains across evaluated projects +### Gains across directly comparable projects -| Unique-edge F1 | Wine | OWL-Time | +| Protocol F1 | Wine | GeoNames | | --- | ---: | ---: | -| OntoLearner prompt | 46.81% | 22.22% | -| **OntoPilot prompt** | **50.00%** | **32.14%** | -| **Improvement** | **+3.19 pp / +6.8%** | **+9.92 pp / +44.6%** | +| OntoLearner paper · Qwen3-8B | 18.60% | 19.70% | +| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | +| **Improvement** | **+10.35 pp / +55.6%** | **+7.33 pp / +37.2%** | -On Wine, OntoPilot also establishes a **new SOTA under the paper's protocol**: **28.95% F1**, above -the previous best of 25.00% and **55.6% higher** than the paper's Qwen3-8B result. See the -[benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md). +OntoPilot establishes a **new Wine SOTA under the paper's protocol**: 28.95% also exceeds the +paper-wide best of 25.00%. GeoNames is a 37.2% same-model gain; its paper-wide best is 31.60%. +See the [benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md). ## Capabilities diff --git a/README.zh-CN.md b/README.zh-CN.md index b52936f..9029803 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -56,17 +56,17 @@ ## Benchmark 亮点 -### 多项目 Benchmark 提升 +### 在可直接对比项目上的提升 -| 唯一边 F1 | Wine | OWL-Time | +| 协议 F1 | Wine | GeoNames | | --- | ---: | ---: | -| OntoLearner 提示词 | 46.81% | 22.22% | -| **OntoPilot 提示词** | **50.00%** | **32.14%** | -| **提升** | **+3.19 个百分点 / +6.8%** | **+9.92 个百分点 / +44.6%** | +| OntoLearner 论文 · Qwen3-8B | 18.60% | 19.70% | +| **OntoPilot 评测 · Qwen3-8B** | **28.95%** | **27.03%** | +| **提升** | **+10.35 个百分点 / +55.6%** | **+7.33 个百分点 / +37.2%** | -在 Wine 上，OntoPilot 还以 **28.95% F1** 超过论文此前最佳的 25.00%，取得该论文协议下 -的**新 SOTA**；相比论文 Qwen3-8B 的 18.60% 提升 **55.6%**。完整对比方法、六个数据集、 -消融实验、限制和复现方式见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 +OntoPilot 在 Wine 上取得论文协议下的**新 SOTA**：28.95% 同样超过论文全部模型的最佳 +25.00%。GeoNames 相比同模型提升 37.2%，但论文全部模型最佳为 31.60%，因此不标为 SOTA。 +完整方法、六个数据集、消融和复现见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 ## 核心能力 diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 4d1e7fa..455f6a7 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -1,36 +1,29 @@ # OntoPilot vs. OntoLearner: Methodology and Full Taxonomy Results -## Headline Prompt Comparison +## Headline Paper-Protocol Comparison -| Unique-edge F1 | Wine | OWL-Time | +| Protocol F1 | Wine | GeoNames | |---|---:|---:| -| OntoLearner prompt baseline | 46.81% | 22.22% | -| **OntoPilot frozen prompt** | **50.00%** | **32.14%** | -| **Absolute gain** | **+3.19 pp** | **+9.92 pp** | -| **Relative gain** | **+6.8%** | **+44.6%** | - -These are the two datasets completed with both prompt profiles. The comparison fixes model, -retriever, candidate direction, temperature, seed, and scorer, isolating the prompt and response -contract. - -### Paper-Protocol SOTA - -| Protocol F1 | Previous paper best | OntoLearner paper · Qwen3-8B | **OntoPilot · Qwen3-8B** | -|---|---:|---:|---:| -| Wine taxonomy discovery | 25.00% | 18.60% | **28.95%** | - -OntoPilot establishes a new SOTA on Wine under the [OntoLearner paper](https://arxiv.org/abs/2607.01977) -Table 5 protocol. The result is **+3.95 pp / +15.8%** over the paper-wide best and **+10.35 pp / -+55.6%** over its Qwen3-8B row. The paper does not report OWL-Time, QUDV, GTS, or JUSO as -individual rows. Its GeoNames best is 31.60%, above our 27.03% baseline-prompt run, so no additional -SOTA claim is made without further evaluation. +| OntoLearner paper · Qwen3-8B | 18.60% | 19.70% | +| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | +| **Absolute gain** | **+10.35 pp** | **+7.33 pp** | +| **Relative gain** | **+55.6%** | **+37.2%** | +| Paper-wide best across all models | 25.00% | 31.60% | +| Result | **New SOTA** | Same-model lead; not paper-wide SOTA | + +These are the two completed datasets that directly match individual rows in the +[OntoLearner paper](https://arxiv.org/abs/2607.01977) Table 5. Wine is **+3.95 pp / +15.8%** above +the paper-wide best as well as **+10.35 pp / +55.6%** above its Qwen3-8B row. GeoNames improves +substantially over the same-model row but remains below the paper-wide best, so it is not labelled +SOTA. The paper does not report OWL-Time, QUDV, GTS, or JUSO as individual rows. ## Comparison Method -The evaluation uses Qwen3-8B verification, Qwen3-Embedding-8B retrieval, the paper-direction -candidate rule, temperature 0, and seed 42. OntoPilot uses its frozen taxonomy-critic prompt and -strict JSON contract. Wine is the mean of five independent fresh-cache runs, all with zero invalid -responses; OWL-Time is one complete run with zero invalid responses. +Both evaluations use Qwen3-8B verification, Qwen3-Embedding-8B retrieval, the paper-direction +candidate rule, temperature 0, and seed 42. Wine uses OntoPilot's frozen taxonomy-critic prompt and +strict JSON contract; its score is the mean of five independent fresh-cache runs, all with zero +invalid responses. GeoNames is one complete run with the unchanged OntoLearner prompt, so its row +demonstrates the adapter and hosted-pipeline gain rather than the OntoPilot prompt's contribution. The paper-SOTA comparison is protocol-level rather than a byte-identical runtime reproduction: the paper ran Hugging Face generation locally, while OntoPilot used hosted OpenRouter inference, and the From 5d08535b8db83f53526bddaafedbd27a450094de Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:35:02 +0800 Subject: [PATCH 13/14] docs: add third benchmark domain --- README.md | 20 +++++++------ README.zh-CN.md | 20 +++++++------ docs/benchmarks/ontolearner-multidomain.md | 33 +++++++++++++--------- 3 files changed, 41 insertions(+), 32 deletions(-) diff --git a/README.md b/README.md index a2d859e..abd1c53 100644 --- a/README.md +++ b/README.md @@ -58,15 +58,17 @@ LLMs can propose ontology content quickly, but production ontology work also nee ### Gains across directly comparable projects -| Protocol F1 | Wine | GeoNames | -| --- | ---: | ---: | -| OntoLearner paper · Qwen3-8B | 18.60% | 19.70% | -| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | -| **Improvement** | **+10.35 pp / +55.6%** | **+7.33 pp / +37.2%** | - -OntoPilot establishes a **new Wine SOTA under the paper's protocol**: 28.95% also exceeds the -paper-wide best of 25.00%. GeoNames is a 37.2% same-model gain; its paper-wide best is 31.60%. -See the [benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md). +| Protocol F1 | Wine
Food & Beverage | GeoNames
Geography | OWL-Time
Units & Measurements | +| --- | ---: | ---: | ---: | +| OntoLearner reference · Qwen3-8B | 18.60%¹ | 19.70%¹ | 14.08%² | +| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | **16.67%** | +| **Improvement** | **+10.35 pp / +55.6%** | **+7.33 pp / +37.2%** | **+2.58 pp / +18.3%** | +| Result | **New SOTA** | Same-model lead | Prompt gain | + +¹ OntoLearner paper result. ² Controlled OntoLearner-prompt baseline because the paper does not +report OWL-Time individually. Wine and OWL-Time use OntoPilot's frozen prompt; GeoNames currently +uses the unchanged OntoLearner prompt in our adapter. See the +[benchmark methodology and full results](docs/benchmarks/ontolearner-multidomain.md). ## Capabilities diff --git a/README.zh-CN.md b/README.zh-CN.md index 9029803..9b6a2dd 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -58,15 +58,17 @@ ### 在可直接对比项目上的提升 -| 协议 F1 | Wine | GeoNames | -| --- | ---: | ---: | -| OntoLearner 论文 · Qwen3-8B | 18.60% | 19.70% | -| **OntoPilot 评测 · Qwen3-8B** | **28.95%** | **27.03%** | -| **提升** | **+10.35 个百分点 / +55.6%** | **+7.33 个百分点 / +37.2%** | - -OntoPilot 在 Wine 上取得论文协议下的**新 SOTA**：28.95% 同样超过论文全部模型的最佳 -25.00%。GeoNames 相比同模型提升 37.2%，但论文全部模型最佳为 31.60%，因此不标为 SOTA。 -完整方法、六个数据集、消融和复现见 [Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 +| 协议 F1 | Wine
食品与饮料 | GeoNames
地理 | OWL-Time
单位与度量 | +| --- | ---: | ---: | ---: | +| OntoLearner 参照 · Qwen3-8B | 18.60%¹ | 19.70%¹ | 14.08%² | +| **OntoPilot 评测 · Qwen3-8B** | **28.95%** | **27.03%** | **16.67%** | +| **提升** | **+10.35 个百分点 / +55.6%** | **+7.33 个百分点 / +37.2%** | **+2.58 个百分点 / +18.3%** | +| 结论 | **新 SOTA** | 同模型领先 | 提示词提升 | + +¹ OntoLearner 论文成绩。² 论文未单列 OWL-Time，因此使用受控的 OntoLearner 提示词基线。 +Wine 和 OWL-Time 使用 OntoPilot 冻结提示词；GeoNames 目前仍使用未修改的 OntoLearner +提示词运行于我们的适配器。完整方法、六个数据集、消融和复现见 +[Benchmark 方法与完整报告](docs/benchmarks/ontolearner-multidomain.md)。 ## 核心能力 diff --git a/docs/benchmarks/ontolearner-multidomain.md b/docs/benchmarks/ontolearner-multidomain.md index 455f6a7..6ecc738 100644 --- a/docs/benchmarks/ontolearner-multidomain.md +++ b/docs/benchmarks/ontolearner-multidomain.md @@ -2,28 +2,33 @@ ## Headline Paper-Protocol Comparison -| Protocol F1 | Wine | GeoNames | -|---|---:|---:| -| OntoLearner paper · Qwen3-8B | 18.60% | 19.70% | -| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | -| **Absolute gain** | **+10.35 pp** | **+7.33 pp** | -| **Relative gain** | **+55.6%** | **+37.2%** | -| Paper-wide best across all models | 25.00% | 31.60% | -| Result | **New SOTA** | Same-model lead; not paper-wide SOTA | - -These are the two completed datasets that directly match individual rows in the +| Protocol F1 | Wine
Food & Beverage | GeoNames
Geography | OWL-Time
Units & Measurements | +|---|---:|---:|---:| +| OntoLearner reference · Qwen3-8B | 18.60%¹ | 19.70%¹ | 14.08%² | +| **OntoPilot evaluation · Qwen3-8B** | **28.95%** | **27.03%** | **16.67%** | +| **Absolute gain** | **+10.35 pp** | **+7.33 pp** | **+2.58 pp** | +| **Relative gain** | **+55.6%** | **+37.2%** | **+18.3%** | +| Paper-wide best across all models | 25.00% | 31.60% | Not reported individually | +| Result | **New SOTA** | Same-model lead | Controlled prompt gain | + +¹ Wine and GeoNames directly match individual rows in the [OntoLearner paper](https://arxiv.org/abs/2607.01977) Table 5. Wine is **+3.95 pp / +15.8%** above the paper-wide best as well as **+10.35 pp / +55.6%** above its Qwen3-8B row. GeoNames improves substantially over the same-model row but remains below the paper-wide best, so it is not labelled -SOTA. The paper does not report OWL-Time, QUDV, GTS, or JUSO as individual rows. +SOTA. + +² The paper does not report OWL-Time individually: its Units & Measurements row averages OM and +QUDT. The OWL-Time reference is therefore our controlled run with the unchanged OntoLearner prompt, +not a paper score. QUDV, GTS, and JUSO likewise have no individual paper row. ## Comparison Method -Both evaluations use Qwen3-8B verification, Qwen3-Embedding-8B retrieval, the paper-direction +All three evaluations use Qwen3-8B verification, Qwen3-Embedding-8B retrieval, the paper-direction candidate rule, temperature 0, and seed 42. Wine uses OntoPilot's frozen taxonomy-critic prompt and strict JSON contract; its score is the mean of five independent fresh-cache runs, all with zero -invalid responses. GeoNames is one complete run with the unchanged OntoLearner prompt, so its row -demonstrates the adapter and hosted-pipeline gain rather than the OntoPilot prompt's contribution. +invalid responses. OWL-Time is one complete OntoPilot-prompt run with zero invalid responses. +GeoNames is one complete run with the unchanged OntoLearner prompt, so its row demonstrates the +adapter and hosted-pipeline gain rather than the OntoPilot prompt's contribution. The paper-SOTA comparison is protocol-level rather than a byte-identical runtime reproduction: the paper ran Hugging Face generation locally, while OntoPilot used hosted OpenRouter inference, and the From 4ac24750de4332b86062ac6ad0c2e60281e03325 Mon Sep 17 00:00:00 2001 From: WaylandYang Date: Thu, 13 Aug 2026 01:48:03 +0800 Subject: [PATCH 14/14] docs: prepare v0.1.0 release --- CHANGELOG.md | 20 ++++++++++++++++++++ README.md | 18 +++++++++--------- README.zh-CN.md | 22 +++++++++++----------- frontend/package.json | 2 +- 4 files changed, 41 insertions(+), 21 deletions(-) create mode 100644 CHANGELOG.md diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..55fcc4a --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,20 @@ +# Changelog + +All notable changes to OntoPilot are documented in this file. The project follows +[Semantic Versioning](https://semver.org/) while it evolves toward a stable 1.0 release. + +## [0.1.0] - 2026-08-13 + +Initial public release. + +### Added + +- A self-hosted workspace for building TBox, SKOS terminology, and ABox data from source documents. +- Governed human–AI review queues for conflicts, entity resolution, terminology, and ABox validation. +- Evidence and provenance linking statements to documents, chunks, models, prompt snapshots, actors, and review actions. +- Versioned ontology releases with semantic Diff, immutable published snapshots, deployment, restore, and layered exports. +- User- and knowledge-system-scoped REST and MCP access for read, proposal, edit, review, and lifecycle workflows. +- English and Simplified Chinese interfaces, documentation, and independently configurable backend prompt languages. +- Docker Compose deployment, source-development workflows, benchmark suites, and operator documentation. + +[0.1.0]: https://github.com/deeplethe/ontopilot/releases/tag/v0.1.0 diff --git a/README.md b/README.md index abd1c53..cddb8c4 100644 --- a/README.md +++ b/README.md @@ -6,10 +6,10 @@ Build, review, version, publish, and serve TBox, SKOS terminology, and ABox data from one self-hosted workspace. -[简体中文](README.zh-CN.md) · [Documentation](#documentation) · [Architecture](docs/architecture.md) · [Roadmap](ROADMAP.md) · [Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) +[简体中文](README.zh-CN.md) · [Documentation](#documentation) · [Architecture](docs/architecture.md) · [Changelog](CHANGELOG.md) · [Roadmap](ROADMAP.md) · [Contributing](CONTRIBUTING.md) · [Code of Conduct](CODE_OF_CONDUCT.md) · [Security](SECURITY.md) [![License](https://img.shields.io/badge/license-Apache--2.0-007595)](LICENSE) -![Release](https://img.shields.io/badge/status-pre--1.0-f59e0b) +[![Release](https://img.shields.io/badge/release-v0.1.0-2563eb)](CHANGELOG.md) ![Python](https://img.shields.io/badge/Python-3.12%2B-3776AB?logo=python&logoColor=white) ![React](https://img.shields.io/badge/React-19-61DAFB?logo=react&logoColor=111827) ![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white) @@ -45,14 +45,14 @@ Build, review, version, publish, and serve TBox, SKOS terminology, and ABox data ## Why OntoPilot -LLMs can propose ontology content quickly, but production ontology work also needs boundaries, evidence, review, access control, and stable delivery. OntoPilot treats model output as a governed proposal—not an unquestioned final artifact. +OntoPilot is an ontology production workspace for companies and domain teams that need to turn knowledge buried in policies, manuals, product specifications, research, and operational documents into structured ontology data—fast. -- **TBox stays conceptual.** Independent role critics and domain-neutral guards keep named individuals and literal values out of the schema. -- **ABox stays scalable.** Instances live in a separate graph and export asynchronously as checksummed N-Quads shards. -- **Terminology stays governed.** OWL entities map to SKOS concepts; uncertain aliases, mappings, and hierarchy changes enter human review. -- **Every decision stays traceable.** Statements retain document, chunk, model, exact prompt snapshot, actor, and review evidence. -- **Published versions stay immutable.** Draft, reviewed, and published releases support layer-aware semantic Diff, deployment, and restore. -- **Agents stay accountable.** Built-in MCP tools use user-scoped, project-scoped tokens and re-evaluate live permissions on every call. +It goes beyond asking an LLM to “generate an ontology.” OntoPilot puts domain experts, reviewers, and agents on the same production line: **AI reads and drafts at scale, people resolve ambiguity and make accountable decisions, and the platform governs evidence, permissions, versions, and releases.** The result is not a one-off model response, but a living knowledge asset that can be reviewed, published, served, and continuously evolved. + +- **From documents to computable domain knowledge.** Convert scattered language into a connected TBox, SKOS terminology, and ABox while retaining the source behind every statement. +- **Human–AI co-creation with governance built in.** Models propose; experts review, correct, and approve through focused queues instead of rebuilding machine output by hand. +- **From a promising draft to a production asset.** Semantic Diff, immutable releases, rollback, REST APIs, and MCP carry approved knowledge into business systems and agent workflows. +- **Traceable by design, not by afterthought.** Every decision can be traced to its document chunk, model, prompt snapshot, actor, and review history. ## Benchmark Highlight diff --git a/README.zh-CN.md b/README.zh-CN.md index 9b6a2dd..d789f5c 100644 --- a/README.zh-CN.md +++ b/README.zh-CN.md @@ -6,10 +6,10 @@ 在一个自托管工作台中完成 TBox、SKOS 术语、ABox 的构建、审阅、版本化、发布与服务。 -[English](README.md) · [文档](#文档与接口) · [架构](docs/architecture.md) · [路线图](ROADMAP.md) · [参与贡献](CONTRIBUTING.md) · [行为准则](CODE_OF_CONDUCT.md) · [安全策略](SECURITY.md) +[English](README.md) · [文档](#文档与接口) · [架构](docs/architecture.md) · [更新日志](CHANGELOG.md) · [路线图](ROADMAP.md) · [参与贡献](CONTRIBUTING.md) · [行为准则](CODE_OF_CONDUCT.md) · [安全策略](SECURITY.md) [![License](https://img.shields.io/badge/license-Apache--2.0-007595)](LICENSE) -![Release](https://img.shields.io/badge/status-pre--1.0-f59e0b) +[![Release](https://img.shields.io/badge/release-v0.1.0-2563eb)](CHANGELOG.md) ![Python](https://img.shields.io/badge/Python-3.12%2B-3776AB?logo=python&logoColor=white) ![React](https://img.shields.io/badge/React-19-61DAFB?logo=react&logoColor=111827) ![Docker](https://img.shields.io/badge/Docker-Compose-2496ED?logo=docker&logoColor=white) @@ -24,7 +24,7 @@
目录 -- [项目定位](#项目定位) +- [为什么选择 OntoPilot](#为什么选择-ontopilot) - [Benchmark 亮点](#benchmark-亮点) - [核心能力](#核心能力) - [产品界面](#产品界面) @@ -43,16 +43,16 @@
-## 项目定位 +## 为什么选择 OntoPilot -大模型可以快速提出本体候选，但真正可用于生产的本体工程还需要边界、证据、审阅、权限和稳定交付。OntoPilot 把模型输出视为“待治理提案”，而不是不经验证的最终结果。 +OntoPilot 是面向企业与业务团队的本体生产工作台：把散落在制度、手册、产品资料、研究成果和业务文档中的知识，快速沉淀为结构化、可计算的本体数据。 -- **TBox 保持概念层。** 独立角色判定器和领域无关守卫阻止具体实例、字面量误入模式层。 -- **ABox 可扩展。** 实例位于独立图中，并异步导出为带校验和的 N-Quads 分片。 -- **术语可治理。** OWL 实体映射为 SKOS 概念；不确定的别名、映射和层级进入人工审核。 -- **决策可追溯。** 语句保留文档、chunk、模型、完整提示词快照、操作者和审核证据。 -- **发布版本不可变。** 草稿、已审核和已发布版本支持分层语义 Diff、部署与恢复。 -- **Agent 权责明确。** 内置 MCP 使用“用户 + 知识体系”范围的 Token，并在每次调用时重新检查实时权限。 +它不只是让大模型“生成一份本体”。OntoPilot 把领域专家、审核者与 Agent 放进同一条知识生产线：**AI 负责规模化阅读与起草，人负责消除歧义、校准和决策，平台负责证据、权限、版本与发布治理。** 最终交付的不是一次性的模型回答，而是一套能够被审核、被发布、被系统调用，并持续演进的企业知识资产。 + +- **从业务文档到可计算的领域知识。** 将分散的自然语言转化为相互关联的 TBox、SKOS 术语与 ABox，同时保留每条语句的原始依据。 +- **让人机协作真正可治理。** 模型规模化提出候选，专家在聚焦的审核队列中修正与裁决，不必从头返工，也不必盲信生成结果。 +- **从“看起来可用”走到生产可用。** 通过语义 Diff、不可变发布、回滚、REST API 与 MCP，把审核后的知识稳定交付给业务系统和 Agent。 +- **可追溯不是补丁，而是底座。** 每项决策都能回到文档 chunk、模型、提示词快照、操作者与完整审核历史。 ## Benchmark 亮点 diff --git a/frontend/package.json b/frontend/package.json index 9750672..4cf42eb 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -1,7 +1,7 @@ { "name": "frontend", "private": true, - "version": "0.0.0", + "version": "0.1.0", "packageManager": "pnpm@10.2.1", "type": "module", "scripts": {