diff --git a/pyproject.toml b/pyproject.toml index 02d4f00..e911c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803..4caf94c 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -156,13 +156,34 @@ def _normalize_token(raw: str) -> str: return str(float(raw.replace(",", ""))) except ValueError: pass - core = raw.strip(_STRIP_EDGE) + core = _strip_edges_keeping_leading_decimal(raw) try: return str(float(core.replace(",", ""))) except ValueError: return _PUNCT.sub("", raw) +def _strip_edges_keeping_leading_decimal(raw: str) -> str: + """Strip edge punctuation like raw.strip(_STRIP_EDGE), except that a . + starting a leading-decimal number (. directly followed by a digit) is kept. + + A plain strip treats the . of ".5." or "$.5" as wrapping noise and + yields "5", so the token normalizes to "5.0" — equal to a gold "5" + (false positive) and unequal to the value-identical "0.5" (false negative). + The official DROP normalizer never corrupts a number this way, so the left-edge + scan stops as soon as stripping one more character would break a leading decimal. + """ + end = len(raw) + while end > 0 and raw[end - 1] in _STRIP_EDGE: + end -= 1 + start = 0 + while start < end and raw[start] in _STRIP_EDGE: + if raw[start] == "." and start + 1 < end and raw[start + 1].isdigit(): + break + start += 1 + return raw[start:end] + + def _split_internal_hyphens(token: str) -> list[str]: """Split ``token`` on INTERNAL hyphens, matching DROP's ``re.split(" |-", ...)``. diff --git a/src/trinity/standings.py b/src/trinity/standings.py index f535a7d..9f6dfff 100644 --- a/src/trinity/standings.py +++ b/src/trinity/standings.py @@ -117,12 +117,16 @@ def compute_standings(leaderboard: Mapping[str, Any]) -> Standings: per_benchmark = h.get("per_benchmark") if miner is None or not isinstance(per_benchmark, dict): continue - best = per_miner.setdefault(str(miner), {}) + miner_key = str(miner) for bench, score in per_benchmark.items(): if not _is_num(score): continue b = str(bench) seen_benches.add(b) + # Register the miner only once a real numeric score is confirmed. A merged win + # whose per_benchmark is empty or all-non-numeric contributes nothing and must + # not create a phantom standing (matching the non-dict per_benchmark case). + best = per_miner.setdefault(miner_key, {}) if b not in best or float(score) > best[b]: best[b] = float(score) diff --git a/tests/test_standings.py b/tests/test_standings.py index 7bca49f..22cf582 100644 --- a/tests/test_standings.py +++ b/tests/test_standings.py @@ -151,6 +151,32 @@ def test_non_dict_per_benchmark_is_skipped(): assert [m.miner for m in s.miners] == ["r"] +def test_win_with_no_numeric_score_creates_no_phantom_leader(): + # Regression: a merged win whose per_benchmark is empty (or entirely non-numeric) + # must NOT register the miner. Pre-fix, setdefault ran before any numeric score was + # confirmed, so such a win produced a phantom MinerStanding(overall=0.0, n_competed=0, + # rank=1) that was falsely reported as the standings leader. This must match the + # non-dict per_benchmark case, which drops the record entirely. + lb = _lb([ + {"miner": "ghost", "merged": True, "per_benchmark": {}, "pr": 1, + "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + {"miner": "nul", "merged": True, "per_benchmark": {"math500": None, "mmlu": "x"}, + "pr": 2, "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + ]) + s = compute_standings(lb) + assert s.miners == [] # neither win registers a miner + assert s.leader is None # no phantom leader + + # And a phantom win must not shadow a real one: only the scored miner ranks/leads. + lb2 = _lb([ + {"miner": "ghost", "merged": True, "per_benchmark": {}, "pr": 1, + "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + _win("real", {"math500": 0.7, "mmlu": 0.8}, 2), + ]) + s2 = compute_standings(lb2) + assert [m.miner for m in s2.miners] == ["real"] and s2.leader == "real" + + # --------------------------------------------------------------------------- # # load + render # --------------------------------------------------------------------------- #