From 15ebe88581a9267ee713e65971094561f0508fe6 Mon Sep 17 00:00:00 2001 From: rsnetworkinginc <272110387+rsnetworkinginc@users.noreply.github.com> Date: Fri, 24 Jul 2026 02:01:22 +0300 Subject: [PATCH 1/3] fix(standings): register a miner only after a numeric score is confirmed compute_standings registered a miner via per_miner.setdefault before any numeric per-benchmark score was seen. A merged competition win whose per_benchmark is empty or entirely non-numeric therefore created a phantom MinerStanding(overall=0.0, n_competed=0, rank=1) that could be falsely reported as standings.leader, contradicting the module contract that an overall leader must be strong on every benchmark and diverging from the non-dict per_benchmark case, which already drops such records. Move the setdefault inside the numeric-score branch so a miner is registered only once a real score is confirmed; empty/all-non-numeric wins now contribute nothing, matching the non-dict case. Add a regression test asserting such a win creates no phantom miner and no false leader. --- src/trinity/standings.py | 6 +++++- tests/test_standings.py | 26 ++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/src/trinity/standings.py b/src/trinity/standings.py index f535a7d..9f6dfff 100644 --- a/src/trinity/standings.py +++ b/src/trinity/standings.py @@ -117,12 +117,16 @@ def compute_standings(leaderboard: Mapping[str, Any]) -> Standings: per_benchmark = h.get("per_benchmark") if miner is None or not isinstance(per_benchmark, dict): continue - best = per_miner.setdefault(str(miner), {}) + miner_key = str(miner) for bench, score in per_benchmark.items(): if not _is_num(score): continue b = str(bench) seen_benches.add(b) + # Register the miner only once a real numeric score is confirmed. A merged win + # whose per_benchmark is empty or all-non-numeric contributes nothing and must + # not create a phantom standing (matching the non-dict per_benchmark case). + best = per_miner.setdefault(miner_key, {}) if b not in best or float(score) > best[b]: best[b] = float(score) diff --git a/tests/test_standings.py b/tests/test_standings.py index 7bca49f..22cf582 100644 --- a/tests/test_standings.py +++ b/tests/test_standings.py @@ -151,6 +151,32 @@ def test_non_dict_per_benchmark_is_skipped(): assert [m.miner for m in s.miners] == ["r"] +def test_win_with_no_numeric_score_creates_no_phantom_leader(): + # Regression: a merged win whose per_benchmark is empty (or entirely non-numeric) + # must NOT register the miner. Pre-fix, setdefault ran before any numeric score was + # confirmed, so such a win produced a phantom MinerStanding(overall=0.0, n_competed=0, + # rank=1) that was falsely reported as the standings leader. This must match the + # non-dict per_benchmark case, which drops the record entirely. + lb = _lb([ + {"miner": "ghost", "merged": True, "per_benchmark": {}, "pr": 1, + "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + {"miner": "nul", "merged": True, "per_benchmark": {"math500": None, "mmlu": "x"}, + "pr": 2, "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + ]) + s = compute_standings(lb) + assert s.miners == [] # neither win registers a miner + assert s.leader is None # no phantom leader + + # And a phantom win must not shadow a real one: only the scored miner ranks/leads. + lb2 = _lb([ + {"miner": "ghost", "merged": True, "per_benchmark": {}, "pr": 1, + "score": 0.0, "generation": 1, "timestamp": "2026-07-13T00:00:00Z"}, + _win("real", {"math500": 0.7, "mmlu": 0.8}, 2), + ]) + s2 = compute_standings(lb2) + assert [m.miner for m in s2.miners] == ["real"] and s2.leader == "real" + + # --------------------------------------------------------------------------- # # load + render # --------------------------------------------------------------------------- # From 1da5fcf0355772df7a165e2728541696b39d988c Mon Sep 17 00:00:00 2001 From: rsnetworkinginc <272110387+rsnetworkinginc@users.noreply.github.com> Date: Sun, 26 Jul 2026 01:56:26 +0300 Subject: [PATCH 2/3] ci: pin ruff below 0.16 so lint stays reproducible ruff is unpinned (>=0.5) in the dev extras, so CI picked up the new ruff 0.16.0 release, which flags 300+ pre-existing violations across src/ and scripts/ on main (unrelated to this change). Pin to <0.16, the newest series main passes cleanly, so the lint gate checks this PR against the same ruleset the rest of the tree was written for. --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 02d4f00..e911c0d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -18,7 +18,7 @@ dependencies = [ ] [project.optional-dependencies] -dev = ["pytest>=8.0", "ruff>=0.5", "mypy>=1.10"] +dev = ["pytest>=8.0", "ruff>=0.5,<0.16", "mypy>=1.10"] [build-system] requires = ["hatchling"] From d66df4ae7aa3d2eb18b18c2ddb507a4810db8c11 Mon Sep 17 00:00:00 2001 From: rsnetworkinginc <272110387+rsnetworkinginc@users.noreply.github.com> Date: Sun, 26 Jul 2026 02:12:11 +0300 Subject: [PATCH 3/3] fix(scoring): keep a leading-decimal number intact when stripping edge punctuation tests/test_drop_leading_decimal_point.py (merged with #423) and the hyphen/edge-punctuation tokenizer changes landed from separate branches and semantically collided: raw.strip() treats the . of ".5." or "$.5" as wrapping noise, so the token normalizes to 5.0 - equal to a gold 5 (false positive) and unequal to the value-identical 0.5 (false negative). Main currently fails those three tests, which keeps every PR red at the Pytest step. Stop the left-edge strip as soon as removing one more character would break a leading decimal (a . directly followed by a digit); trailing punctuation still strips as before. All drop/BBH scoring tests pass. --- src/trinity/adapters/drop.py | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/src/trinity/adapters/drop.py b/src/trinity/adapters/drop.py index 1314803..4caf94c 100644 --- a/src/trinity/adapters/drop.py +++ b/src/trinity/adapters/drop.py @@ -156,13 +156,34 @@ def _normalize_token(raw: str) -> str: return str(float(raw.replace(",", ""))) except ValueError: pass - core = raw.strip(_STRIP_EDGE) + core = _strip_edges_keeping_leading_decimal(raw) try: return str(float(core.replace(",", ""))) except ValueError: return _PUNCT.sub("", raw) +def _strip_edges_keeping_leading_decimal(raw: str) -> str: + """Strip edge punctuation like raw.strip(_STRIP_EDGE), except that a . + starting a leading-decimal number (. directly followed by a digit) is kept. + + A plain strip treats the . of ".5." or "$.5" as wrapping noise and + yields "5", so the token normalizes to "5.0" — equal to a gold "5" + (false positive) and unequal to the value-identical "0.5" (false negative). + The official DROP normalizer never corrupts a number this way, so the left-edge + scan stops as soon as stripping one more character would break a leading decimal. + """ + end = len(raw) + while end > 0 and raw[end - 1] in _STRIP_EDGE: + end -= 1 + start = 0 + while start < end and raw[start] in _STRIP_EDGE: + if raw[start] == "." and start + 1 < end and raw[start + 1].isdigit(): + break + start += 1 + return raw[start:end] + + def _split_internal_hyphens(token: str) -> list[str]: """Split ``token`` on INTERNAL hyphens, matching DROP's ``re.split(" |-", ...)``.