From 97016f1e2869ad24e4a34717ef9d7efb8d86ca6e Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:30:00 +0100 Subject: [PATCH 1/7] feat: add windowed metrics averaging for Guardian governor --- backend/config.py | 5 ++ backend/db.py | 21 ++++++ backend/guardian.py | 36 ++++++++-- backend/main.py | 1 + .../src/components/settings/GeneralTab.tsx | 25 +++++-- .../src/components/settings/SettingsForm.tsx | 3 + frontend-react/src/lib/types.ts | 5 ++ tests/test_guardian.py | 71 +++++++++++++++++++ 8 files changed, 157 insertions(+), 10 deletions(-) diff --git a/backend/config.py b/backend/config.py index 5561640..5f5006a 100644 --- a/backend/config.py +++ b/backend/config.py @@ -157,6 +157,11 @@ class GuardianCfg: # after a live frequency change before the next decision is taken. interval_seconds: int = 300 + # Time window (in seconds) used by the governor to average hashrate, + # temperature, and power metrics, preventing false-positive throttling + # from transient dips/peaks. Set to 0 to disable and use instantaneous values. + hashrate_average_window_seconds: int = 30 + # ---- Control thresholds (the friend's field-tested values). ---- # VR temperature is the primary lever: nothing else in MinerWatch # governs it in a closed loop (the fan PID watches the chip, the diff --git a/backend/db.py b/backend/db.py index 9190f78..2b24f37 100644 --- a/backend/db.py +++ b/backend/db.py @@ -752,6 +752,27 @@ async def get_latest_raw(miner_id: int) -> dict[str, Any] | None: return dict(row) if row else None +async def get_recent_metrics_average(miner_id: int, window_seconds: int) -> dict[str, float | None]: + """Calculate the average hashrate, power, chip temp, and VR temp over the last N seconds.""" + cutoff = int(time.time()) - window_seconds + sql = ( + "SELECT AVG(hashrate_ths), AVG(power_w), AVG(temp_chip_c), AVG(temp_vr_c) " + "FROM metrics WHERE miner_id = ? AND ts >= ?" + ) + async with connect() as conn: + async with conn.execute(sql, (miner_id, cutoff)) as cur: + row = await cur.fetchone() + if row: + return { + "hashrate_ths": row[0], + "power_w": row[1], + "temp_chip_c": row[2], + "temp_vr_c": row[3], + } + return {} + + + # ---------- Ambient (room) temperature time-series ---------- async def insert_ambient_metric(sensor_id: str, ts: int, temp_c: float) -> None: diff --git a/backend/guardian.py b/backend/guardian.py index 26fb68f..da084e5 100644 --- a/backend/guardian.py +++ b/backend/guardian.py @@ -523,16 +523,42 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None # behaviour. In chip mode the chip is also driven by the fan PID and the # 75°C watchdog, so the governor only bites once the fan is saturated. source = "chip" if str(miner.get("guardian_temp_source") or "").lower() == "chip" else "vr" - temp_c = sample.temp_chip_c if source == "chip" else sample.temp_vr_c source_label = "Chip" if source == "chip" else "VR" + # Fetch recent averages over the configured window to avoid transient dips/spikes + window = int(getattr(gcfg, "hashrate_average_window_seconds", 30)) + avg_metrics = {} + if window > 0: + try: + avg_metrics = await db.get_recent_metrics_average(miner_id, window) + except Exception: + log.exception("guardian: failed to fetch recent averages for miner=%s", miner.get("name")) + + # Fallback to instantaneous values if not found or window is 0 + hashrate_ths = avg_metrics.get("hashrate_ths") + if hashrate_ths is None: + hashrate_ths = sample.hashrate_ths + + temp_chip_c = avg_metrics.get("temp_chip_c") + if temp_chip_c is None: + temp_chip_c = sample.temp_chip_c + + temp_vr_c = avg_metrics.get("temp_vr_c") + if temp_vr_c is None: + temp_vr_c = sample.temp_vr_c + + power_w = avg_metrics.get("power_w") + if power_w is None: + power_w = sample.power_w + + temp_c = temp_chip_c if source == "chip" else temp_vr_c + # Effective hashrate (TH/s) and the ASIC hardware-error counter — the # signals behind the regression brake. ``hashrate_ths`` is AxeOS's # reported (real) hashrate; ``hw_errors`` is the summed per-ASIC invalid- # nonce count, which climbs when an overclock starts producing garbage # (those bad nonces crater real hashrate but never reach the pool, so the # reject-% term stays blind to them). - hashrate_ths = sample.hashrate_ths hw_errors = sample.hw_errors err_delta = None if ( @@ -676,11 +702,11 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None hashrate_invalid=hashrate_invalid, valid=allow_up, instability_label=instab_label, - chip_c=sample.temp_chip_c, + chip_c=temp_chip_c, chip_cutoff_c=float(gcfg.chip_cutoff_c), - vr_c=sample.temp_vr_c, + vr_c=temp_vr_c, vr_cutoff_c=float(gcfg.vr_cutoff_c), - power_w=sample.power_w, + power_w=power_w, power_cutoff_w=power_cut, vin_mv=sample.input_voltage_mv, vin_min_mv=vin_lo, diff --git a/backend/main.py b/backend/main.py index f39965a..5f19719 100644 --- a/backend/main.py +++ b/backend/main.py @@ -2102,6 +2102,7 @@ async def api_get_settings() -> dict: "storage": asdict(cfg.storage), "network": asdict(cfg.network), "auth_enabled": cfg.auth.enabled, + "guardian": asdict(cfg.guardian), }, "stored": stored, } diff --git a/frontend-react/src/components/settings/GeneralTab.tsx b/frontend-react/src/components/settings/GeneralTab.tsx index 59609c4..2227a57 100644 --- a/frontend-react/src/components/settings/GeneralTab.tsx +++ b/frontend-react/src/components/settings/GeneralTab.tsx @@ -53,11 +53,26 @@ export function GeneralTab({ form, setForm }: Props) { max={3650} onChange={(v) => setForm({ ...form, retentionDays: v })} /> -

- Hashrate smoothing: tau (time constant) of the - server-side EMA. 60s is a good trade-off between responsiveness and stability. Set to 0 to see raw - firmware values. -

+ setForm({ ...form, guardianHashrateAverageWindow: v })} + /> +
+
+ Hashrate smoothing: tau (time constant) of the + server-side EMA. 60s is a good trade-off between responsiveness and stability. Set to 0 to see raw + firmware values. +
+
+ Guardian averaging window: The time window used by the + Guardian governor to average hashrate, temperature, and power metrics. This prevents false-positive + throttling from transient dips/peaks. Set to 0 to use instantaneous values. +
+
diff --git a/frontend-react/src/components/settings/SettingsForm.tsx b/frontend-react/src/components/settings/SettingsForm.tsx index 1e9392f..f94e9c0 100644 --- a/frontend-react/src/components/settings/SettingsForm.tsx +++ b/frontend-react/src/components/settings/SettingsForm.tsx @@ -34,6 +34,7 @@ export interface SettingsFormState { scanCidr: string; authEnabled: boolean; authPassword: string; // write-only + guardianHashrateAverageWindow: number; } export function useSettingsForm(current: SettingsCurrent | null | undefined) { @@ -65,6 +66,7 @@ export function useSettingsForm(current: SettingsCurrent | null | undefined) { scanCidr: current.network.scan_cidr, authEnabled: current.auth_enabled, authPassword: '', + guardianHashrateAverageWindow: current.guardian?.hashrate_average_window_seconds ?? 30, }); }, [current]); @@ -132,6 +134,7 @@ export function formToOverrides(form: SettingsFormState): Record Date: Fri, 26 Jun 2026 20:37:15 +0100 Subject: [PATCH 2/7] docs: add CHANGELOG entry for Guardian windowed metrics averaging --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a69bccf..420f707 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- **Guardian windowed metrics averaging.** Added windowed averaging of stochastic/noisy metrics (hashrate, temperature, and power) inside the Guardian governor loop, preventing false-positive throttling from transient dips or peaks. The averaging window can be configured via a new global setting in the general tab (defaults to 30 seconds, set to 0 to disable). - **NMAxe miner family (NMAxe / NMAxeGamma / NMQAxe++).** Support for the NMAxe AxeOS fork, whose REST surface is fully nested — `GET /api/system/info` groups `power` / `temps` / `asic` / `miner` / `identity` / `stratum` / `fans[]` — with From 8a84ae16c29f4d2ac5e8805cb296f27ff1940d47 Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:37:32 +0100 Subject: [PATCH 3/7] docs: document windowed metrics averaging in guardian-design.md --- docs/guardian-design.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/guardian-design.md b/docs/guardian-design.md index 58d5d4b..a782047 100644 --- a/docs/guardian-design.md +++ b/docs/guardian-design.md @@ -146,6 +146,12 @@ la banda morta 65–70 °C si parcheggia su una frequenza d'equilibrio e smette scrivere. Le scritture avvengono solo quando l'ambiente deriva oltre soglia — un numero limitato e sotto controllo. +### 3.2 Media mobile delle metriche (windowed metrics averaging) + +Per prevenire falsi positivi causati da oscillazioni temporanee o transitorie delle metriche (es. picchi improvvisi di temperatura o cali temporanei dell'hashrate), il Guardian non lavora sulle metriche istantanee dell'ultimo poll, ma calcola una media dei valori registrati nel database in una finestra temporale (di default 30 secondi). + +La finestra temporale è configurabile globalmente tramite la chiave `guardian.hashrate_average_window_seconds` nelle impostazioni generali. Impostando il valore a `0`, la funzionalità viene disattivata e il governor torna ad utilizzare i valori istantanei. + ## 5. Stato per-miner e ciclo del controller `GuardianController` (in `guardian.py`) è speculare ad `AutoFanController`: From 581e6075a50ef0aedd66d7101ea9c0835965ee57 Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:45:30 +0100 Subject: [PATCH 4/7] feat: add windowed averaging for hardware error percentage (error_pct) --- CHANGELOG.md | 2 +- backend/db.py | 13 +++++++++---- backend/guardian.py | 10 +++++++--- backend/miners/base.py | 1 + docs/guardian-design.md | 2 +- tests/test_guardian.py | 3 +++ 6 files changed, 22 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 420f707..6aa5ccc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,7 +56,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added -- **Guardian windowed metrics averaging.** Added windowed averaging of stochastic/noisy metrics (hashrate, temperature, and power) inside the Guardian governor loop, preventing false-positive throttling from transient dips or peaks. The averaging window can be configured via a new global setting in the general tab (defaults to 30 seconds, set to 0 to disable). +- **Guardian windowed metrics averaging.** Added windowed averaging of stochastic/noisy metrics (hashrate, temperature, power, and hardware error percentage) inside the Guardian governor loop, preventing false-positive throttling from transient dips or peaks. The averaging window can be configured via a new global setting in the general tab (defaults to 30 seconds, set to 0 to disable). - **NMAxe miner family (NMAxe / NMAxeGamma / NMQAxe++).** Support for the NMAxe AxeOS fork, whose REST surface is fully nested — `GET /api/system/info` groups `power` / `temps` / `asic` / `miner` / `identity` / `stratum` / `fans[]` — with diff --git a/backend/db.py b/backend/db.py index 2b24f37..aad51c0 100644 --- a/backend/db.py +++ b/backend/db.py @@ -83,6 +83,7 @@ best_difficulty REAL, pool_url TEXT, worker TEXT, + error_pct REAL, raw TEXT, -- original payload as JSON FOREIGN KEY (miner_id) REFERENCES miners(id) ON DELETE CASCADE ); @@ -461,6 +462,8 @@ def _init_db_sync() -> None: "ALTER TABLE miners ADD COLUMN ambient_sensor_name TEXT", # Per-trophy dashboard visibility (see block_finds DDL). "ALTER TABLE block_finds ADD COLUMN hidden INTEGER NOT NULL DEFAULT 0", + # Hardware error percentage column in metrics table. + "ALTER TABLE metrics ADD COLUMN error_pct REAL", ]: try: conn.execute(column_def) @@ -693,8 +696,8 @@ async def insert_metric(miner_id: int, ts: int, sample: dict[str, Any]) -> None: INSERT INTO metrics (miner_id, ts, hashrate_ths, power_w, temp_chip_c, temp_vr_c, fan_rpm, fan_pct, frequency_mhz, voltage_mv, uptime_s, - accepted, rejected, best_difficulty, pool_url, worker) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + accepted, rejected, best_difficulty, pool_url, worker, error_pct) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) """, ( miner_id, @@ -713,6 +716,7 @@ async def insert_metric(miner_id: int, ts: int, sample: dict[str, Any]) -> None: sample.get("best_difficulty"), sample.get("pool_url"), sample.get("worker"), + sample.get("error_pct"), ), ) if raw is not None: @@ -753,10 +757,10 @@ async def get_latest_raw(miner_id: int) -> dict[str, Any] | None: async def get_recent_metrics_average(miner_id: int, window_seconds: int) -> dict[str, float | None]: - """Calculate the average hashrate, power, chip temp, and VR temp over the last N seconds.""" + """Calculate the average hashrate, power, chip temp, VR temp, and error rate over the last N seconds.""" cutoff = int(time.time()) - window_seconds sql = ( - "SELECT AVG(hashrate_ths), AVG(power_w), AVG(temp_chip_c), AVG(temp_vr_c) " + "SELECT AVG(hashrate_ths), AVG(power_w), AVG(temp_chip_c), AVG(temp_vr_c), AVG(error_pct) " "FROM metrics WHERE miner_id = ? AND ts >= ?" ) async with connect() as conn: @@ -768,6 +772,7 @@ async def get_recent_metrics_average(miner_id: int, window_seconds: int) -> dict "power_w": row[1], "temp_chip_c": row[2], "temp_vr_c": row[3], + "error_pct": row[4], } return {} diff --git a/backend/guardian.py b/backend/guardian.py index da084e5..678beb1 100644 --- a/backend/guardian.py +++ b/backend/guardian.py @@ -551,6 +551,10 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None if power_w is None: power_w = sample.power_w + error_pct = avg_metrics.get("error_pct") + if error_pct is None: + error_pct = sample.error_pct + temp_c = temp_chip_c if source == "chip" else temp_vr_c # Effective hashrate (TH/s) and the ASIC hardware-error counter — the @@ -645,8 +649,8 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None ) valid_hr = bool(can_validate and hashrate_ths >= expected_ths * float(gcfg.valid_pct)) error_high = ( - sample.error_pct is not None - and float(sample.error_pct) > float(gcfg.error_pct_max) + error_pct is not None + and float(error_pct) > float(gcfg.error_pct_max) ) # Either signal means "unstable": back off (frequency-only) or cure with # voltage (co-tuner). The ASIC error % climbs when pushing frequency at @@ -658,7 +662,7 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None ) tele["expected_ths"] = round(expected_ths, 2) if expected_ths is not None else None tele["valid"] = valid_hr if can_validate else None - tele["error_pct"] = round(sample.error_pct, 2) if sample.error_pct is not None else None + tele["error_pct"] = round(error_pct, 2) if error_pct is not None else None # ---- Phase 2: voltage co-tuner path (per-miner opt-in) ---- # When the voltage lever is enabled (global master + per-miner opt-in) diff --git a/backend/miners/base.py b/backend/miners/base.py index 6f31935..ed020af 100644 --- a/backend/miners/base.py +++ b/backend/miners/base.py @@ -356,6 +356,7 @@ def to_db_sample(self) -> dict[str, Any]: "best_difficulty": self.best_difficulty, "pool_url": self.pool_url, "worker": self.worker, + "error_pct": self.error_pct, "raw": self.raw, } diff --git a/docs/guardian-design.md b/docs/guardian-design.md index a782047..d4705fb 100644 --- a/docs/guardian-design.md +++ b/docs/guardian-design.md @@ -148,7 +148,7 @@ un numero limitato e sotto controllo. ### 3.2 Media mobile delle metriche (windowed metrics averaging) -Per prevenire falsi positivi causati da oscillazioni temporanee o transitorie delle metriche (es. picchi improvvisi di temperatura o cali temporanei dell'hashrate), il Guardian non lavora sulle metriche istantanee dell'ultimo poll, ma calcola una media dei valori registrati nel database in una finestra temporale (di default 30 secondi). +Per prevenire falsi positivi causati da oscillazioni temporanee o transitorie delle metriche (es. picchi improvvisi di temperatura, di errore hardware o cali temporanei dell'hashrate), il Guardian non lavora sulle metriche istantanee dell'ultimo poll, ma calcola una media dei valori registrati nel database in una finestra temporale (di default 30 secondi) per hashrate, temperatura, potenza e percentuale di errore hardware (error_pct). La finestra temporale è configurabile globalmente tramite la chiave `guardian.hashrate_average_window_seconds` nelle impostazioni generali. Impostando il valore a `0`, la funzionalità viene disattivata e il governor torna ad utilizzare i valori istantanei. diff --git a/tests/test_guardian.py b/tests/test_guardian.py index 53e04de..9317c1f 100644 --- a/tests/test_guardian.py +++ b/tests/test_guardian.py @@ -436,6 +436,7 @@ def test_govern_one_uses_recent_averages(): "power_w": 20.0, "temp_chip_c": 60.0, "temp_vr_c": 68.0, + "error_pct": 1.2, } async def run(): @@ -458,6 +459,8 @@ async def run(): mock_decide.assert_called_once() kwargs = mock_decide.call_args.kwargs assert kwargs["hashrate_invalid"] is True + mock_publish.assert_called_once() + assert mock_publish.call_args.kwargs["error_pct"] == 1.2 asyncio.run(run()) From aa249bb738b3289a879c4ac18eb015cfeb3fdfc3 Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:46:04 +0100 Subject: [PATCH 5/7] docs: update comments and descriptions to reflect error_pct averaging --- backend/config.py | 5 +++-- backend/guardian.py | 8 ++++---- frontend-react/src/components/settings/GeneralTab.tsx | 4 ++-- 3 files changed, 9 insertions(+), 8 deletions(-) diff --git a/backend/config.py b/backend/config.py index 5f5006a..c5fb998 100644 --- a/backend/config.py +++ b/backend/config.py @@ -158,8 +158,9 @@ class GuardianCfg: interval_seconds: int = 300 # Time window (in seconds) used by the governor to average hashrate, - # temperature, and power metrics, preventing false-positive throttling - # from transient dips/peaks. Set to 0 to disable and use instantaneous values. + # temperature, power, and hardware error percentage metrics, preventing + # false-positive throttling from transient dips/peaks. Set to 0 to disable + # and use instantaneous values. hashrate_average_window_seconds: int = 30 # ---- Control thresholds (the friend's field-tested values). ---- diff --git a/backend/guardian.py b/backend/guardian.py index 678beb1..5b0c9e9 100644 --- a/backend/guardian.py +++ b/backend/guardian.py @@ -883,10 +883,10 @@ def _publish( ``temp_c`` is the governed sensor's reading and ``source`` says which sensor it is ("vr" | "chip"), so the UI can label it correctly. The legacy ``vr_temp_c`` key is kept (populated only in VR mode) so any - older consumer keeps working. ``hashrate_ths`` / ``asic_errors`` are the - effective-hashrate and ASIC hardware-error readings the regression brake - watches; ``soft_ceiling`` is the in-memory cap pinned after a regression - (``ceiling`` already reflects it — this is for an explicit UI hint). + older consumer keeps working. ``hashrate_ths`` / ``error_pct`` / ``asic_errors`` are + the effective-hashrate, ASIC hardware-error percentage, and ASIC hardware-error + readings the regression brake watches; ``soft_ceiling`` is the in-memory cap pinned + after a regression (``ceiling`` already reflects it — this is for an explicit UI hint). """ temp_r = round(temp_c, 1) if temp_c is not None else None self._status[miner_id] = { diff --git a/frontend-react/src/components/settings/GeneralTab.tsx b/frontend-react/src/components/settings/GeneralTab.tsx index 2227a57..cf1ffcd 100644 --- a/frontend-react/src/components/settings/GeneralTab.tsx +++ b/frontend-react/src/components/settings/GeneralTab.tsx @@ -69,8 +69,8 @@ export function GeneralTab({ form, setForm }: Props) {
Guardian averaging window: The time window used by the - Guardian governor to average hashrate, temperature, and power metrics. This prevents false-positive - throttling from transient dips/peaks. Set to 0 to use instantaneous values. + Guardian governor to average hashrate, temperature, power, and hardware error percentage metrics. This + prevents false-positive throttling from transient dips/peaks. Set to 0 to use instantaneous values.
From 7af1ca5b9ff140c929b1a184a002bbd9a1183e69 Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:48:52 +0100 Subject: [PATCH 6/7] docs: update comment in guardian.py regarding error_pct in regression brake --- backend/guardian.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/backend/guardian.py b/backend/guardian.py index 5b0c9e9..e146034 100644 --- a/backend/guardian.py +++ b/backend/guardian.py @@ -557,12 +557,9 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None temp_c = temp_chip_c if source == "chip" else temp_vr_c - # Effective hashrate (TH/s) and the ASIC hardware-error counter — the - # signals behind the regression brake. ``hashrate_ths`` is AxeOS's - # reported (real) hashrate; ``hw_errors`` is the summed per-ASIC invalid- - # nonce count, which climbs when an overclock starts producing garbage - # (those bad nonces crater real hashrate but never reach the pool, so the - # reject-% term stays blind to them). + # Effective hashrate (TH/s) and the ASIC hardware-error percentage (error_pct) + # — the signals behind the regression brake. ``hw_errors`` is the cumulative + # ASIC invalid-nonce count, which is tracked as a delta for telemetry. hw_errors = sample.hw_errors err_delta = None if ( From 8bbbda6c44e603bb1a47a118b98354aee5885ade Mon Sep 17 00:00:00 2001 From: aetherwraith Date: Fri, 26 Jun 2026 20:53:31 +0100 Subject: [PATCH 7/7] docs: preserve explanations of hashrate_ths and hw_errors in guardian.py --- backend/guardian.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/backend/guardian.py b/backend/guardian.py index e146034..56f6d1e 100644 --- a/backend/guardian.py +++ b/backend/guardian.py @@ -558,8 +558,12 @@ async def _govern_one(self, miner: dict, sample: MinerSample, gcfg, cfg) -> None temp_c = temp_chip_c if source == "chip" else temp_vr_c # Effective hashrate (TH/s) and the ASIC hardware-error percentage (error_pct) - # — the signals behind the regression brake. ``hw_errors`` is the cumulative - # ASIC invalid-nonce count, which is tracked as a delta for telemetry. + # — the signals behind the regression brake. + # + # ``hashrate_ths`` is AxeOS's reported (real) hashrate. + # ``hw_errors`` is the cumulative summed per-ASIC invalid-nonce count, which climbs + # when an overclock starts producing garbage (cratering real hashrate while the + # pool-reject % stays blind). We track ``hw_errors`` as a delta for telemetry. hw_errors = sample.hw_errors err_delta = None if (