From 8cf5b043b022964d471e536b64f1b1363c9fd341 Mon Sep 17 00:00:00 2001 From: ThymeKeeper Date: Fri, 10 Jul 2026 09:00:31 -0600 Subject: [PATCH 1/3] Auto-restart to recover from wedged ASIC (Flatline of Death, #1053) Detect the FoD state in firmware: if the stratum link is demonstrably alive (we just received pool traffic) but the ASIC has not returned a single nonce for 10 minutes, log and esp_restart(). - ASIC_result_task stamps last_asic_result_time on every nonce result; register reads deliberately do not count, since the register path can stay responsive while nonce production is wedged. - Both stratum receive loops (v1 and SV2) perform the check only after successfully receiving pool traffic, so a pool outage can never cause a restart loop. - The timer is parked while ASIC_initalized is false, so boot/self-test/ overheat pauses cannot trip a stale-timestamp restart on resume. Diagnosed on an affected Gamma 601 mid-flatline: TCP session ESTABLISHED, mining.notify still arriving in the device log, zero asic_result lines, full mining power draw, share counter frozen for 14+ hours until a manual restart recovered it - the recovery this change automates. Co-Authored-By: Claude Fable 5 --- main/global_state.h | 12 ++++++++++++ main/tasks/asic_result_task.c | 8 ++++++++ main/tasks/stratum_v1_task.c | 13 +++++++++++++ main/tasks/stratum_v2_task.c | 14 ++++++++++++++ 4 files changed, 47 insertions(+) diff --git a/main/global_state.h b/main/global_state.h index 7ef93594a4..756b3feb88 100644 --- a/main/global_state.h +++ b/main/global_state.h @@ -197,6 +197,18 @@ typedef struct char network_diff_string[DIFF_STRING_SIZE]; char block_signals[MAX_BLOCK_SIGNALS][MAX_BLOCK_SIGNAL_LEN]; int block_signals_count; + + // Timestamp (esp_timer_get_time) of the last nonce result read back from + // the ASIC. Register reads deliberately don't count: on wedged units the + // register path can stay responsive while nonce production is dead. + // Watched from the stratum tasks to detect a wedged ASIC (issue #1053). + int64_t last_asic_result_time; } GlobalState; +// How long the ASIC may go without returning a single nonce - while the pool +// link is demonstrably alive - before we conclude it is wedged ("Flatline of +// Death", issue #1053) and restart. At any realistic difficulty the chip +// answers many times a minute, so 10 minutes of true silence is a wedge. +#define FLATLINE_RESTART_TIMEOUT_US (10LL * 60 * 1000000) + #endif /* GLOBAL_STATE_H_ */ diff --git a/main/tasks/asic_result_task.c b/main/tasks/asic_result_task.c index 68f916ee85..16329294a5 100644 --- a/main/tasks/asic_result_task.c +++ b/main/tasks/asic_result_task.c @@ -15,6 +15,7 @@ #include "freertos/task.h" #include "scoreboard.h" #include "self_test.h" +#include "esp_timer.h" static const char *TAG = "asic_result"; @@ -26,6 +27,10 @@ void ASIC_result_task(void *pvParameters) { // Check if ASIC is initialized before trying to process work if (!GLOBAL_STATE->ASIC_initalized) { + // Keep the flatline watchdog timer parked while the ASIC is not + // supposed to be producing (boot, self-test, overheat pause) so + // resuming can never trip a stale-timestamp restart. + GLOBAL_STATE->last_asic_result_time = esp_timer_get_time(); vTaskDelay(100 / portTICK_PERIOD_MS); continue; } @@ -42,6 +47,9 @@ void ASIC_result_task(void *pvParameters) continue; } + // A nonce came back: the ASIC/serial result path is alive. + GLOBAL_STATE->last_asic_result_time = esp_timer_get_time(); + uint8_t job_id = asic_result->job_id; // Snapshot the job while holding the lock. The shared slot diff --git a/main/tasks/stratum_v1_task.c b/main/tasks/stratum_v1_task.c index 3658255799..f56f4cb37b 100644 --- a/main/tasks/stratum_v1_task.c +++ b/main/tasks/stratum_v1_task.c @@ -323,6 +323,19 @@ void stratum_v1_task(void *pvParameters) break; } + // Flatline of Death detection (#1053): we just received a line, + // so the pool link is alive - but if the ASIC has not returned a + // single nonce in FLATLINE_RESTART_TIMEOUT_US the chip is wedged + // and only a reset recovers it. Gating on pool traffic means a + // pool outage can never cause a restart loop. + if (GLOBAL_STATE->last_asic_result_time > 0 && + esp_timer_get_time() - GLOBAL_STATE->last_asic_result_time > FLATLINE_RESTART_TIMEOUT_US) { + ESP_LOGE(TAG, "No ASIC results for %d minutes despite a healthy pool connection - " + "restarting to recover from wedged ASIC (#1053)", + (int) (FLATLINE_RESTART_TIMEOUT_US / 60000000)); + esp_restart(); + } + int64_t receive_time_us = esp_timer_get_time(); bool reconnect_requested = false; diff --git a/main/tasks/stratum_v2_task.c b/main/tasks/stratum_v2_task.c index 7520145aa4..52065a8a7e 100644 --- a/main/tasks/stratum_v2_task.c +++ b/main/tasks/stratum_v2_task.c @@ -1,6 +1,7 @@ #include "esp_log.h" #include "esp_transport.h" #include "esp_transport_tcp.h" +#include "esp_system.h" #include #include "esp_timer.h" #include "system.h" @@ -936,6 +937,19 @@ void stratum_v2_task(void *pvParameters) sv2_parse_frame_header(hdr_buf, &hdr); + // Flatline of Death detection (#1053): we just received a frame, + // so the pool link is alive - but if the ASIC has not returned a + // single nonce in FLATLINE_RESTART_TIMEOUT_US the chip is wedged + // and only a reset recovers it. + if (GLOBAL_STATE->ASIC_initalized && + GLOBAL_STATE->last_asic_result_time > 0 && + esp_timer_get_time() - GLOBAL_STATE->last_asic_result_time > FLATLINE_RESTART_TIMEOUT_US) { + ESP_LOGE(TAG, "No ASIC results for %d minutes despite a healthy pool connection - " + "restarting to recover from wedged ASIC (#1053)", + (int) (FLATLINE_RESTART_TIMEOUT_US / 60000000)); + esp_restart(); + } + switch (hdr.msg_type) { case SV2_MSG_NEW_MINING_JOB: stratum_v2_handle_new_mining_job(GLOBAL_STATE, conn, recv_buf, hdr.msg_length); From bf81fed70dc9d622bfae44a4d642fca545842694 Mon Sep 17 00:00:00 2001 From: ThymeKeeper Date: Sun, 12 Jul 2026 08:36:45 -0600 Subject: [PATCH 2/3] Watchdog v2: judge by pool-accepted shares with difficulty-scaled timeout Field finding within 48h of running v1 on the affected Gamma 601: the wedge has a second mode. Instead of going silent the chip can babble garbage results, which refreshed the nonce-based aliveness stamp (and inflated the share-based hashrate estimator ~17x) while producing zero valid shares - the silence check never fired through a 2+ hour wedge. Garbage can fool every local counter but can never pass pool validation, so track the last pool-ACCEPTED share instead, via the existing SYSTEM_notify_accepted_share path (shared by SV1 and SV2). The timeout is 25x the expected share interval at the current pool difficulty, floored at 10 minutes, so vardiff cannot cause false restarts at any difficulty. The check lives in SYSTEM_check_flatline_watchdog, called from both stratum receive loops; the clock resets on every fresh connection and is skipped during overheat_mode / uninitialized ASIC. Co-Authored-By: Claude Fable 5 --- main/global_state.h | 18 ++++++++++-------- main/system.c | 35 +++++++++++++++++++++++++++++++++++ main/system.h | 5 +++++ main/tasks/asic_result_task.c | 8 -------- main/tasks/stratum_v1_task.c | 21 ++++++++++----------- main/tasks/stratum_v2_task.c | 20 +++++++++----------- 6 files changed, 69 insertions(+), 38 deletions(-) diff --git a/main/global_state.h b/main/global_state.h index 756b3feb88..23afd4b6ce 100644 --- a/main/global_state.h +++ b/main/global_state.h @@ -198,17 +198,19 @@ typedef struct char block_signals[MAX_BLOCK_SIGNALS][MAX_BLOCK_SIGNAL_LEN]; int block_signals_count; - // Timestamp (esp_timer_get_time) of the last nonce result read back from - // the ASIC. Register reads deliberately don't count: on wedged units the - // register path can stay responsive while nonce production is dead. + // Timestamp (esp_timer_get_time) of the last share the pool accepted. // Watched from the stratum tasks to detect a wedged ASIC (issue #1053). - int64_t last_asic_result_time; + // A wedged chip may go silent or babble garbage nonces; garbage can fool + // local counters but never passes pool validation, so accepted shares + // are the ground truth for "actually mining". + int64_t last_accepted_share_time; } GlobalState; -// How long the ASIC may go without returning a single nonce - while the pool -// link is demonstrably alive - before we conclude it is wedged ("Flatline of -// Death", issue #1053) and restart. At any realistic difficulty the chip -// answers many times a minute, so 10 minutes of true silence is a wedge. +// Minimum time without a pool-accepted share - while the pool link is +// demonstrably alive - before we conclude the ASIC is wedged ("Flatline of +// Death", issue #1053) and restart. The effective timeout scales up with +// pool difficulty (see SYSTEM_check_flatline_watchdog) so vardiff cannot +// cause false restarts; this floor applies at typical difficulties. #define FLATLINE_RESTART_TIMEOUT_US (10LL * 60 * 1000000) #endif /* GLOBAL_STATE_H_ */ diff --git a/main/system.c b/main/system.c index d9488e6e01..f26b66bc36 100644 --- a/main/system.c +++ b/main/system.c @@ -14,6 +14,7 @@ #include "driver/gpio.h" #include "esp_app_desc.h" +#include "esp_system.h" #include "esp_timer.h" #include "esp_wifi.h" #include "lwip/inet.h" @@ -290,6 +291,40 @@ void SYSTEM_notify_accepted_share(GlobalState * GLOBAL_STATE) SystemModule * module = &GLOBAL_STATE->SYSTEM_MODULE; module->shares_accepted++; + GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time(); +} + +void SYSTEM_check_flatline_watchdog(GlobalState * GLOBAL_STATE) +{ + // Called from the stratum tasks right after pool traffic is received, so + // the pool link is known-alive. Judge mining health by pool-ACCEPTED + // shares: a wedged ASIC may go silent or babble garbage nonces, and + // garbage can fool local counters but never passes pool validation. The + // timeout scales with pool difficulty (25x the expected share interval, + // floored at FLATLINE_RESTART_TIMEOUT_US) so vardiff cannot cause false + // restarts, and a pool outage cannot cause a restart loop because no + // traffic arrives while the pool is down. + if (GLOBAL_STATE->SYSTEM_MODULE.overheat_mode || !GLOBAL_STATE->ASIC_initalized || + GLOBAL_STATE->last_accepted_share_time <= 0) { + return; + } + + int64_t timeout_us = FLATLINE_RESTART_TIMEOUT_US; + double expected_hs = (double) GLOBAL_STATE->POWER_MANAGEMENT_MODULE.frequency_value * 1.0e6 + * GLOBAL_STATE->DEVICE_CONFIG.family.asic.small_core_count; + if (expected_hs > 0) { + int64_t scaled_us = (int64_t) (25.0 * GLOBAL_STATE->pool_difficulty * 4294967296.0 / expected_hs * 1.0e6); + if (scaled_us > timeout_us) { + timeout_us = scaled_us; + } + } + + if (esp_timer_get_time() - GLOBAL_STATE->last_accepted_share_time > timeout_us) { + ESP_LOGE(TAG, "No pool-accepted shares for %lld minutes despite a healthy pool connection - " + "restarting to recover from wedged ASIC (#1053)", + (long long) (timeout_us / 60000000)); + esp_restart(); + } } static int compare_rejected_reason_stats(const void *a, const void *b) { diff --git a/main/system.h b/main/system.h index 8bde97ed5f..12a0509080 100644 --- a/main/system.h +++ b/main/system.h @@ -16,6 +16,11 @@ void SYSTEM_clean_jobs_queue(GlobalState * GLOBAL_STATE); void SYSTEM_notify_accepted_share(GlobalState * GLOBAL_STATE); void SYSTEM_notify_rejected_share(GlobalState * GLOBAL_STATE, char * error_msg); + +// Flatline-of-Death watchdog (#1053): restart if the pool link is alive but +// no share has been accepted for a difficulty-scaled timeout. Call from the +// stratum tasks right after pool traffic is received. +void SYSTEM_check_flatline_watchdog(GlobalState * GLOBAL_STATE); void SYSTEM_notify_found_nonce(GlobalState * GLOBAL_STATE, double diff, uint32_t target); void SYSTEM_notify_new_ntime(GlobalState * GLOBAL_STATE, uint32_t ntime); diff --git a/main/tasks/asic_result_task.c b/main/tasks/asic_result_task.c index 16329294a5..68f916ee85 100644 --- a/main/tasks/asic_result_task.c +++ b/main/tasks/asic_result_task.c @@ -15,7 +15,6 @@ #include "freertos/task.h" #include "scoreboard.h" #include "self_test.h" -#include "esp_timer.h" static const char *TAG = "asic_result"; @@ -27,10 +26,6 @@ void ASIC_result_task(void *pvParameters) { // Check if ASIC is initialized before trying to process work if (!GLOBAL_STATE->ASIC_initalized) { - // Keep the flatline watchdog timer parked while the ASIC is not - // supposed to be producing (boot, self-test, overheat pause) so - // resuming can never trip a stale-timestamp restart. - GLOBAL_STATE->last_asic_result_time = esp_timer_get_time(); vTaskDelay(100 / portTICK_PERIOD_MS); continue; } @@ -47,9 +42,6 @@ void ASIC_result_task(void *pvParameters) continue; } - // A nonce came back: the ASIC/serial result path is alive. - GLOBAL_STATE->last_asic_result_time = esp_timer_get_time(); - uint8_t job_id = asic_result->job_id; // Snapshot the job while holding the lock. The shared slot diff --git a/main/tasks/stratum_v1_task.c b/main/tasks/stratum_v1_task.c index f56f4cb37b..7187b07257 100644 --- a/main/tasks/stratum_v1_task.c +++ b/main/tasks/stratum_v1_task.c @@ -298,6 +298,10 @@ void stratum_v1_task(void *pvParameters) //mining.authorize - ID: 3 STRATUM_V1_authorize(GLOBAL_STATE->transport, authorize_message_id, username, password); + // Fresh connection: restart the flatline watchdog clock so time + // spent disconnected is never counted against the ASIC. + GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time(); + while (1) { // Check if coordinator wants us to shut down if (protocol_coordinator_v1_should_shutdown()) { @@ -324,17 +328,12 @@ void stratum_v1_task(void *pvParameters) } // Flatline of Death detection (#1053): we just received a line, - // so the pool link is alive - but if the ASIC has not returned a - // single nonce in FLATLINE_RESTART_TIMEOUT_US the chip is wedged - // and only a reset recovers it. Gating on pool traffic means a - // pool outage can never cause a restart loop. - if (GLOBAL_STATE->last_asic_result_time > 0 && - esp_timer_get_time() - GLOBAL_STATE->last_asic_result_time > FLATLINE_RESTART_TIMEOUT_US) { - ESP_LOGE(TAG, "No ASIC results for %d minutes despite a healthy pool connection - " - "restarting to recover from wedged ASIC (#1053)", - (int) (FLATLINE_RESTART_TIMEOUT_US / 60000000)); - esp_restart(); - } + // so the pool link is alive - restart if no share has been + // accepted for a difficulty-scaled timeout (a wedged ASIC may go + // silent or babble garbage; garbage never passes pool + // validation). Gating on pool traffic means a pool outage can + // never cause a restart loop. + SYSTEM_check_flatline_watchdog(GLOBAL_STATE); int64_t receive_time_us = esp_timer_get_time(); diff --git a/main/tasks/stratum_v2_task.c b/main/tasks/stratum_v2_task.c index 52065a8a7e..41f9f63b52 100644 --- a/main/tasks/stratum_v2_task.c +++ b/main/tasks/stratum_v2_task.c @@ -925,6 +925,10 @@ void stratum_v2_task(void *pvParameters) elapsed_ms, stratum_url, port); } + // Fresh connection: restart the flatline watchdog clock so time + // spent disconnected is never counted against the ASIC. + GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time(); + // --- Main receive loop --- while (1) { if (sv2_noise_recv(noise_ctx, transport, hdr_buf, recv_buf, @@ -938,17 +942,11 @@ void stratum_v2_task(void *pvParameters) sv2_parse_frame_header(hdr_buf, &hdr); // Flatline of Death detection (#1053): we just received a frame, - // so the pool link is alive - but if the ASIC has not returned a - // single nonce in FLATLINE_RESTART_TIMEOUT_US the chip is wedged - // and only a reset recovers it. - if (GLOBAL_STATE->ASIC_initalized && - GLOBAL_STATE->last_asic_result_time > 0 && - esp_timer_get_time() - GLOBAL_STATE->last_asic_result_time > FLATLINE_RESTART_TIMEOUT_US) { - ESP_LOGE(TAG, "No ASIC results for %d minutes despite a healthy pool connection - " - "restarting to recover from wedged ASIC (#1053)", - (int) (FLATLINE_RESTART_TIMEOUT_US / 60000000)); - esp_restart(); - } + // so the pool link is alive - restart if no share has been + // accepted for a difficulty-scaled timeout (a wedged ASIC may go + // silent or babble garbage; garbage never passes pool + // validation). + SYSTEM_check_flatline_watchdog(GLOBAL_STATE); switch (hdr.msg_type) { case SV2_MSG_NEW_MINING_JOB: From 316e00ddce16855df5a02b09f4176478d377e5a5 Mon Sep 17 00:00:00 2001 From: ThymeKeeper Date: Sun, 12 Jul 2026 14:52:03 -0600 Subject: [PATCH 3/3] Retry ASIC init via restart instead of stranding a web-UI zombie Field-observed on the FoD test unit: one cold boot failed chip detection (flaky serial link), app_main returned, and the device sat mining nothing for 2.2h with a fully working dashboard. The flatline watchdog cannot help there - stratum never starts. The very next soft reset initialized fine. Keep the UI reachable for 5 minutes for debugging, then esp_restart and retry. Self-test path unchanged. Co-Authored-By: Claude Fable 5 --- main/main.c | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/main/main.c b/main/main.c index e0582b8313..2803ebe692 100644 --- a/main/main.c +++ b/main/main.c @@ -4,6 +4,7 @@ #include "esp_log.h" #include "esp_psram.h" #include "esp_heap_caps.h" +#include "esp_system.h" #include "cJSON.h" #include "asic_result_task.h" @@ -171,7 +172,15 @@ void app_main(void) if (system_init_ret == ESP_OK) { if (asic_initialize(&GLOBAL_STATE, ASIC_INIT_COLD_BOOT, 0) == 0) { if (!GLOBAL_STATE.SELF_TEST_MODULE.is_active) { - return; + // A flaky ASIC serial link can fail chip detection on one + // boot and succeed on the next; returning here leaves a + // web-UI zombie that never mines until someone notices + // (#1053 family; field-observed on the test unit: 2.2 h + // dead, recovered by the very next soft reset). Keep the UI + // reachable for a while for debugging, then retry. + ESP_LOGE(TAG, "ASIC init failed - restarting in 5 minutes to retry"); + vTaskDelay(5 * 60 * 1000 / portTICK_PERIOD_MS); + esp_restart(); } self_test_show_message(&GLOBAL_STATE, GLOBAL_STATE.SYSTEM_MODULE.asic_status);