Auto-restart to recover from wedged ASIC (Flatline of Death, #1053)

main/global_state.h

@@ -197,6 +197,20 @@ typedef struct
     char network_diff_string[DIFF_STRING_SIZE];
     char block_signals[MAX_BLOCK_SIGNALS][MAX_BLOCK_SIGNAL_LEN];
     int block_signals_count;
+
+    // Timestamp (esp_timer_get_time) of the last share the pool accepted.
+    // Watched from the stratum tasks to detect a wedged ASIC (issue #1053).
+    // A wedged chip may go silent or babble garbage nonces; garbage can fool
+    // local counters but never passes pool validation, so accepted shares
+    // are the ground truth for "actually mining".
+    int64_t last_accepted_share_time;
 } GlobalState;
 
+// Minimum time without a pool-accepted share - while the pool link is
+// demonstrably alive - before we conclude the ASIC is wedged ("Flatline of
+// Death", issue #1053) and restart. The effective timeout scales up with
+// pool difficulty (see SYSTEM_check_flatline_watchdog) so vardiff cannot
+// cause false restarts; this floor applies at typical difficulties.
+#define FLATLINE_RESTART_TIMEOUT_US (10LL * 60 * 1000000)
+
 #endif /* GLOBAL_STATE_H_ */

main/main.c

@@ -4,6 +4,7 @@
 #include "esp_log.h"
 #include "esp_psram.h"
 #include "esp_heap_caps.h"
+#include "esp_system.h"
 #include "cJSON.h"
 
 #include "asic_result_task.h"
@@ -171,7 +172,15 @@ void app_main(void)
     if (system_init_ret == ESP_OK) {
         if (asic_initialize(&GLOBAL_STATE, ASIC_INIT_COLD_BOOT, 0) == 0) {
             if (!GLOBAL_STATE.SELF_TEST_MODULE.is_active) {
-                return;
+                // A flaky ASIC serial link can fail chip detection on one
+                // boot and succeed on the next; returning here leaves a
+                // web-UI zombie that never mines until someone notices
+                // (#1053 family; field-observed on the test unit: 2.2 h
+                // dead, recovered by the very next soft reset). Keep the UI
+                // reachable for a while for debugging, then retry.
+                ESP_LOGE(TAG, "ASIC init failed - restarting in 5 minutes to retry");
+                vTaskDelay(5 * 60 * 1000 / portTICK_PERIOD_MS);
+                esp_restart();
             }
 
             self_test_show_message(&GLOBAL_STATE, GLOBAL_STATE.SYSTEM_MODULE.asic_status);

main/system.c

@@ -14,6 +14,7 @@
 
 #include "driver/gpio.h"
 #include "esp_app_desc.h"
+#include "esp_system.h"
 #include "esp_timer.h"
 #include "esp_wifi.h"
 #include "lwip/inet.h"
@@ -290,6 +291,40 @@ void SYSTEM_notify_accepted_share(GlobalState * GLOBAL_STATE)
     SystemModule * module = &GLOBAL_STATE->SYSTEM_MODULE;
 
     module->shares_accepted++;
+    GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time();
+}
+
+void SYSTEM_check_flatline_watchdog(GlobalState * GLOBAL_STATE)
+{
+    // Called from the stratum tasks right after pool traffic is received, so
+    // the pool link is known-alive. Judge mining health by pool-ACCEPTED
+    // shares: a wedged ASIC may go silent or babble garbage nonces, and
+    // garbage can fool local counters but never passes pool validation. The
+    // timeout scales with pool difficulty (25x the expected share interval,
+    // floored at FLATLINE_RESTART_TIMEOUT_US) so vardiff cannot cause false
+    // restarts, and a pool outage cannot cause a restart loop because no
+    // traffic arrives while the pool is down.
+    if (GLOBAL_STATE->SYSTEM_MODULE.overheat_mode || !GLOBAL_STATE->ASIC_initalized ||
+        GLOBAL_STATE->last_accepted_share_time <= 0) {
+        return;
+    }
+
+    int64_t timeout_us = FLATLINE_RESTART_TIMEOUT_US;
+    double expected_hs = (double) GLOBAL_STATE->POWER_MANAGEMENT_MODULE.frequency_value * 1.0e6
+                       * GLOBAL_STATE->DEVICE_CONFIG.family.asic.small_core_count;
+    if (expected_hs > 0) {
+        int64_t scaled_us = (int64_t) (25.0 * GLOBAL_STATE->pool_difficulty * 4294967296.0 / expected_hs * 1.0e6);
+        if (scaled_us > timeout_us) {
+            timeout_us = scaled_us;
+        }
+    }
+
+    if (esp_timer_get_time() - GLOBAL_STATE->last_accepted_share_time > timeout_us) {
+        ESP_LOGE(TAG, "No pool-accepted shares for %lld minutes despite a healthy pool connection - "
+                 "restarting to recover from wedged ASIC (#1053)",
+                 (long long) (timeout_us / 60000000));
+        esp_restart();
+    }
 }
 
 static int compare_rejected_reason_stats(const void *a, const void *b) {

main/system.h

@@ -16,6 +16,11 @@ void SYSTEM_clean_jobs_queue(GlobalState * GLOBAL_STATE);
 
 void SYSTEM_notify_accepted_share(GlobalState * GLOBAL_STATE);
 void SYSTEM_notify_rejected_share(GlobalState * GLOBAL_STATE, char * error_msg);
+
+// Flatline-of-Death watchdog (#1053): restart if the pool link is alive but
+// no share has been accepted for a difficulty-scaled timeout. Call from the
+// stratum tasks right after pool traffic is received.
+void SYSTEM_check_flatline_watchdog(GlobalState * GLOBAL_STATE);
 void SYSTEM_notify_found_nonce(GlobalState * GLOBAL_STATE, double diff, uint32_t target);
 void SYSTEM_notify_new_ntime(GlobalState * GLOBAL_STATE, uint32_t ntime);
 

main/tasks/stratum_v1_task.c

@@ -298,6 +298,10 @@ void stratum_v1_task(void *pvParameters)
         //mining.authorize - ID: 3
         STRATUM_V1_authorize(GLOBAL_STATE->transport, authorize_message_id, username, password);
 
+        // Fresh connection: restart the flatline watchdog clock so time
+        // spent disconnected is never counted against the ASIC.
+        GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time();
+
         while (1) {
             // Check if coordinator wants us to shut down
             if (protocol_coordinator_v1_should_shutdown()) {
@@ -323,6 +327,14 @@ void stratum_v1_task(void *pvParameters)
                 break;
             }
 
+            // Flatline of Death detection (#1053): we just received a line,
+            // so the pool link is alive - restart if no share has been
+            // accepted for a difficulty-scaled timeout (a wedged ASIC may go
+            // silent or babble garbage; garbage never passes pool
+            // validation). Gating on pool traffic means a pool outage can
+            // never cause a restart loop.
+            SYSTEM_check_flatline_watchdog(GLOBAL_STATE);
+
             int64_t receive_time_us = esp_timer_get_time();
 
             bool reconnect_requested = false;

main/tasks/stratum_v2_task.c

@@ -1,6 +1,7 @@
 #include "esp_log.h"
 #include "esp_transport.h"
 #include "esp_transport_tcp.h"
+#include "esp_system.h"
 #include <lwip/sockets.h>
 #include "esp_timer.h"
 #include "system.h"
@@ -924,6 +925,10 @@ void stratum_v2_task(void *pvParameters)
                      elapsed_ms, stratum_url, port);
         }
 
+        // Fresh connection: restart the flatline watchdog clock so time
+        // spent disconnected is never counted against the ASIC.
+        GLOBAL_STATE->last_accepted_share_time = esp_timer_get_time();
+
         // --- Main receive loop ---
         while (1) {
             if (sv2_noise_recv(noise_ctx, transport, hdr_buf, recv_buf,
@@ -936,6 +941,13 @@ void stratum_v2_task(void *pvParameters)
 
             sv2_parse_frame_header(hdr_buf, &hdr);
 
+            // Flatline of Death detection (#1053): we just received a frame,
+            // so the pool link is alive - restart if no share has been
+            // accepted for a difficulty-scaled timeout (a wedged ASIC may go
+            // silent or babble garbage; garbage never passes pool
+            // validation).
+            SYSTEM_check_flatline_watchdog(GLOBAL_STATE);
+
             switch (hdr.msg_type) {
                 case SV2_MSG_NEW_MINING_JOB:
                     stratum_v2_handle_new_mining_job(GLOBAL_STATE, conn, recv_buf, hdr.msg_length);