minimal auto reset (#51)

* Revert to v0.1.6 - undo PR #48, #49, and #50

* keep bun init

* add health check

* custom webhook http client

* auto nonce reset

* health check interval MS

* fallback directly to nonce reset

* remove script

* remove

Author: d4mr

Cargo.lock

@@ -27,4 +27,3 @@ futures = "0.3.31"
 moka = { version = "0.12.10", features = ["future"] }
 prometheus = "0.13.4"
 lazy_static = "1.5.0"
-tokio-retry2 = {version = "0.6.0", features = ["jitter"]}

executors/Cargo.toml

@@ -38,9 +38,6 @@ pub enum EoaExecutionError {
         message: String,
         inner_error: Option<EngineError>,
     },
-
-    /// Thirdweb support error that requires retry in place
-    ThirdwebSupportError { message: String },
 }
 
 /// Recovery strategy for an EOA execution error
@@ -118,10 +115,6 @@ impl EoaErrorMapper {
                     EoaExecutionError::AccountError {
                         message: message.to_string(),
                     }
-                } else if msg_lower.contains("we are not able to process your request at this time") {
-                    EoaExecutionError::ThirdwebSupportError {
-                        message: message.to_string(),
-                    }
                 } else {
                     // Not an actionable error - let engine error handle it
                     EoaExecutionError::RpcError {
@@ -211,14 +204,6 @@ impl EoaErrorMapper {
                 retry_delay: None,
             },
 
-            EoaExecutionError::ThirdwebSupportError { .. } => RecoveryStrategy {
-                queue_confirmation: false,
-                recycle_nonce: false,
-                needs_resync: false,
-                retryable: true,
-                retry_delay: Some(Duration::from_secs(1)), // Short delay for retry in place
-            },
-
             EoaExecutionError::RpcError { .. } => {
                 // This should not be used - let engine error handle it
                 RecoveryStrategy {
@@ -246,7 +231,6 @@ impl EoaExecutionError {
             | EoaExecutionError::GasError { message }
             | EoaExecutionError::PoolLimitExceeded { message }
             | EoaExecutionError::AccountError { message }
-            | EoaExecutionError::ThirdwebSupportError { message }
             | EoaExecutionError::RpcError { message, .. } => message,
         }
     }

executors/src/eoa/error_classifier.rs

@@ -503,8 +503,7 @@ impl AtomicEoaExecutorStore {
             // Add new hash:id to submitted (keeping old ones)
             pipeline.zadd(&submitted_key, &submitted_transaction_string, nonce);
 
-            // Create hash-to-ID mapping for the new gas bump hash
-            // Keep ALL hashes until transaction is fully cleaned up for re-org protection
+            // Still maintain separate hash-to-ID mapping for backward compatibility
             pipeline.set(&hash_to_id_key, &submitted_transaction.transaction_id);
 
             // Simply push the new attempt to the attempts list
@@ -555,7 +554,6 @@ impl AtomicEoaExecutorStore {
         pending_transaction: &PendingTransaction,
         error: EoaExecutorWorkerError,
         webhook_queue: Arc<twmq::Queue<WebhookJobHandler>>,
-        failed_transaction_expiry_seconds: u64,
     ) -> Result<(), TransactionStoreError> {
         self.with_lock_check(|pipeline| {
             let pending_key = self.pending_transactions_zset_name();
@@ -565,23 +563,11 @@ impl AtomicEoaExecutorStore {
             // Remove from pending state
             pipeline.zrem(&pending_key, &pending_transaction.transaction_id);
 
-            // Update transaction data with failure and set expiry
+            // Update transaction data with failure
             pipeline.hset(&tx_data_key, "completed_at", now);
             pipeline.hset(&tx_data_key, "failure_reason", error.to_string());
             pipeline.hset(&tx_data_key, "status", "failed");
 
-            // Set expiry on all related keys for failed transactions
-            let transaction_keys =
-                self.get_all_transaction_keys(&pending_transaction.transaction_id);
-
-            let ttl: i64 = i64::try_from(failed_transaction_expiry_seconds)
-                .unwrap_or(i64::MAX)
-                .max(1);
-
-            for key in transaction_keys {
-                pipeline.expire(&key, ttl);
-            }
-
             let event = EoaExecutorEvent {
                 transaction_id: pending_transaction.transaction_id.clone(),
                 address: pending_transaction.user_request.from,
@@ -627,14 +613,12 @@ impl AtomicEoaExecutorStore {
         &self,
         results: Vec<SubmissionResult>,
         webhook_queue: Arc<twmq::Queue<WebhookJobHandler>>,
-        failed_transaction_expiry_seconds: u64,
     ) -> Result<BorrowedProcessingReport, TransactionStoreError> {
         self.execute_with_watch_and_retry(&ProcessBorrowedTransactions {
             results,
             keys: &self.keys,
             webhook_queue,
             eoa_metrics: &self.eoa_metrics,
-            failed_transaction_expiry_seconds,
         })
         .await
     }

executors/src/eoa/store/atomic.rs

@@ -12,7 +12,7 @@ use crate::eoa::{
     },
     worker::error::EoaExecutorWorkerError,
 };
-use crate::metrics::{EoaMetrics, calculate_duration_seconds, current_timestamp_ms};
+use crate::metrics::{current_timestamp_ms, calculate_duration_seconds, EoaMetrics};
 use crate::webhook::{WebhookJobHandler, queue_webhook_envelopes};
 
 #[derive(Debug, Clone)]
@@ -41,7 +41,6 @@ pub struct ProcessBorrowedTransactions<'a> {
     pub keys: &'a EoaExecutorStoreKeys,
     pub webhook_queue: Arc<Queue<WebhookJobHandler>>,
     pub eoa_metrics: &'a EoaMetrics,
-    pub failed_transaction_expiry_seconds: u64,
 }
 
 #[derive(Debug, Default)]
@@ -122,13 +121,15 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> {
                 SubmissionResultType::Success => {
                     // Record metrics: transaction queued to sent
                     let sent_timestamp = current_timestamp_ms();
-                    let queued_to_sent_duration =
-                        calculate_duration_seconds(result.transaction.queued_at, sent_timestamp);
+                    let queued_to_sent_duration = calculate_duration_seconds(
+                        result.transaction.queued_at, 
+                        sent_timestamp
+                    );
                     // Record metrics using the clean EoaMetrics abstraction
                     self.eoa_metrics.record_transaction_sent(
                         self.keys.eoa,
                         self.keys.chain_id,
-                        queued_to_sent_duration,
+                        queued_to_sent_duration
                     );
 
                     // Add to submitted zset
@@ -188,7 +189,7 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> {
                     // Update transaction data status
                     let tx_data_key = self.keys.transaction_data_key_name(transaction_id);
                     pipeline.hset(&tx_data_key, "status", "pending");
-
+                    
                     // ask for this nonce to be recycled because we did not consume the nonce
                     pipeline.zadd(self.keys.recycled_nonces_zset_name(), nonce, nonce);
 
@@ -218,22 +219,12 @@ impl SafeRedisTransaction for ProcessBorrowedTransactions<'_> {
                     report.moved_to_pending += 1;
                 }
                 SubmissionResultType::Fail(err) => {
-                    // Mark as failed and set expiry
+                    // Mark as failed
                     let tx_data_key = self.keys.transaction_data_key_name(transaction_id);
                     pipeline.hset(&tx_data_key, "status", "failed");
                     pipeline.hset(&tx_data_key, "completed_at", now);
                     pipeline.hset(&tx_data_key, "failure_reason", err.to_string());
 
-                    let ttl: i64 = i64::try_from(self.failed_transaction_expiry_seconds)
-                        .unwrap_or(i64::MAX)
-                        .max(1);
-
-                    // Set expiry on all related keys for failed transactions
-                    let transaction_keys = self.keys.get_all_transaction_keys(transaction_id);
-                    for key in transaction_keys {
-                        pipeline.expire(&key, ttl);
-                    }
-
                     // ask for this nonce to be recycled because we did not consume the nonce
                     pipeline.zadd(self.keys.recycled_nonces_zset_name(), nonce, nonce);
 

executors/src/eoa/store/borrowed.rs

@@ -186,26 +186,6 @@ impl EoaExecutorStoreKeys {
         }
     }
 
-    /// Get all Redis keys related to a transaction for cleanup
-    pub fn get_all_transaction_keys(&self, transaction_id: &str) -> Vec<String> {
-        vec![
-            self.transaction_data_key_name(transaction_id),
-            self.transaction_attempts_list_name(transaction_id),
-        ]
-    }
-
-    /// Get all Redis keys related to a transaction including hash mappings for cleanup
-    pub fn get_all_transaction_keys_with_hashes(&self, transaction_id: &str, transaction_hashes: &[String]) -> Vec<String> {
-        let mut keys = self.get_all_transaction_keys(transaction_id);
-        
-        // Add hash-to-id mappings
-        for hash in transaction_hashes {
-            keys.push(self.transaction_hash_to_id_key_name(hash));
-        }
-        
-        keys
-    }
-
     /// Name of the hashmap that maps `transaction_id` -> `BorrowedTransactionData`
     ///
     /// This is used for crash recovery. Before submitting a transaction, we atomically move from pending to this borrowed hashmap.
@@ -420,6 +400,13 @@ impl NonceType {
     }
 }
 
+pub struct EoaExecutorCounts {
+    pub pending_transactions: u64,
+    pub submitted_transactions: u64,
+    pub borrowed_transactions: u64,
+    pub recycled_nonces: u64,
+}
+
 impl EoaExecutorStore {
     /// Aggressively acquire EOA lock, forcefully taking over from stalled workers
     ///
@@ -533,6 +520,24 @@ impl EoaExecutorStore {
         }
     }
 
+    pub async fn get_all_counts(&self) -> Result<EoaExecutorCounts, TransactionStoreError> {
+        let mut conn = self.redis.clone();
+        let mut pipeline = twmq::redis::pipe();
+
+        pipeline.zcard(self.pending_transactions_zset_name());
+        pipeline.zcard(self.submitted_transactions_zset_name());
+        pipeline.hlen(self.borrowed_transactions_hashmap_name());
+        pipeline.zcard(self.recycled_nonces_zset_name());
+
+        let counts: (u64, u64, u64, u64) = pipeline.query_async(&mut conn).await?;
+        Ok(EoaExecutorCounts {
+            pending_transactions: counts.0,
+            submitted_transactions: counts.1,
+            borrowed_transactions: counts.2,
+            recycled_nonces: counts.3,
+        })
+    }
+
     /// Peek at pending transactions without removing them (safe for planning)
     pub async fn peek_pending_transactions(
         &self,

executors/src/eoa/store/mod.rs

@@ -354,15 +354,6 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> {
                     (id, Some(confirmed_tx)) => {
                         // Clean up confirmed transaction from Redis
                         self.remove_transaction_from_redis_submitted_zset(pipeline, tx);
-
-                        // IMMEDIATE CLEANUP: Delete all transaction data since it's confirmed
-                        // Note: Hash mappings will be cleaned up periodically by maintenance script
-                        // TODO: Buggy deletions, fix later
-                        // let keys_to_delete = self.keys.get_all_transaction_keys(id);
-                        // for key in keys_to_delete {
-                        //     pipeline.del(&key);
-                        // }
-
                         let data_key_name = self.keys.transaction_data_key_name(id);
                         pipeline.hset(&data_key_name, "status", "confirmed");
                         pipeline.hset(&data_key_name, "completed_at", now);
@@ -377,6 +368,7 @@ impl SafeRedisTransaction for CleanSubmittedTransactions<'_> {
                             let confirmed_timestamp = current_timestamp_ms();
                             let queued_to_mined_duration =
                                 calculate_duration_seconds(tx.queued_at, confirmed_timestamp);
+                            // Record metrics using the clean EoaMetrics abstraction
                             self.eoa_metrics.record_transaction_confirmed(
                                 self.keys.eoa,
                                 self.keys.chain_id,