feat: enhance subscription management with metrics and reconnection

Cargo.toml

@@ -91,7 +91,7 @@ jsonrpc-pubsub = "18.0.0"
 jsonrpc-ws-server = "18.0.0"
 lazy_static = "1.4.0"
 libc = "0.2.153"
-log = { version = "0.4.20", features = ["release_max_level_info"] }
+log = { version = "0.4.20" }
 lru = "0.16.0"
 macrotest = "1"
 magic-domain-program = { git = "https://github.com/magicblock-labs/magic-domain-program.git", rev = "ea04d46", default-features = false }
@@ -151,7 +151,7 @@ serde = "1.0.217"
 serde_derive = "1.0"
 serde_json = "1.0"
 sha3 = "0.10.8"
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "f454d4a" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "731fa50" }
 solana-account-decoder = { version = "2.2" }
 solana-accounts-db = { version = "2.2" }
 solana-account-decoder-client-types = { version = "2.2" }
@@ -227,6 +227,6 @@ features = ["dev-context-only-utils"]
 # some solana dependencies have solana-storage-proto as dependency
 # we need to patch them with our version, because they use protobuf-src v1.1.0
 # and we use protobuf-src v2.1.1. Otherwise compilation fails
-solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "f454d4a" }
+solana-account = { git = "https://github.com/magicblock-labs/solana-account.git", rev = "731fa50" }
 solana-storage-proto = { path = "./storage-proto" }
 solana-svm = { git = "https://github.com/magicblock-labs/magicblock-svm.git", rev = "11bbaf2" }

magicblock-account-cloner/src/lib.rs

@@ -173,12 +173,15 @@ impl ChainlinkCloner {
                 // Create and initialize the program account in retracted state
                 // and then deploy it and finally set the authority to match the
                 // one on chain
+                let slot = self.accounts_db.slot();
                 let DeployableV4Program {
                     pre_deploy_loader_state,
                     deploy_instruction,
                     post_deploy_loader_state,
-                } = program
-                    .try_into_deploy_data_and_ixs_v4(validator_kp.pubkey())?;
+                } = program.try_into_deploy_data_and_ixs_v4(
+                    slot,
+                    validator_kp.pubkey(),
+                )?;
 
                 let lamports = Rent::default()
                     .minimum_balance(pre_deploy_loader_state.len());

magicblock-accounts-db/src/lib.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, sync::Arc};
+use std::{collections::HashSet, path::Path, sync::Arc};
 
 use error::AccountsDbError;
 use index::{
@@ -356,7 +356,7 @@ impl AccountsBank for AccountsDb {
             .iter_all()
             .filter(|(pk, acc)| predicate(pk, acc))
             .map(|(pk, _)| pk)
-            .collect::<Vec<_>>();
+            .collect::<HashSet<_>>();
         let removed = to_remove.len();
         for pk in to_remove {
             self.remove_account(&pk);

magicblock-aperture/src/requests/http/mod.rs

@@ -112,7 +112,7 @@ impl HttpDispatcher {
             .inspect_err(|e| {
                 // There is nothing we can do if fetching the account fails
                 // Log the error and return whatever is in the accounts db
-                warn!("Failed to ensure account {pubkey}: {e}");
+                debug!("Failed to ensure account {pubkey}: {e}");
             });
         self.accountsdb.get_account(pubkey)
     }

magicblock-aperture/src/requests/http/send_transaction.rs

@@ -1,4 +1,4 @@
-use log::{debug, trace};
+use log::*;
 use magicblock_metrics::metrics::{
     TRANSACTION_PROCESSING_TIME, TRANSACTION_SKIP_PREFLIGHT,
 };

magicblock-aperture/src/tests.rs

@@ -42,6 +42,7 @@ fn chainlink(accounts_db: &Arc<AccountsDb>) -> ChainlinkImpl {
         None,
         Pubkey::new_unique(),
         Pubkey::new_unique(),
+        0,
     )
     .expect("Failed to create Chainlink")
 }

magicblock-aperture/tests/setup.rs

@@ -62,6 +62,7 @@ fn chainlink(accounts_db: &Arc<AccountsDb>) -> Arc<ChainlinkImpl> {
             None,
             Pubkey::new_unique(),
             Pubkey::new_unique(),
+            0,
         )
         .expect("Failed to create Chainlink"),
     )

magicblock-api/src/magic_validator.rs

@@ -441,6 +441,7 @@ impl MagicValidator {
             validator_pubkey,
             faucet_pubkey,
             chainlink_config,
+            config.accounts.clone.auto_airdrop_lamports,
         )
         .await?;
 

magicblock-api/src/tickers.rs

@@ -101,7 +101,6 @@ async fn handle_scheduled_commits<C: ScheduledCommitsProcessor>(
         error!("Failed to process scheduled commits: {:?}", err);
     }
 }
-
 #[allow(unused_variables)]
 pub fn init_system_metrics_ticker(
     tick_duration: Duration,

magicblock-chainlink/Cargo.toml

@@ -4,6 +4,7 @@ version.workspace = true
 edition.workspace = true
 
 [dependencies]
+arc-swap = "1.7"
 async-trait = { workspace = true }
 bincode = { workspace = true }
 env_logger = { workspace = true }
@@ -12,7 +13,8 @@ log = { workspace = true }
 lru = { workspace = true }
 magicblock-core = { workspace = true }
 magicblock-magic-program-api = { workspace = true }
-magicblock-delegation-program = { workspace = true }
+magicblock-metrics = { workspace = true }
+ magicblock-delegation-program = { workspace = true }
 serde_json = { workspace = true }
 solana-account = { workspace = true }
 solana-account-decoder = { workspace = true }

magicblock-chainlink/src/chainlink/blacklisted_accounts.rs

@@ -11,7 +11,6 @@ pub fn blacklisted_accounts(
     // want to take a dependency on that crate just for this ID which won't change
     const NATIVE_SOL_ID: Pubkey =
         solana_sdk::pubkey!("So11111111111111111111111111111111111111112");
-
     let mut blacklisted_accounts = sysvar_accounts()
         .into_iter()
         .chain(native_program_accounts())
@@ -48,6 +47,9 @@ pub fn sysvar_accounts() -> HashSet<Pubkey> {
 }
 
 pub fn native_program_accounts() -> HashSet<Pubkey> {
+    const NATIVE_TOKEN_PROGRAM_ID: Pubkey =
+        solana_sdk::pubkey!("TokenkegQfeZyiNwAJbNbGKPFXCWuBvf9Ss623VQ5DA");
+
     let mut blacklisted_programs = HashSet::new();
     blacklisted_programs.insert(solana_sdk::address_lookup_table::program::ID);
     blacklisted_programs.insert(solana_sdk::bpf_loader::ID);
@@ -63,5 +65,6 @@ pub fn native_program_accounts() -> HashSet<Pubkey> {
     blacklisted_programs.insert(solana_sdk::stake::program::ID);
     blacklisted_programs.insert(solana_sdk::system_program::ID);
     blacklisted_programs.insert(solana_sdk::vote::program::ID);
+    blacklisted_programs.insert(NATIVE_TOKEN_PROGRAM_ID);
     blacklisted_programs
 }

magicblock-chainlink/src/chainlink/errors.rs

@@ -18,7 +18,7 @@ pub enum ChainlinkError {
     #[error("Cloner error: {0}")]
     ClonerError(#[from] crate::cloner::errors::ClonerError),
 
-    #[error("Delegation could not be decoded: {0} ({1:?})")]
+    #[error("Delegation record could not be decoded: {0} ({1:?})")]
     InvalidDelegationRecord(Pubkey, ProgramError),
 
     #[error("Failed to resolve one or more accounts {0} when getting delegation records")]