diff --git a/Cargo.lock b/Cargo.lock index e0b4f5dae..41265d0ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -17585,6 +17585,7 @@ dependencies = [ "frame-system", "futures", "hex", + "hex-literal", "hydradx", "hydradx-runtime", "indicatif", diff --git a/integration-tests/src/lib.rs b/integration-tests/src/lib.rs index ba56e2c68..4e51ed003 100644 --- a/integration-tests/src/lib.rs +++ b/integration-tests/src/lib.rs @@ -40,6 +40,7 @@ mod parameters; mod polkadot_test_net; mod referrals; mod router; +mod slim_snapshot; mod stableswap; mod stableswap_curve_comparison; mod staking; diff --git a/integration-tests/src/slim_snapshot.rs b/integration-tests/src/slim_snapshot.rs new file mode 100644 index 000000000..ed64ddee2 --- /dev/null +++ b/integration-tests/src/slim_snapshot.rs @@ -0,0 +1,728 @@ +#![cfg(test)] + +use crate::polkadot_test_net::{hydra_live_ext, ALICE, BOB}; +use frame_support::assert_ok; +use hydradx_runtime::{AssetId, Balance, Currencies, Omnipool, Router, RuntimeOrigin, Stableswap, System, XYK}; +use hydradx_traits::router::{PoolType, Trade}; +use orml_traits::MultiCurrency; + +const PATH_TO_SNAPSHOT: &str = "snapshots/slim/SNAPSHOT"; + +const HDX: AssetId = 0; +const HDX_UNITS: Balance = 1_000_000_000_000; + +/// Reset block number and clear stale DynamicFees entries to avoid debug_assert panics. +fn init_block() { + System::set_block_number(1); + pallet_timestamp::Pallet::::set_timestamp(1); + // Clear DynamicFees.AssetFee entries to avoid timestamp mismatch with reset oracle entries + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"DynamicFees")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"AssetFee")); + p + }; + sp_io::storage::clear_prefix(&prefix, None); +} + +/// Find the first asset in the Omnipool (besides `exclude`). +fn find_omnipool_asset(exclude: AssetId) -> Option<(AssetId, Balance)> { + let omnipool_account = Omnipool::protocol_account(); + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Omnipool")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Assets")); + p + }; + let mut key = sp_io::storage::next_key(&prefix); + while let Some(k) = key { + if !k.starts_with(&prefix) { + break; + } + if k.len() >= 52 { + let asset_id = u32::from_le_bytes(k[48..52].try_into().unwrap()); + if asset_id != exclude { + let balance = Currencies::free_balance(asset_id, &omnipool_account); + if balance > 0 { + return Some((asset_id, balance)); + } + } + } + key = sp_io::storage::next_key(&k); + } + None +} + +/// Find a Stableswap pool and its constituent assets. +/// Returns (pool_id, asset_a, asset_b) where asset_a and asset_b are the first two pool assets. +fn find_stableswap_pool() -> Option<(AssetId, AssetId, AssetId)> { + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Stableswap")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Pools")); + p + }; + let mut key = sp_io::storage::next_key(&prefix); + while let Some(ref k) = key { + if !k.starts_with(&prefix) { + break; + } + if k.len() >= 52 { + let pool_id = u32::from_le_bytes(k[48..52].try_into().unwrap()); + if let Some(val) = sp_io::storage::get(k) { + // Pool struct starts with: assets: BoundedVec + // SCALE-encoded BoundedVec starts with compact length prefix + if val.len() >= 10 { + let (asset_count, offset) = if val[0] < 0xFC { + ((val[0] >> 2) as usize, 1usize) + } else { + key = sp_io::storage::next_key(k); + continue; + }; + if asset_count >= 2 && val.len() >= offset + 8 { + let asset_a = u32::from_le_bytes(val[offset..offset + 4].try_into().unwrap()); + let asset_b = u32::from_le_bytes(val[offset + 4..offset + 8].try_into().unwrap()); + + let pool_account = { + let mut hash_input = Vec::new(); + hash_input.extend_from_slice(b"sts"); + hash_input.extend_from_slice(&pool_id.to_le_bytes()); + sp_runtime::AccountId32::new(sp_io::hashing::blake2_256(&hash_input)) + }; + let bal_a = Currencies::free_balance(asset_a, &pool_account); + let bal_b = Currencies::free_balance(asset_b, &pool_account); + // Require meaningful but not huge liquidity to avoid overflow + // Skip pools with very large balances (likely 18-decimal EVM assets) + let max_bal = 1_000_000_000_000_000_000_000u128; // 10^21 + if bal_a > 1_000_000_000_000_000 + && bal_b > 1_000_000_000_000_000 + && bal_a < max_bal && bal_b < max_bal + { + return Some((pool_id, asset_a, asset_b)); + } + } + } + } + } + key = sp_io::storage::next_key(k); + } + None +} + +/// Find an XYK pool with liquidity. +/// Returns (asset_a, asset_b). +fn find_xyk_pool() -> Option<(AssetId, AssetId)> { + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"XYK")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"PoolAssets")); + p + }; + let mut key = sp_io::storage::next_key(&prefix); + while let Some(k) = key { + if !k.starts_with(&prefix) { + break; + } + if let Some(val) = sp_io::storage::get(&k) { + if val.len() >= 8 { + let asset_a = u32::from_le_bytes(val[0..4].try_into().unwrap()); + let asset_b = u32::from_le_bytes(val[4..8].try_into().unwrap()); + + // Check the pool account has liquidity + let pool_account_bytes = &k[k.len() - 32..]; + let pool_account = sp_runtime::AccountId32::new(pool_account_bytes.try_into().unwrap()); + let bal_a = Currencies::free_balance(asset_a, &pool_account); + let bal_b = Currencies::free_balance(asset_b, &pool_account); + // Prefer pools where both assets are well-known (id < 100) + let has_known_asset = asset_a < 100 && asset_b < 100; + let max_bal = 1_000_000_000_000_000_000_000u128; + if has_known_asset + && bal_a > 1_000_000_000_000_000 + && bal_b > 1_000_000_000_000_000 + && bal_a < max_bal + && bal_b < max_bal + { + return Some((asset_a, asset_b)); + } + } + } + key = sp_io::storage::next_key(&k); + } + None +} + +// ============================================================================= +// Snapshot loading +// ============================================================================= + +#[test] +fn slim_snapshot_should_load() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + let block_number = frame_system::Pallet::::block_number(); + assert!(block_number > 0, "Block number should be > 0 from production snapshot"); + }); +} + +#[test] +fn protocol_accounts_have_balances() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + let omnipool_account = Omnipool::protocol_account(); + let omnipool_hdx = Currencies::free_balance(HDX, &omnipool_account); + assert!(omnipool_hdx > 0, "Omnipool should hold HDX: {omnipool_hdx}"); + + let treasury = hydradx_runtime::Treasury::account_id(); + let treasury_hdx = Currencies::free_balance(HDX, &treasury); + assert!(treasury_hdx > 0, "Treasury should hold HDX: {treasury_hdx}"); + }); +} + +// ============================================================================= +// Omnipool trades +// ============================================================================= + +#[test] +fn omnipool_sell_hdx_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (asset_out, _) = find_omnipool_asset(HDX).expect("No tradeable Omnipool asset found"); + + assert_ok!(Currencies::deposit(HDX, &alice, 10_000 * HDX_UNITS)); + let balance_before = Currencies::free_balance(asset_out, &alice); + + assert_ok!(Omnipool::sell( + RuntimeOrigin::signed(alice.clone()), + HDX, + asset_out, + 1_000 * HDX_UNITS, + 0u128, + )); + + let balance_after = Currencies::free_balance(asset_out, &alice); + assert!( + balance_after > balance_before, + "Alice should have received asset {asset_out}" + ); + }); +} + +#[test] +fn omnipool_buy_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (asset_out, _) = find_omnipool_asset(HDX).expect("No tradeable Omnipool asset found"); + + // Buy 0.1% of what the Omnipool holds + let omnipool_account = Omnipool::protocol_account(); + let pool_bal = Currencies::free_balance(asset_out, &omnipool_account); + let buy_amount = pool_bal / 1000; + assert_ok!(Currencies::deposit(HDX, &alice, 1_000_000 * HDX_UNITS)); + let balance_before = Currencies::free_balance(asset_out, &alice); + + assert_ok!(Omnipool::buy( + RuntimeOrigin::signed(alice.clone()), + asset_out, + HDX, + buy_amount, + u128::MAX, + )); + + let balance_after = Currencies::free_balance(asset_out, &alice); + assert!( + balance_after > balance_before, + "Alice should have bought asset {asset_out}" + ); + }); +} + +#[test] +fn omnipool_sell_between_two_non_hdx_assets_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let bob = sp_runtime::AccountId32::from(BOB); + + let (asset_a, _) = find_omnipool_asset(HDX).expect("No tradeable Omnipool asset found"); + let (asset_b, _) = match find_omnipool_asset(asset_a) { + Some(ab) => ab, + None => { + println!("Only one non-HDX Omnipool asset found, skipping"); + return; + } + }; + + // Use 0.1% of the Omnipool's balance of asset_a to avoid liquidity limits + let omnipool_account = Omnipool::protocol_account(); + let pool_balance = Currencies::free_balance(asset_a, &omnipool_account); + let sell_amount = pool_balance / 1000; // 0.1% + assert_ok!(Currencies::deposit(asset_a, &bob, sell_amount * 2)); + let balance_before = Currencies::free_balance(asset_b, &bob); + + assert_ok!(Omnipool::sell( + RuntimeOrigin::signed(bob.clone()), + asset_a, + asset_b, + sell_amount, + 0u128, + )); + + let balance_after = Currencies::free_balance(asset_b, &bob); + assert!( + balance_after > balance_before, + "Bob should have received asset {asset_b}" + ); + }); +} + +// ============================================================================= +// Router trades +// ============================================================================= + +#[test] +fn router_sell_via_omnipool_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (asset_out, _) = find_omnipool_asset(HDX).expect("No tradeable Omnipool asset found"); + + assert_ok!(Currencies::deposit(HDX, &alice, 10_000 * HDX_UNITS)); + let balance_before = Currencies::free_balance(asset_out, &alice); + + assert_ok!(Router::sell( + RuntimeOrigin::signed(alice.clone()), + HDX, + asset_out, + 500 * HDX_UNITS, + 0, + vec![Trade { + pool: PoolType::Omnipool, + asset_in: HDX, + asset_out, + }] + .try_into() + .unwrap(), + )); + + let balance_after = Currencies::free_balance(asset_out, &alice); + assert!(balance_after > balance_before, "Router Omnipool sell should work"); + }); +} + +// ============================================================================= +// Stableswap trades +// ============================================================================= + +#[test] +fn stableswap_sell_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (pool_id, asset_in, asset_out) = match find_stableswap_pool() { + Some(pool) => pool, + None => { + println!("No Stableswap pool with liquidity found, skipping"); + return; + } + }; + println!("Stableswap sell: pool={pool_id}, {asset_in} -> {asset_out}"); + + // Use a small fraction of pool liquidity + let pool_account = { + let mut hash_input = Vec::new(); + hash_input.extend_from_slice(b"sts"); + hash_input.extend_from_slice(&pool_id.to_le_bytes()); + sp_runtime::AccountId32::new(sp_io::hashing::blake2_256(&hash_input)) + }; + let pool_bal = Currencies::free_balance(asset_in, &pool_account); + let sell_amount = pool_bal / 100; // 1% of pool + assert_ok!(Currencies::deposit(asset_in, &alice, sell_amount * 2)); + + let balance_before = Currencies::free_balance(asset_out, &alice); + + assert_ok!(Stableswap::sell( + RuntimeOrigin::signed(alice.clone()), + pool_id, + asset_in, + asset_out, + sell_amount, + 0u128, + )); + + let balance_after = Currencies::free_balance(asset_out, &alice); + assert!( + balance_after > balance_before, + "Alice should have received asset {asset_out} from Stableswap" + ); + }); +} + +#[test] +fn stableswap_buy_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let bob = sp_runtime::AccountId32::from(BOB); + + let (pool_id, asset_in, asset_out) = match find_stableswap_pool() { + Some(pool) => pool, + None => { + println!("No Stableswap pool with liquidity found, skipping"); + return; + } + }; + println!("Stableswap buy: pool={pool_id}, pay {asset_in} -> buy {asset_out}"); + + let pool_account = { + let mut hash_input = Vec::new(); + hash_input.extend_from_slice(b"sts"); + hash_input.extend_from_slice(&pool_id.to_le_bytes()); + sp_runtime::AccountId32::new(sp_io::hashing::blake2_256(&hash_input)) + }; + let pool_bal_out = Currencies::free_balance(asset_out, &pool_account); + let buy_amount = pool_bal_out / 100; // 1% of pool + assert_ok!(Currencies::deposit(asset_in, &bob, pool_bal_out)); // enough to cover + + let balance_before = Currencies::free_balance(asset_out, &bob); + + assert_ok!(Stableswap::buy( + RuntimeOrigin::signed(bob.clone()), + pool_id, + asset_out, + asset_in, + buy_amount, + u128::MAX, + )); + + let balance_after = Currencies::free_balance(asset_out, &bob); + assert!( + balance_after > balance_before, + "Bob should have received asset {asset_out} from Stableswap buy" + ); + }); +} + +#[test] +fn router_sell_via_stableswap_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (pool_id, asset_in, asset_out) = match find_stableswap_pool() { + Some(pool) => pool, + None => { + println!("No Stableswap pool with liquidity found, skipping"); + return; + } + }; + println!("Router Stableswap sell: pool={pool_id}, {asset_in} -> {asset_out}"); + + let pool_account = { + let mut hash_input = Vec::new(); + hash_input.extend_from_slice(b"sts"); + hash_input.extend_from_slice(&pool_id.to_le_bytes()); + sp_runtime::AccountId32::new(sp_io::hashing::blake2_256(&hash_input)) + }; + let pool_bal = Currencies::free_balance(asset_in, &pool_account); + let sell_amount = pool_bal / 1000; + assert_ok!(Currencies::deposit(asset_in, &alice, sell_amount * 2)); + let balance_before = Currencies::free_balance(asset_out, &alice); + + assert_ok!(Router::sell( + RuntimeOrigin::signed(alice.clone()), + asset_in, + asset_out, + sell_amount, + 0, + vec![Trade { + pool: PoolType::Stableswap(pool_id), + asset_in, + asset_out, + }] + .try_into() + .unwrap(), + )); + + let balance_after = Currencies::free_balance(asset_out, &alice); + assert!(balance_after > balance_before, "Router Stableswap sell should work"); + }); +} + +// ============================================================================= +// XYK trades +// ============================================================================= + +#[test] +fn xyk_sell_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (asset_a, asset_b) = match find_xyk_pool() { + Some(pool) => pool, + None => { + println!("No XYK pool with liquidity found, skipping"); + return; + } + }; + println!("XYK sell: {asset_a} -> {asset_b}"); + + // Use a fraction of pool liquidity to avoid ED/slippage issues + let pool_account_bytes = { + // The XYK pool account is determined by the find_xyk_pool key — re-derive it + let mut buf: Vec = b"xyk".to_vec(); + let (min, max) = if asset_a < asset_b { + (asset_a, asset_b) + } else { + (asset_b, asset_a) + }; + buf.extend_from_slice(&min.to_le_bytes()); + buf.extend_from_slice(&max.to_le_bytes()); + sp_io::hashing::blake2_256(&buf) + }; + let pool_account = sp_runtime::AccountId32::new(pool_account_bytes); + let pool_bal = Currencies::free_balance(asset_a, &pool_account); + let sell_amount = pool_bal / 100; // 1% of pool + assert_ok!(Currencies::deposit(asset_a, &alice, sell_amount * 2)); + + let balance_before = Currencies::free_balance(asset_b, &alice); + + assert_ok!(XYK::sell( + RuntimeOrigin::signed(alice.clone()), + asset_a, + asset_b, + sell_amount, + 0u128, + false, + )); + + let balance_after = Currencies::free_balance(asset_b, &alice); + assert!( + balance_after > balance_before, + "Alice should have received asset {asset_b} from XYK sell" + ); + }); +} + +#[test] +fn xyk_buy_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let bob = sp_runtime::AccountId32::from(BOB); + + let (asset_a, asset_b) = match find_xyk_pool() { + Some(pool) => pool, + None => { + println!("No XYK pool with liquidity found, skipping"); + return; + } + }; + println!("XYK buy: buy {asset_b} with {asset_a}"); + + let pool_account_bytes = { + let mut buf: Vec = b"xyk".to_vec(); + let (min, max) = if asset_a < asset_b { + (asset_a, asset_b) + } else { + (asset_b, asset_a) + }; + buf.extend_from_slice(&min.to_le_bytes()); + buf.extend_from_slice(&max.to_le_bytes()); + sp_io::hashing::blake2_256(&buf) + }; + let pool_account = sp_runtime::AccountId32::new(pool_account_bytes); + let pool_bal_b = Currencies::free_balance(asset_b, &pool_account); + let buy_amount = pool_bal_b / 10; // 10% of pool + assert_ok!(Currencies::deposit( + asset_a, + &bob, + Currencies::free_balance(asset_a, &pool_account) + )); + + let balance_before = Currencies::free_balance(asset_b, &bob); + + assert_ok!(XYK::buy( + RuntimeOrigin::signed(bob.clone()), + asset_b, + asset_a, + buy_amount, + u128::MAX, + false, + )); + + let balance_after = Currencies::free_balance(asset_b, &bob); + assert!( + balance_after > balance_before, + "Bob should have received asset {asset_b} from XYK buy" + ); + }); +} + +#[test] +fn router_sell_via_xyk_should_work() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + init_block(); + let alice = sp_runtime::AccountId32::from(ALICE); + + let (asset_a, asset_b) = match find_xyk_pool() { + Some(pool) => pool, + None => { + println!("No XYK pool with liquidity found, skipping"); + return; + } + }; + println!("Router XYK sell: {asset_a} -> {asset_b}"); + + let pool_account_bytes = { + let mut buf: Vec = b"xyk".to_vec(); + let (min, max) = if asset_a < asset_b { + (asset_a, asset_b) + } else { + (asset_b, asset_a) + }; + buf.extend_from_slice(&min.to_le_bytes()); + buf.extend_from_slice(&max.to_le_bytes()); + sp_io::hashing::blake2_256(&buf) + }; + let pool_account = sp_runtime::AccountId32::new(pool_account_bytes); + let pool_bal = Currencies::free_balance(asset_a, &pool_account); + let sell_amount = pool_bal / 100; // 1% of pool + assert_ok!(Currencies::deposit(asset_a, &alice, sell_amount * 2)); + + let balance_before = Currencies::free_balance(asset_b, &alice); + + assert_ok!(Router::sell( + RuntimeOrigin::signed(alice.clone()), + asset_a, + asset_b, + sell_amount, + 0, + vec![Trade { + pool: PoolType::XYK, + asset_in: asset_a, + asset_out: asset_b, + }] + .try_into() + .unwrap(), + )); + + let balance_after = Currencies::free_balance(asset_b, &alice); + assert!(balance_after > balance_before, "Router XYK sell should work"); + }); +} + +// ============================================================================= +// Aave (supply via EVM) +// ============================================================================= + +#[test] +fn aave_supply_dot_should_work() { + use fp_evm::ExitReason::Succeed; + use fp_evm::ExitSucceed::Returned; + use hydradx_runtime::{ + evm::{precompiles::erc20_mapping::HydraErc20Mapping, Executor}, + AccountId, EVMAccounts, Runtime, + }; + use hydradx_traits::evm::{CallContext, Erc20Encoding, InspectEvmAccounts, EVM}; + use liquidation_worker_support::*; + use pallet_liquidation::BorrowingContract; + use sp_core::U256; + + const DOT: AssetId = 5; + const DOT_UNIT: Balance = 10_000_000_000; + + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + // Don't call init_block() — Aave needs realistic timestamps for interest calculations + + // BorrowingContract and ApprovedContract are already set from production state + let pool_contract = >::get(); + println!("Pool contract from snapshot: {pool_contract:?}"); + + let alice = sp_runtime::AccountId32::from(ALICE); + assert_ok!(Currencies::deposit(DOT, &alice, 1_000 * DOT_UNIT)); + + assert_ok!(EVMAccounts::bind_evm_address(RuntimeOrigin::signed(alice.clone()))); + let alice_evm = EVMAccounts::evm_address(&AccountId::from(ALICE)); + + // Supply DOT to Aave + let evm_dot = HydraErc20Mapping::encode_evm_address(DOT); + let context = CallContext::new_call(pool_contract, alice_evm); + let data = hydradx_runtime::evm::precompiles::handle::EvmDataWriter::new_with_selector(Function::Supply) + .write(evm_dot) + .write(100 * DOT_UNIT) + .write(alice_evm) + .write(0u32) + .build(); + + let call_result = Executor::::call(context, data, U256::zero(), 5_000_000); + assert_eq!( + call_result.exit_reason, + Succeed(Returned), + "Aave supply failed: {:?}", + hex::encode(call_result.value) + ); + println!("Aave supply succeeded!"); + }); +} + +// ============================================================================= +// Discovery (diagnostic, ignored by default) +// ============================================================================= + +#[test] +#[ignore] +fn discover_pools_and_assets() { + let mut ext = hydra_live_ext(PATH_TO_SNAPSHOT); + ext.execute_with(|| { + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Stableswap")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Pools")); + p + }; + let mut key = sp_io::storage::next_key(&prefix); + while let Some(k) = key { + if !k.starts_with(&prefix) { + break; + } + if k.len() >= 52 { + let pool_id = u32::from_le_bytes(k[48..52].try_into().unwrap()); + println!("Stableswap pool_id: {pool_id}"); + } + key = sp_io::storage::next_key(&k); + } + + let prefix = { + let mut p = Vec::new(); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Omnipool")); + p.extend_from_slice(&sp_io::hashing::twox_128(b"Assets")); + p + }; + let mut key = sp_io::storage::next_key(&prefix); + while let Some(k) = key { + if !k.starts_with(&prefix) { + break; + } + if k.len() >= 52 { + let asset_id = u32::from_le_bytes(k[48..52].try_into().unwrap()); + println!("Omnipool asset: {asset_id}"); + } + key = sp_io::storage::next_key(&k); + } + }); +} diff --git a/scraper/Cargo.toml b/scraper/Cargo.toml index fdb996d6c..f4df19d76 100644 --- a/scraper/Cargo.toml +++ b/scraper/Cargo.toml @@ -38,6 +38,7 @@ pallet-balances = { workspace = true } serde_json = { version = "1.0", features = ["preserve_order"] } hex = "0.4" +hex-literal = { workspace = true } futures = "0.3.30" [features] diff --git a/scraper/src/lib.rs b/scraper/src/lib.rs index 9b147d85e..96da137ad 100644 --- a/scraper/src/lib.rs +++ b/scraper/src/lib.rs @@ -51,32 +51,33 @@ where } pub type SnapshotVersion = Compact; -pub const SNAPSHOT_VERSION: SnapshotVersion = Compact(3); +pub const SNAPSHOT_VERSION: SnapshotVersion = Compact(4); /// The snapshot that we store on disk. +/// Must match the format from `frame-remote-externalities`. #[derive(Decode, Encode, Clone)] pub struct Snapshot { snapshot_version: SnapshotVersion, state_version: StateVersion, - block_hash: B::Hash, // > raw_storage: Vec<(Vec, (Vec, i32))>, storage_root: B::Hash, + header: B::Header, } impl Snapshot { pub fn new( state_version: StateVersion, - block_hash: B::Hash, raw_storage: Vec<(Vec, (Vec, i32))>, storage_root: B::Hash, + header: B::Header, ) -> Self { Self { snapshot_version: SNAPSHOT_VERSION, state_version, - block_hash, raw_storage, storage_root, + header, } } @@ -111,7 +112,12 @@ pub fn save_externalities>(ext: TestExternalities, path: let state_version = ext.state_version; let (raw_storage, storage_root) = ext.into_raw_snapshot(); - let snapshot = Snapshot::::new(state_version, B::Hash::default(), raw_storage, storage_root); + // Construct a minimal header for the snapshot format. + // This is only used in tests; production snapshots use save_slim_snapshot which gets the real header. + let header = B::Header::decode(&mut frame_support::sp_runtime::traits::TrailingZeroInput::new(&[0u8])) + .expect("infinite input; qed"); + + let snapshot = Snapshot::::new(state_version, raw_storage, storage_root, header); let encoded = snapshot.encode(); fs::write(path, encoded).map_err(|_| "fs::write failed")?; @@ -167,9 +173,9 @@ pub fn filter_snapshot_by_excluded_pallets>( // Create new snapshot with filtered storage let filtered_snapshot = Snapshot::::new( snapshot.state_version, - snapshot.block_hash, filtered_storage, snapshot.storage_root, + snapshot.header, ); // Save the filtered snapshot @@ -182,7 +188,7 @@ pub fn filter_snapshot_by_excluded_pallets>( pub fn load_snapshot>(path: PathBuf) -> Result { let Snapshot { snapshot_version: _, - block_hash: _, + header: _, state_version, raw_storage, storage_root, @@ -196,7 +202,7 @@ pub fn load_snapshot>(path: PathBuf) -> Result>(bytes: Vec) -> Result { let Snapshot { snapshot_version: _, - block_hash: _, + header: _, state_version, raw_storage, storage_root, @@ -216,7 +222,7 @@ pub fn construct_backend_from_snapshot>( ) -> Result<(sp_trie::PrefixedMemoryDB, StateVersion, H256), &'static str> { let Snapshot { snapshot_version: _, - block_hash: _, + header: _, state_version, raw_storage, storage_root, @@ -260,7 +266,7 @@ pub fn create_externalities_from_snapshot>( ) -> Result { let Snapshot { snapshot_version: _, - block_hash: _, + header: _, state_version, raw_storage, storage_root, @@ -433,6 +439,327 @@ pub async fn fetch_all_storage( Ok(all_pairs) } +/// Slim snapshot filtering. +/// +/// We use `sp_io::storage::clear()` inside `execute_with()` instead of filtering raw trie entries +/// directly. The raw snapshot is a Merkle-Patricia trie — removing leaf entries from the raw bytes +/// leaves parent/branch nodes still referencing them, breaking the trie with "Database missing +/// expected key" errors. `sp_io::storage::clear()` goes through Substrate's storage layer which +/// properly updates the trie structure. +mod slim { + use std::collections::HashSet; + + /// Compute `twox128(a) ++ twox128(b)` as a 32-byte storage prefix. + pub fn storage_prefix(pallet: &str, item: &str) -> Vec { + let mut prefix = Vec::with_capacity(32); + prefix.extend_from_slice(&sp_io::hashing::twox_128(pallet.as_bytes())); + prefix.extend_from_slice(&sp_io::hashing::twox_128(item.as_bytes())); + prefix + } + + /// Build a PalletId-derived account: `b"modl" + id_bytes + zero_padding` to 32 bytes. + fn pallet_account(id: &[u8; 8]) -> [u8; 32] { + let mut account = [0u8; 32]; + account[..4].copy_from_slice(b"modl"); + account[4..12].copy_from_slice(id); + account + } + + /// Build a truncated EVM account: `b"ETH\0" + h160 + 8_zero_bytes`. + fn evm_truncated_account(h160: &[u8; 20]) -> [u8; 32] { + let mut account = [0u8; 32]; + account[..4].copy_from_slice(b"ETH\0"); + account[4..24].copy_from_slice(h160); + account + } + + /// Extract account from end of key (for single-key maps like System.Account, XYK.PoolAssets). + /// Key = prefix(32) + blake2_128_concat(account)(48). Account at last 32 bytes. + fn account_from_key_tail(key: &[u8]) -> Option<[u8; 32]> { + if key.len() < 32 { + return None; + } + let mut account = [0u8; 32]; + account.copy_from_slice(&key[key.len() - 32..]); + Some(account) + } + + /// Extract account from a Blake2_128Concat first key position. + /// Key = prefix(32) + blake2_128(account)(16) + account(32) + ...rest. + /// Account is at offset 48..80. + fn account_from_first_key(key: &[u8]) -> Option<[u8; 32]> { + if key.len() < 80 { + return None; + } + let mut account = [0u8; 32]; + account.copy_from_slice(&key[48..80]); + Some(account) + } + + /// Iterate all storage keys under a given prefix using sp_io. + fn iter_keys_with_prefix(prefix: &[u8]) -> Vec> { + let mut keys = Vec::new(); + let mut current = sp_io::storage::next_key(prefix); + while let Some(key) = current { + if !key.starts_with(prefix) { + break; + } + keys.push(key.clone()); + current = sp_io::storage::next_key(&key); + } + keys + } + + /// Build allow-list by reading storage via sp_io (called inside execute_with). + pub fn build_allow_list() -> HashSet<[u8; 32]> { + let mut accounts = HashSet::new(); + + // Hardcoded dev accounts + for a in [ + hex_literal::hex!("d43593c715fdd31c61141abd04a99fd6822c8558854ccde39a5684e7a56da27d"), + hex_literal::hex!("8eaf04151687736326c9fea17e25fc5287613693c912909cb226aa4794f26a48"), + hex_literal::hex!("90b5ab205c6974c9ea841be688864633dc9ca8a6e38e35e40ef95fd7d98de856"), + hex_literal::hex!("306721211d5404bd9da88e0204360a1a9ab8b87c66c1bc2fcdd37f3c2222cc20"), + hex_literal::hex!("e659a7a1628cdd93febc04a4e0646ea20e9f5f0ce097d9a05290d4a9e054df4e"), + hex_literal::hex!("1cbd2d43530a44705ad088af313e18f80b53ef16b36177cd4b77b846f2a5f07c"), + hex_literal::hex!("aa7e0000000000000000000000000000000aa7e0000000000000000000000000"), + hex_literal::hex!("aa7e0000000000000000000000000000000aa7e1000000000000000000000000"), + ] { + accounts.insert(a); + } + + // EVM dev accounts + for h160 in [ + hex_literal::hex!("8202C0aF5962B750123CE1A9B12e1C30A4973557"), + hex_literal::hex!("8aF7764663644989671A71Abe9738a3cF295f384"), + hex_literal::hex!("C19A2970A13ac19898c47d59Cbd0278D428EBC7c"), + hex_literal::hex!("222222ff7Be76052e023Ec1a306fCca8F9659D80"), + hex_literal::hex!("E52567fF06aCd6CBe7BA94dc777a3126e180B6d9"), + ] { + accounts.insert(evm_truncated_account(&h160)); + } + + // PalletId accounts + for id in [ + b"py/trsry", + b"py/vstng", + b"omnipool", + b"stblpool", + b"staking#", + b"PotStake", + b"OmniWhLM", + b"Omni//LM", + b"xykLMpID", + b"XYK///LM", + b"pltbonds", + b"referral", + b"gigahdx!", + b"gigarwd!", + b"feeproc/", + b"py/hsmod", + b"py/signt", + b"py/fucet", + b"curreser", + b"xcm/alte", + b"otcsettl", + b"routerex", + b"lqdation", + ] { + accounts.insert(pallet_account(id)); + } + + // XYK pool accounts + for key in iter_keys_with_prefix(&storage_prefix("XYK", "PoolAssets")) { + if let Some(account) = account_from_key_tail(&key) { + accounts.insert(account); + } + } + + // Stableswap pool accounts + for key in iter_keys_with_prefix(&storage_prefix("Stableswap", "Pools")) { + if key.len() >= 52 { + let mut hash_input = Vec::with_capacity(7); + hash_input.extend_from_slice(b"sts"); + hash_input.extend_from_slice(&key[48..52]); + accounts.insert(sp_io::hashing::blake2_256(&hash_input)); + } + } + + // LBP pool accounts + owners + for key in iter_keys_with_prefix(&storage_prefix("LBP", "PoolData")) { + if let Some(account) = account_from_key_tail(&key) { + accounts.insert(account); + } + if let Some(value) = sp_io::storage::get(&key) { + if value.len() >= 32 { + let mut owner = [0u8; 32]; + owner.copy_from_slice(&value[..32]); + accounts.insert(owner); + } + } + } + + // EVM contract accounts + for key in iter_keys_with_prefix(&storage_prefix("EVM", "AccountCodes")) { + if key.len() >= 68 { + let mut h160 = [0u8; 20]; + h160.copy_from_slice(&key[48..68]); + accounts.insert(evm_truncated_account(&h160)); + } + } + + // LM GlobalFarm owners + for prefix in [ + storage_prefix("OmnipoolWarehouseLM", "GlobalFarm"), + storage_prefix("XYKWarehouseLM", "GlobalFarm"), + ] { + for key in iter_keys_with_prefix(&prefix) { + if let Some(value) = sp_io::storage::get(&key) { + if value.len() >= 36 { + let mut owner = [0u8; 32]; + owner.copy_from_slice(&value[4..36]); + accounts.insert(owner); + } + } + } + } + + // Dispatcher AaveManagerAccount + if let Some(value) = sp_io::storage::get(&storage_prefix("Dispatcher", "AaveManagerAccount")) { + if value.len() >= 32 { + let mut account = [0u8; 32]; + account.copy_from_slice(&value[..32]); + accounts.insert(account); + } + } + + // Signet Admin + if let Some(value) = sp_io::storage::get(&storage_prefix("Signet", "Admin")) { + if value.len() >= 33 && value[0] == 1 { + let mut account = [0u8; 32]; + account.copy_from_slice(&value[1..33]); + accounts.insert(account); + } + } + + println!("Slim allow-list: {} accounts", accounts.len()); + accounts + } + + fn should_keep(account: &[u8; 32], allow_list: &HashSet<[u8; 32]>) -> bool { + if account[..4] == *b"modl" { + return true; + } + allow_list.contains(account) + } + + /// Clear unwanted storage entries. + /// Must be called inside execute_with(). + /// + /// Note: we intentionally do NOT recalculate TotalIssuance. Pools (especially Stableswap) + /// depend on the original share token issuance. Recalculating it from the remaining accounts + /// would set share issuance to zero (since LP holders are removed), breaking pool operations. + /// Slightly inflated issuance is harmless for testing. + pub fn clear_unwanted_entries(allow_list: &HashSet<[u8; 32]>) { + // (pallet, item, account_is_first_key) + // account_is_first_key=true: double map where AccountId is the first key (offset 48..80) + // account_is_first_key=false: single map where AccountId is the only key (last 32 bytes) + let filterable: [(&str, &str, bool); 6] = [ + ("System", "Account", false), + ("Tokens", "Accounts", true), // DoubleMap + ("Balances", "Locks", false), // Map + ("Tokens", "Locks", true), // DoubleMap + ("MultiTransactionPayment", "AccountCurrencyMap", false), + ("Vesting", "VestingSchedules", false), + ]; + + for (pallet, item, account_is_first_key) in &filterable { + let prefix = storage_prefix(pallet, item); + let keys = iter_keys_with_prefix(&prefix); + let total = keys.len(); + let mut removed = 0usize; + + for key in keys { + let account = if *account_is_first_key { + account_from_first_key(&key) + } else { + account_from_key_tail(&key) + }; + + if let Some(account) = account { + if !should_keep(&account, allow_list) { + sp_io::storage::clear(&key); + removed += 1; + } + } + } + + if removed > 0 { + println!(" Removed {removed}/{total} entries from {pallet}.{item}"); + } + } + } +} + +/// Save a slim snapshot: filter out most user accounts, keep only protocol/pool/dev accounts. +/// Uses execute_with() + sp_io::storage to properly modify trie-backed storage, +/// then rebuilds a fresh trie from the filtered entries to avoid dead trie node bloat. +pub fn save_slim_snapshot>( + mut ext: sp_state_machine::TestExternalities>, + header: B::Header, + path: PathBuf, +) -> Result<(), &'static str> { + // Phase 1: Clear unwanted entries through the proper storage API + ext.execute_with(|| { + println!("Building slim allow-list..."); + let allow_list = slim::build_allow_list(); + + println!("Clearing unwanted storage entries..."); + slim::clear_unwanted_entries(&allow_list); + }); + + ext.commit_all().map_err(|_| "Failed to commit storage changes")?; + + // Phase 2: Extract only the live storage key-value pairs and rebuild a clean trie. + // Without this, the PrefixedMemoryDB retains old trie nodes from before filtering, + // bloating the snapshot (e.g. 1.6M trie entries instead of ~270k). + let mut live_storage = BTreeMap::new(); + ext.execute_with(|| { + let child_prefix = sp_core::storage::well_known_keys::CHILD_STORAGE_KEY_PREFIX; + let mut key = sp_io::storage::next_key(&[]); + while let Some(k) = key { + // Skip child storage keys — they can't go in top storage + if !k.starts_with(child_prefix) { + if let Some(v) = sp_io::storage::get(&k) { + live_storage.insert(k.clone(), v.to_vec()); + } + } + key = sp_io::storage::next_key(&k); + } + }); + + println!( + "Rebuilding clean trie from {} live storage entries...", + live_storage.len() + ); + + let fresh_ext = TestExternalities::new(Storage { + top: live_storage, + children_default: Default::default(), + }); + + let state_version = fresh_ext.state_version; + let (raw_storage, storage_root) = fresh_ext.into_raw_snapshot(); + + println!("Saving slim snapshot with {} trie entries...", raw_storage.len()); + let snapshot = Snapshot::::new(state_version, raw_storage, storage_root, header); + let encoded = snapshot.encode(); + fs::write(&path, encoded).map_err(|_| "fs::write failed for slim snapshot")?; + + println!("Slim snapshot saved to {path:?}"); + Ok(()) +} + #[cfg(test)] mod test { use super::*; diff --git a/scraper/src/main.rs b/scraper/src/main.rs index 2c1bb4456..8b8ae650d 100644 --- a/scraper/src/main.rs +++ b/scraper/src/main.rs @@ -18,6 +18,10 @@ struct StorageCmd { /// The pallets to exclude from scraping. #[arg(long, num_args = 0..)] exclude: Vec, + /// Produce a slim snapshot by removing most user accounts from System.Account + /// and Tokens.Accounts, keeping only protocol, pool, and dev accounts. + #[arg(long)] + slim: bool, #[allow(missing_docs)] #[clap(flatten)] shared: SharedParams, @@ -101,33 +105,59 @@ fn main() { Command::SaveStorage(cmd) => { let path = cmd.shared.get_path(); let excluded_pallets = cmd.exclude; - - let snapshot_config = SnapshotConfig::new(path.clone()); + let slim = cmd.slim; let transport = Transport::Uri(cmd.shared.uri); - let online_config = OnlineConfig { - at: cmd.at, - state_snapshot: Some(snapshot_config), - pallets: cmd.pallet, - transport, - ..Default::default() - }; + if slim { + if !excluded_pallets.is_empty() { + println!("Warning: --exclude is ignored in --slim mode (slim filters by account, not by pallet)"); + } + // Slim mode: don't auto-save, capture externalities, filter in memory, write once + let online_config = OnlineConfig { + at: cmd.at, + state_snapshot: None, + pallets: cmd.pallet, + transport, + ..Default::default() + }; - let mode = Mode::Online(online_config); + let mode = Mode::Online(online_config); + let builder = Builder::::new().mode(mode); - let builder = Builder::::new().mode(mode); + let ext = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { builder.build().await.unwrap() }); + + scraper::save_slim_snapshot::(ext.inner_ext, ext.header, path.clone()) + .expect("Failed to save slim snapshot"); + } else { + let snapshot_config = SnapshotConfig::new(path.clone()); + + let online_config = OnlineConfig { + at: cmd.at, + state_snapshot: Some(snapshot_config), + pallets: cmd.pallet, + transport, + ..Default::default() + }; - tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap() - .block_on(async { builder.build().await.unwrap() }); + let mode = Mode::Online(online_config); + let builder = Builder::::new().mode(mode); - // Post-process snapshot to exclude specified pallets - if !excluded_pallets.is_empty() { - println!("Filtering out excluded pallets: {excluded_pallets:?}"); - scraper::filter_snapshot_by_excluded_pallets::(&path, &excluded_pallets) - .expect("Failed to filter snapshot by excluded pallets"); + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() + .block_on(async { builder.build().await.unwrap() }); + + // Post-process snapshot to exclude specified pallets + if !excluded_pallets.is_empty() { + println!("Filtering out excluded pallets: {excluded_pallets:?}"); + scraper::filter_snapshot_by_excluded_pallets::(&path, &excluded_pallets) + .expect("Failed to filter snapshot by excluded pallets"); + } } path