From a3d258927d109022ba20fb8d6bd63febf4d5a816 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 27 Feb 2018 22:19:40 +0800 Subject: [PATCH 1/4] Support partial trie fetching and move mutable to trie-memory This implements two things and fixes several tests: * Support partial trie. DatabaseHandle is changed to allow returning Option, and all op fns will return Error::Require(hash) if one of the hash is missing in the database. Client can use this information to further retrieve the required hash and try again. * trie now contains a mostly stable interface. Mutable structs interface may change a lot, and they're moved to trie-memory crate. --- Cargo.toml | 1 + block/Cargo.toml | 1 + block/src/block.rs | 2 +- block/src/lib.rs | 1 + block/tests/genesis.rs | 3 +- trie/memory/Cargo.toml | 20 ++++++++ trie/{ => memory}/src/cache.rs | 2 +- trie/{ => memory}/src/gc.rs | 17 ++++--- trie/memory/src/lib.rs | 46 ++++++++++++++++++ trie/{ => memory}/src/memory.rs | 18 +++---- trie/{ => memory}/src/mutable.rs | 2 +- trie/src/error.rs | 6 +++ trie/src/lib.rs | 82 +++++++++++++------------------- trie/src/merkle/mod.rs | 1 - trie/src/merkle/nibble.rs | 5 +- trie/src/merkle/node.rs | 7 ++- trie/src/ops/build.rs | 6 +-- trie/src/ops/delete.rs | 38 +++++++-------- trie/src/ops/get.rs | 22 ++++----- trie/src/ops/insert.rs | 24 +++++----- 20 files changed, 177 insertions(+), 127 deletions(-) create mode 100644 trie/memory/Cargo.toml rename trie/{ => memory}/src/cache.rs (97%) rename trie/{ => memory}/src/gc.rs (73%) create mode 100644 trie/memory/src/lib.rs rename trie/{ => memory}/src/memory.rs (92%) rename trie/{ => memory}/src/mutable.rs (100%) create mode 100644 trie/src/error.rs diff --git a/Cargo.toml b/Cargo.toml index d7b1c32..af692d8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,5 +7,6 @@ members = [ "./block-core", "./trie", "./trie/rocksdb", + "./trie/memory", "./bloom", ] diff --git a/block/Cargo.toml b/block/Cargo.toml index 1fa664a..fcb70ab 100644 --- a/block/Cargo.toml +++ b/block/Cargo.toml @@ -16,6 +16,7 @@ etcommon-bigint = { version = "0.2", path = "../bigint" } etcommon-rlp = { version = "0.2", path = "../rlp" } etcommon-bloom = { version = "0.2", path = "../bloom" } etcommon-trie = { version = "0.4", path = "../trie" } +etcommon-trie-memory = { version = "0.4", path = "../trie/memory" } blockchain = "0.2" secp256k1-plus = { version = "0.5", optional = true } diff --git a/block/src/block.rs b/block/src/block.rs index 3c74e87..ce761e9 100644 --- a/block/src/block.rs +++ b/block/src/block.rs @@ -1,7 +1,7 @@ use rlp::{self, Encodable, Decodable, RlpStream, DecoderError, UntrustedRlp}; use bigint::{Address, Gas, H256, U256, B256, H64, H2048}; use bloom::LogsBloom; -use trie::FixedMemoryTrieMut; +use trie_memory::FixedMemoryTrieMut; use sha3::{Keccak256, Digest}; use std::collections::HashMap; use super::{Header, Transaction, Receipt, SignaturePatch}; diff --git a/block/src/lib.rs b/block/src/lib.rs index af484c2..f8965e8 100644 --- a/block/src/lib.rs +++ b/block/src/lib.rs @@ -5,6 +5,7 @@ extern crate secp256k1; extern crate sha3; extern crate blockchain; extern crate trie; +extern crate trie_memory; extern crate block_core; #[cfg(test)] extern crate hexutil; #[cfg(test)] extern crate rand; diff --git a/block/tests/genesis.rs b/block/tests/genesis.rs index 37afafe..0d54a9d 100644 --- a/block/tests/genesis.rs +++ b/block/tests/genesis.rs @@ -5,13 +5,14 @@ extern crate serde; extern crate serde_json; extern crate bigint; extern crate trie; +extern crate trie_memory; extern crate block; extern crate sha3; extern crate rlp; extern crate rand; use bigint::{Address, H256, M256, U256}; -use trie::{FixedSecureMemoryTrieMut, MemoryTrieMut, TrieMut}; +use trie_memory::{FixedSecureMemoryTrieMut, MemoryTrieMut, TrieMut}; use block::Account; use sha3::{Digest, Keccak256}; use rand::Rng; diff --git a/trie/memory/Cargo.toml b/trie/memory/Cargo.toml new file mode 100644 index 0000000..de19be4 --- /dev/null +++ b/trie/memory/Cargo.toml @@ -0,0 +1,20 @@ +[package] +name = "etcommon-trie-memory" +version = "0.4.0" +license = "Apache-2.0" +authors = ["Wei Tang "] +description = "Rocksdb adaptor for trie." +repository = "https://github.com/ethereumproject/etcommon-rs" +keywords = ["ethereum"] + +[lib] +name = "trie_memory" + +[dependencies] +etcommon-trie = { version = "0.4", path = ".." } +etcommon-bigint = { version = "0.2", path = "../../bigint" } +etcommon-rlp = { version = "0.2", path = "../../rlp" } +sha3 = "0.6" + +[dev-dependencies] +etcommon-hexutil = { version = "0.2", path = "../../hexutil" } \ No newline at end of file diff --git a/trie/src/cache.rs b/trie/memory/src/cache.rs similarity index 97% rename from trie/src/cache.rs rename to trie/memory/src/cache.rs index 931c67a..6da424f 100644 --- a/trie/src/cache.rs +++ b/trie/memory/src/cache.rs @@ -1,4 +1,4 @@ -use super::merkle::MerkleNode; +use trie::merkle::MerkleNode; use bigint::H256; use rlp::Rlp; use std::collections::HashMap; diff --git a/trie/src/gc.rs b/trie/memory/src/gc.rs similarity index 73% rename from trie/src/gc.rs rename to trie/memory/src/gc.rs index c042f7e..282e787 100644 --- a/trie/src/gc.rs +++ b/trie/memory/src/gc.rs @@ -1,5 +1,6 @@ use bigint::H256; -use {Change, TrieMut, DatabaseHandle, get, insert, delete}; +use trie::{Change, DatabaseHandle, get, insert, delete}; +use TrieMut; pub trait ItemCounter { fn increase(&mut self, key: H256) -> usize; @@ -11,9 +12,11 @@ pub trait DatabaseMut { fn set(&mut self, key: H256, value: Option<&[u8]>); } -impl<'a, D: DatabaseMut> DatabaseHandle for &'a D { - fn get(&self, key: H256) -> &[u8] { - DatabaseMut::get(*self, key) +struct DatabaseMutHandle<'a, D: DatabaseMut + 'a>(&'a D); + +impl<'a, D: DatabaseMut> DatabaseHandle for DatabaseMutHandle<'a, D> { + fn get(&self, key: H256) -> Option<&[u8]> { + Some(DatabaseMut::get(self.0, key)) } } @@ -62,20 +65,20 @@ impl<'a, D: DatabaseMut> TrieMut for DatabaseTrieMut<'a, D> { } fn insert(&mut self, key: &[u8], value: &[u8]) { - let (new_root, change) = insert(self.root, &self.database, key, value); + let (new_root, change) = insert(self.root, &DatabaseMutHandle(self.database), key, value).unwrap(); self.change.merge(&change); self.root = new_root; } fn delete(&mut self, key: &[u8]) { - let (new_root, change) = delete(self.root, &self.database, key); + let (new_root, change) = delete(self.root, &DatabaseMutHandle(self.database), key).unwrap(); self.change.merge(&change); self.root = new_root; } fn get(&self, key: &[u8]) -> Option> { - get(self.root, &self.database, key).map(|v| v.into()) + get(self.root, &DatabaseMutHandle(self.database), key).unwrap().map(|v| v.into()) } } diff --git a/trie/memory/src/lib.rs b/trie/memory/src/lib.rs new file mode 100644 index 0000000..9a9240d --- /dev/null +++ b/trie/memory/src/lib.rs @@ -0,0 +1,46 @@ +extern crate bigint; +#[macro_use] +extern crate trie; +extern crate rlp; +extern crate sha3; +#[cfg(test)] extern crate hexutil; + +pub mod gc; +mod memory; +mod mutable; +mod cache; + +use cache::Cache; +use bigint::H256; +use trie::DatabaseHandle; + +pub use memory::*; +pub use mutable::*; + +pub trait CachedDatabaseHandle { + fn get(&self, key: H256) -> Vec; +} + +pub struct CachedHandle { + db: D, + cache: Cache, +} + +impl CachedHandle { + pub fn new(db: D) -> Self { + Self { + db, + cache: Cache::new(), + } + } +} + +impl DatabaseHandle for CachedHandle { + fn get(&self, key: H256) -> Option<&[u8]> { + if !self.cache.contains_key(key) { + Some(self.cache.insert(key, self.db.get(key))) + } else { + Some(self.cache.get(key).unwrap()) + } + } +} diff --git a/trie/src/memory.rs b/trie/memory/src/memory.rs similarity index 92% rename from trie/src/memory.rs rename to trie/memory/src/memory.rs index a76e324..c712dfb 100644 --- a/trie/src/memory.rs +++ b/trie/memory/src/memory.rs @@ -1,16 +1,10 @@ use bigint::H256; -use {DatabaseHandle, Change, insert, delete, build, get, - TrieMut, FixedTrieMut, FixedSecureTrieMut, +use trie::{DatabaseHandle, Change, insert, delete, build, get}; +use {TrieMut, FixedTrieMut, FixedSecureTrieMut, AnyTrieMut, AnySecureTrieMut, SecureTrieMut}; use std::collections::HashMap; -impl<'a> DatabaseHandle for &'a HashMap> { - fn get(&self, hash: H256) -> &[u8] { - HashMap::get(self, &hash).unwrap() - } -} - /// A memory-backed trie. #[derive(Clone, Debug)] pub struct MemoryTrieMut { @@ -54,21 +48,21 @@ impl TrieMut for MemoryTrieMut { } fn insert(&mut self, key: &[u8], value: &[u8]) { - let (new_root, change) = insert(self.root, &&self.database, key, value); + let (new_root, change) = insert(self.root, &&self.database, key, value).unwrap(); self.apply_change(change); self.root = new_root; } fn delete(&mut self, key: &[u8]) { - let (new_root, change) = delete(self.root, &&self.database, key); + let (new_root, change) = delete(self.root, &&self.database, key).unwrap(); self.apply_change(change); self.root = new_root; } fn get(&self, key: &[u8]) -> Option> { - get(self.root, &&self.database, key).map(|v| v.into()) + get(self.root, &&self.database, key).unwrap().map(|v| v.into()) } } @@ -99,7 +93,7 @@ impl MemoryTrieMut { mod tests { use {TrieMut}; use super::MemoryTrieMut; - use merkle::MerkleNode; + use trie::merkle::MerkleNode; use rlp::Rlp; use std::collections::HashMap; diff --git a/trie/src/mutable.rs b/trie/memory/src/mutable.rs similarity index 100% rename from trie/src/mutable.rs rename to trie/memory/src/mutable.rs index 19d9752..c76fa7e 100644 --- a/trie/src/mutable.rs +++ b/trie/memory/src/mutable.rs @@ -1,6 +1,6 @@ use bigint::H256; -use rlp::{self, Rlp}; use sha3::{Digest, Keccak256}; +use rlp::{self, Rlp}; use std::marker::PhantomData; diff --git a/trie/src/error.rs b/trie/src/error.rs new file mode 100644 index 0000000..c1f1903 --- /dev/null +++ b/trie/src/error.rs @@ -0,0 +1,6 @@ +use bigint::H256; + +#[derive(Debug)] +pub enum Error { + Require(H256), +} diff --git a/trie/src/lib.rs b/trie/src/lib.rs index b4c61f0..db4ce2c 100644 --- a/trie/src/lib.rs +++ b/trie/src/lib.rs @@ -1,5 +1,10 @@ //! Merkle trie implementation for Ethereum. +#![deny(unused_import_braces, unused_imports, + unused_comparisons, unused_must_use, + unused_variables, non_shorthand_field_patterns, + unreachable_code)] + extern crate bigint; extern crate rlp; extern crate sha3; @@ -9,8 +14,7 @@ use bigint::H256; use rlp::Rlp; use sha3::{Digest, Keccak256}; use std::collections::{HashMap, HashSet}; -use merkle::{MerkleValue, MerkleNode}; -use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; +use merkle::{MerkleValue, MerkleNode, nibble}; macro_rules! empty_nodes { () => ( @@ -25,6 +29,7 @@ macro_rules! empty_nodes { ) } +#[macro_export] macro_rules! empty_trie_hash { () => { { @@ -36,52 +41,31 @@ macro_rules! empty_trie_hash { } pub mod merkle; -pub mod gc; mod ops; -mod memory; -mod mutable; -mod cache; +mod error; use ops::{insert, delete, build, get}; -use cache::Cache; - -pub use memory::*; -pub use mutable::*; +pub use error::Error; -pub trait CachedDatabaseHandle { - fn get(&self, key: H256) -> Vec; -} - -pub struct CachedHandle { - db: D, - cache: Cache, -} +/// An immutable database handle. +pub trait DatabaseHandle { + /// Get a raw value from the database. + fn get<'a>(&'a self, key: H256) -> Option<&'a [u8]>; -impl CachedHandle { - pub fn new(db: D) -> Self { - Self { - db, - cache: Cache::new(), + fn get_with_error<'a>(&'a self, key: H256) -> Result<&'a [u8], Error> { + match self.get(key) { + Some(value) => Ok(value), + None => Err(Error::Require(key)), } } } -impl DatabaseHandle for CachedHandle { - fn get(&self, key: H256) -> &[u8] { - if !self.cache.contains_key(key) { - self.cache.insert(key, self.db.get(key)) - } else { - self.cache.get(key).unwrap() - } +impl<'a> DatabaseHandle for &'a HashMap> { + fn get(&self, hash: H256) -> Option<&[u8]> { + HashMap::get(self, &hash).map(|v| v.as_ref()) } } -/// An immutable database handle. -pub trait DatabaseHandle { - /// Get a raw value from the database. - fn get<'a>(&'a self, key: H256) -> &'a [u8]; -} - /// Change for a merkle trie operation. pub struct Change { /// Additions to the database. @@ -164,22 +148,22 @@ pub fn empty_trie_hash() -> H256 { /// Insert to a merkle trie. Return the new root hash and the changes. pub fn insert( root: H256, database: &D, key: &[u8], value: &[u8] -) -> (H256, Change) { +) -> Result<(H256, Change), Error> { let mut change = Change::default(); let nibble = nibble::from_key(key); let (new, subchange) = if root == empty_trie_hash!() { insert::insert_by_empty(nibble, value) } else { - let old = MerkleNode::decode(&Rlp::new(database.get(root))); + let old = MerkleNode::decode(&Rlp::new(database.get_with_error(root)?)); change.remove_raw(root); - insert::insert_by_node(old, nibble, value, database) + insert::insert_by_node(old, nibble, value, database)? }; change.merge(&subchange); change.add_node(&new); let hash = H256::from(Keccak256::digest(&rlp::encode(&new).to_vec()).as_slice()); - (hash, change) + Ok((hash, change)) } /// Insert to an empty merkle trie. Return the new root hash and the @@ -202,16 +186,16 @@ pub fn insert_empty( /// changes. pub fn delete( root: H256, database: &D, key: &[u8] -) -> (H256, Change) { +) -> Result<(H256, Change), Error> { let mut change = Change::default(); let nibble = nibble::from_key(key); let (new, subchange) = if root == empty_trie_hash!() { - return (root, change) + return Ok((root, change)) } else { - let old = MerkleNode::decode(&Rlp::new(database.get(root))); + let old = MerkleNode::decode(&Rlp::new(database.get_with_error(root)?)); change.remove_raw(root); - delete::delete_by_node(old, nibble, database) + delete::delete_by_node(old, nibble, database)? }; change.merge(&subchange); @@ -220,10 +204,10 @@ pub fn delete( change.add_node(&new); let hash = H256::from(Keccak256::digest(&rlp::encode(&new).to_vec()).as_slice()); - (hash, change) + Ok((hash, change)) }, None => { - (empty_trie_hash!(), change) + Ok((empty_trie_hash!(), change)) }, } } @@ -253,12 +237,12 @@ pub fn build(map: &HashMap, Vec>) -> (H256, Change) { /// Get a value given the root hash and the database. pub fn get<'a, 'b, D: DatabaseHandle>( root: H256, database: &'a D, key: &'b [u8] -) -> Option<&'a [u8]> { +) -> Result, Error> { if root == empty_trie_hash!() { - None + Ok(None) } else { let nibble = nibble::from_key(key); - let node = MerkleNode::decode(&Rlp::new(database.get(root))); + let node = MerkleNode::decode(&Rlp::new(database.get_with_error(root)?)); get::get_by_node(node, nibble, database) } } diff --git a/trie/src/merkle/mod.rs b/trie/src/merkle/mod.rs index b23201e..e2a8c26 100644 --- a/trie/src/merkle/mod.rs +++ b/trie/src/merkle/mod.rs @@ -3,5 +3,4 @@ pub mod nibble; mod node; -use self::nibble::{Nibble, NibbleVec, NibbleSlice, NibbleType}; pub use self::node::{MerkleNode, MerkleValue}; diff --git a/trie/src/merkle/nibble.rs b/trie/src/merkle/nibble.rs index 49f5e09..5f54218 100644 --- a/trie/src/merkle/nibble.rs +++ b/trie/src/merkle/nibble.rs @@ -1,10 +1,7 @@ //! Merkle nibble types. -use rlp::{RlpStream, Encodable, Decodable, Rlp, Prototype}; -use std::ops::Deref; +use rlp::{RlpStream, Rlp}; use std::cmp::min; -use std::hash::{Hash, Hasher}; -use std::fmt::{self, Debug, Formatter}; /// Represents a nibble. A 16-variant value. #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] diff --git a/trie/src/merkle/node.rs b/trie/src/merkle/node.rs index e1162e8..5b49249 100644 --- a/trie/src/merkle/node.rs +++ b/trie/src/merkle/node.rs @@ -1,6 +1,6 @@ -use super::nibble::{self, Nibble, NibbleVec, NibbleSlice, NibbleType}; +use super::nibble::{self, NibbleVec, NibbleType}; -use rlp::{self, RlpStream, Encodable, Decodable, Rlp, Prototype}; +use rlp::{self, RlpStream, Encodable, Rlp, Prototype}; use bigint::H256; use std::borrow::Borrow; @@ -159,8 +159,7 @@ impl<'a> Encodable for MerkleValue<'a> { mod tests { use hexutil::read_hex; use rlp::{self, Rlp}; - use sha3::{Digest, Keccak256}; - use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; + use merkle::nibble; use super::MerkleNode; #[test] diff --git a/trie/src/ops/build.rs b/trie/src/ops/build.rs index 4109506..9d4c43d 100644 --- a/trie/src/ops/build.rs +++ b/trie/src/ops/build.rs @@ -1,11 +1,9 @@ use merkle::{MerkleValue, MerkleNode}; -use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; -use {Change, DatabaseHandle}; +use merkle::nibble::{self, NibbleVec, Nibble}; +use Change; use std::collections::HashMap; -use rlp::{self, Rlp}; - fn make_submap<'a, 'b: 'a, T: Iterator>( common_len: usize, map: T ) -> HashMap { diff --git a/trie/src/ops/delete.rs b/trie/src/ops/delete.rs index 1d13259..ab2e82d 100644 --- a/trie/src/ops/delete.rs +++ b/trie/src/ops/delete.rs @@ -1,25 +1,25 @@ use merkle::{MerkleValue, MerkleNode}; -use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; -use {Change, DatabaseHandle}; +use merkle::nibble::{NibbleVec, Nibble}; +use {Change, DatabaseHandle, Error}; -use rlp::{self, Rlp}; +use rlp::Rlp; fn find_and_remove_child<'a, D: DatabaseHandle>( merkle: MerkleValue<'a>, database: &'a D -) -> (MerkleNode<'a>, Change) { +) -> Result<(MerkleNode<'a>, Change), Error> { let mut change = Change::default(); let node = match merkle { MerkleValue::Empty => panic!(), MerkleValue::Full(ref sub_node) => sub_node.as_ref().clone(), MerkleValue::Hash(h) => { - let sub_node = MerkleNode::decode(&Rlp::new(database.get(h))); + let sub_node = MerkleNode::decode(&Rlp::new(database.get_with_error(h)?)); change.remove_raw(h); sub_node }, }; - (node, change) + Ok((node, change)) } fn collapse_extension<'a>( @@ -59,7 +59,7 @@ fn nonempty_node_count<'a, 'b>( fn collapse_branch<'a, D: DatabaseHandle>( node_nodes: [MerkleValue<'a>; 16], node_additional: Option<&'a [u8]>, database: &'a D -) -> (MerkleNode<'a>, Change) { +) -> Result<(MerkleNode<'a>, Change), Error> { let mut change = Change::default(); let value_count = nonempty_node_count(&node_nodes, &node_additional); @@ -74,7 +74,7 @@ fn collapse_branch<'a, D: DatabaseHandle>( .map(|(i, v)| (i, v.clone())).unwrap(); let subnibble: Nibble = subindex.into(); - let (subnode, subchange) = find_and_remove_child(subvalue, database); + let (subnode, subchange) = find_and_remove_child(subvalue, database)?; change.merge(&subchange); match subnode { @@ -98,12 +98,12 @@ fn collapse_branch<'a, D: DatabaseHandle>( MerkleNode::Branch(node_nodes, node_additional), }; - (node, change) + Ok((node, change)) } pub fn delete_by_child<'a, D: DatabaseHandle>( merkle: MerkleValue<'a>, nibble: NibbleVec, database: &'a D -) -> (Option>, Change) { +) -> Result<(Option>, Change), Error> { let mut change = Change::default(); let new = match merkle { @@ -112,7 +112,7 @@ pub fn delete_by_child<'a, D: DatabaseHandle>( }, MerkleValue::Full(ref sub_node) => { let (new_node, subchange) = delete_by_node( - sub_node.as_ref().clone(), nibble, database); + sub_node.as_ref().clone(), nibble, database)?; change.merge(&subchange); match new_node { Some(new_node) => Some(new_node), @@ -120,10 +120,10 @@ pub fn delete_by_child<'a, D: DatabaseHandle>( } }, MerkleValue::Hash(h) => { - let sub_node = MerkleNode::decode(&Rlp::new(database.get(h))); + let sub_node = MerkleNode::decode(&Rlp::new(database.get_with_error(h)?)); change.remove_raw(h); let (new_node, subchange) = delete_by_node( - sub_node, nibble, database); + sub_node, nibble, database)?; change.merge(&subchange); match new_node { Some(new_node) => Some(new_node), @@ -132,12 +132,12 @@ pub fn delete_by_child<'a, D: DatabaseHandle>( }, }; - (new, change) + Ok((new, change)) } pub fn delete_by_node<'a, D: DatabaseHandle>( node: MerkleNode<'a>, nibble: NibbleVec, database: &'a D -) -> (Option>, Change) { +) -> Result<(Option>, Change), Error> { let mut change = Change::default(); let new = match node { @@ -152,7 +152,7 @@ pub fn delete_by_node<'a, D: DatabaseHandle>( if nibble.starts_with(&node_nibble) { let (subnode, subchange) = delete_by_child( node_value, nibble[node_nibble.len()..].into(), - database); + database)?; change.merge(&subchange); match subnode { @@ -178,7 +178,7 @@ pub fn delete_by_node<'a, D: DatabaseHandle>( let ni: usize = nibble[0].into(); let (new_subnode, subchange) = delete_by_child( node_nodes[ni].clone(), nibble[1..].into(), - database); + database)?; change.merge(&subchange); match new_subnode { @@ -198,7 +198,7 @@ pub fn delete_by_node<'a, D: DatabaseHandle>( if needs_collapse { let value_count = nonempty_node_count(&node_nodes, &node_additional); if value_count > 0 { - let (new, subchange) = collapse_branch(node_nodes, node_additional, database); + let (new, subchange) = collapse_branch(node_nodes, node_additional, database)?; change.merge(&subchange); Some(new) @@ -211,5 +211,5 @@ pub fn delete_by_node<'a, D: DatabaseHandle>( }, }; - (new, change) + Ok((new, change)) } diff --git a/trie/src/ops/get.rs b/trie/src/ops/get.rs index b3b9b62..fbf7b07 100644 --- a/trie/src/ops/get.rs +++ b/trie/src/ops/get.rs @@ -1,19 +1,19 @@ use merkle::{MerkleValue, MerkleNode}; -use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; -use {Change, DatabaseHandle}; +use merkle::nibble::NibbleVec; +use {DatabaseHandle, Error}; -use rlp::{self, Rlp}; +use rlp::Rlp; pub fn get_by_value<'a, D: DatabaseHandle>( merkle: MerkleValue<'a>, nibble: NibbleVec, database: &'a D -) -> Option<&'a [u8]> { +) -> Result, Error> { match merkle { - MerkleValue::Empty => None, + MerkleValue::Empty => Ok(None), MerkleValue::Full(subnode) => { get_by_node(subnode.as_ref().clone(), nibble, database) }, MerkleValue::Hash(h) => { - let subnode = MerkleNode::decode(&Rlp::new(database.get(h))); + let subnode = MerkleNode::decode(&Rlp::new(database.get_with_error(h)?)); get_by_node(subnode, nibble, database) }, } @@ -21,25 +21,25 @@ pub fn get_by_value<'a, D: DatabaseHandle>( pub fn get_by_node<'a, D: DatabaseHandle>( node: MerkleNode<'a>, nibble: NibbleVec, database: &'a D -) -> Option<&'a [u8]> { +) -> Result, Error> { match node { MerkleNode::Leaf(node_nibble, node_value) => { if node_nibble == nibble { - Some(node_value) + Ok(Some(node_value)) } else { - None + Ok(None) } }, MerkleNode::Extension(node_nibble, node_value) => { if nibble.starts_with(&node_nibble) { get_by_value(node_value, nibble[node_nibble.len()..].into(), database) } else { - None + Ok(None) } }, MerkleNode::Branch(node_nodes, node_additional) => { if nibble.len() == 0 { - node_additional + Ok(node_additional) } else { let ni: usize = nibble[0].into(); get_by_value(node_nodes[ni].clone(), nibble[1..].into(), database) diff --git a/trie/src/ops/insert.rs b/trie/src/ops/insert.rs index 2380f6f..fabba8a 100644 --- a/trie/src/ops/insert.rs +++ b/trie/src/ops/insert.rs @@ -1,8 +1,8 @@ use merkle::{MerkleValue, MerkleNode}; -use merkle::nibble::{self, NibbleVec, NibbleSlice, Nibble}; -use {Change, DatabaseHandle}; +use merkle::nibble::{self, NibbleVec}; +use {Change, DatabaseHandle, Error}; -use rlp::{self, Rlp}; +use rlp::Rlp; fn value_and_leaf_branch<'a>( anibble: NibbleVec, avalue: MerkleValue<'a>, bnibble: NibbleVec, bvalue: &'a [u8] @@ -71,7 +71,7 @@ fn two_leaf_branch<'a>( pub fn insert_by_value<'a, D: DatabaseHandle>( merkle: MerkleValue<'a>, nibble: NibbleVec, value: &'a [u8], database: &'a D -) -> (MerkleValue<'a>, Change) { +) -> Result<(MerkleValue<'a>, Change), Error> { let mut change = Change::default(); let new = match merkle { @@ -80,26 +80,26 @@ pub fn insert_by_value<'a, D: DatabaseHandle>( }, MerkleValue::Full(ref sub_node) => { let (new_node, subchange) = insert_by_node( - sub_node.as_ref().clone(), nibble, value, database); + sub_node.as_ref().clone(), nibble, value, database)?; change.merge(&subchange); change.add_value(&new_node) }, MerkleValue::Hash(h) => { - let sub_node = MerkleNode::decode(&Rlp::new(database.get(h))); + let sub_node = MerkleNode::decode(&Rlp::new(database.get_with_error(h)?)); change.remove_raw(h); let (new_node, subchange) = insert_by_node( - sub_node, nibble, value, database); + sub_node, nibble, value, database)?; change.merge(&subchange); change.add_value(&new_node) }, }; - (new, change) + Ok((new, change)) } pub fn insert_by_node<'a, D: DatabaseHandle>( node: MerkleNode<'a>, nibble: NibbleVec, value: &'a [u8], database: &'a D -) -> (MerkleNode<'a>, Change) { +) -> Result<(MerkleNode<'a>, Change), Error> { let mut change = Change::default(); let new = match node { @@ -125,7 +125,7 @@ pub fn insert_by_node<'a, D: DatabaseHandle>( let (subvalue, subchange) = insert_by_value( node_value.clone(), nibble[node_nibble.len()..].into(), - value, database); + value, database)?; change.merge(&subchange); MerkleNode::Extension(node_nibble.clone(), subvalue) @@ -153,7 +153,7 @@ pub fn insert_by_node<'a, D: DatabaseHandle>( let (new, subchange) = insert_by_value( prev, nibble[1..].into(), - value, database); + value, database)?; change.merge(&subchange); nodes[ni] = new; @@ -162,7 +162,7 @@ pub fn insert_by_node<'a, D: DatabaseHandle>( }, }; - (new, change) + Ok((new, change)) } pub fn insert_by_empty<'a>( From e5094adaf498ddcb4e0a713f2641da98ac3b8bd1 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 27 Feb 2018 22:24:36 +0800 Subject: [PATCH 2/4] Fix trie-rocksdb change due to partial trie interface --- trie/rocksdb/Cargo.toml | 1 + trie/rocksdb/src/lib.rs | 14 ++++++++------ 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/trie/rocksdb/Cargo.toml b/trie/rocksdb/Cargo.toml index 6e89436..ab39cff 100644 --- a/trie/rocksdb/Cargo.toml +++ b/trie/rocksdb/Cargo.toml @@ -13,4 +13,5 @@ name = "trie_rocksdb" [dependencies] etcommon-trie = { version = "0.4", path = ".." } etcommon-bigint = { version = "0.2", path = "../../bigint" } +etcommon-trie-memory = { version = "0.4", path = "../memory" } rocksdb = { git = "https://github.com/paritytech/rust-rocksdb" } \ No newline at end of file diff --git a/trie/rocksdb/src/lib.rs b/trie/rocksdb/src/lib.rs index 231ab09..58a6d95 100644 --- a/trie/rocksdb/src/lib.rs +++ b/trie/rocksdb/src/lib.rs @@ -1,9 +1,11 @@ extern crate trie; +extern crate trie_memory; extern crate bigint; extern crate rocksdb; use bigint::H256; -use trie::{CachedDatabaseHandle, CachedHandle, Change, DatabaseHandle, TrieMut, get, insert, delete}; +use trie::{Change, DatabaseHandle, get, insert, delete}; +use trie_memory::{CachedDatabaseHandle, CachedHandle, TrieMut}; use rocksdb::{DB, Writable}; pub struct RocksDatabaseHandle<'a>(&'a DB); @@ -32,9 +34,9 @@ pub struct RocksMemoryTrieMut<'a> { } impl<'a, 'b> DatabaseHandle for &'b RocksMemoryTrieMut<'a> { - fn get(&self, key: H256) -> &[u8] { + fn get(&self, key: H256) -> Option<&[u8]> { if self.change.adds.contains_key(&key) { - self.change.adds.get(&key).unwrap() + self.change.adds.get(&key).map(|v| v.as_ref()) } else { self.handle.get(key) } @@ -49,7 +51,7 @@ impl<'a> TrieMut for RocksMemoryTrieMut<'a> { fn insert(&mut self, key: &[u8], value: &[u8]) { self.clear_cache(); - let (new_root, change) = insert(self.root, &&*self, key, value); + let (new_root, change) = insert(self.root, &&*self, key, value).unwrap(); self.change.merge(&change); self.root = new_root; @@ -58,14 +60,14 @@ impl<'a> TrieMut for RocksMemoryTrieMut<'a> { fn delete(&mut self, key: &[u8]) { self.clear_cache(); - let (new_root, change) = delete(self.root, &&*self, key); + let (new_root, change) = delete(self.root, &&*self, key).unwrap(); self.change.merge(&change); self.root = new_root; } fn get(&self, key: &[u8]) -> Option> { - get(self.root, &self, key).map(|v| v.into()) + get(self.root, &self, key).unwrap().map(|v| v.into()) } } From b6cd747436c8b19f563d46c4b5a82c2de0d65438 Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Tue, 27 Feb 2018 22:39:30 +0800 Subject: [PATCH 3/4] Make EMPTY_TRIE_HASH a constant --- trie/memory/src/memory.rs | 7 ++++--- trie/src/lib.rs | 29 +++++++++-------------------- 2 files changed, 13 insertions(+), 23 deletions(-) diff --git a/trie/memory/src/memory.rs b/trie/memory/src/memory.rs index c712dfb..344baf4 100644 --- a/trie/memory/src/memory.rs +++ b/trie/memory/src/memory.rs @@ -1,5 +1,5 @@ use bigint::H256; -use trie::{DatabaseHandle, Change, insert, delete, build, get}; +use trie::{DatabaseHandle, Change, insert, delete, build, get, EMPTY_TRIE_HASH}; use {TrieMut, FixedTrieMut, FixedSecureTrieMut, AnyTrieMut, AnySecureTrieMut, SecureTrieMut}; @@ -31,7 +31,7 @@ impl Default for MemoryTrieMut { fn default() -> Self { Self { database: HashMap::new(), - root: empty_trie_hash!(), + root: EMPTY_TRIE_HASH, } } } @@ -93,6 +93,7 @@ impl MemoryTrieMut { mod tests { use {TrieMut}; use super::MemoryTrieMut; + use trie::EMPTY_TRIE_HASH; use trie::merkle::MerkleNode; use rlp::Rlp; @@ -134,7 +135,7 @@ mod tests { } assert!(mtrie.database.len() == 0); - assert!(mtrie.root == empty_trie_hash!()); + assert!(mtrie.root == EMPTY_TRIE_HASH); } #[test] diff --git a/trie/src/lib.rs b/trie/src/lib.rs index db4ce2c..3174953 100644 --- a/trie/src/lib.rs +++ b/trie/src/lib.rs @@ -29,16 +29,10 @@ macro_rules! empty_nodes { ) } -#[macro_export] -macro_rules! empty_trie_hash { - () => { - { - use std::str::FromStr; - - H256::from_str("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421").unwrap() - } - } -} +pub const EMPTY_TRIE_HASH: H256 = H256([0x56, 0xe8, 0x1f, 0x17, 0x1b, 0xcc, 0x55, 0xa6, + 0xff, 0x83, 0x45, 0xe6, 0x92, 0xc0, 0xf8, 0x6e, + 0x5b, 0x48, 0xe0, 0x1b, 0x99, 0x6c, 0xad, 0xc0, + 0x01, 0x62, 0x2f, 0xb5, 0xe3, 0x63, 0xb4, 0x21]); pub mod merkle; mod ops; @@ -140,11 +134,6 @@ impl Change { } } -/// Get the empty trie hash for merkle trie. -pub fn empty_trie_hash() -> H256 { - empty_trie_hash!() -} - /// Insert to a merkle trie. Return the new root hash and the changes. pub fn insert( root: H256, database: &D, key: &[u8], value: &[u8] @@ -152,7 +141,7 @@ pub fn insert( let mut change = Change::default(); let nibble = nibble::from_key(key); - let (new, subchange) = if root == empty_trie_hash!() { + let (new, subchange) = if root == EMPTY_TRIE_HASH { insert::insert_by_empty(nibble, value) } else { let old = MerkleNode::decode(&Rlp::new(database.get_with_error(root)?)); @@ -190,7 +179,7 @@ pub fn delete( let mut change = Change::default(); let nibble = nibble::from_key(key); - let (new, subchange) = if root == empty_trie_hash!() { + let (new, subchange) = if root == EMPTY_TRIE_HASH { return Ok((root, change)) } else { let old = MerkleNode::decode(&Rlp::new(database.get_with_error(root)?)); @@ -207,7 +196,7 @@ pub fn delete( Ok((hash, change)) }, None => { - Ok((empty_trie_hash!(), change)) + Ok((EMPTY_TRIE_HASH, change)) }, } } @@ -218,7 +207,7 @@ pub fn build(map: &HashMap, Vec>) -> (H256, Change) { let mut change = Change::default(); if map.len() == 0 { - return (empty_trie_hash!(), change); + return (EMPTY_TRIE_HASH, change); } let mut node_map = HashMap::new(); @@ -238,7 +227,7 @@ pub fn build(map: &HashMap, Vec>) -> (H256, Change) { pub fn get<'a, 'b, D: DatabaseHandle>( root: H256, database: &'a D, key: &'b [u8] ) -> Result, Error> { - if root == empty_trie_hash!() { + if root == EMPTY_TRIE_HASH { Ok(None) } else { let nibble = nibble::from_key(key); From f229bdd7680bdefbdfc43860e59fb5144ff6f87a Mon Sep 17 00:00:00 2001 From: Wei Tang Date: Wed, 28 Feb 2018 23:30:12 +0800 Subject: [PATCH 4/4] Change Cache to use singly linked list instead of Vec Fix #59 --- trie/memory/src/cache.rs | 87 +++++++++++++++++++++++++++++++++++----- 1 file changed, 77 insertions(+), 10 deletions(-) diff --git a/trie/memory/src/cache.rs b/trie/memory/src/cache.rs index 6da424f..ced556b 100644 --- a/trie/memory/src/cache.rs +++ b/trie/memory/src/cache.rs @@ -1,35 +1,102 @@ use trie::merkle::MerkleNode; use bigint::H256; use rlp::Rlp; +use std::ptr; use std::collections::HashMap; -use std::cell::{RefCell, UnsafeCell}; +use std::cell::{RefCell, Cell}; pub struct Cache { - cache: UnsafeCell>>, map: RefCell>, + + cache_head: Cell<*mut Node>, + cache_len: Cell, +} + +struct Node { + next: *mut Node, + value: Vec, +} + +impl Drop for Cache { + fn drop(&mut self) { + if self.cache_head.get().is_null() { + return; + } + + let mut all_ptrs = Vec::new(); + all_ptrs.push(self.cache_head.get()); + + let mut cur_node = unsafe { &*self.cache_head.get() }; + + loop { + if cur_node.next.is_null() { + break; + } + + all_ptrs.push(cur_node.next); + cur_node = unsafe { &*cur_node.next }; + } + + for ptr in all_ptrs { + unsafe { Box::from_raw(ptr); } + } + } } impl Cache { + fn at<'a>(&'a self, index: usize) -> Option<&'a [u8]> { + if self.cache_head.get().is_null() { + return None; + } + + let mut cur_index = self.cache_len.get() - 1; + let mut cur_node = unsafe { &*self.cache_head.get() }; + + loop { + if cur_index < index { + return None; + } + + if cur_index == index { + return Some(cur_node.value.as_ref()); + } + + if cur_node.next.is_null() { + return None; + } + + cur_index -= 1; + cur_node = unsafe { &*cur_node.next }; + } + } + pub fn new() -> Cache { Cache { - cache: UnsafeCell::new(Vec::new()), - map: RefCell::new(HashMap::new()) + map: RefCell::new(HashMap::new()), + + cache_head: Cell::new(ptr::null_mut()), + cache_len: Cell::new(0), } } pub fn insert<'a>(&'a self, key: H256, value: Vec) -> &'a [u8] { - let cache = unsafe { &mut *self.cache.get() }; - let index = cache.len(); + let index = self.cache_len.get(); + self.cache_len.set(self.cache_len.get() + 1); + self.map.borrow_mut().insert(key, index); - cache.push(value); - &cache[index] + let node_ptr = Box::into_raw(Box::new(Node { + next: self.cache_head.get(), + value: value, + })); + self.cache_head.set(node_ptr); + + self.at(index).unwrap() } pub fn get<'a>(&'a self, key: H256) -> Option<&'a [u8]> { - let cache = unsafe { &mut *self.cache.get() }; let mut map = self.map.borrow_mut(); match map.get(&key) { - Some(index) => Some(&cache[*index]), + Some(index) => Some(self.at(*index).unwrap()), None => None, } }