From df3b3b558a3bbbc8eff75b3b8ebbb3bcd1d5ac6e Mon Sep 17 00:00:00 2001 From: Owanikin Date: Tue, 3 Jun 2025 15:52:58 +0100 Subject: [PATCH 1/5] Replace RocksDB with Redb as backing for EVM database --- Cargo.lock | 83 ++-------- bin/portal-bridge/src/bridge/state.rs | 14 +- bin/trin-execution/Cargo.toml | 3 +- bin/trin-execution/src/evm/block_executor.rs | 63 ++++---- bin/trin-execution/src/execution.rs | 4 +- bin/trin-execution/src/storage/account_db.rs | 101 ++++++------ bin/trin-execution/src/storage/error.rs | 35 ++++- bin/trin-execution/src/storage/evm_db.rs | 146 ++++++++++++------ .../src/storage/execution_position.rs | 33 ++-- bin/trin-execution/src/storage/trie_db.rs | 41 +++-- bin/trin-execution/src/storage/utils.rs | 29 ++-- .../src/subcommands/e2ss/export.rs | 19 ++- .../src/subcommands/e2ss/import.rs | 56 +++++-- bin/trin-execution/src/trie_walker/db.rs | 8 +- bin/trin-execution/src/trie_walker/mod.rs | 6 +- 15 files changed, 359 insertions(+), 282 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 0870e10f5..609c83056 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1376,7 +1376,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "61b1d86e7705efe1be1b569bab41d4fa1e14e220b60a160f78de2db687add079" dependencies = [ - "bindgen 0.69.5", + "bindgen", "cc", "cmake", "dunce", @@ -1808,29 +1808,6 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" -[[package]] -name = "bindgen" -version = "0.69.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271383c67ccabffb7381723dea0672a673f292304fcb45c01cc648c7a8d58088" -dependencies = [ - "bitflags 2.9.1", - "cexpr", - "clang-sys", - "itertools 0.12.1", - "lazy_static", - "lazycell", - "log", - "prettyplease", - "proc-macro2", - "quote", - "regex", - "rustc-hash 1.1.0", - "shlex", - "syn 2.0.104", - "which", -] - [[package]] name = "bindgen" version = "0.71.1" @@ -2012,16 +1989,6 @@ dependencies = [ "either", ] -[[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" -dependencies = [ - "cc", - "pkg-config", -] - [[package]] name = "c-kzg" version = "2.1.1" @@ -4764,21 +4731,6 @@ dependencies = [ "libc", ] -[[package]] -name = "librocksdb-sys" -version = "0.17.1+9.9.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b7869a512ae9982f4d46ba482c2a304f1efd80c6412a3d4bf57bb79a619679f" -dependencies = [ - "bindgen 0.69.5", - "bzip2-sys", - "cc", - "libc", - "libz-sys", - "lz4-sys", - "zstd-sys", -] - [[package]] name = "libsecp256k1" version = "0.7.2" @@ -4948,16 +4900,6 @@ dependencies = [ "hashbrown 0.15.4", ] -[[package]] -name = "lz4-sys" -version = "1.11.1+lz4-1.10.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6" -dependencies = [ - "cc", - "libc", -] - [[package]] name = "mac" version = "0.1.1" @@ -6114,6 +6056,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d3edd4d5d42c92f0a659926464d4cce56b562761267ecf0f469d85b7de384175" +[[package]] +name = "redb" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34bc6763177194266fc3773e2b2bb3693f7b02fdf461e285aa33202e3164b74e" +dependencies = [ + "libc", +] + [[package]] name = "redox_syscall" version = "0.2.16" @@ -6585,16 +6536,6 @@ dependencies = [ "rustc-hex", ] -[[package]] -name = "rocksdb" -version = "0.23.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26ec73b20525cb235bad420f911473b69f9fe27cc856c5461bccd7e4af037f43" -dependencies = [ - "libc", - "librocksdb-sys", -] - [[package]] name = "route-recognizer" version = "0.3.1" @@ -8422,13 +8363,14 @@ dependencies = [ "prometheus_exporter", "rand 0.9.1", "rayon", + "redb", "reqwest", "revm", "revm-inspectors", "revm-primitives", - "rocksdb", "serde", "serde_json", + "tempfile", "test-log", "thiserror 2.0.12", "tokio", @@ -9651,7 +9593,6 @@ version = "2.0.15+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" dependencies = [ - "bindgen 0.71.1", "cc", "pkg-config", ] diff --git a/bin/portal-bridge/src/bridge/state.rs b/bin/portal-bridge/src/bridge/state.rs index 359afb9c8..bf6fa2549 100644 --- a/bin/portal-bridge/src/bridge/state.rs +++ b/bin/portal-bridge/src/bridge/state.rs @@ -37,7 +37,7 @@ use trin_execution::{ execution::TrinExecution, storage::{ account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, - utils::setup_rocksdb, + utils::setup_redb, }, subcommands::e2ss::{ import::StateImporter, @@ -181,8 +181,8 @@ impl StateBridge { // 1. Download the e2ss file and import the state snapshot let data_dir = setup_data_dir(APP_NAME, self.data_dir.clone(), false)?; let next_block_number = { - let rocks_db = Arc::new(setup_rocksdb(&data_dir)?); - let execution_position = ExecutionPosition::initialize_from_db(rocks_db.clone())?; + let red_db = Arc::new(setup_redb(&data_dir)?); + let execution_position = ExecutionPosition::initialize_from_db(red_db.clone())?; execution_position.next_block_number() }; @@ -240,15 +240,15 @@ impl StateBridge { // 2. Start the state bridge - let rocks_db = Arc::new(setup_rocksdb(&data_dir)?); + let redb_db = Arc::new(setup_redb(&data_dir)?); - let execution_position = ExecutionPosition::initialize_from_db(rocks_db.clone())?; + let execution_position = ExecutionPosition::initialize_from_db(redb_db.clone())?; ensure!( execution_position.next_block_number() > 0, "Trin execution not initialized!" ); - let mut evm_db = EvmDB::new(StateConfig::default(), rocks_db, &execution_position) + let mut evm_db = EvmDB::new(StateConfig::default(), redb_db, &execution_position) .expect("Failed to create EVM database"); self.gossip_whole_state_snapshot(&mut evm_db, execution_position) @@ -424,7 +424,7 @@ impl StateBridge { // check contract storage content key/value if account.storage_root != EMPTY_ROOT_HASH { - let account_db = AccountDB::new(address_hash, evm_db.db.clone()); + let account_db = AccountDB::new(address_hash, evm_db.db.clone())?; let trie = EthTrie::from(Arc::new(account_db), account.storage_root)?.db; let storage_walker = TrieWalker::new(account.storage_root, trie, None)?; diff --git a/bin/trin-execution/Cargo.toml b/bin/trin-execution/Cargo.toml index 020c45c73..452c1f90b 100644 --- a/bin/trin-execution/Cargo.toml +++ b/bin/trin-execution/Cargo.toml @@ -33,7 +33,7 @@ reqwest = { workspace = true, features = ["stream"] } revm.workspace = true revm-inspectors = "0.23" revm-primitives.workspace = true -rocksdb = "0.23" +redb = "2.5.0" serde = { workspace = true, features = ["rc"] } serde_json.workspace = true thiserror.workspace = true @@ -44,3 +44,4 @@ trin-utils.workspace = true [dev-dependencies] test-log.workspace = true +tempfile = "3" diff --git a/bin/trin-execution/src/evm/block_executor.rs b/bin/trin-execution/src/evm/block_executor.rs index 29719ec3d..57bfb7f4a 100644 --- a/bin/trin-execution/src/evm/block_executor.rs +++ b/bin/trin-execution/src/evm/block_executor.rs @@ -29,7 +29,7 @@ use crate::{ set_int_gauge_vec, start_timer_vec, stop_timer, BLOCK_HEIGHT, BLOCK_PROCESSING_TIMES, TRANSACTION_PROCESSING_TIMES, }, - storage::evm_db::EvmDB, + storage::evm_db::{EvmDB, ACCOUNTS_TABLE, BLOCK_HASHES_TABLE}, }; pub const BLOCKHASH_SERVE_WINDOW: u64 = 256; @@ -127,23 +127,25 @@ impl BlockExecutor { fn process_genesis(&mut self) -> anyhow::Result<()> { let genesis: GenesisConfig = serde_json::from_str(include_str!("../../resources/genesis/mainnet.json"))?; - - for (address, alloc_balance) in genesis.alloc { - let address_hash = keccak256(address); - let mut account = AccountState::default(); - account.balance += alloc_balance.balance; - self.evm - .db() - .database - .trie - .lock() - .insert(address_hash.as_ref(), &alloy::rlp::encode(&account))?; - self.evm - .db() - .database - .db - .put(address_hash, alloy::rlp::encode(account))?; + let db = &self.evm.db().database.db; + let txn = db.begin_write()?; + { + let mut table = txn.open_table(ACCOUNTS_TABLE)?; + for (address, alloc_balance) in genesis.alloc { + let address_hash = keccak256(address); + let mut account = AccountState::default(); + account.balance += alloc_balance.balance; + self.evm + .db() + .database + .trie + .lock() + .insert(address_hash.as_ref(), &alloy::rlp::encode(&account))?; + + table.insert(address_hash.as_slice(), alloy::rlp::encode(account).as_slice())?; + } } + txn.commit()?; Ok(()) } @@ -240,19 +242,22 @@ impl BlockExecutor { /// insert block hash into database and remove old one fn manage_block_hash_serve_window(&mut self, header: &Header) -> anyhow::Result<()> { let timer = start_timer_vec(&BLOCK_PROCESSING_TIMES, &["insert_blockhash"]); - self.evm.db().database.db.put( - keccak256(B256::from(U256::from(header.number))), - header.hash_slow(), - )?; - if header.number >= BLOCKHASH_SERVE_WINDOW { - self.evm - .db() - .database - .db - .delete(keccak256(B256::from(U256::from( - header.number - BLOCKHASH_SERVE_WINDOW, - ))))?; + + let db = &self.evm.db().database.db; + let txn = db.begin_write()?; + { + let mut table = txn.open_table(BLOCK_HASHES_TABLE)?; + let key = keccak256(B256::from(U256::from(header.number))); + let value = header.hash_slow(); + table.insert(key.as_slice(), value.as_slice())?; + + if header.number >= BLOCKHASH_SERVE_WINDOW { + let old_key = keccak256(B256::from(U256::from(header.number - BLOCKHASH_SERVE_WINDOW))); + table.remove(old_key.as_slice())?; + } } + txn.commit()?; + stop_timer(timer); Ok(()) } diff --git a/bin/trin-execution/src/execution.rs b/bin/trin-execution/src/execution.rs index 963703f3e..f72b00b15 100644 --- a/bin/trin-execution/src/execution.rs +++ b/bin/trin-execution/src/execution.rs @@ -18,7 +18,7 @@ use crate::{ e2hs::manager::E2HSManager, evm::block_executor::BlockExecutor, metrics::{start_timer_vec, stop_timer, BLOCK_PROCESSING_TIMES}, - storage::{evm_db::EvmDB, execution_position::ExecutionPosition, utils::setup_rocksdb}, + storage::{evm_db::EvmDB, execution_position::ExecutionPosition, utils::setup_redb}, }; pub struct TrinExecution { @@ -31,7 +31,7 @@ pub struct TrinExecution { impl TrinExecution { pub async fn new(data_dir: &Path, config: StateConfig) -> anyhow::Result { - let db = Arc::new(setup_rocksdb(data_dir)?); + let db = Arc::new(setup_redb(data_dir)?); let execution_position = ExecutionPosition::initialize_from_db(db.clone())?; let database = EvmDB::new(config.clone(), db, &execution_position) diff --git a/bin/trin-execution/src/storage/account_db.rs b/bin/trin-execution/src/storage/account_db.rs index fe5aafb5e..42fc658cc 100644 --- a/bin/trin-execution/src/storage/account_db.rs +++ b/bin/trin-execution/src/storage/account_db.rs @@ -2,98 +2,83 @@ use std::sync::Arc; use alloy::primitives::{keccak256, B256}; use eth_trie::DB; -use rocksdb::DB as RocksDB; +use redb::{Database as ReDB, TableDefinition}; use super::error::EVMError; static NULL_RLP_STATIC: [u8; 1] = [0x80; 1]; +const ACCOUNT_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("account_storage"); + #[derive(Debug, Clone)] pub struct AccountDB { pub address_hash: B256, - /// storage trie - pub db: Arc, + pub db: Arc } impl AccountDB { - pub fn new(address_hash: B256, db: Arc) -> Self { - Self { address_hash, db } + pub fn new(address_hash: B256, db: Arc) -> Result { + let txn = db.begin_write().map_err(EVMError::from)?; + txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; + txn.commit().map_err(EVMError::from)?; + + Ok(Self {address_hash, db}) } - fn get_db_key(&self, key: &[u8]) -> Vec { + pub fn get_db_key(&self, key: &[u8]) -> Vec { [self.address_hash.as_slice(), key].concat() } } impl DB for AccountDB { type Error = EVMError; - fn get(&self, key: &[u8]) -> Result>, EVMError> { + + fn get(&self, key: &[u8]) -> Result>, Self::Error> { if B256::from_slice(key) == keccak256([]) { return Ok(Some(NULL_RLP_STATIC.to_vec())); } - self.db.get(self.get_db_key(key)).map_err(|err| err.into()) + let txn = self.db.begin_read().map_err(EVMError::from)?; + let table = txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; + let db_key = self.get_db_key(key); + + match table.get(db_key.as_slice()).map_err(EVMError::from)? { + Some(access_guard) => Ok(Some(access_guard.value().to_vec())), + None => Ok(None) + } } - fn insert(&self, key: &[u8], value: Vec) -> Result<(), EVMError> { + fn insert(&self, key: &[u8], value: Vec) -> Result<(), Self::Error> { if B256::from_slice(key) == keccak256([]) { return Ok(()); } - self.db - .put(self.get_db_key(key), value) - .map_err(|err| err.into()) + + let txn = self.db.begin_write().map_err(EVMError::from)?; + { + let mut table = txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; + let db_key = self.get_db_key(key); + table.insert(db_key.as_slice(), value.as_slice()).map_err(EVMError::from)?; + } + txn.commit().map_err(EVMError::from)?; + Ok(()) } - fn remove(&self, key: &[u8]) -> Result<(), EVMError> { + fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { if B256::from_slice(key) == keccak256([]) { return Ok(()); } - self.db - .delete(self.get_db_key(key)) - .map_err(|err| err.into()) - } - - fn flush(&self) -> Result<(), EVMError> { - self.db.flush().map_err(|err| err.into()) - } -} -#[cfg(test)] -mod test_account_db { - - use eth_trie::DB; - use trin_utils::dir::create_temp_test_dir; - - use super::*; - use crate::storage::utils::setup_rocksdb; - - #[test] - fn test_account_db_get() { - let temp_directory = create_temp_test_dir().unwrap(); - let rocksdb = setup_rocksdb(temp_directory.path()).unwrap(); - let accdb = AccountDB::new(B256::ZERO, Arc::new(rocksdb)); - accdb - .insert(keccak256(b"test-key").as_slice(), b"test-value".to_vec()) - .unwrap(); - let v = accdb - .get(keccak256(b"test-key").as_slice()) - .unwrap() - .unwrap(); - assert_eq!(v, b"test-value"); - temp_directory.close().unwrap(); + let txn = self.db.begin_write().map_err(EVMError::from)?; + { + let mut table = txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; + let db_key = self.get_db_key(key); + table.remove(db_key.as_slice()).map_err(EVMError::from)?; + } + txn.commit().map_err(EVMError::from)?; + Ok(()) } - #[test] - fn test_account_db_remove() { - let temp_directory = create_temp_test_dir().unwrap(); - let rocksdb = setup_rocksdb(temp_directory.path()).unwrap(); - let accdb = AccountDB::new(B256::ZERO, Arc::new(rocksdb)); - accdb - .insert(keccak256(b"test").as_slice(), b"test".to_vec()) - .unwrap(); - accdb.remove(keccak256(b"test").as_slice()).unwrap(); - let contains = accdb.get(keccak256(b"test").as_slice()).unwrap(); - assert_eq!(contains, None); - temp_directory.close().unwrap(); + fn flush(&self) -> Result<(), Self::Error> { + Ok(()) } -} +} \ No newline at end of file diff --git a/bin/trin-execution/src/storage/error.rs b/bin/trin-execution/src/storage/error.rs index f1f6e4041..c049a7c69 100644 --- a/bin/trin-execution/src/storage/error.rs +++ b/bin/trin-execution/src/storage/error.rs @@ -9,8 +9,8 @@ pub enum EVMError { #[error("rlp error {0}")] RLP(#[from] alloy::rlp::Error), - #[error("rocksdb error {0}")] - DB(#[from] rocksdb::Error), + #[error("redb error {0}")] + DB(#[from] redb::Error), #[error("ethportal error {0}")] ANYHOW(#[from] anyhow::Error), @@ -23,3 +23,34 @@ pub enum EVMError { } impl DBErrorMarker for EVMError {} + + +impl From for EVMError { + fn from(err: redb::DatabaseError) -> Self { + EVMError::ANYHOW(anyhow::anyhow!("redb database error: {}", err)) + } +} + +impl From for EVMError { + fn from(err: redb::TransactionError) -> Self { + EVMError::ANYHOW(anyhow::anyhow!("redb transaction error: {}", err)) + } +} + +impl From for EVMError { + fn from(err: redb::TableError) -> Self { + EVMError::ANYHOW(anyhow::anyhow!("redb table error: {}", err)) + } +} + +impl From for EVMError { + fn from(err: redb::StorageError) -> Self { + EVMError::ANYHOW(anyhow::anyhow!("redb storage error: {}", err)) + } +} + +impl From for EVMError { + fn from(err: redb::CommitError) -> Self { + EVMError::ANYHOW(anyhow::anyhow!("redb commit error: {}", err)) + } +} diff --git a/bin/trin-execution/src/storage/evm_db.rs b/bin/trin-execution/src/storage/evm_db.rs index 5d4f4fdb9..a2ac39f01 100644 --- a/bin/trin-execution/src/storage/evm_db.rs +++ b/bin/trin-execution/src/storage/evm_db.rs @@ -16,10 +16,10 @@ use revm::{ Database, DatabaseRef, }; use revm_primitives::KECCAK_EMPTY; -use rocksdb::DB as RocksDB; +use redb::{Database as ReDB, Table, TableDefinition}; use tracing::info; -use super::{account_db::AccountDB, execution_position::ExecutionPosition, trie_db::TrieRocksDB}; +use super::{account_db::AccountDB, execution_position::ExecutionPosition, trie_db::TrieReDB}; use crate::{ config::StateConfig, metrics::{ @@ -28,6 +28,11 @@ use crate::{ storage::error::EVMError, }; +pub const ACCOUNTS_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("accounts"); +pub const CONTRACTS_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("contracts"); +pub const STORAGE_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("storage"); +pub const BLOCK_HASHES_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("block_hashes"); + fn start_commit_timer(name: &str) -> HistogramTimer { start_timer_vec(&BUNDLE_COMMIT_PROCESSING_TIMES, &[name]) } @@ -45,26 +50,36 @@ pub struct EvmDB { /// Cache for newly created contracts required for gossiping stat diffs, keyed by code hash. newly_created_contracts: Arc>>, /// The underlying database. - pub db: Arc, + pub db: Arc, /// To get proofs and to verify trie state. - pub trie: Arc>>, -} + pub trie: Arc>>, +} impl EvmDB { pub fn new( config: StateConfig, - db: Arc, + db: Arc, execution_position: &ExecutionPosition, ) -> anyhow::Result { - db.put(KECCAK_EMPTY, Bytecode::new().bytes().as_ref())?; - db.put(B256::ZERO, Bytecode::new().bytes().as_ref())?; + // Initialize empty byte code in the database + let txn = db.begin_write()?; + { + let mut contracts_table = txn.open_table(CONTRACTS_TABLE)?; + let empty_bytecode = Bytecode::new().bytes(); + contracts_table.insert(KECCAK_EMPTY.as_slice(), empty_bytecode.as_ref())?; + contracts_table.insert(B256::ZERO.as_slice(), empty_bytecode.as_ref())?; + } + txn.commit()?; + + // db.put(KECCAK_EMPTY, Bytecode::new().bytes().as_ref())?; + // db.put(B256::ZERO, Bytecode::new().bytes().as_ref())?; let trie = Arc::new(Mutex::new( if execution_position.state_root() == EMPTY_ROOT_HASH { - EthTrie::new(Arc::new(TrieRocksDB::new(false, db.clone()))) + EthTrie::new(Arc::new(TrieReDB::new(false, db.clone()))) } else { EthTrie::from( - Arc::new(TrieRocksDB::new(false, db.clone())), + Arc::new(TrieReDB::new(false, db.clone())), execution_position.state_root(), )? }, @@ -84,24 +99,21 @@ impl EvmDB { pub fn get_storage_trie_diff(&self, address_hash: B256) -> BrownHashMap> { let mut trie_diff = BrownHashMap::new(); + let txn = self.db.begin_read().expect("Redb read transaction failed"); + let storage_table = txn.open_table(STORAGE_TABLE).expect("Failed to open Redb storage table"); + for key in self .storage_cache .lock() .get(&address_hash) .unwrap_or(&HashSet::new()) { - // storage trie keys are prefixed with the address hash in the database - let value = self - .db - .get( - [address_hash.as_slice(), key.as_slice()] - .concat() - .as_slice(), - ) - .expect("Getting storage value should never fail"); - - if let Some(raw_value) = value { - trie_diff.insert(*key, raw_value); + let mut full_key = [0u8; 64]; + full_key[..32].copy_from_slice(address_hash.as_slice()); + full_key[32..].copy_from_slice(key.as_slice()); + + if let Ok(Some(value)) = storage_table.get(&full_key[..]) { + trie_diff.insert(*key, value.value().to_vec()); } } trie_diff @@ -151,8 +163,16 @@ impl EvmDB { stop_timer(timer); let timer = start_commit_timer("account:put_account_into_db"); - self.db - .put(address_hash, alloy::rlp::encode(account_state))?; + { + let txn = self.db.begin_write()?; + { + let mut table: Table<&[u8], &[u8]> = txn.open_table(ACCOUNTS_TABLE)?; + let key: &[u8] = address_hash.as_slice(); + let value: Vec = alloy::rlp::encode(&account_state); + table.insert(key, value.as_slice())?; + } + txn.commit()?; + } stop_timer(timer); stop_timer(plain_state_some_account_timer); @@ -173,7 +193,7 @@ impl EvmDB { // wipe storage trie and db if account_state.storage_root != EMPTY_ROOT_HASH { - let account_db = AccountDB::new(address_hash, self.db.clone()); + let account_db = AccountDB::new(address_hash, self.db.clone())?; let mut trie = EthTrie::from(Arc::new(account_db), account_state.storage_root)?; trie.clear_trie_from_db()?; account_state.storage_root = EMPTY_ROOT_HASH; @@ -181,11 +201,22 @@ impl EvmDB { // update account trie and db if delete_account { - self.db.delete(address_hash)?; + let txn = self.db.begin_write()?; + { + let mut table = txn.open_table(ACCOUNTS_TABLE)?; + table.remove(address_hash.as_slice())?; + } + txn.commit()?; + let _ = self.trie.lock().remove(address_hash.as_ref()); } else { - self.db - .put(address_hash, alloy::rlp::encode(&account_state))?; + let txn = self.db.begin_write()?; + { + let mut table = txn.open_table(ACCOUNTS_TABLE)?; + table.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice())?; + } + txn.commit()?; + let _ = self .trie .lock() @@ -218,7 +249,7 @@ impl EvmDB { ) -> anyhow::Result<()> { let timer = start_commit_timer("storage:apply_updates"); - let account_db = AccountDB::new(address_hash, self.db.clone()); + let account_db = AccountDB::new(address_hash, self.db.clone())?; let mut account_state = self.fetch_account(address_hash)?.unwrap_or_default(); let mut trie = if account_state.storage_root == EMPTY_ROOT_HASH { @@ -257,8 +288,12 @@ impl EvmDB { .lock() .insert(address_hash.as_ref(), &alloy::rlp::encode(&account_state)); - self.db - .put(address_hash, alloy::rlp::encode(account_state))?; + let txn = self.db.begin_write()?; + { + let mut table = txn.open_table(ACCOUNTS_TABLE)?; + table.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice())?; + } + txn.commit()?; stop_timer(timer); Ok(()) } @@ -316,9 +351,14 @@ impl EvmDB { .insert(hash, bytecode.clone()); } let timer = start_commit_timer("committing_contract"); - self.db - .put(hash, bytecode.original_bytes().as_ref()) - .expect("Inserting contract code should never fail"); + + let txn = self.db.begin_write()?; + { + let mut table = txn.open_table(CONTRACTS_TABLE)?; + table.insert(hash.as_slice(), bytecode.original_bytes().as_ref()).expect("Inserting contract code should never fail"); + } + txn.commit()?; + stop_timer(timer); } stop_timer(timer); @@ -332,9 +372,14 @@ impl EvmDB { } fn fetch_account(&self, address_hash: B256) -> anyhow::Result> { - match self.db.get(address_hash)? { - Some(raw_account) => Ok(Some(AccountState::decode(&mut raw_account.as_slice())?)), - None => Ok(None), + let txn = self.db.begin_read()?; + let table = txn.open_table(ACCOUNTS_TABLE)?; + + if let Some(raw_account) = table.get(address_hash.as_slice())? { + let decoded = AccountState::decode(&mut raw_account.value())?; + Ok(Some(decoded)) + } else { + Ok(None) } } } @@ -379,9 +424,14 @@ impl DatabaseRef for EvmDB { fn code_by_hash_ref(&self, code_hash: B256) -> Result { let timer = start_processing_timer("database_get_code_by_hash"); - let result = match self.db.get(code_hash)? { - Some(raw_code) => Ok(Bytecode::new_raw(raw_code.into())), - None => Err(Self::Error::NotFound("code_by_hash".to_string())), + + let txn = self.db.begin_read()?; + let table = txn.open_table(CONTRACTS_TABLE)?; + + let result = match table.get(code_hash.as_slice()) { + Ok(Some(value)) => Ok(Bytecode::new_raw(value.value().to_vec().into())), + Ok(None) => Err(Self::Error::NotFound("code_by_hash".to_string())), + Err(e) => Err(Self::Error::DB(e.into())), }; stop_timer(timer); result @@ -394,7 +444,7 @@ impl DatabaseRef for EvmDB { Some(account) => account, None => return Err(Self::Error::NotFound("storage".to_string())), }; - let account_db = AccountDB::new(address_hash, self.db.clone()); + let account_db = AccountDB::new(address_hash, self.db.clone())?; let raw_value = if account.storage_root == EMPTY_ROOT_HASH { None } else { @@ -411,10 +461,18 @@ impl DatabaseRef for EvmDB { fn block_hash_ref(&self, number: u64) -> Result { let timer = start_processing_timer("database_get_block_hash"); - let result = match self.db.get(keccak256(B256::from(U256::from(number))))? { - Some(raw_hash) => Ok(B256::from_slice(&raw_hash)), - None => Err(Self::Error::NotFound("block_hash".to_string())), + + let txn = self.db.begin_read()?; + let table = txn.open_table(BLOCK_HASHES_TABLE)?; + + let key = keccak256(B256::from(U256::from(number))); + + let result = match table.get(key.as_slice()) { + Ok(Some(value)) => Ok(B256::from_slice(value.value())), + Ok(None) => Err(Self::Error::NotFound("block_hash".to_string())), + Err(e) => Err(Self::Error::DB(e.into())), }; + stop_timer(timer); result } diff --git a/bin/trin-execution/src/storage/execution_position.rs b/bin/trin-execution/src/storage/execution_position.rs index 916ff67c7..c16d497bc 100644 --- a/bin/trin-execution/src/storage/execution_position.rs +++ b/bin/trin-execution/src/storage/execution_position.rs @@ -5,9 +5,10 @@ use alloy::{ rlp::{Decodable, RlpDecodable, RlpEncodable}, }; use revm_primitives::B256; -use rocksdb::DB as RocksDB; +use redb::{Database as ReDB, TableDefinition}; use serde::{Deserialize, Serialize}; +const TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("execution"); // The location in the database which describes the current execution position. pub const EXECUTION_POSITION_DB_KEY: &[u8; 18] = b"EXECUTION_POSITION"; @@ -23,13 +24,13 @@ pub struct ExecutionPosition { } impl ExecutionPosition { - pub fn initialize_from_db(db: Arc) -> anyhow::Result { - Ok(match db.get(EXECUTION_POSITION_DB_KEY)? { - Some(raw_execution_position) => { - Decodable::decode(&mut raw_execution_position.as_slice())? - } - None => Self::default(), - }) + pub fn initialize_from_db(db: Arc) -> anyhow::Result { + let txn = db.begin_read()?; + let table = txn.open_table(TABLE)?; + match table.get(EXECUTION_POSITION_DB_KEY.as_slice())? { + Some(value) => Ok(Decodable::decode(&mut value.value().as_ref())?), + None => Ok(Self::default()) + } } pub fn version(&self) -> u8 { @@ -44,12 +45,22 @@ impl ExecutionPosition { self.state_root } - pub fn update_position(&mut self, db: Arc, header: &Header) -> anyhow::Result<()> { + pub fn update_position(&mut self, db: Arc, header: &Header) -> anyhow::Result<()> { self.next_block_number = header.number + 1; self.state_root = header.state_root; - db.put(EXECUTION_POSITION_DB_KEY, alloy::rlp::encode(self))?; + + let txn = db.begin_write()?; + { + let mut table = txn.open_table(TABLE)?; + table.insert( + EXECUTION_POSITION_DB_KEY.as_slice(), + &alloy::rlp::encode(self)[..] + )?; + } + txn.commit()?; Ok(()) } + } impl Default for ExecutionPosition { @@ -60,4 +71,4 @@ impl Default for ExecutionPosition { state_root: EMPTY_ROOT_HASH, } } -} +} \ No newline at end of file diff --git a/bin/trin-execution/src/storage/trie_db.rs b/bin/trin-execution/src/storage/trie_db.rs index 4c119ccf6..a46ccc7d3 100644 --- a/bin/trin-execution/src/storage/trie_db.rs +++ b/bin/trin-execution/src/storage/trie_db.rs @@ -1,35 +1,52 @@ use std::sync::Arc; use eth_trie::DB; -use rocksdb::DB as RocksDB; + +use redb::{Database as ReDB, TableDefinition}; + +// Define a table type: key and value are byte arrays +const TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("trie"); #[derive(Debug)] -pub struct TrieRocksDB { +pub struct TrieReDB { // If "light" is true, the data is deleted from the database at the time of submission. light: bool, - storage: Arc, + storage: Arc, } -impl TrieRocksDB { - pub fn new(light: bool, storage: Arc) -> Self { - TrieRocksDB { light, storage } +impl TrieReDB { + pub fn new(light: bool, storage: Arc) -> Self { + TrieReDB { light, storage } } } -impl DB for TrieRocksDB { - type Error = rocksdb::Error; +impl DB for TrieReDB { + type Error = redb::Error; fn get(&self, key: &[u8]) -> Result>, Self::Error> { - self.storage.get(key) + let txn = self.storage.begin_read()?; + let table = txn.open_table(TABLE)?; + Ok(table.get(key)?.map(|val| val.value().to_vec())) } fn insert(&self, key: &[u8], value: Vec) -> Result<(), Self::Error> { - self.storage.put(key, value) + let txn = self.storage.begin_write()?; + { + let mut table = txn.open_table(TABLE)?; + table.insert(key, value.as_slice())?; + } + txn.commit()?; + Ok(()) } fn remove(&self, key: &[u8]) -> Result<(), Self::Error> { if self.light { - self.storage.delete(key)?; + let txn = self.storage.begin_write()?; + { + let mut table = txn.open_table(TABLE)?; + table.remove(key)?; + } + txn.commit()?; } Ok(()) } @@ -37,4 +54,4 @@ impl DB for TrieRocksDB { fn flush(&self) -> Result<(), Self::Error> { Ok(()) } -} +} \ No newline at end of file diff --git a/bin/trin-execution/src/storage/utils.rs b/bin/trin-execution/src/storage/utils.rs index 0a317d7d4..9b29010cd 100644 --- a/bin/trin-execution/src/storage/utils.rs +++ b/bin/trin-execution/src/storage/utils.rs @@ -1,24 +1,19 @@ use std::path::Path; -use rocksdb::{Options, DB as RocksDB}; +use redb::{Database as ReDB, Error}; use tracing::info; -/// Helper function for opening a RocksDB connection for the radius-constrained db. -pub fn setup_rocksdb(path: &Path) -> anyhow::Result { - let rocksdb_path = path.join("rocksdb"); - info!(path = %rocksdb_path.display(), "Setting up RocksDB"); - let cache_size = 1024 * 1024 * 1024; // 1GB +/// Helper function for opening a ReDB database at the specified path. +pub fn setup_redb(path: &Path) -> Result { + let redb_path = path.join("redb"); + info!(path = %redb_path.display(), "Setting up ReDB"); - let mut db_opts = Options::default(); - db_opts.create_if_missing(true); - db_opts.set_write_buffer_size(cache_size / 4); - let mut factory = rocksdb::BlockBasedOptions::default(); - factory.set_block_cache(&rocksdb::Cache::new_lru_cache(cache_size / 2)); - db_opts.set_block_based_table_factory(&factory); + let db = if redb_path.exists() { + ReDB::open(redb_path)? + } else { + ReDB::create(redb_path)? + }; - // Set the max number of open files to 150. MacOs has a default limit of 256 open files per - // process. This limit prevents issues with the OS running out of file descriptors. - db_opts.set_max_open_files(150); - Ok(RocksDB::open(&db_opts, rocksdb_path)?) -} + Ok(db) +} \ No newline at end of file diff --git a/bin/trin-execution/src/subcommands/e2ss/export.rs b/bin/trin-execution/src/subcommands/e2ss/export.rs index 8c1d67ae3..d02604a74 100644 --- a/bin/trin-execution/src/subcommands/e2ss/export.rs +++ b/bin/trin-execution/src/subcommands/e2ss/export.rs @@ -23,11 +23,13 @@ use crate::{ e2hs::manager::E2HSManager, storage::{ account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, - utils::setup_rocksdb, + utils::setup_redb }, subcommands::e2ss::utils::percentage_from_address_hash, }; +use crate::storage::evm_db::CONTRACTS_TABLE; + pub struct StateExporter { config: ExportStateConfig, header: Header, @@ -36,9 +38,9 @@ pub struct StateExporter { impl StateExporter { pub async fn new(config: ExportStateConfig, data_dir: &Path) -> anyhow::Result { - let rocks_db = Arc::new(setup_rocksdb(data_dir)?); + let red_db = Arc::new(setup_redb(data_dir)?); - let execution_position = ExecutionPosition::initialize_from_db(rocks_db.clone())?; + let execution_position = ExecutionPosition::initialize_from_db(red_db.clone())?; ensure!( execution_position.next_block_number() > 0, "Trin execution not initialized!" @@ -53,7 +55,7 @@ impl StateExporter { .header .clone(); - let evm_db = EvmDB::new(StateConfig::default(), rocks_db, &execution_position) + let evm_db = EvmDB::new(StateConfig::default(), red_db, &execution_position) .expect("Failed to create EVM database"); ensure!( evm_db.trie.lock().root_hash()? == header.state_root, @@ -82,9 +84,10 @@ impl StateExporter { let account_state = AccountState::decode(&mut account_state.as_slice())?; let bytecode = if account_state.code_hash != KECCAK_EMPTY { - self.evm_db - .db - .get(account_state.code_hash)? + let txn = self.evm_db.db.begin_read()?; + let table = txn.open_table(CONTRACTS_TABLE)?; + table.get(account_state.code_hash.as_slice())? + .map(|val| val.value().to_vec()) .expect("If code hash is not empty, code must be present") } else { vec![] @@ -92,7 +95,7 @@ impl StateExporter { let mut storage: Vec = vec![]; if account_state.storage_root != EMPTY_ROOT_HASH { - let account_db = AccountDB::new(account_hash, self.evm_db.db.clone()); + let account_db = AccountDB::new(account_hash, self.evm_db.db.clone())?; let account_trie = Arc::new(Mutex::new(EthTrie::from( Arc::new(account_db), account_state.storage_root, diff --git a/bin/trin-execution/src/subcommands/e2ss/import.rs b/bin/trin-execution/src/subcommands/e2ss/import.rs index 3c53d6b69..cbce5c78d 100644 --- a/bin/trin-execution/src/subcommands/e2ss/import.rs +++ b/bin/trin-execution/src/subcommands/e2ss/import.rs @@ -14,11 +14,13 @@ use crate::{ evm::block_executor::BLOCKHASH_SERVE_WINDOW, storage::{ account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, - utils::setup_rocksdb, + utils::setup_redb, }, subcommands::e2ss::utils::percentage_from_address_hash, }; +use crate::storage::evm_db::{ACCOUNTS_TABLE, BLOCK_HASHES_TABLE, CONTRACTS_TABLE, STORAGE_TABLE}; + pub struct StateImporter { config: ImportStateConfig, evm_db: EvmDB, @@ -26,15 +28,15 @@ pub struct StateImporter { impl StateImporter { pub async fn new(config: ImportStateConfig, data_dir: &Path) -> anyhow::Result { - let rocks_db = Arc::new(setup_rocksdb(data_dir)?); + let red_db = Arc::new(setup_redb(data_dir)?); - let execution_position = ExecutionPosition::initialize_from_db(rocks_db.clone())?; + let execution_position = ExecutionPosition::initialize_from_db(red_db.clone())?; ensure!( execution_position.next_block_number() == 0, "Cannot import state from .e2ss, database is not empty", ); - let evm_db = EvmDB::new(StateConfig::default(), rocks_db, &execution_position) + let evm_db = EvmDB::new(StateConfig::default(), red_db, &execution_position) .expect("Failed to create EVM database"); Ok(Self { config, evm_db }) @@ -72,7 +74,7 @@ impl StateImporter { } = account; // Build storage trie - let account_db = AccountDB::new(address_hash, self.evm_db.db.clone()); + let account_db = AccountDB::new(address_hash, self.evm_db.db.clone())?; let mut storage_trie = EthTrie::new(Arc::new(account_db)); for _ in 0..storage_count { let Some(AccountOrStorageEntry::Storage(storage_entry)) = e2ss.next() else { @@ -100,20 +102,34 @@ impl StateImporter { account_state.code_hash == keccak256(&bytecode), "Code hash mismatch, .e2ss import failed" ); - if !bytecode.is_empty() { - self.evm_db.db.put(keccak256(&bytecode), bytecode.clone())?; - } + + let db = &self.evm_db.db; + let txn = db.begin_write()?; + + { + let mut accounts = txn.open_table(ACCOUNTS_TABLE)?; + let mut contracts = txn.open_table(CONTRACTS_TABLE)?; - // Insert account into state trie + if !bytecode.is_empty() { + contracts.insert(keccak256(&bytecode).as_slice(), bytecode.as_slice())?; + } + + // Insert account into accounts table + accounts.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice(),)?; + } + txn.commit()?; + self.evm_db .trie .lock() .insert(address_hash.as_slice(), &alloy::rlp::encode(&account_state))?; - self.evm_db - .db - .put(address_hash, alloy::rlp::encode(account_state)) - .expect("Inserting account should never fail"); + let txn = self.evm_db.db.begin_write()?; + { + let mut accounts = txn.open_table(ACCOUNTS_TABLE)?; + accounts.insert(address_hash.as_slice(), alloy::rlp::encode(account_state).as_slice()).expect("Inserting account should never fail"); + } + txn.commit()?; accounts_imported += 1; if accounts_imported % 1000 == 0 { @@ -139,6 +155,7 @@ impl StateImporter { /// insert the last 256 block hashes into the database async fn import_last_256_block_hashes(&self, block_number: u64) -> anyhow::Result<()> { let first_block_hash_to_add = block_number.saturating_sub(BLOCKHASH_SERVE_WINDOW); +<<<<<<< HEAD let mut e2hs_manager = E2HSManager::new(first_block_hash_to_add).await?; while e2hs_manager.next_block_number() <= block_number { let block = e2hs_manager.get_next_block().await?; @@ -146,7 +163,20 @@ impl StateImporter { keccak256(B256::from(U256::from(block.header.number))), block.header.hash_slow(), )? +======= + let mut era_manager = EraManager::new(first_block_hash_to_add).await?; + + let txn = self.evm_db.db.begin_write()?; + { + let mut table = txn.open_table(BLOCK_HASHES_TABLE)?; + + while era_manager.next_block_number() <= block_number { + let block = era_manager.get_next_block().await?; + table.insert(keccak256(B256::from(U256::from(block.header.number))).as_slice(), block.header.hash_slow().as_slice())?; + } +>>>>>>> 7c448cc7 (Replace RocksDB with Redb as backing for EVM database) } + txn.commit()?; Ok(()) } diff --git a/bin/trin-execution/src/trie_walker/db.rs b/bin/trin-execution/src/trie_walker/db.rs index 5af388d25..b1b40025a 100644 --- a/bin/trin-execution/src/trie_walker/db.rs +++ b/bin/trin-execution/src/trie_walker/db.rs @@ -3,7 +3,7 @@ use anyhow::anyhow; use eth_trie::DB; use hashbrown::HashMap; -use crate::storage::{account_db::AccountDB, trie_db::TrieRocksDB}; +use crate::storage::{account_db::AccountDB, trie_db::TrieReDB}; pub trait TrieWalkerDb { fn get(&self, key: &[u8]) -> anyhow::Result>; @@ -15,11 +15,11 @@ impl TrieWalkerDb for HashMap> { } } -impl TrieWalkerDb for TrieRocksDB { +impl TrieWalkerDb for TrieReDB { fn get(&self, key: &[u8]) -> anyhow::Result> { DB::get(self, key) .map(|result| result.map(Bytes::from)) - .map_err(|err| anyhow!("Failed to read key value from TrieRocksDB {err}")) + .map_err(|err| anyhow!("Failed to read key value from TrieReDB {err}")) } } @@ -27,6 +27,6 @@ impl TrieWalkerDb for AccountDB { fn get(&self, key: &[u8]) -> anyhow::Result> { DB::get(self, key) .map(|result| result.map(Bytes::from)) - .map_err(|err| anyhow!("Failed to read key value from TrieRocksDB {err}")) + .map_err(|err| anyhow!("Failed to read key value from TrieReDB {err}")) } } diff --git a/bin/trin-execution/src/trie_walker/mod.rs b/bin/trin-execution/src/trie_walker/mod.rs index 90579dcfa..1d731fb77 100644 --- a/bin/trin-execution/src/trie_walker/mod.rs +++ b/bin/trin-execution/src/trie_walker/mod.rs @@ -182,15 +182,15 @@ mod tests { use crate::{ config::StateConfig, execution::TrinExecution, - storage::{trie_db::TrieRocksDB, utils::setup_rocksdb}, + storage::{trie_db::TrieReDB, utils::setup_redb}, utils::full_nibble_path_to_address_hash, }; #[tokio::test] async fn test_state_walker() { let temp_directory = create_temp_test_dir().unwrap(); - let db = Arc::new(setup_rocksdb(temp_directory.path()).unwrap()); - let mut trie = EthTrie::new(Arc::new(TrieRocksDB::new(false, db.clone()))); + let db = Arc::new(setup_redb(temp_directory.path()).unwrap()); + let mut trie = EthTrie::new(Arc::new(TrieReDB::new(false, db.clone()))); for i in 1..=18 { trie.insert( From 99f8055a997e852c71162b28e5f7e83cc62298df Mon Sep 17 00:00:00 2001 From: Owanikin Date: Tue, 3 Jun 2025 19:10:25 +0100 Subject: [PATCH 2/5] fix: remove unused STORAGE_TABLE import to fix CI --- bin/trin-execution/src/subcommands/e2ss/import.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/trin-execution/src/subcommands/e2ss/import.rs b/bin/trin-execution/src/subcommands/e2ss/import.rs index cbce5c78d..f21988f63 100644 --- a/bin/trin-execution/src/subcommands/e2ss/import.rs +++ b/bin/trin-execution/src/subcommands/e2ss/import.rs @@ -19,7 +19,7 @@ use crate::{ subcommands::e2ss::utils::percentage_from_address_hash, }; -use crate::storage::evm_db::{ACCOUNTS_TABLE, BLOCK_HASHES_TABLE, CONTRACTS_TABLE, STORAGE_TABLE}; +use crate::storage::evm_db::{ACCOUNTS_TABLE, BLOCK_HASHES_TABLE, CONTRACTS_TABLE}; pub struct StateImporter { config: ImportStateConfig, From 7bb1be9e562e22e2862d4af58aa7ef3164749286 Mon Sep 17 00:00:00 2001 From: Owanikin Date: Tue, 3 Jun 2025 21:02:11 +0100 Subject: [PATCH 3/5] Fix remaining issues before rebase --- bin/trin-execution/src/evm/block_executor.rs | 15 ++++++--- bin/trin-execution/src/storage/account_db.rs | 18 ++++++++--- bin/trin-execution/src/storage/error.rs | 3 +- bin/trin-execution/src/storage/evm_db.rs | 32 ++++++++++++------- .../src/storage/execution_position.rs | 11 +++---- bin/trin-execution/src/storage/trie_db.rs | 4 +-- bin/trin-execution/src/storage/utils.rs | 3 +- .../src/subcommands/e2ss/export.rs | 5 +-- .../src/subcommands/e2ss/import.rs | 23 +++++++++---- 9 files changed, 73 insertions(+), 41 deletions(-) diff --git a/bin/trin-execution/src/evm/block_executor.rs b/bin/trin-execution/src/evm/block_executor.rs index 57bfb7f4a..d6c985fef 100644 --- a/bin/trin-execution/src/evm/block_executor.rs +++ b/bin/trin-execution/src/evm/block_executor.rs @@ -129,9 +129,9 @@ impl BlockExecutor { serde_json::from_str(include_str!("../../resources/genesis/mainnet.json"))?; let db = &self.evm.db().database.db; let txn = db.begin_write()?; - { - let mut table = txn.open_table(ACCOUNTS_TABLE)?; - for (address, alloc_balance) in genesis.alloc { + { + let mut table = txn.open_table(ACCOUNTS_TABLE)?; + for (address, alloc_balance) in genesis.alloc { let address_hash = keccak256(address); let mut account = AccountState::default(); account.balance += alloc_balance.balance; @@ -142,7 +142,10 @@ impl BlockExecutor { .lock() .insert(address_hash.as_ref(), &alloy::rlp::encode(&account))?; - table.insert(address_hash.as_slice(), alloy::rlp::encode(account).as_slice())?; + table.insert( + address_hash.as_slice(), + alloy::rlp::encode(account).as_slice(), + )?; } } txn.commit()?; @@ -252,7 +255,9 @@ impl BlockExecutor { table.insert(key.as_slice(), value.as_slice())?; if header.number >= BLOCKHASH_SERVE_WINDOW { - let old_key = keccak256(B256::from(U256::from(header.number - BLOCKHASH_SERVE_WINDOW))); + let old_key = keccak256(B256::from(U256::from( + header.number - BLOCKHASH_SERVE_WINDOW, + ))); table.remove(old_key.as_slice())?; } } diff --git a/bin/trin-execution/src/storage/account_db.rs b/bin/trin-execution/src/storage/account_db.rs index 42fc658cc..02679e3fa 100644 --- a/bin/trin-execution/src/storage/account_db.rs +++ b/bin/trin-execution/src/storage/account_db.rs @@ -13,7 +13,13 @@ const ACCOUNT_TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("accou #[derive(Debug, Clone)] pub struct AccountDB { pub address_hash: B256, - pub db: Arc + pub db: Arc, +} + +impl From for EVMError { + fn from(e: redb::Error) -> Self { + EVMError::DB(Box::new(e)) + } } impl AccountDB { @@ -22,7 +28,7 @@ impl AccountDB { txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; txn.commit().map_err(EVMError::from)?; - Ok(Self {address_hash, db}) + Ok(Self { address_hash, db }) } pub fn get_db_key(&self, key: &[u8]) -> Vec { @@ -44,7 +50,7 @@ impl DB for AccountDB { match table.get(db_key.as_slice()).map_err(EVMError::from)? { Some(access_guard) => Ok(Some(access_guard.value().to_vec())), - None => Ok(None) + None => Ok(None), } } @@ -57,7 +63,9 @@ impl DB for AccountDB { { let mut table = txn.open_table(ACCOUNT_TABLE).map_err(EVMError::from)?; let db_key = self.get_db_key(key); - table.insert(db_key.as_slice(), value.as_slice()).map_err(EVMError::from)?; + table + .insert(db_key.as_slice(), value.as_slice()) + .map_err(EVMError::from)?; } txn.commit().map_err(EVMError::from)?; Ok(()) @@ -81,4 +89,4 @@ impl DB for AccountDB { fn flush(&self) -> Result<(), Self::Error> { Ok(()) } -} \ No newline at end of file +} diff --git a/bin/trin-execution/src/storage/error.rs b/bin/trin-execution/src/storage/error.rs index c049a7c69..89afa3dab 100644 --- a/bin/trin-execution/src/storage/error.rs +++ b/bin/trin-execution/src/storage/error.rs @@ -10,7 +10,7 @@ pub enum EVMError { RLP(#[from] alloy::rlp::Error), #[error("redb error {0}")] - DB(#[from] redb::Error), + DB(#[from] Box), #[error("ethportal error {0}")] ANYHOW(#[from] anyhow::Error), @@ -24,7 +24,6 @@ pub enum EVMError { impl DBErrorMarker for EVMError {} - impl From for EVMError { fn from(err: redb::DatabaseError) -> Self { EVMError::ANYHOW(anyhow::anyhow!("redb database error: {}", err)) diff --git a/bin/trin-execution/src/storage/evm_db.rs b/bin/trin-execution/src/storage/evm_db.rs index a2ac39f01..76d144012 100644 --- a/bin/trin-execution/src/storage/evm_db.rs +++ b/bin/trin-execution/src/storage/evm_db.rs @@ -10,13 +10,13 @@ use ethportal_api::types::state_trie::account_state::AccountState; use hashbrown::{HashMap as BrownHashMap, HashSet}; use parking_lot::Mutex; use prometheus_exporter::prometheus::HistogramTimer; +use redb::{Database as ReDB, Table, TableDefinition}; use revm::{ database::{states::PlainStorageChangeset, BundleState, OriginalValuesKnown}, state::{AccountInfo, Bytecode}, Database, DatabaseRef, }; use revm_primitives::KECCAK_EMPTY; -use redb::{Database as ReDB, Table, TableDefinition}; use tracing::info; use super::{account_db::AccountDB, execution_position::ExecutionPosition, trie_db::TrieReDB}; @@ -53,7 +53,7 @@ pub struct EvmDB { pub db: Arc, /// To get proofs and to verify trie state. pub trie: Arc>>, -} +} impl EvmDB { pub fn new( @@ -61,7 +61,7 @@ impl EvmDB { db: Arc, execution_position: &ExecutionPosition, ) -> anyhow::Result { - // Initialize empty byte code in the database + // Initialize empty byte code in the database let txn = db.begin_write()?; { let mut contracts_table = txn.open_table(CONTRACTS_TABLE)?; @@ -70,7 +70,7 @@ impl EvmDB { contracts_table.insert(B256::ZERO.as_slice(), empty_bytecode.as_ref())?; } txn.commit()?; - + // db.put(KECCAK_EMPTY, Bytecode::new().bytes().as_ref())?; // db.put(B256::ZERO, Bytecode::new().bytes().as_ref())?; @@ -100,7 +100,9 @@ impl EvmDB { let mut trie_diff = BrownHashMap::new(); let txn = self.db.begin_read().expect("Redb read transaction failed"); - let storage_table = txn.open_table(STORAGE_TABLE).expect("Failed to open Redb storage table"); + let storage_table = txn + .open_table(STORAGE_TABLE) + .expect("Failed to open Redb storage table"); for key in self .storage_cache @@ -213,7 +215,10 @@ impl EvmDB { let txn = self.db.begin_write()?; { let mut table = txn.open_table(ACCOUNTS_TABLE)?; - table.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice())?; + table.insert( + address_hash.as_slice(), + alloy::rlp::encode(&account_state).as_slice(), + )?; } txn.commit()?; @@ -291,7 +296,10 @@ impl EvmDB { let txn = self.db.begin_write()?; { let mut table = txn.open_table(ACCOUNTS_TABLE)?; - table.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice())?; + table.insert( + address_hash.as_slice(), + alloy::rlp::encode(&account_state).as_slice(), + )?; } txn.commit()?; stop_timer(timer); @@ -355,10 +363,12 @@ impl EvmDB { let txn = self.db.begin_write()?; { let mut table = txn.open_table(CONTRACTS_TABLE)?; - table.insert(hash.as_slice(), bytecode.original_bytes().as_ref()).expect("Inserting contract code should never fail"); + table + .insert(hash.as_slice(), bytecode.original_bytes().as_ref()) + .expect("Inserting contract code should never fail"); } txn.commit()?; - + stop_timer(timer); } stop_timer(timer); @@ -431,7 +441,7 @@ impl DatabaseRef for EvmDB { let result = match table.get(code_hash.as_slice()) { Ok(Some(value)) => Ok(Bytecode::new_raw(value.value().to_vec().into())), Ok(None) => Err(Self::Error::NotFound("code_by_hash".to_string())), - Err(e) => Err(Self::Error::DB(e.into())), + Err(e) => Err(Self::Error::DB(Box::new(e.into()))), }; stop_timer(timer); result @@ -470,7 +480,7 @@ impl DatabaseRef for EvmDB { let result = match table.get(key.as_slice()) { Ok(Some(value)) => Ok(B256::from_slice(value.value())), Ok(None) => Err(Self::Error::NotFound("block_hash".to_string())), - Err(e) => Err(Self::Error::DB(e.into())), + Err(e) => Err(Self::Error::DB(Box::new(e.into()))), }; stop_timer(timer); diff --git a/bin/trin-execution/src/storage/execution_position.rs b/bin/trin-execution/src/storage/execution_position.rs index c16d497bc..75302a2d0 100644 --- a/bin/trin-execution/src/storage/execution_position.rs +++ b/bin/trin-execution/src/storage/execution_position.rs @@ -4,8 +4,8 @@ use alloy::{ consensus::{Header, EMPTY_ROOT_HASH}, rlp::{Decodable, RlpDecodable, RlpEncodable}, }; -use revm_primitives::B256; use redb::{Database as ReDB, TableDefinition}; +use revm_primitives::B256; use serde::{Deserialize, Serialize}; const TABLE: TableDefinition<&[u8], &[u8]> = TableDefinition::new("execution"); @@ -29,7 +29,7 @@ impl ExecutionPosition { let table = txn.open_table(TABLE)?; match table.get(EXECUTION_POSITION_DB_KEY.as_slice())? { Some(value) => Ok(Decodable::decode(&mut value.value().as_ref())?), - None => Ok(Self::default()) + None => Ok(Self::default()), } } @@ -53,14 +53,13 @@ impl ExecutionPosition { { let mut table = txn.open_table(TABLE)?; table.insert( - EXECUTION_POSITION_DB_KEY.as_slice(), - &alloy::rlp::encode(self)[..] + EXECUTION_POSITION_DB_KEY.as_slice(), + &alloy::rlp::encode(self)[..], )?; } txn.commit()?; Ok(()) } - } impl Default for ExecutionPosition { @@ -71,4 +70,4 @@ impl Default for ExecutionPosition { state_root: EMPTY_ROOT_HASH, } } -} \ No newline at end of file +} diff --git a/bin/trin-execution/src/storage/trie_db.rs b/bin/trin-execution/src/storage/trie_db.rs index a46ccc7d3..edcc9bcfb 100644 --- a/bin/trin-execution/src/storage/trie_db.rs +++ b/bin/trin-execution/src/storage/trie_db.rs @@ -26,7 +26,7 @@ impl DB for TrieReDB { fn get(&self, key: &[u8]) -> Result>, Self::Error> { let txn = self.storage.begin_read()?; let table = txn.open_table(TABLE)?; - Ok(table.get(key)?.map(|val| val.value().to_vec())) + Ok(table.get(key)?.map(|val| val.value().to_vec())) } fn insert(&self, key: &[u8], value: Vec) -> Result<(), Self::Error> { @@ -54,4 +54,4 @@ impl DB for TrieReDB { fn flush(&self) -> Result<(), Self::Error> { Ok(()) } -} \ No newline at end of file +} diff --git a/bin/trin-execution/src/storage/utils.rs b/bin/trin-execution/src/storage/utils.rs index 9b29010cd..6553ecdaf 100644 --- a/bin/trin-execution/src/storage/utils.rs +++ b/bin/trin-execution/src/storage/utils.rs @@ -3,7 +3,6 @@ use std::path::Path; use redb::{Database as ReDB, Error}; use tracing::info; - /// Helper function for opening a ReDB database at the specified path. pub fn setup_redb(path: &Path) -> Result { let redb_path = path.join("redb"); @@ -16,4 +15,4 @@ pub fn setup_redb(path: &Path) -> Result { }; Ok(db) -} \ No newline at end of file +} diff --git a/bin/trin-execution/src/subcommands/e2ss/export.rs b/bin/trin-execution/src/subcommands/e2ss/export.rs index d02604a74..ce6adaf38 100644 --- a/bin/trin-execution/src/subcommands/e2ss/export.rs +++ b/bin/trin-execution/src/subcommands/e2ss/export.rs @@ -23,7 +23,7 @@ use crate::{ e2hs::manager::E2HSManager, storage::{ account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, - utils::setup_redb + utils::setup_redb, }, subcommands::e2ss::utils::percentage_from_address_hash, }; @@ -86,7 +86,8 @@ impl StateExporter { let bytecode = if account_state.code_hash != KECCAK_EMPTY { let txn = self.evm_db.db.begin_read()?; let table = txn.open_table(CONTRACTS_TABLE)?; - table.get(account_state.code_hash.as_slice())? + table + .get(account_state.code_hash.as_slice())? .map(|val| val.value().to_vec()) .expect("If code hash is not empty, code must be present") } else { diff --git a/bin/trin-execution/src/subcommands/e2ss/import.rs b/bin/trin-execution/src/subcommands/e2ss/import.rs index f21988f63..04963b595 100644 --- a/bin/trin-execution/src/subcommands/e2ss/import.rs +++ b/bin/trin-execution/src/subcommands/e2ss/import.rs @@ -102,7 +102,7 @@ impl StateImporter { account_state.code_hash == keccak256(&bytecode), "Code hash mismatch, .e2ss import failed" ); - + let db = &self.evm_db.db; let txn = db.begin_write()?; @@ -111,14 +111,17 @@ impl StateImporter { let mut contracts = txn.open_table(CONTRACTS_TABLE)?; if !bytecode.is_empty() { - contracts.insert(keccak256(&bytecode).as_slice(), bytecode.as_slice())?; + contracts.insert(keccak256(&bytecode).as_slice(), bytecode.as_slice())?; } // Insert account into accounts table - accounts.insert(address_hash.as_slice(), alloy::rlp::encode(&account_state).as_slice(),)?; + accounts.insert( + address_hash.as_slice(), + alloy::rlp::encode(&account_state).as_slice(), + )?; } txn.commit()?; - + self.evm_db .trie .lock() @@ -127,7 +130,12 @@ impl StateImporter { let txn = self.evm_db.db.begin_write()?; { let mut accounts = txn.open_table(ACCOUNTS_TABLE)?; - accounts.insert(address_hash.as_slice(), alloy::rlp::encode(account_state).as_slice()).expect("Inserting account should never fail"); + accounts + .insert( + address_hash.as_slice(), + alloy::rlp::encode(account_state).as_slice(), + ) + .expect("Inserting account should never fail"); } txn.commit()?; @@ -172,7 +180,10 @@ impl StateImporter { while era_manager.next_block_number() <= block_number { let block = era_manager.get_next_block().await?; - table.insert(keccak256(B256::from(U256::from(block.header.number))).as_slice(), block.header.hash_slow().as_slice())?; + table.insert( + keccak256(B256::from(U256::from(block.header.number))).as_slice(), + block.header.hash_slow().as_slice(), + )?; } >>>>>>> 7c448cc7 (Replace RocksDB with Redb as backing for EVM database) } From 99d9108dc2559116110de4bc33785cdac04a9d9a Mon Sep 17 00:00:00 2001 From: Owanikin Date: Tue, 3 Jun 2025 21:30:51 +0100 Subject: [PATCH 4/5] style: fix formatting to satisfy cargo fmt --- bin/trin-execution/src/storage/execution_position.rs | 2 +- bin/trin-execution/src/storage/utils.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/trin-execution/src/storage/execution_position.rs b/bin/trin-execution/src/storage/execution_position.rs index 75302a2d0..b493b2f88 100644 --- a/bin/trin-execution/src/storage/execution_position.rs +++ b/bin/trin-execution/src/storage/execution_position.rs @@ -28,7 +28,7 @@ impl ExecutionPosition { let txn = db.begin_read()?; let table = txn.open_table(TABLE)?; match table.get(EXECUTION_POSITION_DB_KEY.as_slice())? { - Some(value) => Ok(Decodable::decode(&mut value.value().as_ref())?), + Some(value) => Ok(Decodable::decode(&mut value.value())?), None => Ok(Self::default()), } } diff --git a/bin/trin-execution/src/storage/utils.rs b/bin/trin-execution/src/storage/utils.rs index 6553ecdaf..80d698df1 100644 --- a/bin/trin-execution/src/storage/utils.rs +++ b/bin/trin-execution/src/storage/utils.rs @@ -4,6 +4,7 @@ use redb::{Database as ReDB, Error}; use tracing::info; /// Helper function for opening a ReDB database at the specified path. +#[allow(clippy::result_large_err)] pub fn setup_redb(path: &Path) -> Result { let redb_path = path.join("redb"); info!(path = %redb_path.display(), "Setting up ReDB"); From 74e67493241f9d93077b082acc9d35d8e5229d77 Mon Sep 17 00:00:00 2001 From: Owanikin Date: Tue, 3 Jun 2025 21:49:24 +0100 Subject: [PATCH 5/5] style: fix formatting to satisfy cargo fmt --- bin/trin-execution/src/storage/trie_db.rs | 1 - .../src/subcommands/e2ss/export.rs | 6 +++--- .../src/subcommands/e2ss/import.rs | 20 +++++-------------- 3 files changed, 8 insertions(+), 19 deletions(-) diff --git a/bin/trin-execution/src/storage/trie_db.rs b/bin/trin-execution/src/storage/trie_db.rs index edcc9bcfb..0d831a2b0 100644 --- a/bin/trin-execution/src/storage/trie_db.rs +++ b/bin/trin-execution/src/storage/trie_db.rs @@ -1,7 +1,6 @@ use std::sync::Arc; use eth_trie::DB; - use redb::{Database as ReDB, TableDefinition}; // Define a table type: key and value are byte arrays diff --git a/bin/trin-execution/src/subcommands/e2ss/export.rs b/bin/trin-execution/src/subcommands/e2ss/export.rs index ce6adaf38..da0b5c4c4 100644 --- a/bin/trin-execution/src/subcommands/e2ss/export.rs +++ b/bin/trin-execution/src/subcommands/e2ss/export.rs @@ -22,14 +22,14 @@ use crate::{ config::StateConfig, e2hs::manager::E2HSManager, storage::{ - account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, + account_db::AccountDB, + evm_db::{EvmDB, CONTRACTS_TABLE}, + execution_position::ExecutionPosition, utils::setup_redb, }, subcommands::e2ss::utils::percentage_from_address_hash, }; -use crate::storage::evm_db::CONTRACTS_TABLE; - pub struct StateExporter { config: ExportStateConfig, header: Header, diff --git a/bin/trin-execution/src/subcommands/e2ss/import.rs b/bin/trin-execution/src/subcommands/e2ss/import.rs index 04963b595..51ccb8e0c 100644 --- a/bin/trin-execution/src/subcommands/e2ss/import.rs +++ b/bin/trin-execution/src/subcommands/e2ss/import.rs @@ -13,14 +13,14 @@ use crate::{ e2hs::manager::E2HSManager, evm::block_executor::BLOCKHASH_SERVE_WINDOW, storage::{ - account_db::AccountDB, evm_db::EvmDB, execution_position::ExecutionPosition, + account_db::AccountDB, + evm_db::{EvmDB, ACCOUNTS_TABLE, BLOCK_HASHES_TABLE, CONTRACTS_TABLE}, + execution_position::ExecutionPosition, utils::setup_redb, }, subcommands::e2ss::utils::percentage_from_address_hash, }; -use crate::storage::evm_db::{ACCOUNTS_TABLE, BLOCK_HASHES_TABLE, CONTRACTS_TABLE}; - pub struct StateImporter { config: ImportStateConfig, evm_db: EvmDB, @@ -163,29 +163,19 @@ impl StateImporter { /// insert the last 256 block hashes into the database async fn import_last_256_block_hashes(&self, block_number: u64) -> anyhow::Result<()> { let first_block_hash_to_add = block_number.saturating_sub(BLOCKHASH_SERVE_WINDOW); -<<<<<<< HEAD let mut e2hs_manager = E2HSManager::new(first_block_hash_to_add).await?; - while e2hs_manager.next_block_number() <= block_number { - let block = e2hs_manager.get_next_block().await?; - self.evm_db.db.put( - keccak256(B256::from(U256::from(block.header.number))), - block.header.hash_slow(), - )? -======= - let mut era_manager = EraManager::new(first_block_hash_to_add).await?; let txn = self.evm_db.db.begin_write()?; { let mut table = txn.open_table(BLOCK_HASHES_TABLE)?; - while era_manager.next_block_number() <= block_number { - let block = era_manager.get_next_block().await?; + while e2hs_manager.next_block_number() <= block_number { + let block = e2hs_manager.get_next_block().await?; table.insert( keccak256(B256::from(U256::from(block.header.number))).as_slice(), block.header.hash_slow().as_slice(), )?; } ->>>>>>> 7c448cc7 (Replace RocksDB with Redb as backing for EVM database) } txn.commit()?;