From 1635231d12fffa5a8ce9291d7cc57da0568cd50b Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 24 Jun 2025 11:59:47 +0800 Subject: [PATCH 01/36] feat: proper ledger metric counters --- magicblock-api/src/tickers.rs | 31 -------- magicblock-metrics/src/metrics/mod.rs | 110 +++++++++++++------------- 2 files changed, 55 insertions(+), 86 deletions(-) diff --git a/magicblock-api/src/tickers.rs b/magicblock-api/src/tickers.rs index 6a951fae6..6054048fd 100644 --- a/magicblock-api/src/tickers.rs +++ b/magicblock-api/src/tickers.rs @@ -117,36 +117,6 @@ pub fn init_system_metrics_ticker( bank: &Arc, token: CancellationToken, ) -> tokio::task::JoinHandle<()> { - fn try_set_ledger_counts(ledger: &Ledger) { - macro_rules! try_set_ledger_count { - ($name:ident) => { - paste::paste! { - match ledger.[< count_ $name >]() { - Ok(count) => { - metrics::[< set_ledger_ $name _count >](count); - } - Err(err) => warn!( - "Failed to get ledger {} count: {:?}", - stringify!($name), - err - ), - } - } - }; - } - try_set_ledger_count!(block_times); - try_set_ledger_count!(blockhashes); - try_set_ledger_count!(slot_signatures); - try_set_ledger_count!(address_signatures); - try_set_ledger_count!(transaction_status); - try_set_ledger_count!(transaction_successful_status); - try_set_ledger_count!(transaction_failed_status); - try_set_ledger_count!(transactions); - try_set_ledger_count!(transaction_memos); - try_set_ledger_count!(perf_samples); - try_set_ledger_count!(account_mod_data); - } - fn try_set_ledger_storage_size(ledger: &Ledger) { match ledger.storage_size() { Ok(byte_size) => metrics::set_ledger_size(byte_size), @@ -169,7 +139,6 @@ pub fn init_system_metrics_ticker( _ = tokio::time::sleep(tick_duration) => { try_set_ledger_storage_size(&ledger); set_accounts_storage_size(&bank); - try_set_ledger_counts(&ledger); set_accounts_count(&bank); }, _ = token.cancelled() => { diff --git a/magicblock-metrics/src/metrics/mod.rs b/magicblock-metrics/src/metrics/mod.rs index 71efb02d1..cf9130958 100644 --- a/magicblock-metrics/src/metrics/mod.rs +++ b/magicblock-metrics/src/metrics/mod.rs @@ -87,38 +87,38 @@ lazy_static::lazy_static! { static ref LEDGER_SIZE_GAUGE: IntGauge = IntGauge::new( "ledger_size", "Ledger size in Bytes", ).unwrap(); - static ref LEDGER_BLOCK_TIMES_GAUGE: IntGauge = IntGauge::new( - "ledger_blocktimes_gauge", "Ledger Blocktimes Gauge", + static ref LEDGER_BLOCK_TIMES_COUNT: IntCounter = IntCounter::new( + "ledger_blocktimes_count", "Ledger Blocktimes Count", ).unwrap(); - static ref LEDGER_BLOCKHASHES_GAUGE: IntGauge = IntGauge::new( - "ledger_blockhashes_gauge", "Ledger Blockhashes Gauge", + static ref LEDGER_BLOCKHASHES_COUNT: IntCounter = IntCounter::new( + "ledger_blockhashes_count", "Ledger Blockhashes Count", ).unwrap(); - static ref LEDGER_SLOT_SIGNATURES_GAUGE: IntGauge = IntGauge::new( - "ledger_slot_signatures_gauge", "Ledger Slot Signatures Gauge", + static ref LEDGER_SLOT_SIGNATURES_COUNT: IntCounter = IntCounter::new( + "ledger_slot_signatures_count", "Ledger Slot Signatures Count", ).unwrap(); - static ref LEDGER_ADDRESS_SIGNATURES_GAUGE: IntGauge = IntGauge::new( - "ledger_address_signatures_gauge", "Ledger Address Signatures Gauge", + static ref LEDGER_ADDRESS_SIGNATURES_COUNT: IntCounter = IntCounter::new( + "ledger_address_signatures_count", "Ledger Address Signatures Count", ).unwrap(); - static ref LEDGER_TRANSACTION_STATUS_GAUGE: IntGauge = IntGauge::new( - "ledger_transaction_status_gauge", "Ledger Transaction Status Gauge", + static ref LEDGER_TRANSACTION_STATUS_COUNT: IntCounter = IntCounter::new( + "ledger_transaction_status_count", "Ledger Transaction Status Count", ).unwrap(); - static ref LEDGER_TRANSACTION_SUCCESSFUL_STATUS_GAUGE: IntGauge = IntGauge::new( - "ledger_transaction_successful_status_gauge", "Ledger Successful Transaction Status Gauge", + static ref LEDGER_TRANSACTION_SUCCESSFUL_STATUS_COUNT: IntCounter = IntCounter::new( + "ledger_transaction_successful_status_count", "Ledger Successful Transaction Status Count", ).unwrap(); - static ref LEDGER_TRANSACTION_FAILED_STATUS_GAUGE: IntGauge = IntGauge::new( - "ledger_transaction_failed_status_gauge", "Ledger Failed Transaction Status Gauge", + static ref LEDGER_TRANSACTION_FAILED_STATUS_COUNT: IntCounter = IntCounter::new( + "ledger_transaction_failed_status_count", "Ledger Failed Transaction Status Count", ).unwrap(); - static ref LEDGER_TRANSACTIONS_GAUGE: IntGauge = IntGauge::new( - "ledger_transactions_gauge", "Ledger Transactions Gauge", + static ref LEDGER_TRANSACTIONS_COUNT: IntCounter = IntCounter::new( + "ledger_transactions_count", "Ledger Transactions Count", ).unwrap(); - static ref LEDGER_TRANSACTION_MEMOS_GAUGE: IntGauge = IntGauge::new( - "ledger_transaction_memos_gauge", "Ledger Transaction Memos Gauge", + static ref LEDGER_TRANSACTION_MEMOS_COUNT: IntCounter = IntCounter::new( + "ledger_transaction_memos_count", "Ledger Transaction Memos Count", ).unwrap(); - static ref LEDGER_PERF_SAMPLES_GAUGE: IntGauge = IntGauge::new( - "ledger_perf_samples_gauge", "Ledger Perf Samples Gauge", + static ref LEDGER_PERF_SAMPLES_COUNT: IntCounter = IntCounter::new( + "ledger_perf_samples_count", "Ledger Perf Samples Count", ).unwrap(); - static ref LEDGER_ACCOUNT_MOD_DATA_GAUGE: IntGauge = IntGauge::new( - "ledger_account_mod_data_gauge", "Ledger Account Mod Data Gauge", + static ref LEDGER_ACCOUNT_MOD_DATA_COUNT: IntCounter = IntCounter::new( + "ledger_account_mod_data_count", "Ledger Account Mod Data Count", ).unwrap(); // ----------------- @@ -221,17 +221,17 @@ pub(crate) fn register() { register!(ACCOUNT_COMMIT_TIME_HISTOGRAM); register!(CACHED_CLONE_OUTPUTS_COUNT); register!(LEDGER_SIZE_GAUGE); - register!(LEDGER_BLOCK_TIMES_GAUGE); - register!(LEDGER_BLOCKHASHES_GAUGE); - register!(LEDGER_SLOT_SIGNATURES_GAUGE); - register!(LEDGER_ADDRESS_SIGNATURES_GAUGE); - register!(LEDGER_TRANSACTION_STATUS_GAUGE); - register!(LEDGER_TRANSACTION_SUCCESSFUL_STATUS_GAUGE); - register!(LEDGER_TRANSACTION_FAILED_STATUS_GAUGE); - register!(LEDGER_TRANSACTIONS_GAUGE); - register!(LEDGER_TRANSACTION_MEMOS_GAUGE); - register!(LEDGER_PERF_SAMPLES_GAUGE); - register!(LEDGER_ACCOUNT_MOD_DATA_GAUGE); + register!(LEDGER_BLOCK_TIMES_COUNT); + register!(LEDGER_BLOCKHASHES_COUNT); + register!(LEDGER_SLOT_SIGNATURES_COUNT); + register!(LEDGER_ADDRESS_SIGNATURES_COUNT); + register!(LEDGER_TRANSACTION_STATUS_COUNT); + register!(LEDGER_TRANSACTION_SUCCESSFUL_STATUS_COUNT); + register!(LEDGER_TRANSACTION_FAILED_STATUS_COUNT); + register!(LEDGER_TRANSACTIONS_COUNT); + register!(LEDGER_TRANSACTION_MEMOS_COUNT); + register!(LEDGER_PERF_SAMPLES_COUNT); + register!(LEDGER_ACCOUNT_MOD_DATA_COUNT); register!(ACCOUNTS_SIZE_GAUGE); register!(ACCOUNTS_COUNT_GAUGE); register!(INMEM_ACCOUNTS_SIZE_GAUGE); @@ -343,48 +343,48 @@ pub fn set_ledger_size(size: u64) { LEDGER_SIZE_GAUGE.set(size as i64); } -pub fn set_ledger_block_times_count(count: i64) { - LEDGER_BLOCK_TIMES_GAUGE.set(count); +pub fn inc_ledger_block_times_count() { + LEDGER_BLOCK_TIMES_COUNT.inc(); } -pub fn set_ledger_blockhashes_count(count: i64) { - LEDGER_BLOCKHASHES_GAUGE.set(count); +pub fn inc_ledger_blockhashes_count() { + LEDGER_BLOCKHASHES_COUNT.inc(); } -pub fn set_ledger_slot_signatures_count(count: i64) { - LEDGER_SLOT_SIGNATURES_GAUGE.set(count); +pub fn inc_ledger_slot_signatures_count() { + LEDGER_SLOT_SIGNATURES_COUNT.inc(); } -pub fn set_ledger_address_signatures_count(count: i64) { - LEDGER_ADDRESS_SIGNATURES_GAUGE.set(count); +pub fn inc_ledger_address_signatures_count() { + LEDGER_ADDRESS_SIGNATURES_COUNT.inc(); } -pub fn set_ledger_transaction_status_count(count: i64) { - LEDGER_TRANSACTION_STATUS_GAUGE.set(count); +pub fn inc_ledger_transaction_status_count() { + LEDGER_TRANSACTION_STATUS_COUNT.inc(); } -pub fn set_ledger_transaction_successful_status_count(count: i64) { - LEDGER_TRANSACTION_SUCCESSFUL_STATUS_GAUGE.set(count); +pub fn inc_ledger_transaction_successful_status_count() { + LEDGER_TRANSACTION_SUCCESSFUL_STATUS_COUNT.inc(); } -pub fn set_ledger_transaction_failed_status_count(count: i64) { - LEDGER_TRANSACTION_FAILED_STATUS_GAUGE.set(count); +pub fn inc_ledger_transaction_failed_status_count() { + LEDGER_TRANSACTION_FAILED_STATUS_COUNT.inc(); } -pub fn set_ledger_transactions_count(count: i64) { - LEDGER_TRANSACTIONS_GAUGE.set(count); +pub fn inc_ledger_transactions_count() { + LEDGER_TRANSACTIONS_COUNT.inc(); } -pub fn set_ledger_transaction_memos_count(count: i64) { - LEDGER_TRANSACTION_MEMOS_GAUGE.set(count); +pub fn inc_ledger_transaction_memos_count() { + LEDGER_TRANSACTION_MEMOS_COUNT.inc(); } -pub fn set_ledger_perf_samples_count(count: i64) { - LEDGER_PERF_SAMPLES_GAUGE.set(count); +pub fn inc_ledger_perf_samples_count() { + LEDGER_PERF_SAMPLES_COUNT.inc(); } -pub fn set_ledger_account_mod_data_count(count: i64) { - LEDGER_ACCOUNT_MOD_DATA_GAUGE.set(count); +pub fn inc_ledger_account_mod_data_count(count: u64) { + LEDGER_ACCOUNT_MOD_DATA_COUNT.inc_by(count); } pub fn set_accounts_size(size: u64) { From 430670417e49305acccb7c57685cf8921cab0210 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 24 Jun 2025 12:00:14 +0800 Subject: [PATCH 02/36] feat: increase metrics directly from ledger --- Cargo.lock | 1 + magicblock-ledger/Cargo.toml | 1 + magicblock-ledger/src/store/api.rs | 29 +++++++++++++++++++++++++++++ test-integration/Cargo.lock | 1 + 4 files changed, 32 insertions(+) diff --git a/Cargo.lock b/Cargo.lock index 30dedaa99..dc53d06c8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3803,6 +3803,7 @@ dependencies = [ "magicblock-accounts-db", "magicblock-bank", "magicblock-core", + "magicblock-metrics", "num-format", "num_cpus", "prost 0.11.9", diff --git a/magicblock-ledger/Cargo.toml b/magicblock-ledger/Cargo.toml index aa29c3c40..7e48bc7d8 100644 --- a/magicblock-ledger/Cargo.toml +++ b/magicblock-ledger/Cargo.toml @@ -20,6 +20,7 @@ serde = { workspace = true } magicblock-bank = { workspace = true } magicblock-accounts-db = { workspace = true } magicblock-core = { workspace = true } +magicblock-metrics = { workspace = true } solana-account-decoder = { workspace = true } solana-measure = { workspace = true } solana-metrics = { workspace = true } diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 697d17911..3dcc48646 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -10,6 +10,7 @@ use std::{ use bincode::{deserialize, serialize}; use log::*; +use magicblock_metrics::metrics; use rocksdb::{Direction as IteratorDirection, FlushOptions}; use solana_measure::measure::Measure; use solana_sdk::{ @@ -313,9 +314,13 @@ impl Ledger { blockhash: Hash, ) -> LedgerResult<()> { self.blocktime_cf.put(slot, ×tamp)?; + metrics::inc_ledger_block_times_count(); + // TODO: @@@ do we need this? self.blocktime_cf.try_increase_entry_counter(1); self.blockhash_cf.put(slot, &blockhash)?; + metrics::inc_ledger_blockhashes_count(); + // TODO: @@@ do we need this? self.blockhash_cf.try_increase_entry_counter(1); Ok(()) } @@ -825,6 +830,9 @@ impl Ledger { self.transaction_cf .put_protobuf((signature, slot), &transaction)?; + metrics::inc_ledger_transactions_count(); + + // TODO: @@@ do we need this? self.transaction_cf.try_increase_entry_counter(1); Ok(()) @@ -864,6 +872,9 @@ impl Ledger { memos: String, ) -> LedgerResult<()> { let res = self.transaction_memos_cf.put((*signature, slot), &memos); + metrics::inc_ledger_transaction_memos_count(); + + // TODO: @@@ do we need this? self.transaction_memos_cf.try_increase_entry_counter(1); res } @@ -948,6 +959,8 @@ impl Ledger { (*address, slot, transaction_slot_index, signature), &AddressSignatureMeta { writeable: true }, )?; + metrics::inc_ledger_address_signatures_count(); + // TODO: @@@ do we need this? self.address_signatures_cf.try_increase_entry_counter(1); } for address in readonly_keys { @@ -955,24 +968,37 @@ impl Ledger { (*address, slot, transaction_slot_index, signature), &AddressSignatureMeta { writeable: false }, )?; + metrics::inc_ledger_address_signatures_count(); + // TODO: @@@ do we need this? self.address_signatures_cf.try_increase_entry_counter(1); } self.slot_signatures_cf .put((slot, transaction_slot_index), &signature)?; + metrics::inc_ledger_slot_signatures_count(); + // TODO: @@@ do we need this? self.slot_signatures_cf.try_increase_entry_counter(1); let status = status.into(); self.transaction_status_cf .put_protobuf((signature, slot), &status)?; + metrics::inc_ledger_transaction_status_count(); + + // TODO: @@@ do we need this? self.transaction_status_cf.try_increase_entry_counter(1); if status.err.is_none() { + metrics::inc_ledger_transaction_successful_status_count(); + + // TODO: @@@ do we need this? try_increase_entry_counter( &self.transaction_successful_status_count, 1, ); } else { + metrics::inc_ledger_transaction_failed_status_count(); + + // TODO: @@@ do we need this? try_increase_entry_counter( &self.transaction_failed_status_count, 1, @@ -1102,6 +1128,9 @@ impl Ledger { let bytes = serialize(perf_sample) .expect("`PerfSample` can be serialized with `bincode`"); self.perf_samples_cf.put_bytes(index, &bytes)?; + metrics::inc_ledger_perf_samples_count(); + + // TODO: @@@ do we need this? self.perf_samples_cf.try_increase_entry_counter(1); Ok(()) diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 6805c6095..1db01db7c 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3723,6 +3723,7 @@ dependencies = [ "magicblock-accounts-db", "magicblock-bank", "magicblock-core", + "magicblock-metrics", "num-format", "num_cpus", "prost", From 822349abc623c51b05c92f9edbb73224ec82ce0b Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 24 Jun 2025 12:11:48 +0800 Subject: [PATCH 03/36] chore: remove high overhead ledger counters --- .../src/database/ledger_column.rs | 81 +------------- magicblock-ledger/src/store/api.rs | 104 ++---------------- 2 files changed, 8 insertions(+), 177 deletions(-) diff --git a/magicblock-ledger/src/database/ledger_column.rs b/magicblock-ledger/src/database/ledger_column.rs index 4061a7728..16388ccb3 100644 --- a/magicblock-ledger/src/database/ledger_column.rs +++ b/magicblock-ledger/src/database/ledger_column.rs @@ -7,7 +7,7 @@ use std::{ }; use bincode::{deserialize, serialize}; -use log::{error, warn}; +use log::error; use prost::Message; use rocksdb::{properties as RocksProperties, ColumnFamily}; use serde::de::DeserializeOwned; @@ -295,20 +295,6 @@ where } else { val as i64 }) .inspect(|updated| self.entry_counter.store(*updated, Ordering::Relaxed)) } - - /// Increases entries counter if it's not [`DIRTY_COUNT`] - /// Otherwise just skips it until it is set - #[inline(always)] - pub fn try_increase_entry_counter(&self, by: u64) { - try_increase_entry_counter(&self.entry_counter, by); - } - - /// Decreases entries counter if it's not [`DIRTY_COUNT`] - /// Otherwise just skips it until it is set - #[inline(always)] - pub fn try_decrease_entry_counter(&self, by: u64) { - try_decrease_entry_counter(&self.entry_counter, by); - } } impl LedgerColumn @@ -539,68 +525,3 @@ where }) } } - -/// Increases entries counter if it's not [`DIRTY_COUNT`] -/// Otherwise just skips it until it is set -pub fn try_increase_entry_counter(entry_counter: &AtomicI64, by: u64) { - loop { - let prev = entry_counter.load(Ordering::Acquire); - if prev == DIRTY_COUNT { - return; - } - - // In case value changed to [`DIRTY_COUNT`] in between - if entry_counter - .compare_exchange( - prev, - prev + by as i64, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .is_ok() - { - return; - } - } -} - -/// Decreases entries counter if it's not [`DIRTY_COUNT`] -/// Otherwise just skips it until it is set -pub fn try_decrease_entry_counter(entry_counter: &AtomicI64, by: u64) { - loop { - let prev = entry_counter.load(Ordering::Acquire); - if prev == DIRTY_COUNT { - return; - } - - let new = prev - by as i64; - if new >= 0 { - // In case value changed to [`DIRTY_COUNT`] in between - if entry_counter - .compare_exchange( - prev, - new, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .is_ok() - { - return; - } - } else { - warn!("Negative entry counter!"); - // In case value fixed to valid one in between - if entry_counter - .compare_exchange( - prev, - DIRTY_COUNT, - Ordering::AcqRel, - Ordering::Relaxed, - ) - .is_ok() - { - return; - } - } - } -} diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 3dcc48646..09db930f0 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -2,10 +2,7 @@ use std::{ collections::HashMap, fmt, fs, path::{Path, PathBuf}, - sync::{ - atomic::{AtomicI64, Ordering}, - Arc, RwLock, - }, + sync::{atomic::Ordering, Arc, RwLock}, }; use bincode::{deserialize, serialize}; @@ -31,10 +28,10 @@ use crate::{ conversions::transaction, database::{ columns as cf, - columns::{Column, ColumnName, DIRTY_COUNT}, + columns::{Column, ColumnName}, db::Database, iterator::IteratorMode, - ledger_column::{try_increase_entry_counter, LedgerColumn}, + ledger_column::LedgerColumn, meta::{AccountModData, AddressSignatureMeta, PerfSample}, options::LedgerOptions, }, @@ -64,9 +61,6 @@ pub struct Ledger { perf_samples_cf: LedgerColumn, account_mod_datas_cf: LedgerColumn, - transaction_successful_status_count: AtomicI64, - transaction_failed_status_count: AtomicI64, - lowest_cleanup_slot: RwLock, rpc_api_metrics: LedgerRpcApiMetrics, } @@ -159,9 +153,6 @@ impl Ledger { perf_samples_cf, account_mod_datas_cf, - transaction_successful_status_count: AtomicI64::new(DIRTY_COUNT), - transaction_failed_status_count: AtomicI64::new(DIRTY_COUNT), - lowest_cleanup_slot: RwLock::::default(), rpc_api_metrics: LedgerRpcApiMetrics::default(), }; @@ -315,13 +306,9 @@ impl Ledger { ) -> LedgerResult<()> { self.blocktime_cf.put(slot, ×tamp)?; metrics::inc_ledger_block_times_count(); - // TODO: @@@ do we need this? - self.blocktime_cf.try_increase_entry_counter(1); self.blockhash_cf.put(slot, &blockhash)?; metrics::inc_ledger_blockhashes_count(); - // TODO: @@@ do we need this? - self.blockhash_cf.try_increase_entry_counter(1); Ok(()) } @@ -832,9 +819,6 @@ impl Ledger { .put_protobuf((signature, slot), &transaction)?; metrics::inc_ledger_transactions_count(); - // TODO: @@@ do we need this? - self.transaction_cf.try_increase_entry_counter(1); - Ok(()) } @@ -873,9 +857,6 @@ impl Ledger { ) -> LedgerResult<()> { let res = self.transaction_memos_cf.put((*signature, slot), &memos); metrics::inc_ledger_transaction_memos_count(); - - // TODO: @@@ do we need this? - self.transaction_memos_cf.try_increase_entry_counter(1); res } @@ -960,8 +941,6 @@ impl Ledger { &AddressSignatureMeta { writeable: true }, )?; metrics::inc_ledger_address_signatures_count(); - // TODO: @@@ do we need this? - self.address_signatures_cf.try_increase_entry_counter(1); } for address in readonly_keys { self.address_signatures_cf.put( @@ -969,40 +948,21 @@ impl Ledger { &AddressSignatureMeta { writeable: false }, )?; metrics::inc_ledger_address_signatures_count(); - // TODO: @@@ do we need this? - self.address_signatures_cf.try_increase_entry_counter(1); } self.slot_signatures_cf .put((slot, transaction_slot_index), &signature)?; metrics::inc_ledger_slot_signatures_count(); - // TODO: @@@ do we need this? - self.slot_signatures_cf.try_increase_entry_counter(1); let status = status.into(); self.transaction_status_cf .put_protobuf((signature, slot), &status)?; metrics::inc_ledger_transaction_status_count(); - // TODO: @@@ do we need this? - self.transaction_status_cf.try_increase_entry_counter(1); - if status.err.is_none() { metrics::inc_ledger_transaction_successful_status_count(); - - // TODO: @@@ do we need this? - try_increase_entry_counter( - &self.transaction_successful_status_count, - 1, - ); } else { metrics::inc_ledger_transaction_failed_status_count(); - - // TODO: @@@ do we need this? - try_increase_entry_counter( - &self.transaction_failed_status_count, - 1, - ); } Ok(()) @@ -1069,34 +1029,13 @@ impl Ledger { } pub fn count_transaction_successful_status(&self) -> LedgerResult { - if self - .transaction_status_cf - .entry_counter - .load(Ordering::Relaxed) - == DIRTY_COUNT - { - let count = self.count_outcome_transaction_status(true)?; - self.transaction_successful_status_count - .store(count, Ordering::Relaxed); - Ok(count) - } else { - Ok(self - .transaction_successful_status_count - .load(Ordering::Relaxed)) - } + let count = self.count_outcome_transaction_status(true)?; + Ok(count) } pub fn count_transaction_failed_status(&self) -> LedgerResult { - if self.transaction_failed_status_count.load(Ordering::Relaxed) - == DIRTY_COUNT - { - let count = self.count_outcome_transaction_status(false)?; - self.transaction_failed_status_count - .store(count, Ordering::Relaxed); - Ok(count) - } else { - Ok(self.transaction_failed_status_count.load(Ordering::Relaxed)) - } + let count = self.count_outcome_transaction_status(false)?; + Ok(count) } // ----------------- @@ -1130,9 +1069,6 @@ impl Ledger { self.perf_samples_cf.put_bytes(index, &bytes)?; metrics::inc_ledger_perf_samples_count(); - // TODO: @@@ do we need this? - self.perf_samples_cf.try_increase_entry_counter(1); - Ok(()) } @@ -1149,7 +1085,6 @@ impl Ledger { data: &AccountModData, ) -> LedgerResult<()> { self.account_mod_datas_cf.put(id, data)?; - self.account_mod_datas_cf.try_increase_entry_counter(1); Ok(()) } @@ -1189,7 +1124,6 @@ impl Ledger { .expect(Self::LOWEST_CLEANUP_SLOT_POISONED); *lowest_cleanup_slot = std::cmp::max(*lowest_cleanup_slot, to_slot); - let num_deleted_slots = to_slot + 1 - from_slot; self.blocktime_cf.delete_range_in_batch( &mut batch, from_slot, @@ -1257,30 +1191,6 @@ impl Ledger { self.db.write(batch)?; - self.blocktime_cf - .try_decrease_entry_counter(num_deleted_slots); - self.blockhash_cf - .try_decrease_entry_counter(num_deleted_slots); - self.perf_samples_cf - .try_decrease_entry_counter(num_deleted_slots); - self.slot_signatures_cf - .try_decrease_entry_counter(slot_signatures_deleted); - self.transaction_status_cf - .try_decrease_entry_counter(transaction_status_deleted); - self.transaction_cf - .try_decrease_entry_counter(transactions_deleted); - self.transaction_memos_cf - .try_decrease_entry_counter(transaction_memos_deleted); - self.address_signatures_cf - .try_decrease_entry_counter(address_signatures_deleted); - - // To not spend time querying DB for value we set drop the counter - // This shouldn't happen very often due to rarity of actual truncations. - self.transaction_successful_status_count - .store(DIRTY_COUNT, Ordering::Release); - self.transaction_failed_status_count - .store(DIRTY_COUNT, Ordering::Release); - Ok(()) } From 4e2cb81145fbe81e173321879387453c87bcd9e8 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 24 Jun 2025 12:54:55 +0800 Subject: [PATCH 04/36] chore: properly cleanout remaining ledger code relying on cached counts --- magicblock-ledger/src/database/columns.rs | 5 ---- magicblock-ledger/src/database/db.rs | 12 ++------- .../src/database/ledger_column.rs | 25 ++----------------- 3 files changed, 4 insertions(+), 38 deletions(-) diff --git a/magicblock-ledger/src/database/columns.rs b/magicblock-ledger/src/database/columns.rs index 42bfb10b5..399b0607c 100644 --- a/magicblock-ledger/src/database/columns.rs +++ b/magicblock-ledger/src/database/columns.rs @@ -665,8 +665,3 @@ impl TypedColumn for AccountModDatas { pub fn should_enable_compression() -> bool { C::NAME == TransactionStatus::NAME } - -// ----------------- -// Column Queries -// ----------------- -pub(crate) const DIRTY_COUNT: i64 = -1; diff --git a/magicblock-ledger/src/database/db.rs b/magicblock-ledger/src/database/db.rs index 44042a16c..e607d4d70 100644 --- a/magicblock-ledger/src/database/db.rs +++ b/magicblock-ledger/src/database/db.rs @@ -1,8 +1,4 @@ -use std::{ - marker::PhantomData, - path::Path, - sync::{atomic::AtomicI64, Arc}, -}; +use std::{marker::PhantomData, path::Path, sync::Arc}; use bincode::deserialize; use rocksdb::{ColumnFamily, DBRawIterator, LiveFile}; @@ -16,10 +12,7 @@ use super::{ rocks_db::Rocks, write_batch::WriteBatch, }; -use crate::{ - database::columns::DIRTY_COUNT, errors::LedgerError, - metrics::PerfSamplingStatus, -}; +use crate::{errors::LedgerError, metrics::PerfSamplingStatus}; #[derive(Debug)] pub struct Database { @@ -97,7 +90,6 @@ impl Database { column_options: Arc::clone(&self.column_options), read_perf_status: PerfSamplingStatus::default(), write_perf_status: PerfSamplingStatus::default(), - entry_counter: AtomicI64::new(DIRTY_COUNT), } } diff --git a/magicblock-ledger/src/database/ledger_column.rs b/magicblock-ledger/src/database/ledger_column.rs index 16388ccb3..9b22bf195 100644 --- a/magicblock-ledger/src/database/ledger_column.rs +++ b/magicblock-ledger/src/database/ledger_column.rs @@ -1,10 +1,4 @@ -use std::{ - marker::PhantomData, - sync::{ - atomic::{AtomicI64, Ordering}, - Arc, - }, -}; +use std::{marker::PhantomData, sync::Arc}; use bincode::{deserialize, serialize}; use log::error; @@ -21,7 +15,7 @@ use super::{ rocks_db::Rocks, }; use crate::{ - database::{columns::DIRTY_COUNT, write_batch::WriteBatch}, + database::write_batch::WriteBatch, errors::{LedgerError, LedgerResult}, metrics::{ maybe_enable_rocksdb_perf, report_rocksdb_read_perf, @@ -41,15 +35,6 @@ where pub column_options: Arc, pub read_perf_status: PerfSamplingStatus, pub write_perf_status: PerfSamplingStatus, - // We are caching the column item counts since they are expensive to obtain. - // `-1` indicates that they are "dirty" // - // // We are using an i64 to make this work even though the counts are usize, - // // however if we had 50,000 transactions/sec and 50ms slots for 100 years then: - // // - // // slots: 200 * 3600 * 24 * 365 * 100 = 630,720,000,000 - // // txs: 50,000 * 3600 * 24 * 365 * 100 = 157,680,000,000,000 - // // i64::MAX = 9,223,372,036,854,775,807 - pub entry_counter: AtomicI64, } impl LedgerColumn { @@ -278,11 +263,6 @@ where } pub fn count_column_using_cache(&self) -> LedgerResult { - let cached = self.entry_counter.load(Ordering::Relaxed); - if cached != DIRTY_COUNT { - return Ok(cached); - } - self .iter(IteratorMode::Start) .map(Iterator::count) @@ -293,7 +273,6 @@ where error!("Column {} count is too large: {} for metrics, returning max.", C::NAME, val); i64::MAX } else { val as i64 }) - .inspect(|updated| self.entry_counter.store(*updated, Ordering::Relaxed)) } } From 01fea1b6211c41e5b386d0422d78a910d315887f Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 24 Jun 2025 19:01:51 +0800 Subject: [PATCH 05/36] feat: set ledger metrics when we get size and remove from ticker --- magicblock-api/src/magic_validator.rs | 1 - magicblock-api/src/tickers.rs | 9 --------- magicblock-ledger/src/database/db.rs | 5 ++++- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 169ce0cdb..2ec9ec010 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -250,7 +250,6 @@ impl MagicValidator { Duration::from_secs( metrics_config.system_metrics_tick_interval_secs, ), - &ledger, &bank, token.clone(), ); diff --git a/magicblock-api/src/tickers.rs b/magicblock-api/src/tickers.rs index 6054048fd..4a64a0427 100644 --- a/magicblock-api/src/tickers.rs +++ b/magicblock-api/src/tickers.rs @@ -113,16 +113,9 @@ pub fn init_commit_accounts_ticker( pub fn init_system_metrics_ticker( tick_duration: Duration, - ledger: &Arc, bank: &Arc, token: CancellationToken, ) -> tokio::task::JoinHandle<()> { - fn try_set_ledger_storage_size(ledger: &Ledger) { - match ledger.storage_size() { - Ok(byte_size) => metrics::set_ledger_size(byte_size), - Err(err) => warn!("Failed to get ledger storage size: {:?}", err), - } - } fn set_accounts_storage_size(bank: &Bank) { let byte_size = bank.accounts_db_storage_size(); metrics::set_accounts_size(byte_size); @@ -131,13 +124,11 @@ pub fn init_system_metrics_ticker( metrics::set_accounts_count(bank.accounts_db.get_accounts_count()); } - let ledger = ledger.clone(); let bank = bank.clone(); tokio::task::spawn(async move { loop { tokio::select! { _ = tokio::time::sleep(tick_duration) => { - try_set_ledger_storage_size(&ledger); set_accounts_storage_size(&bank); set_accounts_count(&bank); }, diff --git a/magicblock-ledger/src/database/db.rs b/magicblock-ledger/src/database/db.rs index e607d4d70..9d8686c02 100644 --- a/magicblock-ledger/src/database/db.rs +++ b/magicblock-ledger/src/database/db.rs @@ -1,6 +1,7 @@ use std::{marker::PhantomData, path::Path, sync::Arc}; use bincode::deserialize; +use magicblock_metrics::metrics; use rocksdb::{ColumnFamily, DBRawIterator, LiveFile}; use solana_sdk::clock::Slot; @@ -113,7 +114,9 @@ impl Database { } pub fn storage_size(&self) -> Result { - Ok(fs_extra::dir::get_size(&self.path)?) + let size = fs_extra::dir::get_size(&self.path)?; + metrics::set_ledger_size(size); + Ok(size) } /// Adds a \[`from`, `to`\] range that deletes all entries between the `from` slot From 88fbb9816b8f057972500ab440faa6c45622099f Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 25 Jun 2025 17:22:00 +0800 Subject: [PATCH 06/36] feat: initial watermark based ledger size manager --- magicblock-ledger/src/database/db.rs | 2 - magicblock-ledger/src/ledger_size_manager.rs | 432 +++++++++++++++++++ magicblock-ledger/src/lib.rs | 1 + magicblock-ledger/src/store/api.rs | 21 +- 4 files changed, 453 insertions(+), 3 deletions(-) create mode 100644 magicblock-ledger/src/ledger_size_manager.rs diff --git a/magicblock-ledger/src/database/db.rs b/magicblock-ledger/src/database/db.rs index 9d8686c02..bffa37588 100644 --- a/magicblock-ledger/src/database/db.rs +++ b/magicblock-ledger/src/database/db.rs @@ -1,7 +1,6 @@ use std::{marker::PhantomData, path::Path, sync::Arc}; use bincode::deserialize; -use magicblock_metrics::metrics; use rocksdb::{ColumnFamily, DBRawIterator, LiveFile}; use solana_sdk::clock::Slot; @@ -115,7 +114,6 @@ impl Database { pub fn storage_size(&self) -> Result { let size = fs_extra::dir::get_size(&self.path)?; - metrics::set_ledger_size(size); Ok(size) } diff --git a/magicblock-ledger/src/ledger_size_manager.rs b/magicblock-ledger/src/ledger_size_manager.rs new file mode 100644 index 000000000..f304f301d --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager.rs @@ -0,0 +1,432 @@ +#![allow(unused)] +use log::*; +use std::{collections::VecDeque, sync::Arc, thread::JoinHandle}; + +use magicblock_metrics::metrics; +use solana_sdk::clock::Slot; +use thiserror::Error; +use tokio::task::JoinError; +use tokio_util::sync::CancellationToken; + +use crate::Ledger; + +// ----------------- +// LedgerManagerError +// ----------------- +#[derive(Error, Debug)] +pub enum LedgerSizeManagerError { + #[error("Ledger needs to be provided to start the manager")] + LedgerNotProvided, + #[error("Failed to join worker: {0}")] + JoinError(#[from] JoinError), +} +pub type LedgerSizeManagerResult = Result; + +// ----------------- +// LedgerSizeManagerConfig +// ----------------- +/// Percentage of ledger to keep when resizing. +pub enum ResizePercentage { + /// Keep 75% of the ledger size. + Large, + /// Keep 66% of the ledger size. + Medium, + /// Keep 50% of the ledger size. + Small, +} + +impl ResizePercentage { + /// The portion of ledger we cut on each resize. + pub fn watermark_size_percent(&self) -> u64 { + use ResizePercentage::*; + match self { + Large => 25, + Medium => 34, + Small => 50, + } + } + + /// The number of watermarks to track + pub fn watermark_count(&self) -> u64 { + use ResizePercentage::*; + match self { + Large => 3, + Medium => 2, + Small => 1, + } + } +} + +pub struct LedgerSizeManagerConfig { + /// Max ledger size to maintain. + /// The [LedgerSizeManager] will attempt to respect this size,, but + /// it may grow larger temporarily in between size checks. + pub max_size: u64, + + /// Interval at which the size is checked in milliseconds. + pub size_check_interval_ms: u64, + + /// Percentage of the ledger to keep when resizing + pub resize_percentage: ResizePercentage, +} + +// ----------------- +// Watermarks +// ----------------- +#[derive(Debug, PartialEq, Eq)] +struct Watermark { + /// The slot at which this watermark was captured + slot: u64, + /// Account mod ID at which this watermark was captured + mod_id: u64, + /// The size of the ledger when this watermark was captured + size: u64, +} + +#[derive(Debug, PartialEq, Eq)] +struct Watermarks { + /// The watermarks captured + marks: VecDeque, + /// The maximum number of watermarks to keep + count: u64, + /// The targeted size difference for each watermark + mark_size: u64, + /// The maximum ledger size to maintain + max_ledger_size: u64, +} + +pub struct ExistingLedgerState { + /// The current size of the ledger + size: u64, + /// The last slot in the ledger at time of restart + slot: Slot, + /// The last account mod ID in the ledger at time of restart + mod_id: u64, +} + +impl Watermarks { + /// Creates a new set of watermarks based on the resize percentage and max ledger size. + /// - * `percentage`: The resize percentage to use. + /// - * `max_ledger_size`: The maximum size of the ledger to try to maintain. + /// - * `ledger_state`: The current ledger state which is + /// only available during restart with an existing ledger + fn new( + percentage: &ResizePercentage, + max_ledger_size: u64, + ledger_state: Option, + ) -> Self { + let count = percentage.watermark_count(); + let mut marks = VecDeque::with_capacity(count as usize); + let mark_size = + (max_ledger_size * percentage.watermark_size_percent()) / 100; + if let Some(ExistingLedgerState { size, slot, mod_id }) = ledger_state { + // Since we don't know the actual ledger sizes at each slot we must assume + // they were evenly distributed. + let mark_size_delta = size / count; + let mod_id_delta = mod_id / count; + let slot_delta = slot / count; + for i in 0..count { + let size = (i + 1) * mark_size_delta; + let mod_id = (i + 1) * mod_id_delta; + let slot = (i + 1) * slot_delta; + marks.push_back(Watermark { slot, mod_id, size }); + } + } + // In case we don't have an existing ledger state, we assume that the ledger size is + // still zero and we won't need any fabricated watermarks. + Watermarks { + marks, + count, + mark_size, + max_ledger_size, + } + } + + fn reached_max(&self, size: u64) -> bool { + size >= self.max_ledger_size + } + + fn update(&mut self, slot: u64, mod_id: u64, size: u64) { + // We try to record a watermark as close as possible (but below) the ideal + // watermark cutoff size. + let mark_idx = (size as f64 / self.mark_size as f64).ceil() as u64 - 1; + if mark_idx < self.count { + let watermark = Watermark { slot, mod_id, size }; + if let Some(mark) = self.marks.get_mut(mark_idx as usize) { + *mark = watermark; + } else { + self.marks.push_back(watermark); + } + } + } + + fn adjust_for_truncation(&mut self) { + // The sizes recorded at specific slots need to be adjusted since we truncated + // the slots before + + for mut mark in self.marks.iter_mut() { + mark.size = mark.size.saturating_sub(self.mark_size); + } + } + + fn consume_next(&mut self) -> Option { + self.marks.pop_front() + } +} + +// ----------------- +// LedgerSizeManager +// ----------------- +enum ServiceState { + Created, + Running { + cancellation_token: CancellationToken, + worker_handle: JoinHandle<()>, + }, + Stopped { + worker_handle: JoinHandle<()>, + }, +} + +pub struct LedgerSizeManager { + ledger: Option>, + watermarks: Watermarks, + servie_state: ServiceState, +} + +impl LedgerSizeManager { + pub(crate) fn new( + ledger: Option>, + ledger_state: Option, + config: LedgerSizeManagerConfig, + ) -> Self { + let watermarks = Watermarks::new( + &config.resize_percentage, + config.max_size, + ledger_state, + ); + LedgerSizeManager { + ledger, + watermarks, + servie_state: ServiceState::Created, + } + } + + fn ensure_initial_max_ledger_size() { + // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which + // includes a _fat_ ledger truncate + // We will run that first to get below the ledger max size _before_ switching to + // the watermark strategy. + } + + pub fn try_start(mut self) -> LedgerSizeManagerResult<()> { + // TODO: @@@ async loop + let Some(ledger) = self.ledger.take() else { + return Err(LedgerSizeManagerError::LedgerNotProvided); + }; + loop { + let Ok(ledger_size) = ledger.storage_size() else { + eprintln!( + "Failed to get ledger size, cannot start LedgerSizeManager" + ); + continue; + }; + metrics::set_ledger_size(ledger_size); + if let Some(mark) = self.get_truncation_mark( + ledger_size, + ledger.last_slot(), + ledger.last_mod_id(), + ) { + self.truncate(&ledger, mark); + } + } + } + + fn get_truncation_mark( + &mut self, + ledger_size: u64, + slot: Slot, + mod_id: u64, + ) -> Option { + if ledger_size == 0 { + return None; + } + if self.watermarks.reached_max(ledger_size) { + debug!( + "Ledger size {} reached maximum size {}, resizing...", + ledger_size, self.watermarks.max_ledger_size + ); + self.watermarks.consume_next() + } else { + self.watermarks.update(slot, mod_id, ledger_size); + None + } + } + + fn truncate(&self, ledger: &Arc, mark: Watermark) { + // This is where the truncation logic would go. + // For now, we just print the truncation mark. + debug!( + "Truncating ledger at slot {}, mod_id {}, size {}", + mark.slot, mark.mod_id, mark.size + ); + } +} + +#[cfg(test)] +mod tests { + use test_tools_core::init_logger; + + use super::*; + + macro_rules! mark { + ($slot:expr, $mod_id:expr, $size:expr) => {{ + Watermark { + slot: $slot, + mod_id: $mod_id, + size: $size, + } + }}; + ($idx:expr, $size:expr) => {{ + mark!($idx, $idx, $size) + }}; + } + + macro_rules! marks { + ($($slot:expr, $mod_id:expr, $size:expr);+) => {{ + let mut marks = VecDeque::::new(); + $( + marks.push_back(mark!($slot, $mod_id, $size)); + )+ + Watermarks { + marks, + count: 3, + mark_size: 250, + max_ledger_size: 1000, + } + }}; + } + + macro_rules! truncate_ledger { + ($slot:expr, $mod_id:expr, $sut:ident, $mark:expr, $size:ident) => {{ + // These steps are usually performed in _actual_ ledger truncate method + $size -= $mark.size; + $sut.watermarks.adjust_for_truncation(); + $sut.watermarks.update($slot, $mod_id, $size); + debug!( + "Truncated ledger to size {} -> {:#?}", + $size, $sut.watermarks + ); + }} + } + + #[test] + fn test_size_manager_new_ledger() { + init_logger!(); + + let percentage = ResizePercentage::Large; + const MAX_SIZE: u64 = 1_000; + const STEP_SIZE: u64 = MAX_SIZE / 20; + let mut sut = LedgerSizeManager::new( + None, + None, + LedgerSizeManagerConfig { + max_size: MAX_SIZE, + size_check_interval_ms: 1000, + resize_percentage: percentage, + }, + ); + + // 1. Go up to right below the ledger size + let mut size = 0; + for i in 0..19 { + size += STEP_SIZE; + let mark = sut.get_truncation_mark(size, i, i); + assert!( + mark.is_none(), + "Expected no truncation mark at size {}", + size + ); + } + + assert_eq!(sut.watermarks, marks!(4, 4, 250; 9, 9, 500; 14, 14, 750)); + + // 2. Hit ledger max size + size += STEP_SIZE; + let mark = sut.get_truncation_mark(size, 20, 20); + assert_eq!(sut.watermarks, marks!(9, 9, 500; 14, 14, 750)); + assert_eq!(mark, Some(mark!(4, 4, 250))); + + truncate_ledger!(20, 20, sut, mark.unwrap(), size); + assert_eq!(sut.watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); + + // 3. Go up to right below the next truncation mark (also ledger max size) + for i in 21..=24 { + size += STEP_SIZE; + let mark = sut.get_truncation_mark(size, i, i); + assert!( + mark.is_none(), + "Expected no truncation mark at size {}", + size + ); + } + assert_eq!(sut.watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); + + // 4. Hit next truncation mark (also ledger max size) + size += STEP_SIZE; + let mark = sut.get_truncation_mark(size, 25, 25); + assert_eq!(mark, Some(mark!(9, 9, 250))); + + truncate_ledger!(25, 25, sut, mark.unwrap(), size); + assert_eq!( + sut.watermarks, + marks!(14, 14, 250; 20, 20, 500; 25, 25, 750) + ); + + // 5. Go past 3 truncation marks + for i in 26..=40 { + size += STEP_SIZE; + let mark = sut.get_truncation_mark(size, i, i); + if mark.is_some() { + truncate_ledger!(i, i, sut, mark.unwrap(), size); + } + } + + assert_eq!( + sut.watermarks, + marks!(30, 30, 250; 35, 35, 500; 40, 40, 750) + ); + } + + #[test] + fn test_size_manager_existing_ledger() { + init_logger!(); + + let percentage = ResizePercentage::Large; + const MAX_SIZE: u64 = 1_000; + const STEP_SIZE: u64 = MAX_SIZE / 20; + let ledger_state = ExistingLedgerState { + // NOTE: that the watermarks will always be adjusted to have the size + // lower than the max size before we start using the watermark strategy. + // See [`ensure_initial_max_ledger_size`]. + size: 900, + slot: 150, + mod_id: 150, + }; + let mut sut = LedgerSizeManager::new( + None, + Some(ledger_state), + LedgerSizeManagerConfig { + max_size: MAX_SIZE, + size_check_interval_ms: 1000, + resize_percentage: percentage, + }, + ); + + // Initial watermarks should be based on the existing ledger state + assert_eq!( + sut.watermarks, + marks!(50, 50, 300; 100, 100, 600; 150, 150, 900) + ); + } +} diff --git a/magicblock-ledger/src/lib.rs b/magicblock-ledger/src/lib.rs index 110705c95..6f0b72ef4 100644 --- a/magicblock-ledger/src/lib.rs +++ b/magicblock-ledger/src/lib.rs @@ -2,6 +2,7 @@ pub mod blockstore_processor; mod conversions; mod database; pub mod errors; +pub mod ledger_size_manager; pub mod ledger_truncator; mod metrics; mod store; diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 09db930f0..85958413f 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -2,7 +2,10 @@ use std::{ collections::HashMap, fmt, fs, path::{Path, PathBuf}, - sync::{atomic::Ordering, Arc, RwLock}, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, RwLock, + }, }; use bincode::{deserialize, serialize}; @@ -63,6 +66,9 @@ pub struct Ledger { lowest_cleanup_slot: RwLock, rpc_api_metrics: LedgerRpcApiMetrics, + + last_slot: AtomicU64, + last_mod_id: AtomicU64, } impl fmt::Display for Ledger { @@ -155,11 +161,22 @@ impl Ledger { lowest_cleanup_slot: RwLock::::default(), rpc_api_metrics: LedgerRpcApiMetrics::default(), + + last_slot: AtomicU64::new(0), + last_mod_id: AtomicU64::new(0), }; Ok(ledger) } + pub fn last_slot(&self) -> Slot { + self.last_slot.load(Ordering::Relaxed) + } + + pub fn last_mod_id(&self) -> u64 { + self.last_mod_id.load(Ordering::Relaxed) + } + /// Collects and reports [`BlockstoreRocksDbColumnFamilyMetrics`] for /// all the column families. /// @@ -309,6 +326,7 @@ impl Ledger { self.blockhash_cf.put(slot, &blockhash)?; metrics::inc_ledger_blockhashes_count(); + self.last_slot.store(slot, Ordering::Relaxed); Ok(()) } @@ -1085,6 +1103,7 @@ impl Ledger { data: &AccountModData, ) -> LedgerResult<()> { self.account_mod_datas_cf.put(id, data)?; + self.last_mod_id.store(id, Ordering::Relaxed); Ok(()) } From f7e16bffa45cf9abbe61808c184deb5bd8623318 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 09:37:07 +0800 Subject: [PATCH 07/36] wip: need to separate a few things --- magicblock-ledger/src/ledger_size_manager.rs | 107 +++++++++++++++---- 1 file changed, 85 insertions(+), 22 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager.rs b/magicblock-ledger/src/ledger_size_manager.rs index f304f301d..c14c7d785 100644 --- a/magicblock-ledger/src/ledger_size_manager.rs +++ b/magicblock-ledger/src/ledger_size_manager.rs @@ -1,11 +1,14 @@ #![allow(unused)] use log::*; -use std::{collections::VecDeque, sync::Arc, thread::JoinHandle}; +use std::{collections::VecDeque, sync::Arc, time::Duration}; use magicblock_metrics::metrics; use solana_sdk::clock::Slot; use thiserror::Error; -use tokio::task::JoinError; +use tokio::{ + task::{JoinError, JoinHandle}, + time::interval, +}; use tokio_util::sync::CancellationToken; use crate::Ledger; @@ -15,6 +18,8 @@ use crate::Ledger; // ----------------- #[derive(Error, Debug)] pub enum LedgerSizeManagerError { + #[error(transparent)] + LedgerError(#[from] crate::errors::LedgerError), #[error("Ledger needs to be provided to start the manager")] LedgerNotProvided, #[error("Failed to join worker: {0}")] @@ -142,6 +147,10 @@ impl Watermarks { } } + fn is_empty(&self) -> bool { + self.marks.is_empty() + } + fn reached_max(&self, size: u64) -> bool { size >= self.max_ledger_size } @@ -178,7 +187,10 @@ impl Watermarks { // LedgerSizeManager // ----------------- enum ServiceState { - Created, + Created { + size_check_interval: Duration, + resize_percentage: ResizePercentage, + }, Running { cancellation_token: CancellationToken, worker_handle: JoinHandle<()>, @@ -191,7 +203,7 @@ enum ServiceState { pub struct LedgerSizeManager { ledger: Option>, watermarks: Watermarks, - servie_state: ServiceState, + service_state: ServiceState, } impl LedgerSizeManager { @@ -208,37 +220,88 @@ impl LedgerSizeManager { LedgerSizeManager { ledger, watermarks, - servie_state: ServiceState::Created, + service_state: ServiceState::Created { + size_check_interval: Duration::from_millis( + config.size_check_interval_ms, + ), + resize_percentage: config.resize_percentage, + }, } } - fn ensure_initial_max_ledger_size() { + fn ensure_initial_max_ledger_size(&self) { // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which // includes a _fat_ ledger truncate // We will run that first to get below the ledger max size _before_ switching to // the watermark strategy. } - pub fn try_start(mut self) -> LedgerSizeManagerResult<()> { - // TODO: @@@ async loop + pub fn try_start(self) -> LedgerSizeManagerResult { let Some(ledger) = self.ledger.take() else { return Err(LedgerSizeManagerError::LedgerNotProvided); }; - loop { - let Ok(ledger_size) = ledger.storage_size() else { - eprintln!( - "Failed to get ledger size, cannot start LedgerSizeManager" - ); - continue; + if let ServiceState::Created { + size_check_interval, + resize_percentage, + } = self.service_state + { + let cancellation_token = CancellationToken::new(); + let worker_handle = { + ledger.initialize_lowest_cleanup_slot()?; + let mut interval = interval(size_check_interval); + let cancellation_token = cancellation_token.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = cancellation_token.cancelled() => { + return; + } + _ = interval.tick() => { + let Ok(ledger_size) = ledger.storage_size() else { + eprintln!( + "Failed to get ledger size, cannot start LedgerSizeManager" + ); + continue; + }; + metrics::set_ledger_size(ledger_size); + if ledger_size > self.watermarks.max_ledger_size { + self.ensure_initial_max_ledger_size(); + continue; + } + if self.watermarks.is_empty() { + self.watermarks = Watermarks::new( + &resize_percentage, + self.watermarks.max_ledger_size, + Some(ExistingLedgerState { + size: ledger_size, + slot: ledger.last_slot(), + mod_id: ledger.last_mod_id(), + }), + ); + } + + + + if let Some(mark) = self.get_truncation_mark( + ledger_size, + ledger.last_slot(), + ledger.last_mod_id(), + ) { + self.truncate(&ledger, mark); + } + } + } + } + }) }; - metrics::set_ledger_size(ledger_size); - if let Some(mark) = self.get_truncation_mark( - ledger_size, - ledger.last_slot(), - ledger.last_mod_id(), - ) { - self.truncate(&ledger, mark); - } + self.service_state = ServiceState::Running { + cancellation_token, + worker_handle, + }; + todo!() + } else { + warn!("LedgerSizeManager already running, no need to start."); + todo!() } } From 2a04350ddf566e130d3bdd687abff61c9c54db47 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 10:36:08 +0800 Subject: [PATCH 08/36] chore: separate modules --- magicblock-ledger/src/ledger_size_manager.rs | 495 ------------------ .../src/ledger_size_manager/config.rs | 55 ++ .../src/ledger_size_manager/errors.rs | 13 + .../src/ledger_size_manager/mod.rs | 154 ++++++ .../src/ledger_size_manager/watermarks.rs | 263 ++++++++++ 5 files changed, 485 insertions(+), 495 deletions(-) delete mode 100644 magicblock-ledger/src/ledger_size_manager.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/config.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/errors.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/mod.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/watermarks.rs diff --git a/magicblock-ledger/src/ledger_size_manager.rs b/magicblock-ledger/src/ledger_size_manager.rs deleted file mode 100644 index c14c7d785..000000000 --- a/magicblock-ledger/src/ledger_size_manager.rs +++ /dev/null @@ -1,495 +0,0 @@ -#![allow(unused)] -use log::*; -use std::{collections::VecDeque, sync::Arc, time::Duration}; - -use magicblock_metrics::metrics; -use solana_sdk::clock::Slot; -use thiserror::Error; -use tokio::{ - task::{JoinError, JoinHandle}, - time::interval, -}; -use tokio_util::sync::CancellationToken; - -use crate::Ledger; - -// ----------------- -// LedgerManagerError -// ----------------- -#[derive(Error, Debug)] -pub enum LedgerSizeManagerError { - #[error(transparent)] - LedgerError(#[from] crate::errors::LedgerError), - #[error("Ledger needs to be provided to start the manager")] - LedgerNotProvided, - #[error("Failed to join worker: {0}")] - JoinError(#[from] JoinError), -} -pub type LedgerSizeManagerResult = Result; - -// ----------------- -// LedgerSizeManagerConfig -// ----------------- -/// Percentage of ledger to keep when resizing. -pub enum ResizePercentage { - /// Keep 75% of the ledger size. - Large, - /// Keep 66% of the ledger size. - Medium, - /// Keep 50% of the ledger size. - Small, -} - -impl ResizePercentage { - /// The portion of ledger we cut on each resize. - pub fn watermark_size_percent(&self) -> u64 { - use ResizePercentage::*; - match self { - Large => 25, - Medium => 34, - Small => 50, - } - } - - /// The number of watermarks to track - pub fn watermark_count(&self) -> u64 { - use ResizePercentage::*; - match self { - Large => 3, - Medium => 2, - Small => 1, - } - } -} - -pub struct LedgerSizeManagerConfig { - /// Max ledger size to maintain. - /// The [LedgerSizeManager] will attempt to respect this size,, but - /// it may grow larger temporarily in between size checks. - pub max_size: u64, - - /// Interval at which the size is checked in milliseconds. - pub size_check_interval_ms: u64, - - /// Percentage of the ledger to keep when resizing - pub resize_percentage: ResizePercentage, -} - -// ----------------- -// Watermarks -// ----------------- -#[derive(Debug, PartialEq, Eq)] -struct Watermark { - /// The slot at which this watermark was captured - slot: u64, - /// Account mod ID at which this watermark was captured - mod_id: u64, - /// The size of the ledger when this watermark was captured - size: u64, -} - -#[derive(Debug, PartialEq, Eq)] -struct Watermarks { - /// The watermarks captured - marks: VecDeque, - /// The maximum number of watermarks to keep - count: u64, - /// The targeted size difference for each watermark - mark_size: u64, - /// The maximum ledger size to maintain - max_ledger_size: u64, -} - -pub struct ExistingLedgerState { - /// The current size of the ledger - size: u64, - /// The last slot in the ledger at time of restart - slot: Slot, - /// The last account mod ID in the ledger at time of restart - mod_id: u64, -} - -impl Watermarks { - /// Creates a new set of watermarks based on the resize percentage and max ledger size. - /// - * `percentage`: The resize percentage to use. - /// - * `max_ledger_size`: The maximum size of the ledger to try to maintain. - /// - * `ledger_state`: The current ledger state which is - /// only available during restart with an existing ledger - fn new( - percentage: &ResizePercentage, - max_ledger_size: u64, - ledger_state: Option, - ) -> Self { - let count = percentage.watermark_count(); - let mut marks = VecDeque::with_capacity(count as usize); - let mark_size = - (max_ledger_size * percentage.watermark_size_percent()) / 100; - if let Some(ExistingLedgerState { size, slot, mod_id }) = ledger_state { - // Since we don't know the actual ledger sizes at each slot we must assume - // they were evenly distributed. - let mark_size_delta = size / count; - let mod_id_delta = mod_id / count; - let slot_delta = slot / count; - for i in 0..count { - let size = (i + 1) * mark_size_delta; - let mod_id = (i + 1) * mod_id_delta; - let slot = (i + 1) * slot_delta; - marks.push_back(Watermark { slot, mod_id, size }); - } - } - // In case we don't have an existing ledger state, we assume that the ledger size is - // still zero and we won't need any fabricated watermarks. - Watermarks { - marks, - count, - mark_size, - max_ledger_size, - } - } - - fn is_empty(&self) -> bool { - self.marks.is_empty() - } - - fn reached_max(&self, size: u64) -> bool { - size >= self.max_ledger_size - } - - fn update(&mut self, slot: u64, mod_id: u64, size: u64) { - // We try to record a watermark as close as possible (but below) the ideal - // watermark cutoff size. - let mark_idx = (size as f64 / self.mark_size as f64).ceil() as u64 - 1; - if mark_idx < self.count { - let watermark = Watermark { slot, mod_id, size }; - if let Some(mark) = self.marks.get_mut(mark_idx as usize) { - *mark = watermark; - } else { - self.marks.push_back(watermark); - } - } - } - - fn adjust_for_truncation(&mut self) { - // The sizes recorded at specific slots need to be adjusted since we truncated - // the slots before - - for mut mark in self.marks.iter_mut() { - mark.size = mark.size.saturating_sub(self.mark_size); - } - } - - fn consume_next(&mut self) -> Option { - self.marks.pop_front() - } -} - -// ----------------- -// LedgerSizeManager -// ----------------- -enum ServiceState { - Created { - size_check_interval: Duration, - resize_percentage: ResizePercentage, - }, - Running { - cancellation_token: CancellationToken, - worker_handle: JoinHandle<()>, - }, - Stopped { - worker_handle: JoinHandle<()>, - }, -} - -pub struct LedgerSizeManager { - ledger: Option>, - watermarks: Watermarks, - service_state: ServiceState, -} - -impl LedgerSizeManager { - pub(crate) fn new( - ledger: Option>, - ledger_state: Option, - config: LedgerSizeManagerConfig, - ) -> Self { - let watermarks = Watermarks::new( - &config.resize_percentage, - config.max_size, - ledger_state, - ); - LedgerSizeManager { - ledger, - watermarks, - service_state: ServiceState::Created { - size_check_interval: Duration::from_millis( - config.size_check_interval_ms, - ), - resize_percentage: config.resize_percentage, - }, - } - } - - fn ensure_initial_max_ledger_size(&self) { - // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which - // includes a _fat_ ledger truncate - // We will run that first to get below the ledger max size _before_ switching to - // the watermark strategy. - } - - pub fn try_start(self) -> LedgerSizeManagerResult { - let Some(ledger) = self.ledger.take() else { - return Err(LedgerSizeManagerError::LedgerNotProvided); - }; - if let ServiceState::Created { - size_check_interval, - resize_percentage, - } = self.service_state - { - let cancellation_token = CancellationToken::new(); - let worker_handle = { - ledger.initialize_lowest_cleanup_slot()?; - let mut interval = interval(size_check_interval); - let cancellation_token = cancellation_token.clone(); - tokio::spawn(async move { - loop { - tokio::select! { - _ = cancellation_token.cancelled() => { - return; - } - _ = interval.tick() => { - let Ok(ledger_size) = ledger.storage_size() else { - eprintln!( - "Failed to get ledger size, cannot start LedgerSizeManager" - ); - continue; - }; - metrics::set_ledger_size(ledger_size); - if ledger_size > self.watermarks.max_ledger_size { - self.ensure_initial_max_ledger_size(); - continue; - } - if self.watermarks.is_empty() { - self.watermarks = Watermarks::new( - &resize_percentage, - self.watermarks.max_ledger_size, - Some(ExistingLedgerState { - size: ledger_size, - slot: ledger.last_slot(), - mod_id: ledger.last_mod_id(), - }), - ); - } - - - - if let Some(mark) = self.get_truncation_mark( - ledger_size, - ledger.last_slot(), - ledger.last_mod_id(), - ) { - self.truncate(&ledger, mark); - } - } - } - } - }) - }; - self.service_state = ServiceState::Running { - cancellation_token, - worker_handle, - }; - todo!() - } else { - warn!("LedgerSizeManager already running, no need to start."); - todo!() - } - } - - fn get_truncation_mark( - &mut self, - ledger_size: u64, - slot: Slot, - mod_id: u64, - ) -> Option { - if ledger_size == 0 { - return None; - } - if self.watermarks.reached_max(ledger_size) { - debug!( - "Ledger size {} reached maximum size {}, resizing...", - ledger_size, self.watermarks.max_ledger_size - ); - self.watermarks.consume_next() - } else { - self.watermarks.update(slot, mod_id, ledger_size); - None - } - } - - fn truncate(&self, ledger: &Arc, mark: Watermark) { - // This is where the truncation logic would go. - // For now, we just print the truncation mark. - debug!( - "Truncating ledger at slot {}, mod_id {}, size {}", - mark.slot, mark.mod_id, mark.size - ); - } -} - -#[cfg(test)] -mod tests { - use test_tools_core::init_logger; - - use super::*; - - macro_rules! mark { - ($slot:expr, $mod_id:expr, $size:expr) => {{ - Watermark { - slot: $slot, - mod_id: $mod_id, - size: $size, - } - }}; - ($idx:expr, $size:expr) => {{ - mark!($idx, $idx, $size) - }}; - } - - macro_rules! marks { - ($($slot:expr, $mod_id:expr, $size:expr);+) => {{ - let mut marks = VecDeque::::new(); - $( - marks.push_back(mark!($slot, $mod_id, $size)); - )+ - Watermarks { - marks, - count: 3, - mark_size: 250, - max_ledger_size: 1000, - } - }}; - } - - macro_rules! truncate_ledger { - ($slot:expr, $mod_id:expr, $sut:ident, $mark:expr, $size:ident) => {{ - // These steps are usually performed in _actual_ ledger truncate method - $size -= $mark.size; - $sut.watermarks.adjust_for_truncation(); - $sut.watermarks.update($slot, $mod_id, $size); - debug!( - "Truncated ledger to size {} -> {:#?}", - $size, $sut.watermarks - ); - }} - } - - #[test] - fn test_size_manager_new_ledger() { - init_logger!(); - - let percentage = ResizePercentage::Large; - const MAX_SIZE: u64 = 1_000; - const STEP_SIZE: u64 = MAX_SIZE / 20; - let mut sut = LedgerSizeManager::new( - None, - None, - LedgerSizeManagerConfig { - max_size: MAX_SIZE, - size_check_interval_ms: 1000, - resize_percentage: percentage, - }, - ); - - // 1. Go up to right below the ledger size - let mut size = 0; - for i in 0..19 { - size += STEP_SIZE; - let mark = sut.get_truncation_mark(size, i, i); - assert!( - mark.is_none(), - "Expected no truncation mark at size {}", - size - ); - } - - assert_eq!(sut.watermarks, marks!(4, 4, 250; 9, 9, 500; 14, 14, 750)); - - // 2. Hit ledger max size - size += STEP_SIZE; - let mark = sut.get_truncation_mark(size, 20, 20); - assert_eq!(sut.watermarks, marks!(9, 9, 500; 14, 14, 750)); - assert_eq!(mark, Some(mark!(4, 4, 250))); - - truncate_ledger!(20, 20, sut, mark.unwrap(), size); - assert_eq!(sut.watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); - - // 3. Go up to right below the next truncation mark (also ledger max size) - for i in 21..=24 { - size += STEP_SIZE; - let mark = sut.get_truncation_mark(size, i, i); - assert!( - mark.is_none(), - "Expected no truncation mark at size {}", - size - ); - } - assert_eq!(sut.watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); - - // 4. Hit next truncation mark (also ledger max size) - size += STEP_SIZE; - let mark = sut.get_truncation_mark(size, 25, 25); - assert_eq!(mark, Some(mark!(9, 9, 250))); - - truncate_ledger!(25, 25, sut, mark.unwrap(), size); - assert_eq!( - sut.watermarks, - marks!(14, 14, 250; 20, 20, 500; 25, 25, 750) - ); - - // 5. Go past 3 truncation marks - for i in 26..=40 { - size += STEP_SIZE; - let mark = sut.get_truncation_mark(size, i, i); - if mark.is_some() { - truncate_ledger!(i, i, sut, mark.unwrap(), size); - } - } - - assert_eq!( - sut.watermarks, - marks!(30, 30, 250; 35, 35, 500; 40, 40, 750) - ); - } - - #[test] - fn test_size_manager_existing_ledger() { - init_logger!(); - - let percentage = ResizePercentage::Large; - const MAX_SIZE: u64 = 1_000; - const STEP_SIZE: u64 = MAX_SIZE / 20; - let ledger_state = ExistingLedgerState { - // NOTE: that the watermarks will always be adjusted to have the size - // lower than the max size before we start using the watermark strategy. - // See [`ensure_initial_max_ledger_size`]. - size: 900, - slot: 150, - mod_id: 150, - }; - let mut sut = LedgerSizeManager::new( - None, - Some(ledger_state), - LedgerSizeManagerConfig { - max_size: MAX_SIZE, - size_check_interval_ms: 1000, - resize_percentage: percentage, - }, - ); - - // Initial watermarks should be based on the existing ledger state - assert_eq!( - sut.watermarks, - marks!(50, 50, 300; 100, 100, 600; 150, 150, 900) - ); - } -} diff --git a/magicblock-ledger/src/ledger_size_manager/config.rs b/magicblock-ledger/src/ledger_size_manager/config.rs new file mode 100644 index 000000000..31eeefff6 --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/config.rs @@ -0,0 +1,55 @@ +use solana_sdk::clock::Slot; + +/// Percentage of ledger to keep when resizing. +pub enum ResizePercentage { + /// Keep 75% of the ledger size. + Large, + /// Keep 66% of the ledger size. + Medium, + /// Keep 50% of the ledger size. + Small, +} + +impl ResizePercentage { + /// The portion of ledger we cut on each resize. + pub fn watermark_size_percent(&self) -> u64 { + use ResizePercentage::*; + match self { + Large => 25, + Medium => 34, + Small => 50, + } + } + + /// The number of watermarks to track + pub fn watermark_count(&self) -> u64 { + use ResizePercentage::*; + match self { + Large => 3, + Medium => 2, + Small => 1, + } + } +} + +pub struct LedgerSizeManagerConfig { + /// Max ledger size to maintain. + /// The [LedgerSizeManager] will attempt to respect this size,, but + /// it may grow larger temporarily in between size checks. + pub max_size: u64, + + /// Interval at which the size is checked in milliseconds. + pub size_check_interval_ms: u64, + + /// Percentage of the ledger to keep when resizing + pub resize_percentage: ResizePercentage, +} + +pub struct ExistingLedgerState { + /// The current size of the ledger + pub(crate) size: u64, + /// The last slot in the ledger at time of restart + pub(crate) slot: Slot, + /// The last account mod ID in the ledger at time of restart + pub(crate) mod_id: u64, +} diff --git a/magicblock-ledger/src/ledger_size_manager/errors.rs b/magicblock-ledger/src/ledger_size_manager/errors.rs new file mode 100644 index 000000000..a396936d4 --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/errors.rs @@ -0,0 +1,13 @@ +use thiserror::Error; +use tokio::task::JoinError; + +#[derive(Error, Debug)] +pub enum LedgerSizeManagerError { + #[error(transparent)] + LedgerError(#[from] crate::errors::LedgerError), + #[error("Ledger needs to be provided to start the manager")] + LedgerNotProvided, + #[error("Failed to join worker: {0}")] + JoinError(#[from] JoinError), +} +pub type LedgerSizeManagerResult = Result; diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs new file mode 100644 index 000000000..c03409dff --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -0,0 +1,154 @@ +#![allow(unused)] + +pub mod config; +pub mod errors; +mod watermarks; + +use config::{ExistingLedgerState, LedgerSizeManagerConfig, ResizePercentage}; +use errors::LedgerSizeManagerResult; +use log::*; +use std::{collections::VecDeque, sync::Arc, time::Duration}; +use watermarks::{Watermark, Watermarks}; + +use magicblock_metrics::metrics; +use solana_sdk::clock::Slot; +use thiserror::Error; +use tokio::{ + task::{JoinError, JoinHandle}, + time::interval, +}; +use tokio_util::sync::CancellationToken; + +use crate::Ledger; + +enum ServiceState { + Created { + size_check_interval: Duration, + resize_percentage: ResizePercentage, + }, + Running { + cancellation_token: CancellationToken, + worker_handle: JoinHandle<()>, + }, + Stopped { + worker_handle: JoinHandle<()>, + }, +} + +pub struct LedgerSizeManager { + ledger: Option>, + watermarks: Watermarks, + service_state: ServiceState, +} + +impl LedgerSizeManager { + pub(crate) fn new( + ledger: Option>, + ledger_state: Option, + config: LedgerSizeManagerConfig, + ) -> Self { + let watermarks = Watermarks::new( + &config.resize_percentage, + config.max_size, + ledger_state, + ); + LedgerSizeManager { + ledger, + watermarks, + service_state: ServiceState::Created { + size_check_interval: Duration::from_millis( + config.size_check_interval_ms, + ), + resize_percentage: config.resize_percentage, + }, + } + } + + fn ensure_initial_max_ledger_size(&self) { + // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which + // includes a _fat_ ledger truncate + // We will run that first to get below the ledger max size _before_ switching to + // the watermark strategy. + } + + pub fn try_start(self) -> LedgerSizeManagerResult { + /* + let Some(ledger) = self.ledger.take() else { + return Err(LedgerSizeManagerError::LedgerNotProvided); + }; + if let ServiceState::Created { + size_check_interval, + resize_percentage, + } = self.service_state + { + let cancellation_token = CancellationToken::new(); + let worker_handle = { + ledger.initialize_lowest_cleanup_slot()?; + let mut interval = interval(size_check_interval); + let cancellation_token = cancellation_token.clone(); + tokio::spawn(async move { + loop { + tokio::select! { + _ = cancellation_token.cancelled() => { + return; + } + _ = interval.tick() => { + let Ok(ledger_size) = ledger.storage_size() else { + eprintln!( + "Failed to get ledger size, cannot start LedgerSizeManager" + ); + continue; + }; + metrics::set_ledger_size(ledger_size); + if ledger_size > self.watermarks.max_ledger_size { + self.ensure_initial_max_ledger_size(); + continue; + } + if self.watermarks.is_empty() { + self.watermarks = Watermarks::new( + &resize_percentage, + self.watermarks.max_ledger_size, + Some(ExistingLedgerState { + size: ledger_size, + slot: ledger.last_slot(), + mod_id: ledger.last_mod_id(), + }), + ); + } + + + + if let Some(mark) = self.get_truncation_mark( + ledger_size, + ledger.last_slot(), + ledger.last_mod_id(), + ) { + self.truncate(&ledger, mark); + } + } + } + } + }) + }; + self.service_state = ServiceState::Running { + cancellation_token, + worker_handle, + }; + todo!() + } else { + warn!("LedgerSizeManager already running, no need to start."); + todo!() + } + */ + todo!() + } + + fn truncate(&self, ledger: &Arc, mark: Watermark) { + // This is where the truncation logic would go. + // For now, we just print the truncation mark. + debug!( + "Truncating ledger at slot {}, mod_id {}, size {}", + mark.slot, mark.mod_id, mark.size + ); + } +} diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs new file mode 100644 index 000000000..f13cd3d43 --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -0,0 +1,263 @@ +use log::*; +use std::collections::VecDeque; + +use solana_sdk::clock::Slot; + +use super::config::{ExistingLedgerState, ResizePercentage}; + +// ----------------- +// Watermarks +// ----------------- +#[derive(Debug, PartialEq, Eq)] +pub(super) struct Watermark { + /// The slot at which this watermark was captured + pub(crate) slot: u64, + /// Account mod ID at which this watermark was captured + pub(crate) mod_id: u64, + /// The size of the ledger when this watermark was captured + pub(crate) size: u64, +} + +#[derive(Debug, PartialEq, Eq)] +pub(super) struct Watermarks { + /// The watermarks captured + pub(crate) marks: VecDeque, + /// The maximum number of watermarks to keep + count: u64, + /// The targeted size difference for each watermark + mark_size: u64, + /// The maximum ledger size to maintain + max_ledger_size: u64, +} + +impl Watermarks { + /// Creates a new set of watermarks based on the resize percentage and max ledger size. + /// - * `percentage`: The resize percentage to use. + /// - * `max_ledger_size`: The maximum size of the ledger to try to maintain. + /// - * `ledger_state`: The current ledger state which is + /// only available during restart with an existing ledger + pub(super) fn new( + percentage: &ResizePercentage, + max_ledger_size: u64, + ledger_state: Option, + ) -> Self { + let count = percentage.watermark_count(); + let mut marks = VecDeque::with_capacity(count as usize); + let mark_size = + (max_ledger_size * percentage.watermark_size_percent()) / 100; + if let Some(ExistingLedgerState { size, slot, mod_id }) = ledger_state { + // Since we don't know the actual ledger sizes at each slot we must assume + // they were evenly distributed. + let mark_size_delta = size / count; + let mod_id_delta = mod_id / count; + let slot_delta = slot / count; + for i in 0..count { + let size = (i + 1) * mark_size_delta; + let mod_id = (i + 1) * mod_id_delta; + let slot = (i + 1) * slot_delta; + marks.push_back(Watermark { slot, mod_id, size }); + } + } + // In case we don't have an existing ledger state, we assume that the ledger size is + // still zero and we won't need any fabricated watermarks. + Watermarks { + marks, + count, + mark_size, + max_ledger_size, + } + } + + fn is_empty(&self) -> bool { + self.marks.is_empty() + } + + fn reached_max(&self, size: u64) -> bool { + size >= self.max_ledger_size + } + + fn get_truncation_mark( + &mut self, + ledger_size: u64, + slot: Slot, + mod_id: u64, + ) -> Option { + if ledger_size == 0 { + return None; + } + if self.reached_max(ledger_size) { + debug!( + "Ledger size {} reached maximum size {}, resizing...", + ledger_size, self.max_ledger_size + ); + self.consume_next() + } else { + self.update(slot, mod_id, ledger_size); + None + } + } + + fn update(&mut self, slot: u64, mod_id: u64, size: u64) { + // We try to record a watermark as close as possible (but below) the ideal + // watermark cutoff size. + let mark_idx = (size as f64 / self.mark_size as f64).ceil() as u64 - 1; + if mark_idx < self.count { + let watermark = Watermark { slot, mod_id, size }; + if let Some(mark) = self.marks.get_mut(mark_idx as usize) { + *mark = watermark; + } else { + self.marks.push_back(watermark); + } + } + } + + fn adjust_for_truncation(&mut self) { + // The sizes recorded at specific slots need to be adjusted since we truncated + // the slots before + + for mut mark in self.marks.iter_mut() { + mark.size = mark.size.saturating_sub(self.mark_size); + } + } + + fn consume_next(&mut self) -> Option { + self.marks.pop_front() + } +} + +#[cfg(test)] +mod tests { + use test_tools_core::init_logger; + + use super::*; + + macro_rules! mark { + ($slot:expr, $mod_id:expr, $size:expr) => {{ + Watermark { + slot: $slot, + mod_id: $mod_id, + size: $size, + } + }}; + ($idx:expr, $size:expr) => {{ + mark!($idx, $idx, $size) + }}; + } + + macro_rules! marks { + ($($slot:expr, $mod_id:expr, $size:expr);+) => {{ + let mut marks = VecDeque::::new(); + $( + marks.push_back(mark!($slot, $mod_id, $size)); + )+ + Watermarks { + marks, + count: 3, + mark_size: 250, + max_ledger_size: 1000, + } + }}; + } + + macro_rules! truncate_ledger { + ($slot:expr, $mod_id:expr, $watermarks:ident, $mark:expr, $size:ident) => {{ + // These steps are usually performed in _actual_ ledger truncate method + $size -= $mark.size; + $watermarks.adjust_for_truncation(); + $watermarks.update($slot, $mod_id, $size); + debug!( + "Truncated ledger to size {} -> {:#?}", + $size, $watermarks + ); + }} + } + + #[test] + fn test_watermarks_new_ledger() { + init_logger!(); + + let percentage = ResizePercentage::Large; + const MAX_SIZE: u64 = 1_000; + const STEP_SIZE: u64 = MAX_SIZE / 20; + let mut watermarks = Watermarks::new(&percentage, MAX_SIZE, None); + + // 1. Go up to right below the ledger size + let mut size = 0; + for i in 0..19 { + size += STEP_SIZE; + let mark = watermarks.get_truncation_mark(size, i, i); + assert!( + mark.is_none(), + "Expected no truncation mark at size {}", + size + ); + } + + assert_eq!(watermarks, marks!(4, 4, 250; 9, 9, 500; 14, 14, 750)); + + // 2. Hit ledger max size + size += STEP_SIZE; + let mark = watermarks.get_truncation_mark(size, 20, 20); + assert_eq!(watermarks, marks!(9, 9, 500; 14, 14, 750)); + assert_eq!(mark, Some(mark!(4, 4, 250))); + + truncate_ledger!(20, 20, watermarks, mark.unwrap(), size); + assert_eq!(watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); + + // 3. Go up to right below the next truncation mark (also ledger max size) + for i in 21..=24 { + size += STEP_SIZE; + let mark = watermarks.get_truncation_mark(size, i, i); + assert!( + mark.is_none(), + "Expected no truncation mark at size {}", + size + ); + } + assert_eq!(watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); + + // 4. Hit next truncation mark (also ledger max size) + size += STEP_SIZE; + let mark = watermarks.get_truncation_mark(size, 25, 25); + assert_eq!(mark, Some(mark!(9, 9, 250))); + + truncate_ledger!(25, 25, watermarks, mark.unwrap(), size); + assert_eq!(watermarks, marks!(14, 14, 250; 20, 20, 500; 25, 25, 750)); + + // 5. Go past 3 truncation marks + for i in 26..=40 { + size += STEP_SIZE; + let mark = watermarks.get_truncation_mark(size, i, i); + if mark.is_some() { + truncate_ledger!(i, i, watermarks, mark.unwrap(), size); + } + } + + assert_eq!(watermarks, marks!(30, 30, 250; 35, 35, 500; 40, 40, 750)); + } + + #[test] + fn test_watermarks_existing_ledger() { + init_logger!(); + + let percentage = ResizePercentage::Large; + const MAX_SIZE: u64 = 1_000; + const STEP_SIZE: u64 = MAX_SIZE / 20; + let ledger_state = ExistingLedgerState { + // NOTE: that the watermarks will always be adjusted to have the size + // lower than the max size before we start using the watermark strategy. + // See [`ensure_initial_max_ledger_size`]. + size: 900, + slot: 150, + mod_id: 150, + }; + let watermarks = + Watermarks::new(&percentage, MAX_SIZE, Some(ledger_state)); + + // Initial watermarks should be based on the existing ledger state + assert_eq!( + watermarks, + marks!(50, 50, 300; 100, 100, 600; 150, 150, 900) + ); + } +} From 92ec9ea0bf316e8abab88a969365f5ede0d8bd43 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 15:12:47 +0800 Subject: [PATCH 09/36] feat: truncator as managable ledger + added impl from fix/ledger/delete-using-compaction-filter --- Cargo.lock | 1 + magicblock-ledger/Cargo.toml | 1 + .../src/database/cf_descriptors.rs | 41 ++++-- .../src/database/compaction_filter.rs | 111 +++++++++++++++ magicblock-ledger/src/database/db.rs | 6 + magicblock-ledger/src/database/mod.rs | 1 + magicblock-ledger/src/database/rocks_db.rs | 34 ++++- .../src/ledger_size_manager/errors.rs | 2 - .../src/ledger_size_manager/mod.rs | 132 +++++++++++------- .../src/ledger_size_manager/traits.rs | 14 ++ .../src/ledger_size_manager/truncator.rs | 118 ++++++++++++++++ .../src/ledger_size_manager/watermarks.rs | 2 +- magicblock-ledger/src/store/api.rs | 19 +++ 13 files changed, 412 insertions(+), 70 deletions(-) create mode 100644 magicblock-ledger/src/database/compaction_filter.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/traits.rs create mode 100644 magicblock-ledger/src/ledger_size_manager/truncator.rs diff --git a/Cargo.lock b/Cargo.lock index dc53d06c8..1df547ede 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3795,6 +3795,7 @@ dependencies = [ name = "magicblock-ledger" version = "0.1.5" dependencies = [ + "async-trait", "bincode", "byteorder", "fs_extra", diff --git a/magicblock-ledger/Cargo.toml b/magicblock-ledger/Cargo.toml index 7e48bc7d8..163599d3b 100644 --- a/magicblock-ledger/Cargo.toml +++ b/magicblock-ledger/Cargo.toml @@ -8,6 +8,7 @@ license.workspace = true edition.workspace = true [dependencies] +async-trait = { workspace = true } bincode = { workspace = true } log = { workspace = true } byteorder = { workspace = true } diff --git a/magicblock-ledger/src/database/cf_descriptors.rs b/magicblock-ledger/src/database/cf_descriptors.rs index a81b1234a..ca9f7ac19 100644 --- a/magicblock-ledger/src/database/cf_descriptors.rs +++ b/magicblock-ledger/src/database/cf_descriptors.rs @@ -1,4 +1,8 @@ -use std::{collections::HashSet, path::Path}; +use std::{ + collections::HashSet, + path::Path, + sync::{atomic::AtomicU64, Arc}, +}; use log::*; use rocksdb::{ColumnFamilyDescriptor, DBCompressionType, Options, DB}; @@ -9,7 +13,9 @@ use super::{ options::{LedgerColumnOptions, LedgerOptions}, rocksdb_options::should_disable_auto_compactions, }; -use crate::database::{columns, options::AccessType}; +use crate::database::{ + columns, compaction_filter::PurgedSlotFilterFactory, options::AccessType, +}; /// Create the column family (CF) descriptors necessary to open the database. /// @@ -23,19 +29,20 @@ use crate::database::{columns, options::AccessType}; pub fn cf_descriptors( path: &Path, options: &LedgerOptions, + oldest_slot: &Arc, ) -> Vec { use columns::*; let mut cf_descriptors = vec![ - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), - new_cf_descriptor::(options), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), + new_cf_descriptor::(options, oldest_slot), ]; // If the access type is Secondary, we don't need to open all of the @@ -87,13 +94,18 @@ pub fn cf_descriptors( fn new_cf_descriptor( options: &LedgerOptions, + oldest_slot: &Arc, ) -> ColumnFamilyDescriptor { - ColumnFamilyDescriptor::new(C::NAME, get_cf_options::(options)) + ColumnFamilyDescriptor::new( + C::NAME, + get_cf_options::(options, oldest_slot), + ) } // FROM ledger/src/blockstore_db.rs :2010 fn get_cf_options( options: &LedgerOptions, + oldest_slot: &Arc, ) -> Options { let mut cf_options = Options::default(); // 256 * 8 = 2GB. 6 of these columns should take at most 12GB of RAM @@ -111,7 +123,12 @@ fn get_cf_options( ); cf_options.set_max_bytes_for_level_base(total_size_base); cf_options.set_target_file_size_base(file_size_base); + cf_options.set_compaction_filter_factory( + PurgedSlotFilterFactory::::new(oldest_slot.clone()), + ); + // TODO(edwin): check if needed + // cf_options.set_max_total_wal_size(4 * 1024 * 1024 * 1024); let disable_auto_compactions = should_disable_auto_compactions(&options.access_type); if disable_auto_compactions { diff --git a/magicblock-ledger/src/database/compaction_filter.rs b/magicblock-ledger/src/database/compaction_filter.rs new file mode 100644 index 000000000..353a45eb3 --- /dev/null +++ b/magicblock-ledger/src/database/compaction_filter.rs @@ -0,0 +1,111 @@ +use std::{ + ffi::{CStr, CString}, + marker::PhantomData, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; + +use log::trace; +use rocksdb::{ + compaction_filter::CompactionFilter, + compaction_filter_factory::{ + CompactionFilterContext, CompactionFilterFactory, + }, + CompactionDecision, +}; +use solana_sdk::clock::Slot; + +use crate::database::columns::{Column, ColumnName}; + +/// Factory that produces PurgedSlotFilter +/// This struct is used for deleting truncated slots from the DB during +/// RocksDB's scheduled compaction. The Factory creates the Filter. +/// We maintain oldest_slot that signals what slots were truncated and are safe to remove +pub(crate) struct PurgedSlotFilterFactory { + oldest_slot: Arc, + name: CString, + _phantom: PhantomData, +} + +impl PurgedSlotFilterFactory { + pub fn new(oldest_slot: Arc) -> Self { + let name = CString::new(format!( + "purged_slot_filter({}, {:?})", + C::NAME, + oldest_slot + )) + .unwrap(); + Self { + oldest_slot, + name, + _phantom: PhantomData, + } + } +} + +impl CompactionFilterFactory + for PurgedSlotFilterFactory +{ + type Filter = PurgedSlotFilter; + + fn create(&mut self, _context: CompactionFilterContext) -> Self::Filter { + let copied_oldest_slot = self.oldest_slot.load(Ordering::Relaxed); + let name = CString::new(format!( + "purged_slot_filter({}, {:?})", + C::NAME, + copied_oldest_slot + )) + .unwrap(); + + PurgedSlotFilter:: { + oldest_slot: copied_oldest_slot, + name, + _phantom: PhantomData, + } + } + + fn name(&self) -> &CStr { + &self.name + } +} + +/// A CompactionFilter implementation to remove keys older than a given slot. +pub(crate) struct PurgedSlotFilter { + /// The oldest slot to keep; any slot < oldest_slot will be removed + oldest_slot: Slot, + name: CString, + _phantom: PhantomData, +} + +impl CompactionFilter for PurgedSlotFilter { + fn filter( + &mut self, + level: u32, + key: &[u8], + _value: &[u8], + ) -> CompactionDecision { + use rocksdb::CompactionDecision::*; + trace!("CompactionFilter: triggered!"); + + let slot_in_key = C::slot(C::index(key)); + if slot_in_key < self.oldest_slot { + trace!( + "CompactionFilter: removing key. level: {}, slot: {}", + level, + slot_in_key + ); + + // It is safe to delete this key + // since those slots were truncated anyway + Remove + } else { + Keep + } + } + + fn name(&self) -> &CStr { + &self.name + } +} diff --git a/magicblock-ledger/src/database/db.rs b/magicblock-ledger/src/database/db.rs index bffa37588..3c798c0e4 100644 --- a/magicblock-ledger/src/database/db.rs +++ b/magicblock-ledger/src/database/db.rs @@ -179,4 +179,10 @@ impl Database { ) -> std::result::Result, LedgerError> { self.backend.live_files_metadata() } + + /// Stores oldest maintained slot in db + /// Used in CompactionFilter to decide if slot can be safely removed + pub fn set_oldest_slot(&self, slot: Slot) { + self.backend.set_oldest_slot(slot); + } } diff --git a/magicblock-ledger/src/database/mod.rs b/magicblock-ledger/src/database/mod.rs index ff1e99d1d..8a046f746 100644 --- a/magicblock-ledger/src/database/mod.rs +++ b/magicblock-ledger/src/database/mod.rs @@ -1,5 +1,6 @@ pub mod cf_descriptors; pub mod columns; +pub mod compaction_filter; mod consts; pub mod db; pub mod iterator; diff --git a/magicblock-ledger/src/database/rocks_db.rs b/magicblock-ledger/src/database/rocks_db.rs index 15334fcdb..be4be97d1 100644 --- a/magicblock-ledger/src/database/rocks_db.rs +++ b/magicblock-ledger/src/database/rocks_db.rs @@ -1,10 +1,18 @@ -use std::{fs, path::Path}; +use std::{ + fs, + path::Path, + sync::{ + atomic::{AtomicU64, Ordering}, + Arc, + }, +}; use rocksdb::{ AsColumnFamilyRef, ColumnFamily, DBIterator, DBPinnableSlice, DBRawIterator, FlushOptions, IteratorMode as RocksIteratorMode, LiveFile, Options, WriteBatch as RWriteBatch, DB, }; +use solana_sdk::clock::Slot; use super::{ cf_descriptors::cf_descriptors, @@ -22,15 +30,20 @@ use crate::errors::{LedgerError, LedgerResult}; pub struct Rocks { pub db: DB, access_type: AccessType, + /// Oldest slot we want to keep in DB, slots before will be removed + oldest_slot: Arc, } impl Rocks { pub fn open(path: &Path, options: LedgerOptions) -> LedgerResult { + const DEFAULT_OLD_SLOT: Slot = 0; + let access_type = options.access_type.clone(); fs::create_dir_all(path)?; + let oldest_slot = Arc::new(DEFAULT_OLD_SLOT.into()); let db_options = get_rocksdb_options(&access_type); - let descriptors = cf_descriptors(path, &options); + let descriptors = cf_descriptors(path, &options, &oldest_slot); let db = match access_type { AccessType::Primary => { @@ -39,7 +52,11 @@ impl Rocks { _ => unreachable!("Only primary access is supported"), }; - Ok(Self { db, access_type }) + Ok(Self { + db, + access_type, + oldest_slot, + }) } pub fn destroy(path: &Path) -> LedgerResult<()> { @@ -235,6 +252,12 @@ impl Rocks { Err(e) => Err(LedgerError::RocksDb(e)), } } + + /// Stores oldest maintained slot in db + /// Used in CompactionFilter to decide if slot can be safely removed + pub fn set_oldest_slot(&self, slot: Slot) { + self.oldest_slot.store(slot, Ordering::Relaxed); + } } #[cfg(test)] @@ -254,7 +277,10 @@ mod tests { // The names and descriptors don't need to be in the same order for our use cases; // however, there should be the same number of each. For example, adding a new column // should update both lists. - assert_eq!(columns().len(), cf_descriptors(&path, &options,).len()); + assert_eq!( + columns().len(), + cf_descriptors(&path, &options, &Arc::new(0.into())).len() + ); } #[test] diff --git a/magicblock-ledger/src/ledger_size_manager/errors.rs b/magicblock-ledger/src/ledger_size_manager/errors.rs index a396936d4..126276b96 100644 --- a/magicblock-ledger/src/ledger_size_manager/errors.rs +++ b/magicblock-ledger/src/ledger_size_manager/errors.rs @@ -5,8 +5,6 @@ use tokio::task::JoinError; pub enum LedgerSizeManagerError { #[error(transparent)] LedgerError(#[from] crate::errors::LedgerError), - #[error("Ledger needs to be provided to start the manager")] - LedgerNotProvided, #[error("Failed to join worker: {0}")] JoinError(#[from] JoinError), } diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index c03409dff..c9e527631 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -2,12 +2,15 @@ pub mod config; pub mod errors; +pub mod traits; +mod truncator; mod watermarks; use config::{ExistingLedgerState, LedgerSizeManagerConfig, ResizePercentage}; -use errors::LedgerSizeManagerResult; +use errors::{LedgerSizeManagerError, LedgerSizeManagerResult}; use log::*; use std::{collections::VecDeque, sync::Arc, time::Duration}; +use traits::ManagableLedger; use watermarks::{Watermark, Watermarks}; use magicblock_metrics::metrics; @@ -25,6 +28,8 @@ enum ServiceState { Created { size_check_interval: Duration, resize_percentage: ResizePercentage, + max_ledger_size: u64, + existing_ledger_state: Option, }, Running { cancellation_token: CancellationToken, @@ -35,57 +40,46 @@ enum ServiceState { }, } -pub struct LedgerSizeManager { - ledger: Option>, - watermarks: Watermarks, +pub struct LedgerSizeManager { + ledger: Arc, service_state: ServiceState, } -impl LedgerSizeManager { +impl LedgerSizeManager { pub(crate) fn new( - ledger: Option>, + ledger: Arc, ledger_state: Option, config: LedgerSizeManagerConfig, ) -> Self { - let watermarks = Watermarks::new( - &config.resize_percentage, - config.max_size, - ledger_state, - ); LedgerSizeManager { ledger, - watermarks, service_state: ServiceState::Created { size_check_interval: Duration::from_millis( config.size_check_interval_ms, ), resize_percentage: config.resize_percentage, + max_ledger_size: config.max_size, + existing_ledger_state: ledger_state, }, } } - fn ensure_initial_max_ledger_size(&self) { - // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which - // includes a _fat_ ledger truncate - // We will run that first to get below the ledger max size _before_ switching to - // the watermark strategy. - } - pub fn try_start(self) -> LedgerSizeManagerResult { - /* - let Some(ledger) = self.ledger.take() else { - return Err(LedgerSizeManagerError::LedgerNotProvided); - }; if let ServiceState::Created { size_check_interval, resize_percentage, + max_ledger_size, + mut existing_ledger_state, } = self.service_state { let cancellation_token = CancellationToken::new(); let worker_handle = { + let ledger = self.ledger.clone(); ledger.initialize_lowest_cleanup_slot()?; let mut interval = interval(size_check_interval); - let cancellation_token = cancellation_token.clone(); + let mut watermarks = None::; + + let mut cancellation_token = cancellation_token.clone(); tokio::spawn(async move { loop { tokio::select! { @@ -94,56 +88,92 @@ impl LedgerSizeManager { } _ = interval.tick() => { let Ok(ledger_size) = ledger.storage_size() else { - eprintln!( - "Failed to get ledger size, cannot start LedgerSizeManager" + error!( + "Failed to get ledger size, cannot manage its size" ); continue; }; + metrics::set_ledger_size(ledger_size); - if ledger_size > self.watermarks.max_ledger_size { - self.ensure_initial_max_ledger_size(); + + // If we restarted with an existing ledger we need to make sure that the + // ledger size is not far above the max size before we can + // start using the watermark strategy. + if watermarks.is_none() && + existing_ledger_state.is_some() && + ledger_size > max_ledger_size { + warn!( + "Ledger size {} is above the max size {}, \ + waiting for truncation to start using watermarks.", + ledger_size, max_ledger_size + ); + Self::ensure_initial_max_ledger_size_below( + &ledger, + max_ledger_size); continue; } - if self.watermarks.is_empty() { - self.watermarks = Watermarks::new( - &resize_percentage, - self.watermarks.max_ledger_size, - Some(ExistingLedgerState { - size: ledger_size, - slot: ledger.last_slot(), - mod_id: ledger.last_mod_id(), - }), - ); - } - + // We either started new or trimmed the existing ledger to + // below the max size and now will keep it so using watermarks. + let wms = watermarks + .get_or_insert_with(|| Watermarks::new( + &resize_percentage, + max_ledger_size, + existing_ledger_state.take(), + )); - if let Some(mark) = self.get_truncation_mark( + if let Some(mark) = wms.get_truncation_mark( ledger_size, ledger.last_slot(), ledger.last_mod_id(), ) { - self.truncate(&ledger, mark); + Self::truncate_ledger(&ledger, mark); } } } } }) }; - self.service_state = ServiceState::Running { - cancellation_token, - worker_handle, - }; - todo!() + Ok(Self { + ledger: self.ledger, + service_state: ServiceState::Running { + cancellation_token, + worker_handle, + }, + }) } else { warn!("LedgerSizeManager already running, no need to start."); - todo!() + Ok(self) + } + } + + pub fn stop(self) -> Self { + match self.service_state { + ServiceState::Running { + cancellation_token, + worker_handle, + } => { + cancellation_token.cancel(); + Self { + ledger: self.ledger, + service_state: ServiceState::Stopped { worker_handle }, + } + } + _ => { + warn!("LedgerSizeManager is not running, cannot stop."); + self + } } - */ - todo!() } - fn truncate(&self, ledger: &Arc, mark: Watermark) { + fn ensure_initial_max_ledger_size_below(ledger: &Arc, max_size: u64) { + // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which + // includes a _fat_ ledger truncate + // We will run that first to get below the ledger max size _before_ switching to + // the watermark strategy. + } + + fn truncate_ledger(ledger: &Arc, mark: Watermark) { // This is where the truncation logic would go. // For now, we just print the truncation mark. debug!( diff --git a/magicblock-ledger/src/ledger_size_manager/traits.rs b/magicblock-ledger/src/ledger_size_manager/traits.rs new file mode 100644 index 000000000..b8b549c7a --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/traits.rs @@ -0,0 +1,14 @@ +use async_trait::async_trait; +use solana_sdk::clock::Slot; + +use crate::errors::{LedgerError, LedgerResult}; + +#[async_trait] +pub trait ManagableLedger: Send + Sync + 'static { + fn storage_size(&self) -> LedgerResult; + fn last_slot(&self) -> Slot; + fn last_mod_id(&self) -> u64; + fn initialize_lowest_cleanup_slot(&self) -> LedgerResult<()>; + async fn compact_slot_range(&self, from: Slot, to: Slot); + async fn truncate_fat_ledger(&self, lowest_slot: u64); +} diff --git a/magicblock-ledger/src/ledger_size_manager/truncator.rs b/magicblock-ledger/src/ledger_size_manager/truncator.rs new file mode 100644 index 000000000..c5042dff0 --- /dev/null +++ b/magicblock-ledger/src/ledger_size_manager/truncator.rs @@ -0,0 +1,118 @@ +use async_trait::async_trait; +use log::*; +use solana_measure::measure::Measure; +use std::sync::Arc; +use tokio::task::JoinSet; + +use solana_sdk::clock::Slot; + +use crate::{ + database::columns::{ + AddressSignatures, Blockhash, Blocktime, PerfSamples, SlotSignatures, + Transaction, TransactionMemos, TransactionStatus, + }, + errors::{LedgerError, LedgerResult}, + Ledger, +}; + +use super::traits::ManagableLedger; + +pub struct Truncator { + ledger: Arc, +} + +impl Truncator { + /// Synchronous utility function that triggers and awaits compaction on all the columns + /// Compacts [from_slot; to_slot] inclusive + pub async fn compact_slot_range( + ledger: &Arc, + from_slot: u64, + to_slot: u64, + ) { + debug_assert!(to_slot >= from_slot, "to_slot >= from_slot"); + if to_slot < from_slot { + error!("BUG: to_slot < from_slot, not compacting"); + return; + } + + // Compaction can be run concurrently for different cf + // but it utilizes rocksdb threads, in order not to drain + // our tokio rt threads, we split the effort in just 3 tasks + let mut measure = Measure::start("Manual compaction"); + let mut join_set = JoinSet::new(); + join_set.spawn_blocking({ + let ledger = ledger.clone(); + move || { + ledger.compact_slot_range_cf::( + Some(from_slot), + Some(to_slot + 1), + ); + ledger.compact_slot_range_cf::( + Some(from_slot), + Some(to_slot + 1), + ); + ledger.compact_slot_range_cf::( + Some(from_slot), + Some(to_slot + 1), + ); + ledger.compact_slot_range_cf::( + Some((from_slot, u32::MIN)), + Some((to_slot + 1, u32::MAX)), + ); + } + }); + + // The below we cannot compact with specific range since they keys don't + // start with the slot value + join_set.spawn_blocking({ + let ledger = ledger.clone(); + move || { + ledger.compact_slot_range_cf::(None, None); + ledger.compact_slot_range_cf::(None, None); + } + }); + join_set.spawn_blocking({ + let ledger = ledger.clone(); + move || { + ledger.compact_slot_range_cf::(None, None); + ledger.compact_slot_range_cf::(None, None); + } + }); + + join_set.join_all().await; + measure.stop(); + debug!("Manual compaction took: {measure}"); + } +} + +#[async_trait] +impl ManagableLedger for Truncator { + fn storage_size(&self) -> Result { + self.ledger.storage_size() + } + + fn last_slot(&self) -> Slot { + self.ledger.last_slot() + } + + fn last_mod_id(&self) -> u64 { + self.ledger.last_mod_id() + } + + fn initialize_lowest_cleanup_slot(&self) -> Result<(), LedgerError> { + self.ledger.initialize_lowest_cleanup_slot() + } + + async fn compact_slot_range(&self, from: Slot, to: Slot) { + Self::compact_slot_range(&self.ledger, from, to).await; + } + + async fn truncate_fat_ledger(&self, lowest_slot: u64) { + self.ledger.set_lowest_cleanup_slot(lowest_slot); + if let Err(err) = self.ledger.flush() { + // We will still compact + error!("Failed to flush: {}", err); + } + Self::compact_slot_range(&self.ledger, 0, lowest_slot).await; + } +} diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index f13cd3d43..636b11ed0 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -76,7 +76,7 @@ impl Watermarks { size >= self.max_ledger_size } - fn get_truncation_mark( + pub(super) fn get_truncation_mark( &mut self, ledger_size: u64, slot: Slot, diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 85958413f..9f3ed7519 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -248,6 +248,25 @@ impl Ledger { .read() .expect(Self::LOWEST_CLEANUP_SLOT_POISONED) } + /// + /// Updates both lowest_cleanup_slot and oldest_slot for CompactionFilter + /// All slots less or equal to argument will be removed during compaction + pub fn set_lowest_cleanup_slot(&self, slot: Slot) { + let mut lowest_cleanup_slot = self + .lowest_cleanup_slot + .write() + .expect(Self::LOWEST_CLEANUP_SLOT_POISONED); + + let new_lowest_cleanup_slot = std::cmp::max(*lowest_cleanup_slot, slot); + *lowest_cleanup_slot = new_lowest_cleanup_slot; + + if new_lowest_cleanup_slot == 0 { + // fresh db case + self.db.set_oldest_slot(new_lowest_cleanup_slot); + } else { + self.db.set_oldest_slot(new_lowest_cleanup_slot + 1); + } + } /// Initializes lowest slot to cleanup from pub fn initialize_lowest_cleanup_slot(&self) -> Result<(), LedgerError> { From 2ccd3cb67d8a079aff12af057365c430e7467146 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 16:38:42 +0800 Subject: [PATCH 10/36] fix: don't remove account mods (not based on slots) --- magicblock-ledger/src/database/columns.rs | 6 ++++++ magicblock-ledger/src/database/compaction_filter.rs | 4 ++++ 2 files changed, 10 insertions(+) diff --git a/magicblock-ledger/src/database/columns.rs b/magicblock-ledger/src/database/columns.rs index 399b0607c..8f4ceb51b 100644 --- a/magicblock-ledger/src/database/columns.rs +++ b/magicblock-ledger/src/database/columns.rs @@ -129,6 +129,9 @@ pub trait Column { // first item in the key. fn as_index(slot: Slot) -> Self::Index; fn slot(index: Self::Index) -> Slot; + fn keep_all_on_compaction() -> bool { + false + } } pub trait ColumnName { @@ -651,6 +654,9 @@ impl Column for AccountModDatas { fn as_index(slot: Slot) -> Self::Index { slot } + fn keep_all_on_compaction() -> bool { + true + } } impl TypedColumn for AccountModDatas { diff --git a/magicblock-ledger/src/database/compaction_filter.rs b/magicblock-ledger/src/database/compaction_filter.rs index 353a45eb3..fe6b52d79 100644 --- a/magicblock-ledger/src/database/compaction_filter.rs +++ b/magicblock-ledger/src/database/compaction_filter.rs @@ -87,6 +87,10 @@ impl CompactionFilter for PurgedSlotFilter { _value: &[u8], ) -> CompactionDecision { use rocksdb::CompactionDecision::*; + if C::keep_all_on_compaction() { + return Keep; + } + trace!("CompactionFilter: triggered!"); let slot_in_key = C::slot(C::index(key)); From 4c03e434836072fae5be94dc0de1d9e3f7582ca1 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 16:45:38 +0800 Subject: [PATCH 11/36] feat: adapted ManagableLedger and made truncator impl it --- .../src/ledger_size_manager/mod.rs | 62 +++++++++++++++++++ .../src/ledger_size_manager/traits.rs | 4 +- .../src/ledger_size_manager/truncator.rs | 29 +++++---- 3 files changed, 81 insertions(+), 14 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index c9e527631..4e20f0349 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -182,3 +182,65 @@ impl LedgerSizeManager { ); } } + +#[cfg(test)] +mod tests { + use std::sync::Mutex; + + use async_trait::async_trait; + + use super::*; + use crate::{errors::LedgerResult, Ledger}; + + struct ManageableLedgerMock { + first_slot: Mutex, + last_slot: Mutex, + last_mod_id: Mutex, + } + impl ManageableLedgerMock { + fn new(first_slot: Slot, last_slot: Slot, last_mod_id: u64) -> Self { + ManageableLedgerMock { + first_slot: Mutex::new(first_slot), + last_slot: Mutex::new(last_slot), + last_mod_id: Mutex::new(last_mod_id), + } + } + + fn slots(&self) -> Slot { + let first_slot = *self.first_slot.lock().unwrap(); + let last_slot = *self.last_slot.lock().unwrap(); + last_slot - first_slot + } + } + + #[async_trait] + impl ManagableLedger for ManageableLedgerMock { + fn storage_size(&self) -> LedgerResult { + Ok(self.slots() * 100) + } + + fn last_slot(&self) -> Slot { + *self.last_slot.lock().unwrap() + } + + fn last_mod_id(&self) -> u64 { + *self.last_mod_id.lock().unwrap() + } + + fn initialize_lowest_cleanup_slot(&self) -> LedgerResult<()> { + Ok(()) + } + + async fn compact_slot_range(&self, to: Slot) { + assert!(to >= self.last_slot()); + *self.first_slot.lock().unwrap() = to; + } + + async fn truncate_fat_ledger(&self, lowest_slot: Slot) { + *self.first_slot.lock().unwrap() = lowest_slot; + } + } + + #[tokio::test] + async fn test_ledger_size_manager() {} +} diff --git a/magicblock-ledger/src/ledger_size_manager/traits.rs b/magicblock-ledger/src/ledger_size_manager/traits.rs index b8b549c7a..0a5e5aa2c 100644 --- a/magicblock-ledger/src/ledger_size_manager/traits.rs +++ b/magicblock-ledger/src/ledger_size_manager/traits.rs @@ -9,6 +9,6 @@ pub trait ManagableLedger: Send + Sync + 'static { fn last_slot(&self) -> Slot; fn last_mod_id(&self) -> u64; fn initialize_lowest_cleanup_slot(&self) -> LedgerResult<()>; - async fn compact_slot_range(&self, from: Slot, to: Slot); - async fn truncate_fat_ledger(&self, lowest_slot: u64); + async fn compact_slot_range(&self, to: Slot); + async fn truncate_fat_ledger(&self, lowest_slot: Slot); } diff --git a/magicblock-ledger/src/ledger_size_manager/truncator.rs b/magicblock-ledger/src/ledger_size_manager/truncator.rs index c5042dff0..3cbe5cad5 100644 --- a/magicblock-ledger/src/ledger_size_manager/truncator.rs +++ b/magicblock-ledger/src/ledger_size_manager/truncator.rs @@ -24,16 +24,21 @@ pub struct Truncator { impl Truncator { /// Synchronous utility function that triggers and awaits compaction on all the columns /// Compacts [from_slot; to_slot] inclusive - pub async fn compact_slot_range( + pub async fn truncate( ledger: &Arc, - from_slot: u64, - to_slot: u64, + from_slot: Slot, + to_slot: Slot, ) { - debug_assert!(to_slot >= from_slot, "to_slot >= from_slot"); - if to_slot < from_slot { - error!("BUG: to_slot < from_slot, not compacting"); + debug_assert!(from_slot <= to_slot); + if from_slot > to_slot { + error!( + "Truncation requested with from_slot: {from_slot} > to_slot: {to_slot}, skipping" + ); return; } + // The compaction filter uses lowest_cleanup_slot to determine what slots + // to remove so we need to set this _before_ we run compaction + ledger.set_lowest_cleanup_slot(to_slot); // Compaction can be run concurrently for different cf // but it utilizes rocksdb threads, in order not to drain @@ -103,16 +108,16 @@ impl ManagableLedger for Truncator { self.ledger.initialize_lowest_cleanup_slot() } - async fn compact_slot_range(&self, from: Slot, to: Slot) { - Self::compact_slot_range(&self.ledger, from, to).await; + async fn compact_slot_range(&self, to: Slot) { + let from_slot = self.ledger.get_lowest_cleanup_slot() + 1; + Self::truncate(&self.ledger, from_slot, to).await; } async fn truncate_fat_ledger(&self, lowest_slot: u64) { - self.ledger.set_lowest_cleanup_slot(lowest_slot); if let Err(err) = self.ledger.flush() { - // We will still compact - error!("Failed to flush: {}", err); + error!("Failed to flush, but compaction will still run: {}", err); } - Self::compact_slot_range(&self.ledger, 0, lowest_slot).await; + + Self::truncate(&self.ledger, 0, lowest_slot).await; } } From 547b7162f7a80906375758d2f81864e736b5f85d Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Thu, 26 Jun 2025 17:16:40 +0800 Subject: [PATCH 12/36] feat: completed implementation and more mock prep --- .../src/ledger_size_manager/mod.rs | 157 ++++++++++++------ .../src/ledger_size_manager/traits.rs | 1 + .../src/ledger_size_manager/truncator.rs | 4 + 3 files changed, 109 insertions(+), 53 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 4e20f0349..d92118b6c 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -76,9 +76,10 @@ impl LedgerSizeManager { let worker_handle = { let ledger = self.ledger.clone(); ledger.initialize_lowest_cleanup_slot()?; + let mut interval = interval(size_check_interval); - let mut watermarks = None::; + let mut watermarks = None::; let mut cancellation_token = cancellation_token.clone(); tokio::spawn(async move { loop { @@ -87,47 +88,15 @@ impl LedgerSizeManager { return; } _ = interval.tick() => { - let Ok(ledger_size) = ledger.storage_size() else { - error!( - "Failed to get ledger size, cannot manage its size" - ); - continue; - }; - - metrics::set_ledger_size(ledger_size); - - // If we restarted with an existing ledger we need to make sure that the - // ledger size is not far above the max size before we can - // start using the watermark strategy. - if watermarks.is_none() && - existing_ledger_state.is_some() && - ledger_size > max_ledger_size { - warn!( - "Ledger size {} is above the max size {}, \ - waiting for truncation to start using watermarks.", - ledger_size, max_ledger_size - ); - Self::ensure_initial_max_ledger_size_below( - &ledger, - max_ledger_size); - continue; - } - - // We either started new or trimmed the existing ledger to - // below the max size and now will keep it so using watermarks. - let wms = watermarks - .get_or_insert_with(|| Watermarks::new( - &resize_percentage, - max_ledger_size, - existing_ledger_state.take(), - )); - - if let Some(mark) = wms.get_truncation_mark( - ledger_size, - ledger.last_slot(), - ledger.last_mod_id(), - ) { - Self::truncate_ledger(&ledger, mark); + let ledger_size = Self::tick( + &ledger, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ).await; + if let Some(ledger_size) = ledger_size { + metrics::set_ledger_size(ledger_size); } } } @@ -147,6 +116,63 @@ impl LedgerSizeManager { } } + async fn tick( + ledger: &Arc, + watermarks: &mut Option, + resize_percentage: &ResizePercentage, + max_ledger_size: u64, + existing_ledger_state: &mut Option, + ) -> Option { + // This function is called on each tick to manage the ledger size. + // It checks the current ledger size and truncates it if necessary. + let Ok(ledger_size) = ledger.storage_size() else { + error!("Failed to get ledger size, cannot manage its size"); + return None; + }; + + // If we restarted with an existing ledger we need to make sure that the + // ledger size is not far above the max size before we can + // start using the watermark strategy. + if watermarks.is_none() + && existing_ledger_state.is_some() + && ledger_size > max_ledger_size + { + warn!( + "Ledger size {} is above the max size {}, \ + waiting for truncation before using watermarks.", + ledger_size, max_ledger_size + ); + Self::ensure_initial_max_ledger_size_below( + ledger, + ledger_size, + max_ledger_size, + ) + .await; + return Some(ledger_size); + } + + // We either started new or trimmed the existing ledger to + // below the max size and now will keep it so using watermarks. + let wms = watermarks.get_or_insert_with(|| { + Watermarks::new( + resize_percentage, + max_ledger_size, + existing_ledger_state.take(), + ) + }); + + if let Some(mark) = wms.get_truncation_mark( + ledger_size, + ledger.last_slot(), + ledger.last_mod_id(), + ) { + Self::truncate_ledger(ledger, mark); + ledger.storage_size().ok() + } else { + Some(ledger_size) + } + } + pub fn stop(self) -> Self { match self.service_state { ServiceState::Running { @@ -166,20 +192,28 @@ impl LedgerSizeManager { } } - fn ensure_initial_max_ledger_size_below(ledger: &Arc, max_size: u64) { - // TODO: @@@ wait for fix/ledger/delete-using-compaction-filter to be merged which - // includes a _fat_ ledger truncate - // We will run that first to get below the ledger max size _before_ switching to - // the watermark strategy. + async fn ensure_initial_max_ledger_size_below( + ledger: &Arc, + current_size: u64, + max_size: u64, + ) { + let total_slots = ledger.last_slot(); + let avg_size_per_slot = current_size as f64 / total_slots as f64; + let max_slots = (max_size as f64 / avg_size_per_slot).floor() as Slot; + let cut_slots = total_slots - max_slots; + let current_lowest_slot = ledger.get_lowest_cleanup_slot(); + + let max_slot = (current_lowest_slot + cut_slots).min(total_slots); + + ledger.truncate_fat_ledger(max_slot).await; } - fn truncate_ledger(ledger: &Arc, mark: Watermark) { - // This is where the truncation logic would go. - // For now, we just print the truncation mark. + async fn truncate_ledger(ledger: &Arc, mark: Watermark) { debug!( - "Truncating ledger at slot {}, mod_id {}, size {}", - mark.slot, mark.mod_id, mark.size + "Truncating ledger at slot {}, size {}", + mark.slot, mark.size ); + ledger.compact_slot_range(mark.slot).await; } } @@ -192,11 +226,16 @@ mod tests { use super::*; use crate::{errors::LedgerResult, Ledger}; + // ----------------- + // ManageableLedgerMock + // ----------------- + const BYTES_PER_SLOT: u64 = 100; struct ManageableLedgerMock { first_slot: Mutex, last_slot: Mutex, last_mod_id: Mutex, } + impl ManageableLedgerMock { fn new(first_slot: Slot, last_slot: Slot, last_mod_id: u64) -> Self { ManageableLedgerMock { @@ -211,12 +250,17 @@ mod tests { let last_slot = *self.last_slot.lock().unwrap(); last_slot - first_slot } + + fn add_slots(&self, slots: Slot) { + let mut last_slot = self.last_slot.lock().unwrap(); + *last_slot += slots; + } } #[async_trait] impl ManagableLedger for ManageableLedgerMock { fn storage_size(&self) -> LedgerResult { - Ok(self.slots() * 100) + Ok(self.slots() * BYTES_PER_SLOT) } fn last_slot(&self) -> Slot { @@ -231,6 +275,10 @@ mod tests { Ok(()) } + fn get_lowest_cleanup_slot(&self) -> Slot { + *self.first_slot.lock().unwrap() + } + async fn compact_slot_range(&self, to: Slot) { assert!(to >= self.last_slot()); *self.first_slot.lock().unwrap() = to; @@ -241,6 +289,9 @@ mod tests { } } + // ----------------- + // Tests + // ----------------- #[tokio::test] async fn test_ledger_size_manager() {} } diff --git a/magicblock-ledger/src/ledger_size_manager/traits.rs b/magicblock-ledger/src/ledger_size_manager/traits.rs index 0a5e5aa2c..40e598a8b 100644 --- a/magicblock-ledger/src/ledger_size_manager/traits.rs +++ b/magicblock-ledger/src/ledger_size_manager/traits.rs @@ -11,4 +11,5 @@ pub trait ManagableLedger: Send + Sync + 'static { fn initialize_lowest_cleanup_slot(&self) -> LedgerResult<()>; async fn compact_slot_range(&self, to: Slot); async fn truncate_fat_ledger(&self, lowest_slot: Slot); + fn get_lowest_cleanup_slot(&self) -> Slot; } diff --git a/magicblock-ledger/src/ledger_size_manager/truncator.rs b/magicblock-ledger/src/ledger_size_manager/truncator.rs index 3cbe5cad5..0771b2e9e 100644 --- a/magicblock-ledger/src/ledger_size_manager/truncator.rs +++ b/magicblock-ledger/src/ledger_size_manager/truncator.rs @@ -108,6 +108,10 @@ impl ManagableLedger for Truncator { self.ledger.initialize_lowest_cleanup_slot() } + fn get_lowest_cleanup_slot(&self) -> Slot { + self.ledger.get_lowest_cleanup_slot() + } + async fn compact_slot_range(&self, to: Slot) { let from_slot = self.ledger.get_lowest_cleanup_slot() + 1; Self::truncate(&self.ledger, from_slot, to).await; From f62412aa72f051753d745c667240c5d4d41624b0 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Sun, 29 Jun 2025 18:16:46 +0800 Subject: [PATCH 13/36] chore: simplify updating watermarks --- .../src/ledger_size_manager/watermarks.rs | 143 ++++++++++-------- 1 file changed, 83 insertions(+), 60 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index 636b11ed0..6a4e6abf1 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -14,14 +14,18 @@ pub(super) struct Watermark { pub(crate) slot: u64, /// Account mod ID at which this watermark was captured pub(crate) mod_id: u64, - /// The size of the ledger when this watermark was captured - pub(crate) size: u64, + /// The size delta relative to the previous captured watermark + /// This tells us how much size we gain by removing all slots + /// added since the last watermark. + pub(crate) size_delta: u64, } #[derive(Debug, PartialEq, Eq)] pub(super) struct Watermarks { /// The watermarks captured pub(crate) marks: VecDeque, + /// The size of the ledger when the last watermark was captured. + pub(crate) size_at_last_capture: u64, /// The maximum number of watermarks to keep count: u64, /// The targeted size difference for each watermark @@ -45,19 +49,32 @@ impl Watermarks { let mut marks = VecDeque::with_capacity(count as usize); let mark_size = (max_ledger_size * percentage.watermark_size_percent()) / 100; - if let Some(ExistingLedgerState { size, slot, mod_id }) = ledger_state { - // Since we don't know the actual ledger sizes at each slot we must assume - // they were evenly distributed. - let mark_size_delta = size / count; - let mod_id_delta = mod_id / count; - let slot_delta = slot / count; - for i in 0..count { - let size = (i + 1) * mark_size_delta; - let mod_id = (i + 1) * mod_id_delta; - let slot = (i + 1) * slot_delta; - marks.push_back(Watermark { slot, mod_id, size }); - } - } + let initial_size = + if let Some(ExistingLedgerState { size, slot, mod_id }) = + ledger_state + { + // Since we don't know the actual ledger sizes at each slot we must assume + // they were evenly distributed. + let mark_size_delta = size / count; + let mod_id_delta = mod_id / count; + let slot_delta = slot / count; + + let mut last_size = 0; + for i in 0..count { + let size = (i + 1) * mark_size_delta; + let mod_id = (i + 1) * mod_id_delta; + let slot = (i + 1) * slot_delta; + marks.push_back(Watermark { + slot, + mod_id, + size_delta: size - last_size, + }); + last_size = size; + } + last_size + } else { + 0 + }; // In case we don't have an existing ledger state, we assume that the ledger size is // still zero and we won't need any fabricated watermarks. Watermarks { @@ -65,6 +82,7 @@ impl Watermarks { count, mark_size, max_ledger_size, + size_at_last_capture: initial_size, } } @@ -85,38 +103,37 @@ impl Watermarks { if ledger_size == 0 { return None; } - if self.reached_max(ledger_size) { + let mark = if self.reached_max(ledger_size) { debug!( - "Ledger size {} reached maximum size {}, resizing...", + "Ledger size {} exceeded maximum size {}, resizing...", ledger_size, self.max_ledger_size ); self.consume_next() } else { - self.update(slot, mod_id, ledger_size); None - } - } + }; + self.update(slot, mod_id, ledger_size); - fn update(&mut self, slot: u64, mod_id: u64, size: u64) { - // We try to record a watermark as close as possible (but below) the ideal - // watermark cutoff size. - let mark_idx = (size as f64 / self.mark_size as f64).ceil() as u64 - 1; - if mark_idx < self.count { - let watermark = Watermark { slot, mod_id, size }; - if let Some(mark) = self.marks.get_mut(mark_idx as usize) { - *mark = watermark; - } else { - self.marks.push_back(watermark); - } + if let Some(mark) = mark.as_ref() { + // We assume that the ledger will be truncated since we return a mark + // Thus we adjust the size at last capture to represent what it would + // have been if we'd have truncated the ledger before capturing it + self.size_at_last_capture = + ledger_size.saturating_sub(mark.size_delta); } + mark } - fn adjust_for_truncation(&mut self) { - // The sizes recorded at specific slots need to be adjusted since we truncated - // the slots before - - for mut mark in self.marks.iter_mut() { - mark.size = mark.size.saturating_sub(self.mark_size); + fn update(&mut self, slot: u64, mod_id: u64, size: u64) { + let size_delta = size.saturating_sub(self.size_at_last_capture); + if size_delta >= self.mark_size { + let mark = Watermark { + slot, + mod_id, + size_delta, + }; + self.marks.push_back(mark); + self.size_at_last_capture = size; } } @@ -132,44 +149,39 @@ mod tests { use super::*; macro_rules! mark { - ($slot:expr, $mod_id:expr, $size:expr) => {{ + ($slot:expr, $mod_id:expr, $size_delta:expr) => {{ Watermark { slot: $slot, mod_id: $mod_id, - size: $size, + size_delta: $size_delta, } }}; - ($idx:expr, $size:expr) => {{ - mark!($idx, $idx, $size) + ($idx:expr, $size_delta:expr) => {{ + mark!($idx, $idx, $size_delta) }}; } macro_rules! marks { - ($($slot:expr, $mod_id:expr, $size:expr);+) => {{ + ($size:expr; $($slot:expr, $mod_id:expr, $size_delta:expr);+) => {{ let mut marks = VecDeque::::new(); $( - marks.push_back(mark!($slot, $mod_id, $size)); + marks.push_back(mark!($slot, $mod_id, $size_delta)); )+ Watermarks { marks, count: 3, + size_at_last_capture: $size, mark_size: 250, max_ledger_size: 1000, } }}; } - macro_rules! truncate_ledger { ($slot:expr, $mod_id:expr, $watermarks:ident, $mark:expr, $size:ident) => {{ - // These steps are usually performed in _actual_ ledger truncate method - $size -= $mark.size; - $watermarks.adjust_for_truncation(); - $watermarks.update($slot, $mod_id, $size); - debug!( - "Truncated ledger to size {} -> {:#?}", - $size, $watermarks - ); - }} + // This step is usually performed in _actual_ ledger truncate method + $size -= $mark.size_delta; + debug!("Truncated ledger to size {} -> {:#?}", $size, $watermarks); + }}; } #[test] @@ -193,16 +205,18 @@ mod tests { ); } - assert_eq!(watermarks, marks!(4, 4, 250; 9, 9, 500; 14, 14, 750)); + assert_eq!(watermarks, marks!(750; 4, 4, 250; 9, 9, 250; 14, 14, 250)); // 2. Hit ledger max size size += STEP_SIZE; let mark = watermarks.get_truncation_mark(size, 20, 20); - assert_eq!(watermarks, marks!(9, 9, 500; 14, 14, 750)); assert_eq!(mark, Some(mark!(4, 4, 250))); + assert_eq!( + watermarks, + marks!(750; 9, 9, 250; 14, 14, 250; 20, 20, 250) + ); truncate_ledger!(20, 20, watermarks, mark.unwrap(), size); - assert_eq!(watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); // 3. Go up to right below the next truncation mark (also ledger max size) for i in 21..=24 { @@ -214,7 +228,10 @@ mod tests { size ); } - assert_eq!(watermarks, marks!(9, 9, 250; 14, 14, 500; 20, 20, 750)); + assert_eq!( + watermarks, + marks!(750; 9, 9, 250; 14, 14, 250; 20, 20, 250) + ); // 4. Hit next truncation mark (also ledger max size) size += STEP_SIZE; @@ -222,7 +239,10 @@ mod tests { assert_eq!(mark, Some(mark!(9, 9, 250))); truncate_ledger!(25, 25, watermarks, mark.unwrap(), size); - assert_eq!(watermarks, marks!(14, 14, 250; 20, 20, 500; 25, 25, 750)); + assert_eq!( + watermarks, + marks!(750; 14, 14, 250; 20, 20, 250; 25, 25, 250) + ); // 5. Go past 3 truncation marks for i in 26..=40 { @@ -233,7 +253,10 @@ mod tests { } } - assert_eq!(watermarks, marks!(30, 30, 250; 35, 35, 500; 40, 40, 750)); + assert_eq!( + watermarks, + marks!(750; 30, 30, 250; 35, 35, 250; 40, 40, 250) + ); } #[test] @@ -257,7 +280,7 @@ mod tests { // Initial watermarks should be based on the existing ledger state assert_eq!( watermarks, - marks!(50, 50, 300; 100, 100, 600; 150, 150, 900) + marks!(900; 50, 50, 300; 100, 100, 300; 150, 150, 300) ); } } From 8c0ef275a10bf3201ba16da33634e03ff69c83b9 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Sun, 29 Jun 2025 18:55:34 +0800 Subject: [PATCH 14/36] test: detailed new ledger size manager tests --- .../src/ledger_size_manager/mod.rs | 123 +++++++++++++++--- 1 file changed, 104 insertions(+), 19 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index d92118b6c..f882d8499 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -161,15 +161,27 @@ impl LedgerSizeManager { ) }); - if let Some(mark) = wms.get_truncation_mark( - ledger_size, - ledger.last_slot(), - ledger.last_mod_id(), - ) { - Self::truncate_ledger(ledger, mark); - ledger.storage_size().ok() - } else { - Some(ledger_size) + let mut ledger_size = ledger_size; + // If ledger exceeded size we downsize until we either reach below the max size + // or run out of watermarks. + loop { + let last_slot = ledger.last_slot(); + + if let Some(mark) = wms.get_truncation_mark( + ledger_size, + last_slot, + ledger.last_mod_id(), + ) { + Self::truncate_ledger(ledger, mark).await; + + if let Ok(ls) = ledger.storage_size() { + ledger_size = ls; + } else { + break Some(ledger_size); + } + } else { + break Some(ledger_size); + } } } @@ -210,8 +222,8 @@ impl LedgerSizeManager { async fn truncate_ledger(ledger: &Arc, mark: Watermark) { debug!( - "Truncating ledger at slot {}, size {}", - mark.slot, mark.size + "Truncating ledger up to slot {} gaining {} bytes", + mark.slot, mark.size_delta ); ledger.compact_slot_range(mark.slot).await; } @@ -222,6 +234,7 @@ mod tests { use std::sync::Mutex; use async_trait::async_trait; + use test_tools_core::init_logger; use super::*; use crate::{errors::LedgerResult, Ledger}; @@ -231,7 +244,7 @@ mod tests { // ----------------- const BYTES_PER_SLOT: u64 = 100; struct ManageableLedgerMock { - first_slot: Mutex, + lowest_slot: Mutex, last_slot: Mutex, last_mod_id: Mutex, } @@ -239,14 +252,14 @@ mod tests { impl ManageableLedgerMock { fn new(first_slot: Slot, last_slot: Slot, last_mod_id: u64) -> Self { ManageableLedgerMock { - first_slot: Mutex::new(first_slot), + lowest_slot: Mutex::new(first_slot), last_slot: Mutex::new(last_slot), last_mod_id: Mutex::new(last_mod_id), } } fn slots(&self) -> Slot { - let first_slot = *self.first_slot.lock().unwrap(); + let first_slot = *self.lowest_slot.lock().unwrap(); let last_slot = *self.last_slot.lock().unwrap(); last_slot - first_slot } @@ -276,16 +289,21 @@ mod tests { } fn get_lowest_cleanup_slot(&self) -> Slot { - *self.first_slot.lock().unwrap() + *self.lowest_slot.lock().unwrap() } async fn compact_slot_range(&self, to: Slot) { - assert!(to >= self.last_slot()); - *self.first_slot.lock().unwrap() = to; + let lowest_slot = self.get_lowest_cleanup_slot(); + assert!( + to >= lowest_slot, + "{to} must be >= last slot {lowest_slot}", + ); + debug!("Setting lowest cleanup slot to {}", to); + *self.lowest_slot.lock().unwrap() = to; } async fn truncate_fat_ledger(&self, lowest_slot: Slot) { - *self.first_slot.lock().unwrap() = lowest_slot; + *self.lowest_slot.lock().unwrap() = lowest_slot; } } @@ -293,5 +311,72 @@ mod tests { // Tests // ----------------- #[tokio::test] - async fn test_ledger_size_manager() {} + async fn test_ledger_size_manager_new_ledger() { + init_logger!(); + + let ledger = Arc::new(ManageableLedgerMock::new(0, 0, 0)); + let mut watermarks = None::; + let resize_percentage = ResizePercentage::Large; + let max_ledger_size = 800; + let mut existing_ledger_state = None::; + + macro_rules! tick { + ($tick:expr) => {{ + let ledger_size = + LedgerSizeManager::::tick( + &ledger, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); + debug!( + "Ledger after tick {}: Size {} {:#?}", + $tick, ledger_size, watermarks + ); + ledger_size + }}; + } + info!("Slot: 0, New Ledger"); + let ledger_size = tick!(1); + assert_eq!(ledger_size, 0); + + info!("Slot: 1 added 1 slot -> 100 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(2); + assert_eq!(ledger_size, 100); + + info!("Slot: 4, added 3 slots -> 400 bytes marked (delta: 400)"); + ledger.add_slots(3); + let ledger_size = tick!(3); + assert_eq!(ledger_size, 400); + + info!("Slot: 6, added 2 slots -> 600 bytes marked (delta: 200)"); + ledger.add_slots(2); + let ledger_size = tick!(4); + assert_eq!(ledger_size, 600); + + info!("Slot: 7, added 1 slot -> 700 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(5); + assert_eq!(ledger_size, 700); + + // Here we go to 900 and truncate using the first watermark which removes 400 bytes + info!("Slot 9, added 2 slots -> 900 bytes marked (delta: 300) -> remove 400 -> 500 bytes "); + ledger.add_slots(2); + let ledger_size = tick!(6); + assert_eq!(ledger_size, 500); + + info!("Slot 10, added 1 slot -> 600 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(7); + assert_eq!(ledger_size, 600); + + info!("Slot 14, added 4 slots -> 1000 bytes marked (delta: 500) -> remove 200 -> remove 300"); + ledger.add_slots(4); + let ledger_size = tick!(8); + assert_eq!(ledger_size, 500); + } } From a641ea1260f34e4de5c7675275c1b284ceb0e686 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Mon, 30 Jun 2025 10:26:26 +0800 Subject: [PATCH 15/36] fix: existing ledger state when smaller than target size --- .../src/ledger_size_manager/config.rs | 1 + .../src/ledger_size_manager/mod.rs | 114 +++++++++++++++--- .../src/ledger_size_manager/watermarks.rs | 31 ++--- 3 files changed, 115 insertions(+), 31 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/config.rs b/magicblock-ledger/src/ledger_size_manager/config.rs index 31eeefff6..9d23aa8df 100644 --- a/magicblock-ledger/src/ledger_size_manager/config.rs +++ b/magicblock-ledger/src/ledger_size_manager/config.rs @@ -45,6 +45,7 @@ pub struct LedgerSizeManagerConfig { pub resize_percentage: ResizePercentage, } +#[derive(Debug)] pub struct ExistingLedgerState { /// The current size of the ledger pub(crate) size: u64, diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index f882d8499..e7e848c80 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -133,32 +133,33 @@ impl LedgerSizeManager { // If we restarted with an existing ledger we need to make sure that the // ledger size is not far above the max size before we can // start using the watermark strategy. - if watermarks.is_none() - && existing_ledger_state.is_some() - && ledger_size > max_ledger_size - { + if watermarks.is_none() && existing_ledger_state.is_some() { warn!( "Ledger size {} is above the max size {}, \ waiting for truncation before using watermarks.", ledger_size, max_ledger_size ); - Self::ensure_initial_max_ledger_size_below( - ledger, - ledger_size, + + if ledger_size > max_ledger_size { + Self::ensure_initial_max_ledger_size_below( + ledger, + ledger_size, + max_ledger_size, + ) + .await; + } + watermarks.replace(Watermarks::new( + resize_percentage, max_ledger_size, - ) - .await; + existing_ledger_state.take(), + )); return Some(ledger_size); } - // We either started new or trimmed the existing ledger to - // below the max size and now will keep it so using watermarks. + // If we started with an existing ledger state we already added watermarks, + // otherwise we do this here the during the first loop iteration let wms = watermarks.get_or_insert_with(|| { - Watermarks::new( - resize_percentage, - max_ledger_size, - existing_ledger_state.take(), - ) + Watermarks::new(resize_percentage, max_ledger_size, None) }); let mut ledger_size = ledger_size; @@ -379,4 +380,85 @@ mod tests { let ledger_size = tick!(8); assert_eq!(ledger_size, 500); } + + #[tokio::test] + async fn test_ledger_size_manager_existing_ledger_below_max_size() { + init_logger!(); + + let ledger = Arc::new(ManageableLedgerMock::new(0, 6, 6)); + let mut watermarks = None::; + let resize_percentage = ResizePercentage::Large; + let max_ledger_size = 1000; + let mut existing_ledger_state = Some(ExistingLedgerState { + size: 600, + slot: 6, + mod_id: 6, + }); + + macro_rules! tick { + ($tick:expr) => {{ + let ledger_size = + LedgerSizeManager::::tick( + &ledger, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); + debug!( + "Ledger after tick {}: Size {} {:#?}", + $tick, ledger_size, watermarks + ); + ledger_size + }}; + } + + info!("Slot: 6, existing ledger"); + let ledger_size = tick!(1); + assert_eq!(ledger_size, 600); + assert_eq!( + watermarks.as_ref().unwrap(), + &Watermarks { + marks: [ + Watermark { + slot: 2, + mod_id: 2, + size_delta: 200, + }, + Watermark { + slot: 4, + mod_id: 4, + size_delta: 200, + }, + Watermark { + slot: 6, + mod_id: 6, + size_delta: 200, + }, + ] + .into(), + size_at_last_capture: 600, + count: 3, + mark_size: 250, + max_ledger_size: 1000, + }, + ); + + info!("Slot: 10, added 1 slot -> 700 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(2); + assert_eq!(ledger_size, 700); + + info!("Slot: 12, added 2 slots -> 900 bytes marked (delta: 200)"); + ledger.add_slots(2); + let ledger_size = tick!(3); + assert_eq!(ledger_size, 900); + + info!("Slot: 14, added 2 slots -> 1100 bytes marked (delta: 200) -> remove 200 -> 900 bytes"); + ledger.add_slots(2); + let ledger_size = tick!(4); + assert_eq!(ledger_size, 900); + } } diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index 6a4e6abf1..f7c392d20 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -27,11 +27,11 @@ pub(super) struct Watermarks { /// The size of the ledger when the last watermark was captured. pub(crate) size_at_last_capture: u64, /// The maximum number of watermarks to keep - count: u64, + pub(crate) count: u64, /// The targeted size difference for each watermark - mark_size: u64, + pub(crate) mark_size: u64, /// The maximum ledger size to maintain - max_ledger_size: u64, + pub(crate) max_ledger_size: u64, } impl Watermarks { @@ -55,23 +55,24 @@ impl Watermarks { { // Since we don't know the actual ledger sizes at each slot we must assume // they were evenly distributed. - let mark_size_delta = size / count; + let mark_size_delta = + (size as f64 / count as f64).round() as u64; let mod_id_delta = mod_id / count; - let slot_delta = slot / count; + let slot_delta = (slot as f64 / count as f64).round() as u64; - let mut last_size = 0; - for i in 0..count { - let size = (i + 1) * mark_size_delta; - let mod_id = (i + 1) * mod_id_delta; - let slot = (i + 1) * slot_delta; - marks.push_back(Watermark { + for i in 1..=count { + let mod_id = i * mod_id_delta; + let slot = i * slot_delta; + let mark = Watermark { slot, mod_id, - size_delta: size - last_size, - }); - last_size = size; + size_delta: mark_size_delta, + }; + debug!("Adding initial watermark: {:#?}", mark); + + marks.push_back(mark); } - last_size + size } else { 0 }; From 14600b8d2fdff8395b45aa15bc93f1b1a85f890a Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Mon, 30 Jun 2025 10:27:52 +0800 Subject: [PATCH 16/36] chore: remove not needed count property --- magicblock-ledger/src/ledger_size_manager/mod.rs | 1 - magicblock-ledger/src/ledger_size_manager/watermarks.rs | 4 ---- 2 files changed, 5 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index e7e848c80..db65f215c 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -440,7 +440,6 @@ mod tests { ] .into(), size_at_last_capture: 600, - count: 3, mark_size: 250, max_ledger_size: 1000, }, diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index f7c392d20..b20d73f01 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -26,8 +26,6 @@ pub(super) struct Watermarks { pub(crate) marks: VecDeque, /// The size of the ledger when the last watermark was captured. pub(crate) size_at_last_capture: u64, - /// The maximum number of watermarks to keep - pub(crate) count: u64, /// The targeted size difference for each watermark pub(crate) mark_size: u64, /// The maximum ledger size to maintain @@ -80,7 +78,6 @@ impl Watermarks { // still zero and we won't need any fabricated watermarks. Watermarks { marks, - count, mark_size, max_ledger_size, size_at_last_capture: initial_size, @@ -170,7 +167,6 @@ mod tests { )+ Watermarks { marks, - count: 3, size_at_last_capture: $size, mark_size: 250, max_ledger_size: 1000, From c1cf10aaecda4e6f3359e756b148550d70abe8f1 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Mon, 30 Jun 2025 15:02:51 +0800 Subject: [PATCH 17/36] feat: improve last captured size tracking --- .../src/ledger_size_manager/config.rs | 7 + .../src/ledger_size_manager/mod.rs | 248 ++++++++++++++---- .../src/ledger_size_manager/watermarks.rs | 68 +++-- 3 files changed, 242 insertions(+), 81 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/config.rs b/magicblock-ledger/src/ledger_size_manager/config.rs index 9d23aa8df..06951ca19 100644 --- a/magicblock-ledger/src/ledger_size_manager/config.rs +++ b/magicblock-ledger/src/ledger_size_manager/config.rs @@ -30,6 +30,13 @@ impl ResizePercentage { Small => 1, } } + + pub fn upper_mark_size(&self, max_ledger_size: u64) -> u64 { + (max_ledger_size as f64 + * (self.watermark_size_percent() as f64 / 100.00) + * self.watermark_count() as f64) + .round() as u64 + } } pub struct LedgerSizeManagerConfig { diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index db65f215c..88235de31 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -123,6 +123,49 @@ impl LedgerSizeManager { max_ledger_size: u64, existing_ledger_state: &mut Option, ) -> Option { + // If we restarted with an existing ledger we need to make sure that the + // ledger size is not far above the max size before we can + // start using the watermark strategy. + // NOTE: that watermarks are set during the first tick + if watermarks.is_none() { + if let Some(existing_ledger_state) = existing_ledger_state.take() { + let prev_size = existing_ledger_state.size; + + let (adjusted_ledger_size, lowest_slot) = + if prev_size > max_ledger_size { + warn!( + "Existing ledger size {} is above the max size {}, \ + waiting for truncation before using watermarks.", + prev_size, max_ledger_size + ); + + Self::ensure_initial_max_ledger_size_below( + ledger, + &existing_ledger_state, + resize_percentage, + max_ledger_size, + ) + .await + } else { + (prev_size, ledger.get_lowest_cleanup_slot()) + }; + + watermarks.replace({ + let mut marks = Watermarks::new( + resize_percentage, + max_ledger_size, + Some(existing_ledger_state), + ); + marks.size_at_last_capture = adjusted_ledger_size; + // Remove watermarks that are below the lowest cleanup slot + marks.marks.retain(|mark| mark.slot > lowest_slot); + marks + }); + + return Some(adjusted_ledger_size); + } + } + // This function is called on each tick to manage the ledger size. // It checks the current ledger size and truncates it if necessary. let Ok(ledger_size) = ledger.storage_size() else { @@ -130,35 +173,9 @@ impl LedgerSizeManager { return None; }; - // If we restarted with an existing ledger we need to make sure that the - // ledger size is not far above the max size before we can - // start using the watermark strategy. - if watermarks.is_none() && existing_ledger_state.is_some() { - warn!( - "Ledger size {} is above the max size {}, \ - waiting for truncation before using watermarks.", - ledger_size, max_ledger_size - ); - - if ledger_size > max_ledger_size { - Self::ensure_initial_max_ledger_size_below( - ledger, - ledger_size, - max_ledger_size, - ) - .await; - } - watermarks.replace(Watermarks::new( - resize_percentage, - max_ledger_size, - existing_ledger_state.take(), - )); - return Some(ledger_size); - } - - // If we started with an existing ledger state we already added watermarks, - // otherwise we do this here the during the first loop iteration - let wms = watermarks.get_or_insert_with(|| { + // If we started with an existing ledger state we already added watermarks + // above, otherwise we do this here the during the first tick + let mut wms = watermarks.get_or_insert_with(|| { Watermarks::new(resize_percentage, max_ledger_size, None) }); @@ -167,20 +184,27 @@ impl LedgerSizeManager { // or run out of watermarks. loop { let last_slot = ledger.last_slot(); - - if let Some(mark) = wms.get_truncation_mark( + let (mark, captured) = wms.get_truncation_mark( ledger_size, last_slot, ledger.last_mod_id(), - ) { + ); + if let Some(mark) = mark.as_ref() { Self::truncate_ledger(ledger, mark).await; if let Ok(ls) = ledger.storage_size() { ledger_size = ls; } else { - break Some(ledger_size); + // If we cannot get the ledger size we guess it + ledger_size = ledger_size.saturating_sub(mark.size_delta); + } + if captured { + wms.size_at_last_capture = ledger_size; } } else { + if captured { + wms.size_at_last_capture = ledger_size; + } break Some(ledger_size); } } @@ -205,23 +229,43 @@ impl LedgerSizeManager { } } + /// Downsizes the ledger to the percentage we want to truncate to whenever we reach or + /// exceed the maximum ledger size. + /// Returns the adjusted ledger size after truncation and the lowest cleanup slot. async fn ensure_initial_max_ledger_size_below( ledger: &Arc, - current_size: u64, + existing_ledger_state: &ExistingLedgerState, + resize_percentage: &ResizePercentage, max_size: u64, - ) { - let total_slots = ledger.last_slot(); - let avg_size_per_slot = current_size as f64 / total_slots as f64; - let max_slots = (max_size as f64 / avg_size_per_slot).floor() as Slot; - let cut_slots = total_slots - max_slots; - let current_lowest_slot = ledger.get_lowest_cleanup_slot(); - - let max_slot = (current_lowest_slot + cut_slots).min(total_slots); - - ledger.truncate_fat_ledger(max_slot).await; + ) -> (u64, Slot) { + let ExistingLedgerState { + size: current_size, + slot: total_slots, + mod_id, + } = existing_ledger_state; + let mut ledger_size = *current_size; + let mut total_slots = *total_slots; + + while ledger_size >= max_size { + let avg_size_per_slot = *current_size as f64 / total_slots as f64; + let target_size = resize_percentage.upper_mark_size(max_size); + let target_slot = + (target_size as f64 / avg_size_per_slot).floor() as Slot; + let cut_slots = total_slots.saturating_sub(target_slot); + let current_lowest_slot = ledger.get_lowest_cleanup_slot(); + + let max_slot = current_lowest_slot + .saturating_add(cut_slots) + .min(total_slots); + ledger.truncate_fat_ledger(max_slot).await; + + ledger_size = ledger.storage_size().unwrap_or(target_size); + total_slots -= cut_slots; + } + (ledger_size, ledger.get_lowest_cleanup_slot()) } - async fn truncate_ledger(ledger: &Arc, mark: Watermark) { + async fn truncate_ledger(ledger: &Arc, mark: &Watermark) { debug!( "Truncating ledger up to slot {} gaining {} bytes", mark.slot, mark.size_delta @@ -396,7 +440,7 @@ mod tests { }); macro_rules! tick { - ($tick:expr) => {{ + () => {{ let ledger_size = LedgerSizeManager::::tick( &ledger, @@ -407,16 +451,13 @@ mod tests { ) .await .unwrap(); - debug!( - "Ledger after tick {}: Size {} {:#?}", - $tick, ledger_size, watermarks - ); + debug!("Ledger Size {} {:#?}", ledger_size, watermarks); ledger_size }}; } info!("Slot: 6, existing ledger"); - let ledger_size = tick!(1); + let ledger_size = tick!(); assert_eq!(ledger_size, 600); assert_eq!( watermarks.as_ref().unwrap(), @@ -445,19 +486,110 @@ mod tests { }, ); - info!("Slot: 10, added 1 slot -> 700 bytes"); + info!("Slot: 7, added 1 slot -> 700 bytes"); ledger.add_slots(1); - let ledger_size = tick!(2); + let ledger_size = tick!(); assert_eq!(ledger_size, 700); - info!("Slot: 12, added 2 slots -> 900 bytes marked (delta: 200)"); + info!("Slot: 9, added 2 slots -> 900 bytes marked (delta: 200)"); ledger.add_slots(2); - let ledger_size = tick!(3); + let ledger_size = tick!(); assert_eq!(ledger_size, 900); - info!("Slot: 14, added 2 slots -> 1100 bytes marked (delta: 200) -> remove 200 -> 900 bytes"); + info!("Slot: 12, added 3 slots -> 1200 bytes marked (delta: 300) -> remove 200 -> 1000 bytes -> remove 200 -> 800 bytes"); + ledger.add_slots(3); + let ledger_size = tick!(); + assert_eq!(ledger_size, 800); + } + + #[tokio::test] + async fn test_ledger_size_manager_existing_ledger_above_max_size() { + init_logger!(); + + let ledger = Arc::new(ManageableLedgerMock::new(0, 12, 12)); + let mut watermarks = None::; + let resize_percentage = ResizePercentage::Large; + let max_ledger_size = 1000; + let mut existing_ledger_state = Some(ExistingLedgerState { + size: 1200, + slot: 12, + mod_id: 12, + }); + + macro_rules! tick { + () => {{ + let ledger_size = + LedgerSizeManager::::tick( + &ledger, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); + debug!("Ledger Size {} {:#?}", ledger_size, watermarks); + ledger_size + }}; + } + + info!("Slot: 12, existing ledger above max size"); + let ledger_size = tick!(); + assert_eq!(ledger_size, 700); + assert_eq!( + watermarks.as_ref().unwrap(), + &Watermarks { + marks: [ + Watermark { + slot: 8, + mod_id: 8, + size_delta: 400, + }, + Watermark { + slot: 12, + mod_id: 12, + size_delta: 400, + }, + ] + .into(), + size_at_last_capture: 700, + mark_size: 250, + max_ledger_size: 1000, + }, + ); + + info!("Slot: 13, added 1 slot -> 800 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 800); + + info!( + "Slot: 15, added 2 slots -> 1000 bytes -> remove estimated 400 (really 300) -> 700 bytes" + ); ledger.add_slots(2); - let ledger_size = tick!(4); - assert_eq!(ledger_size, 900); + let ledger_size = tick!(); + assert_eq!(ledger_size, 700); + + assert_eq!( + watermarks.as_ref().unwrap(), + &Watermarks { + marks: [ + Watermark { + slot: 12, + mod_id: 12, + size_delta: 400, + }, + Watermark { + slot: 15, + mod_id: 12, + size_delta: 300, + }, + ] + .into(), + size_at_last_capture: 700, + mark_size: 250, + max_ledger_size: 1000, + }, + ); } } diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index b20d73f01..cf8c142c1 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -13,6 +13,8 @@ pub(super) struct Watermark { /// The slot at which this watermark was captured pub(crate) slot: u64, /// Account mod ID at which this watermark was captured + /// NOTE: this tells us up to where to clean account mods, however + /// the size of that column is very small and thus we don't clean it yet pub(crate) mod_id: u64, /// The size delta relative to the previous captured watermark /// This tells us how much size we gain by removing all slots @@ -25,6 +27,9 @@ pub(super) struct Watermarks { /// The watermarks captured pub(crate) marks: VecDeque, /// The size of the ledger when the last watermark was captured. + /// NOTE: this is updated by the watermarks client, NOT by the watermark + /// implementation itself due to the fact that it isn't aware of the + /// true ledger size at time of capture or after it was truncated pub(crate) size_at_last_capture: u64, /// The targeted size difference for each watermark pub(crate) mark_size: u64, @@ -66,16 +71,12 @@ impl Watermarks { mod_id, size_delta: mark_size_delta, }; - debug!("Adding initial watermark: {:#?}", mark); - marks.push_back(mark); } size } else { 0 }; - // In case we don't have an existing ledger state, we assume that the ledger size is - // still zero and we won't need any fabricated watermarks. Watermarks { marks, mark_size, @@ -92,14 +93,22 @@ impl Watermarks { size >= self.max_ledger_size } + /// Returns a tuple containing two items: + /// - an optional watermark specifying the slot to truncate to and + /// an indicator of how much size should be recuperated as a result + /// - a boolean indicating if a new watermark was captured. + /// + /// NOTE: if a new watermark is captured the caller needs to update the + /// [Watermarks::size_at_last_capture] with the current ledger size or the size + /// resulting from the truncation applied when a watermark is returned. pub(super) fn get_truncation_mark( &mut self, ledger_size: u64, slot: Slot, mod_id: u64, - ) -> Option { + ) -> (Option, bool) { if ledger_size == 0 { - return None; + return (None, false); } let mark = if self.reached_max(ledger_size) { debug!( @@ -110,19 +119,11 @@ impl Watermarks { } else { None }; - self.update(slot, mod_id, ledger_size); - - if let Some(mark) = mark.as_ref() { - // We assume that the ledger will be truncated since we return a mark - // Thus we adjust the size at last capture to represent what it would - // have been if we'd have truncated the ledger before capturing it - self.size_at_last_capture = - ledger_size.saturating_sub(mark.size_delta); - } - mark + let captured = self.update(slot, mod_id, ledger_size); + (mark, captured) } - fn update(&mut self, slot: u64, mod_id: u64, size: u64) { + fn update(&mut self, slot: u64, mod_id: u64, size: u64) -> bool { let size_delta = size.saturating_sub(self.size_at_last_capture); if size_delta >= self.mark_size { let mark = Watermark { @@ -131,7 +132,9 @@ impl Watermarks { size_delta, }; self.marks.push_back(mark); - self.size_at_last_capture = size; + true + } else { + false } } @@ -181,6 +184,13 @@ mod tests { }}; } + macro_rules! adjust_last_capture { + ($watermarks:ident, $size:ident, $mark:ident ) => {{ + $watermarks.size_at_last_capture = + $size - $mark.as_ref().map(|x| x.size_delta).unwrap_or(0); + }}; + } + #[test] fn test_watermarks_new_ledger() { init_logger!(); @@ -194,20 +204,26 @@ mod tests { let mut size = 0; for i in 0..19 { size += STEP_SIZE; - let mark = watermarks.get_truncation_mark(size, i, i); + let (mark, captured) = watermarks.get_truncation_mark(size, i, i); assert!( mark.is_none(), "Expected no truncation mark at size {}", size ); + if captured { + watermarks.size_at_last_capture = size; + } } assert_eq!(watermarks, marks!(750; 4, 4, 250; 9, 9, 250; 14, 14, 250)); // 2. Hit ledger max size size += STEP_SIZE; - let mark = watermarks.get_truncation_mark(size, 20, 20); + let (mark, captured) = watermarks.get_truncation_mark(size, 20, 20); assert_eq!(mark, Some(mark!(4, 4, 250))); + assert!(captured, "Expected to capture a truncation mark"); + adjust_last_capture!(watermarks, size, mark); + assert_eq!( watermarks, marks!(750; 9, 9, 250; 14, 14, 250; 20, 20, 250) @@ -218,12 +234,13 @@ mod tests { // 3. Go up to right below the next truncation mark (also ledger max size) for i in 21..=24 { size += STEP_SIZE; - let mark = watermarks.get_truncation_mark(size, i, i); + let (mark, captured) = watermarks.get_truncation_mark(size, i, i); assert!( mark.is_none(), "Expected no truncation mark at size {}", size ); + assert!(!captured, "Expected no truncation mark captured"); } assert_eq!( watermarks, @@ -232,8 +249,10 @@ mod tests { // 4. Hit next truncation mark (also ledger max size) size += STEP_SIZE; - let mark = watermarks.get_truncation_mark(size, 25, 25); + let (mark, captured) = watermarks.get_truncation_mark(size, 25, 25); assert_eq!(mark, Some(mark!(9, 9, 250))); + assert!(captured, "Expected to capture a truncation mark"); + adjust_last_capture!(watermarks, size, mark); truncate_ledger!(25, 25, watermarks, mark.unwrap(), size); assert_eq!( @@ -244,7 +263,10 @@ mod tests { // 5. Go past 3 truncation marks for i in 26..=40 { size += STEP_SIZE; - let mark = watermarks.get_truncation_mark(size, i, i); + let (mark, captured) = watermarks.get_truncation_mark(size, i, i); + if captured { + adjust_last_capture!(watermarks, size, mark); + } if mark.is_some() { truncate_ledger!(i, i, watermarks, mark.unwrap(), size); } From 2e06f5cf808995409e92f72b1d57f75293d4b92f Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Mon, 30 Jun 2025 16:10:15 +0800 Subject: [PATCH 18/36] feat: initial ledger size manager integration --- magicblock-api/src/errors.rs | 3 + magicblock-api/src/magic_validator.rs | 40 +++++++++---- .../src/ledger_size_manager/config.rs | 11 +++- .../src/ledger_size_manager/mod.rs | 56 ++++++++++--------- .../src/ledger_size_manager/truncator.rs | 2 +- magicblock-ledger/src/store/api.rs | 11 +++- test-integration/Cargo.lock | 1 + 7 files changed, 82 insertions(+), 42 deletions(-) diff --git a/magicblock-api/src/errors.rs b/magicblock-api/src/errors.rs index ee6c08fa3..2903a8679 100644 --- a/magicblock-api/src/errors.rs +++ b/magicblock-api/src/errors.rs @@ -26,6 +26,9 @@ pub enum ApiError { #[error("Ledger error: {0}")] LedgerError(#[from] magicblock_ledger::errors::LedgerError), + #[error("LedgerSizeManager error: {0}")] + LedgerSizeManagerError(#[from] magicblock_ledger::ledger_size_manager::errors::LedgerSizeManagerError), + #[error("Failed to load programs into bank: {0}")] FailedToLoadProgramsIntoBank(String), diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 2ec9ec010..bac9cc130 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -41,7 +41,13 @@ use magicblock_config::{EphemeralConfig, LifecycleMode, ProgramConfig}; use magicblock_geyser_plugin::rpc::GeyserRpcService; use magicblock_ledger::{ blockstore_processor::process_ledger, - ledger_truncator::{LedgerTruncator, DEFAULT_TRUNCATION_TIME_INTERVAL}, + ledger_size_manager::{ + config::{ + ExistingLedgerState, LedgerSizeManagerConfig, ResizePercentage, + CHECK_LEDGER_SIZE_INTERVAL_MS, + }, + TruncatingLedgerSizeManager, + }, Ledger, }; use magicblock_metrics::MetricsService; @@ -127,7 +133,7 @@ pub struct MagicValidator { token: CancellationToken, bank: Arc, ledger: Arc, - ledger_truncator: LedgerTruncator, + ledger_size_manager: TruncatingLedgerSizeManager, slot_ticker: Option>, pubsub_handle: RwLock>>, pubsub_close_handle: PubsubServiceCloseHandle, @@ -189,6 +195,23 @@ impl MagicValidator { config.validator_config.ledger.reset, )?; + let existing_ledger_state = (!config.validator_config.ledger.reset) + .then_some(ExistingLedgerState { + size: ledger.storage_size()?, + slot: ledger.last_slot(), + mod_id: ledger.last_mod_id(), + }); + let ledger_size_manager: TruncatingLedgerSizeManager = + TruncatingLedgerSizeManager::new_from_ledger( + ledger.clone(), + existing_ledger_state, + LedgerSizeManagerConfig { + max_size: config.validator_config.ledger.size, + size_check_interval_ms: CHECK_LEDGER_SIZE_INTERVAL_MS, + resize_percentage: ResizePercentage::Large, + }, + ); + let exit = Arc::::default(); // SAFETY: // this code will never panic as the ledger_path always appends the @@ -208,13 +231,6 @@ impl MagicValidator { ledger.get_max_blockhash().map(|(slot, _)| slot)?, )?; - let ledger_truncator = LedgerTruncator::new( - ledger.clone(), - bank.clone(), - DEFAULT_TRUNCATION_TIME_INTERVAL, - config.validator_config.ledger.size, - ); - fund_validator_identity(&bank, &validator_pubkey); fund_magic_context(&bank); let faucet_keypair = funded_faucet( @@ -366,7 +382,7 @@ impl MagicValidator { token, bank, ledger, - ledger_truncator, + ledger_size_manager, accounts_manager, transaction_listener, transaction_status_sender, @@ -654,7 +670,7 @@ impl MagicValidator { self.start_remote_account_updates_worker(); self.start_remote_account_cloner_worker().await?; - self.ledger_truncator.start(); + self.ledger_size_manager.try_start()?; self.rpc_service.start().map_err(|err| { ApiError::FailedToStartJsonRpcService(format!("{:?}", err)) @@ -755,7 +771,7 @@ impl MagicValidator { self.rpc_service.close(); PubsubService::close(&self.pubsub_close_handle); self.token.cancel(); - self.ledger_truncator.stop(); + self.ledger_size_manager.stop(); // wait a bit for services to stop thread::sleep(Duration::from_secs(1)); diff --git a/magicblock-ledger/src/ledger_size_manager/config.rs b/magicblock-ledger/src/ledger_size_manager/config.rs index 06951ca19..d96af65ba 100644 --- a/magicblock-ledger/src/ledger_size_manager/config.rs +++ b/magicblock-ledger/src/ledger_size_manager/config.rs @@ -1,5 +1,10 @@ +use std::time::Duration; + use solana_sdk::clock::Slot; +pub const CHECK_LEDGER_SIZE_INTERVAL_MS: u64 = + Duration::from_secs(2 * 60).as_millis() as u64; + /// Percentage of ledger to keep when resizing. pub enum ResizePercentage { /// Keep 75% of the ledger size. @@ -55,9 +60,9 @@ pub struct LedgerSizeManagerConfig { #[derive(Debug)] pub struct ExistingLedgerState { /// The current size of the ledger - pub(crate) size: u64, + pub size: u64, /// The last slot in the ledger at time of restart - pub(crate) slot: Slot, + pub slot: Slot, /// The last account mod ID in the ledger at time of restart - pub(crate) mod_id: u64, + pub mod_id: u64, } diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 88235de31..ffb5f502f 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -11,6 +11,7 @@ use errors::{LedgerSizeManagerError, LedgerSizeManagerResult}; use log::*; use std::{collections::VecDeque, sync::Arc, time::Duration}; use traits::ManagableLedger; +use truncator::Truncator; use watermarks::{Watermark, Watermarks}; use magicblock_metrics::metrics; @@ -40,37 +41,47 @@ enum ServiceState { }, } +pub type TruncatingLedgerSizeManager = LedgerSizeManager; pub struct LedgerSizeManager { ledger: Arc, - service_state: ServiceState, + service_state: Option, } impl LedgerSizeManager { + pub fn new_from_ledger( + ledger: Arc, + ledger_state: Option, + config: LedgerSizeManagerConfig, + ) -> LedgerSizeManager { + let managed_ledger = Truncator { ledger }; + LedgerSizeManager::new(Arc::new(managed_ledger), ledger_state, config) + } + pub(crate) fn new( - ledger: Arc, + managed_ledger: Arc, ledger_state: Option, config: LedgerSizeManagerConfig, ) -> Self { LedgerSizeManager { - ledger, - service_state: ServiceState::Created { + ledger: managed_ledger, + service_state: Some(ServiceState::Created { size_check_interval: Duration::from_millis( config.size_check_interval_ms, ), resize_percentage: config.resize_percentage, max_ledger_size: config.max_size, existing_ledger_state: ledger_state, - }, + }), } } - pub fn try_start(self) -> LedgerSizeManagerResult { - if let ServiceState::Created { + pub fn try_start(&mut self) -> LedgerSizeManagerResult<()> { + if let Some(ServiceState::Created { size_check_interval, resize_percentage, max_ledger_size, mut existing_ledger_state, - } = self.service_state + }) = self.service_state.take() { let cancellation_token = CancellationToken::new(); let worker_handle = { @@ -103,16 +114,14 @@ impl LedgerSizeManager { } }) }; - Ok(Self { - ledger: self.ledger, - service_state: ServiceState::Running { - cancellation_token, - worker_handle, - }, - }) + self.service_state = Some(ServiceState::Running { + cancellation_token, + worker_handle, + }); + Ok(()) } else { warn!("LedgerSizeManager already running, no need to start."); - Ok(self) + Ok(()) } } @@ -210,21 +219,18 @@ impl LedgerSizeManager { } } - pub fn stop(self) -> Self { - match self.service_state { - ServiceState::Running { + pub fn stop(&mut self) { + match self.service_state.take() { + Some(ServiceState::Running { cancellation_token, worker_handle, - } => { + }) => { cancellation_token.cancel(); - Self { - ledger: self.ledger, - service_state: ServiceState::Stopped { worker_handle }, - } + self.service_state = + Some(ServiceState::Stopped { worker_handle }); } _ => { warn!("LedgerSizeManager is not running, cannot stop."); - self } } } diff --git a/magicblock-ledger/src/ledger_size_manager/truncator.rs b/magicblock-ledger/src/ledger_size_manager/truncator.rs index 0771b2e9e..221de5620 100644 --- a/magicblock-ledger/src/ledger_size_manager/truncator.rs +++ b/magicblock-ledger/src/ledger_size_manager/truncator.rs @@ -18,7 +18,7 @@ use crate::{ use super::traits::ManagableLedger; pub struct Truncator { - ledger: Arc, + pub(crate) ledger: Arc, } impl Truncator { diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 9f3ed7519..a2d4b0aad 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -145,7 +145,7 @@ impl Ledger { measure.stop(); info!("Opening ledger done; {measure}"); - let ledger = Ledger { + let mut ledger = Ledger { ledger_path: ledger_path.to_path_buf(), db, @@ -166,6 +166,15 @@ impl Ledger { last_mod_id: AtomicU64::new(0), }; + ledger.last_slot = AtomicU64::new(ledger.get_max_blockhash()?.0); + ledger.last_mod_id = AtomicU64::new( + ledger + .account_mod_datas_cf + .iter(IteratorMode::End)? + .next() + .map_or(0, |(mod_id, _)| mod_id), + ); + Ok(ledger) } diff --git a/test-integration/Cargo.lock b/test-integration/Cargo.lock index 1db01db7c..2f8b3fad8 100644 --- a/test-integration/Cargo.lock +++ b/test-integration/Cargo.lock @@ -3715,6 +3715,7 @@ dependencies = [ name = "magicblock-ledger" version = "0.1.5" dependencies = [ + "async-trait", "bincode", "byteorder", "fs_extra", From caaf2860c889e27eae2bef66f22110ab3e631031 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Tue, 1 Jul 2025 18:01:48 +0800 Subject: [PATCH 19/36] feat: respect finality slot part 1 --- magicblock-api/src/magic_validator.rs | 22 +- .../src/ledger_size_manager/mod.rs | 275 +++++++++++++++--- .../src/ledger_size_manager/watermarks.rs | 4 + magicblock-ledger/src/lib.rs | 1 - 4 files changed, 247 insertions(+), 55 deletions(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index bac9cc130..bd33405f2 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -201,16 +201,6 @@ impl MagicValidator { slot: ledger.last_slot(), mod_id: ledger.last_mod_id(), }); - let ledger_size_manager: TruncatingLedgerSizeManager = - TruncatingLedgerSizeManager::new_from_ledger( - ledger.clone(), - existing_ledger_state, - LedgerSizeManagerConfig { - max_size: config.validator_config.ledger.size, - size_check_interval_ms: CHECK_LEDGER_SIZE_INTERVAL_MS, - resize_percentage: ResizePercentage::Large, - }, - ); let exit = Arc::::default(); // SAFETY: @@ -253,6 +243,18 @@ impl MagicValidator { Some(TransactionNotifier::new(geyser_manager)), ); + let ledger_size_manager: TruncatingLedgerSizeManager = + TruncatingLedgerSizeManager::new_from_ledger( + ledger.clone(), + bank.clone(), + existing_ledger_state, + LedgerSizeManagerConfig { + max_size: config.validator_config.ledger.size, + size_check_interval_ms: CHECK_LEDGER_SIZE_INTERVAL_MS, + resize_percentage: ResizePercentage::Large, + }, + ); + let metrics_config = &config.validator_config.metrics; let metrics = if metrics_config.enabled { let metrics_service = diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index ffb5f502f..7e336a68c 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -9,6 +9,8 @@ mod watermarks; use config::{ExistingLedgerState, LedgerSizeManagerConfig, ResizePercentage}; use errors::{LedgerSizeManagerError, LedgerSizeManagerResult}; use log::*; +use magicblock_bank::bank::Bank; +use magicblock_core::traits::FinalityProvider; use std::{collections::VecDeque, sync::Arc, time::Duration}; use traits::ManagableLedger; use truncator::Truncator; @@ -41,29 +43,38 @@ enum ServiceState { }, } -pub type TruncatingLedgerSizeManager = LedgerSizeManager; -pub struct LedgerSizeManager { +pub type TruncatingLedgerSizeManager = LedgerSizeManager; +pub struct LedgerSizeManager { ledger: Arc, + finality_provider: Arc, service_state: Option, } -impl LedgerSizeManager { +impl LedgerSizeManager { pub fn new_from_ledger( ledger: Arc, + finality_provider: Arc, ledger_state: Option, config: LedgerSizeManagerConfig, - ) -> LedgerSizeManager { + ) -> LedgerSizeManager { let managed_ledger = Truncator { ledger }; - LedgerSizeManager::new(Arc::new(managed_ledger), ledger_state, config) + LedgerSizeManager::new( + Arc::new(managed_ledger), + finality_provider, + ledger_state, + config, + ) } pub(crate) fn new( managed_ledger: Arc, + finality_provider: Arc, ledger_state: Option, config: LedgerSizeManagerConfig, ) -> Self { LedgerSizeManager { ledger: managed_ledger, + finality_provider, service_state: Some(ServiceState::Created { size_check_interval: Duration::from_millis( config.size_check_interval_ms, @@ -88,6 +99,8 @@ impl LedgerSizeManager { let ledger = self.ledger.clone(); ledger.initialize_lowest_cleanup_slot()?; + let finality_provider = self.finality_provider.clone(); + let mut interval = interval(size_check_interval); let mut watermarks = None::; @@ -101,6 +114,7 @@ impl LedgerSizeManager { _ = interval.tick() => { let ledger_size = Self::tick( &ledger, + &finality_provider, &mut watermarks, &resize_percentage, max_ledger_size, @@ -127,6 +141,7 @@ impl LedgerSizeManager { async fn tick( ledger: &Arc, + finality_provider: &Arc, watermarks: &mut Option, resize_percentage: &ResizePercentage, max_ledger_size: u64, @@ -150,6 +165,7 @@ impl LedgerSizeManager { Self::ensure_initial_max_ledger_size_below( ledger, + finality_provider, &existing_ledger_state, resize_percentage, max_ledger_size, @@ -198,8 +214,57 @@ impl LedgerSizeManager { last_slot, ledger.last_mod_id(), ); - if let Some(mark) = mark.as_ref() { - Self::truncate_ledger(ledger, mark).await; + if let Some(mark) = mark { + let latest_final_slot = + finality_provider.get_latest_final_slot(); + + let lowest_cleanup_slot = ledger.get_lowest_cleanup_slot(); + if lowest_cleanup_slot >= latest_final_slot { + warn!( + "Lowest cleanup slot {} is at or above the latest final slot {}. \ + Cannot truncate above the lowest cleanup slot.", + lowest_cleanup_slot, latest_final_slot + ); + wms.push_front(mark); + return Some(ledger_size); + } + + if mark.slot > latest_final_slot { + warn!("Truncation would remove data above the latest final slot {}. \ + Adjusting truncation for mark: {mark:?} to cut up to the latest final slot.", + latest_final_slot); + + // Estimate the size delta based on the ratio of the slots + // that we can remove + let original_diff = + mark.slot.saturating_sub(lowest_cleanup_slot); + let applied_diff = + latest_final_slot.saturating_sub(lowest_cleanup_slot); + let size_delta = (applied_diff as f64 + / original_diff as f64 + * mark.size_delta as f64) + as u64; + Self::truncate_ledger( + ledger, + latest_final_slot, + size_delta, + ) + .await; + + // Since we didn't truncate the full mark, we need to put one + // back so it will be processed to remove the remaining space + // when possible + // Otherwise we would process the following mark which would + // cause us to truncate too many slots + wms.push_front(Watermark { + slot: mark.slot, + mod_id: mark.mod_id, + size_delta: mark.size_delta.saturating_sub(size_delta), + }); + } else { + Self::truncate_ledger(ledger, mark.slot, mark.size_delta) + .await; + } if let Ok(ls) = ledger.storage_size() { ledger_size = ls; @@ -214,7 +279,7 @@ impl LedgerSizeManager { if captured { wms.size_at_last_capture = ledger_size; } - break Some(ledger_size); + return Some(ledger_size); } } } @@ -240,6 +305,7 @@ impl LedgerSizeManager { /// Returns the adjusted ledger size after truncation and the lowest cleanup slot. async fn ensure_initial_max_ledger_size_below( ledger: &Arc, + finality_provider: &Arc, existing_ledger_state: &ExistingLedgerState, resize_percentage: &ResizePercentage, max_size: u64, @@ -252,31 +318,61 @@ impl LedgerSizeManager { let mut ledger_size = *current_size; let mut total_slots = *total_slots; + let finality_slot = finality_provider.get_latest_final_slot(); + while ledger_size >= max_size { let avg_size_per_slot = *current_size as f64 / total_slots as f64; let target_size = resize_percentage.upper_mark_size(max_size); let target_slot = (target_size as f64 / avg_size_per_slot).floor() as Slot; + let cut_slots = total_slots.saturating_sub(target_slot); let current_lowest_slot = ledger.get_lowest_cleanup_slot(); let max_slot = current_lowest_slot .saturating_add(cut_slots) .min(total_slots); - ledger.truncate_fat_ledger(max_slot).await; + // We can either truncate up to the calculated slot and repeat this until + // we reach the target size or we can truncate up to the latest final slot + // and then have to stop. + if max_slot > finality_slot { + let lowest_cleanup_slot = ledger.get_lowest_cleanup_slot(); + if lowest_cleanup_slot >= finality_slot { + warn!( + "Lowest cleanup slot {} is above the latest final slot {}. \ + Initial truncation cannot truncate above the lowest cleanup slot.", + lowest_cleanup_slot, finality_slot + ); + return (ledger_size, lowest_cleanup_slot); + } - ledger_size = ledger.storage_size().unwrap_or(target_size); - total_slots -= cut_slots; + warn!( + "Initial truncation would remove data above the latest final slot {}. \ + Truncating only up to the latest final slot {}.", + finality_slot, max_slot + ); + ledger.truncate_fat_ledger(finality_slot).await; + + let ledger_size = ledger.storage_size().unwrap_or({ + (avg_size_per_slot * finality_slot as f64) as u64 + }); + return (ledger_size, ledger.get_lowest_cleanup_slot()); + } else { + ledger.truncate_fat_ledger(max_slot).await; + + ledger_size = ledger.storage_size().unwrap_or(target_size); + total_slots -= cut_slots; + } } (ledger_size, ledger.get_lowest_cleanup_slot()) } - async fn truncate_ledger(ledger: &Arc, mark: &Watermark) { + async fn truncate_ledger(ledger: &Arc, slot: Slot, size_delta: u64) { debug!( "Truncating ledger up to slot {} gaining {} bytes", - mark.slot, mark.size_delta + slot, size_delta ); - ledger.compact_slot_range(mark.slot).await; + ledger.compact_slot_range(slot).await; } } @@ -358,6 +454,24 @@ mod tests { } } + struct FinalityProviderMock { + finality_slot: Mutex, + } + + impl Default for FinalityProviderMock { + fn default() -> Self { + FinalityProviderMock { + finality_slot: Mutex::new(u64::MAX), + } + } + } + + impl FinalityProvider for FinalityProviderMock { + fn get_latest_final_slot(&self) -> u64 { + *self.finality_slot.lock().unwrap() + } + } + // ----------------- // Tests // ----------------- @@ -366,6 +480,7 @@ mod tests { init_logger!(); let ledger = Arc::new(ManageableLedgerMock::new(0, 0, 0)); + let finality_provider = Arc::new(FinalityProviderMock::default()); let mut watermarks = None::; let resize_percentage = ResizePercentage::Large; let max_ledger_size = 800; @@ -373,16 +488,19 @@ mod tests { macro_rules! tick { ($tick:expr) => {{ - let ledger_size = - LedgerSizeManager::::tick( - &ledger, - &mut watermarks, - &resize_percentage, - max_ledger_size, - &mut existing_ledger_state, - ) - .await - .unwrap(); + let ledger_size = LedgerSizeManager::< + ManageableLedgerMock, + FinalityProviderMock, + >::tick( + &ledger, + &finality_provider, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); debug!( "Ledger after tick {}: Size {} {:#?}", $tick, ledger_size, watermarks @@ -431,11 +549,73 @@ mod tests { assert_eq!(ledger_size, 500); } + #[tokio::test] + async fn test_ledger_size_manager_new_ledger_reaching_finality_slot() { + init_logger!(); + + let ledger = Arc::new(ManageableLedgerMock::new(0, 0, 0)); + let finality_provider = Arc::new(FinalityProviderMock { + finality_slot: Mutex::new(4), + }); + let mut watermarks = None::; + let resize_percentage = ResizePercentage::Large; + let max_ledger_size = 800; + let mut existing_ledger_state = None::; + + macro_rules! tick { + ($tick:expr) => {{ + let ledger_size = LedgerSizeManager::< + ManageableLedgerMock, + FinalityProviderMock, + >::tick( + &ledger, + &finality_provider, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); + debug!( + "Ledger after tick {}: Size {} {:#?}", + $tick, ledger_size, watermarks + ); + ledger_size + }}; + } + + info!("Slot: 0, New Ledger"); + let ledger_size = tick!(1); + assert_eq!(ledger_size, 0); + + info!("Slot: 1 added 1 slot -> 100 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(2); + assert_eq!(ledger_size, 100); + + info!("Slot: 2, added 1 slot -> 200 bytes marked (delta: 200)"); + ledger.add_slots(1); + let ledger_size = tick!(3); + assert_eq!(ledger_size, 200); + + info!("Slot: 8, added 6 slots -> 800 bytes marked (delta: 600)"); + ledger.add_slots(6); + let ledger_size = tick!(4); + assert_eq!(ledger_size, 600); + + info!("Slot: 12, added 4 slots -> 1000 bytes marked (delta: 400)"); + ledger.add_slots(4); + let ledger_size = tick!(5); + assert_eq!(ledger_size, 800); + } + #[tokio::test] async fn test_ledger_size_manager_existing_ledger_below_max_size() { init_logger!(); let ledger = Arc::new(ManageableLedgerMock::new(0, 6, 6)); + let finality_provider = Arc::new(FinalityProviderMock::default()); let mut watermarks = None::; let resize_percentage = ResizePercentage::Large; let max_ledger_size = 1000; @@ -447,16 +627,19 @@ mod tests { macro_rules! tick { () => {{ - let ledger_size = - LedgerSizeManager::::tick( - &ledger, - &mut watermarks, - &resize_percentage, - max_ledger_size, - &mut existing_ledger_state, - ) - .await - .unwrap(); + let ledger_size = LedgerSizeManager::< + ManageableLedgerMock, + FinalityProviderMock, + >::tick( + &ledger, + &finality_provider, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); debug!("Ledger Size {} {:#?}", ledger_size, watermarks); ledger_size }}; @@ -513,6 +696,7 @@ mod tests { init_logger!(); let ledger = Arc::new(ManageableLedgerMock::new(0, 12, 12)); + let finality_provider = Arc::new(FinalityProviderMock::default()); let mut watermarks = None::; let resize_percentage = ResizePercentage::Large; let max_ledger_size = 1000; @@ -524,16 +708,19 @@ mod tests { macro_rules! tick { () => {{ - let ledger_size = - LedgerSizeManager::::tick( - &ledger, - &mut watermarks, - &resize_percentage, - max_ledger_size, - &mut existing_ledger_state, - ) - .await - .unwrap(); + let ledger_size = LedgerSizeManager::< + ManageableLedgerMock, + FinalityProviderMock, + >::tick( + &ledger, + &finality_provider, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); debug!("Ledger Size {} {:#?}", ledger_size, watermarks); ledger_size }}; diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index cf8c142c1..95542ad2a 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -123,6 +123,10 @@ impl Watermarks { (mark, captured) } + pub fn push_front(&mut self, mark: Watermark) { + self.marks.push_front(mark); + } + fn update(&mut self, slot: u64, mod_id: u64, size: u64) -> bool { let size_delta = size.saturating_sub(self.size_at_last_capture); if size_delta >= self.mark_size { diff --git a/magicblock-ledger/src/lib.rs b/magicblock-ledger/src/lib.rs index 6f0b72ef4..55e7976ac 100644 --- a/magicblock-ledger/src/lib.rs +++ b/magicblock-ledger/src/lib.rs @@ -3,7 +3,6 @@ mod conversions; mod database; pub mod errors; pub mod ledger_size_manager; -pub mod ledger_truncator; mod metrics; mod store; From 231fa83e519fa09b7f1996c9190b785fb8cd13b5 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 10:58:30 +0800 Subject: [PATCH 20/36] feat: partial truncation working + tested --- .../src/ledger_size_manager/mod.rs | 84 ++++++++++++++++--- 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 7e336a68c..4f2740e40 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -226,11 +226,14 @@ impl LedgerSizeManager { lowest_cleanup_slot, latest_final_slot ); wms.push_front(mark); + if captured { + wms.size_at_last_capture = ledger_size; + } return Some(ledger_size); } if mark.slot > latest_final_slot { - warn!("Truncation would remove data above the latest final slot {}. \ + warn!("Truncation would remove data at or above the latest final slot {}. \ Adjusting truncation for mark: {mark:?} to cut up to the latest final slot.", latest_final_slot); @@ -250,6 +253,8 @@ impl LedgerSizeManager { size_delta, ) .await; + wms.size_at_last_capture = + wms.size_at_last_capture.saturating_sub(size_delta); // Since we didn't truncate the full mark, we need to put one // back so it will be processed to remove the remaining space @@ -264,6 +269,9 @@ impl LedgerSizeManager { } else { Self::truncate_ledger(ledger, mark.slot, mark.size_delta) .await; + wms.size_at_last_capture = wms + .size_at_last_capture + .saturating_sub(mark.size_delta); } if let Ok(ls) = ledger.storage_size() { @@ -563,7 +571,7 @@ mod tests { let mut existing_ledger_state = None::; macro_rules! tick { - ($tick:expr) => {{ + () => {{ let ledger_size = LedgerSizeManager::< ManageableLedgerMock, FinalityProviderMock, @@ -577,37 +585,89 @@ mod tests { ) .await .unwrap(); - debug!( - "Ledger after tick {}: Size {} {:#?}", - $tick, ledger_size, watermarks - ); + debug!("Ledger Size {} {:#?}", ledger_size, watermarks); ledger_size }}; } + macro_rules! wms { + ($size:expr, $len:expr) => { + let wms = watermarks.as_ref().unwrap(); + assert_eq!(wms.size_at_last_capture, $size); + assert_eq!(wms.marks.len(), $len); + }; + } + info!("Slot: 0, New Ledger"); - let ledger_size = tick!(1); + let ledger_size = tick!(); assert_eq!(ledger_size, 0); info!("Slot: 1 added 1 slot -> 100 bytes"); ledger.add_slots(1); - let ledger_size = tick!(2); + let ledger_size = tick!(); assert_eq!(ledger_size, 100); info!("Slot: 2, added 1 slot -> 200 bytes marked (delta: 200)"); ledger.add_slots(1); - let ledger_size = tick!(3); + let ledger_size = tick!(); assert_eq!(ledger_size, 200); info!("Slot: 8, added 6 slots -> 800 bytes marked (delta: 600)"); ledger.add_slots(6); - let ledger_size = tick!(4); + let ledger_size = tick!(); assert_eq!(ledger_size, 600); - info!("Slot: 12, added 4 slots -> 1000 bytes marked (delta: 400)"); + // It would normally remove 600 bytes, but the finality slot is 4 so + // it can only remove up to 4 instead 8 + info!("Slot: 12, added 4 slots -> 1000 bytes marked (delta: 400) -> remove 200 -> 800 bytes"); ledger.add_slots(4); - let ledger_size = tick!(5); + let ledger_size = tick!(); assert_eq!(ledger_size, 800); + wms!(800, 2); + + info!("Slot: 13, added 1 slot -> 900 bytes -> cannot remove anything"); + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 900); + wms!(800, 2); + + info!("Slot: 14 - 15, adding slots, but finality slot blocks removal until it is increased"); + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 1_000); + wms!(1_000, 3); + + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 1_100); + wms!(1_000, 3); + + *finality_provider.finality_slot.lock().unwrap() = 14; + let ledger_size = tick!(); + assert_eq!(ledger_size, 700); + // We cut 400 bytes, so the size at last capture is adjusted down + wms!(600, 2); + + info!( + "Slot: 16, added 1 slot -> marked (delta: 200) -> remove 400 -> 400 bytes" + ); + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 400); + wms!(400, 2); + + info!("Slot: 17-20, added 3 slots -> 700 bytes marked (delta: 300) + set finality 19"); + ledger.add_slots(3); + *finality_provider.finality_slot.lock().unwrap() = 19; + let ledger_size = tick!(); + assert_eq!(ledger_size, 700); + wms!(700, 3); + + info!("Slot: 21, added 1 slot -> remove 200 -> 600 bytes"); + ledger.add_slots(1); + let ledger_size = tick!(); + assert_eq!(ledger_size, 600); + wms!(500, 2); } #[tokio::test] From ef7fd9f0a55789a6be44f20c1372f5eb1a2462ca Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 11:04:10 +0800 Subject: [PATCH 21/36] feat: initial partial truncation working + tested --- .../src/ledger_size_manager/mod.rs | 47 ++++++++++++++++++- 1 file changed, 45 insertions(+), 2 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 4f2740e40..6b4a38f05 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -666,8 +666,6 @@ mod tests { info!("Slot: 21, added 1 slot -> remove 200 -> 600 bytes"); ledger.add_slots(1); let ledger_size = tick!(); - assert_eq!(ledger_size, 600); - wms!(500, 2); } #[tokio::test] @@ -845,4 +843,49 @@ mod tests { }, ); } + + #[tokio::test] + async fn test_ledger_size_manager_existing_ledger_above_max_size_finality_slot_blocking_full_truncation( + ) { + init_logger!(); + + let ledger = Arc::new(ManageableLedgerMock::new(0, 12, 12)); + let finality_provider = Arc::new(FinalityProviderMock { + finality_slot: Mutex::new(3), + }); + let mut watermarks = None::; + let resize_percentage = ResizePercentage::Large; + let max_ledger_size = 1000; + let mut existing_ledger_state = Some(ExistingLedgerState { + size: 1200, + slot: 12, + mod_id: 12, + }); + + macro_rules! tick { + () => {{ + let ledger_size = LedgerSizeManager::< + ManageableLedgerMock, + FinalityProviderMock, + >::tick( + &ledger, + &finality_provider, + &mut watermarks, + &resize_percentage, + max_ledger_size, + &mut existing_ledger_state, + ) + .await + .unwrap(); + debug!("Ledger Size {} {:#?}", ledger_size, watermarks); + ledger_size + }}; + } + + info!("Slot: 12, existing ledger above max size"); + let ledger_size = tick!(); + // We cannot truncate above the finality slot, so we only get 300 bytes back and + // stay above the max size + assert_eq!(ledger_size, 900); + } } From 0d340530849d27ae4336a74e0d885f2d18fa2e15 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 11:04:30 +0800 Subject: [PATCH 22/36] chore: remove test_ledger_truncator.rs --- .../tests/test_ledger_truncator.rs | 296 ------------------ 1 file changed, 296 deletions(-) delete mode 100644 magicblock-ledger/tests/test_ledger_truncator.rs diff --git a/magicblock-ledger/tests/test_ledger_truncator.rs b/magicblock-ledger/tests/test_ledger_truncator.rs deleted file mode 100644 index 799891e9d..000000000 --- a/magicblock-ledger/tests/test_ledger_truncator.rs +++ /dev/null @@ -1,296 +0,0 @@ -mod common; -use std::{ - sync::{ - atomic::{AtomicU64, Ordering}, - Arc, - }, - time::Duration, -}; - -use magicblock_core::traits::FinalityProvider; -use magicblock_ledger::{ledger_truncator::LedgerTruncator, Ledger}; -use solana_sdk::{hash::Hash, signature::Signature}; - -use crate::common::{setup, write_dummy_transaction}; - -const TEST_TRUNCATION_TIME_INTERVAL: Duration = Duration::from_millis(50); -#[derive(Default)] -pub struct TestFinalityProvider { - pub latest_final_slot: AtomicU64, -} - -impl FinalityProvider for TestFinalityProvider { - fn get_latest_final_slot(&self) -> u64 { - self.latest_final_slot.load(Ordering::Relaxed) - } -} - -fn verify_transactions_state( - ledger: &Ledger, - start_slot: u64, - signatures: &[Signature], - shall_exist: bool, -) { - for (offset, signature) in signatures.iter().enumerate() { - let slot = start_slot + offset as u64; - assert_eq!( - ledger.read_slot_signature((slot, 0)).unwrap().is_some(), - shall_exist - ); - assert_eq!( - ledger - .read_transaction((*signature, slot)) - .unwrap() - .is_some(), - shall_exist - ); - assert_eq!( - ledger - .read_transaction_status((*signature, slot)) - .unwrap() - .is_some(), - shall_exist - ) - } -} - -/// Tests that ledger is not truncated if finality slot - 0 -#[tokio::test] -async fn test_truncator_not_purged_finality() { - const SLOT_TRUNCATION_INTERVAL: u64 = 5; - - let ledger = Arc::new(setup()); - let finality_provider = TestFinalityProvider { - latest_final_slot: 0.into(), - }; - - let mut ledger_truncator = LedgerTruncator::new( - ledger.clone(), - Arc::new(finality_provider), - TEST_TRUNCATION_TIME_INTERVAL, - 0, - ); - - for i in 0..SLOT_TRUNCATION_INTERVAL { - write_dummy_transaction(&ledger, i, 0); - ledger.write_block(i, 0, Hash::new_unique()).unwrap() - } - let signatures = (0..SLOT_TRUNCATION_INTERVAL) - .map(|i| { - let signature = ledger.read_slot_signature((i, 0)).unwrap(); - assert!(signature.is_some()); - - signature.unwrap() - }) - .collect::>(); - - ledger_truncator.start(); - tokio::time::sleep(Duration::from_millis(10)).await; - ledger_truncator.stop(); - assert!(ledger_truncator.join().await.is_ok()); - - // Not truncated due to final_slot 0 - verify_transactions_state(&ledger, 0, &signatures, true); -} - -// Tests that ledger is not truncated while there is still enough space -#[tokio::test] -async fn test_truncator_not_purged_size() { - const NUM_TRANSACTIONS: u64 = 100; - - let ledger = Arc::new(setup()); - let finality_provider = TestFinalityProvider { - latest_final_slot: 0.into(), - }; - - let mut ledger_truncator = LedgerTruncator::new( - ledger.clone(), - Arc::new(finality_provider), - TEST_TRUNCATION_TIME_INTERVAL, - 1 << 30, // 1 GB - ); - - for i in 0..NUM_TRANSACTIONS { - write_dummy_transaction(&ledger, i, 0); - ledger.write_block(i, 0, Hash::new_unique()).unwrap() - } - let signatures = (0..NUM_TRANSACTIONS) - .map(|i| { - let signature = ledger.read_slot_signature((i, 0)).unwrap(); - assert!(signature.is_some()); - - signature.unwrap() - }) - .collect::>(); - - ledger_truncator.start(); - tokio::time::sleep(Duration::from_millis(10)).await; - ledger_truncator.stop(); - assert!(ledger_truncator.join().await.is_ok()); - - // Not truncated due to final_slot 0 - verify_transactions_state(&ledger, 0, &signatures, true); -} - -// Tests that ledger got truncated but not after finality slot -#[tokio::test] -async fn test_truncator_non_empty_ledger() { - const FINAL_SLOT: u64 = 80; - - let ledger = Arc::new(setup()); - let signatures = (0..FINAL_SLOT + 20) - .map(|i| { - let (_, signature) = write_dummy_transaction(&ledger, i, 0); - ledger.write_block(i, 0, Hash::new_unique()).unwrap(); - signature - }) - .collect::>(); - - let finality_provider = Arc::new(TestFinalityProvider { - latest_final_slot: FINAL_SLOT.into(), - }); - - let mut ledger_truncator = LedgerTruncator::new( - ledger.clone(), - finality_provider, - TEST_TRUNCATION_TIME_INTERVAL, - 0, - ); - - ledger_truncator.start(); - tokio::time::sleep(TEST_TRUNCATION_TIME_INTERVAL).await; - - ledger_truncator.stop(); - assert!(ledger_truncator.join().await.is_ok()); - - let cleanup_slot = ledger.get_lowest_cleanup_slot(); - assert_ne!(ledger.get_lowest_cleanup_slot(), 0); - verify_transactions_state( - &ledger, - 0, - &signatures[..(cleanup_slot + 1) as usize], - false, - ); - verify_transactions_state( - &ledger, - cleanup_slot + 1, - &signatures[(cleanup_slot + 1) as usize..], - true, - ); -} - -async fn transaction_spammer( - ledger: Arc, - finality_provider: Arc, - num_of_iterations: usize, - tx_per_operation: usize, -) -> Vec { - let mut signatures = - Vec::with_capacity(num_of_iterations * tx_per_operation); - for _ in 0..num_of_iterations { - for _ in 0..tx_per_operation { - let slot = signatures.len() as u64; - let (_, signature) = write_dummy_transaction(&ledger, slot, 0); - ledger.write_block(slot, 0, Hash::new_unique()).unwrap(); - signatures.push(signature); - } - - finality_provider - .latest_final_slot - .store(signatures.len() as u64 - 1, Ordering::Relaxed); - tokio::time::sleep(Duration::from_millis(10)).await; - } - - signatures -} - -// Tests if ledger truncated correctly during tx spamming with finality slot increments -#[tokio::test] -async fn test_truncator_with_tx_spammer() { - let ledger = Arc::new(setup()); - let finality_provider = Arc::new(TestFinalityProvider { - latest_final_slot: 0.into(), - }); - - let mut ledger_truncator = LedgerTruncator::new( - ledger.clone(), - finality_provider.clone(), - TEST_TRUNCATION_TIME_INTERVAL, - 0, - ); - - ledger_truncator.start(); - let handle = tokio::spawn(transaction_spammer( - ledger.clone(), - finality_provider.clone(), - 10, - 20, - )); - - // Sleep some time - tokio::time::sleep(Duration::from_secs(3)).await; - - let signatures_result = handle.await; - assert!(signatures_result.is_ok()); - let signatures = signatures_result.unwrap(); - - // Stop truncator assuming that complete after sleep - ledger_truncator.stop(); - assert!(ledger_truncator.join().await.is_ok()); - - assert!(ledger.flush().is_ok()); - - let lowest_existing = - finality_provider.latest_final_slot.load(Ordering::Relaxed); - assert_eq!(ledger.get_lowest_cleanup_slot(), lowest_existing - 1); - verify_transactions_state( - &ledger, - 0, - &signatures[..lowest_existing as usize], - false, - ); - verify_transactions_state( - &ledger, - lowest_existing, - &signatures[lowest_existing as usize..], - true, - ); -} - -#[ignore = "Long running test"] -#[tokio::test] -async fn test_with_1gb_db() { - const DB_SIZE: u64 = 1 << 30; - const CHECK_RATE: u64 = 100; - - // let ledger = Arc::new(Ledger::open(Path::new("/var/folders/r9/q7l5l9ks1vs1nlv10vlpkhw80000gn/T/.tmp00LEDc/rocksd")).unwrap()); - let ledger = Arc::new(setup()); - - let mut slot = 0; - loop { - if slot % CHECK_RATE == 0 && ledger.storage_size().unwrap() >= DB_SIZE { - break; - } - - write_dummy_transaction(&ledger, slot, 0); - ledger.write_block(slot, 0, Hash::new_unique()).unwrap(); - slot += 1 - } - - let finality_provider = Arc::new(TestFinalityProvider { - latest_final_slot: AtomicU64::new(slot - 1), - }); - - let mut ledger_truncator = LedgerTruncator::new( - ledger.clone(), - finality_provider.clone(), - TEST_TRUNCATION_TIME_INTERVAL, - DB_SIZE, - ); - - ledger_truncator.start(); - tokio::time::sleep(Duration::from_secs(1)).await; - ledger_truncator.stop(); - - ledger_truncator.join().await.unwrap(); -} From 4043fa9e70706a1f5295d814e709965a61a68178 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 11:15:23 +0800 Subject: [PATCH 23/36] chore: fmt --- magicblock-ledger/src/ledger_size_manager/mod.rs | 10 +++++----- magicblock-ledger/src/ledger_size_manager/truncator.rs | 9 ++++----- .../src/ledger_size_manager/watermarks.rs | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 6b4a38f05..705cdb36b 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -6,16 +6,13 @@ pub mod traits; mod truncator; mod watermarks; +use std::{collections::VecDeque, sync::Arc, time::Duration}; + use config::{ExistingLedgerState, LedgerSizeManagerConfig, ResizePercentage}; use errors::{LedgerSizeManagerError, LedgerSizeManagerResult}; use log::*; use magicblock_bank::bank::Bank; use magicblock_core::traits::FinalityProvider; -use std::{collections::VecDeque, sync::Arc, time::Duration}; -use traits::ManagableLedger; -use truncator::Truncator; -use watermarks::{Watermark, Watermarks}; - use magicblock_metrics::metrics; use solana_sdk::clock::Slot; use thiserror::Error; @@ -24,6 +21,9 @@ use tokio::{ time::interval, }; use tokio_util::sync::CancellationToken; +use traits::ManagableLedger; +use truncator::Truncator; +use watermarks::{Watermark, Watermarks}; use crate::Ledger; diff --git a/magicblock-ledger/src/ledger_size_manager/truncator.rs b/magicblock-ledger/src/ledger_size_manager/truncator.rs index 221de5620..d010de078 100644 --- a/magicblock-ledger/src/ledger_size_manager/truncator.rs +++ b/magicblock-ledger/src/ledger_size_manager/truncator.rs @@ -1,11 +1,12 @@ +use std::sync::Arc; + use async_trait::async_trait; use log::*; use solana_measure::measure::Measure; -use std::sync::Arc; -use tokio::task::JoinSet; - use solana_sdk::clock::Slot; +use tokio::task::JoinSet; +use super::traits::ManagableLedger; use crate::{ database::columns::{ AddressSignatures, Blockhash, Blocktime, PerfSamples, SlotSignatures, @@ -15,8 +16,6 @@ use crate::{ Ledger, }; -use super::traits::ManagableLedger; - pub struct Truncator { pub(crate) ledger: Arc, } diff --git a/magicblock-ledger/src/ledger_size_manager/watermarks.rs b/magicblock-ledger/src/ledger_size_manager/watermarks.rs index 95542ad2a..b84be3c74 100644 --- a/magicblock-ledger/src/ledger_size_manager/watermarks.rs +++ b/magicblock-ledger/src/ledger_size_manager/watermarks.rs @@ -1,6 +1,6 @@ -use log::*; use std::collections::VecDeque; +use log::*; use solana_sdk::clock::Slot; use super::config::{ExistingLedgerState, ResizePercentage}; From 2dd795fa93d9b8bc5c8385c08668bd6d2b08fcb8 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 11:33:24 +0800 Subject: [PATCH 24/36] chore: apply greptile suggestions --- magicblock-ledger/src/ledger_size_manager/config.rs | 2 +- magicblock-ledger/src/ledger_size_manager/mod.rs | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/magicblock-ledger/src/ledger_size_manager/config.rs b/magicblock-ledger/src/ledger_size_manager/config.rs index d96af65ba..8ba643043 100644 --- a/magicblock-ledger/src/ledger_size_manager/config.rs +++ b/magicblock-ledger/src/ledger_size_manager/config.rs @@ -46,7 +46,7 @@ impl ResizePercentage { pub struct LedgerSizeManagerConfig { /// Max ledger size to maintain. - /// The [LedgerSizeManager] will attempt to respect this size,, but + /// The [LedgerSizeManager] will attempt to respect this size, but /// it may grow larger temporarily in between size checks. pub max_size: u64, diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 705cdb36b..5941e0f28 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -666,6 +666,7 @@ mod tests { info!("Slot: 21, added 1 slot -> remove 200 -> 600 bytes"); ledger.add_slots(1); let ledger_size = tick!(); + assert_eq!(ledger_size, 600); } #[tokio::test] From 271cf654b1f9efc5e43e30339f628ee88d52e4fc Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Wed, 2 Jul 2025 11:43:58 +0800 Subject: [PATCH 25/36] fix: count account mod data write metrics --- magicblock-ledger/src/store/api.rs | 1 + magicblock-metrics/src/metrics/mod.rs | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index a2d4b0aad..48547990d 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -1130,6 +1130,7 @@ impl Ledger { id: u64, data: &AccountModData, ) -> LedgerResult<()> { + metrics::inc_ledger_account_mod_data_count(); self.account_mod_datas_cf.put(id, data)?; self.last_mod_id.store(id, Ordering::Relaxed); Ok(()) diff --git a/magicblock-metrics/src/metrics/mod.rs b/magicblock-metrics/src/metrics/mod.rs index cf9130958..3592678e3 100644 --- a/magicblock-metrics/src/metrics/mod.rs +++ b/magicblock-metrics/src/metrics/mod.rs @@ -383,8 +383,8 @@ pub fn inc_ledger_perf_samples_count() { LEDGER_PERF_SAMPLES_COUNT.inc(); } -pub fn inc_ledger_account_mod_data_count(count: u64) { - LEDGER_ACCOUNT_MOD_DATA_COUNT.inc_by(count); +pub fn inc_ledger_account_mod_data_count() { + LEDGER_ACCOUNT_MOD_DATA_COUNT.inc(); } pub fn set_accounts_size(size: u64) { From ef0eb03296d811e6bce75e9964cbfa4e0a48fdeb Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 09:49:46 +0530 Subject: [PATCH 26/36] fix: taco-paco's review comment 1 Amp-Thread: https://ampcode.com/threads/T-ac49f6d4-4ea8-465a-a7c1-0f5a72ccde2c Co-authored-by: Amp --- magicblock-ledger/src/store/api.rs | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index 48547990d..b78ae2f87 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -279,20 +279,18 @@ impl Ledger { /// Initializes lowest slot to cleanup from pub fn initialize_lowest_cleanup_slot(&self) -> Result<(), LedgerError> { - match self.blockhash_cf.iter(IteratorMode::Start)?.next() { - Some((lowest_slot, _)) => { - *self - .lowest_cleanup_slot - .write() - .expect(Self::LOWEST_CLEANUP_SLOT_POISONED) = lowest_slot; - } - None => { - *self - .lowest_cleanup_slot - .write() - .expect(Self::LOWEST_CLEANUP_SLOT_POISONED) = 0; - } - } + let lowest_cleanup_slot = match self.blockhash_cf.iter(IteratorMode::Start)?.next() { + Some((lowest_slot, _)) if lowest_slot > 0 => lowest_slot - 1, + _ => 0, + }; + + info!("initializing lowest cleanup slot: {}", lowest_cleanup_slot); + *self + .lowest_cleanup_slot + .write() + .expect(Self::LOWEST_CLEANUP_SLOT_POISONED) = lowest_cleanup_slot; + + self.db.set_oldest_slot(lowest_cleanup_slot); Ok(()) } From 50ff3140badff84a97d0014607126b65e01d47bc Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 11:44:56 +0530 Subject: [PATCH 27/36] fix: ef0eb032 solution was slightly incorrect --- magicblock-ledger/src/store/api.rs | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/magicblock-ledger/src/store/api.rs b/magicblock-ledger/src/store/api.rs index b78ae2f87..3312d1bb5 100644 --- a/magicblock-ledger/src/store/api.rs +++ b/magicblock-ledger/src/store/api.rs @@ -279,10 +279,11 @@ impl Ledger { /// Initializes lowest slot to cleanup from pub fn initialize_lowest_cleanup_slot(&self) -> Result<(), LedgerError> { - let lowest_cleanup_slot = match self.blockhash_cf.iter(IteratorMode::Start)?.next() { - Some((lowest_slot, _)) if lowest_slot > 0 => lowest_slot - 1, - _ => 0, - }; + let lowest_cleanup_slot = + match self.blockhash_cf.iter(IteratorMode::Start)?.next() { + Some((lowest_slot, _)) if lowest_slot > 0 => lowest_slot - 1, + _ => 0, + }; info!("initializing lowest cleanup slot: {}", lowest_cleanup_slot); *self @@ -290,7 +291,7 @@ impl Ledger { .write() .expect(Self::LOWEST_CLEANUP_SLOT_POISONED) = lowest_cleanup_slot; - self.db.set_oldest_slot(lowest_cleanup_slot); + self.set_lowest_cleanup_slot(lowest_cleanup_slot); Ok(()) } From 6d1793ed13b31ae0ceb8f173909bbb60acb90279 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 09:51:01 +0530 Subject: [PATCH 28/36] fix: taco-paco's review comment 2 Amp-Thread: https://ampcode.com/threads/T-ac49f6d4-4ea8-465a-a7c1-0f5a72ccde2c Co-authored-by: Amp --- magicblock-ledger/src/ledger_truncator.rs | 339 ---------------------- 1 file changed, 339 deletions(-) delete mode 100644 magicblock-ledger/src/ledger_truncator.rs diff --git a/magicblock-ledger/src/ledger_truncator.rs b/magicblock-ledger/src/ledger_truncator.rs deleted file mode 100644 index dad285cc9..000000000 --- a/magicblock-ledger/src/ledger_truncator.rs +++ /dev/null @@ -1,339 +0,0 @@ -use std::{cmp::min, sync::Arc, time::Duration}; - -use log::{error, info, warn}; -use magicblock_core::traits::FinalityProvider; -use tokio::{ - task::{JoinError, JoinHandle, JoinSet}, - time::interval, -}; -use tokio_util::sync::CancellationToken; - -use crate::{ - database::columns::{ - AddressSignatures, Blockhash, Blocktime, PerfSamples, SlotSignatures, - Transaction, TransactionMemos, TransactionStatus, - }, - errors::LedgerResult, - Ledger, -}; - -pub const DEFAULT_TRUNCATION_TIME_INTERVAL: Duration = - Duration::from_secs(2 * 60); -const PERCENTAGE_TO_TRUNCATE: u8 = 10; - -struct LedgerTrunctationWorker { - finality_provider: Arc, - ledger: Arc, - truncation_time_interval: Duration, - ledger_size: u64, - cancellation_token: CancellationToken, -} - -impl LedgerTrunctationWorker { - pub fn new( - ledger: Arc, - finality_provider: Arc, - truncation_time_interval: Duration, - ledger_size: u64, - cancellation_token: CancellationToken, - ) -> Self { - Self { - ledger, - finality_provider, - truncation_time_interval, - ledger_size, - cancellation_token, - } - } - - pub async fn run(self) { - self.ledger - .initialize_lowest_cleanup_slot() - .expect("Lowest cleanup slot initialization"); - let mut interval = interval(self.truncation_time_interval); - loop { - tokio::select! { - _ = self.cancellation_token.cancelled() => { - return; - } - _ = interval.tick() => { - // Note: since we clean 10%, tomstones will take around 10% as well - const FILLED_PERCENTAGE_LIMIT: u8 = 100 - PERCENTAGE_TO_TRUNCATE; - - let current_size = match self.ledger.storage_size() { - Ok(value) => value, - Err(err) => { - error!("Failed to check truncation condition: {err}"); - continue; - } - }; - - // Check if we should truncate - if current_size < (self.ledger_size / 100) * FILLED_PERCENTAGE_LIMIT as u64 { - continue; - } - - info!("Ledger size: {current_size}"); - match self.estimate_truncation_range(current_size) { - Ok(Some((from_slot, to_slot))) => Self::truncate_slot_range(&self.ledger, from_slot, to_slot).await, - Ok(None) => warn!("Could not estimate truncation range"), - Err(err) => error!("Failed to estimate truncation range: {:?}", err), - } - } - } - } - } - - /// Returns range to truncate [from_slot, to_slot] - fn estimate_truncation_range( - &self, - current_ledger_size: u64, - ) -> LedgerResult> { - let (from_slot, to_slot) = - if let Some(val) = self.available_truncation_range() { - val - } else { - return Ok(None); - }; - - let num_slots = self.ledger.count_blockhashes()?; - if num_slots == 0 { - info!("No slot were written yet. Nothing to truncate!"); - return Ok(None); - } - - let slot_size = current_ledger_size / num_slots as u64; - let size_to_truncate = - (current_ledger_size / 100) * PERCENTAGE_TO_TRUNCATE as u64; - let num_slots_to_truncate = size_to_truncate / slot_size; - - let to_slot = min(from_slot + num_slots_to_truncate, to_slot); - Ok(Some((from_slot, to_slot))) - } - - /// Returns [from_slot, to_slot] range that's safe to truncate - fn available_truncation_range(&self) -> Option<(u64, u64)> { - let lowest_cleanup_slot = self.ledger.get_lowest_cleanup_slot(); - let latest_final_slot = self.finality_provider.get_latest_final_slot(); - - if latest_final_slot <= lowest_cleanup_slot { - // Could both be 0 at startup, no need to report - if lowest_cleanup_slot != 0 { - // This could not happen because of Truncator - warn!("Slots after latest final slot have been truncated!"); - } - - info!( - "Lowest cleanup slot ge than latest final slot. {}, {}", - lowest_cleanup_slot, latest_final_slot - ); - return None; - } - // Nothing to truncate - if latest_final_slot == lowest_cleanup_slot + 1 { - info!("Nothing to truncate"); - return None; - } - - // Fresh start case - let next_from_slot = if lowest_cleanup_slot == 0 { - 0 - } else { - lowest_cleanup_slot + 1 - }; - - // we don't clean latest final slot - Some((next_from_slot, latest_final_slot - 1)) - } - - /// Utility function for splitting truncation into smaller chunks - /// Cleans slots [from_slot; to_slot] inclusive range - pub async fn truncate_slot_range( - ledger: &Arc, - from_slot: u64, - to_slot: u64, - ) { - // In order not to torture RocksDB's WriteBatch we split large tasks into chunks - const SINGLE_TRUNCATION_LIMIT: usize = 300; - - if to_slot < from_slot { - warn!("LedgerTruncator: Nani?"); - return; - } - - info!( - "LedgerTruncator: truncating slot range [{from_slot}; {to_slot}]" - ); - (from_slot..=to_slot) - .step_by(SINGLE_TRUNCATION_LIMIT) - .for_each(|cur_from_slot| { - let num_slots_to_truncate = min( - to_slot - cur_from_slot + 1, - SINGLE_TRUNCATION_LIMIT as u64, - ); - let truncate_to_slot = - cur_from_slot + num_slots_to_truncate - 1; - - if let Err(err) = - ledger.delete_slot_range(cur_from_slot, truncate_to_slot) - { - warn!( - "Failed to truncate slots {}-{}: {}", - cur_from_slot, truncate_to_slot, err - ); - } - }); - // Flush memtables with tombstones prior to compaction - if let Err(err) = ledger.flush() { - error!("Failed to flush ledger: {err}"); - } - - Self::compact_slot_range(ledger, from_slot, to_slot).await; - } - - /// Synchronous utility function that triggers and awaits compaction on all the columns - pub async fn compact_slot_range( - ledger: &Arc, - from_slot: u64, - to_slot: u64, - ) { - if to_slot < from_slot { - warn!("LedgerTruncator: Nani2?"); - return; - } - - // Compaction can be run concurrently for different cf - // but it utilizes rocksdb threads, in order not to drain - // our tokio rt threads, we split the effort in just 3 tasks - let mut join_set = JoinSet::new(); - join_set.spawn({ - let ledger = ledger.clone(); - async move { - ledger.compact_slot_range_cf::( - Some(from_slot), - Some(to_slot + 1), - ); - ledger.compact_slot_range_cf::( - Some(from_slot), - Some(to_slot + 1), - ); - ledger.compact_slot_range_cf::( - Some(from_slot), - Some(to_slot + 1), - ); - ledger.compact_slot_range_cf::( - Some((from_slot, u32::MIN)), - Some((to_slot + 1, u32::MAX)), - ); - } - }); - - // Can not compact with specific range - join_set.spawn({ - let ledger = ledger.clone(); - async move { - ledger.compact_slot_range_cf::(None, None); - ledger.compact_slot_range_cf::(None, None); - } - }); - join_set.spawn({ - let ledger = ledger.clone(); - async move { - ledger.compact_slot_range_cf::(None, None); - ledger.compact_slot_range_cf::(None, None); - } - }); - - let _ = join_set.join_all().await; - } -} - -#[derive(Debug)] -struct WorkerController { - cancellation_token: CancellationToken, - worker_handle: JoinHandle<()>, -} - -#[derive(Debug)] -enum ServiceState { - Created, - Running(WorkerController), - Stopped(JoinHandle<()>), -} - -pub struct LedgerTruncator { - finality_provider: Arc, - ledger: Arc, - ledger_size: u64, - truncation_time_interval: Duration, - state: ServiceState, -} - -impl LedgerTruncator { - pub fn new( - ledger: Arc, - finality_provider: Arc, - truncation_time_interval: Duration, - ledger_size: u64, - ) -> Self { - Self { - ledger, - finality_provider, - truncation_time_interval, - ledger_size, - state: ServiceState::Created, - } - } - - pub fn start(&mut self) { - if let ServiceState::Created = self.state { - let cancellation_token = CancellationToken::new(); - let worker = LedgerTrunctationWorker::new( - self.ledger.clone(), - self.finality_provider.clone(), - self.truncation_time_interval, - self.ledger_size, - cancellation_token.clone(), - ); - let worker_handle = tokio::spawn(worker.run()); - - self.state = ServiceState::Running(WorkerController { - cancellation_token, - worker_handle, - }) - } else { - warn!("LedgerTruncator already running, no need to start."); - } - } - - pub fn stop(&mut self) { - let state = std::mem::replace(&mut self.state, ServiceState::Created); - if let ServiceState::Running(controller) = state { - controller.cancellation_token.cancel(); - self.state = ServiceState::Stopped(controller.worker_handle); - } else { - warn!("LedgerTruncator not running, can not be stopped."); - self.state = state; - } - } - - pub async fn join(mut self) -> Result<(), LedgerTruncatorError> { - if matches!(self.state, ServiceState::Running(_)) { - self.stop(); - } - - if let ServiceState::Stopped(worker_handle) = self.state { - worker_handle.await?; - Ok(()) - } else { - warn!("LedgerTruncator was not running, nothing to stop"); - Ok(()) - } - } -} - -#[derive(thiserror::Error, Debug)] -pub enum LedgerTruncatorError { - #[error("Failed to join worker: {0}")] - JoinError(#[from] JoinError), -} From 8662b314950f31042ea8faa5de3dde9e47675ca4 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 10:18:38 +0530 Subject: [PATCH 29/36] fix: taco-paco's review comment 3 Amp-Thread: https://ampcode.com/threads/T-ac49f6d4-4ea8-465a-a7c1-0f5a72ccde2c Co-authored-by: Amp --- magicblock-ledger/src/ledger_size_manager/mod.rs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 5941e0f28..4feb8843a 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -102,10 +102,9 @@ impl LedgerSizeManager { let finality_provider = self.finality_provider.clone(); let mut interval = interval(size_check_interval); - - let mut watermarks = None::; let mut cancellation_token = cancellation_token.clone(); tokio::spawn(async move { + let mut watermarks = None::; loop { tokio::select! { _ = cancellation_token.cancelled() => { From 74540b69a050c5adfdfc9a4ace80c8ec2f3c647a Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 10:20:03 +0530 Subject: [PATCH 30/36] fix: taco-paco's review comment 6 Amp-Thread: https://ampcode.com/threads/T-ac49f6d4-4ea8-465a-a7c1-0f5a72ccde2c Co-authored-by: Amp --- magicblock-ledger/src/ledger_size_manager/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 4feb8843a..8c9909036 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -101,9 +101,9 @@ impl LedgerSizeManager { let finality_provider = self.finality_provider.clone(); - let mut interval = interval(size_check_interval); let mut cancellation_token = cancellation_token.clone(); tokio::spawn(async move { + let mut interval = interval(size_check_interval); let mut watermarks = None::; loop { tokio::select! { From 9d843d42b10e27785f064385a87bd032960b028e Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 10:43:36 +0530 Subject: [PATCH 31/36] fix: taco-paco's review comment 7 --- magicblock-api/src/magic_validator.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index 6ebec9b19..ab02c8433 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -211,7 +211,8 @@ impl MagicValidator { config.validator_config.ledger.reset, )?; - let existing_ledger_state = (!config.validator_config.ledger.reset) + let existing_ledger_state = (!config.validator_config.ledger.reset + && ledger.last_slot() > 0) .then_some(ExistingLedgerState { size: ledger.storage_size()?, slot: ledger.last_slot(), From 428d829a39d14192d710ffe35bb6ada04ab3ab46 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 11:17:15 +0530 Subject: [PATCH 32/36] fix: taco-paco's review comment 5 --- .../src/ledger_size_manager/mod.rs | 82 +++++++++++-------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 8c9909036..442dab826 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -138,6 +138,47 @@ impl LedgerSizeManager { } } + async fn prepare_watermarks_for_existing_ledger( + ledger: &Arc, + finality_provider: &Arc, + existing_ledger_state: ExistingLedgerState, + resize_percentage: &ResizePercentage, + max_ledger_size: u64, + ) -> (Watermarks, u64) { + let prev_size = existing_ledger_state.size; + + let (adjusted_ledger_size, lowest_slot) = + if prev_size > max_ledger_size { + warn!( + "Existing ledger size {} is above the max size {}, \ + waiting for truncation before using watermarks.", + prev_size, max_ledger_size + ); + + Self::ensure_initial_max_ledger_size_below( + ledger, + finality_provider, + &existing_ledger_state, + resize_percentage, + max_ledger_size, + ) + .await + } else { + (prev_size, ledger.get_lowest_cleanup_slot()) + }; + + let mut marks = Watermarks::new( + resize_percentage, + max_ledger_size, + Some(existing_ledger_state), + ); + marks.size_at_last_capture = adjusted_ledger_size; + // Remove watermarks that are below the lowest cleanup slot + marks.marks.retain(|mark| mark.slot > lowest_slot); + + (marks, adjusted_ledger_size) + } + async fn tick( ledger: &Arc, finality_provider: &Arc, @@ -152,40 +193,15 @@ impl LedgerSizeManager { // NOTE: that watermarks are set during the first tick if watermarks.is_none() { if let Some(existing_ledger_state) = existing_ledger_state.take() { - let prev_size = existing_ledger_state.size; - - let (adjusted_ledger_size, lowest_slot) = - if prev_size > max_ledger_size { - warn!( - "Existing ledger size {} is above the max size {}, \ - waiting for truncation before using watermarks.", - prev_size, max_ledger_size - ); - - Self::ensure_initial_max_ledger_size_below( - ledger, - finality_provider, - &existing_ledger_state, - resize_percentage, - max_ledger_size, - ) - .await - } else { - (prev_size, ledger.get_lowest_cleanup_slot()) - }; - - watermarks.replace({ - let mut marks = Watermarks::new( - resize_percentage, - max_ledger_size, - Some(existing_ledger_state), - ); - marks.size_at_last_capture = adjusted_ledger_size; - // Remove watermarks that are below the lowest cleanup slot - marks.marks.retain(|mark| mark.slot > lowest_slot); - marks - }); + let (prepared_watermarks, adjusted_ledger_size) = Self::prepare_watermarks_for_existing_ledger( + ledger, + finality_provider, + existing_ledger_state, + resize_percentage, + max_ledger_size, + ).await; + watermarks.replace(prepared_watermarks); return Some(adjusted_ledger_size); } } From 1af014d64b825d55026aadd1bfc26bb71d72adcf Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 11:30:27 +0530 Subject: [PATCH 33/36] fix: taco-paco's review comment 10 part 1 --- .../src/ledger_size_manager/mod.rs | 78 +++++++++++-------- 1 file changed, 47 insertions(+), 31 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 442dab826..e2512666a 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -179,6 +179,48 @@ impl LedgerSizeManager { (marks, adjusted_ledger_size) } + async fn handle_partial_truncation( + ledger: &Arc, + wms: &mut Watermarks, + mark: &Watermark, + latest_final_slot: u64, + lowest_cleanup_slot: u64, + ) { + warn!("Truncation would remove data at or above the latest final slot {}. \ + Adjusting truncation for mark: {mark:?} to cut up to the latest final slot.", + latest_final_slot); + + // Estimate the size delta based on the ratio of the slots + // that we can remove + let original_diff = + mark.slot.saturating_sub(lowest_cleanup_slot); + let applied_diff = + latest_final_slot.saturating_sub(lowest_cleanup_slot); + let size_delta = (applied_diff as f64 + / original_diff as f64 + * mark.size_delta as f64) + as u64; + Self::truncate_ledger( + ledger, + latest_final_slot, + size_delta, + ) + .await; + wms.size_at_last_capture = + wms.size_at_last_capture.saturating_sub(size_delta); + + // Since we didn't truncate the full mark, we need to put one + // back so it will be processed to remove the remaining space + // when possible + // Otherwise we would process the following mark which would + // cause us to truncate too many slots + wms.push_front(Watermark { + slot: mark.slot, + mod_id: mark.mod_id, + size_delta: mark.size_delta.saturating_sub(size_delta), + }); + } + async fn tick( ledger: &Arc, finality_provider: &Arc, @@ -248,39 +290,13 @@ impl LedgerSizeManager { } if mark.slot > latest_final_slot { - warn!("Truncation would remove data at or above the latest final slot {}. \ - Adjusting truncation for mark: {mark:?} to cut up to the latest final slot.", - latest_final_slot); - - // Estimate the size delta based on the ratio of the slots - // that we can remove - let original_diff = - mark.slot.saturating_sub(lowest_cleanup_slot); - let applied_diff = - latest_final_slot.saturating_sub(lowest_cleanup_slot); - let size_delta = (applied_diff as f64 - / original_diff as f64 - * mark.size_delta as f64) - as u64; - Self::truncate_ledger( + Self::handle_partial_truncation( ledger, + wms, + &mark, latest_final_slot, - size_delta, - ) - .await; - wms.size_at_last_capture = - wms.size_at_last_capture.saturating_sub(size_delta); - - // Since we didn't truncate the full mark, we need to put one - // back so it will be processed to remove the remaining space - // when possible - // Otherwise we would process the following mark which would - // cause us to truncate too many slots - wms.push_front(Watermark { - slot: mark.slot, - mod_id: mark.mod_id, - size_delta: mark.size_delta.saturating_sub(size_delta), - }); + lowest_cleanup_slot, + ).await; } else { Self::truncate_ledger(ledger, mark.slot, mark.size_delta) .await; From f40d75eb09094fa911830d0a13a363a4166d26d4 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 11:34:33 +0530 Subject: [PATCH 34/36] fix: taco-paco's review comment 10 part 2 --- .../src/ledger_size_manager/mod.rs | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index e2512666a..82c7c2bd0 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -221,6 +221,18 @@ impl LedgerSizeManager { }); } + async fn handle_full_truncation( + ledger: &Arc, + wms: &mut Watermarks, + mark: &Watermark, + ) { + Self::truncate_ledger(ledger, mark.slot, mark.size_delta) + .await; + wms.size_at_last_capture = wms + .size_at_last_capture + .saturating_sub(mark.size_delta); + } + async fn tick( ledger: &Arc, finality_provider: &Arc, @@ -298,11 +310,7 @@ impl LedgerSizeManager { lowest_cleanup_slot, ).await; } else { - Self::truncate_ledger(ledger, mark.slot, mark.size_delta) - .await; - wms.size_at_last_capture = wms - .size_at_last_capture - .saturating_sub(mark.size_delta); + Self::handle_full_truncation(ledger, wms, &mark).await; } if let Ok(ls) = ledger.storage_size() { From 06716aac8ca4eb67cb409c469e3cf7af4f0d272e Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 12:24:36 +0530 Subject: [PATCH 35/36] fix: taco-paco review comments regarding Bank generic --- .../src/ledger_size_manager/mod.rs | 77 +++++++++---------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/magicblock-ledger/src/ledger_size_manager/mod.rs b/magicblock-ledger/src/ledger_size_manager/mod.rs index 82c7c2bd0..60bda059d 100644 --- a/magicblock-ledger/src/ledger_size_manager/mod.rs +++ b/magicblock-ledger/src/ledger_size_manager/mod.rs @@ -53,10 +53,10 @@ pub struct LedgerSizeManager { impl LedgerSizeManager { pub fn new_from_ledger( ledger: Arc, - finality_provider: Arc, + finality_provider: Arc, ledger_state: Option, config: LedgerSizeManagerConfig, - ) -> LedgerSizeManager { + ) -> LedgerSizeManager { let managed_ledger = Truncator { ledger }; LedgerSizeManager::new( Arc::new(managed_ledger), @@ -147,25 +147,25 @@ impl LedgerSizeManager { ) -> (Watermarks, u64) { let prev_size = existing_ledger_state.size; - let (adjusted_ledger_size, lowest_slot) = - if prev_size > max_ledger_size { - warn!( + let (adjusted_ledger_size, lowest_slot) = if prev_size > max_ledger_size + { + warn!( "Existing ledger size {} is above the max size {}, \ waiting for truncation before using watermarks.", prev_size, max_ledger_size ); - Self::ensure_initial_max_ledger_size_below( - ledger, - finality_provider, - &existing_ledger_state, - resize_percentage, - max_ledger_size, - ) - .await - } else { - (prev_size, ledger.get_lowest_cleanup_slot()) - }; + Self::ensure_initial_max_ledger_size_below( + ledger, + finality_provider, + &existing_ledger_state, + resize_percentage, + max_ledger_size, + ) + .await + } else { + (prev_size, ledger.get_lowest_cleanup_slot()) + }; let mut marks = Watermarks::new( resize_percentage, @@ -175,7 +175,7 @@ impl LedgerSizeManager { marks.size_at_last_capture = adjusted_ledger_size; // Remove watermarks that are below the lowest cleanup slot marks.marks.retain(|mark| mark.slot > lowest_slot); - + (marks, adjusted_ledger_size) } @@ -192,20 +192,12 @@ impl LedgerSizeManager { // Estimate the size delta based on the ratio of the slots // that we can remove - let original_diff = - mark.slot.saturating_sub(lowest_cleanup_slot); + let original_diff = mark.slot.saturating_sub(lowest_cleanup_slot); let applied_diff = latest_final_slot.saturating_sub(lowest_cleanup_slot); - let size_delta = (applied_diff as f64 - / original_diff as f64 - * mark.size_delta as f64) - as u64; - Self::truncate_ledger( - ledger, - latest_final_slot, - size_delta, - ) - .await; + let size_delta = (applied_diff as f64 / original_diff as f64 + * mark.size_delta as f64) as u64; + Self::truncate_ledger(ledger, latest_final_slot, size_delta).await; wms.size_at_last_capture = wms.size_at_last_capture.saturating_sub(size_delta); @@ -226,11 +218,9 @@ impl LedgerSizeManager { wms: &mut Watermarks, mark: &Watermark, ) { - Self::truncate_ledger(ledger, mark.slot, mark.size_delta) - .await; - wms.size_at_last_capture = wms - .size_at_last_capture - .saturating_sub(mark.size_delta); + Self::truncate_ledger(ledger, mark.slot, mark.size_delta).await; + wms.size_at_last_capture = + wms.size_at_last_capture.saturating_sub(mark.size_delta); } async fn tick( @@ -247,13 +237,15 @@ impl LedgerSizeManager { // NOTE: that watermarks are set during the first tick if watermarks.is_none() { if let Some(existing_ledger_state) = existing_ledger_state.take() { - let (prepared_watermarks, adjusted_ledger_size) = Self::prepare_watermarks_for_existing_ledger( - ledger, - finality_provider, - existing_ledger_state, - resize_percentage, - max_ledger_size, - ).await; + let (prepared_watermarks, adjusted_ledger_size) = + Self::prepare_watermarks_for_existing_ledger( + ledger, + finality_provider, + existing_ledger_state, + resize_percentage, + max_ledger_size, + ) + .await; watermarks.replace(prepared_watermarks); return Some(adjusted_ledger_size); @@ -308,7 +300,8 @@ impl LedgerSizeManager { &mark, latest_final_slot, lowest_cleanup_slot, - ).await; + ) + .await; } else { Self::handle_full_truncation(ledger, wms, &mark).await; } From 426e512be0533f592722c138ac1ec72e0e830897 Mon Sep 17 00:00:00 2001 From: Thorsten Lorenz Date: Fri, 11 Jul 2025 12:28:07 +0530 Subject: [PATCH 36/36] chore: fmt --- magicblock-api/src/magic_validator.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/magicblock-api/src/magic_validator.rs b/magicblock-api/src/magic_validator.rs index ab02c8433..069bf59d2 100644 --- a/magicblock-api/src/magic_validator.rs +++ b/magicblock-api/src/magic_validator.rs @@ -211,7 +211,7 @@ impl MagicValidator { config.validator_config.ledger.reset, )?; - let existing_ledger_state = (!config.validator_config.ledger.reset + let existing_ledger_state = (!config.validator_config.ledger.reset && ledger.last_slot() > 0) .then_some(ExistingLedgerState { size: ledger.storage_size()?,