From 6296d164f5a3503febd4e44e9182e1ca35c2da1b Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 18 Mar 2026 21:58:16 +0800 Subject: [PATCH 1/6] perf: add lru cache for load_executed_tipset to speed up hot queries --- src/rpc/methods/eth.rs | 15 ++------ src/shim/executor.rs | 6 ++-- src/state_manager/mod.rs | 76 ++++++++++++++++++++------------------- src/utils/get_size/mod.rs | 4 +++ 4 files changed, 50 insertions(+), 51 deletions(-) diff --git a/src/rpc/methods/eth.rs b/src/rpc/methods/eth.rs index 801486b68ffd..39263ee50600 100644 --- a/src/rpc/methods/eth.rs +++ b/src/rpc/methods/eth.rs @@ -500,10 +500,7 @@ impl Block { let ExecutedTipset { state_root, executed_messages, - } = ctx - .state_manager - .load_executed_tipset_without_events(&tipset) - .await?; + } = ctx.state_manager.load_executed_tipset(&tipset).await?; let has_transactions = !executed_messages.is_empty(); let state_tree = ctx.state_manager.get_state_tree(&state_root)?; @@ -1419,10 +1416,7 @@ async fn get_block_receipts( let ExecutedTipset { state_root, executed_messages, - } = ctx - .state_manager - .load_executed_tipset_without_events(&ts_ref) - .await?; + } = ctx.state_manager.load_executed_tipset(&ts_ref).await?; // Load the state tree let state_tree = ctx.state_manager.get_state_tree(&state_root)?; @@ -1933,10 +1927,7 @@ async fn eth_fee_history( let base_fee = &ts.block_headers().first().parent_base_fee; let ExecutedTipset { executed_messages, .. - } = ctx - .state_manager - .load_executed_tipset_without_events(&ts) - .await?; + } = ctx.state_manager.load_executed_tipset(&ts).await?; let mut tx_gas_rewards = Vec::with_capacity(executed_messages.len()); for ExecutedMessage { message, receipt, .. diff --git a/src/shim/executor.rs b/src/shim/executor.rs index 8c61dc085344..3e52454e54fb 100644 --- a/src/shim/executor.rs +++ b/src/shim/executor.rs @@ -22,7 +22,7 @@ use fvm_shared4::receipt::Receipt as Receipt_v4; use fvm2::executor::ApplyRet as ApplyRet_v2; use fvm3::executor::ApplyRet as ApplyRet_v3; use fvm4::executor::ApplyRet as ApplyRet_v4; -use serde::Serialize; +use serde::{Deserialize, Serialize}; use spire_enum::prelude::delegated_enum; use std::borrow::Borrow as _; @@ -79,7 +79,7 @@ impl ApplyRet { // Note: it's impossible to properly derive Deserialize. // To deserialize into `Receipt`, refer to `fn get_parent_receipt` #[delegated_enum(impl_conversions)] -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] pub enum Receipt { V2(Receipt_v2), @@ -225,7 +225,7 @@ impl ActorEvent { /// Event with extra information stamped by the FVM. #[delegated_enum(impl_conversions)] -#[derive(Clone, Debug, Serialize)] +#[derive(Clone, Debug, Serialize, Deserialize)] #[serde(untagged)] pub enum StampedEvent { V3(StampedEvent_v3), diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index a60b9aaa23d4..9f2e3b50a21d 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -54,7 +54,8 @@ use crate::shim::{ use crate::state_manager::cache::TipsetStateCache; use crate::state_manager::chain_rand::draw_randomness; use crate::state_migration::run_state_migrations; -use crate::utils::get_size::GetSize; +use crate::utils::cache::SizeTrackingLruCache; +use crate::utils::get_size::{GetSize, vec_heap_size_helper}; use ahash::{HashMap, HashMapExt}; use anyhow::{Context as _, bail, ensure}; use bls_signatures::{PublicKey as BlsPublicKey, Serialize as _}; @@ -78,6 +79,7 @@ use rayon::prelude::ParallelBridge; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; use std::ops::RangeInclusive; +use std::sync::LazyLock; use std::time::Duration; use std::{num::NonZeroUsize, sync::Arc}; use tokio::sync::{RwLock, broadcast::error::RecvError}; @@ -93,26 +95,40 @@ type CidPair = (Cid, Cid); /// /// Includes the executed message itself, the execution receipt, and /// optional events emitted by the actor during execution. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExecutedMessage { pub message: ChainMessage, pub receipt: Receipt, pub events: Option>, } +impl GetSize for ExecutedMessage { + fn get_heap_size(&self) -> usize { + self.message.get_heap_size() + + self.receipt.get_heap_size() + + self + .events + .as_ref() + .map(vec_heap_size_helper) + .unwrap_or_default() + } +} + /// Aggregated execution result for a tipset. /// /// `state_root` is the resulting state tree root after message execution /// and `executed_messages` contains per-message execution details. +#[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExecutedTipset { pub state_root: Cid, pub executed_messages: Vec, } -/// Options controlling how `load_executed_tipset` fetches extra execution data. -/// -/// `include_events` toggles whether event logs are loaded from receipts. -pub struct LoadExecutedTipsetOptions { - pub include_events: bool, +impl GetSize for ExecutedTipset { + fn get_heap_size(&self) -> usize { + // state_root(Cid) has no heap allocation, so we only calculate the heap size of executed_messages + vec_heap_size_helper(&self.executed_messages) + } } #[derive(Debug, Default, Clone, GetSize)] @@ -471,38 +487,28 @@ where .await } - /// Load an executed tipset, including message receipts and state root, - /// without loading event logs from receipts. - pub async fn load_executed_tipset_without_events( - self: &Arc, - ts: &Tipset, - ) -> anyhow::Result { - let receipt_ts = self.chain_store().load_child_tipset(ts).ok(); - self.load_executed_tipset_inner( - ts, - receipt_ts.as_ref(), - LoadExecutedTipsetOptions { - include_events: false, - }, - ) - .await - } - - /// Load an executed tipset, including message receipts and state root, - /// with event logs loaded when available. + /// Load an executed tipset, including state root, message receipts and events with caching. pub async fn load_executed_tipset( self: &Arc, ts: &Tipset, ) -> anyhow::Result { + // A tipset key should always map to a deterministic state output, so it's safe to cache the entire executed tipset with the same key. + static CACHE: LazyLock> = + LazyLock::new(|| { + SizeTrackingLruCache::new_with_metrics( + "executed_tipset".into(), + nonzero!(1024usize), + ) + }); + if let Some(cached) = CACHE.get_cloned(ts.key()) { + return Ok(cached); + } let receipt_ts = self.chain_store().load_child_tipset(ts).ok(); - self.load_executed_tipset_inner( - ts, - receipt_ts.as_ref(), - LoadExecutedTipsetOptions { - include_events: true, - }, - ) - .await + let result = self + .load_executed_tipset_inner(ts, receipt_ts.as_ref()) + .await?; + CACHE.push(ts.key().clone(), result.clone()); + Ok(result) } async fn load_executed_tipset_inner( @@ -510,9 +516,7 @@ where msg_ts: &Tipset, // when `msg_ts` is the current head, `receipt_ts` is `None` receipt_ts: Option<&Tipset>, - options: LoadExecutedTipsetOptions, ) -> anyhow::Result { - let LoadExecutedTipsetOptions { include_events } = options; if let Some(receipt_ts) = receipt_ts { anyhow::ensure!( msg_ts.key() == receipt_ts.parents(), @@ -546,7 +550,7 @@ where ); let mut executed_messages = Vec::with_capacity(messages.len()); for (message, receipt) in messages.into_iter().zip(receipts.into_iter()) { - let events = if include_events && let Some(events_root) = receipt.events_root() { + let events = if let Some(events_root) = receipt.events_root() { Some( match StampedEvent::get_events(self.cs.blockstore(), &events_root) { Ok(events) => events, diff --git a/src/utils/get_size/mod.rs b/src/utils/get_size/mod.rs index 6c35b3f2714a..b5a696a3a1eb 100644 --- a/src/utils/get_size/mod.rs +++ b/src/utils/get_size/mod.rs @@ -35,6 +35,10 @@ macro_rules! impl_vec_alike_heap_size_helper { }; } +pub fn vec_heap_size_helper(v: &Vec) -> usize { + impl_vec_alike_heap_size_helper!(v, T) +} + pub fn vec_heap_size_with_fn_helper(v: &Vec, get_heap_size: impl Fn(&T) -> usize) -> usize { impl_vec_alike_heap_size_with_fn_helper!(v, T, std::mem::size_of::, get_heap_size) } From 5fdd7b1c736adafbdebd3c2a676b55af3b7f821a Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 18 Mar 2026 22:22:19 +0800 Subject: [PATCH 2/6] resolve AI comments --- src/shim/executor.rs | 6 +++--- src/state_manager/mod.rs | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/shim/executor.rs b/src/shim/executor.rs index 3e52454e54fb..8c61dc085344 100644 --- a/src/shim/executor.rs +++ b/src/shim/executor.rs @@ -22,7 +22,7 @@ use fvm_shared4::receipt::Receipt as Receipt_v4; use fvm2::executor::ApplyRet as ApplyRet_v2; use fvm3::executor::ApplyRet as ApplyRet_v3; use fvm4::executor::ApplyRet as ApplyRet_v4; -use serde::{Deserialize, Serialize}; +use serde::Serialize; use spire_enum::prelude::delegated_enum; use std::borrow::Borrow as _; @@ -79,7 +79,7 @@ impl ApplyRet { // Note: it's impossible to properly derive Deserialize. // To deserialize into `Receipt`, refer to `fn get_parent_receipt` #[delegated_enum(impl_conversions)] -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize)] #[serde(untagged)] pub enum Receipt { V2(Receipt_v2), @@ -225,7 +225,7 @@ impl ActorEvent { /// Event with extra information stamped by the FVM. #[delegated_enum(impl_conversions)] -#[derive(Clone, Debug, Serialize, Deserialize)] +#[derive(Clone, Debug, Serialize)] #[serde(untagged)] pub enum StampedEvent { V3(StampedEvent_v3), diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index 9f2e3b50a21d..12b76bcc2c7f 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -95,7 +95,7 @@ type CidPair = (Cid, Cid); /// /// Includes the executed message itself, the execution receipt, and /// optional events emitted by the actor during execution. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] pub struct ExecutedMessage { pub message: ChainMessage, pub receipt: Receipt, @@ -118,7 +118,7 @@ impl GetSize for ExecutedMessage { /// /// `state_root` is the resulting state tree root after message execution /// and `executed_messages` contains per-message execution details. -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone)] pub struct ExecutedTipset { pub state_root: Cid, pub executed_messages: Vec, From d0b182bd48645bd4e34e0f972021099f33193600 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 18 Mar 2026 23:42:11 +0800 Subject: [PATCH 3/6] populate cache after state computation --- src/chain/store/index.rs | 2 +- src/interpreter/vm.rs | 13 ++++++-- src/rpc/methods/eth.rs | 2 ++ src/state_manager/mod.rs | 70 +++++++++++++++++++++++++++++----------- 4 files changed, 64 insertions(+), 23 deletions(-) diff --git a/src/chain/store/index.rs b/src/chain/store/index.rs index 65c8d80fa582..2d8febeb00d6 100644 --- a/src/chain/store/index.rs +++ b/src/chain/store/index.rs @@ -130,7 +130,7 @@ impl ChainIndex { SizeTrackingLruCache::new_with_metrics( "tipset_by_height".into(), // 20480 * 900 = 18432000 which is sufficient for mainnet - 20480.try_into().expect("infallible"), + nonzero!(20480_usize), ) }); diff --git a/src/interpreter/vm.rs b/src/interpreter/vm.rs index 6985c695577a..3057d3e2f860 100644 --- a/src/interpreter/vm.rs +++ b/src/interpreter/vm.rs @@ -77,8 +77,11 @@ type ForestExecutorV4 = DefaultExecutor_v4>; pub type ApplyResult = anyhow::Result<(ApplyRet, Duration)>; -pub type ApplyBlockResult = - anyhow::Result<(Vec, Vec>, Vec>), anyhow::Error>; +pub type ApplyBlockResult = anyhow::Result<( + Vec, + Vec>>, + Vec>, +)>; /// Comes from pub const IMPLICIT_MESSAGE_GAS_LIMIT: i64 = i64::MAX / 2; @@ -387,7 +390,11 @@ where receipts.push(msg_receipt.clone()); events_roots.push(ret.msg_receipt().events_root()); - events.push(ret.events()); + if ret.msg_receipt().events_root().is_some() { + events.push(Some(ret.events())); + } else { + events.push(None); + } // Add processed Cid to set of processed messages processed.insert(cid); diff --git a/src/rpc/methods/eth.rs b/src/rpc/methods/eth.rs index 39263ee50600..83ba7058270e 100644 --- a/src/rpc/methods/eth.rs +++ b/src/rpc/methods/eth.rs @@ -500,6 +500,7 @@ impl Block { let ExecutedTipset { state_root, executed_messages, + .. } = ctx.state_manager.load_executed_tipset(&tipset).await?; let has_transactions = !executed_messages.is_empty(); let state_tree = ctx.state_manager.get_state_tree(&state_root)?; @@ -1416,6 +1417,7 @@ async fn get_block_receipts( let ExecutedTipset { state_root, executed_messages, + .. } = ctx.state_manager.load_executed_tipset(&ts_ref).await?; // Load the state tree diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index 9f2e3b50a21d..4fab3dbd0566 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -91,6 +91,14 @@ pub const EVENTS_AMT_BITWIDTH: u32 = 5; /// Intermediary for retrieving state objects and updating actor states. type CidPair = (Cid, Cid); +fn executed_tipset_cache() -> &'static SizeTrackingLruCache { + // A tipset key should always map to a deterministic state output, so it's safe to cache the entire executed tipset with the same key. + static CACHE: LazyLock> = LazyLock::new(|| { + SizeTrackingLruCache::new_with_metrics("executed_tipset".into(), nonzero!(1024usize)) + }); + &CACHE +} + /// Result of executing an individual chain message in a tipset. /// /// Includes the executed message itself, the execution receipt, and @@ -121,6 +129,7 @@ impl GetSize for ExecutedMessage { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct ExecutedTipset { pub state_root: Cid, + pub receipt_root: Cid, pub executed_messages: Vec, } @@ -492,22 +501,15 @@ where self: &Arc, ts: &Tipset, ) -> anyhow::Result { - // A tipset key should always map to a deterministic state output, so it's safe to cache the entire executed tipset with the same key. - static CACHE: LazyLock> = - LazyLock::new(|| { - SizeTrackingLruCache::new_with_metrics( - "executed_tipset".into(), - nonzero!(1024usize), - ) - }); - if let Some(cached) = CACHE.get_cloned(ts.key()) { + let cache = executed_tipset_cache(); + if let Some(cached) = cache.get_cloned(ts.key()) { return Ok(cached); } let receipt_ts = self.chain_store().load_child_tipset(ts).ok(); let result = self .load_executed_tipset_inner(ts, receipt_ts.as_ref()) .await?; - CACHE.push(ts.key().clone(), result.clone()); + cache.push(ts.key().clone(), result.clone()); Ok(result) } @@ -525,12 +527,13 @@ where } let messages = self.chain_store().messages_for_tipset(msg_ts)?; let mut recomputed = false; - let (state_root, receipts) = match receipt_ts.and_then(|ts| { - Receipt::get_receipts(self.cs.blockstore(), *ts.parent_message_receipts()) + let (state_root, receipt_root, receipts) = match receipt_ts.and_then(|ts| { + let receipt_root = *ts.parent_message_receipts(); + Receipt::get_receipts(self.cs.blockstore(), receipt_root) .ok() - .map(|r| (*ts.parent_state(), r)) + .map(|r| (*ts.parent_state(), receipt_root, r)) }) { - Some((state_root, receipts)) => (state_root, receipts), + Some((state_root, receipt_root, receipts)) => (state_root, receipt_root, receipts), None => { let state_output = self .compute_tipset_state(msg_ts.clone(), NO_CALLBACK, VMTrace::NotTraced) @@ -538,6 +541,7 @@ where recomputed = true; ( state_output.state_root, + state_output.receipt_root, Receipt::get_receipts(self.cs.blockstore(), state_output.receipt_root)?, ) } @@ -578,6 +582,7 @@ where } Ok(ExecutedTipset { state_root, + receipt_root, executed_messages, }) } @@ -2094,22 +2099,24 @@ where vm.apply_block_messages(&block_messages, epoch, callback)?; // step 5: construct receipt root from receipts - let receipt_root = Amtv0::new_from_iter(chain_index.db(), receipts)?; + let receipt_root = Amtv0::new_from_iter(chain_index.db(), receipts.iter())?; // step 6: store events AMTs in the blockstore - for (msg_events, events_root) in events.iter().zip(events_roots.iter()) { - if let Some(event_root) = events_root { + for (events, events_root) in events.iter().zip(events_roots.iter()) { + if let Some(events) = events { + let event_root = + events_root.context("events root should be present when events present")?; // Store the events AMT - the root CID should match the one computed by FVM let derived_event_root = Amt::new_from_iter_with_bit_width( chain_index.db(), EVENTS_AMT_BITWIDTH, - msg_events.iter(), + events.iter(), ) .map_err(|e| Error::Other(format!("failed to store events AMT: {e}")))?; // Verify the stored root matches the FVM-computed root ensure!( - derived_event_root.eq(event_root), + derived_event_root == event_root, "Events AMT root mismatch: derived={derived_event_root}, actual={event_root}." ); } @@ -2117,6 +2124,31 @@ where let state_root = vm.flush()?; + // Update executed tipset cache + let messages: Vec = block_messages + .into_iter() + .flat_map(|bm| bm.messages) + .collect_vec(); + anyhow::ensure!( + messages.len() == receipts.len() && messages.len() == events.len(), + "length of messages, receipts, and events should match", + ); + let executed_tipset = ExecutedTipset { + state_root, + receipt_root, + executed_messages: messages + .into_iter() + .zip(receipts) + .zip(events) + .map(|((message, receipt), events)| ExecutedMessage { + message, + receipt, + events, + }) + .collect(), + }; + executed_tipset_cache().push(tipset.key().clone(), executed_tipset); + Ok(StateOutput { state_root, receipt_root, From 62eb46cf4d0d7a02f699523f7d69e2da670cb10b Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Wed, 18 Mar 2026 23:59:36 +0800 Subject: [PATCH 4/6] fix --- src/state_manager/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index fd1ea2ebc934..c5f8fef99f81 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -129,6 +129,7 @@ impl GetSize for ExecutedMessage { #[derive(Debug, Clone)] pub struct ExecutedTipset { pub state_root: Cid, + #[allow(dead_code)] pub receipt_root: Cid, pub executed_messages: Vec, } From a12b2b25b38e149b1dc7423448d5ae716d28a3d3 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 19 Mar 2026 00:12:45 +0800 Subject: [PATCH 5/6] code comment --- src/state_manager/mod.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index c5f8fef99f81..0a4538601b3c 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -94,6 +94,7 @@ type CidPair = (Cid, Cid); fn executed_tipset_cache() -> &'static SizeTrackingLruCache { // A tipset key should always map to a deterministic state output, so it's safe to cache the entire executed tipset with the same key. static CACHE: LazyLock> = LazyLock::new(|| { + // 100-200MiB on mainet with capacity 1024 SizeTrackingLruCache::new_with_metrics("executed_tipset".into(), nonzero!(1024usize)) }); &CACHE From 4ec750ca8eb1445716068ed9c6d9fb027e2f2b31 Mon Sep 17 00:00:00 2001 From: hanabi1224 Date: Thu, 19 Mar 2026 09:33:38 +0800 Subject: [PATCH 6/6] fix comment --- src/state_manager/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state_manager/mod.rs b/src/state_manager/mod.rs index 0a4538601b3c..c62cfcf7d385 100644 --- a/src/state_manager/mod.rs +++ b/src/state_manager/mod.rs @@ -137,7 +137,7 @@ pub struct ExecutedTipset { impl GetSize for ExecutedTipset { fn get_heap_size(&self) -> usize { - // state_root(Cid) has no heap allocation, so we only calculate the heap size of executed_messages + // state_root (Cid) and receipt_root (Cid) have no heap allocation, so we only calculate the heap size of executed_messages vec_heap_size_helper(&self.executed_messages) } }