From 3ecd843d08e1d7a0899672cfc10e154e86689815 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 10:38:34 -0400 Subject: [PATCH 01/14] [glue] Own the databases and fail closed The actor now owns its databases outright and doomed verification jobs find out from the database itself, through a typed error, instead of being classified and cancelled by the scheduler. - Shared (Arc>>), WriteSlot, BatchContext, and the take/put mutation dance are gone. split(db) -> (Writer, Reader) gives the actor a sole non-Clone mutation handle whose mutations read as by-value code; jobs and peer serving hold cloneable Readers whose guards cover exactly one storage call. - The Disposition/FinalizationBoundary/VerificationProgress classification, the invalidation channels, and the quiesce-before-apply barrier are deleted. Finalization applies immediately; a job on a losing fork dies at its next database read, which returns StaleRead, or when its requester drops the response channel. - Application::{propose, verify, apply} return Result<_, ExecutionError> ({Stale, Shutdown, Fatal}); applications propagate storage errors with ? and never interpret them. Every storage failure maps to Fatal and panics, proposals included; Shutdown is application-signaled only and never inferred from storage errors. The verifier maps Stale by re-entering its canonical-state recheck (a block that finalized itself mid-verify answers true, competing blocks answer false), and an invalid-looking attempt whose anchor moved mid-walk retries instead of answering false. - fork_batches refuses while a finalization is mid-flight (ExecutionState's finalizing window) and re-checks its anchor after per-database forks, so a batch set can never straddle a boundary. PrepareFailure::Invalid re-runs check_processed before answering false. - Mailbox::subscribe_databases is gone: the actor no longer hands its databases out, and peers observe only the published snapshot. The syncer delivers its artifact exactly once on the completion channel (SyncResult loses Clone; update_targets returns UpdateOutcome), and DatabaseSet::committed_targets becomes applied_targets. - DatabaseSet mutations take self and return the successor; ManagedDb mirrors. Mocks, the reshare harness, and the example migrate from .unwrap() to ?. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- examples/reshare/src/application.rs | 41 +- examples/reshare/src/types.rs | 6 +- glue/src/dkg/tests/reshare/harness.rs | 49 +- glue/src/stateful/actor/core/mailbox.rs | 55 +- glue/src/stateful/actor/core/mod.rs | 12 +- glue/src/stateful/actor/core/processing.rs | 1384 ++++++----- glue/src/stateful/actor/core/syncing.rs | 201 +- glue/src/stateful/actor/core/verifications.rs | 196 +- glue/src/stateful/actor/processor/mod.rs | 2117 +++++------------ glue/src/stateful/actor/processor/verifier.rs | 222 +- glue/src/stateful/actor/syncer/actor.rs | 101 +- glue/src/stateful/actor/syncer/mailbox.rs | 126 +- glue/src/stateful/actor/syncer/mod.rs | 34 +- glue/src/stateful/db/any.rs | 255 +- glue/src/stateful/db/cell.rs | 261 ++ glue/src/stateful/db/current.rs | 276 +-- glue/src/stateful/db/immutable/compact.rs | 98 +- glue/src/stateful/db/immutable/standard.rs | 75 +- glue/src/stateful/db/keyless/compact.rs | 105 +- glue/src/stateful/db/keyless/standard.rs | 162 +- glue/src/stateful/db/mod.rs | 1272 +++++----- glue/src/stateful/mod.rs | 133 +- glue/src/stateful/tests/mocks.rs | 84 +- glue/src/stateful/tests/mod.rs | 367 +-- glue/src/stateful/tests/multi_db_app.rs | 65 +- glue/src/stateful/tests/ownership.rs | 128 + glue/src/stateful/tests/single_db_app.rs | 50 +- 27 files changed, 3681 insertions(+), 4194 deletions(-) create mode 100644 glue/src/stateful/db/cell.rs create mode 100644 glue/src/stateful/tests/ownership.rs diff --git a/examples/reshare/src/application.rs b/examples/reshare/src/application.rs index 019f1ad1dae..ed1847c1c0b 100644 --- a/examples/reshare/src/application.rs +++ b/examples/reshare/src/application.rs @@ -10,8 +10,8 @@ use commonware_cryptography::{ use commonware_glue::{ dkg::reshare::Input as ReshareInput, stateful::{ - Application, Input, Proposed, - db::{DatabaseSet, Merkleized as _, Unmerkleized as _}, + Application, ExecutionError, Input, Proposed, + db::{DatabaseSet, Merkleized as _, MerkleizedOf, Unmerkleized as _, UnmerkleizedOf}, }, }; use commonware_runtime::{BufferPooler, Clock, Metrics, Spawner, Storage}; @@ -36,13 +36,12 @@ impl App { async fn execute( height: Height, - batches: as DatabaseSet>::Unmerkleized, - ) -> as DatabaseSet>::Merkleized { - batches + batches: UnmerkleizedOf, E>, + ) -> Result, E>, ExecutionError> { + Ok(batches .write(HEIGHT_KEY, Some(U64::new(height.get()))) .merkleize() - .await - .expect("height write must merkleize") + .await?) } } @@ -65,14 +64,16 @@ where &mut self, context: (E, Self::Context), mut ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { // The `reshare::Application` wrapper selected and fetched the payload. let payload = input.upstream.payload; - let parent = ancestry.next().await?; + let Some(parent) = ancestry.next().await else { + return Ok(None); + }; let height = parent.height().next(); - let merkleized = Self::execute(height, batches).await; + let merkleized = Self::execute(height, batches).await?; let bounds = merkleized.bounds(); let block = Block { context: context.1, @@ -82,31 +83,33 @@ where range: non_empty_range!(bounds.inactivity_floor, bounds.tip.size), payload, }; - Some(Proposed { block, merkleized }) + Ok(Some(Proposed { block, merkleized })) } async fn verify( &mut self, _context: (E, Self::Context), mut ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> { // Validation from higher layers: // - Epoch validation is handled by `Deferred` // - QMDB root / range validation is handled by `stateful::Application` // - Reshare `Payload` validation is handled by `reshare::Application` - let block = ancestry.next().await?; - let merkleized = Self::execute(block.height(), batches).await; - Some(merkleized) + let Some(block) = ancestry.next().await else { + return Ok(None); + }; + let merkleized = Self::execute(block.height(), batches).await?; + Ok(Some(merkleized)) } async fn apply( &mut self, _context: (E, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> >::Merkleized { + batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { Self::execute(block.height(), batches).await } diff --git a/examples/reshare/src/types.rs b/examples/reshare/src/types.rs index 79e0c108d50..beec7fa42bb 100644 --- a/examples/reshare/src/types.rs +++ b/examples/reshare/src/types.rs @@ -28,7 +28,7 @@ use commonware_cryptography::{ use commonware_formatting::{from_hex, hex}; use commonware_glue::{ dkg::{self, ParticipantsProvider, Registrar as RegistrarTrait, ReshareBlock, types::Payload}, - stateful::db::{Shared, SyncEngineConfig}, + stateful::db::{Single, SyncEngineConfig}, }; use commonware_parallel::Sequential; use commonware_runtime::{Buf, BufMut, Quota, buffer::paged::CacheRef}; @@ -62,8 +62,8 @@ use tracing::info; pub type Scheme = simplex::scheme::bls12381_threshold::vrf::Scheme; /// QMDB holding the application state. pub type Qmdb = fixed::Db; -/// Shared handle to the application QMDB. -pub type Database = Shared>; +/// Database set containing a single QMDB. +pub type Database = Single>; /// Globally unique namespace for every message signed by this example. pub const NAMESPACE: &[u8] = b"_COMMONWARE_RESHARE_EXAMPLE"; /// Number of blocks in each epoch. diff --git a/glue/src/dkg/tests/reshare/harness.rs b/glue/src/dkg/tests/reshare/harness.rs index 65c3da9c720..deb0f5e9d71 100644 --- a/glue/src/dkg/tests/reshare/harness.rs +++ b/glue/src/dkg/tests/reshare/harness.rs @@ -20,11 +20,11 @@ use crate::{ reporter::MonitorReporter, }, stateful::{ - Application, Config as StatefulConfig, Input, Proposed, Stateful as StatefulActor, - SyncPlan, + Application, Config as StatefulConfig, ExecutionError, Input, Proposed, + Stateful as StatefulActor, SyncPlan, db::{ - DatabaseSet, Merkleized as _, Shared, SyncEngineConfig, Unmerkleized as _, - p2p as qmdb_resolver, + DatabaseSet, Merkleized as _, MerkleizedOf, Publisher, ReadersOf, Single, + SyncEngineConfig, Unmerkleized as _, UnmerkleizedOf, p2p as qmdb_resolver, }, }, }; @@ -97,7 +97,7 @@ use std::{ type Qmdb = fixed::Db; -type Database = Shared>; +type Database = Single>; type Scheme = simplex::scheme::bls12381_threshold::vrf::Scheme; type MarshalVariant = Standard; type Marshal = MarshalMailbox; @@ -362,11 +362,11 @@ struct App { impl App { async fn execute( height: Height, - mut batches: as DatabaseSet>::Unmerkleized, - ) -> as DatabaseSet>::Merkleized { + mut batches: UnmerkleizedOf, E>, + ) -> Result, E>, ExecutionError> { let key = Sha256::hash(&[b"height"]); batches = batches.write(key, Some(u64_to_digest(height.get()))); - batches.merkleize().await.unwrap() + Ok(batches.merkleize().await?) } } @@ -386,14 +386,16 @@ impl Application &mut self, context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, input: Input, - ) -> Option> { - let parent = ancestry.peek()?.clone(); + ) -> Result>, ExecutionError> { + let Some(parent) = ancestry.peek().cloned() else { + return Ok(None); + }; let height = Height::new(parent.height().get() + 1); // The reshare::Application wrapper selected and fetched the payload. let payload = input.upstream.payload; - let merkleized = Self::execute(height, batches).await; + let merkleized = Self::execute(height, batches).await?; let bounds = merkleized.bounds(); let block = Block { context: context.1, @@ -403,28 +405,30 @@ impl Application range: non_empty_range!(bounds.inactivity_floor, bounds.tip.size), payload, }; - Some(Proposed { block, merkleized }) + Ok(Some(Proposed { block, merkleized })) } async fn verify( &mut self, _context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> { // Reshare final-block payload validation is enforced by the surrounding // reshare::Application wrapper; this inner app only executes state. - let tip = ancestry.peek()?.clone(); - let merkleized = Self::execute(tip.height(), batches).await; - Some(merkleized) + let Some(tip) = ancestry.peek().cloned() else { + return Ok(None); + }; + let merkleized = Self::execute(tip.height(), batches).await?; + Ok(Some(merkleized)) } async fn apply( &mut self, _context: (E, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> >::Merkleized { + batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { Self::execute(block.height(), batches).await } @@ -432,7 +436,7 @@ impl Application &mut self, context: (E, Self::Context), block: &Self::Block, - _readers: >::Readers, + _readers: ReadersOf, ) { self.processed .lock() @@ -1019,8 +1023,7 @@ impl EngineDefinition for ReshareEngine { }; let publication_context = context.child("publication"); - let (snapshot_publisher, snapshot_subscriber) = - crate::stateful::db::Publisher::new(&publication_context); + let (snapshot_publisher, snapshot_subscriber) = Publisher::new(&publication_context); let (qmdb_resolver_actor, qmdb_sync_resolver) = qmdb_resolver::Actor::new( context.child("qmdb_resolver"), qmdb_resolver::Config { diff --git a/glue/src/stateful/actor/core/mailbox.rs b/glue/src/stateful/actor/core/mailbox.rs index 3d17babacd8..daba9b332e1 100644 --- a/glue/src/stateful/actor/core/mailbox.rs +++ b/glue/src/stateful/actor/core/mailbox.rs @@ -22,8 +22,6 @@ use rand_core::Rng; use std::{collections::VecDeque, sync::Arc}; use tracing::{Span, info_span}; -type RetryMailbox = Arc) + Send + Sync>; - /// A verification is scoped to its caller. pub(in crate::stateful::actor) struct Verification { response: oneshot::Sender, @@ -71,15 +69,6 @@ where span: Span, block: Arc, acknowledgement: Exact, - retry_mailbox: RetryMailbox, - }, - - /// Requests the database set. - /// - /// The actor replies once startup handoff has produced the database set, - /// or immediately if that has already happened. - SubscribeDatabases { - response: oneshot::Sender, }, } @@ -92,7 +81,6 @@ where match self { Self::Propose { response, .. } => response.is_closed(), Self::Verify { verification, .. } => verification.is_cancelled(), - Self::SubscribeDatabases { response } => response.is_closed(), Self::Finalized { .. } => false, } } @@ -164,7 +152,6 @@ where A: Application, { sender: Sender>, - retry_mailbox: RetryMailbox, } impl Clone for Mailbox @@ -175,7 +162,6 @@ where fn clone(&self) -> Self { Self { sender: self.sender.clone(), - retry_mailbox: self.retry_mailbox.clone(), } } } @@ -186,44 +172,8 @@ where A: Application, { /// Create a mailbox from the send half of the actor's message channel. - pub(super) fn new(sender: Sender>) -> Self { - let retry_sender = sender.clone(); - let retry_mailbox = Arc::new(move |message| { - let _ = retry_sender.enqueue(message); - }); - Self { - sender, - retry_mailbox, - } - } -} - -impl Mailbox -where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - /// Wait for the database set. - /// - /// This resolves once startup handoff has produced the database set. Late - /// callers receive the current database set immediately. - /// - /// ## Safety - /// - /// Holders must never manually prune these databases. Stateful uses - /// [`Config::prune_config`](crate::stateful::Config::prune_config) to - /// schedule safe pruning without pruning past the rewind window needed for - /// crash reconciliation. With pruning enabled, glue keeps a - /// `max_pending_acks + 1` finalized-target window plus the configured - /// extra block windows before pruning. - pub async fn subscribe_databases(&self) -> A::Databases { - let (response, receiver) = oneshot::channel(); - let _ = self - .sender - .enqueue(Message::SubscribeDatabases { response }); - receiver - .await - .expect("stateful actor dropped during subscribe_databases") + pub(super) const fn new(sender: Sender>) -> Self { + Self { sender } } } @@ -305,7 +255,6 @@ where span, block, acknowledgement, - retry_mailbox: self.retry_mailbox.clone(), } } }; diff --git a/glue/src/stateful/actor/core/mod.rs b/glue/src/stateful/actor/core/mod.rs index 3f5a7283247..7d83dc147f2 100644 --- a/glue/src/stateful/actor/core/mod.rs +++ b/glue/src/stateful/actor/core/mod.rs @@ -250,7 +250,6 @@ where sync_metadata, syncer: syncer_mailbox, deferred_verifications: Vec::new(), - database_subscribers: Vec::new(), artifact: None, snapshot_publisher: self.snapshot_publisher, sync_completed, @@ -278,22 +277,19 @@ where let metrics = StatefulMetrics::new(self.context.as_present()); let _ = metrics.sync_done.try_set(1); let processor = Processor::new(self.application, databases, anchor, metrics, self.pruning); - - // The recovered state alone must publish before the loop starts, so - // serving begins before the next finalization. + // Publish the recovered state before the loop starts, so serving begins + // before the next finalization. let mut snapshot_publisher = self.snapshot_publisher; - processor.publish_snapshot(&mut snapshot_publisher).await; + let processor = processor.publish_snapshot(&mut snapshot_publisher).await; Processing { context: self.context, mailbox: self.mailbox, provider: self.provider, marshal, - processor, snapshot_publisher, - deferred_verifications: Vec::new(), skip_finalized_until, } - .start() + .start(processor, Vec::new()) .await } } diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index 7559848f80f..7210dd5699f 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -1,3 +1,25 @@ +//! The post-sync processing loop of the stateful actor. +//! +//! The loop owns the database set and is the only thing that mutates it. +//! Verification jobs hold readers instead, so they are never cancelled. A +//! job that is mid-read when a finalized block arrives finishes that read, the +//! apply runs, and the job continues against the state it installs. A job on +//! the losing side of that apply is refused at its next batch operation +//! ([`ExecutionError::Stale`](crate::stateful::ExecutionError::Stale)) and +//! answered from the canonical chain. +//! +//! Each finalized block is applied to the databases, its flush deferred to a +//! pool, and a snapshot of the applied state published for serving as soon as +//! the apply completes. Served state may run ahead of disk, which is safe +//! because peers verify what they fetch against a finalized root. The block is +//! acknowledged to marshal only once its flush is durable, so marshal's floor +//! never gets ahead of disk. +//! +//! Pruning is maintenance, run only while the mailbox is idle. A prune waits +//! until the pruned range is durable, prunes, and publishes fresh snapshots +//! right away, since the served snapshots pin the pruned storage (see +//! [`Publisher`]). + use crate::stateful::{ Application, Input, actor::{ @@ -19,9 +41,9 @@ use commonware_consensus::{ types::Height, }; use commonware_cryptography::certificate::Scheme; -use commonware_macros::{select, select_loop}; +use commonware_macros::select; use commonware_runtime::{Clock, ContextCell, Metrics, Spawner}; -use commonware_utils::{Acknowledgement as _, channel::fallible::OneshotExt, futures::Pool}; +use commonware_utils::{Acknowledgement as _, futures::Pool}; use futures::{ FutureExt as _, future::{Either, ready}, @@ -46,34 +68,6 @@ fn complete(pending: &mut BTreeSet, (height, durable): (Height, bool)) - durable } -fn requeue_verifications( - mailbox: &(dyn Fn(Message) + Send + Sync), - requests: Vec>, -) where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - // FIFO puts each retry behind work accepted while the mutation ran and - // ahead of later arrivals, without waiting for the mailbox to become idle. - for VerificationRequest { - span, - context, - ancestry, - verification, - } in requests - { - if verification.is_cancelled() { - continue; - } - mailbox(Message::Verify { - span, - context, - ancestry, - verification, - }); - } -} - pub(super) struct Processing where E: Rng + Spawner + Metrics + Clock, @@ -93,15 +87,9 @@ where /// Marshal mailbox used for lazy block lookup. pub(super) marshal: MarshalMailbox, - /// The processing state of the actor. - pub(super) processor: Processor, - /// Publishes the latest snapshots for serving. pub(super) snapshot_publisher: Publisher>, - /// Verification requests deferred until processing starts. - pub(super) deferred_verifications: Vec>, - /// Finalized marshal blocks at or below this height were already reflected /// in the selected database anchor and should be acknowledged only. pub(super) skip_finalized_until: Option, @@ -109,97 +97,116 @@ where impl Processing where - E: Rng + Spawner + Metrics + Clock, - A: Application, - S: Scheme, - V: Variant, + E: Rng + Spawner + Metrics + Clock + 'static, + A: Application + 'static, + S: Scheme + 'static, + V: Variant + 'static, MarshalMailbox: BlockProvider, { - pub async fn start(mut self) { + /// Run the loop until shutdown. + /// + /// `deferred` holds verification requests that arrived during state sync + /// and have not started yet. + pub async fn start( + mut self, + mut processor: Processor, + deferred: Vec>, + ) { let mut pending_prune = None; let mut deferred_message = None; let mut verifications = Verifications::new(self.marshal.clone()); - for request in std::mem::take(&mut self.deferred_verifications) { - verifications.schedule(self.processor.verifier(), request); + for request in deferred { + verifications.schedule(processor.verifier(), request); } + // One signal for the actor's whole life. Re-creating it per iteration + // would record an extra auditor event on the deterministic runtime each + // time. + let mut shutdown = self.context.stopped(); + // Deferred finalize flushes, each releasing its block's marshal // acknowledgement once the flush completes (see `Barrier`). let mut syncs = Pool::<(Height, bool)>::default(); let mut pending_syncs = BTreeSet::new(); - select_loop! { - self.context, - on_start => { - // Observe every already-completed flush (releasing its marshal - // acknowledgement) before taking the next unit of work, so - // acknowledgements keep flowing even while the mailbox is - // never idle. - while let Some(completion) = syncs.next_completed().now_or_never() { - if !complete(&mut pending_syncs, completion) { - return; - } + + loop { + // Observe every already-completed flush (releasing its marshal + // acknowledgement) before taking the next unit of work, so + // acknowledgements keep flowing even while the mailbox is + // never idle. + while let Some(completion) = syncs.next_completed().now_or_never() { + if !complete(&mut pending_syncs, completion) { + return; } + } - // Publish completed verdicts before admitting another message. - // A later finalization cannot retroactively invalidate them. - verifications.complete_ready(); - - // A message deferred by an active proposal is the FIFO barrier - // for subsequent mailbox work, so handle it before later arrivals. - let message = match deferred_message.take() { - Some(message) => Ok(message), - None => self.mailbox.try_recv(), - }; - - // Pruning is non-critical work. We only run it when the mailbox is idle, and - // it is never raced against the mailbox due to its internal lock acquisition. - // If a message is ready, it is always processed immediately. - let next = match message { - Ok(message) => Either::Left(ready(Some(Step::Message(message)))), - Err(TryRecvError::Empty) => { - match pending_prune.take() { - // No message, but a prune is queued: run it. - Some(prune) => Either::Left(ready(Some(Step::Prune(prune)))), - // No message and nothing to prune: wait on the mailbox, driving flush - // completions while idle. - None => { - let mailbox = &mut self.mailbox; - let syncs = &mut syncs; - let pending_syncs = &mut pending_syncs; - let verifications = &mut verifications; - Either::Right(async move { - loop { - select! { - message = mailbox.recv() => { - break message.map(Step::Message); - }, - completion = syncs.next_completed() => { - if !complete(pending_syncs, completion) { - return None; - } - }, - _ = verifications.next_completed() => { - continue; - }, + // Publish completed verdicts before admitting another message, so + // continuous mailbox traffic cannot starve them under the biased + // `select!`. + verifications.complete_ready(); + + // A message deferred by an active proposal is the FIFO barrier + // for subsequent mailbox work, so handle it before later arrivals. + let message = match deferred_message.take() { + Some(message) => Ok(message), + None => self.mailbox.try_recv(), + }; + + // Pruning is non-critical work, run only when the mailbox is idle. + // If a message is ready, it is always processed immediately. + let next = match message { + // A message is ready. Handle it now, regardless of any queued prune. + Ok(message) => Either::Left(ready(Some(Step::Message(message)))), + Err(TryRecvError::Empty) => match pending_prune.take() { + // No message, but a prune is queued. Run it. + Some(prune) => Either::Left(ready(Some(Step::Prune(prune)))), + // No message and nothing to prune. Wait on the mailbox, driving + // flush completions and verification jobs while idle. + None => { + let mailbox = &mut self.mailbox; + let syncs = &mut syncs; + let pending_syncs = &mut pending_syncs; + let verifications = &mut verifications; + Either::Right(async move { + loop { + select! { + message = mailbox.recv() => { + if message.is_none() { + debug!("mailbox closed, stopping processing"); } - } - }) + break message.map(Step::Message); + }, + completion = syncs.next_completed() => { + if !complete(pending_syncs, completion) { + return None; + } + }, + _ = verifications.next_completed() => { + continue; + }, + } } - } - } - Err(TryRecvError::Disconnected) => { - debug!("mailbox closed, stopping processing"); - return; + }) } - }; - }, - on_stopped => { - debug!("shutdown signal received, stopping processing"); - }, - Some(step) = next else { - debug!("mailbox closed, stopping processing"); - break; - } => match step { + }, + Err(TryRecvError::Disconnected) => { + debug!("mailbox closed, stopping processing"); + return; + } + }; + + let step = select! { + _ = &mut shutdown => { + debug!("shutdown signal received, stopping processing"); + return; + }, + step = next => step, + }; + let Some(step) = step else { + return; + }; + + match step { Step::Message(Message::Propose { span, context, @@ -212,14 +219,11 @@ where upstream, provider: self.provider.clone(), }; - let verifier = self.processor.verifier(); let actor_context = self.context.as_present(); - let marshal = self.marshal.clone(); - let proposal = self - .processor + let proposal = processor .propose( actor_context, - marshal.clone(), + self.marshal.clone(), context, ancestry, input, @@ -239,7 +243,7 @@ where ancestry, verification, }) => verifications.schedule( - verifier.clone(), + processor.verifier(), VerificationRequest { span, context, @@ -273,7 +277,7 @@ where verification, }) => { verifications.schedule( - self.processor.verifier(), + processor.verifier(), VerificationRequest { span, context, @@ -286,88 +290,78 @@ where span, block, acknowledgement, - retry_mailbox, + .. }) => { let process = info_span!(parent: &span, "stateful.actor.finalized"); if skip_finalized_block(&mut self.skip_finalized_until, block.height()) { + let notify = + processor.notify_finalized(self.context.as_present(), block.as_ref()); async { - verifications - .drive(self.processor.notify_finalized( - self.context.as_present(), - block.as_ref(), - )) - .await; + verifications.drive(notify).await; acknowledgement.acknowledge(); } .instrument(process) .await; - } else { - let boundary = self.processor.finalization_boundary(block.as_ref()); - let (retry, reject) = verifications - .quiesce_where(|progress| boundary.disposition(progress)) - .await; - drop(boundary); - async { - let applied = verifications - .drive(self.processor.finalize(&self.context, block.as_ref())) - .await; - let Some(Applied { - snapshots, - barrier, - prune, - }) = applied - else { - // Duplicate report: marshal redelivers a processed - // height only after a restart, where startup aligned - // the databases to durable state. - acknowledgement.acknowledge(); - return; - }; - debug!( - height = block.height().get(), - "applied finalized database batch" - ); - - // The snapshots serve immediately; peers verify what - // they fetch against a finalized root, so serving - // safely runs ahead of disk. - let height = block.height(); - self.snapshot_publisher.publish(height, snapshots); - - // Acknowledge marshal only once the batch's flush - // completes, so marshal's processed floor never runs - // ahead of flushed database state (the startup rewind - // contract), without blocking the loop on the flush. - // Marshal's ack window bounds the flush backlog. A - // false `Barrier::durable` leaves the block - // unacknowledged, and marshal redelivers it on restart. - assert!( - pending_syncs.insert(height), - "finalize flush height must be unique", - ); - syncs.push(async move { - let durable = barrier.durable().await; - if durable { - acknowledgement.acknowledge(); - } - (height, durable) - }); - if let Some(prune) = prune { - pending_prune = Some((prune, retry_mailbox.clone())); - } - } - .instrument(process) + continue; + } + + // The apply owns mutation. Live verification jobs pause at + // their next batch operation and resume afterward, so they + // keep being polled throughout. + let applied; + (processor, applied) = verifications + .drive(processor.finalize(self.context.as_present(), block.as_ref())) + .instrument(process.clone()) .await; - for verification in reject { - verification.respond(false); + + // Keep the publication bookkeeping under the same span. + let _span = process.entered(); + let Some(Applied { + snapshots, + barrier, + prune, + }) = applied + else { + // A duplicate report. Marshal redelivers a processed height + // only after a restart, where startup aligned the databases + // to durable state. + acknowledgement.acknowledge(); + continue; + }; + debug!( + height = block.height().get(), + "applied finalized database batch" + ); + + // The snapshots serve immediately; peers verify what they + // fetch against a finalized root, so serving safely runs + // ahead of disk. + let height = block.height(); + self.snapshot_publisher.publish(height, snapshots); + + // Acknowledge marshal only once the batch's flush completes, + // so marshal's processed floor never runs ahead of flushed + // database state (the startup rewind contract), without + // blocking the loop on the flush. Marshal's ack window + // bounds the flush backlog. A false `Barrier::durable` + // leaves the block unacknowledged, and marshal redelivers + // it on restart. + assert!( + pending_syncs.insert(height), + "finalize flush height must be unique", + ); + syncs.push(async move { + let durable = barrier.durable().await; + if durable { + acknowledgement.acknowledge(); } - requeue_verifications(retry_mailbox.as_ref(), retry); + (height, durable) + }); + if let Some(prune) = prune { + pending_prune = Some(prune); } } - Step::Message(Message::SubscribeDatabases { response }) => { - response.send_lossy(self.processor.databases().clone()); - } - Step::Prune((prune, retry_mailbox)) => { + Step::Prune(prune) => { // The prune target must be durable, but later blocks remain available in // marshal for replay and do not delay maintenance. Verification may complete // during this wait. Later mailbox work remains ordered behind the prune. @@ -384,22 +378,17 @@ where _ = verifications.next_completed() => {}, } } - let retry = verifications.quiesce().await; - assert!( - self.processor.replays_idle(), - "verification replay remained active after quiescence" - ); - prune - .run(self.processor.databases(), &self.marshal) + + processor = verifications + .drive(processor.prune(prune, &self.marshal)) .await; // The published snapshots predate this prune and pin the pruned // storage, so capture and publish afresh right away. - self.processor - .publish_snapshot(&mut self.snapshot_publisher) + processor = verifications + .drive(processor.publish_snapshot(&mut self.snapshot_publisher)) .await; - requeue_verifications(retry_mailbox.as_ref(), retry); } - }, + } } } } @@ -422,13 +411,13 @@ fn skip_finalized_block(skip_until: &mut Option, height: Height) -> bool mod tests { use super::{Message, Processing, VerificationRequest, skip_finalized_block}; use crate::stateful::{ - Application, Input, Proposed, PruneConfig, + Application, ExecutionError, Input, Proposed, PruneConfig, actor::{ core::mailbox::Mailbox, metrics::Metrics as StatefulMetrics, processor::{Processor, Pruning}, }, - db::{DatabaseSet, Publisher, Shared, Subscriber}, + db::{Publisher, ReadersOf, Single, SnapshotsOf, Subscriber}, tests::{ fixtures, mocks::{ @@ -478,6 +467,10 @@ mod tests { verify_gates: Arc>>, proposal_gate: Arc>>, verify_valid: bool, + /// Verifications that return [`ExecutionError::Stale`] after their gate + /// releases, standing in for a batch read refused by a competing + /// finalization. + stale_verifies: Arc>, observed_contexts: Arc>>, } @@ -503,13 +496,13 @@ mod tests { _ancestry: impl Ancestry, _batches: TestUnmerkleized, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { let gate = self.proposal_gate.lock().take(); if let Some(mut gate) = gate { let _ = gate.started.send(()); let _ = (&mut gate.release).await; } - None + Ok(None) } async fn verify( @@ -517,10 +510,12 @@ mod tests { context: (deterministic::Context, Self::Context), ancestry: impl Ancestry, _batches: TestUnmerkleized, - ) -> Option { + ) -> Result, ExecutionError> { self.observed_contexts.lock().push(context.0.name()); let mut ancestry = Box::pin(ancestry); - let _block = ancestry.next().await?; + let Some(_block) = ancestry.next().await else { + return Ok(None); + }; let mut gate = self .verify_gates .lock() @@ -528,7 +523,14 @@ mod tests { .expect("unexpected verification"); let _ = gate.started.send(()); let _ = (&mut gate.release).await; - self.verify_valid.then_some(TestMerkleized) + { + let mut stale = self.stale_verifies.lock(); + if *stale > 0 { + *stale -= 1; + return Err(ExecutionError::Stale); + } + } + Ok(self.verify_valid.then_some(TestMerkleized)) } async fn apply( @@ -536,8 +538,8 @@ mod tests { _context: (deterministic::Context, Self::Context), _block: &Self::Block, _batches: TestUnmerkleized, - ) -> TestMerkleized { - TestMerkleized + ) -> Result { + Ok(TestMerkleized) } } @@ -573,7 +575,7 @@ mod tests { _ancestry: impl Ancestry, _batches: TestUnmerkleized, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { panic!("replay-gated application proposal is not used") } @@ -582,14 +584,14 @@ mod tests { _context: (deterministic::Context, Self::Context), _ancestry: impl Ancestry, _batches: TestUnmerkleized, - ) -> Option { + ) -> Result, ExecutionError> { self.verify_calls.fetch_add(1, Ordering::SeqCst); let gate = self.verify_gate.lock().take(); if let Some(mut gate) = gate { let _ = gate.started.send(()); let _ = (&mut gate.release).await; } - Some(TestMerkleized) + Ok(Some(TestMerkleized)) } async fn apply( @@ -597,7 +599,7 @@ mod tests { _context: (deterministic::Context, Self::Context), block: &Self::Block, _batches: TestUnmerkleized, - ) -> TestMerkleized { + ) -> Result { self.apply_calls.fetch_add(1, Ordering::SeqCst); let gate = (block.height() == self.gate_height) .then(|| self.gates.lock().pop_front()) @@ -606,14 +608,14 @@ mod tests { let _ = gate.started.send(()); let _ = (&mut gate.release).await; } - TestMerkleized + Ok(TestMerkleized) } async fn finalized( &mut self, _context: (deterministic::Context, Self::Context), _block: &Self::Block, - _readers: >::Readers, + _readers: ReadersOf, ) { let gate = self.finalized_gate.lock().take(); if let Some(mut gate) = gate { @@ -642,7 +644,7 @@ mod tests { app: GatedApp, ) -> ( Mailbox, - Subscriber, + Subscriber>, Box, Handle<()>, ) { @@ -666,19 +668,20 @@ mod tests { None, ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); - let (publisher, subscriber) = Publisher::new(context); + let publication_context = context.child("publication"); + let (publisher, reader) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, snapshot_publisher: publisher, - deferred_verifications: Vec::new(), skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); - (Mailbox::new(sender), subscriber, marshal.guards, actor) + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + (Mailbox::new(sender), reader, marshal.guards, actor) } /// Spawn a [`Processing`] loop over a gated [`TestDb`], returning its @@ -722,15 +725,15 @@ mod tests { false, ) .await; - let control = FlushControl::default(); - let databases = Shared::new("test", TestDb::gated(control.clone())); + let databases = Single::from(TestDb::gated(control.clone())); let pruning = prune_config .map(|config| Pruning::build(config, marshal.mailbox.max_pending_acks(), 0)); let app = GatedApp { verify_gates: Arc::new(Mutex::new(verify_gates)), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let processor = Processor::new( @@ -740,27 +743,21 @@ mod tests { StatefulMetrics::new(context), pruning, ); + let (mut publisher, reader) = Publisher::new(context); + let processor = processor.publish_snapshot(&mut publisher).await; let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); - let (mut publisher, subscriber) = Publisher::new(context); - processor.publish_snapshot(&mut publisher).await; let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, snapshot_publisher: publisher, - deferred_verifications: Vec::new(), skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); - ( - Mailbox::new(sender), - control, - subscriber, - marshal.guards, - actor, - ) + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + (Mailbox::new(sender), control, reader, marshal.guards, actor) } /// The value of the `publications` counter. @@ -783,6 +780,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([first_gate, second_gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -840,6 +838,155 @@ mod tests { }); } + /// A verification that goes stale because its own block finalized mid-execution + /// is answered from the canonical chain as true, not false. + #[test] + fn stale_verification_of_finalized_block_answers_true() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let (gate, started, release) = application_gate(); + let app = GatedApp { + verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), + proposal_gate: Arc::new(Mutex::new(None)), + verify_valid: true, + stale_verifies: Arc::new(Mutex::new(1)), + observed_contexts: Arc::default(), + }; + let (mut mailbox, _subscriber, marshal, actor) = + spawn_gated_application(&context, "stale-self-finalized", app).await; + + let genesis = TestBlock::new(0, 0); + let block = TestBlock::child(&genesis, 1); + let block_context = block.context(); + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), block_context), + ancestry::from_iter([Arc::new(block.clone()), Arc::new(genesis)]), + )); + assert!(poll!(&mut verify).is_pending()); + started.await.expect("verification should start"); + + // The block itself finalizes while its verification is parked. + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(block), acknowledgement)); + waiter.await.expect("finalization should be acknowledged"); + + // The parked execution resumes and refuses with Stale, and the verifier + // re-checks canonical state and answers true. + release.send(()).expect("verification should remain active"); + assert!(verify.await); + actor.abort(); + drop(marshal); + }); + } + + /// A verification that goes stale because a competing block finalized is + /// answered from the canonical chain as false. + #[test] + fn stale_verification_of_competing_block_answers_false() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let (gate, started, release) = application_gate(); + let app = GatedApp { + verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), + proposal_gate: Arc::new(Mutex::new(None)), + verify_valid: true, + stale_verifies: Arc::new(Mutex::new(1)), + observed_contexts: Arc::default(), + }; + let (mut mailbox, _subscriber, marshal, actor) = + spawn_gated_application(&context, "stale-competing", app).await; + + let genesis = TestBlock::new(0, 0); + let block = TestBlock::child(&genesis, 1); + let winner = TestBlock::child(&genesis, 2); + let block_context = block.context(); + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), block_context), + ancestry::from_iter([Arc::new(block), Arc::new(genesis)]), + )); + assert!(poll!(&mut verify).is_pending()); + started.await.expect("verification should start"); + + // A competing block at the same height finalizes while the + // verification is parked. + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(winner), acknowledgement)); + waiter.await.expect("finalization should be acknowledged"); + + release.send(()).expect("verification should remain active"); + assert!(!verify.await); + actor.abort(); + drop(marshal); + }); + } + + /// A stale attempt whose candidate is still above the new anchor re-executes + /// against the post-finalization state and completes with a verdict. (The + /// staleness is injected through the mock, while the storage-refused shape is + /// pinned by `fork_refuses_inside_the_finalize_window`.) + #[test] + fn stale_verification_reexecutes_and_answers_true() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let (parent_gate, parent_started, parent_release) = application_gate(); + let (first, first_started, first_release) = application_gate(); + let (second, second_started, second_release) = application_gate(); + let app = GatedApp { + verify_gates: Arc::new(Mutex::new(VecDeque::from([parent_gate, first, second]))), + proposal_gate: Arc::new(Mutex::new(None)), + verify_valid: true, + stale_verifies: Arc::default(), + observed_contexts: Arc::default(), + }; + let staleness = app.stale_verifies.clone(); + let (mut mailbox, _subscriber, marshal, actor) = + spawn_gated_application(&context, "stale-reexecute", app).await; + + // Verify the parent first so the candidate forks from pending state. + let genesis = TestBlock::new(0, 0); + let parent = TestBlock::child(&genesis, 1); + let block = TestBlock::child(&parent, 2); + let mut parent_verifier = mailbox.clone(); + let mut parent_verify = Box::pin(parent_verifier.verify( + (context.child("verify_parent"), parent.context()), + ancestry::from_iter([Arc::new(parent.clone()), Arc::new(genesis)]), + )); + assert!(poll!(&mut parent_verify).is_pending()); + parent_started + .await + .expect("parent verification should start"); + parent_release + .send(()) + .expect("parent verification should remain active"); + assert!(parent_verify.await); + + let block_context = block.context(); + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), block_context), + ancestry::from_iter([Arc::new(block), Arc::new(parent.clone())]), + )); + assert!(poll!(&mut verify).is_pending()); + first_started.await.expect("verification should start"); + *staleness.lock() = 1; + + // The candidate's parent finalizes while the candidate executes. + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(parent), acknowledgement)); + waiter.await.expect("finalization should be acknowledged"); + + // The stale attempt re-classifies (still above the anchor) and + // re-executes against the new state. + first_release + .send(()) + .expect("verification should remain active"); + second_started.await.expect("retry should re-execute"); + second_release.send(()).expect("retry should remain active"); + assert!(verify.await); + actor.abort(); + drop(marshal); + }); + } + #[test] fn verification_preserves_request_attributes() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { @@ -849,6 +996,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: observed_contexts.clone(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -896,6 +1044,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -930,6 +1079,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([first_gate, second_gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -991,6 +1141,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: false, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1047,66 +1198,6 @@ mod tests { }); } - #[test] - fn pending_proposal_does_not_block_active_verification() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let (verify_gate, verify_started, verify_release) = application_gate(); - let (proposal_gate, proposal_started, proposal_release) = application_gate(); - let app = GatedApp { - verify_gates: Arc::new(Mutex::new(VecDeque::from([verify_gate]))), - proposal_gate: Arc::new(Mutex::new(Some(proposal_gate))), - verify_valid: true, - observed_contexts: Arc::default(), - }; - let (mut mailbox, _subscriber, _marshal, actor) = - spawn_gated_application(&context, "propose-verify", app).await; - - let genesis = TestBlock::new(0, 0); - let block = TestBlock::child(&genesis, 1); - let mut verifier = mailbox.clone(); - let verify_genesis = genesis.clone(); - let consensus_context = block.context(); - let mut verify = Box::pin(verifier.verify( - (context.child("verify"), consensus_context), - ancestry::from_iter([Arc::new(block), Arc::new(verify_genesis)]), - )); - let subscriber = mailbox.clone(); - assert!(poll!(&mut verify).is_pending()); - verify_started.await.expect("verification should start"); - - let proposal_context = TestBlock::child(&genesis, 2).context(); - let mut proposal = Box::pin(mailbox.propose( - (context.child("propose"), proposal_context), - ancestry::from_iter([Arc::new(genesis)]), - (), - )); - assert!(poll!(&mut proposal).is_pending()); - proposal_started.await.expect("proposal should start"); - let mut databases = Box::pin(subscriber.subscribe_databases()); - assert!(poll!(&mut databases).is_pending()); - context.sleep(Duration::from_millis(10)).await; - verify_release - .send(()) - .expect("verification should remain active"); - select! { - result = &mut verify => { - assert!(result); - }, - _ = context.sleep(Duration::from_millis(100)) => { - panic!("pending proposal blocked active verification"); - }, - } - assert!(poll!(&mut databases).is_pending()); - - proposal_release - .send(()) - .expect("proposal should remain active"); - assert!(proposal.await.is_none()); - drop(databases.await); - actor.abort(); - }); - } - #[test] fn pending_proposal_does_not_block_new_verification() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { @@ -1116,6 +1207,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([verify_gate]))), proposal_gate: Arc::new(Mutex::new(Some(proposal_gate))), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1170,6 +1262,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([parent_gate, child_gate]))), proposal_gate: Arc::new(Mutex::new(Some(proposal_gate))), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1245,19 +1338,15 @@ mod tests { } #[test] - fn finalization_keeps_compatible_verification_active() { + fn finalization_keeps_compatible_verification_running() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let (parent_gate, parent_started, parent_release) = application_gate(); let (child_gate, child_started, child_release) = application_gate(); - let (retry_gate, _retry_started, _retry_release) = application_gate(); let app = GatedApp { - verify_gates: Arc::new(Mutex::new(VecDeque::from([ - parent_gate, - child_gate, - retry_gate, - ]))), + verify_gates: Arc::new(Mutex::new(VecDeque::from([parent_gate, child_gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1298,23 +1387,27 @@ mod tests { waiter .await .expect("finalized parent should be acknowledged"); + + // The apply does not touch the child's attempt. It is still waiting + // in the application and answers from that same attempt. child_release .send(()) - .expect("compatible verification should remain active"); + .expect("the attempt should still be live after the apply"); assert!(verify_child.await); actor.abort(); }); } #[test] - fn finalization_invalidates_incompatible_verification() { + fn finalization_refuses_incompatible_verification_result() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let (fork_gate, fork_started, fork_release) = application_gate(); - let (child_gate, child_started, mut child_release) = application_gate(); + let (child_gate, child_started, child_release) = application_gate(); let app = GatedApp { verify_gates: Arc::new(Mutex::new(VecDeque::from([fork_gate, child_gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1351,12 +1444,6 @@ mod tests { let (acknowledgement, waiter) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(winner), acknowledgement)); - select! { - _ = child_release.closed() => {}, - _ = context.sleep(Duration::from_millis(100)) => { - panic!("incompatible verification was not cancelled"); - }, - } let mut waiter = Box::pin(waiter); select! { result = &mut waiter => { @@ -1366,12 +1453,19 @@ mod tests { panic!("winning block was not acknowledged"); }, } + + // The losing child runs to completion. Its parent is gone from the + // pending set, so caching its result is refused and the verdict is + // false. + child_release + .send(()) + .expect("the attempt should still be live after the apply"); select! { valid = &mut verify_child => { assert!(!valid, "verification on a finalized-away fork must fail"); }, _ = context.sleep(Duration::from_millis(100)) => { - panic!("incompatible verification retry did not resolve"); + panic!("incompatible verification did not resolve"); }, } actor.abort(); @@ -1383,7 +1477,7 @@ mod tests { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let (parent_gate, parent_started, parent_release) = application_gate(); let (child_gate, child_started, child_release) = application_gate(); - let (grandchild_gate, grandchild_started, mut grandchild_release) = application_gate(); + let (grandchild_gate, grandchild_started, grandchild_release) = application_gate(); let app = GatedApp { verify_gates: Arc::new(Mutex::new(VecDeque::from([ parent_gate, @@ -1392,6 +1486,7 @@ mod tests { ]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let (mut mailbox, _subscriber, _marshal, actor) = @@ -1446,8 +1541,10 @@ mod tests { let (acknowledgement, waiter) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(winner), acknowledgement)); - grandchild_release.closed().await; waiter.await.expect("winning block should be acknowledged"); + grandchild_release + .send(()) + .expect("the attempt should still be live after the apply"); let result = select! { valid = &mut verify_grandchild => Some(valid), @@ -1499,17 +1596,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: Some(finalized.height()), }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let mut verifier = mailbox.clone(); let mut verify_child = Box::pin(verifier.verify( @@ -1562,6 +1661,7 @@ mod tests { verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), proposal_gate: Arc::new(Mutex::new(None)), verify_valid: true, + stale_verifies: Arc::default(), observed_contexts: Arc::default(), }; let mut signing = context.child("signing"); @@ -1617,17 +1717,19 @@ mod tests { }; // Resume the deferred verification after state sync. + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: vec![request], + snapshot_publisher: publisher, skip_finalized_until: Some(Height::new(0)), }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, vec![request])); started.await.expect("deferred verification should resume"); release @@ -1644,7 +1746,7 @@ mod tests { } #[test] - fn finalization_reuses_active_replay() { + fn finalization_does_not_wait_for_a_shared_replay() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let genesis = TestBlock::new(0, 0); let parent = TestBlock::child(&genesis, 1); @@ -1684,17 +1786,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let consensus_context = first_child.context(); let mut first_verifier = mailbox.clone(); @@ -1711,63 +1815,243 @@ mod tests { assert!(poll!(&mut second).is_pending()); apply_started.await.expect("replay should start"); context.sleep(Duration::from_millis(10)).await; + assert_eq!( + apply_calls.load(Ordering::SeqCst), + 1, + "siblings needing the same parent should share one replay", + ); let (acknowledgement, waiter) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(parent), acknowledgement)); context.sleep(Duration::from_millis(10)).await; - apply_release - .send(()) - .expect("finalization should keep the replay active"); - verify_started - .await - .expect("compatible verification should start"); + + // The apply does not wait for the shared replay. Because that replay + // has not cached the winner yet, the finalization reconstructs it. finalized_started .await .expect("finalization hook should start"); - context.sleep(Duration::from_millis(10)).await; - assert_eq!(apply_calls.load(Ordering::SeqCst), 1); - verify_release - .send(()) - .expect("compatible verification should remain active"); - assert!(first.await); finalized_release .send(()) .expect("finalization hook should remain active"); waiter .await .expect("finalized parent should be acknowledged"); + + // The shared replay is still live afterward. Its parent is now the + // processed anchor, so both siblings continue from it. + apply_release + .send(()) + .expect("the shared replay should still be live after the apply"); + verify_started + .await + .expect("verification should reach the application"); + let _ = verify_release.send(()); + assert!(first.await); assert!(second.await); - assert_eq!(apply_calls.load(Ordering::SeqCst), 1); + assert_eq!( + apply_calls.load(Ordering::SeqCst), + 2, + "the finalization reconstructs the winner once, and the siblings \ + need no further replay", + ); assert_eq!(verify_calls.load(Ordering::SeqCst), 2); actor.abort(); drop(marshal.guards); }); } + /// A candidate whose own finalization completes while its parent replay is + /// parked is answered true from the canonical chain, even though the replay + /// resolves as invalid ancestry afterward. #[test] - fn finalization_does_not_bypass_active_winner_replay() { + fn candidate_finalized_during_parent_replay_answers_true() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let genesis = TestBlock::new(0, 0); - let finalized = TestBlock::child(&genesis, 1); - let child = TestBlock::child(&finalized, 2); + let parent = TestBlock::child(&genesis, 1); + let candidate = TestBlock::child(&parent, 2); let mut signing = context.child("signing"); - let scheme = scheme_mocks::fixture(&mut signing, b"finalize-pending-replay", 1).schemes - [0] - .clone(); + let scheme = + scheme_mocks::fixture(&mut signing, b"finalized-mid-replay", 1).schemes[0].clone(); let marshal = fixtures::marshal_fixture_with_finalized_block( context.child("marshal"), - "finalize-pending-replay", + "finalized-mid-replay", scheme, &genesis, NZUsize!(1), true, ) .await; - let (replay_gate, replay_started, replay_release) = application_gate(); - let apply_calls = Arc::new(AtomicUsize::new(0)); - let verify_calls = Arc::new(AtomicUsize::new(0)); + let (gate, apply_started, apply_release) = application_gate(); let app = ReplayGatedApp { - gates: Arc::new(Mutex::new(VecDeque::from([replay_gate]))), + gates: Arc::new(Mutex::new(VecDeque::from([gate]))), + verify_gate: Arc::new(Mutex::new(None)), + finalized_gate: Arc::new(Mutex::new(None)), + gate_height: parent.height(), + apply_calls: Arc::new(AtomicUsize::new(0)), + verify_calls: Arc::new(AtomicUsize::new(0)), + }; + let processor = Processor::new( + app, + test_databases(), + anchor(0, 0), + StatefulMetrics::new(&context), + None, + ); + let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); + let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); + let processing = Processing { + context: ContextCell::new(context.child("processing")), + mailbox: receiver, + provider: (), + marshal: marshal.mailbox, + snapshot_publisher: publisher, + skip_finalized_until: None, + }; + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + + // The candidate's verification parks inside the shared replay of its + // unknown parent. + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), candidate.context()), + ancestry::from_iter([Arc::new(candidate.clone()), Arc::new(parent)]), + )); + assert!(poll!(&mut verify).is_pending()); + apply_started.await.expect("parent replay should start"); + + // The candidate itself finalizes (reconstructed by the finalization, + // since the parked replay has not cached its parent). + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(candidate), acknowledgement)); + waiter + .await + .expect("finalized candidate should be acknowledged"); + + // The replay resumes, resolves as invalid ancestry (its parent is + // below the new anchor), and the verifier answers from the canonical + // chain instead of voting false. + apply_release + .send(()) + .expect("the parent replay should still be live"); + assert!(verify.await); + actor.abort(); + drop(marshal.guards); + }); + } + + /// A valid descendant whose parent replay crosses the parent's own + /// finalization is retried against the new anchor, not answered false. The + /// interrupted attempt resolves as stale or invalid ancestry, and either way + /// the verifier re-runs and forks the candidate from applied state. + #[test] + fn parent_finalized_during_replay_retries_and_answers_true() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let genesis = TestBlock::new(0, 0); + let parent = TestBlock::child(&genesis, 1); + let candidate = TestBlock::child(&parent, 2); + let mut signing = context.child("signing"); + let scheme = + scheme_mocks::fixture(&mut signing, b"anchor-mid-replay", 1).schemes[0].clone(); + let marshal = fixtures::marshal_fixture_with_finalized_block( + context.child("marshal"), + "anchor-mid-replay", + scheme, + &genesis, + NZUsize!(1), + true, + ) + .await; + let (gate, apply_started, apply_release) = application_gate(); + let app = ReplayGatedApp { + gates: Arc::new(Mutex::new(VecDeque::from([gate]))), + verify_gate: Arc::new(Mutex::new(None)), + finalized_gate: Arc::new(Mutex::new(None)), + gate_height: parent.height(), + apply_calls: Arc::new(AtomicUsize::new(0)), + verify_calls: Arc::new(AtomicUsize::new(0)), + }; + let processor = Processor::new( + app, + test_databases(), + anchor(0, 0), + StatefulMetrics::new(&context), + None, + ); + let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); + let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); + let processing = Processing { + context: ContextCell::new(context.child("processing")), + mailbox: receiver, + provider: (), + marshal: marshal.mailbox, + snapshot_publisher: publisher, + skip_finalized_until: None, + }; + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + + // The candidate's verification parks inside the shared replay of its + // unknown parent. + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), candidate.context()), + ancestry::from_iter([Arc::new(candidate.clone()), Arc::new(parent.clone())]), + )); + assert!(poll!(&mut verify).is_pending()); + apply_started.await.expect("parent replay should start"); + + // The PARENT finalizes while the replay is parked, moving the anchor + // past the walk this attempt started from. The candidate remains a + // valid, unfinalized descendant of the new anchor. + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(parent), acknowledgement)); + waiter + .await + .expect("finalized parent should be acknowledged"); + + // The parked replay resumes and fails against the moved anchor, so + // the verifier retries, forks the candidate from the new anchor, and + // answers true. + apply_release + .send(()) + .expect("the parent replay should still be live"); + assert!(verify.await); + actor.abort(); + drop(marshal.guards); + }); + } + + #[test] + fn finalization_does_not_bypass_active_winner_replay() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let genesis = TestBlock::new(0, 0); + let finalized = TestBlock::child(&genesis, 1); + let child = TestBlock::child(&finalized, 2); + let mut signing = context.child("signing"); + let scheme = scheme_mocks::fixture(&mut signing, b"finalize-pending-replay", 1).schemes + [0] + .clone(); + let marshal = fixtures::marshal_fixture_with_finalized_block( + context.child("marshal"), + "finalize-pending-replay", + scheme, + &genesis, + NZUsize!(1), + true, + ) + .await; + let (replay_gate, replay_started, replay_release) = application_gate(); + let apply_calls = Arc::new(AtomicUsize::new(0)); + let verify_calls = Arc::new(AtomicUsize::new(0)); + let app = ReplayGatedApp { + gates: Arc::new(Mutex::new(VecDeque::from([replay_gate]))), verify_gate: Arc::new(Mutex::new(None)), finalized_gate: Arc::new(Mutex::new(None)), gate_height: finalized.height(), @@ -1783,17 +2067,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let mut child_verifier = mailbox.clone(); let mut verify_child = Box::pin(child_verifier.verify( @@ -1815,18 +2101,13 @@ mod tests { ); let (acknowledgement, waiter) = Exact::handle(); - let mut waiter = Box::pin(waiter); let _ = mailbox.report(Update::Block(Arc::new(finalized), acknowledgement)); - assert!( - poll!(&mut waiter).is_pending(), - "finalization must wait for the existing winner computation", - ); + waiter + .await + .expect("the cached winner should finalize while its replay runs"); replay_release .send(()) .expect("winner replay should remain active"); - waiter - .await - .expect("cached winner should finalize after replay completes"); let valid = verify_child.await; actor.abort(); drop(marshal.guards); @@ -1837,7 +2118,7 @@ mod tests { } #[test] - fn consecutive_finalizations_preserve_descendant_replay() { + fn consecutive_finalizations_retry_descendant_replay() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let genesis = TestBlock::new(0, 0); let first = TestBlock::child(&genesis, 1); @@ -1877,17 +2158,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let mut child_verifier = mailbox.clone(); let mut verify_child = Box::pin(child_verifier.verify( @@ -1922,26 +2205,32 @@ mod tests { .expect("first finalized block should be acknowledged"); verify_started .await - .expect("descendant verification should start"); - assert!(poll!(&mut verify_child).is_pending()); + .expect("descendant verification should re-run after the first finalization"); let (acknowledgement, second_waiter) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(second), acknowledgement)); second_waiter .await .expect("second finalized block should be acknowledged"); - verify_release - .send(()) - .expect("descendant verification should remain active"); + // Each cancellation drops the attempt holding this gate, so releasing + // it is best-effort. + let _ = verify_release.send(()); let valid = verify_child.await; actor.abort(); drop(marshal.guards); assert!( valid, - "descendant replay must remain valid across consecutive finalizations", + "a descendant of both finalized blocks must verify, not be rejected", + ); + assert!( + verify_calls.load(Ordering::SeqCst) > 1, + "the descendant should have re-run against the applied state", + ); + assert_eq!( + apply_calls.load(Ordering::SeqCst), + 2, + "replay should not repeat work already cached as pending state", ); - assert_eq!(apply_calls.load(Ordering::SeqCst), 2); - assert_eq!(verify_calls.load(Ordering::SeqCst), 2); }); } @@ -1984,17 +2273,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let mut first_verifier = mailbox.clone(); let mut first_attempt = Box::pin(first_verifier.verify( @@ -2059,7 +2350,7 @@ mod tests { } #[test] - fn pruning_quiesces_replay_before_database_prune() { + fn pruning_does_not_disturb_a_live_replay() { deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { let genesis = TestBlock::new(0, 0); let block1 = TestBlock::child(&genesis, 1); @@ -2077,12 +2368,11 @@ mod tests { true, ) .await; - let (first_gate, first_started, mut first_release) = application_gate(); - let (second_gate, second_started, second_release) = application_gate(); + let (replay_gate, replay_started, replay_release) = application_gate(); let apply_calls = Arc::new(AtomicUsize::new(0)); let verify_calls = Arc::new(AtomicUsize::new(0)); let app = ReplayGatedApp { - gates: Arc::new(Mutex::new(VecDeque::from([first_gate, second_gate]))), + gates: Arc::new(Mutex::new(VecDeque::from([replay_gate]))), verify_gate: Arc::new(Mutex::new(None)), finalized_gate: Arc::new(Mutex::new(None)), gate_height: parent.height(), @@ -2090,7 +2380,7 @@ mod tests { verify_calls: verify_calls.clone(), }; let control = FlushControl::default(); - let databases = Shared::new("prune-replay", TestDb::gated(control.clone())); + let databases = Single::from(TestDb::gated(control.clone())); let pruning = Pruning::build( PruneConfig { maintenance_interval: NZUsize!(1), @@ -2109,17 +2399,19 @@ mod tests { ); let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); let processing = Processing { context: ContextCell::new(context.child("processing")), mailbox: receiver, provider: (), marshal: marshal.mailbox.clone(), - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), + snapshot_publisher: publisher, skip_finalized_until: None, }; - let actor = context.child("loop").spawn(move |_| processing.start()); + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); let (acknowledgement, waiter1) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(block1), acknowledgement)); @@ -2137,7 +2429,9 @@ mod tests { assert!(poll!(&mut verify).is_pending()); select! { - result = first_started => result.expect("verification should start before pruning"), + result = replay_started => { + result.expect("verification should start before pruning"); + }, _ = context.sleep(Duration::from_millis(100)) => { panic!( "verification did not start: flushes={} pruned={}", @@ -2152,20 +2446,22 @@ mod tests { .send(Ok(())) .expect("target flush should be pending"); waiter1.await.expect("target block should be acknowledged"); - first_release.closed().await; + + // The prune runs while the replay is still parked in the + // application, and the replay then finishes on its first attempt. while control.pruned.lock().is_empty() { context.sleep(Duration::from_millis(10)).await; } - - second_started - .await - .expect("live replay should restart after pruning"); - second_release + replay_release .send(()) - .expect("restarted replay should remain active"); + .expect("the replay should still be live after pruning"); assert!(verify.await); assert_eq!(control.pruned.lock().clone(), vec![1]); - assert_eq!(apply_calls.load(Ordering::SeqCst), 4); + assert_eq!( + apply_calls.load(Ordering::SeqCst), + 3, + "two finalizations reconstruct their own blocks, plus the replay", + ); assert_eq!(verify_calls.load(Ordering::SeqCst), 1); let release = control.flushes.lock().remove(0); @@ -2176,165 +2472,23 @@ mod tests { }); } - #[test] - fn prune_retries_wait_for_queued_finalization() { - deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { - let genesis = TestBlock::new(0, 0); - let block1 = TestBlock::child(&genesis, 1); - let block2 = TestBlock::child(&block1, 2); - let losing = TestBlock::child(&block2, 3); - let winner = TestBlock::child(&block2, 4); - let mut signing = context.child("signing"); - let scheme = scheme_mocks::fixture(&mut signing, b"prune-retry", 1).schemes[0].clone(); - let marshal = fixtures::marshal_fixture( - context.child("marshal"), - "prune-retry", - scheme, - None, - NZUsize!(1), - false, - ) - .await; - let (verify_gate, verify_started, mut verify_release) = application_gate(); - let (proposal_gate, proposal_started, proposal_release) = application_gate(); - let observed_contexts: Arc>> = Arc::default(); - let app = GatedApp { - verify_gates: Arc::new(Mutex::new(VecDeque::from([verify_gate]))), - proposal_gate: Arc::new(Mutex::new(Some(proposal_gate))), - verify_valid: true, - observed_contexts: observed_contexts.clone(), - }; - let control = FlushControl::default(); - let (prune_started, prune_release) = control.gate_prune(); - let databases = Shared::new("prune-retry", TestDb::gated(control.clone())); - let pruning = Pruning::build( - PruneConfig { - maintenance_interval: NZUsize!(1), - retained_marshal_blocks: 0, - retained_qmdb_blocks: 0, - }, - 1, - 0, - ); - let processor = Processor::new( - app, - databases, - anchor(0, 0), - StatefulMetrics::new(&context), - Some(pruning), - ); - let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); - let mut mailbox = Mailbox::new(sender); - let processing = Processing { - context: ContextCell::new(context.child("processing")), - mailbox: receiver, - provider: (), - marshal: marshal.mailbox, - processor, - snapshot_publisher: Publisher::new(&context).0, - deferred_verifications: Vec::new(), - skip_finalized_until: None, - }; - let actor = context.child("loop").spawn(move |_| processing.start()); - - let (acknowledgement, waiter1) = Exact::handle(); - let _ = mailbox.report(Update::Block(Arc::new(block1), acknowledgement)); - let (acknowledgement, waiter2) = Exact::handle(); - let _ = mailbox.report(Update::Block(Arc::new(block2.clone()), acknowledgement)); - let mut verifier = mailbox.clone(); - let mut verify = Box::pin(verifier.verify( - (context.child("verify"), losing.context()), - ancestry::from_iter([Arc::new(losing), Arc::new(block2)]), - )); - assert!(poll!(&mut verify).is_pending()); - verify_started - .await - .expect("verification should start before pruning"); - - while control.flushes.lock().len() < 2 { - context.sleep(Duration::from_millis(10)).await; - } - control - .flushes - .lock() - .remove(0) - .send(Ok(())) - .expect("target flush should remain pending"); - waiter1.await.expect("target block should be acknowledged"); - prune_started.await.expect("prune should start"); - verify_release.closed().await; - - let (acknowledgement, winner_waiter) = Exact::handle(); - let _ = mailbox.report(Update::Block(Arc::new(winner.clone()), acknowledgement)); - let proposal_context = TestBlock::child(&winner, 5).context(); - let mut proposer = mailbox.clone(); - let mut proposal = Box::pin(proposer.propose( - (context.child("propose"), proposal_context), - ancestry::from_iter([Arc::new(winner)]), - (), - )); - assert!(poll!(&mut proposal).is_pending()); - prune_release.send(()).expect("prune should remain active"); - proposal_started - .await - .expect("proposal queued behind finalization should start"); - - let subscriber = mailbox.clone(); - let mut databases = Box::pin(subscriber.subscribe_databases()); - assert!(poll!(&mut databases).is_pending()); - - let result = select! { - valid = &mut verify => Some(valid), - _ = context.sleep(Duration::from_millis(100)) => None, - }; - assert_eq!( - result, - Some(false), - "prune retry must observe the queued finalization", - ); - assert!(poll!(&mut databases).is_pending()); - assert_eq!(observed_contexts.lock().len(), 1); - assert_eq!(control.pruned.lock().as_slice(), [1]); - - proposal_release - .send(()) - .expect("proposal should remain active"); - assert!(proposal.await.is_none()); - drop(databases.await); - - while control.flushes.lock().len() < 2 { - context.sleep(Duration::from_millis(10)).await; - } - for release in control.flushes.lock().drain(..) { - release - .send(Ok(())) - .expect("finalize flush should remain pending"); - } - waiter2.await.expect("block 2 should be acknowledged"); - winner_waiter.await.expect("winner should be acknowledged"); - actor.abort(); - drop(marshal.guards); - }); - } - /// Pruning waits for the flush that covers its target without waiting for newer state. #[test] fn prune_waits_only_for_target_flush() { deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { // Marshal only receives prune requests here. Its actor never runs. let (verify_gate, verify_started, verify_release) = application_gate(); - let (mut mailbox, control, _subscriber, _marshal, _actor) = - spawn_processing_with_gates( - &context, - "gated-prune", - Some(PruneConfig { - maintenance_interval: NZUsize!(1), - retained_marshal_blocks: 0, - retained_qmdb_blocks: 0, - }), - VecDeque::from([verify_gate]), - ) - .await; + let (mut mailbox, control, subscriber, _marshal, _actor) = spawn_processing_with_gates( + &context, + "gated-prune", + Some(PruneConfig { + maintenance_interval: NZUsize!(1), + retained_marshal_blocks: 0, + retained_qmdb_blocks: 0, + }), + VecDeque::from([verify_gate]), + ) + .await; let genesis = TestBlock::new(0, 0); let block1 = TestBlock::child(&genesis, 1); @@ -2368,6 +2522,11 @@ mod tests { poll!(&mut waiter1).is_pending() && poll!(&mut waiter2).is_pending(), "acknowledgements must wait for pending flushes", ); + assert_eq!( + subscriber.latest(), + Some(2), + "snapshots serve at apply, ahead of their flushes", + ); // Block 2 filled the retention window, but pruning must remain blocked behind the // target at block 1. @@ -2468,7 +2627,7 @@ mod tests { #[test] fn aborted_target_flush_prevents_prune() { deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { - let (mut mailbox, control, _subscriber, _marshal, actor) = spawn_processing( + let (mut mailbox, control, subscriber, _marshal, actor) = spawn_processing( &context, "gated-aborted-prune", Some(PruneConfig { @@ -2507,6 +2666,10 @@ mod tests { control.pruned.lock().is_empty(), "aborted flush must prevent pruning", ); + assert!( + subscriber.latest().is_none(), + "serving must shut off after an aborted flush", + ); }); } @@ -2558,7 +2721,7 @@ mod tests { #[test] fn idle_acks_follow_flush_outcome() { deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { - let (mut mailbox, control, _subscriber, _marshal, actor) = + let (mut mailbox, control, subscriber, _marshal, actor) = spawn_processing(&context, "gated-idle", None).await; // Park the loop idle with block 1's flush pending. @@ -2598,6 +2761,10 @@ mod tests { waiter2.await.is_err(), "unflushed block acknowledgement must be canceled", ); + assert!( + subscriber.latest().is_none(), + "sources must decline after the loop stops", + ); }); } @@ -2640,7 +2807,7 @@ mod tests { #[test] fn shutdown_cancels_pending_flush_ack() { deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { - let (mut mailbox, control, _subscriber, _marshal, actor) = + let (mut mailbox, control, subscriber, _marshal, actor) = spawn_processing(&context, "gated-shutdown", None).await; let (acknowledgement, waiter) = Exact::handle(); @@ -2658,6 +2825,10 @@ mod tests { waiter.await.is_err(), "shutdown must cancel in-flight acknowledgements", ); + assert!( + subscriber.latest().is_none(), + "serving must shut off once the actor stops", + ); }); } @@ -2688,6 +2859,113 @@ mod tests { }); } + /// An application whose proposal path observes fatal storage. + #[derive(Clone)] + struct FatalProposeApp; + + impl Application for FatalProposeApp { + type SigningScheme = TestScheme; + type Context = >::Context; + type Block = TestBlock; + type Databases = TestDatabases; + type Provider = (); + type Input = (); + + fn sync_targets(block: &Self::Block) -> u64 { + block.height().get() + } + + async fn genesis(&mut self) -> Self::Block { + panic!("fatal-propose application genesis is not used") + } + + async fn propose( + &mut self, + _context: (deterministic::Context, Self::Context), + _ancestry: impl Ancestry, + _batches: TestUnmerkleized, + _input: Input, + ) -> Result>, ExecutionError> { + Err(ExecutionError::Fatal("disk failed".into())) + } + + async fn verify( + &mut self, + _context: (deterministic::Context, Self::Context), + _ancestry: impl Ancestry, + _batches: TestUnmerkleized, + ) -> Result, ExecutionError> { + panic!("fatal-propose application verify is not used") + } + + async fn apply( + &mut self, + _context: (deterministic::Context, Self::Context), + _block: &Self::Block, + _batches: TestUnmerkleized, + ) -> Result { + Ok(TestMerkleized) + } + } + + /// Fatal storage observed during a proposal takes the actor down instead + /// of masking a broken database behind an ordinary decline. + #[test] + #[should_panic(expected = "application proposal failed")] + fn fatal_proposal_panics_processing() { + deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { + let mut signing = context.child("signing"); + let scheme = + scheme_mocks::fixture(&mut signing, b"fatal-propose", 1).schemes[0].clone(); + let marshal = fixtures::marshal_fixture( + context.child("marshal_fixture"), + "fatal-propose", + scheme, + None, + NZUsize!(1), + false, + ) + .await; + let processor = Processor::new( + FatalProposeApp, + test_databases(), + anchor(0, 0), + StatefulMetrics::new(&context), + None, + ); + let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); + let mut mailbox = Mailbox::new(sender); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); + let processing = Processing { + context: ContextCell::new(context.child("processing")), + mailbox: receiver, + provider: (), + marshal: marshal.mailbox, + snapshot_publisher: publisher, + skip_finalized_until: None, + }; + let _actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + + let genesis = TestBlock::new(0, 0); + let proposal_context = TestBlock::child(&genesis, 1).context(); + let _ = mailbox + .propose( + (context.child("propose"), proposal_context), + ancestry::from_iter([Arc::new(genesis)]), + (), + ) + .await; + + // The actor panics while handling the proposal. + loop { + context.sleep(Duration::from_millis(100)).await; + } + }); + } + #[test] fn skip_finalized_block_skips_through_target_height() { let mut skip_until = Some(Height::new(3)); diff --git a/glue/src/stateful/actor/core/syncing.rs b/glue/src/stateful/actor/core/syncing.rs index d2dc5446795..0b06c4efc6b 100644 --- a/glue/src/stateful/actor/core/syncing.rs +++ b/glue/src/stateful/actor/core/syncing.rs @@ -1,3 +1,12 @@ +//! The state-sync phase of the stateful actor. +//! +//! Finalized blocks are retained with their acknowledgements while a sync runs. +//! Each time marshal's ack window fills, the newest retained block becomes the +//! live sync target and the whole window is acknowledged once the sync engines +//! observe that target. When sync completes, retained blocks are classified +//! against the artifact's anchor, applied durably, and the loop hands off to +//! [`Processing`]. + use crate::stateful::{ Application, actor::{ @@ -79,13 +88,10 @@ where /// Verification requests deferred until state sync completes. pub(super) deferred_verifications: Vec>, - /// Open subscriptions to the synced databases. - pub(super) database_subscribers: Vec>, - /// The cached [`SyncResult`], populated when sync completes. pub(super) artifact: Option>, - /// Publishes the latest snapshots. + /// Publishes the latest snapshots for serving. pub(super) snapshot_publisher: Publisher>, /// Signals that the syncer has produced a usable artifact. @@ -115,11 +121,9 @@ where on_start => { self.deferred_verifications .retain(|request| !request.verification.is_cancelled()); - self.database_subscribers - .retain(|subscriber| !subscriber.is_closed()); }, on_stopped => { - debug!("processor received shutdown signal"); + debug!("syncing loop received shutdown signal"); }, Ok(artifact) = &mut self.sync_completed else { error!("syncer stopped before publishing state sync artifact"); @@ -132,7 +136,7 @@ where return; }, Some(message) = self.mailbox.recv() else { - debug!("mailbox closed, shutting down processor"); + debug!("mailbox closed, shutting down syncing loop"); break; } => match message { Message::Propose { @@ -185,13 +189,6 @@ where return; } } - Message::SubscribeDatabases { response } => { - self.database_subscribers - .retain(|subscriber| !subscriber.is_closed()); - if !response.is_closed() { - self.database_subscribers.push(response); - } - } }, } } @@ -245,15 +242,33 @@ where /// Record `block` as the live sync target. async fn update_target(mut self, block: &Arc) -> (Self, bool) { - let artifact = select! { + let outcome = select! { _ = self.context.stopped() => return (self, false), - artifact = self.syncer.update_targets( + outcome = self.syncer.update_targets( Anchor::from(block.as_ref()), A::sync_targets(block.as_ref()), - ) => artifact, + ) => outcome, }; - if let Some(artifact) = artifact { - self.artifact = Some(artifact); + match outcome { + syncer::UpdateOutcome::Observed => {} + syncer::UpdateOutcome::SyncCompleted => { + // The syncer sent the artifact on the completion channel. Collect it + // here so the pending finalizations are handed off under the newest + // recorded target. + let artifact = select! { + _ = self.context.stopped() => return (self, false), + artifact = &mut self.sync_completed => match artifact { + Ok(artifact) => artifact, + Err(_) => { + // The consumed receiver must not be polled again. Leave a + // dead one so the loop's completion arm logs and stops. + self.sync_completed = oneshot::channel().1; + return (self, false); + } + }, + }; + self.artifact = Some(artifact); + } } (self, true) } @@ -307,27 +322,29 @@ where /// Transitions to [`Processing`] state once the database set has converged /// on the state sync [`Anchor`]. - async fn transition( - mut self, - handoffs: impl IntoIterator>>, - ) { - let artifact = self.artifact.take().expect("transition must have artifact"); - let mut completed_height = artifact.anchor.height; - - let _ = self.metrics.sync_done.try_set(1); - let mut processor = Processor::new( - self.application, - artifact.databases, - artifact.anchor, - self.metrics, - self.pruning, - ); - - // Serving must not wait for the next finalization, so the synced state - // alone publishes first. - processor - .publish_snapshot(&mut self.snapshot_publisher) - .await; + async fn transition(self, handoffs: impl IntoIterator>>) { + let Self { + context, + mailbox, + application, + provider, + marshal, + sync_metadata, + syncer: _, + deferred_verifications, + artifact, + mut snapshot_publisher, + sync_completed: _, + pending_finalizations: _, + pruning, + metrics, + } = self; + let SyncResult { databases, anchor } = artifact.expect("transition must have artifact"); + let mut completed_height = anchor.height; + + let _ = metrics.sync_done.try_set(1); + let mut processor = Processor::new(application, databases, anchor, metrics, pruning); + processor = processor.publish_snapshot(&mut snapshot_publisher).await; let mut pending_prune = None; @@ -336,19 +353,20 @@ where FinalizedHandoff::Covered(block, acknowledgement) | FinalizedHandoff::Reflected(block, acknowledgement) => { processor - .notify_finalized(self.context.as_present(), block.as_ref()) + .notify_finalized(context.as_present(), block.as_ref()) .await; acknowledgement.acknowledge(); } FinalizedHandoff::Apply(block, acknowledgement) => { + let applied; + (processor, applied) = processor + .finalize(context.as_present(), block.as_ref()) + .await; let Applied { snapshots, barrier, prune, - } = processor - .finalize(self.context.as_present(), block.as_ref()) - .await - .expect("sync handoff block cannot be a duplicate"); + } = applied.expect("sync handoff block cannot be a duplicate"); // The processing loop's flush pool does not exist yet, so observe the // deferred flush inline. Keep state-sync metadata in progress until every @@ -356,7 +374,7 @@ where if !barrier.durable().await { return; } - self.snapshot_publisher.publish(block.height(), snapshots); + snapshot_publisher.publish(block.height(), snapshots); acknowledgement.acknowledge(); pending_prune = prune.or(pending_prune); completed_height = block.height(); @@ -369,33 +387,28 @@ where } // Every applied handoff is durable, so completion can advance through the last one before - // pruning or exposing the databases to other actors. - self.sync_metadata = self.sync_metadata.set_complete(completed_height).await; + // pruning. + let _ = sync_metadata.set_complete(completed_height).await; + // Defensive only. The handoff applies at most a full ack window, one short of + // what the prune cadence needs, so this fires only in tests that feed + // more handoffs than the window holds. if let Some(prune) = pending_prune { - prune.run(processor.databases(), &self.marshal).await; + processor = processor.prune(prune, &marshal).await; // The published snapshots were captured before this prune. Republish // so serving stops pinning the pruned state. Every handoff barrier // was awaited above, so the republished state is already durable. - processor - .publish_snapshot(&mut self.snapshot_publisher) - .await; - } - - for subscriber in self.database_subscribers.drain(..) { - subscriber.send_lossy(processor.databases().clone()); + processor = processor.publish_snapshot(&mut snapshot_publisher).await; } Processing { - context: self.context, - mailbox: self.mailbox, - provider: self.provider, - marshal: self.marshal, - processor, - snapshot_publisher: self.snapshot_publisher, - deferred_verifications: self.deferred_verifications, + context, + mailbox, + provider, + marshal, + snapshot_publisher, skip_finalized_until: Some(completed_height), } - .start() + .start(processor, deferred_verifications) .await } } @@ -412,7 +425,7 @@ mod tests { processor::Pruning, syncer::{self, StateSyncMetadata, SyncResult}, }, - db::{Anchor, Publisher, Shared}, + db::{Anchor, Publisher, Single, Subscriber}, tests::{ fixtures::{self, MarshalFixture}, mocks::{ @@ -451,6 +464,7 @@ mod tests { E: rand_core::Rng + commonware_runtime::Spawner + commonware_storage::Context, { syncing: Syncing, + subscriber: Subscriber, } impl TestHarness { @@ -499,6 +513,7 @@ mod tests { } let (acknowledgement, mut newest_waiter) = Exact::handle(); + let subscriber = self.subscriber.clone(); let process = context.child("full_window").spawn(move |_| { self.syncing .process_finalized(Arc::new(TestBlock::new(10, 12)), acknowledgement) @@ -512,7 +527,7 @@ mod tests { assert!(poll!(waiter).is_pending()); } assert!(poll!(&mut newest_waiter).is_pending()); - assert!(response.send(None).is_ok()); + assert!(response.send(syncer::UpdateOutcome::Observed).is_ok()); for waiter in &mut waiters { assert!(poll!(waiter).is_pending()); } @@ -529,7 +544,10 @@ mod tests { } assert!(newest_waiter.await.is_ok()); assert!(syncing.pending_finalizations.is_empty()); - Self { syncing } + Self { + syncing, + subscriber, + } } } @@ -552,6 +570,7 @@ mod tests { let (syncer_sender, syncer_receiver) = actor_mailbox::new(syncing_context.child("syncer_mailbox"), NZUsize!(1)); let (sync_complete, sync_completed) = oneshot::channel(); + let (snapshot_publisher, snapshot_subscriber) = Publisher::new(&syncing_context); let harness = Self { syncing: Syncing { @@ -563,14 +582,14 @@ mod tests { sync_metadata: StateSyncMetadata::init(&syncing_context, "syncing-test").await, syncer: syncer::Mailbox::new(syncer_sender), deferred_verifications: Vec::new(), - database_subscribers: Vec::new(), artifact: None, - snapshot_publisher: Publisher::new(&syncing_context).0, + snapshot_publisher, sync_completed, pending_finalizations: VecDeque::new(), pruning: None, metrics: StatefulMetrics::new(&context), }, + subscriber: snapshot_subscriber, }; ( harness, @@ -606,6 +625,7 @@ mod tests { let (syncer_sender, _syncer_receiver) = actor_mailbox::new(context.child("syncer_mailbox"), NZUsize!(1)); let (_sync_complete, sync_completed) = oneshot::channel(); + let (snapshot_publisher, snapshot_subscriber) = Publisher::new(&syncing_context); Self { syncing: Syncing { @@ -617,17 +637,17 @@ mod tests { sync_metadata: StateSyncMetadata::init(&syncing_context, "syncing-test").await, syncer: syncer::Mailbox::new(syncer_sender), deferred_verifications: Vec::new(), - database_subscribers: Vec::new(), artifact: Some(SyncResult { databases: test_databases(), anchor, }), - snapshot_publisher: Publisher::new(&syncing_context).0, + snapshot_publisher, sync_completed, pending_finalizations: VecDeque::new(), pruning: None, metrics: StatefulMetrics::new(&context), }, + subscriber: snapshot_subscriber, } } } @@ -731,7 +751,7 @@ mod tests { .artifact .as_mut() .expect("harness must contain a sync artifact") - .databases = Shared::new("test", TestDb::gated(control.clone())); + .databases = Single::from(TestDb::gated(control.clone())); // Completion metadata must not be written until the handoff batch is durable. pending.arm(); @@ -739,6 +759,7 @@ mod tests { let (reflected_acknowledgement, mut reflected_waiter) = Exact::handle(); let (first_acknowledgement, mut first_waiter) = Exact::handle(); let (second_acknowledgement, mut second_waiter) = Exact::handle(); + let subscriber = harness.subscriber.clone(); let transition = context.child("transition").spawn(move |_| { harness.syncing.transition([ FinalizedHandoff::Reflected( @@ -764,6 +785,11 @@ mod tests { 0, "completion metadata must not be written before the handoff is durable", ); + assert_eq!( + subscriber.latest(), + Some(0), + "the synced state must serve first, before any handoff flush", + ); let first_flush = control.flushes.lock().remove(0); first_flush .send(Ok(())) @@ -773,6 +799,11 @@ mod tests { } assert!(poll!(&mut first_waiter).is_ready()); assert!(poll!(&mut second_waiter).is_pending()); + assert_eq!( + subscriber.latest(), + Some(1), + "each handoff block's snapshots must serve once its flush is durable", + ); let second_flush = control.flushes.lock().remove(0); second_flush .send(Ok(())) @@ -803,10 +834,9 @@ mod tests { fn aborted_handoff_flush_cancels_ack_and_keeps_sync_incomplete() { deterministic::Runner::default().start(|context| async move { let mut harness = TestHarness::new(context.child("harness"), anchor(7, 9)).await; - let databases = Shared::new( - "test", - TestDb::with_finalize(Handle::ready(Err(RuntimeError::Aborted))), - ); + let databases = Single::from(TestDb::with_finalize(Handle::ready(Err( + RuntimeError::Aborted, + )))); harness .syncing .artifact @@ -827,6 +857,11 @@ mod tests { waiter.await.is_err(), "an aborted handoff must cancel marshal's acknowledgement", ); + assert!( + harness.subscriber.latest().is_none(), + "an aborted handoff must never serve, and the subscriber must decline \ + once the writer is gone", + ); let reopened = StateSyncMetadata::<_, TestScheme, Sha256Digest>::init(&context, "syncing-test") .await; @@ -853,7 +888,7 @@ mod tests { true, ) .await; - let (harness, _mailbox, mut syncer_receiver, _sync_complete) = + let (harness, _mailbox, mut syncer_receiver, sync_complete) = TestHarness::new_syncing(context.child("harness"), marshal).await; let (acknowledgement, mut waiter) = Exact::handle(); @@ -869,12 +904,16 @@ mod tests { }; assert!(poll!(&mut waiter).is_pending()); assert!( - response - .send(Some(SyncResult { + sync_complete + .send(SyncResult { databases: test_databases(), anchor: anchor(7, 9), - })) + }) .is_ok(), + "completion receiver must be alive", + ); + assert!( + response.send(syncer::UpdateOutcome::SyncCompleted).is_ok(), "target update must still await its response", ); drop(update); diff --git a/glue/src/stateful/actor/core/verifications.rs b/glue/src/stateful/actor/core/verifications.rs index b551e242a39..10aeac10d48 100644 --- a/glue/src/stateful/actor/core/verifications.rs +++ b/glue/src/stateful/actor/core/verifications.rs @@ -2,7 +2,7 @@ use crate::stateful::{ Application, actor::{ core::mailbox::Verification, - processor::{Disposition, PendingDigest, VerificationProgress, Verifier}, + processor::{VerificationResult, Verifier}, }, }; use commonware_consensus::marshal::{ @@ -12,13 +12,13 @@ use commonware_consensus::marshal::{ use commonware_cryptography::certificate::Scheme; use commonware_macros::select; use commonware_runtime::{Clock, Metrics, Spawner}; -use commonware_utils::{channel::oneshot, futures::Pool}; +use commonware_utils::futures::Pool; use futures::FutureExt as _; use rand_core::Rng; -use std::{collections::BTreeMap, future::Future}; +use std::future::Future; use tracing::{Instrument as _, Span, info_span}; -/// A verification request that can be deferred or retried. +/// A verification request handed to a job. pub(super) struct Request where E: Rng + Spawner + Metrics + Clock, @@ -30,122 +30,71 @@ where pub(super) verification: Verification, } -enum JobResult -where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - Finished { - id: u64, - request: Request, - valid: Option, - }, - Invalidated { - id: u64, - request: Request, - }, -} - -impl JobResult -where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - const fn id(&self) -> u64 { - match self { - Self::Finished { id, .. } | Self::Invalidated { id, .. } => *id, - } - } -} - -struct JobControl { - invalidation: Option>, - progress: VerificationProgress, -} - -/// Owns independently-polled verification requests and their cancellation handles. -pub(super) struct Handler -where - E: Rng + Spawner + Metrics + Clock, - A: Application, - S: Scheme, - V: Variant, -{ +/// Owns independently-polled verification jobs. +/// +/// A job runs to its own conclusion. The only thing that ends one early is its +/// caller leaving. +pub(super) struct Handler { marshal: MarshalMailbox, - jobs: Pool>, - controls: BTreeMap>>, - next_id: u64, + jobs: Pool<(Verification, VerificationResult)>, } -impl Handler +impl Handler where - E: Rng + Spawner + Metrics + Clock, - A: Application, - S: Scheme, - V: Variant, - MarshalMailbox: BlockProvider, + S: Scheme + 'static, + V: Variant + 'static, { pub(super) fn new(marshal: MarshalMailbox) -> Self { Self { marshal, jobs: Pool::default(), - controls: BTreeMap::new(), - next_id: 0, } } - pub(super) fn schedule(&mut self, mut verifier: Verifier, mut request: Request) { - let id = self.next_id; - self.next_id = self - .next_id - .checked_add(1) - .expect("verification request ID overflowed"); - let (invalidate, invalidated) = oneshot::channel(); - let progress = VerificationProgress::default(); - assert!( - self.controls - .insert( - id, - JobControl { - invalidation: Some(invalidate), - progress: progress.clone(), - }, - ) - .is_none() - ); - + /// Starts verifying a request as a job the actor loop polls alongside its + /// own work. + pub(super) fn schedule( + &mut self, + mut verifier: Verifier, + mut request: Request, + ) where + E: Rng + Spawner + Metrics + Clock + 'static, + A: Application + 'static, + V: Variant, + MarshalMailbox: BlockProvider, + { let marshal = self.marshal.clone(); let process = info_span!(parent: &request.span, "stateful.actor.verify"); self.jobs.push( async move { - let ancestry = request.ancestry.clone(); - select! { - _ = invalidated => JobResult::Invalidated { id, request }, - valid = verifier.run( + let outcome = verifier + .run( &request.context.0, marshal, - request.context.1.clone(), - ancestry, - &progress, + request.context.1, + request.ancestry, &mut request.verification, - ) => JobResult::Finished { id, request, valid }, - } + ) + .await; + (request.verification, outcome) } .instrument(process), ); } + /// Answers every job that has already finished, without waiting. pub(super) fn complete_ready(&mut self) { while let Some(result) = self.jobs.next_completed().now_or_never() { - self.handle(result); + Self::respond(result); } } pub(super) async fn next_completed(&mut self) { let result = self.jobs.next_completed().await; - self.handle(result); + Self::respond(result); } + /// Runs `operation` while still answering verification jobs. pub(super) async fn drive(&mut self, operation: impl Future) -> T { futures::pin_mut!(operation); loop { @@ -156,75 +105,10 @@ where } } - /// Cancels every active attempt and waits for verification work to stop. - /// - /// Pruning uses this full barrier because it can remove history needed by - /// every branch. Live requests are returned for rescheduling afterward. - pub(super) async fn quiesce(&mut self) -> Vec> { - let (retry, reject) = self.quiesce_where(|_| Disposition::Retry).await; - assert!(reject.is_empty()); - retry - } - - pub(super) async fn quiesce_where( - &mut self, - disposition: impl Fn(&VerificationProgress>) -> Disposition, - ) -> (Vec>, Vec) { - let mut pending = BTreeMap::new(); - for (&id, control) in &mut self.controls { - let disposition = disposition(&control.progress); - if disposition == Disposition::Retain { - continue; - } - assert!(control.invalidation.take().is_some()); - assert!(pending.insert(id, disposition).is_none()); - } - - let mut retry = Vec::with_capacity(pending.len()); - let mut reject = Vec::with_capacity(pending.len()); - while !pending.is_empty() { - let result = self.jobs.next_completed().await; - let id = result.id(); - let Some(disposition) = pending.remove(&id) else { - self.handle(result); - continue; - }; - let control = self - .controls - .remove(&id) - .expect("completed verification must have an invalidation handle"); - assert!(control.invalidation.is_none()); - let request = match result { - JobResult::Finished { request, .. } | JobResult::Invalidated { request, .. } => { - request - } - }; - match disposition { - Disposition::Retain => { - unreachable!("retained verification cannot be invalidated") - } - Disposition::Retry => { - if !request.verification.is_cancelled() { - retry.push(request); - } - } - Disposition::Reject => reject.push(request.verification), - } - } - (retry, reject) - } - - fn handle(&mut self, result: JobResult) { - let control = self - .controls - .remove(&result.id()) - .expect("completed verification must have an invalidation handle"); - assert!(control.invalidation.is_some()); - let JobResult::Finished { request, valid, .. } = result else { - panic!("verification cannot finish through the actor loop after invalidation"); - }; - if let Some(valid) = valid { - request.verification.respond(valid); + fn respond((verification, outcome): (Verification, VerificationResult)) { + match outcome { + VerificationResult::Decided(valid) => verification.respond(valid), + VerificationResult::Cancelled => {} } } } diff --git a/glue/src/stateful/actor/processor/mod.rs b/glue/src/stateful/actor/processor/mod.rs index 9ea5a4d1b77..73de9cc4874 100644 --- a/glue/src/stateful/actor/processor/mod.rs +++ b/glue/src/stateful/actor/processor/mod.rs @@ -1,8 +1,7 @@ //! Speculative execution engine for the [`Stateful`](super::Stateful) actor. //! //! The [`Processor`] owns the in-memory pending-tip DAG and the applied -//! database set. It is the workhorse behind the actor's `Processing` mode, -//! handling three operations: +//! database set, and does the work behind the actor's `Processing` mode. //! //! - Propose/Verify: fork unmerkleized batches from a parent's pending //! state (or from applied state), delegate to the [`Application`], and @@ -14,18 +13,21 @@ //! inserting each intermediate result into the pending map. //! //! - Finalization: apply the winning fork's merkleized batches to the -//! databases and start flushing them (durability is reported via -//! [`Barrier`]), retaining only pending descendants of the finalized +//! databases, start flushing them (durability is reported via [`Barrier`]), +//! capture a snapshot of the database set for publication (returned in +//! [`Applied`]), then retain only pending descendants of the finalized //! winner. //! -//! Verification jobs are polled independently and scoped to their callers. -//! Verification-owned lazy recovery shares [`Application::apply`] by block -//! digest. Proposal recovery remains actor-owned. +//! - Maintenance -- [`Processor::prune`] runs due prunes and +//! [`Processor::publish_snapshot`] publishes fresh snapshots afterwards. use crate::stateful::{ - Application, Input, Proposed, PruneConfig, + Application, ExecutionError, Input, Proposed, PruneConfig, actor::{core::Verification, metrics::Metrics as StatefulMetrics}, - db::{Anchor, Barrier, DatabaseSet, Publisher, SnapshotsOf}, + db::{ + Anchor, Barrier, DatabaseSet, MerkleizedOf, Publisher, ReadersOf, SnapshotsOf, + SyncTargetsOf, UnmerkleizedOf, + }, }; use commonware_consensus::{ Block, CertifiableBlock, Heightable, Roundable, @@ -51,19 +53,17 @@ use rand_core::Rng; use std::{ collections::{BTreeMap, HashSet, VecDeque}, future::Future, - hash::Hash, sync::Arc, }; -use tracing::{debug, warn}; +use tracing::{Instrument as _, debug, info_span, warn}; mod verifier; pub(super) use verifier::Verifier; pub(super) type PendingDigest = <>::Block as Digestible>::Digest; -type PendingBatches = <>::Databases as DatabaseSet>::Merkleized; +type PendingBatches = MerkleizedOf<>::Databases, E>; type PendingMap = BTreeMap, PendingEntry>; -pub(super) type PendingSyncTargets = - <>::Databases as DatabaseSet>::SyncTargets; +pub(super) type PendingSyncTargets = SyncTargetsOf<>::Databases, E>; type DeferredPrune = Option>; type ReplayResult = Result<(), PrepareBatchesError>; type ReplayWaiterSlots = Vec>>; @@ -79,111 +79,12 @@ struct ReplayFlight { vacant_slots: Vec, } -/// Last observed phase used to classify live verification across finalization. -#[derive(Clone, Copy)] -enum VerificationPhase { - Acquiring, - Replaying { digest: D, parent: D, round: Round }, - Verifying { digest: D, parent: D, round: Round }, -} - -/// How tracked verification work crosses an incoming finalization. -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -pub(super) enum Disposition { - /// Continue polling work proven to descend from the finalized block. - Retain, - /// Re-evaluate work whose branch is unknown or whose active phase cannot cross finalization. - Retry, - /// Return false for work already proven to use an incompatible parent. - Reject, -} - -/// Progress needed to decide whether an active verification remains valid -/// across an incoming finalization. -#[derive(Clone)] -pub(super) struct VerificationProgress(Arc>>); - -impl Default for VerificationProgress { - fn default() -> Self { - Self(Arc::new(Mutex::new(VerificationPhase::Acquiring))) - } -} - -impl VerificationProgress { - fn replaying(&self, digest: D, parent: D, round: Round) { - *self.0.lock() = VerificationPhase::Replaying { - digest, - parent, - round, - }; - } - - fn verifying(&self, digest: D, parent: D, round: Round) { - *self.0.lock() = VerificationPhase::Verifying { - digest, - parent, - round, - }; - } - - fn phase(&self) -> VerificationPhase { - *self.0.lock() - } -} - -/// Pending parents whose branch-scoped batches remain valid after one -/// finalized block is applied. -/// -/// Marshal delivers finalized blocks in height order. Canonical older blocks -/// are therefore already covered by the processed anchor and resolve while -/// their verification is still acquiring. The exact processed phase is the -/// only older replay or verification phase that can survive from the preceding -/// finalization. -pub(super) struct FinalizationBoundary { - digest: D, - round: Round, - processed_digest: D, - processed_round: Round, - compatible: HashSet, -} - -impl FinalizationBoundary { - pub(super) fn disposition(&self, progress: &VerificationProgress) -> Disposition { - match progress.phase() { - VerificationPhase::Acquiring => Disposition::Retry, - VerificationPhase::Replaying { - digest, - parent, - round, - } => { - if digest == self.processed_digest && round == self.processed_round { - return Disposition::Retry; - } - match digest == self.digest { - true => Disposition::Retain, - false if round > self.round && self.compatible.contains(&parent) => { - Disposition::Retain - } - false => Disposition::Reject, - } - } - VerificationPhase::Verifying { - digest, - parent, - round, - } => { - if digest == self.digest - || (digest == self.processed_digest && round == self.processed_round) - { - Disposition::Retry - } else if round > self.round && self.compatible.contains(&parent) { - Disposition::Retain - } else { - Disposition::Reject - } - } - } - } +/// What one verification attempt concluded. +pub(in crate::stateful::actor) enum VerificationResult { + /// A verdict to return to the caller. + Decided(bool), + /// The caller left, so there is nothing left to answer. + Cancelled, } /// Cached speculative state for a block digest. @@ -198,10 +99,6 @@ where } /// Speculative state shared by independently-polled verification jobs. -/// -/// During finalization, the winning batch remains available as a branch parent -/// while a clone is applied. `finalizing_compatible` admits late state only -/// when its recorded parent already belongs to that branch. struct ExecutionState where E: Rng + Spawner + Metrics + Clock, @@ -211,12 +108,13 @@ where pending: PendingMap, /// Latest canonical anchor whose finalization hook has completed. last_processed: Anchor>, - /// Winner currently being applied, if finalization is active. - finalizing: Option>>, - /// Winner batch retained while a clone is applied to the databases. - finalizing_batch: Option>, - /// Winner and descendants allowed in pending state during finalization. - finalizing_compatible: HashSet>, + /// Set from the start of a finalization's apply phase (before its replay + /// and database mutations) until the anchor move. Forks from the anchor + /// during that window could take post-apply state under the pre-apply + /// anchor, so they refuse. + finalizing: bool, + /// Woken when the anchor moves and the finalizing window closes. + anchor_waiters: Vec>, } /// Returns the winner and pending descendants whose state survives finalization. @@ -258,13 +156,16 @@ where compatible } -/// Shared execution inputs and actor-owned speculative state. +/// Read capability and speculative state, shared by every verification job. +/// +/// Deliberately carries no authority to mutate the database set. Jobs holding +/// one are therefore `'static` and can outlive any number of applies. struct Execution where E: Rng + Spawner + Metrics + Clock, A: Application, { - databases: A::Databases, + readers: ReadersOf, state: Arc>>, metrics: StatefulMetrics, } @@ -276,7 +177,7 @@ where { fn clone(&self) -> Self { Self { - databases: self.databases.clone(), + readers: self.readers.clone(), state: self.state.clone(), metrics: self.metrics.clone(), } @@ -291,12 +192,6 @@ struct ReplayFlights { entries: ReplayRegistry, } -#[derive(Clone, Copy)] -struct ReplayTracking<'a, D: Copy + Ord> { - flights: &'a ReplayFlights, - progress: &'a VerificationProgress, -} - impl Default for ReplayFlights { fn default() -> Self { Self { @@ -306,6 +201,8 @@ impl Default for ReplayFlights { } impl ReplayFlights { + /// Whether no replay is in flight. + #[cfg(test)] fn is_empty(&self) -> bool { self.entries.lock().is_empty() } @@ -372,13 +269,6 @@ enum ReplayClaim { Wait(ReplayWaiter), } -/// Claim on the finalizing winner's batch construction. -enum FinalizationClaim { - Cached, - Wait(ReplayWaiter), - Reconstruct(ReplayOwner), -} - /// Registration that removes its own waiter slot when dropped. struct ReplayWaiter { flights: ReplayFlights, @@ -458,6 +348,9 @@ enum PrepareBatchesError { Incomplete, /// The attempt was cancelled while waiting. Cancelled, + /// A competing finalization landed mid-preparation. The caller re-checks + /// against the new canonical state. + Stale, } /// Provides a cancellation signal for speculative actor work. @@ -497,23 +390,6 @@ pub(super) struct Prune { qmdb_target: T, } -impl Prune { - /// Run database and marshal pruning. - /// - /// Every finalize barrier through `barrier_height` is durable before this runs. The marshal - /// prune that follows retains every later block a restart could replay. - pub(super) async fn run(self, databases: &DBs, marshal: &MarshalMailbox) - where - E: Rng + Spawner + Metrics + Clock, - DBs: DatabaseSet, - S: Scheme, - V: MarshalVariant, - { - databases.prune(&self.qmdb_target).await; - marshal.prune(self.marshal_height); - } -} - /// Tracks the configured prune cadence and finalized sync targets needed to /// make pruning safe. pub(super) struct Pruning { @@ -614,6 +490,7 @@ where A: Application, { app: A, + databases: A::Databases, execution: Execution, replays: ReplayFlights>, pruning: Option>>, @@ -636,16 +513,16 @@ where Self { app, execution: Execution { - databases, + readers: databases.readers(), state: Arc::new(Mutex::new(ExecutionState { pending: BTreeMap::new(), last_processed, - finalizing: None, - finalizing_batch: None, - finalizing_compatible: HashSet::new(), + finalizing: false, + anchor_waiters: Vec::new(), })), metrics, }, + databases, replays: ReplayFlights::default(), pruning, } @@ -660,49 +537,57 @@ where } } - /// Returns whether every verification-owned replay has released its owner. - pub(super) fn replays_idle(&self) -> bool { - self.replays.is_empty() - } - - /// Snapshot the pending bases that remain branch-valid after `block` is - /// finalized. - pub(super) fn finalization_boundary( - &self, - block: &A::Block, - ) -> FinalizationBoundary> { - let digest = block.digest(); - let round = block.context().round(); - let state = self.execution.state.lock(); - FinalizationBoundary { - digest, - round, - processed_digest: state.last_processed.digest, - processed_round: state.last_processed.round, - compatible: compatible_pending(&state, digest, round), - } + /// The height of the last finalized block applied to the databases. + pub(super) fn processed_height(&self) -> Height { + self.execution.last_processed().height } - /// Returns a reference to the database set. - pub(super) const fn databases(&self) -> &A::Databases { - &self.execution.databases + /// Prune `self.databases` and `marshal` to the `prune` target. + /// + /// # Invariant + /// + /// Databases must be durable through `prune.barrier_height`. + pub(super) async fn prune( + mut self, + prune: Prune>, + marshal: &MarshalMailbox, + ) -> Self + where + S: Scheme, + V: MarshalVariant, + { + self.databases = self.databases.prune(&prune.qmdb_target).await; + marshal.prune(prune.marshal_height); + self } /// Capture a snapshot of the database set's applied state and publish it /// at the processed height. - /// - /// Returns a future that captures a clone of the set rather than `&self`, - /// so it stays `Send` without requiring `Application: Sync`. - pub(super) fn publish_snapshot<'p>( + pub(super) async fn publish_snapshot( + mut self, + publisher: &mut Publisher>, + ) -> Self { + let snapshots; + (self.databases, snapshots) = self.databases.snapshot().await; + publisher.publish(self.processed_height(), snapshots); + self + } + + #[cfg(test)] + fn readers(&self) -> ReadersOf { + self.execution.readers.clone() + } + + #[cfg(test)] + fn cache_pending( &self, - publisher: &'p mut Publisher>, - ) -> impl Future + Send + 'p { - let databases = self.execution.databases.clone(); - let height = self.execution.last_processed().height; - async move { - let snapshots = databases.snapshot().await; - publisher.publish(height, snapshots); - } + digest: PendingDigest, + parent: PendingDigest, + round: Round, + merkleized: PendingBatches, + ) -> bool { + self.execution + .cache_pending(digest, parent, round, merkleized) } #[cfg(test)] @@ -721,132 +606,12 @@ where self.execution.update_pending_metric(); } - /// Prepare parent-relative batches and delegate to the application to - /// build a new block proposal. The resulting block and its merkleized - /// state are cached in `pending`. Sends `None` on `response` if the - /// ancestry is invalid or the application declines to propose. - pub(super) async fn propose( - &mut self, - context: &E, - marshal: MarshalMailbox, - (runtime_context, consensus_context): (E, A::Context), - mut ancestry: impl Ancestry, - input: Input, - mut response: oneshot::Sender>, - ) where - S: Scheme, - V: MarshalVariant, - MarshalMailbox: BlockProvider, - { - let timer = self.execution.metrics.propose_duration.timer(context); - - let parent = match fetch_ancestor(&mut response, &mut ancestry).await { - Some(Some(parent)) => parent, - Some(None) => { - response.send_lossy(None); - return; - } - None => { - debug!("proposal request cancelled before initial ancestry arrived"); - return; - } - }; - let parent_digest = parent.digest(); - let ancestry = marshal_ancestry::with_prefix([Arc::clone(&parent)], ancestry); - - let round = consensus_context.round(); - let batches = match self - .prepare_batches(context, marshal, parent, &mut response) - .await - { - Ok(batches) => batches, - Err(PrepareBatchesError::Invalid) => { - response.send_lossy(None); - return; - } - Err(PrepareBatchesError::Incomplete) => { - debug!( - ?parent_digest, - "proposal request waiting on incomplete ancestry during prepare_batches" - ); - response.closed().await; - return; - } - Err(PrepareBatchesError::Cancelled) => { - debug!( - ?parent_digest, - "proposal request cancelled during prepare_batches" - ); - return; - } - }; - - let proposed = match await_or_cancel( - &mut response, - self.app.propose( - (runtime_context, consensus_context), - ancestry, - batches, - input, - ), - ) - .await - { - Some(result) => result, - None => { - debug!(?parent_digest, "proposal request cancelled during propose"); - return; - } - }; - - let Some(Proposed { block, merkleized }) = proposed else { - response.send_lossy(None); - return; - }; - assert!( - A::Databases::matches_sync_targets(&merkleized, &A::sync_targets(&block)), - "proposed state must match block commitments", - ); - assert!( - self.cache_pending(block.digest(), parent_digest, round, merkleized), - "proposal parent must remain compatible until the proposal completes", - ); - self.execution.update_pending_metric(); - timer.observe(context); - response.send_lossy(Some(block)); - } - - /// Ensure parent state exists, then prepare unmerkleized batches for execution. - #[tracing::instrument( - name = "stateful.processor.prepare_batches", - level = "info", - skip_all, - fields(parent = %parent.digest()) - )] - async fn prepare_batches( - &mut self, - context: &E, - marshal: MarshalMailbox, - parent: Arc, - cancellation: &mut C, - ) -> Result<>::Unmerkleized, PrepareBatchesError> - where - S: Scheme, - V: MarshalVariant, - MarshalMailbox: BlockProvider, - C: Cancellation, - { - self.execution - .prepare_batches(&mut self.app, context, marshal, parent, cancellation, None) - .await - } - /// Fork unmerkleized batches from known parent state. #[cfg(test)] async fn fork_batches( &self, parent: &::Digest, - ) -> Result<>::Unmerkleized, PrepareBatchesError> { + ) -> Result, PrepareBatchesError> { self.execution.fork_batches(parent).await } @@ -870,15 +635,22 @@ where /// Apply finalized state, start persisting it, and prune dead in-memory forks. /// - /// Returns [`None`] when the block was already processed (a duplicate - /// report). + /// Returns the processor, the snapshot to publish, the barrier proving that + /// snapshot durable, and any prune now due. [`None`] means the block was + /// already applied, which happens when marshal reports it twice. + /// + /// The block's state comes from its verification when that is cached, and is + /// replayed here otherwise. Verification jobs keep running throughout, so + /// this leaves the block reachable as a parent until the anchor moves. pub(super) async fn finalize( - &mut self, + mut self, context: &E, block: &A::Block, - ) -> Option, SnapshotsOf>> { - let finalized = Anchor::from(block); - let (height, digest) = (finalized.height, finalized.digest); + ) -> ( + Self, + Option, SnapshotsOf>>, + ) { + let (height, digest) = (block.height(), block.digest()); let last_processed = self.execution.last_processed(); if height < last_processed.height { panic!( @@ -892,169 +664,233 @@ where digest, last_processed.digest, "received conflicting finalized block at processed height", ); - return None; + return (self, None); } let timer = self.execution.metrics.finalize_duration.timer(context); let block_context = block.context(); + let round = block_context.round(); let sync_targets = A::sync_targets(block); - self.execution.begin_finalization(finalized); - // Marshal finalization is ordered. If the winner is not cached, - // reconstruct its batch from the current finalized database state. + // Marshal finalization is ordered. A pending miss means we can replay + // this block on top of finalized state. // - // Safety contract: reconstructed `Application::apply` output must - // match the block commitments previously enforced by `Application::verify`. - let reconstruction = loop { - match self - .execution - .claim_finalization_batch(&self.replays, digest) - { - FinalizationClaim::Cached => break None, - FinalizationClaim::Reconstruct(owner) => break Some(owner), - FinalizationClaim::Wait(mut waiter) => match (&mut waiter.completion).await { - Ok(Ok(())) | Err(_) => continue, - Ok(Err(error)) => { - warn!( - ?digest, - ?error, - "finalization could not reuse active verification replay" - ); - continue; - } - }, + // The entry stays in the pending map until the retention sweep below. + // Verification jobs run throughout this call, and one that forks from + // this block must find it rather than rebuild it on top of itself. + // + // Safety contract. Replayed `Application::apply` output must match the + // block commitments previously enforced by `Application::verify`. + // Every path from here must reach `advance_to_finalized` (which closes + // the window) or take the actor down, because a stranded window parks every + // later verification forever. + self.execution.state.lock().finalizing = true; + let batch = match self.execution.pending_batch(&digest) { + Some(merkleized) => merkleized, + None => { + let batches = A::Databases::new_batches(&self.execution.readers).await; + let batch = match self + .app + .apply( + (context.child("finalize_replay"), block_context), + block, + batches, + ) + .await + { + Ok(batch) => batch, + // Impossible on a correct node, since the batches were just forked + // from applied state, mutation authority is unique, and + // there is no caller to answer with a refusal. + Err(err) => panic!("finalize replay failed: {err}"), + }; + assert!( + A::Databases::matches_sync_targets(&batch, &sync_targets), + "finalize replay state root must match block commitments", + ); + batch } }; - let reconstructed = if reconstruction.is_none() { - None - } else { - let batches = self.execution.databases.new_batches().await; - let batch = self - .app - .apply( - (context.child("finalize_reconstruct"), block_context), - block, - batches, - ) - .await; - assert!( - A::Databases::matches_sync_targets(&batch, &sync_targets), - "finalize reconstruction must match block commitments", - ); - Some(batch) - }; - let batch = self - .execution - .secure_finalization_batch(digest, reconstructed); - if let Some(owner) = reconstruction { - owner.finish(Ok(())); - } - let (snapshots, barrier) = self.execution.databases.finalize(batch).await; + let (snapshots, barrier); + (self.databases, snapshots, barrier) = self.databases.finalize(batch).await; self.notify_finalized(context, block).await; let prune = self .pruning .as_mut() .and_then(|pruning| pruning.observe_finalized(height, sync_targets)); - self.execution.finish_finalization(finalized); + self.execution.advance_to_finalized(Anchor { + height, + round, + digest, + }); timer.observe(context); - Some(Applied { - snapshots, - barrier, - prune, - }) + ( + self, + Some(Applied { + snapshots, + barrier, + prune, + }), + ) } /// Notify the application that marshal delivered a finalized block already /// reflected in the database set. - pub(super) async fn notify_finalized(&mut self, context: &E, block: &A::Block) { - self.app - .finalized( + pub(super) fn notify_finalized( + &self, + context: &E, + block: &A::Block, + ) -> impl Future + Send { + let mut app = self.app.clone(); + let readers = self.execution.readers.clone(); + async move { + app.finalized( (context.child("finalized"), block.context()), block, - self.execution.databases.readers(), + readers, ) .await; + } } - /// Cache merkleized pending state for a block digest. - fn cache_pending( + /// Prepare parent-relative batches and delegate to the application to + /// build a new block proposal. The resulting block and its merkleized + /// state are cached in `pending`. Sends `None` on `response` if the + /// ancestry is invalid, the application declines to propose, or the + /// application errors. + pub(super) fn propose( &self, - digest: PendingDigest, - parent: PendingDigest, - round: Round, - merkleized: PendingBatches, - ) -> bool { - self.execution - .cache_pending(digest, parent, round, merkleized) - } -} + context: &E, + marshal: MarshalMailbox, + (runtime_context, consensus_context): (E, A::Context), + mut ancestry: impl Ancestry, + input: Input, + mut response: oneshot::Sender>, + ) -> impl Future + Send + where + S: Scheme, + V: MarshalVariant, + MarshalMailbox: BlockProvider, + { + let mut app = self.app.clone(); + let execution = &self.execution; + async move { + let timer = execution.metrics.propose_duration.timer(context); -impl Execution -where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - fn last_processed(&self) -> Anchor> { - self.state.lock().last_processed - } + let parent = match fetch_ancestor(&mut response, &mut ancestry).await { + Some(Some(parent)) => parent, + Some(None) => { + response.send_lossy(None); + return; + } + None => { + debug!("proposal request cancelled before initial ancestry arrived"); + return; + } + }; + let parent_digest = parent.digest(); + let prepare = info_span!( + "stateful.processor.prepare_batches", + parent = %parent_digest, + ); + let ancestry = marshal_ancestry::with_prefix([Arc::clone(&parent)], ancestry); - /// Records the compatible pending state for a serialized finalization. - fn begin_finalization(&self, anchor: Anchor>) { - let mut state = self.state.lock(); - let compatible = compatible_pending(&state, anchor.digest, anchor.round); - assert!( - state.finalizing.replace(anchor).is_none(), - "finalization must be serialized", - ); - assert!(state.finalizing_batch.is_none()); - assert!(state.finalizing_compatible.is_empty()); - state.finalizing_compatible = compatible; - } + let round = consensus_context.round(); + let batches = match execution + .prepare_batches(&mut app, context, marshal, parent, &mut response, None) + .instrument(prepare) + .await + { + Ok(batches) => batches, + Err(PrepareBatchesError::Invalid) => { + response.send_lossy(None); + return; + } + Err(PrepareBatchesError::Incomplete) => { + debug!( + ?parent_digest, + "proposal request waiting on incomplete ancestry during prepare_batches" + ); + response.closed().await; + return; + } + Err(PrepareBatchesError::Cancelled) => { + debug!( + ?parent_digest, + "proposal request cancelled during prepare_batches" + ); + return; + } + // Unreachable, since the actor admits no finalization while a + // proposal runs (it becomes the FIFO barrier), so nothing can + // go stale. Decline loudly rather than hide a broken barrier. + Err(PrepareBatchesError::Stale) => { + warn!(?parent_digest, "proposal went stale during prepare_batches"); + debug_assert!(false, "no finalization can interleave a proposal"); + response.send_lossy(None); + return; + } + }; - /// Retain the winner as a branch parent and discard incompatible state. - fn secure_finalization_batch( - &self, - digest: PendingDigest, - reconstructed: Option>, - ) -> PendingBatches { - let mut state = self.state.lock(); - assert_eq!( - state.finalizing.map(|anchor| anchor.digest), - Some(digest), - "secured batch must match active finalization", - ); - assert!(state.finalizing_batch.is_none()); - let ExecutionState { - pending, - finalizing_batch, - finalizing_compatible, - .. - } = &mut *state; - let batch = pending - .remove(&digest) - .map(|entry| entry.merkleized) - .or(reconstructed) - .expect("finalization must have a cached or reconstructed batch"); - *finalizing_batch = Some(batch.clone()); - let before = pending.len(); - pending.retain(|candidate_digest, _| finalizing_compatible.contains(candidate_digest)); - let pruned = before - pending.len(); - let pending = pending.len(); - drop(state); - self.metrics.pruned_forks.inc_by(pruned as u64); - let _ = self.metrics.pending_blocks.try_set(pending); - batch + let proposed = match await_or_cancel( + &mut response, + app.propose( + (runtime_context, consensus_context), + ancestry, + batches, + input, + ), + ) + .await + { + Some(Ok(result)) => result, + Some(Err(err @ ExecutionError::Fatal(_))) => { + panic!("application proposal failed: {err}") + } + Some(Err(err)) => { + // Stale is unreachable for the same reason as above. + warn!(?parent_digest, ?err, "proposal declined by error"); + debug_assert!( + !matches!(err, ExecutionError::Stale), + "no finalization can interleave a proposal", + ); + response.send_lossy(None); + return; + } + None => { + debug!(?parent_digest, "proposal request cancelled during propose"); + return; + } + }; + + let Some(Proposed { block, merkleized }) = proposed else { + response.send_lossy(None); + return; + }; + assert!( + A::Databases::matches_sync_targets(&merkleized, &A::sync_targets(&block)), + "proposed state must match block commitments", + ); + assert!( + execution.cache_pending(block.digest(), parent_digest, round, merkleized), + "proposal parent must remain compatible until the proposal completes", + ); + execution.update_pending_metric(); + timer.observe(context); + response.send_lossy(Some(block)); + } } +} - /// Publish the finalized anchor after its application hook completes. - fn finish_finalization(&self, finalized: Anchor>) { - let mut state = self.state.lock(); - assert_eq!(state.finalizing.take(), Some(finalized)); - assert!(state.finalizing_batch.take().is_some()); - state.finalizing_compatible.clear(); - state.last_processed = finalized; +impl Execution +where + E: Rng + Spawner + Metrics + Clock, + A: Application, +{ + fn last_processed(&self) -> Anchor> { + self.state.lock().last_processed } fn summary(&self) -> (Anchor>, usize) { @@ -1088,35 +924,17 @@ where return true; } - // A replay can finish after another copy supplied the batch consumed by finalization. - // Treat it as cached without reinserting the finalized batch. - let winner_already_retained = state.finalizing.is_some_and(|finalizing| { - state.finalizing_batch.is_some() - && finalizing.digest == digest - && finalizing.round == round - }); - if winner_already_retained - || (state.last_processed.digest == digest && state.last_processed.round == round) - { + // A replay of the block the anchor now sits on is already reflected in + // applied state. + if state.last_processed.digest == digest && state.last_processed.round == round { return true; } - // The processed anchor is not advanced until the application hook returns. Retained - // descendants may cache state during that interval, but new work on the old anchor may not. - let compatible = state.finalizing.map_or_else( - || { - round > state.last_processed.round - && (parent == state.last_processed.digest - || state.pending.contains_key(&parent)) - }, - |finalizing| { - (digest == finalizing.digest - && round == finalizing.round - && (parent == state.last_processed.digest - || state.pending.contains_key(&parent))) - || (round > finalizing.round && state.finalizing_compatible.contains(&parent)) - }, - ); + // Verification runs across finalizations, so a verdict can land against + // a newer anchor and pending set than the one it started from. This is + // where such a result is refused because its branch is no longer reachable. + let compatible = round > state.last_processed.round + && (parent == state.last_processed.digest || state.pending.contains_key(&parent)); if !compatible { return false; } @@ -1128,34 +946,9 @@ where merkleized, }, ); - if state.finalizing.is_some() { - state.finalizing_compatible.insert(digest); - } true } - /// Finds, joins, or reserves construction of the finalizing winner batch. - /// - /// Replay completion only tells the caller to check the cache again: the - /// owner caches the batch before notifying its waiters. Execution state is - /// always locked before the replay registry when both are inspected. - fn claim_finalization_batch( - &self, - replays: &ReplayFlights>, - digest: PendingDigest, - ) -> FinalizationClaim> { - let state = self.state.lock(); - let mut entries = replays.entries.lock(); - if let Some(flight) = entries.get_mut(&digest) { - return FinalizationClaim::Wait(replays.waiter(digest, flight)); - } - if state.pending.contains_key(&digest) { - FinalizationClaim::Cached - } else { - FinalizationClaim::Reconstruct(replays.register_owner(digest, &mut entries)) - } - } - /// Reuses known state, joins an active replay, or claims replay ownership. /// /// The state-before-registry lock order prevents a replay registration from @@ -1166,13 +959,7 @@ where digest: PendingDigest, ) -> ReplayClaim> { let state = self.state.lock(); - if state.last_processed.digest == digest - || state.pending.contains_key(&digest) - || (state.finalizing_batch.is_some() - && state - .finalizing - .is_some_and(|finalizing| finalizing.digest == digest)) - { + if state.last_processed.digest == digest || state.pending.contains_key(&digest) { return ReplayClaim::Ready; } @@ -1184,47 +971,61 @@ where ReplayClaim::Owner(replays.register_owner(digest, &mut entries)) } - /// Whether finalization supersedes a winner replay's terminal failure. - fn finalization_recovers_replay(&self, digest: PendingDigest) -> bool { - let state = self.state.lock(); - state.last_processed.digest == digest - || state - .finalizing - .is_some_and(|finalizing| finalizing.digest == digest) - } - /// Forks batches from a known parent. async fn fork_batches( &self, parent: &PendingDigest, - ) -> Result<>::Unmerkleized, PrepareBatchesError> { + ) -> Result, PrepareBatchesError> { { let state = self.state.lock(); if let Some(entry) = state.pending.get(parent) { return Ok(A::Databases::fork_batches(&entry.merkleized)); } - if state - .finalizing - .is_some_and(|finalizing| finalizing.digest == *parent) - { - let batch = state - .finalizing_batch - .as_ref() - .ok_or(PrepareBatchesError::Invalid)?; - return Ok(A::Databases::fork_batches(batch)); - } if state.last_processed.digest != *parent { return Err(PrepareBatchesError::Invalid); } } - Ok(self.databases.new_batches().await) + let batches = A::Databases::new_batches(&self.readers).await; + + // A finalization mutates the databases before the anchor moves, so a + // fork taken meanwhile can hold post-apply state under the pre-apply + // anchor. Refuse whenever one overlapped this fork -- either its window + // is still open, or its anchor move already landed. The caller waits + // out the window and re-checks canonical state. + let state = self.state.lock(); + if state.finalizing || state.last_processed.digest != *parent { + return Err(PrepareBatchesError::Stale); + } + drop(state); + Ok(batches) + } + + /// Wait until no finalization is mid-flight and the anchor differs from `seen`. + /// + /// Returns immediately when that already holds. Used by stale verification + /// attempts, whose staleness proves a finalization opened its window, and that + /// finalization either moves the anchor or takes the actor down, so parking + /// here (instead of retrying immediately) cannot outlive it. + async fn anchor_past(&self, seen: &Anchor>) { + loop { + let waiter = { + let mut state = self.state.lock(); + if !state.finalizing && state.last_processed.digest != seen.digest { + return; + } + let (sender, receiver) = oneshot::channel(); + state.anchor_waiters.push(sender); + receiver + }; + let _ = waiter.await; + } } /// Replays one certified block and caches its commitment-matching state. /// - /// Cancellation caches nothing. A commitment mismatch or state that cannot - /// be cached across active finalization makes the ancestry invalid. + /// Cancellation caches nothing. A commitment mismatch, or state that the + /// applied anchor has already moved past, makes the ancestry invalid. async fn replay_block( &self, app: &mut A, @@ -1240,13 +1041,9 @@ where let consensus_context = block.context(); let round = consensus_context.round(); - let Some(batches) = await_or_cancel(cancellation, self.fork_batches(&parent_digest)).await - else { - return Err(PrepareBatchesError::Cancelled); - }; - let batches = batches?; + let batches = self.fork_batches(&parent_digest).await?; - let Some(merkleized) = await_or_cancel( + let Some(applied) = await_or_cancel( cancellation, app.apply( (context.child("rebuild_pending_apply"), consensus_context), @@ -1258,6 +1055,13 @@ where else { return Err(PrepareBatchesError::Cancelled); }; + let merkleized = match applied { + Ok(merkleized) => merkleized, + // A block finalized while this replay executed. The requester + // re-checks canonical state, so this replay never panics on it. + Err(ExecutionError::Stale) => return Err(PrepareBatchesError::Stale), + Err(err @ ExecutionError::Fatal(_)) => panic!("application replay failed: {err}"), + }; if !A::Databases::matches_sync_targets(&merkleized, &A::sync_targets(&block)) { warn!( @@ -1285,17 +1089,16 @@ where target_digest: PendingDigest, block: Arc, cancellation: &mut C, - replay: ReplayTracking<'_, PendingDigest>, + replays: &ReplayFlights>, ) -> ReplayResult where C: Cancellation, { - let (digest, parent, round) = (block.digest(), block.parent(), block.context().round()); + let digest = block.digest(); loop { - match self.claim_replay(replay.flights, digest) { + match self.claim_replay(replays, digest) { ReplayClaim::Ready => return Ok(()), ReplayClaim::Owner(owner) => { - replay.progress.replaying(digest, parent, round); let result = self .replay_block( app, @@ -1311,7 +1114,6 @@ where return result; } ReplayClaim::Wait(mut waiter) => { - replay.progress.replaying(digest, parent, round); let Some(completion) = await_or_cancel(cancellation, &mut waiter.completion).await else { @@ -1319,12 +1121,7 @@ where }; match completion { Ok(Ok(())) | Err(_) => continue, - Ok(Err(error)) => { - if self.finalization_recovers_replay(digest) { - continue; - } - return Err(error); - } + Ok(Err(error)) => return Err(error), } } } @@ -1333,7 +1130,7 @@ where /// Ensures parent state exists and forks batches for speculative execution. /// - /// Verification supplies replay tracking to share reconstruction by block + /// Verification supplies the replay registry to share reconstruction by block /// digest, while proposals reconstruct independently. `fork_batches` /// revalidates the parent after reconstruction in case finalization /// advanced meanwhile. @@ -1344,7 +1141,7 @@ where marshal: MarshalMailbox, parent: Arc, cancellation: &mut C, - replay: Option>>, + replays: Option<&ReplayFlights>>, ) -> Result<>::Unmerkleized, PrepareBatchesError> where S: Scheme, @@ -1359,13 +1156,11 @@ where || state.pending.contains_key(&parent_digest) }; if !known { - self.rebuild_pending(app, context, marshal, parent, cancellation, replay) + self.rebuild_pending(app, context, marshal, parent, cancellation, replays) .await?; } - await_or_cancel(cancellation, self.fork_batches(&parent_digest)) - .await - .ok_or(PrepareBatchesError::Cancelled)? + self.fork_batches(&parent_digest).await } /// Rebuilds missing ancestry through `target`. @@ -1380,7 +1175,7 @@ where provider: P, target: Arc, cancellation: &mut C, - replay: Option>>, + replays: Option<&ReplayFlights>>, ) -> Result<(), PrepareBatchesError> where P: BlockProvider + Clone, @@ -1451,8 +1246,8 @@ where let depth = replay_path.len(); for block in replay_path.into_iter().rev() { - if let Some(replay) = replay { - self.replay_block_shared(app, context, target_digest, block, cancellation, replay) + if let Some(replays) = replays { + self.replay_block_shared(app, context, target_digest, block, cancellation, replays) .await?; } else { self.replay_block(app, context, target_digest, block, cancellation) @@ -1465,6 +1260,42 @@ where timer.observe(context); Ok(()) } + + /// Take the cached merkleized batch for `digest`, if it was executed. + fn pending_batch(&self, digest: &PendingDigest) -> Option> { + self.state + .lock() + .pending + .get(digest) + .map(|entry| entry.merkleized.clone()) + } + + /// Move the anchor to a finalized block and drop the pending state that + /// block invalidates. A pending block survives only when it descends from + /// the anchor and was created after its round. + /// + /// The sweep, the anchor move, and the finalizing-window close happen under + /// one lock. Verification jobs read this state while a block applies, and a + /// job that saw the pending set already swept but the anchor not yet moved + /// would reject work that is still valid. + fn advance_to_finalized(&self, anchor: Anchor>) { + let mut state = self.state.lock(); + let compatible = compatible_pending(&state, anchor.digest, anchor.round); + let before = state.pending.len(); + state + .pending + .retain(|digest, entry| entry.round > anchor.round && compatible.contains(digest)); + let pruned = before - state.pending.len(); + let remaining = state.pending.len(); + state.last_processed = anchor; + state.finalizing = false; + for waiter in state.anchor_waiters.drain(..) { + waiter.send_lossy(()); + } + drop(state); + self.metrics.pruned_forks.inc_by(pruned as u64); + let _ = self.metrics.pending_blocks.try_set(remaining); + } } /// Returns true when `block` is already covered by applied state. @@ -1539,13 +1370,16 @@ where #[cfg(test)] mod tests { use super::{ - Applied, Disposition, FinalizationBoundary, PrepareBatchesError, Processor, Prune, Pruning, - ReplayClaim, ReplayFlights, ReplayTracking, VerificationProgress, fetch_ancestor, + Applied, PrepareBatchesError, Processor, Prune, Pruning, ReplayClaim, ReplayFlights, + fetch_ancestor, }; use crate::stateful::{ - Application, Input, Proposed, PruneConfig, + Application, ExecutionError, Input, Proposed, PruneConfig, actor::metrics::Metrics as StatefulMetrics, - db::{Anchor, DatabaseSet, Merkleized as _, Shared, Unmerkleized as _}, + db::{ + Anchor, DatabaseSet, Merkleized as _, MerkleizedOf, ReadersOf, Single, SyncTargetsOf, + UnmerkleizedOf, + }, }; use commonware_codec::{Encode, EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; use commonware_consensus::{ @@ -1560,8 +1394,7 @@ mod tests { use commonware_macros::{boxed, select}; use commonware_parallel::Sequential; use commonware_runtime::{ - Clock as _, ContextCell, Runner as _, Supervisor as _, buffer::paged::CacheRef, - deterministic, + ContextCell, Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, }; use commonware_storage::{ journal::contiguous::fixed::Config as FixedLogConfig, @@ -1574,7 +1407,7 @@ mod tests { }; use futures::StreamExt; use std::{ - collections::{BTreeMap, HashSet, VecDeque}, + collections::{BTreeMap, VecDeque}, future::Future, num::NonZeroUsize, sync::{ @@ -1592,41 +1425,11 @@ mod tests { type Qmdb = any::unordered::fixed::Db; - type DbSet = Shared>; + type DbSet = Single>; type TestMerkleized = as DatabaseSet>::Merkleized; - - #[test] - fn finalization_dispositions_preserve_winner_and_descendant_work() { - let boundary = FinalizationBoundary { - digest: 10, - round: Round::new(Epoch::zero(), View::new(10)), - processed_digest: 9, - processed_round: Round::new(Epoch::zero(), View::new(9)), - compatible: HashSet::from([10, 11]), - }; - let progress = VerificationProgress::default(); - assert_eq!(boundary.disposition(&progress), Disposition::Retry,); - - progress.replaying(9, 8, Round::new(Epoch::zero(), View::new(9))); - assert_eq!(boundary.disposition(&progress), Disposition::Retry,); - - progress.replaying(10, 1, Round::new(Epoch::zero(), View::new(10))); - assert_eq!(boundary.disposition(&progress), Disposition::Retain,); - progress.replaying(12, 11, Round::new(Epoch::zero(), View::new(11))); - assert_eq!(boundary.disposition(&progress), Disposition::Retain,); - progress.replaying(20, 19, Round::new(Epoch::zero(), View::new(11))); - assert_eq!(boundary.disposition(&progress), Disposition::Reject,); - - progress.verifying(9, 8, Round::new(Epoch::zero(), View::new(9))); - assert_eq!(boundary.disposition(&progress), Disposition::Retry,); - progress.verifying(10, 1, Round::new(Epoch::zero(), View::new(10))); - assert_eq!(boundary.disposition(&progress), Disposition::Retry,); - progress.verifying(12, 11, Round::new(Epoch::zero(), View::new(11))); - assert_eq!(boundary.disposition(&progress), Disposition::Retain,); - progress.verifying(20, 19, Round::new(Epoch::zero(), View::new(11))); - assert_eq!(boundary.disposition(&progress), Disposition::Reject,); - } + type TestUnmerkleized = + as DatabaseSet>::Unmerkleized; #[derive(Clone, Debug, PartialEq, Eq)] struct Block { @@ -1784,10 +1587,6 @@ mod tests { gate.started.send(()).expect("test must await replay"); let _ = (&mut gate.release).await; } - - fn calls(&self) -> usize { - self.calls.load(Ordering::SeqCst) - } } fn apply_gate() -> (ApplyGate, oneshot::Receiver<()>, oneshot::Sender<()>) { @@ -1807,7 +1606,6 @@ mod tests { struct ExecutionApp { genesis: Block, finalized_observer: Option>>>, - apply_probe: Option, finalized_probe: Option, } @@ -1816,7 +1614,6 @@ mod tests { Self { genesis: Block::genesis(), finalized_observer: None, - apply_probe: None, finalized_probe: None, } } @@ -1827,7 +1624,6 @@ mod tests { Self { genesis: Block::genesis(), finalized_observer: Some(finalized_values.clone()), - apply_probe: None, finalized_probe: None, }, finalized_values, @@ -1837,17 +1633,18 @@ mod tests { async fn execute( height: Height, view: View, - mut batches: as DatabaseSet>::Unmerkleized, - ) -> as DatabaseSet>::Merkleized - { + mut batches: UnmerkleizedOf, deterministic::Context>, + ) -> Result< + MerkleizedOf, deterministic::Context>, + ExecutionError, + > { let current_counter = batches .get(&counter_key()) - .await - .expect("counter read should succeed") + .await? .map_or(0, |digest| digest_to_u64(&digest)); batches = batches.write(counter_key(), Some(u64_to_digest(current_counter + 1))); batches = batches.write(height_key(height), Some(u64_to_digest(view.get()))); - batches.merkleize().await.expect("merkleize should succeed") + Ok(crate::stateful::db::Unmerkleized::merkleize(batches).await?) } } @@ -1867,15 +1664,17 @@ mod tests { &mut self, context: (deterministic::Context, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { let mut ancestry = Box::pin(ancestry); - let parent = ancestry.next().await?; + let Some(parent) = ancestry.next().await else { + return Ok(None); + }; let context = context.1.clone(); let view = context.round.view(); let height = parent.height().next(); - let merkleized = Self::execute(height, view, batches).await; + let merkleized = Self::execute(height, view, batches).await?; let block = Block { context, parent: parent.digest(), @@ -1886,34 +1685,34 @@ mod tests { merkleized.bounds().tip.size ), }; - Some(Proposed { block, merkleized }) + Ok(Some(Proposed { block, merkleized })) } async fn verify( &mut self, _context: (deterministic::Context, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> + { let mut ancestry = Box::pin(ancestry); - let block = ancestry.next().await?; + let Some(block) = ancestry.next().await else { + return Ok(None); + }; let merkleized = - Self::execute(block.height(), block.context.round.view(), batches).await; + Self::execute(block.height(), block.context.round.view(), batches).await?; if merkleized.root() != block.state_root { - return None; + return Ok(None); } - Some(merkleized) + Ok(Some(merkleized)) } async fn apply( &mut self, _context: (deterministic::Context, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> >::Merkleized { - if let Some(probe) = &self.apply_probe { - probe.call(block.digest()).await; - } + batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { Self::execute(block.height(), block.context.round.view(), batches).await } @@ -1921,16 +1720,17 @@ mod tests { &mut self, _context: (deterministic::Context, Self::Context), block: &Self::Block, - readers: >::Readers, + readers: ReadersOf, ) { if let Some(probe) = &self.finalized_probe { probe.call(block.digest()).await; } - let Some(observer) = self.finalized_observer.clone() else { + let Some(observer) = &self.finalized_observer else { return; }; - let db = readers.read().await; - let value = db + let value = readers + .read() + .await .get(&height_key(block.height())) .await .expect("database read should succeed") @@ -1940,7 +1740,7 @@ mod tests { fn sync_targets( block: &Self::Block, - ) -> >::SyncTargets { + ) -> SyncTargetsOf { Target::new(block.state_root, block.range.clone()) } } @@ -2051,10 +1851,19 @@ mod tests { config: any::FixedConfig, app: ExecutionApp, ) -> Self { - let databases = as DatabaseSet< - deterministic::Context, - >>::init(context.child("db_set"), config.clone()) - .await; + Self::with_app_pruned(context, provider, config, app, None).await + } + + async fn with_app_pruned( + context: deterministic::Context, + provider: MapProvider, + config: any::FixedConfig, + app: ExecutionApp, + prune_config: Option, + ) -> Self { + let databases = + DbSet::::init(context.child("db_set"), config.clone()) + .await; let metrics = StatefulMetrics::new(&context); Self { context_cell: ContextCell::new(context), @@ -2067,7 +1876,7 @@ mod tests { digest: Block::genesis().digest(), }, metrics, - None, + prune_config.map(|config| Pruning::build(config, 1, 0)), ), provider, db_config: config, @@ -2082,7 +1891,7 @@ mod tests { .fork_batches(&parent.digest()) .await .expect("parent should be available"); - let merkleized = ExecutionApp::execute(height, view, batches).await; + let merkleized = ExecutionApp::execute(height, view, batches).await.unwrap(); let block = Block { context, parent: parent.digest(), @@ -2096,6 +1905,13 @@ mod tests { (block, merkleized) } + async fn fork_from(&self, parent: &Block) -> TestUnmerkleized { + self.processor + .fork_batches(&parent.digest()) + .await + .expect("parent must be forkable") + } + async fn stage_pending_child(&mut self, parent: &Block, view: View) -> Block { let (block, merkleized) = self.build_child(parent, view).await; let round = Round::new(Epoch::zero(), view); @@ -2113,51 +1929,54 @@ mod tests { /// Returns whether the block was newly applied (`false` for a /// duplicate report). #[boxed] - async fn finalize(&mut self, block: Block) -> bool { - let Some(Applied { barrier, .. }) = self + async fn finalize(mut self, block: Block) -> (Self, bool) { + let applied; + (self.processor, applied) = self .processor .finalize(self.context_cell.as_present(), &block) - .await - else { - return false; + .await; + let Some(Applied { barrier, .. }) = applied else { + return (self, false); }; assert!(barrier.durable().await, "finalize flush must complete"); - true + (self, true) } #[boxed] async fn finalize_with_prune( - &mut self, + mut self, block: Block, - ) -> Option< - Prune< - as DatabaseSet>::SyncTargets, - >, - > { - let Applied { - snapshots: _, - barrier, - prune, - } = self + ) -> ( + Self, + Option, deterministic::Context>>>, + ) { + let applied; + (self.processor, applied) = self .processor .finalize(self.context_cell.as_present(), &block) - .await - .expect("finalized block must apply"); + .await; + let Applied { barrier, prune, .. } = applied.expect("finalized block must apply"); assert!(barrier.durable().await, "finalize flush must complete"); - prune + (self, prune) } async fn height_value(&self, height: Height) -> Option { - let db = self.processor.databases().read().await; - db.get(&height_key(height)) + self.processor + .readers() + .read() + .await + .get(&height_key(height)) .await .expect("database read should succeed") .map(|value| digest_to_u64(&value)) } async fn counter_value(&self) -> Option { - let db = self.processor.databases().read().await; - db.get(&counter_key()) + self.processor + .readers() + .read() + .await + .get(&counter_key()) .await .expect("database read should succeed") .map(|value| digest_to_u64(&value)) @@ -2354,31 +2173,23 @@ mod tests { let provider = MapProvider::default(); let config = qmdb_config("db_config", &context); let app = ExecutionApp::new(); - let mut harness = Harness::with_app(context, provider, config, app).await; - harness.processor = Processor::new( - ExecutionApp::new(), - harness.processor.databases().clone(), - Anchor { - height: Height::zero(), - round: Block::genesis().context().round, - digest: Block::genesis().digest(), - }, - StatefulMetrics::new(harness.context_cell.as_present()), - Some(Pruning::build( - PruneConfig { - maintenance_interval: NZUsize!(1), - retained_marshal_blocks: 1, - retained_qmdb_blocks: 1, - }, - 1, - 0, - )), - ); + let mut harness = Harness::with_app_pruned( + context, + provider, + config, + app, + Some(PruneConfig { + maintenance_interval: NZUsize!(1), + retained_marshal_blocks: 1, + retained_qmdb_blocks: 1, + }), + ) + .await; let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - let prune = harness.finalize_with_prune(block1).await; + let (_, prune) = harness.finalize_with_prune(block1).await; assert_eq!( prune, None, "pruning should wait for the full retention window", @@ -2386,6 +2197,67 @@ mod tests { }); } + /// A fork taken from the anchor while a finalization is mid-flight (databases + /// applied, anchor not yet advanced) refuses instead of handing out the winner's + /// state under the loser's anchor. + #[test] + fn fork_refuses_inside_the_finalize_window() { + deterministic::Runner::default().start(|context| async move { + let mut harness = Harness::new(context).await; + let genesis = Block::genesis(); + let winner = harness.stage_pending_child(&genesis, View::new(1)).await; + + // Park the finalize between its database apply and its anchor move. + let (gate, started, release) = apply_gate(); + harness.processor.app.finalized_probe = + Some(ApplicationProbe::new(winner.digest(), [gate])); + let verifier = harness.processor.verifier(); + let processor = harness.processor; + let finalize = processor.finalize(harness.context_cell.as_present(), &winner); + futures::pin_mut!(finalize); + select! { + _ = &mut finalize => panic!("finalize must park on the probe"), + result = started => result.expect("finalize must reach the probe"), + } + + // Inside the window, a fork from the anchor must refuse, since the databases + // are already at the winner, but the anchor still names genesis. + assert!(matches!( + verifier.execution.fork_batches(&genesis.digest()).await, + Err(PrepareBatchesError::Stale) + )); + + release.send(()).expect("finalize is parked"); + let (processor, applied) = finalize.await; + assert!(applied.is_some()); + drop(processor); + }); + } + + /// A batch forked before a competing finalization refuses its next operation with + /// the typed stale error, end to end through the set wrapper, the database cell, + /// and the storage checks. + #[test] + fn stale_fork_refuses_through_the_set() { + deterministic::Runner::default().start(|context| async move { + let mut harness = Harness::new(context).await; + let genesis = Block::genesis(); + + // Fork from the applied anchor, then finalize a competing child. + let stale = harness.fork_from(&genesis).await; + let winner = harness.stage_pending_child(&genesis, View::new(1)).await; + let applied; + (harness, applied) = harness.finalize(winner).await; + assert!(applied); + + assert!(matches!( + ExecutionApp::execute(Height::new(1), View::new(1), stale).await, + Err(ExecutionError::Stale) + )); + drop(harness); + }); + } + #[test] fn execution_finalization_prunes_losing_fork() { deterministic::Runner::default().start(|context| async move { @@ -2398,10 +2270,9 @@ mod tests { assert!(harness.processor.pending_contains(&winner.digest())); assert!(harness.processor.pending_contains(&loser.digest())); - assert!( - harness.finalize(winner.clone()).await, - "finalization should persist winner state", - ); + let applied; + (harness, applied) = harness.finalize(winner.clone()).await; + assert!(applied, "finalization should persist winner state"); assert!( !harness.processor.pending_contains(&loser.digest()), "losing fork at finalized round should be pruned", @@ -2425,10 +2296,9 @@ mod tests { assert!(harness.processor.pending_contains(&loser.digest())); assert!(harness.processor.pending_contains(&loser_child.digest())); - assert!( - harness.finalize(winner.clone()).await, - "finalization should persist winner state", - ); + let applied; + (harness, applied) = harness.finalize(winner.clone()).await; + assert!(applied, "finalization should persist winner state"); assert!( !harness.processor.pending_contains(&loser.digest()), "losing fork at finalized round should be pruned", @@ -2440,615 +2310,82 @@ mod tests { }); } + /// A verification job holds an [`Execution`] and keeps running while a + /// block is applied. The block being finalized must stay reachable as a + /// parent for that whole window, otherwise a job that forks from it + /// mid-apply rebuilds it on top of itself and rejects a valid descendant. #[test] - fn execution_finalization_prunes_before_finalized_hook_completes() { - deterministic::Runner::default().start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - let loser = harness.stage_pending_child(&block1, View::new(2)).await; - let winner = harness.stage_pending_child(&block1, View::new(3)).await; - let winner_child = harness.stage_pending_child(&winner, View::new(4)).await; - let (gate, started, release) = apply_gate(); - harness.processor.app.finalized_probe = - Some(ApplicationProbe::new(winner.digest(), [gate])); - let execution = harness.processor.execution.clone(); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &winner), - ); - assert!(futures::poll!(&mut finalize).is_pending()); - started.await.expect("finalized hook should start"); - - assert!(execution.pending_contains(&winner_child.digest())); - assert!(!execution.pending_contains(&block1.digest())); - assert!(!execution.pending_contains(&loser.digest())); - - release - .send(()) - .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - }); - } - - #[test] - fn execution_forks_from_finalizing_winner_before_database_apply() { + fn finalized_block_stays_forkable_while_it_applies() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let mut harness = Harness::new(context).await; let genesis = Block::genesis(); let winner = harness.stage_pending_child(&genesis, View::new(1)).await; - let databases = harness.processor.databases().clone(); - let read = databases.read().await; - let execution = harness.processor.execution.clone(); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &winner), - ); - assert!(futures::poll!(&mut finalize).is_pending()); - - let winner_digest = winner.digest(); - let mut fork = Box::pin(execution.fork_batches(&winner_digest)); - let forked = match futures::poll!(&mut fork) { - std::task::Poll::Ready(forked) => forked, - std::task::Poll::Pending => { - panic!("finalizing winner should remain available for child batches") - } - }; - assert!(forked.is_ok(), "finalizing winner should remain forkable"); - - drop(read); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - }); - } - #[test] - fn execution_late_winner_publication_is_a_noop() { - deterministic::Runner::default().start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let view = View::new(1); - let round = Round::new(Epoch::zero(), view); - let (winner, initial_batch) = harness.build_child(&genesis, view).await; - let (_, during_finalization_batch) = harness.build_child(&genesis, view).await; - let (_, after_finalization_batch) = harness.build_child(&genesis, view).await; - assert!(harness.processor.cache_pending( - winner.digest(), - genesis.digest(), - round, - initial_batch, - )); + // A job's view of the world, taken before the apply starts. + let execution = harness.processor.execution.clone(); - let (gate, started, release) = apply_gate(); + // Park the finalization inside its `finalized` hook. The database + // apply is done, and the anchor has not moved yet. + let (gate, mut started, release) = apply_gate(); harness.processor.app.finalized_probe = Some(ApplicationProbe::new(winner.digest(), [gate])); - let execution = harness.processor.execution.clone(); let mut finalize = Box::pin( harness .processor .finalize(harness.context_cell.as_present(), &winner), ); - assert!(futures::poll!(&mut finalize).is_pending()); - started.await.expect("finalized hook should start"); - - assert!(execution.cache_pending( - winner.digest(), - genesis.digest(), - round, - during_finalization_batch, - )); - assert!(!execution.pending_contains(&winner.digest())); - - release - .send(()) - .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - - assert!(execution.cache_pending( - winner.digest(), - genesis.digest(), - round, - after_finalization_batch, - )); - assert!(!execution.pending_contains(&winner.digest())); - }); - } - - #[test] - fn execution_descendant_replay_survives_finalized_parent_removal() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let parent = harness.stage_pending_child(&genesis, View::new(1)).await; - let (child, _) = harness.build_child(&parent, View::new(2)).await; - - let (owner_gate, owner_started, mut owner_release) = apply_gate(); - let (retry_gate, retry_started, retry_release) = apply_gate(); - harness.processor.app.apply_probe = Some(ApplicationProbe::new( - child.digest(), - [owner_gate, retry_gate], - )); - let (finalized_gate, finalized_started, finalized_release) = apply_gate(); - harness.processor.app.finalized_probe = - Some(ApplicationProbe::new(parent.digest(), [finalized_gate])); - - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - let mut owner_app = harness.processor.app.clone(); - let mut waiter_app = harness.processor.app.clone(); - let replay_context = harness.context_cell.as_present(); - let owner_progress = VerificationProgress::default(); - let waiter_progress = VerificationProgress::default(); - let (mut owner_cancellation, owner_alive) = oneshot::channel::<()>(); - let (mut waiter_cancellation, _waiter_alive) = oneshot::channel::<()>(); - - let mut owner = Box::pin(execution.replay_block_shared( - &mut owner_app, - replay_context, - child.digest(), - Arc::new(child.clone()), - &mut owner_cancellation, - ReplayTracking { - flights: &replays, - progress: &owner_progress, - }, - )); - assert!(futures::poll!(&mut owner).is_pending()); - owner_started.await.expect("replay owner should start"); - - let mut waiter = Box::pin(execution.replay_block_shared( - &mut waiter_app, - replay_context, - child.digest(), - Arc::new(child.clone()), - &mut waiter_cancellation, - ReplayTracking { - flights: &replays, - progress: &waiter_progress, - }, - )); - assert!(futures::poll!(&mut waiter).is_pending()); - let boundary = harness.processor.finalization_boundary(&parent); - assert_eq!(boundary.disposition(&owner_progress), Disposition::Retain,); - assert_eq!(boundary.disposition(&waiter_progress), Disposition::Retain,); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &parent), - ); - assert!(futures::poll!(&mut finalize).is_pending()); - finalized_started - .await - .expect("finalized hook should start"); - - drop(owner_alive); - assert_eq!(owner.await, Err(PrepareBatchesError::Cancelled)); - owner_release.closed().await; - select! { - result = &mut waiter => { - panic!("retained replay failed after owner cancellation: {result:?}"); - }, - result = retry_started => { - result.expect("retained replay should restart from finalized state"); - }, + _ = &mut finalize => panic!("finalize completed before its hook returned"), + result = &mut started => result.expect("finalized hook should start"), } - retry_release - .send(()) - .expect("retried replay should remain active"); - assert_eq!(waiter.await, Ok(())); - - finalized_release - .send(()) - .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - }); - } - - #[test] - fn finalized_reader_preserves_retained_replay_base() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let (mut harness, finalized_values) = - Harness::new_with_finalized_observer(context).await; - let genesis = Block::genesis(); - let parent = harness.stage_pending_child(&genesis, View::new(1)).await; - let (child, _) = harness.build_child(&parent, View::new(2)).await; - - let (owner_gate, owner_started, mut owner_release) = apply_gate(); - let (retry_gate, retry_started, retry_release) = apply_gate(); - harness.processor.app.apply_probe = Some(ApplicationProbe::new( - child.digest(), - [owner_gate, retry_gate], - )); - let (finalized_gate, finalized_started, finalized_release) = apply_gate(); - harness.processor.app.finalized_probe = - Some(ApplicationProbe::new(parent.digest(), [finalized_gate])); - - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - let mut owner_app = harness.processor.app.clone(); - let mut waiter_app = harness.processor.app.clone(); - let replay_context = harness.context_cell.as_present(); - let owner_progress = VerificationProgress::default(); - let waiter_progress = VerificationProgress::default(); - let (mut owner_cancellation, owner_alive) = oneshot::channel::<()>(); - let (mut waiter_cancellation, _waiter_alive) = oneshot::channel::<()>(); - - let mut owner = Box::pin(execution.replay_block_shared( - &mut owner_app, - replay_context, - child.digest(), - Arc::new(child.clone()), - &mut owner_cancellation, - ReplayTracking { - flights: &replays, - progress: &owner_progress, - }, - )); - assert!(futures::poll!(&mut owner).is_pending()); - owner_started.await.expect("replay owner should start"); - - let mut waiter = Box::pin(execution.replay_block_shared( - &mut waiter_app, - replay_context, - child.digest(), - Arc::new(child), - &mut waiter_cancellation, - ReplayTracking { - flights: &replays, - progress: &waiter_progress, - }, - )); - assert!(futures::poll!(&mut waiter).is_pending()); - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &parent), + assert!( + execution.fork_batches(&winner.digest()).await.is_ok(), + "the applying block must stay forkable for jobs that are still running", ); - assert!(futures::poll!(&mut finalize).is_pending()); - finalized_started - .await - .expect("finalized hook should start"); - - drop(owner_alive); - assert_eq!(owner.await, Err(PrepareBatchesError::Cancelled)); - owner_release.closed().await; - select! { - result = &mut waiter => { - panic!("retained replay failed after owner cancellation: {result:?}"); - }, - result = retry_started => { - result.expect("retained replay should restart from finalized state"); - }, - } - finalized_release + release .send(()) .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - assert_eq!(finalized_values.lock().as_slice(), [1]); - - retry_release - .send(()) - .expect("retried replay should remain active"); - assert_eq!(waiter.await, Ok(())); - }); - } - - #[test] - fn execution_finalize_self_applies_after_cancelled_winner_replay() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let (winner, _) = harness.build_child(&genesis, View::new(1)).await; - - let (owner_gate, owner_started, mut owner_release) = apply_gate(); - let probe = ApplicationProbe::new(winner.digest(), [owner_gate]); - harness.processor.app.apply_probe = Some(probe.clone()); - - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - let mut owner_app = harness.processor.app.clone(); - let replay_context = harness.context_cell.as_present(); - let (mut owner_cancellation, owner_alive) = oneshot::channel::<()>(); - let owner_progress = VerificationProgress::default(); - - let mut owner = Box::pin(execution.replay_block_shared( - &mut owner_app, - replay_context, - winner.digest(), - Arc::new(winner.clone()), - &mut owner_cancellation, - ReplayTracking { - flights: &replays, - progress: &owner_progress, - }, - )); - assert!(futures::poll!(&mut owner).is_pending()); - owner_started.await.expect("winner replay should start"); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &winner), - ); - assert!( - futures::poll!(&mut finalize).is_pending(), - "finalization should wait on the active winner replay", - ); - - drop(owner_alive); - assert_eq!(owner.await, Err(PrepareBatchesError::Cancelled)); - owner_release.closed().await; - - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - assert_eq!( - probe.calls(), - 2, - "finalization must reconstruct the winner after the owner cancels", - ); - assert_eq!(harness.processor.last_processed().digest, winner.digest()); - assert!(harness.processor.replays_idle()); - }); - } - - #[test] - fn execution_replay_waiter_recovers_from_invalid_finalizing_winner() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let (winner, _) = harness.build_child(&genesis, View::new(1)).await; - let probe = ApplicationProbe::new(winner.digest(), std::iter::empty()); - harness.processor.app.apply_probe = Some(probe.clone()); - - let owner = match harness - .processor - .execution - .claim_replay(&harness.processor.replays, winner.digest()) - { - ReplayClaim::Owner(owner) => owner, - _ => panic!("winner replay claim should own the flight"), - }; - - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - let mut waiter_app = harness.processor.app.clone(); - let replay_context = harness.context_cell.as_present(); - let waiter_progress = VerificationProgress::default(); - let (mut waiter_cancellation, _waiter_alive) = oneshot::channel::<()>(); - let mut waiter = Box::pin(execution.replay_block_shared( - &mut waiter_app, - replay_context, - winner.digest(), - Arc::new(winner.clone()), - &mut waiter_cancellation, - ReplayTracking { - flights: &replays, - progress: &waiter_progress, - }, - )); - assert!(futures::poll!(&mut waiter).is_pending()); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &winner), - ); - assert!( - futures::poll!(&mut finalize).is_pending(), - "finalization should wait on the active winner replay", - ); - - owner.finish(Err(PrepareBatchesError::Invalid)); - assert_eq!( - waiter.await, - Ok(()), - "retained waiter should join recovery of the finalizing winner", - ); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); + let (_processor, applied) = finalize.await; + let Applied { barrier, .. } = applied.expect("finalized block should be newly applied"); assert!(barrier.durable().await, "finalize flush must complete"); - assert_eq!(probe.calls(), 1, "winner should be reconstructed once"); - assert_eq!(harness.processor.last_processed().digest, winner.digest()); - assert!(harness.processor.replays_idle()); }); } #[test] - fn execution_finalization_waits_for_active_cached_winner_replay() { + fn execution_finalize_awaits_its_finalized_hook() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let mut harness = Harness::new(context).await; let genesis = Block::genesis(); - let (winner, merkleized) = harness.build_child(&genesis, View::new(1)).await; - - let owner = match harness - .processor - .execution - .claim_replay(&harness.processor.replays, winner.digest()) - { - ReplayClaim::Owner(owner) => owner, - _ => panic!("winner replay should own the flight"), - }; - assert!(harness.processor.cache_pending( - winner.digest(), - genesis.digest(), - winner.context().round, - merkleized, - )); + let block = harness.stage_pending_child(&genesis, View::new(1)).await; let (gate, mut started, release) = apply_gate(); harness.processor.app.finalized_probe = - Some(ApplicationProbe::new(winner.digest(), [gate])); + Some(ApplicationProbe::new(block.digest(), [gate])); let mut finalize = Box::pin( harness .processor - .finalize(harness.context_cell.as_present(), &winner), + .finalize(harness.context_cell.as_present(), &block), ); - select! { _ = &mut finalize => { - panic!("finalization bypassed active winner replay"); + panic!("finalize completed before its finalized hook returned"); }, result = &mut started => { - result.expect("finalized hook should remain reachable"); - panic!("finalization reached the application hook before replay completed"); - }, - _ = harness.context_cell.as_present().sleep(Duration::from_millis(10)) => {}, - } - - owner.finish(Ok(())); - select! { - result = &mut started => { - result.expect("finalized hook should start after replay completes"); - }, - _ = &mut finalize => { - panic!("finalization completed before its application hook"); - }, - } - release - .send(()) - .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); - assert!(barrier.durable().await, "finalize flush must complete"); - }); - } - - #[test] - fn execution_replay_waiter_reuses_retained_finalizing_winner() { - deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { - let mut harness = Harness::new(context).await; - let genesis = Block::genesis(); - let (winner, _) = harness.build_child(&genesis, View::new(1)).await; - - let (replay_gate, replay_started, replay_release) = apply_gate(); - let probe = ApplicationProbe::new(winner.digest(), [replay_gate]); - harness.processor.app.apply_probe = Some(probe.clone()); - let (finalized_gate, finalized_started, finalized_release) = apply_gate(); - harness.processor.app.finalized_probe = - Some(ApplicationProbe::new(winner.digest(), [finalized_gate])); - - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - let mut owner_app = harness.processor.app.clone(); - let mut waiter_app = harness.processor.app.clone(); - let replay_context = harness.context_cell.as_present(); - let owner_progress = VerificationProgress::default(); - let waiter_progress = VerificationProgress::default(); - let (mut owner_cancellation, _owner_alive) = oneshot::channel::<()>(); - let (mut waiter_cancellation, _waiter_alive) = oneshot::channel::<()>(); - - let mut owner = Box::pin(execution.replay_block_shared( - &mut owner_app, - replay_context, - winner.digest(), - Arc::new(winner.clone()), - &mut owner_cancellation, - ReplayTracking { - flights: &replays, - progress: &owner_progress, - }, - )); - assert!(futures::poll!(&mut owner).is_pending()); - replay_started.await.expect("winner replay should start"); - - let mut waiter = Box::pin(execution.replay_block_shared( - &mut waiter_app, - replay_context, - winner.digest(), - Arc::new(winner.clone()), - &mut waiter_cancellation, - ReplayTracking { - flights: &replays, - progress: &waiter_progress, - }, - )); - assert!(futures::poll!(&mut waiter).is_pending()); - - let mut finalize = Box::pin( - harness - .processor - .finalize(harness.context_cell.as_present(), &winner), - ); - assert!(futures::poll!(&mut finalize).is_pending()); - - replay_release - .send(()) - .expect("winner replay should remain active"); - assert_eq!(owner.await, Ok(())); - select! { - result = finalized_started => { result.expect("finalized hook should start"); }, - _ = &mut finalize => { - panic!("finalization completed before its application hook"); - }, } - assert_eq!( - waiter.await, - Ok(()), - "waiter should reuse the retained winner batch", - ); - assert_eq!(probe.calls(), 1, "winner should be reconstructed once"); - - finalized_release + release .send(()) .expect("finalized hook should remain active"); - let Applied { barrier, .. } = finalize - .await - .expect("finalized block should be newly applied"); + let (_processor, applied) = finalize.await; + let Applied { barrier, .. } = applied.expect("finalized block should be newly applied"); assert!(barrier.durable().await, "finalize flush must complete"); }); } - #[test] - fn execution_finalization_reserves_missing_winner_reconstruction() { - deterministic::Runner::default().start(|context| async move { - let harness = Harness::new(context).await; - let genesis = Block::genesis(); - let (winner, _) = harness.build_child(&genesis, View::new(1)).await; - let execution = harness.processor.execution.clone(); - let replays = harness.processor.replays.clone(); - - execution.begin_finalization(Anchor::from(&winner)); - let finalization = execution.claim_finalization_batch(&replays, winner.digest()); - assert!( - matches!( - execution.claim_replay(&replays, winner.digest()), - ReplayClaim::Wait(_), - ), - "missing winner reconstruction must remain single-flight", - ); - drop(finalization); - }); - } - #[test] fn execution_rejects_late_losing_fork_publication() { deterministic::Runner::default().start(|context| async move { @@ -3060,7 +2397,9 @@ mod tests { let late_view = View::new(4); let (late_child, merkleized) = harness.build_child(&loser, late_view).await; - assert!(harness.finalize(winner).await); + let applied; + (harness, applied) = harness.finalize(winner).await; + assert!(applied); assert!( !harness.processor.cache_pending( late_child.digest(), @@ -3080,7 +2419,9 @@ mod tests { let mut harness = Harness::new(context).await; let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let block2 = harness.stage_pending_child(&block1, View::new(2)).await; let block3 = harness.stage_pending_child(&block2, View::new(3)).await; @@ -3269,203 +2610,6 @@ mod tests { }); } - #[test] - fn overlapping_rebuilds_share_replay_after_owner_cancellation() { - deterministic::Runner::timed(std::time::Duration::from_secs(5)).start( - |context| async move { - let mut harness = Harness::new(context.child("harness")).await; - let genesis = Block::genesis(); - let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - let block2 = harness.stage_pending_child(&block1, View::new(2)).await; - let block3 = harness.stage_pending_child(&block2, View::new(3)).await; - harness.processor.clear_pending(); - harness.provider.insert(genesis); - - let (first_gate, first_started, mut first_release) = apply_gate(); - let (retry_gate, retry_started, retry_release) = apply_gate(); - let (duplicate_gate, mut duplicate_started, _duplicate_release) = apply_gate(); - let probe = ApplicationProbe::new( - block1.digest(), - [first_gate, retry_gate, duplicate_gate], - ); - harness.processor.app.apply_probe = Some(probe.clone()); - - let first_execution = harness.processor.execution.clone(); - let second_execution = harness.processor.execution.clone(); - let third_execution = harness.processor.execution.clone(); - let fourth_execution = harness.processor.execution.clone(); - let mut first_app = harness.processor.app.clone(); - let mut second_app = harness.processor.app.clone(); - let mut third_app = harness.processor.app.clone(); - let mut fourth_app = harness.processor.app.clone(); - let first_provider = harness.provider.clone(); - let second_provider = harness.provider.clone(); - let third_provider = harness.provider.clone(); - let fourth_provider = harness.provider.clone(); - let first_replays = harness.processor.replays.clone(); - let second_replays = harness.processor.replays.clone(); - let third_replays = harness.processor.replays.clone(); - let fourth_replays = harness.processor.replays.clone(); - let (mut first_cancellation, first_live) = oneshot::channel::(); - let (mut second_cancellation, second_live) = oneshot::channel::(); - let (mut third_cancellation, third_live) = oneshot::channel::(); - let (mut fourth_cancellation, fourth_live) = oneshot::channel::(); - let first_progress = VerificationProgress::default(); - let second_progress = VerificationProgress::default(); - let third_progress = VerificationProgress::default(); - let fourth_progress = VerificationProgress::default(); - - let mut first = Box::pin(first_execution.rebuild_pending( - &mut first_app, - &context, - first_provider, - Arc::new(block2.clone()), - &mut first_cancellation, - Some(ReplayTracking { - flights: &first_replays, - progress: &first_progress, - }), - )); - select! { - result = &mut first => panic!("first rebuild completed before replay gate: {result:?}"), - result = first_started => result.expect("first replay should start"), - } - - let mut second = Box::pin(second_execution.rebuild_pending( - &mut second_app, - &context, - second_provider, - Arc::new(block2.clone()), - &mut second_cancellation, - Some(ReplayTracking { - flights: &second_replays, - progress: &second_progress, - }), - )); - let mut third = Box::pin(third_execution.rebuild_pending( - &mut third_app, - &context, - third_provider, - Arc::new(block3), - &mut third_cancellation, - Some(ReplayTracking { - flights: &third_replays, - progress: &third_progress, - }), - )); - let mut fourth = Box::pin(fourth_execution.rebuild_pending( - &mut fourth_app, - &context, - fourth_provider, - Arc::new(block2), - &mut fourth_cancellation, - Some(ReplayTracking { - flights: &fourth_replays, - progress: &fourth_progress, - }), - )); - let mut waiters_registered = false; - for _ in 0..100 { - waiters_registered = harness - .processor - .replays - .entries - .lock() - .get(&block1.digest()) - .is_some_and(|flight| flight.waiters.len() == 3); - if waiters_registered { - break; - } - select! { - result = &mut first => panic!("first rebuild completed before cancellation: {result:?}"), - result = &mut second => panic!("second rebuild completed before cancellation: {result:?}"), - result = &mut third => panic!("third rebuild completed before cancellation: {result:?}"), - result = &mut fourth => panic!("fourth rebuild completed before cancellation: {result:?}"), - result = &mut duplicate_started => { - result.expect("duplicate replay signal should remain available"); - panic!("overlapping rebuilds executed the same ancestor concurrently"); - }, - _ = context.sleep(std::time::Duration::from_millis(1)) => {}, - } - } - assert!( - waiters_registered, - "overlapping replays should wait for the current owner" - ); - assert_eq!(probe.calls(), 1); - - drop(fourth_live); - let fourth_result = select! { - result = &mut fourth => result, - result = &mut first => panic!("owner completed while cancelling waiter: {result:?}"), - result = &mut second => panic!("live waiter completed while cancelling peer: {result:?}"), - result = &mut third => panic!("live waiter completed while cancelling peer: {result:?}"), - _ = context.sleep(std::time::Duration::from_secs(1)) => { - panic!("cancelled replay waiter did not stop"); - }, - }; - assert_eq!(fourth_result, Err(PrepareBatchesError::Cancelled)); - assert!( - harness - .processor - .replays - .entries - .lock() - .get(&block1.digest()) - .is_some_and(|flight| { - flight.waiters.iter().flatten().count() == 2 - && flight.vacant_slots.len() == 1 - }), - "cancelled waiter must unregister while the owner remains active", - ); - - drop(first_live); - let first_result = select! { - result = &mut first => result, - result = &mut second => panic!("waiter completed before owner cancellation: {result:?}"), - result = &mut third => panic!("waiter completed before owner cancellation: {result:?}"), - _ = context.sleep(std::time::Duration::from_secs(1)) => { - panic!("cancelled replay owner did not stop"); - }, - }; - assert_eq!(first_result, Err(PrepareBatchesError::Cancelled)); - first_release.closed().await; - select! { - result = &mut second => panic!("waiter completed before retry gate: {result:?}"), - result = &mut third => panic!("waiter completed before retry gate: {result:?}"), - result = retry_started => result.expect("live waiter should acquire replay ownership"), - result = &mut duplicate_started => { - result.expect("duplicate replay signal should remain available"); - panic!("multiple waiters acquired replay ownership"); - }, - _ = context.sleep(std::time::Duration::from_secs(1)) => { - panic!("live waiter did not retry cancelled replay"); - }, - } - - retry_release - .send(()) - .expect("retried replay should still be running"); - let completed = futures::future::join(second, third); - let (second_result, third_result) = select! { - result = &mut duplicate_started => { - result.expect("duplicate replay signal should remain available"); - panic!("successful replay was not shared with every waiter"); - }, - result = completed => result, - _ = context.sleep(std::time::Duration::from_secs(1)) => { - panic!("waiting rebuilds did not complete"); - }, - }; - assert_eq!(second_result, Ok(())); - assert_eq!(third_result, Ok(())); - assert_eq!(probe.calls(), 2); - assert!(harness.processor.replays_idle()); - drop((second_live, third_live)); - }, - ); - } - #[test] fn execution_rebuild_pending_rejects_stale_ancestor_quickly() { deterministic::Runner::default().start(|context| async move { @@ -3476,7 +2620,9 @@ mod tests { let mut parent = genesis; for view in 1..=5 { let block = harness.stage_pending_child(&parent, View::new(view)).await; - assert!(harness.finalize(block.clone()).await); + let applied; + (harness, applied) = harness.finalize(block.clone()).await; + assert!(applied); parent = block.clone(); chain.push(block); } @@ -3517,7 +2663,9 @@ mod tests { let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let mut block2 = harness.stage_pending_child(&block1, View::new(2)).await; harness.processor.clear_pending(); @@ -3554,7 +2702,9 @@ mod tests { let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let gap_height = Height::new(3); let gap_view = View::new(3); @@ -3563,7 +2713,9 @@ mod tests { .fork_batches(&block1.digest()) .await .expect("processed anchor should be available"); - let merkleized = ExecutionApp::execute(gap_height, gap_view, batches).await; + let merkleized = ExecutionApp::execute(gap_height, gap_view, batches) + .await + .unwrap(); let gap_block = Block { context: consensus_context(block1.digest(), gap_view), parent: block1.digest(), @@ -3607,7 +2759,9 @@ mod tests { let mut harness = Harness::new(context.child("harness")).await; let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let block2 = harness.stage_pending_child(&block1, View::new(2)).await; let block3 = harness.stage_pending_child(&block2, View::new(3)).await; @@ -3645,7 +2799,9 @@ mod tests { let canonical = harness.stage_pending_child(&genesis, View::new(1)).await; let conflicting = harness.stage_pending_child(&genesis, View::new(2)).await; - assert!(harness.finalize(canonical).await); + let applied; + (harness, applied) = harness.finalize(canonical).await; + assert!(applied); let _ = harness.finalize(conflicting).await; }); @@ -3658,8 +2814,12 @@ mod tests { let genesis = Block::genesis(); let canonical = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(canonical.clone()).await); - assert!(!harness.finalize(canonical).await); + let applied; + (harness, applied) = harness.finalize(canonical.clone()).await; + assert!(applied); + let applied; + (harness, applied) = harness.finalize(canonical).await; + assert!(!applied); assert_eq!(harness.counter_value().await, Some(1)); }); } @@ -3671,7 +2831,9 @@ mod tests { let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1).await); + let applied; + (harness, applied) = harness.finalize(block1).await; + assert!(applied); assert_eq!(harness.counter_value().await, Some(1)); assert_eq!( harness @@ -3692,8 +2854,11 @@ mod tests { let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; let block2 = harness.stage_pending_child(&block1, View::new(2)).await; - assert!(harness.finalize(block1).await); - assert!(harness.finalize(block2).await); + let applied; + (harness, applied) = harness.finalize(block1).await; + assert!(applied); + let (_, applied) = harness.finalize(block2).await; + assert!(applied); assert_eq!( finalized_values.lock().clone(), vec![1, 2], @@ -3710,7 +2875,9 @@ mod tests { let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); finalized_values.lock().clear(); harness @@ -3726,8 +2893,8 @@ mod tests { } #[test] - #[should_panic(expected = "finalize reconstruction must match block commitments")] - fn execution_finalize_reconstruction_rejects_state_root_mismatch() { + #[should_panic(expected = "finalize replay state root must match block commitments")] + fn execution_finalize_replay_rejects_state_root_mismatch() { deterministic::Runner::default().start(|context| async move { let mut harness = Harness::new(context).await; let genesis = Block::genesis(); @@ -3756,7 +2923,9 @@ mod tests { let mut harness = Harness::new(context.child("harness")).await; let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let block2 = harness.stage_pending_child(&block1, View::new(2)).await; harness.processor.clear_pending(); @@ -3785,7 +2954,9 @@ mod tests { let mut harness = Harness::new(context.child("harness")).await; let genesis = Block::genesis(); let block1 = harness.stage_pending_child(&genesis, View::new(1)).await; - assert!(harness.finalize(block1.clone()).await); + let applied; + (harness, applied) = harness.finalize(block1.clone()).await; + assert!(applied); let block2 = harness.stage_pending_child(&block1, View::new(2)).await; harness.processor.clear_pending(); diff --git a/glue/src/stateful/actor/processor/verifier.rs b/glue/src/stateful/actor/processor/verifier.rs index 99f5d087b4f..76dc44f135c 100644 --- a/glue/src/stateful/actor/processor/verifier.rs +++ b/glue/src/stateful/actor/processor/verifier.rs @@ -1,8 +1,8 @@ use super::{ Application, Cancellation, Execution, PendingDigest, PrepareBatchesError, ReplayFlights, - ReplayTracking, VerificationProgress, await_or_cancel, fetch_ancestor, is_already_processed, + VerificationResult, await_or_cancel, fetch_ancestor, is_already_processed, }; -use crate::stateful::{actor::core::Verification, db::DatabaseSet}; +use crate::stateful::{ExecutionError, actor::core::Verification, db::DatabaseSet}; use commonware_consensus::{ Heightable, Roundable, marshal::{ @@ -37,6 +37,18 @@ enum PrepareFailure { Invalid, /// Preparation ended without a verdict because its request was cancelled. Cancelled, + /// A competing finalization landed mid-preparation. Re-check the candidate + /// against the new canonical state. + Stale, +} + +/// Outcome of one execution attempt of the candidate against applied state. +enum Attempt { + /// The attempt finished with a result. + Done(VerificationResult), + /// A competing finalization made the attempt's batches stale. Re-check the + /// candidate against the new canonical state and try again. + Stale, } /// A candidate's parent and forked batches, ready for application verification. @@ -54,6 +66,9 @@ where } /// Executes one independently-polled verification request. +/// +/// Carries only read capability and speculative state, so a job outlives any +/// number of applies. Its batch operations pause while one is running. pub(in crate::stateful::actor) struct Verifier where E: Rng + Spawner + Metrics + Clock, @@ -91,9 +106,8 @@ where marshal: MarshalMailbox, consensus_context: A::Context, ancestry: impl Ancestry, - progress: &VerificationProgress>, verification: &mut Verification, - ) -> Option + ) -> VerificationResult where S: Scheme, V: MarshalVariant, @@ -109,68 +123,127 @@ where Some(None) => { debug!("verification request waiting on incomplete block ancestry"); verification.cancelled().await; - return None; + return VerificationResult::Cancelled; } None => { debug!("verification request cancelled before initial block arrived"); - return None; + return VerificationResult::Cancelled; } }; let block_digest = block.digest(); if self.execution.pending_contains(&block_digest) { timer.observe(context); - return Some(true); + return VerificationResult::Decided(true); } - // A finalized candidate cannot be re-executed against newer database - // state. Prove it belongs to the canonical chain before accepting it. - match self - .check_processed(marshal.clone(), block.as_ref(), verification) - .await - { - ProcessedBlock::Continue => {} - ProcessedBlock::Accepted => { - timer.observe(context); - return Some(true); + // Each iteration classifies the candidate against the canonical chain, + // then executes it. A stale or invalid-looking attempt means a + // finalization landed while this one ran, and re-classifying answers + // correctly whether the finalized block was the candidate itself, an + // ancestor, or a competitor. The loop is bounded because each retry + // consumes an anchor move, and classification decides outright once the + // anchor reaches the candidate's height. Each attempt consumes its own + // ancestry clone, so a retry starts from the same position after the + // candidate. + loop { + let seen = self.execution.last_processed(); + + // A finalized candidate cannot be re-executed against newer database + // state. Prove it belongs to the canonical chain before accepting it. + match self + .check_processed(marshal.clone(), block.as_ref(), verification) + .await + { + ProcessedBlock::Continue => {} + ProcessedBlock::Accepted => { + timer.observe(context); + return VerificationResult::Decided(true); + } + ProcessedBlock::Rejected => return VerificationResult::Decided(false), + ProcessedBlock::Cancelled => return VerificationResult::Cancelled, } - ProcessedBlock::Rejected => return Some(false), - ProcessedBlock::Cancelled => return None, - } - // Reconstruct the candidate's parent state. This is the only phase - // shared across requests, keyed by the acquired parent's block digest. - let parent = match self - .prepare_parent( - context, - marshal, - block_digest, - &mut ancestry, - progress, - verification, - ) - .await - { - Ok(parent) => parent, - Err(PrepareFailure::Invalid) => return Some(false), - Err(PrepareFailure::Cancelled) => return None, - }; + // Reconstruct the candidate's parent state. This is the only phase + // shared across requests, keyed by the acquired parent's block digest. + let mut attempt_ancestry = ancestry.clone(); + let parent = match self + .prepare_parent( + context, + marshal.clone(), + block_digest, + &mut attempt_ancestry, + verification, + ) + .await + { + Ok(parent) => parent, + Err(PrepareFailure::Invalid) => { + // A finalization that completed while this attempt waited can + // make valid ancestry look invalid (the parent swept below the + // new anchor), so classify once more before answering false. + match self + .check_processed(marshal.clone(), block.as_ref(), verification) + .await + { + ProcessedBlock::Accepted => { + timer.observe(context); + return VerificationResult::Decided(true); + } + ProcessedBlock::Cancelled => return VerificationResult::Cancelled, + ProcessedBlock::Rejected => return VerificationResult::Decided(false), + // The candidate still sits above the anchor. An anchor + // that moved during this attempt may have invalidated the + // walk itself, so retry against the new anchor. A stable + // anchor means the ancestry is genuinely invalid. + ProcessedBlock::Continue => { + if self.execution.last_processed().digest != seen.digest { + continue; + } + return VerificationResult::Decided(false); + } + } + } + Err(PrepareFailure::Cancelled) => return VerificationResult::Cancelled, + Err(PrepareFailure::Stale) => { + if await_or_cancel(verification, self.execution.anchor_past(&seen)) + .await + .is_none() + { + return VerificationResult::Cancelled; + } + continue; + } + }; - progress.verifying(block_digest, parent.digest, consensus_context.round()); - let result = self - .verify( - context, - consensus_context, - block, - parent, - ancestry, - verification, - ) - .await; - if result == Some(true) { - timer.observe(context); + match self + .verify( + context, + consensus_context.clone(), + Arc::clone(&block), + parent, + attempt_ancestry, + verification, + ) + .await + { + Attempt::Done(result) => { + if matches!(result, VerificationResult::Decided(true)) { + timer.observe(context); + } + return result; + } + Attempt::Stale => { + if await_or_cancel(verification, self.execution.anchor_past(&seen)) + .await + .is_none() + { + return VerificationResult::Cancelled; + } + continue; + } + } } - result } /// Classifies a candidate at or below the applied height without @@ -210,8 +283,8 @@ where verification.cancelled().await; ProcessedBlock::Cancelled } - Err(PrepareBatchesError::Invalid) => { - unreachable!("processed-block check cannot return Invalid") + Err(PrepareBatchesError::Invalid | PrepareBatchesError::Stale) => { + unreachable!("processed-block check cannot return Invalid or Stale") } } } @@ -223,7 +296,6 @@ where marshal: MarshalMailbox, block_digest: PendingDigest, ancestry: &mut impl Ancestry, - progress: &VerificationProgress>, verification: &mut Verification, ) -> Result, PrepareFailure> where @@ -239,8 +311,8 @@ where "verification request waiting on incomplete parent ancestry" ); - // As with incomplete candidate ancestry, only cancellation or - // actor-driven invalidation should release this pending request. + // As with incomplete candidate ancestry, only the caller + // leaving should release this pending request. verification.cancelled().await; return Err(PrepareFailure::Cancelled); } @@ -261,10 +333,7 @@ where marshal, block.clone(), verification, - Some(ReplayTracking { - flights: &self.replays, - progress, - }), + Some(&self.replays), ) .await { @@ -296,6 +365,14 @@ where ); return Err(PrepareFailure::Cancelled); } + Err(PrepareBatchesError::Stale) => { + debug!( + parent_digest = ?digest, + ?block_digest, + "verification went stale during prepare_batches" + ); + return Err(PrepareFailure::Stale); + } }; Ok(PreparedParent { @@ -314,7 +391,7 @@ where parent: PreparedParent, ancestry: impl Ancestry, verification: &mut Verification, - ) -> Option { + ) -> Attempt { let block_digest = block.digest(); let round = consensus_context.round(); @@ -334,13 +411,24 @@ where ) .await { - Some(result) => result, + Some(Ok(result)) => result, + Some(Err(ExecutionError::Stale)) => { + debug!( + parent_digest = ?parent.digest, + ?block_digest, + "verification went stale during application execution" + ); + return Attempt::Stale; + } + Some(Err(err @ ExecutionError::Fatal(_))) => { + panic!("application verification failed: {err}") + } None => { debug!( parent_digest = ?parent.digest, "verification request cancelled during verify" ); - return None; + return Attempt::Done(VerificationResult::Cancelled); } }; @@ -350,7 +438,7 @@ where ?block_digest, "verification rejected: app.verify returned None" ); - return Some(false); + return Attempt::Done(VerificationResult::Decided(false)); }; let tail = info_span!( "stateful.processor.match_commitments", @@ -367,7 +455,7 @@ where ?block_digest, "verification rejected: verified state must match block commitments" ); - return Some(false); + return Attempt::Done(VerificationResult::Decided(false)); } if !self .execution @@ -378,11 +466,11 @@ where ?block_digest, "verification result became incompatible before caching" ); - return Some(false); + return Attempt::Done(VerificationResult::Decided(false)); } self.execution.update_pending_metric(); drop(block); drop(tail); - Some(true) + Attempt::Done(VerificationResult::Decided(true)) } } diff --git a/glue/src/stateful/actor/syncer/actor.rs b/glue/src/stateful/actor/syncer/actor.rs index ae039398b75..1b19c6a68f9 100644 --- a/glue/src/stateful/actor/syncer/actor.rs +++ b/glue/src/stateful/actor/syncer/actor.rs @@ -1,11 +1,11 @@ use super::{ BlockDigest, SyncResult, - mailbox::{Mailbox, Message}, + mailbox::{Mailbox, Message, UpdateOutcome}, resolve_state_sync_floor, }; use crate::stateful::{ Application, - db::{Anchor, DatabaseSet, StateSyncSet, SyncEngineConfig}, + db::{DatabaseSet, StateSyncSet, SyncEngineConfig}, }; use commonware_actor::mailbox::{self as actor_mailbox, Receiver}; use commonware_consensus::{ @@ -70,9 +70,6 @@ where /// The mailbox. mailbox: Receiver>, - /// The produced state sync artifact, if complete. - artifact: Option>, - /// Database configuration for the managed set. db_config: >::Config, @@ -108,7 +105,6 @@ where Self { context: ContextCell::new(config.context), mailbox: receiver, - artifact: None, db_config: config.db_config, sync_config: config.sync_config, resolvers: config.resolvers, @@ -125,6 +121,9 @@ where } pub async fn run(mut self) { + // Everything before the select loop runs outside shutdown handling, so a + // stop during this await tears the task down crash-style, which the + // durable InProgress metadata makes recoverable. let (marshal, floor) = &self.marshal; let resolved_floor = resolve_state_sync_floor::(marshal, *floor, &self.finalization).await; @@ -148,21 +147,24 @@ where }, result = &mut state_sync_task => match result { Ok((databases, anchor)) => { - Self::publish_artifact( - &mut self.artifact, - &mut self.sync_complete, - databases, - anchor, - ); + let sync_complete = self + .sync_complete + .take() + .expect("completion sender present until sync completes"); + sync_complete.send_lossy(SyncResult { databases, anchor }); state_sync_task = None.into(); // A tip update enqueued after the coordinator's final drain has no // receiver left to record it or release its observation barrier. // Dropping the sender frees the ring buffer, so the observer of any - // queued update retries and receives the artifact. + // queued update retries and learns sync completed. tip_updates_tx = None; } Err(err) => { + // Unreachable from adversarial input, since the target root comes + // from a finalized block and fetched operations are + // proof-verified, so bad peer data surfaces as resolver + // feedback and retries, never as an engine error. panic!("state sync task failed: {err:?}"); } }, @@ -171,13 +173,13 @@ where break; } => match message { Message::UpdateTargets { update, response } => { - if let Some(artifact) = self.artifact.clone() { - response.send_lossy(Some(artifact)); + if self.sync_complete.is_none() { + response.send_lossy(UpdateOutcome::SyncCompleted); continue; } // If sync had already completed, the state-sync branch above would - // have published `self.artifact` before this mailbox branch ran. + // have consumed the completion sender before this mailbox branch ran. let tip_updates = tip_updates_tx .as_mut() .expect("ring sender lives until the artifact is published"); @@ -188,48 +190,34 @@ where // publish its artifact", not as a hard failure. match (&mut state_sync_task).await { Ok((databases, anchor)) => { - Self::publish_artifact( - &mut self.artifact, - &mut self.sync_complete, - databases, - anchor, - ); state_sync_task = None.into(); + let sync_complete = self + .sync_complete + .take() + .expect("completion sender present until sync completes"); + sync_complete.send_lossy(SyncResult { databases, anchor }); + response.send_lossy(UpdateOutcome::SyncCompleted); } Err(err) => { panic!("state sync task failed: {err:?}"); } } tip_updates_tx = None; - response.send_lossy(self.artifact.clone()); continue; } - response.send_lossy(None); + response.send_lossy(UpdateOutcome::Observed); } }, } } - - fn publish_artifact( - artifact: &mut Option>, - sync_complete: &mut Option>>, - databases: A::Databases, - anchor: Anchor>, - ) { - let sync_result = SyncResult { databases, anchor }; - *artifact = Some(sync_result.clone()); - if let Some(sync_complete) = sync_complete.take() { - sync_complete.send_lossy(sync_result); - } - } } #[cfg(test)] mod tests { use super::{Config, Syncer, resolve_state_sync_floor}; use crate::stateful::{ - Application, Input, Proposed, - actor::syncer::{StateSyncMetadata, init_databases_from_marshal}, + Application, ExecutionError, Input, Proposed, + actor::syncer::{StateSyncMetadata, UpdateOutcome, init_databases_from_marshal}, db::{Anchor, Barrier, DatabaseSet, StateSyncSet, SyncEngineConfig, TipUpdate}, tests::{ fixtures::{self, MarshalFixture}, @@ -279,7 +267,9 @@ mod tests { 0 } - async fn new_batches(&self) -> Self::Unmerkleized { + fn readers(&self) -> Self::Readers {} + + async fn new_batches(_readers: &Self::Readers) -> Self::Unmerkleized { unreachable!("WedgeSet only serves the syncer harness") } @@ -291,17 +281,15 @@ mod tests { unreachable!("WedgeSet only serves the syncer harness") } - fn readers(&self) -> Self::Readers {} - - async fn finalize(&self, _batches: Self::Merkleized) -> (Self::Snapshots, Barrier) { + async fn finalize(self, _batches: Self::Merkleized) -> (Self, Self::Snapshots, Barrier) { unreachable!("WedgeSet only serves the syncer harness") } - async fn snapshot(&self) -> Self::Snapshots { - unreachable!("WedgeSet only serves the syncer harness") + async fn snapshot(self) -> (Self, Self::Snapshots) { + (self, ()) } - async fn prune(&self, _targets: &Self::SyncTargets) { + async fn prune(self, _targets: &Self::SyncTargets) -> Self { unreachable!("WedgeSet only serves the syncer harness") } @@ -309,8 +297,9 @@ mod tests { self.0 } - async fn rewind_to_targets(&self, targets: Self::SyncTargets) { + async fn rewind_to_targets(self, targets: Self::SyncTargets) -> Self { assert_eq!(targets, self.0, "test database cannot rewind"); + self } } @@ -362,7 +351,7 @@ mod tests { _ancestry: impl Ancestry, _batches: TestUnmerkleized, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { unreachable!("WedgeApp only serves the syncer harness") } @@ -371,7 +360,7 @@ mod tests { _context: (deterministic::Context, Self::Context), _ancestry: impl Ancestry, _batches: TestUnmerkleized, - ) -> Option { + ) -> Result, ExecutionError> { unreachable!("WedgeApp only serves the syncer harness") } @@ -380,7 +369,7 @@ mod tests { _context: (deterministic::Context, Self::Context), _block: &Self::Block, _batches: TestUnmerkleized, - ) -> TestMerkleized { + ) -> Result { unreachable!("WedgeApp only serves the syncer harness") } } @@ -770,14 +759,16 @@ mod tests { // The update is forwarded into the ring buffer and its observation parks before // the sync task completes (the task's clock only advances at quiescence). The - // stranded observation must resolve through a retry that returns the artifact. + // stranded observation must resolve through a retry that reports completion, + // with the artifact arriving on the completion channel. let update = context .child("update") .spawn(move |_| async move { mailbox.update_targets(anchor(1, 1), 1).await }); - let result = update.await.expect("update task failed"); - assert!( - matches!(&result, Some(artifact) if artifact.anchor.height == Height::zero()), - "stranded update must resolve to the completed artifact", + let outcome = update.await.expect("update task failed"); + assert_eq!( + outcome, + UpdateOutcome::SyncCompleted, + "stranded update must report the completed sync" ); let artifact = sync_completed.await.expect("artifact must publish"); diff --git a/glue/src/stateful/actor/syncer/mailbox.rs b/glue/src/stateful/actor/syncer/mailbox.rs index cdc2c30e06e..e139a17f0db 100644 --- a/glue/src/stateful/actor/syncer/mailbox.rs +++ b/glue/src/stateful/actor/syncer/mailbox.rs @@ -1,6 +1,5 @@ //! [`Syncer`](super::Syncer) actor ingress. -use super::SyncResult; use crate::stateful::{ Application, db::{Anchor, DatabaseSet, TipUpdate}, @@ -14,6 +13,17 @@ use rand_core::Rng; type SyncTargets = <>::Databases as DatabaseSet>::SyncTargets; type BlockDigest = <>::Block as Digestible>::Digest; +/// Reply to [`Mailbox::update_targets`]. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) enum UpdateOutcome { + /// The live sync coordinator recorded the update, so the eventual sync + /// artifact reflects this target or a newer one. + Observed, + /// State sync already completed, so no coordinator remains to record the + /// update. The artifact is on the completion channel. + SyncCompleted, +} + pub(crate) enum Message where E: Rng + Spawner + Metrics + Clock, @@ -21,7 +31,8 @@ where { UpdateTargets { update: TipUpdate, SyncTargets>, - response: oneshot::Sender>>, + /// Reports whether the update was recorded or sync already completed. + response: oneshot::Sender, }, } @@ -78,49 +89,51 @@ where /// Sends a target update and waits until the live sync coordinator records it. /// - /// If sync already completed before the update could be observed, returns the - /// completed artifact instead. + /// If sync already completed, the artifact arrives on the completion channel. pub async fn update_targets( &self, anchor: Anchor>, targets: SyncTargets, - ) -> Option> { + ) -> UpdateOutcome { loop { let (update, observed) = TipUpdate::with_observation(anchor, targets.clone()); let (response, receiver) = oneshot::channel(); - let _ = self + let feedback = self .sender .enqueue(Message::UpdateTargets { update, response }); + assert!( + feedback.accepted(), + "syncer must outlive update_targets callers", + ); - match receiver - .await - .expect("Syncer should respond to update_targets") - { - Some(artifact) => return Some(artifact), - None => { - // Wait until the live sync coordinator has recorded the new tip update. - // Enqueueing it into Syncer is not enough to prove the eventual sync - // artifact includes the target or to discard its handoff state. - if observed.await.is_ok() { - return None; - } - - // The active coordinator dropped before recording this update. - // Retry so Syncer can either hand the update to the next coordinator - // or report the completed sync artifact. - } + let Ok(outcome) = receiver.await else { + // A newer queued update displaced this one before the syncer saw + // it (the queue keeps only the newest update), or the syncer died + // with the message queued. + continue; + }; + if outcome == UpdateOutcome::SyncCompleted { + return outcome; } + + // Wait until the live sync coordinator has recorded the new tip update. + // Enqueueing it into Syncer is not enough to prove the eventual sync + // artifact includes the target or to discard its handoff state. + if observed.await.is_ok() { + return UpdateOutcome::Observed; + } + + // The active coordinator dropped before recording this update. + // Retry so Syncer can either hand the update to the next coordinator + // or report the completed sync. } } } #[cfg(test)] mod tests { - use super::{Mailbox, Message}; - use crate::stateful::{ - actor::syncer::SyncResult, - tests::mocks::{TestApp, anchor, test_databases}, - }; + use super::{Mailbox, Message, UpdateOutcome}; + use crate::stateful::tests::mocks::{TestApp, anchor}; use commonware_actor::mailbox as actor_mailbox; use commonware_runtime::{Runner as _, Supervisor as _, deterministic}; use commonware_utils::NZUsize; @@ -139,35 +152,64 @@ mod tests { panic!("first update should be sent"); }; assert!( - response.send(None).is_ok(), + response.send(UpdateOutcome::Observed).is_ok(), "response receiver should be alive" ); drop(update); assert!(update_targets.as_mut().now_or_never().is_none()); - let expected = SyncResult:: { - databases: test_databases(), - anchor: anchor(8, 10), - }; let Some(Message::UpdateTargets { response, .. }) = receiver.recv().await else { panic!("dropped observation should trigger a retry"); }; assert!( - response.send(Some(expected.clone())).is_ok(), + response.send(UpdateOutcome::SyncCompleted).is_ok(), "response receiver should be alive" ); - let result = update_targets.await; assert_eq!( - result.expect("retry should return artifact").anchor, - expected.anchor + update_targets.await, + UpdateOutcome::SyncCompleted, + "retry should report the completed sync" ); }); } #[test] - fn update_targets_returns_none_only_after_observation_is_recorded() { + fn update_targets_retries_when_response_is_displaced() { + deterministic::Runner::default().start(|context| async move { + let (sender, mut receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(1)); + let mailbox = Mailbox::::new(sender); + let mut update_targets = Box::pin(mailbox.update_targets(anchor(7, 9), 7)); + + assert!(update_targets.as_mut().now_or_never().is_none()); + + // Drop the message without responding, as overflow displacement does. + let Some(message) = receiver.recv().await else { + panic!("first update should be sent"); + }; + drop(message); + + assert!(update_targets.as_mut().now_or_never().is_none()); + + let Some(Message::UpdateTargets { response, .. }) = receiver.recv().await else { + panic!("displaced response should trigger a retry"); + }; + assert!( + response.send(UpdateOutcome::SyncCompleted).is_ok(), + "response receiver should be alive" + ); + + assert_eq!( + update_targets.await, + UpdateOutcome::SyncCompleted, + "retry should report the completed sync" + ); + }); + } + + #[test] + fn update_targets_resolves_only_after_observation_is_recorded() { deterministic::Runner::default().start(|context| async move { let (sender, mut receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(1)); let mailbox = Mailbox::::new(sender); @@ -179,7 +221,7 @@ mod tests { panic!("update should be sent"); }; assert!( - response.send(None).is_ok(), + response.send(UpdateOutcome::Observed).is_ok(), "response receiver should be alive" ); @@ -187,7 +229,11 @@ mod tests { update.record(|_, _| {}); - assert!(update_targets.await.is_none()); + assert_eq!( + update_targets.await, + UpdateOutcome::Observed, + "recorded update should report observation" + ); }); } } diff --git a/glue/src/stateful/actor/syncer/mod.rs b/glue/src/stateful/actor/syncer/mod.rs index 2ee7baaef25..20000714dca 100644 --- a/glue/src/stateful/actor/syncer/mod.rs +++ b/glue/src/stateful/actor/syncer/mod.rs @@ -25,7 +25,7 @@ mod actor; pub(crate) use actor::{Config, Syncer}; pub(crate) mod mailbox; -pub(crate) use mailbox::Mailbox; +pub(crate) use mailbox::{Mailbox, UpdateOutcome}; mod plan; pub use plan::SyncPlan; @@ -131,25 +131,12 @@ where E: Rng + Spawner + Metrics + Clock, A: Application, { - /// The database handle set. + /// The owned database set produced by sync. pub databases: A::Databases, /// The anchor at which state sync completed. pub anchor: Anchor>, } -impl Clone for SyncResult -where - E: Rng + Spawner + Metrics + Clock, - A: Application, -{ - fn clone(&self) -> Self { - Self { - databases: self.databases.clone(), - anchor: self.anchor, - } - } -} - /// Resolved state sync floor data derived from the selected finalization and marshal progress. pub(crate) struct ResolvedFloor where @@ -376,7 +363,10 @@ where /// /// If the databases are found to be inconsistent with the marshal floor, this /// function will attempt to repair by rewinding the databases which are ahead. If the -/// databases are entirely inconsistent, this function will panic. +/// databases are entirely inconsistent, this function will panic. That covers a crash +/// between marshal installing a sync floor and the sync metadata recording it, so the +/// operator's sync request must persist across such a restart so startup re-enters the +/// sync path instead of asking a fresh database set to reach the installed floor. pub(crate) async fn init_databases_from_marshal( context: &E, marshal: &MarshalMailbox, @@ -418,20 +408,18 @@ where .chain((floor_block.height() > marshal_floor).then_some(floor_block.height())) .max(); - let databases = A::Databases::init(context.child("db_set"), db_config).await; + let mut databases = A::Databases::init(context.child("db_set"), db_config).await; let processed_targets = A::sync_targets(&floor_block); - // In the case that the committed targets do not match the marshal floor, we may + // In the case that the applied targets do not match the marshal floor, we may // have suffered a crash that left the set in an inconsistent state. In this case, // we attempt to repair by rewinding the databases back to the marshal floor. If // the rewind fails to produce a consistent state, we must crash. This can occur // if the databases were corrupted or pruned too aggressively. - let committed = databases.committed_targets().await; - if committed != processed_targets { - databases.rewind_to_targets(processed_targets.clone()).await; - let rewound_targets = databases.committed_targets().await; + if databases.committed_targets().await != processed_targets { + databases = databases.rewind_to_targets(processed_targets.clone()).await; assert!( - rewound_targets == processed_targets, + databases.committed_targets().await == processed_targets, "databases must be consistent with marshal floor after rewind" ); } diff --git a/glue/src/stateful/db/any.rs b/glue/src/stateful/db/any.rs index 550622fc9b5..ceaaba31979 100644 --- a/glue/src/stateful/db/any.rs +++ b/glue/src/stateful/db/any.rs @@ -1,14 +1,14 @@ //! [`ManagedDb`] implementation for QMDB [`any`](commonware_storage::qmdb::any) databases. //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for -//! read-through to applied state. This module provides wrapper types -//! that capture a [`Shared`] database handle alongside the raw batch so the -//! [`Unmerkleized`](super::Unmerkleized) and [`Merkleized`](super::Merkleized) -//! traits can be implemented without a DB parameter. +//! read-through to applied state. The wrapper types here hold a [`Reader`] +//! to their database and lease it for each such call, so a batch stays usable +//! across applies of compatible batches and never delays a mutation by more +//! than one storage call. use crate::stateful::db::{ - BatchContext, LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, - SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_standard_db, + LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, + Unmerkleized as UnmerkleizedTrait, sync_standard_db, }; use commonware_codec::{Codec, Read as CodecRead}; use commonware_cryptography::Hasher; @@ -48,8 +48,10 @@ use std::{ // Matches commonware_storage::qmdb::any::BITMAP_CHUNK_BYTES, which is crate-private. const ANY_BITMAP_CHUNK_BYTES: usize = 64; -/// Wraps a QMDB [`UnmerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Unmerkleized`](super::Unmerkleized) trait. +/// The `any` database type the wrapper batches read through. +type AnyDb = Db; + +/// Wraps a QMDB [`UnmerkleizedBatch`] to implement [`Unmerkleized`](super::Unmerkleized). pub struct AnyUnmerkleized where F: Family, @@ -62,16 +64,15 @@ where Operation: Codec, { batch: UnmerkleizedBatch, - db: Shared>, + reader: Reader>, metadata: Option, } -/// Staged batch returned by [`AnyUnmerkleized::stage`], wrapping a QMDB [`Staged`] with a -/// reference to the parent database. +/// Staged batch returned by [`AnyUnmerkleized::stage`], wrapping a QMDB [`Staged`]. /// -/// Like any speculative batch, this handle is a branch-scoped view of the shared database: it -/// stays valid only while every batch finalized on the database is an ancestor of this batch -/// (see [`MerkleizedBatch`]'s branch-validity contract). +/// A branch-scoped view of the database. It stays valid only while every batch finalized on +/// the database is an ancestor of this batch (see [`MerkleizedBatch`]'s branch-validity +/// contract). pub struct AnyStaged where F: Family, @@ -84,7 +85,7 @@ where Operation: Codec, { staged: Staged, - db: Shared>, + reader: Reader>, metadata: Option, } @@ -109,7 +110,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get(key, &db).await } @@ -117,7 +118,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get_many(keys, &db).await } @@ -130,18 +131,18 @@ where ) -> Result<(Vec>, AnyStaged), Error> { let Self { batch, - db, + reader, metadata, } = self; let (values, staged) = { - let guard = db.read().await; + let guard = reader.read().await; batch.stage(keys, &guard).await? }; Ok(( values, AnyStaged { staged, - db, + reader, metadata, }, )) @@ -154,8 +155,7 @@ where } } -/// Wraps a QMDB [`MerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Merkleized`](super::Merkleized) trait. +/// Wraps a QMDB [`MerkleizedBatch`] to implement [`Merkleized`](super::Merkleized). pub struct AnyMerkleized where F: Family, @@ -168,7 +168,7 @@ where Operation: Codec, { inner: Arc>, - db: Shared>, + reader: Reader>, } impl Clone for AnyMerkleized @@ -185,7 +185,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -257,11 +257,11 @@ where ) -> Result<(Range, Vec>, Self), Error> { let Self { staged, - db, + reader, metadata, } = self; let (range, values, staged) = { - let guard = db.read().await; + let guard = reader.read().await; staged.expand(keys, &guard).await? }; Ok(( @@ -269,7 +269,7 @@ where values, Self { staged, - db, + reader, metadata, }, )) @@ -291,7 +291,7 @@ where { /// Record updates for staged reads and upserts for unread keys, then merkleize. /// - /// Consumes the staged handle and write vectors. Call [`expand`](AnyStaged::expand) before + /// Consumes the staged batch and write vectors. Call [`expand`](AnyStaged::expand) before /// this method if more keys must be read into the staged index space. /// /// A `Some` value is an upsert. `None` is a delete. Update indices refer to the staged read @@ -309,14 +309,14 @@ where ) -> Result, S>, Error> { let Self { staged, - db, + reader, metadata, } = self; let inner = { - let guard = db.read().await; + let guard = reader.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(AnyMerkleized { inner, db }) + Ok(AnyMerkleized { inner, reader }) } } @@ -335,7 +335,7 @@ where { /// Record updates for staged reads and upserts for unread keys, then merkleize. /// - /// Consumes the staged handle and write vectors. Call [`expand`](AnyStaged::expand) before + /// Consumes the staged batch and write vectors. Call [`expand`](AnyStaged::expand) before /// this method if more keys must be read into the staged index space. /// /// A `Some` value is an upsert. `None` is a delete. Update indices refer to the staged read @@ -353,14 +353,14 @@ where ) -> Result, S>, Error> { let Self { staged, - db, + reader, metadata, } = self; let inner = { - let guard = db.read().await; + let guard = reader.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(AnyMerkleized { inner, db }) + Ok(AnyMerkleized { inner, reader }) } } @@ -378,7 +378,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get(key, &db).await } @@ -386,7 +386,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get_many(keys, &db).await } } @@ -409,11 +409,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(AnyMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -436,11 +436,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(AnyMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -451,12 +451,11 @@ where F: Family, E: Context, U: Update, - C: Mutable>, - I: UnorderedIndex> + 'static, + C: Contiguous>, + I: UnorderedIndex>, H: Hasher, S: Strategy, Operation: Codec, - AnyUnmerkleized: UnmerkleizedTrait, { type Digest = H::Digest; type Unmerkleized = AnyUnmerkleized; @@ -468,18 +467,13 @@ where fn new_batch(&self) -> Self::Unmerkleized { AnyUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, } } } /// Implement [`ManagedDb`] for unordered QMDB databases with fixed-size values. -/// -/// `new_batch` captures the [`Shared`] database handle in the returned -/// wrapper so that `get()` and `merkleize()` can read through to -/// applied state. -/// /// `finalize` applies the merkleized batch's changeset and starts /// persisting it, reporting durability on the returned handle. impl ManagedDb @@ -537,11 +531,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); AnyUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -557,9 +551,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -657,11 +651,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); AnyUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -677,9 +671,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -810,6 +804,8 @@ where #[cfg(test)] mod tests { use super::*; + use crate::stateful::db::{DatabaseSet, Single}; + use commonware_codec::Encode as _; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_parallel::Sequential; use commonware_runtime::{ @@ -820,8 +816,12 @@ mod tests { }; use commonware_storage::{ journal::contiguous::fixed::Config as FixedJournalConfig, - merkle::{full::Config as MerkleConfig, mmr}, - qmdb::{self, any::unordered::fixed}, + merkle::{Location, full::Config as MerkleConfig, mmr}, + qmdb::{ + self, + any::unordered::fixed, + sync::{Request, Response, Source as SyncSource}, + }, translator::TwoCap, }; use commonware_utils::{NZU16, NZU64, NZUsize}; @@ -857,45 +857,11 @@ mod tests { } } - #[test] - fn unmerkleized_batch_refuses_after_competing_finalization() { - deterministic::Runner::default().start(|context| async move { - let config = fixed_config("unordered-fixed-stale-refusal", &context); - let db = >::init(context.child("db"), config) - .await - .unwrap(); - let db = Shared::new("test", db); - - let key = Sha256::hash(&[b"key"]); - let value = Sha256::hash(&[b"winner"]); - let pre_finalization = db.new_batch_for_test::<_>().await; - let winner = db.new_batch_for_test::<_>().await.write(key, Some(value)); - let winner = crate::stateful::db::Unmerkleized::merkleize(winner) - .await - .unwrap(); - - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, winner) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - - // The winner is not an ancestor of the earlier fork, so reading through it - // refuses instead of consulting state the fork never accounted for. - assert!(matches!( - pre_finalization.get(&key).await, - Err(qmdb::Error::StaleRead) - )); - }); - } - /// The glue staged wrapper (`AnyUnmerkleized::stage` -> `AnyStaged::expand` -> /// `AnyStaged::merkleize`) must return the same values and root as an explicit `get_many` + /// `write` + `merkleize`, including a staged delete, an upsert, and metadata flow (both set - /// on the staged handle via `with_metadata` and carried from before staging). This guards - /// metadata flow and db-handle pairing through the wrapper. + /// on the staged batch via `with_metadata` and carried from before staging). This guards + /// metadata flow through the wrapper. #[test] fn unordered_fixed_staged_merkleize_matches_explicit_writes() { deterministic::Runner::default().start(|context| async move { @@ -903,29 +869,21 @@ mod tests { let db = >::init(context.child("db"), config) .await .unwrap(); - let db = Shared::new("test", db); - + let db = Single::from(db); let key = |i: u64| Sha256::hash(&[&i.to_be_bytes()]); let val = |i: u64| Sha256::hash(&[&(i + 10_000).to_be_bytes()]); let metadata = Sha256::hash(&[b"metadata"]); // Seed keys 0..50 and finalize. - let mut seed = db.new_batch_for_test::<_>().await; + let mut seed = >::new_batch(db.reader()).await; for i in 0..50u64 { seed = seed.write(key(i), Some(val(i))); } let merkleized = crate::stateful::db::Unmerkleized::merkleize(seed) .await .unwrap(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } + let (db, _, barrier) = DatabaseSet::finalize(db, merkleized).await; + assert!(barrier.durable().await, "finalize flush failed"); // Read set: key(1) updated, key(2) deleted, key(999) missing -> created. let read_keys = [key(1), key(2), key(999)]; @@ -934,7 +892,7 @@ mod tests { let upserts = vec![(key(3), Some(val(1_002)))]; // Explicit path. - let mut explicit = db.new_batch_for_test::<_>().await; + let mut explicit = >::new_batch(db.reader()).await; let explicit_values = explicit.get_many(&keys).await.unwrap(); for (slot, value) in &indexed_updates { explicit = explicit.write(read_keys[*slot], *value); @@ -948,8 +906,8 @@ mod tests { .unwrap() .root(); - // Staged path, with metadata set on the staged handle. - let staged_batch = db.new_batch_for_test::<_>().await; + // Staged path, with metadata set on the staged batch. + let staged_batch = >::new_batch(db.reader()).await; let split = 2; let (mut staged_values, staged) = staged_batch.stage(&keys[..split]).await.unwrap(); let (range, suffix_values, staged) = staged.expand(&keys[split..]).await.unwrap(); @@ -966,7 +924,9 @@ mod tests { assert_eq!(explicit_root, staged_root); // Metadata set before staging must be carried through to staged merkleize. - let carried_batch = db.new_batch_for_test::<_>().await.with_metadata(metadata); + let carried_batch = >::new_batch(db.reader()) + .await + .with_metadata(metadata); let (carried_values, staged) = carried_batch.stage(&keys).await.unwrap(); let carried_root = staged .merkleize(indexed_updates.clone(), upserts.clone()) @@ -988,9 +948,10 @@ mod tests { Sequential, >; - /// `finalize` must return, with the batch readable through the shared - /// handle, while its flush is still parked at the storage layer. - /// Durability is reported only on the returned handle. + /// `finalize` must return, with the batch readable through the set's + /// readers, while its flush is still parked at the storage layer. + /// Durability is reported only on the returned barrier, and the captured + /// snapshot must already prove the post-apply state. #[test] fn finalize_defers_flush_to_returned_handle() { deterministic::Runner::default().start(|context| async move { @@ -1006,36 +967,60 @@ mod tests { ) .await .unwrap(); - let db = Shared::new("test", db); + let db = Single::from(db); let key = Sha256::hash(&[b"key"]); let value = Sha256::hash(&[b"value"]); - let batch = db.new_batch_for_test::<_>().await.write(key, Some(value)); + let batch = >::new_batch(db.reader()) + .await + .write(key, Some(value)); let merkleized = crate::stateful::db::Unmerkleized::merkleize(batch) .await .unwrap(); - - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); + let (db, snapshot, barrier) = DatabaseSet::finalize(db, merkleized).await; // The flush is parked, yet the batch is already readable. assert!( pending.starts() > pending.completions(), "finalize must leave its flush parked", ); - { - let guard = db.read().await; - assert_eq!(guard.get(&key).await.unwrap(), Some(value)); - } + let db = db.reader(); + assert_eq!(db.read().await.get(&key).await.unwrap(), Some(value)); + + // The snapshot freezes at the post-apply boundary and proves the + // just-applied state, independent of durability. + let size = Location::new(snapshot.bounds().end); + assert_eq!( + size, + db.read().await.bounds().end, + "snapshot must cover the applied batch" + ); + let (response, _) = SyncSource::serve( + &*snapshot, + Request::Boundary { + size, + start: size - 1, + }, + ) + .await + .expect("captured snapshot must serve its tip"); + let Response::Boundary { proof, op, .. } = response else { + panic!("expected a boundary response"); + }; + let root = proof + .reconstruct_root(&qmdb::hasher::(), &[op.encode()], size - 1) + .expect("served proof must reconstruct"); + assert_eq!( + root, + db.read().await.root(), + "snapshot must prove the post-apply root" + ); release_pending_syncs(&pending); - drive_pending_syncs(&pending, sync) - .await - .expect("flush must succeed once released"); + assert!( + drive_pending_syncs(&pending, barrier.durable()).await, + "flush must succeed once released" + ); }); } } diff --git a/glue/src/stateful/db/cell.rs b/glue/src/stateful/db/cell.rs new file mode 100644 index 00000000000..2a2f2dc69ac --- /dev/null +++ b/glue/src/stateful/db/cell.rs @@ -0,0 +1,261 @@ +//! Separates read access to a live database from the authority to mutate it. +//! +//! [`split`] wraps a database and returns two capabilities over it. The +//! [`Writer`] is unique and runs consuming mutations. The [`Reader`] is +//! freely cloned into batches, and grants short leases that cover exactly one +//! storage call. +//! +//! Neither value owns the database. The cell does, and both capabilities keep +//! it alive. What distinguishes them is what they permit. +//! +//! The cell is a tokio read-write lock, whose documented policy is fair and +//! write-preferring, so a waiting mutation blocks later leases and cannot be +//! starved, while leases already granted finish first. The write side covers +//! the whole take-and-restore of a mutation, so a reader can never observe the +//! database missing. A mutation that is interrupted mid-flight leaves the cell +//! poisoned, and the only thing that interrupts one here is the owning task +//! being torn down, so later leases park forever and are dropped along with +//! the tasks holding them. +//! +//! A lease guarantees the database is present and unchanging for one call, not +//! that the caller's batch is still current. A batch operation under a lease +//! can still refuse because a competing batch was applied (see +//! [`commonware_storage::qmdb::Error::StaleRead`]). +//! +//! Two rules keep callers out of trouble. The lock is not reentrant, so never +//! hold a lease while acquiring another, on any cell, because a mutation queued +//! between the two would deadlock both. And a [`Writer`] must outlive the +//! readers from the same cell. Dropping the writer does not drop the database, +//! so readers would otherwise go on answering from a database that can never +//! advance. + +use commonware_utils::sync::{AsyncRwLockReadGuard, TracedAsyncRwLock}; +use futures::future; +use std::{future::Future, ops::Deref, sync::Arc}; + +enum State { + Live(T), + /// A mutation was interrupted before restoring the database. Fatal. + Poisoned, +} + +struct Cell { + state: TracedAsyncRwLock>, +} + +impl Cell { + async fn read(&self) -> ReadGuard<'_, T> { + let guard = self.state.read().await; + match AsyncRwLockReadGuard::try_map(guard, |state| match state { + State::Live(db) => Some(db), + State::Poisoned => None, + }) { + Ok(lease) => ReadGuard(lease), + Err(guard) => { + // Poisoning is only reachable during writer teardown. Park until + // this task is dropped with the rest of the actor, and the trace + // separates that from a bug if the process outlives the cell. + drop(guard); + tracing::error!("database cell poisoned; parking reader"); + future::pending().await + } + } + } +} + +/// Split access to `db` into the sole mutation authority and a cloneable +/// read capability. +pub fn split(db: T) -> (Writer, Reader) { + let writer = Writer::new(db); + let reader = writer.reader(); + (writer, reader) +} + +/// The unique capability to mutate the database behind a cell. +/// +/// This is the only value with [`mutate`](Self::mutate), and it is deliberately +/// not [`Clone`], so at most one exists per cell. It does not own the database. +/// Dropping it leaves readers on a database that can no longer advance, which is +/// why it must outlive the readers taken from the same cell. +pub struct Writer(Arc>); + +impl Writer { + /// Wrap `db` in a fresh cell, returning its sole mutation authority. + pub fn new(db: T) -> Self { + Self(Arc::new(Cell { + state: TracedAsyncRwLock::new("database_cell", State::Live(db)), + })) + } + + /// A read capability over the writer's cell. + pub fn reader(&self) -> Reader { + Reader(self.0.clone()) + } + + /// Run one consuming mutation to completion, returning the capability. + /// + /// New leases queue behind the mutation and leases already granted finish + /// first, so this waits at most one storage call before starting. + /// + /// Consume/produce at both levels. `mutation` takes the database by value and + /// must produce it back, the contract mutable storage operations already use. + /// This method does the same with the capability. An interrupted mutation + /// takes the writer with it, and if the database was already taken out it + /// also poisons the cell, so a second mutation of a poisoned cell is + /// unreachable rather than merely documented. Leases taken afterward park + /// forever. (Dropped while still queued for the lock, the cell stays live + /// but can never advance again -- the same wedge as dropping the writer.) + pub async fn mutate(self, mutation: F) -> (Self, R) + where + F: FnOnce(T) -> Fut, + Fut: Future, + { + let mut guard = self.0.state.write().await; + let State::Live(db) = std::mem::replace(&mut *guard, State::Poisoned) else { + unreachable!("a writer only exists while its cell is live") + }; + let (db, result) = mutation(db).await; + *guard = State::Live(db); + drop(guard); + (self, result) + } +} + +/// A cloneable read capability over the database behind a cell. +pub struct Reader(Arc>); + +impl Clone for Reader { + fn clone(&self) -> Self { + Self(self.0.clone()) + } +} + +impl Reader { + /// Acquire a read lease. + pub async fn read(&self) -> ReadGuard<'_, T> { + self.0.read().await + } +} + +/// A short read lease. Must cover exactly one storage call, never an +/// application await, so a waiting mutation is delayed by at most one call. +pub struct ReadGuard<'a, T>(AsyncRwLockReadGuard<'a, T>); + +impl Deref for ReadGuard<'_, T> { + type Target = T; + + fn deref(&self) -> &T { + &self.0 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use commonware_runtime::{Clock, Runner as _, Spawner as _, Supervisor as _, deterministic}; + use commonware_utils::channel::oneshot; + use futures::FutureExt as _; + use std::time::Duration; + + /// A waiting mutation cannot be starved by a stream of short read leases, + /// and no lease ever observes taken-out state. + #[test] + fn mutation_is_not_starved_by_read_storm() { + deterministic::Runner::default().start(|context| async move { + let (writer, reader) = split(0u64); + + let mut workers = Vec::new(); + for worker in ["r0", "r1", "r2", "r3"] { + let reader = reader.clone(); + workers.push(context.child(worker).spawn(move |ctx| async move { + loop { + { + let lease = reader.read().await; + // Every observation is a full, live value. + assert!(*lease == 0 || *lease == 1); + if *lease == 1 { + return; + } + } + ctx.sleep(Duration::from_millis(1)).await; + } + })); + } + + context.sleep(Duration::from_millis(5)).await; + writer + .mutate(|db| async move { + assert_eq!(db, 0); + (db + 1, ()) + }) + .await; + + for worker in workers { + worker.await.expect("worker should observe the mutation"); + } + }); + } + + /// A lease waits out an in-flight mutation and then sees the mutated + /// state, never a gap. + #[test] + fn leases_wait_out_a_parked_mutation() { + deterministic::Runner::default().start(|context| async move { + let (writer, reader) = split(0u64); + let (release_tx, release) = oneshot::channel::<()>(); + let (acquired_tx, acquired) = oneshot::channel::<()>(); + + let mutation_task = context.child("mutation").spawn(move |_| async move { + writer + .mutate(|db| async move { + let _ = acquired_tx.send(()); + let _ = release.await; + (db + 1, ()) + }) + .await; + }); + + acquired.await.expect("mutation must start"); + let read = reader.read(); + futures::pin_mut!(read); + assert!( + read.as_mut().now_or_never().is_none(), + "a lease must wait while a mutation holds the cell", + ); + + release_tx.send(()).expect("mutation is waiting"); + mutation_task.await.expect("mutation completes"); + assert_eq!(*read.await, 1, "the lease sees the mutated state"); + }); + } + + /// Dropping a mutation mid-flight poisons the cell, and takes the writer + /// with it. Later leases park forever instead of observing missing state. + /// A second mutation is unrepresentable, so there is nothing to assert. + #[test] + fn interrupted_mutation_poisons() { + deterministic::Runner::default().start(|_context| async move { + let (writer, reader) = split(0u64); + let (started_tx, started) = oneshot::channel::<()>(); + + let mut mutation = Box::pin(writer.mutate(|db| async move { + let _ = started_tx.send(()); + future::pending::<()>().await; + (db, ()) + })); + assert!( + mutation.as_mut().now_or_never().is_none(), + "mutation must park mid-flight", + ); + started.await.expect("mutation must reach its closure"); + drop(mutation); + + let read = reader.read(); + futures::pin_mut!(read); + assert!( + read.as_mut().now_or_never().is_none(), + "a lease after poisoning must park, not observe a gap", + ); + }); + } +} diff --git a/glue/src/stateful/db/current.rs b/glue/src/stateful/db/current.rs index 89382585d35..997e3bac239 100644 --- a/glue/src/stateful/db/current.rs +++ b/glue/src/stateful/db/current.rs @@ -1,14 +1,14 @@ //! [`ManagedDb`] implementation for QMDB [`current`](commonware_storage::qmdb::current) databases. //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for -//! read-through to applied state. This module provides wrapper types -//! that capture a [`Shared`] database handle alongside the raw batch so the -//! [`Unmerkleized`](super::Unmerkleized) and [`Merkleized`](super::Merkleized) -//! traits can be implemented without a DB parameter. +//! read-through to applied state. The wrapper types here hold a [`Reader`] +//! to their database and lease it for each such call, so a batch stays usable +//! across applies of compatible batches and never delays a mutation by more +//! than one storage call. use crate::stateful::db::{ - BatchContext, LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, - SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_standard_db, + LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, + Unmerkleized as UnmerkleizedTrait, sync_standard_db, }; use commonware_codec::{Codec, Read as CodecRead}; use commonware_cryptography::Hasher; @@ -48,8 +48,7 @@ use std::{ sync::Arc, }; -/// Wraps a QMDB [`UnmerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Unmerkleized`](super::Unmerkleized) trait. +/// Wraps a QMDB [`UnmerkleizedBatch`] to implement [`Unmerkleized`](super::Unmerkleized). pub struct CurrentUnmerkleized where F: Graftable, @@ -62,16 +61,15 @@ where Operation: Codec, { batch: UnmerkleizedBatch, - db: Shared>, + reader: Reader>, metadata: Option, } -/// Staged batch returned by [`CurrentUnmerkleized::stage`], wrapping a QMDB [`Staged`] with a -/// reference to the parent database. +/// Staged batch returned by [`CurrentUnmerkleized::stage`], wrapping a QMDB [`Staged`]. /// -/// Like any speculative batch, this handle is a branch-scoped view of the shared database: it -/// stays valid only while every batch finalized on the database is an ancestor of this batch -/// (see [`MerkleizedBatch`]'s branch-validity contract). +/// A branch-scoped view of the database. It stays valid only while every batch finalized on +/// the database is an ancestor of this batch (see [`MerkleizedBatch`]'s branch-validity +/// contract). pub struct CurrentStaged where F: Graftable, @@ -84,7 +82,7 @@ where Operation: Codec, { staged: Staged, - db: Shared>, + reader: Reader>, metadata: Option, } @@ -109,7 +107,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get(key, &db).await } @@ -117,7 +115,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get_many(keys, &db).await } @@ -130,18 +128,18 @@ where ) -> Result<(Vec>, CurrentStaged), Error> { let Self { batch, - db, + reader, metadata, } = self; let (values, staged) = { - let guard = db.read().await; + let guard = reader.read().await; batch.stage(keys, &guard).await? }; Ok(( values, CurrentStaged { staged, - db, + reader, metadata, }, )) @@ -154,8 +152,7 @@ where } } -/// Wraps a QMDB [`MerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Merkleized`](super::Merkleized) trait. +/// Wraps a QMDB [`MerkleizedBatch`] to implement [`Merkleized`](super::Merkleized). pub struct CurrentMerkleized where F: Graftable, @@ -168,7 +165,7 @@ where Operation: Codec, { inner: Arc>, - db: Shared>, + reader: Reader>, } impl Clone for CurrentMerkleized @@ -185,7 +182,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -257,11 +254,11 @@ where ) -> Result<(Range, Vec>, Self), Error> { let Self { staged, - db, + reader, metadata, } = self; let (range, values, staged) = { - let guard = db.read().await; + let guard = reader.read().await; staged.expand(keys, &guard).await? }; Ok(( @@ -269,7 +266,7 @@ where values, Self { staged, - db, + reader, metadata, }, )) @@ -292,7 +289,7 @@ where { /// Record updates for staged reads and upserts for unread keys, then merkleize. /// - /// Consumes the staged handle and write vectors. Call [`expand`](CurrentStaged::expand) + /// Consumes the staged batch and write vectors. Call [`expand`](CurrentStaged::expand) /// before this method if more keys must be read into the staged index space. /// /// A `Some` value is an upsert. `None` is a delete. Update indices refer to the staged read @@ -310,14 +307,14 @@ where ) -> Result, N, S>, Error> { let Self { staged, - db, + reader, metadata, } = self; let inner = { - let guard = db.read().await; + let guard = reader.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(CurrentMerkleized { inner, db }) + Ok(CurrentMerkleized { inner, reader }) } } @@ -337,7 +334,7 @@ where { /// Record updates for staged reads and upserts for unread keys, then merkleize. /// - /// Consumes the staged handle and write vectors. Call [`expand`](CurrentStaged::expand) + /// Consumes the staged batch and write vectors. Call [`expand`](CurrentStaged::expand) /// before this method if more keys must be read into the staged index space. /// /// A `Some` value is an upsert. `None` is a delete. Update indices refer to the staged read @@ -355,14 +352,14 @@ where ) -> Result, N, S>, Error> { let Self { staged, - db, + reader, metadata, } = self; let inner = { - let guard = db.read().await; + let guard = reader.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(CurrentMerkleized { inner, db }) + Ok(CurrentMerkleized { inner, reader }) } } @@ -380,7 +377,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get(key, &db).await } @@ -388,7 +385,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get_many(keys, &db).await } } @@ -411,11 +408,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(CurrentMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -438,11 +435,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(CurrentMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -454,12 +451,11 @@ where F: Graftable, E: Context, U: Update, - C: Mutable>, - I: UnorderedIndex> + 'static, + C: Contiguous>, + I: UnorderedIndex>, H: Hasher, S: Strategy, Operation: Codec, - CurrentUnmerkleized: UnmerkleizedTrait, { type Digest = H::Digest; type Unmerkleized = CurrentUnmerkleized; @@ -471,7 +467,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { CurrentUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, } } @@ -535,11 +531,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); CurrentUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -555,9 +551,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -648,11 +644,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); CurrentUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -668,9 +664,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -843,11 +839,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); CurrentUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -863,9 +859,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -965,11 +961,11 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let batch = reader.read().await.new_batch(); CurrentUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, } } @@ -985,9 +981,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -1217,6 +1213,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::stateful::db::{DatabaseSet, Single, split}; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_macros::boxed; use commonware_parallel::Sequential; @@ -1237,12 +1234,22 @@ mod tests { use commonware_utils::{NZU16, NZU64, NZUsize, non_empty_range}; use std::num::{NonZeroU16, NonZeroUsize}; - /// Finalize `batch` into `db` and wait for the deferred flush, boxing the future + /// A fresh batch over the set's single database. + async fn new_batch + 'static>( + db: &Single, + ) -> D::Unmerkleized { + as DatabaseSet>::new_batches(&db.readers()).await + } + + /// Finalize `batch` into `set` and wait for the deferred flush, boxing the future /// ([`ManagedDb::finalize`] embeds the database in its state machine). #[boxed] - async fn finalize>(db: D, batch: D::Merkleized) -> D { - let (db, _snapshot, sync) = D::finalize(db, batch).await.unwrap(); - sync.await.expect("finalize flush failed"); + async fn finalize + 'static>( + db: Single, + batch: D::Merkleized, + ) -> Single { + let (db, _, barrier) = DatabaseSet::finalize(db, batch).await; + assert!(barrier.durable().await, "finalize flush failed"); db } @@ -1351,8 +1358,8 @@ mod tests { assert_managed_db::(); assert_state_sync_db::>(); assert_state_sync_db::>(); - assert_database_set::>(); - assert_database_set::>(); + assert_database_set::>(); + assert_database_set::>(); } #[test] @@ -1362,14 +1369,13 @@ mod tests { let db = >::init(context.child("db"), config) .await .unwrap(); - let db = Shared::new("test", db); + let db = Single::from(db); let key = Sha256::hash(&[b"key"]); let value = Sha256::hash(&[b"value"]); let metadata = Sha256::hash(&[b"metadata"]); let missing = Sha256::hash(&[b"missing"]); - let batch = db - .new_batch_for_test::<_>() + let batch = new_batch(&db) .await .write(key, Some(value)) .with_metadata(metadata); @@ -1378,20 +1384,24 @@ mod tests { .unwrap(); let expected_root = merkleized.root(); - { - let (slot, database) = db.write().await; - slot.put(finalize::(database, merkleized).await); - } + let (db, snapshot, barrier) = DatabaseSet::finalize(db, merkleized).await; + assert!(barrier.durable().await, "finalize flush failed"); - let guard = db.read().await; - assert_eq!(guard.root(), expected_root); - assert_eq!(guard.get(&key).await.unwrap(), Some(value)); + let db = db.reader(); + assert_eq!(db.read().await.root(), expected_root); + assert_eq!(db.read().await.get(&key).await.unwrap(), Some(value)); + assert_eq!( + mmr::Location::new(snapshot.bounds().end), + db.read().await.bounds().end, + "captured snapshot must cover the applied batch", + ); - let proof = guard.exclusion_proof(&missing).await.unwrap(); + let db = db.read().await; + let proof = db.exclusion_proof(&missing).await.unwrap(); assert!(OrderedFixedDb::verify_exclusion_proof( &missing, &proof, - &guard.root(), + &db.root(), )); }); } @@ -1399,8 +1409,8 @@ mod tests { /// The glue staged wrapper (`CurrentUnmerkleized::stage` -> `CurrentStaged::expand` -> /// `CurrentStaged::merkleize`) must return the same values and root as an explicit `get_many` + /// `write` + `merkleize`, including a staged delete, an upsert, and metadata flow (both set - /// on the staged handle via `with_metadata` and carried from before staging). This guards - /// metadata flow and db-handle pairing through the wrapper. + /// on the staged batch via `with_metadata` and carried from before staging). This guards + /// metadata flow through the wrapper. #[test] fn ordered_fixed_staged_merkleize_matches_explicit_writes() { deterministic::Runner::default().start(|context| async move { @@ -1408,24 +1418,21 @@ mod tests { let db = >::init(context.child("db"), config) .await .unwrap(); - let db = Shared::new("test", db); + let db = Single::from(db); let key = |i: u64| Sha256::hash(&[&i.to_be_bytes()]); let val = |i: u64| Sha256::hash(&[&(i + 10_000).to_be_bytes()]); let metadata = Sha256::hash(&[b"metadata"]); // Seed keys 0..50 and finalize. - let mut seed = db.new_batch_for_test::<_>().await; + let mut seed = new_batch(&db).await; for i in 0..50u64 { seed = seed.write(key(i), Some(val(i))); } let merkleized = crate::stateful::db::Unmerkleized::merkleize(seed) .await .unwrap(); - { - let (slot, database) = db.write().await; - slot.put(finalize::(database, merkleized).await); - } + let db = finalize::(db, merkleized).await; // Read set: key(1) updated, key(2) deleted, key(999) missing -> created. let read_keys = [key(1), key(2), key(999)]; @@ -1434,7 +1441,7 @@ mod tests { let upserts = vec![(key(3), Some(val(1_002)))]; // Explicit path. - let mut explicit = db.new_batch_for_test::<_>().await; + let mut explicit = new_batch(&db).await; let explicit_values = explicit.get_many(&keys).await.unwrap(); for (slot, value) in &indexed_updates { explicit = explicit.write(read_keys[*slot], *value); @@ -1448,8 +1455,8 @@ mod tests { .unwrap() .root(); - // Staged path, with metadata set on the staged handle. - let staged_batch = db.new_batch_for_test::<_>().await; + // Staged path, with metadata set on the staged batch. + let staged_batch = new_batch(&db).await; let split = 2; let (mut staged_values, staged) = staged_batch.stage(&keys[..split]).await.unwrap(); let (range, suffix_values, staged) = staged.expand(&keys[split..]).await.unwrap(); @@ -1466,7 +1473,7 @@ mod tests { assert_eq!(explicit_root, staged_root); // Metadata set before staging must be carried through to staged merkleize. - let carried_batch = db.new_batch_for_test::<_>().await.with_metadata(metadata); + let carried_batch = new_batch(&db).await.with_metadata(metadata); let (carried_values, staged) = carried_batch.stage(&keys).await.unwrap(); let carried_root = staged .merkleize(indexed_updates.clone(), upserts.clone()) @@ -1485,14 +1492,13 @@ mod tests { let db = >::init(context.child("db"), config) .await .unwrap(); - let db = Shared::new("test", db); + let db = Single::from(db); let key = Sha256::hash(&[b"key"]); let value = Sha256::hash(&[b"value"]); let metadata = Sha256::hash(&[b"metadata"]); let missing = Sha256::hash(&[b"missing"]); - let batch = db - .new_batch_for_test::<_>() + let batch = new_batch(&db) .await .write(key, Some(value)) .with_metadata(metadata); @@ -1501,20 +1507,18 @@ mod tests { .unwrap(); let expected_root = merkleized.root(); - { - let (slot, database) = db.write().await; - slot.put(finalize::(database, merkleized).await); - } + let db = finalize::(db, merkleized).await; - let guard = db.read().await; - assert_eq!(guard.root(), expected_root); - assert_eq!(guard.get(&key).await.unwrap(), Some(value)); + let db = db.reader(); + assert_eq!(db.read().await.root(), expected_root); + assert_eq!(db.read().await.get(&key).await.unwrap(), Some(value)); - let proof = guard.exclusion_proof(&missing).await.unwrap(); + let db = db.read().await; + let proof = db.exclusion_proof(&missing).await.unwrap(); assert!(OrderedVariableDb::verify_exclusion_proof( &missing, &proof, - &guard.root(), + &db.root(), )); }); } @@ -1526,14 +1530,13 @@ mod tests { let db = >::init(context.child("db"), config.clone()) .await .unwrap(); - let db = Shared::new("test", db); + let (_writer, reader) = split(db); let key = Sha256::hash(&[b"key"]); let value = Sha256::hash(&[b"value"]); let metadata = Sha256::hash(&[b"metadata"]); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader) .await .write(key, Some(value)) .with_metadata(metadata); @@ -1581,59 +1584,35 @@ mod tests { let db = >::init(context.child("db"), config) .await .unwrap(); - let db = Shared::new("test", db); + let db = Single::from(db); let key1 = Sha256::hash(&[b"key1"]); let value1 = Sha256::hash(&[b"value1"]); let metadata1 = Sha256::hash(&[b"metadata1"]); - let batch1 = db - .new_batch_for_test::<_>() + let batch1 = new_batch(&db) .await .write(key1, Some(value1)) .with_metadata(metadata1); let merkleized1 = crate::stateful::db::Unmerkleized::merkleize(batch1) .await .unwrap(); - { - let (slot, database) = db.write().await; - slot.put(finalize::(database, merkleized1).await); - } - let target_after_first = { - let guard = db.read().await; - >::sync_target(&guard) - }; + let db = finalize::(db, merkleized1).await; + let target_after_first = db.committed_targets().await; let key2 = Sha256::hash(&[b"key2"]); let value2 = Sha256::hash(&[b"value2"]); let metadata2 = Sha256::hash(&[b"metadata2"]); - let batch2 = db - .new_batch_for_test::<_>() + let batch2 = new_batch(&db) .await .write(key2, Some(value2)) .with_metadata(metadata2); let merkleized2 = crate::stateful::db::Unmerkleized::merkleize(batch2) .await .unwrap(); - { - let (slot, database) = db.write().await; - slot.put(finalize::(database, merkleized2).await); - } + let db = finalize::(db, merkleized2).await; - { - let (slot, database) = db.write().await; - slot.put( - >::rewind_to_target( - database, - target_after_first.clone(), - ) - .await - .unwrap(), - ); - } - let target_after_rewind = { - let guard = db.read().await; - >::sync_target(&guard) - }; + let db = db.rewind_to_targets(target_after_first.clone()).await; + let target_after_rewind = db.committed_targets().await; assert_eq!(target_after_rewind, target_after_first); }); } @@ -1645,14 +1624,13 @@ mod tests { let db = FixedDb::init(context.child("db"), config.clone()) .await .unwrap(); - let db = Shared::new("test", db); + let (_writer, reader) = split(db); let key = Sha256::hash(&[b"key"]); let value = Sha256::hash(&[b"value"]); let metadata = Sha256::hash(&[b"metadata"]); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader) .await .write(key, Some(value)) .with_metadata(metadata); diff --git a/glue/src/stateful/db/immutable/compact.rs b/glue/src/stateful/db/immutable/compact.rs index 4d3902eccba..528d1bed2df 100644 --- a/glue/src/stateful/db/immutable/compact.rs +++ b/glue/src/stateful/db/immutable/compact.rs @@ -5,7 +5,7 @@ //! adapters expose set and merkleization operations but no historical reads. use crate::stateful::db::{ - BatchContext, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, SyncEngineConfig, + ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_compact_db, }; use commonware_codec::{EncodeShared, Read as CodecRead}; @@ -24,7 +24,7 @@ use commonware_storage::{ initial_root, variable, }, operation::Key, - sync::{self}, + sync, }, }; use commonware_utils::{Array, channel::mpsc}; @@ -44,7 +44,7 @@ where S: Strategy, { batch: CompactUnmerkleizedBatch, - db: Shared>, + reader: Reader>, metadata: Option, inactivity_floor: Location, } @@ -113,7 +113,7 @@ where S: Strategy, { inner: Arc>, - db: Shared>, + reader: Reader>, } impl Clone for ImmutableUnjournaledMerkleized @@ -131,7 +131,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -172,14 +172,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(ImmutableUnjournaledMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -206,7 +206,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { ImmutableUnjournaledUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -223,8 +223,8 @@ where S: Strategy, Operation>: EncodeShared + CodecRead, { - type Unmerkleized = ImmutableUnjournaledUnmerkleized, H, S, ()>; - type Merkleized = ImmutableUnjournaledMerkleized, H, S, ()>; + type Unmerkleized = ImmutableUnjournaledUnmerkleized, H, S>; + type Merkleized = ImmutableUnjournaledMerkleized, H, S>; type Error = Error; type Config = fixed::CompactConfig; type SyncTarget = sync::CompactTarget; @@ -241,13 +241,16 @@ where } } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; ImmutableUnjournaledUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -260,9 +263,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, handle)) + Ok((db, snapshot, sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -290,14 +293,15 @@ where } } -impl ManagedDb for variable::CompactDb +impl ManagedDb for variable::CompactDb where F: Family, E: Context, K: Key, V: VariableValue + 'static, H: Hasher + 'static, - Operation>: EncodeShared + CodecRead, + Operation>: EncodeShared, + Operation>: CodecRead, C: Clone + Send + Sync + 'static, S: Strategy, { @@ -319,13 +323,16 @@ where } } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; ImmutableUnjournaledUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -338,9 +345,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, handle)) + Ok((db, snapshot, sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -412,7 +419,8 @@ where K: Key, V: VariableValue + 'static, H: Hasher + 'static, - Operation>: EncodeShared + CodecRead, + Operation>: EncodeShared, + Operation>: CodecRead, C: Clone + Send + Sync + 'static, S: Strategy, R: sync::SourceFor, @@ -446,6 +454,7 @@ where #[cfg(test)] mod tests { use super::*; + use crate::stateful::db::{DatabaseSet, Single, SyncEngineConfig}; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_macros::select; use commonware_parallel::Sequential; @@ -458,7 +467,7 @@ mod tests { merkle::{full::Config as MerkleConfig, mmr}, translator::TwoCap, }; - use commonware_utils::{NZU16, NZU64, NZUsize}; + use commonware_utils::{NZU16, NZU64, NZUsize, channel::mpsc}; use futures::pin_mut; use std::time::Duration; @@ -580,13 +589,12 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config(&context, "managed-db"); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); let key = Sha256::hash(&[&[1]]); let value = Sha256::hash(&[&[2]]); let metadata = Sha256::hash(&[&[3]]); - let batch = db - .new_batch_for_test::<_>() + let db = Single::from(db); + let batch = >::new_batch(db.reader()) .await .set(key, value) .with_inactivity_floor(mmr::Location::new(1)) @@ -596,23 +604,23 @@ mod tests { .unwrap(); let expected_root = merkleized.root(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } + let (db, snapshot, barrier) = DatabaseSet::finalize(db, merkleized).await; + assert!(barrier.durable().await, "finalize flush failed"); - let guard = db.read().await; - assert_eq!(guard.root(), expected_root); - assert_eq!(guard.get_metadata(), Some(metadata)); + let db = db.reader(); + let db = db.read().await; + assert_eq!(db.root(), expected_root); + assert_eq!(db.get_metadata(), Some(metadata)); - let target = >::sync_target(&guard); - assert_eq!(target.root, guard.root()); + let target = >::sync_target(&db); + assert_eq!(target.root, db.root()); assert_eq!(target.size, mmr::Location::new(3)); + assert_eq!( + snapshot.root(), + expected_root, + "captured snapshot must carry the applied root", + ); + assert_eq!(snapshot.size(), mmr::Location::new(3)); }); } diff --git a/glue/src/stateful/db/immutable/standard.rs b/glue/src/stateful/db/immutable/standard.rs index 7b9954506f2..1d45913d5e7 100644 --- a/glue/src/stateful/db/immutable/standard.rs +++ b/glue/src/stateful/db/immutable/standard.rs @@ -2,12 +2,13 @@ //! [`immutable`](commonware_storage::qmdb::immutable) databases. //! //! Immutable databases support adding new keyed values but not updates or -//! deletions. The wrapper types here capture a [`Shared`] database handle -//! so the batch API can read through to applied state. +//! deletions. Keyed batch reads lease the database through the batch's +//! [`Reader`] because the immutable proof snapshot carries no keyed +//! index. use crate::stateful::db::{ - BatchContext, LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, - SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_standard_db, + LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, + Unmerkleized as UnmerkleizedTrait, sync_standard_db, }; use commonware_codec::{Codec, EncodeShared, Read as CodecRead}; use commonware_cryptography::Hasher; @@ -35,11 +36,11 @@ use commonware_storage::{ use commonware_utils::{Array, channel::mpsc, non_empty_range}; use std::{ops::Deref, sync::Arc}; -/// Shared handle to an immutable database. -type ImmutableDbHandle = Shared>; +/// Reader over the immutable database a wrapper batch reads through. +type ImmutableDbHandle = Reader>; -/// Wraps an immutable [`UnmerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Unmerkleized`](crate::stateful::db::Unmerkleized) trait. +/// Wraps an immutable [`UnmerkleizedBatch`] to implement +/// [`Unmerkleized`](crate::stateful::db::Unmerkleized). pub struct ImmutableUnmerkleized where F: Family, @@ -53,7 +54,7 @@ where Operation: EncodeShared, { batch: UnmerkleizedBatch, - db: ImmutableDbHandle, + reader: ImmutableDbHandle, metadata: Option, inactivity_floor: Location, } @@ -104,7 +105,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &K) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get(key, &db).await } @@ -112,7 +113,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&K]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get_many(keys, &db).await } @@ -123,8 +124,8 @@ where } } -/// Wraps an immutable [`MerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Merkleized`](crate::stateful::db::Merkleized) trait. +/// Wraps an immutable [`MerkleizedBatch`] to implement +/// [`Merkleized`](crate::stateful::db::Merkleized). pub struct ImmutableMerkleized where F: Family, @@ -138,7 +139,7 @@ where Operation: EncodeShared, { inner: Arc>, - db: ImmutableDbHandle, + reader: ImmutableDbHandle, } impl Clone for ImmutableMerkleized @@ -156,7 +157,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -194,7 +195,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &K) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get(key, &db).await } @@ -202,7 +203,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&K]) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get_many(keys, &db).await } } @@ -223,14 +224,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(ImmutableMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -257,7 +258,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { ImmutableUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -310,13 +311,16 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; ImmutableUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -331,9 +335,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -413,13 +417,16 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; ImmutableUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -434,9 +441,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/keyless/compact.rs b/glue/src/stateful/db/keyless/compact.rs index 106e4c83ab3..b5a4b0babc3 100644 --- a/glue/src/stateful/db/keyless/compact.rs +++ b/glue/src/stateful/db/keyless/compact.rs @@ -5,7 +5,7 @@ //! adapters expose append and merkleization operations but no historical reads. use crate::stateful::db::{ - BatchContext, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, SyncEngineConfig, + ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_compact_db, }; use commonware_codec::{EncodeShared, Read as CodecRead}; @@ -23,7 +23,7 @@ use commonware_storage::{ CompactDb, CompactMerkleizedBatch, CompactUnmerkleizedBatch, Operation, fixed, initial_root, variable, }, - sync::{self}, + sync, }, }; use commonware_utils::channel::mpsc; @@ -42,7 +42,7 @@ where S: Strategy, { batch: CompactUnmerkleizedBatch, - db: Shared>, + reader: Reader>, metadata: Option, inactivity_floor: Location, } @@ -108,7 +108,7 @@ where S: Strategy, { inner: Arc>, - db: Shared>, + reader: Reader>, } impl Clone for KeylessUnjournaledMerkleized @@ -125,7 +125,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -163,14 +163,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(KeylessUnjournaledMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -196,7 +196,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { KeylessUnjournaledUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -212,8 +212,8 @@ where S: Strategy, Operation>: EncodeShared + CodecRead, { - type Unmerkleized = KeylessUnjournaledUnmerkleized, H, S, ()>; - type Merkleized = KeylessUnjournaledMerkleized, H, S, ()>; + type Unmerkleized = KeylessUnjournaledUnmerkleized, H, S>; + type Merkleized = KeylessUnjournaledMerkleized, H, S>; type Error = Error; type Config = fixed::CompactConfig; type SyncTarget = sync::CompactTarget; @@ -230,13 +230,16 @@ where } } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; KeylessUnjournaledUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -249,9 +252,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, handle)) + Ok((db, snapshot, sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -279,13 +282,14 @@ where } } -impl ManagedDb for variable::CompactDb +impl ManagedDb for variable::CompactDb where F: Family, E: Context, V: VariableValue + 'static, H: Hasher + 'static, - Operation>: EncodeShared + CodecRead, + Operation>: EncodeShared, + Operation>: CodecRead, C: Clone + Send + Sync + 'static, S: Strategy, { @@ -307,13 +311,16 @@ where } } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; KeylessUnjournaledUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -326,9 +333,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, handle)) + Ok((db, snapshot, sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -398,7 +405,8 @@ where E: Context + Spawner, V: VariableValue + 'static, H: Hasher + 'static, - Operation>: EncodeShared + CodecRead, + Operation>: EncodeShared, + Operation>: CodecRead, C: Clone + Send + Sync + 'static, S: Strategy, R: sync::SourceFor, @@ -432,6 +440,10 @@ where #[cfg(test)] mod tests { use super::*; + use crate::stateful::{ + db::{SyncEngineConfig, split}, + tests::mocks::finalize, + }; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_macros::select; use commonware_parallel::Sequential; @@ -444,7 +456,7 @@ mod tests { merkle::{full::Config as MerkleConfig, mmr}, qmdb::keyless as storage_keyless, }; - use commonware_utils::{NZU16, NZU64, NZUsize, sequence::U64}; + use commonware_utils::{NZU16, NZU64, NZUsize, channel::mpsc, sequence::U64}; use futures::pin_mut; use std::time::Duration; @@ -575,10 +587,9 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config(&context, "managed-db"); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); + let (writer, reader) = split(db); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader.clone()) .await .append(U64::new(7)) .with_inactivity_floor(mmr::Location::new(1)) @@ -588,23 +599,22 @@ mod tests { .unwrap(); let expected_root = merkleized.root(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } + let (_writer, snapshot, durability) = finalize(writer, merkleized).await; + durability.await.expect("finalize flush failed"); - let guard = db.read().await; - assert_eq!(guard.root(), expected_root); - assert_eq!(guard.get_metadata(), Some(U64::new(9))); + let db = reader.read().await; + assert_eq!(db.root(), expected_root); + assert_eq!(db.get_metadata(), Some(U64::new(9))); - let target = >::sync_target(&guard); - assert_eq!(target.root, guard.root()); + let target = >::sync_target(&db); + assert_eq!(target.root, db.root()); assert_eq!(target.size, mmr::Location::new(3)); + assert_eq!( + snapshot.root(), + expected_root, + "captured snapshot must carry the applied root", + ); + assert_eq!(snapshot.size(), mmr::Location::new(3)); }); } @@ -613,10 +623,9 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config(&context, "matches-sync-target"); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); + let (_writer, reader) = split(db); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader) .await .append(U64::new(7)) .with_inactivity_floor(mmr::Location::new(1)) diff --git a/glue/src/stateful/db/keyless/standard.rs b/glue/src/stateful/db/keyless/standard.rs index c46def56216..e1d9a729c24 100644 --- a/glue/src/stateful/db/keyless/standard.rs +++ b/glue/src/stateful/db/keyless/standard.rs @@ -2,13 +2,12 @@ //! [`keyless`](commonware_storage::qmdb::keyless) databases. //! //! Keyless databases are append-only. Operations are addressed by -//! [`Location`] rather than by key. -//! The wrapper types here capture a [`Shared`] database handle so the batch API -//! can read through to applied state. +//! [`Location`] rather than by key. Positional batch reads lease the +//! database through the batch's [`Reader`]. use crate::stateful::db::{ - BatchContext, LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Shared, StateSyncDb, - SyncEngineConfig, Unmerkleized as UnmerkleizedTrait, sync_standard_db, + LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, + Unmerkleized as UnmerkleizedTrait, sync_standard_db, }; use commonware_codec::{EncodeShared, Read as CodecRead}; use commonware_cryptography::Hasher; @@ -34,8 +33,8 @@ use commonware_storage::{ use commonware_utils::{channel::mpsc, non_empty_range}; use std::{ops::Deref, sync::Arc}; -/// Wraps a keyless [`UnmerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Unmerkleized`](crate::stateful::db::Unmerkleized) trait. +/// Wraps a keyless [`UnmerkleizedBatch`] to implement +/// [`Unmerkleized`](crate::stateful::db::Unmerkleized). pub struct KeylessUnmerkleized where F: Family, @@ -47,7 +46,7 @@ where Operation: EncodeShared, { batch: UnmerkleizedBatch, - db: Shared>, + reader: Reader>, metadata: Option, inactivity_floor: Location, } @@ -94,7 +93,7 @@ where /// Read a value by location, falling back to applied state. pub async fn get(&self, location: Location) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get(location, &db).await } @@ -106,7 +105,7 @@ where &self, locations: &[Location], ) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.batch.get_many(locations, &db).await } @@ -117,8 +116,8 @@ where } } -/// Wraps a keyless [`MerkleizedBatch`] with a reference to the parent -/// database, implementing the [`Merkleized`](crate::stateful::db::Merkleized) trait. +/// Wraps a keyless [`MerkleizedBatch`] to implement +/// [`Merkleized`](crate::stateful::db::Merkleized). pub struct KeylessMerkleized where F: Family, @@ -130,7 +129,7 @@ where Operation: EncodeShared, { inner: Arc>, - db: Shared>, + reader: Reader>, } impl Clone for KeylessMerkleized @@ -146,7 +145,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - db: self.db.clone(), + reader: self.reader.clone(), } } } @@ -180,7 +179,7 @@ where { /// Read a value by location, falling back to applied state. pub async fn get(&self, location: Location) -> Result, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get(location, &db).await } @@ -192,7 +191,7 @@ where &self, locations: &[Location], ) -> Result>, Error> { - let db = self.db.read().await; + let db = self.reader.read().await; self.inner.get_many(locations, &db).await } } @@ -211,14 +210,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.db.read().await; + let db = self.reader.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(KeylessMerkleized { inner: merkleized, - db: self.db.clone(), + reader: self.reader.clone(), }) } } @@ -243,7 +242,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { KeylessUnmerkleized { batch: self.inner.new_batch::(), - db: self.db.clone(), + reader: self.reader.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -278,13 +277,16 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; KeylessUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -299,9 +301,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -374,13 +376,16 @@ where ) } - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized { - let (database, shared) = database.into_parts(); + async fn new_batch(reader: Reader) -> Self::Unmerkleized { + let (batch, inactivity_floor) = { + let db = reader.read().await; + (db.new_batch(), db.inactivity_floor_loc()) + }; KeylessUnmerkleized { - batch: database.new_batch(), - db: shared, + batch, + reader, metadata: None, - inactivity_floor: database.inactivity_floor_loc(), + inactivity_floor, } } @@ -395,9 +400,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, handle) = db.start_sync().await?; + let (db, sync) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), handle)) + Ok((db, Arc::new(snapshot), sync)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -503,14 +508,17 @@ where #[cfg(test)] mod tests { use super::*; + use crate::stateful::{db::split, tests::mocks::finalize}; use commonware_cryptography::Sha256; use commonware_parallel::Sequential; use commonware_runtime::{ BufferPooler, Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, }; use commonware_storage::{ - journal::contiguous::fixed::Config as FixedJournalConfig, - merkle::full::Config as MerkleConfig, mmr, qmdb::keyless as storage_keyless, + journal::contiguous::{Contiguous as _, fixed::Config as FixedJournalConfig}, + merkle::full::Config as MerkleConfig, + mmr, + qmdb::keyless as storage_keyless, }; use commonware_utils::{NZU16, NZU64, NZUsize, non_empty_range, sequence::U64}; use std::num::{NonZeroU16, NonZeroUsize}; @@ -563,10 +571,9 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config("stateful-keyless-managed-db", &context); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); + let (writer, reader) = split(db); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader.clone()) .await .append(U64::new(7)) .with_inactivity_floor(mmr::Location::new(1)) @@ -575,27 +582,25 @@ mod tests { .await .unwrap(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } - - let guard = db.read().await; + let (_writer, snapshot, durability) = finalize(writer, merkleized).await; + durability.await.expect("finalize flush failed"); + + let db = reader.read().await; assert_eq!( - guard.get(mmr::Location::new(1)).await.unwrap(), + db.get(mmr::Location::new(1)).await.unwrap(), Some(U64::new(7)) ); - assert_eq!(guard.get_metadata().await.unwrap(), Some(U64::new(9))); + assert_eq!(db.get_metadata().await.unwrap(), Some(U64::new(9))); - let target = >::sync_target(&guard); - assert_eq!(target.root, guard.root()); + let target = >::sync_target(&db); + assert_eq!(target.root, db.root()); assert_eq!(target.range.start(), mmr::Location::new(1)); assert_eq!(target.range.end(), mmr::Location::new(3)); + assert_eq!( + mmr::Location::new(snapshot.bounds().end), + *target.range.end(), + "captured snapshot must cover the applied batch", + ); }); } @@ -604,10 +609,9 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config("stateful-keyless-matches-sync-target", &context); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); + let (_writer, reader) = split(db); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader) .await .append(U64::new(7)) .with_inactivity_floor(mmr::Location::new(1)) @@ -658,43 +662,30 @@ mod tests { deterministic::Runner::default().start(|context| async move { let config = fixed_config("stateful-keyless-floor-carry", &context); let db = FixedDb::init(context.child("db"), config).await.unwrap(); - let db = Shared::new("test", db); + let (writer, reader) = split(db); - let batch = db - .new_batch_for_test::<_>() + let batch = >::new_batch(reader.clone()) .await .append(U64::new(7)) .with_inactivity_floor(mmr::Location::new(1)); let merkleized = crate::stateful::db::Unmerkleized::merkleize(batch) .await .unwrap(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } + let (writer, _, durability) = finalize(writer, merkleized).await; + durability.await.expect("finalize flush failed"); // A fresh batch without an explicit floor must commit at the raised // floor instead of regressing it to zero. - let batch = db.new_batch_for_test::<_>().await.append(U64::new(8)); + let batch = >::new_batch(reader.clone()) + .await + .append(U64::new(8)); let merkleized = crate::stateful::db::Unmerkleized::merkleize(batch) .await .unwrap(); let fork = MerkleizedTrait::new_batch(&merkleized); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } - let target = >::sync_target(&*db.read().await); + let (writer, _, durability) = finalize(writer, merkleized).await; + durability.await.expect("finalize flush failed"); + let target = >::sync_target(&*reader.read().await); assert_eq!(target.range.start(), mmr::Location::new(1)); // The same holds for a batch forked from a merkleized parent. @@ -702,16 +693,9 @@ mod tests { let merkleized = crate::stateful::db::Unmerkleized::merkleize(fork) .await .unwrap(); - { - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = - >::finalize(database, merkleized) - .await - .unwrap(); - slot.put(database); - sync.await.expect("finalize flush failed"); - } - let target = >::sync_target(&*db.read().await); + let (_writer, _, durability) = finalize(writer, merkleized).await; + durability.await.expect("finalize flush failed"); + let target = >::sync_target(&*reader.read().await); assert_eq!(target.range.start(), mmr::Location::new(1)); }); } diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index e8d143b724b..c3421dd8738 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -9,11 +9,34 @@ //! 1. [`Unmerkleized`]: mutable, in-progress batch (concrete types expose reads and writes). //! 2. [`Merkleized`]: a sealed batch with a computed root. //! 3. Finalization: apply the sealed batch and start persisting it via -//! [`ManagedDb::finalize`], observing durability via [`Barrier`]. +//! [`ManagedDb::finalize`], observing durability via [`Barrier`]. Finalize also +//! captures each database's serving snapshot, published for resolver serving (see +//! [`Subscriber`]) as soon as the apply completes. //! //! [`DatabaseSet`] groups one or more [`ManagedDb`] instances into one logical //! unit for execution and commit. //! +//! # Read access and mutation +//! +//! Each database is split ([`split`]) into two capabilities over one shared +//! cell. The set holds the only [`Writer`], which is not [`Clone`], so +//! mutation is uniquely permitted. Batches hold a [`Reader`], which is freely +//! cloned and grants a [`ReadGuard`] covering exactly one storage call. +//! +//! Because a lease never spans application code, a mutation waits at most one +//! storage call to start, and work holding a reader is never cancelled, so it +//! pauses at its next read and resumes once the mutation completes. A mutation +//! that is interrupted leaves the cell poisoned, which is reachable only while +//! the mutating task is being torn down. +//! +//! Two invariants keep this sound. A batch handed to [`ManagedDb::finalize`] +//! must not read through its own reader, because that call runs while the write +//! side is held. And a [`Writer`] must outlive the readers from the same cell. +//! Dropping the writer does not drop the database, so readers would otherwise +//! go on answering from a database that can never advance. Both invariants +//! hold structurally today, because the set holds the [`Writer`] and every +//! reader lives in a batch the set outlives. +//! //! # State Sync //! //! State sync orchestration is expressed by two traits: @@ -84,12 +107,9 @@ use commonware_macros::select; use commonware_runtime::{Error as RuntimeError, Handle, Metrics, Spawner, reschedule}; use commonware_storage::{ journal::{authenticated, contiguous::Snapshottable}, - qmdb::sync::{self, FeedbackTx, Request, Response, Source}, -}; -use commonware_utils::{ - channel::{fallible::AsyncFallibleExt, mpsc, oneshot, ring}, - sync::{AsyncRwLockReadGuard, AsyncRwLockWriteGuard, TracedAsyncRwLock}, + qmdb::sync, }; +use commonware_utils::channel::{fallible::AsyncFallibleExt, mpsc, oneshot, ring}; use futures::{ future::{Either, pending, try_join_all}, join, @@ -99,7 +119,6 @@ use std::{ fmt::Debug, future::Future, num::{NonZeroU64, NonZeroUsize}, - ops::Deref, sync::Arc, }; use tracing::debug; @@ -107,193 +126,23 @@ use tracing::debug; const MAX_CHANNEL_DRAIN_PER_TICK: usize = 32; pub mod any; +mod cell; pub mod current; pub mod immutable; pub mod keyless; pub mod p2p; mod snapshot; +pub use cell::{ReadGuard, Reader, Writer, split}; pub use snapshot::{Publisher, Subscriber}; -/// A database shared across tasks. -/// -/// Owned mutations (finalize, prune, rewind) take the database out of the cell under the -/// write lock ([Self::write]) and put it back on success ([WriteSlot::put]); a failure, -/// panic, or cancellation mid-operation leaves the cell empty permanently, and every -/// later [Self::read] or [Self::write] panics: a lost database is fatal here by design; -/// restart to recover. Serve calls instead report the source as missing, so remote -/// sync degrades without crashing. -pub struct Shared(Inner); - -/// The lock wrapped by [`Shared`]. Storage implements its sync source traits on -/// this shape, so [`Shared`]'s source impls delegate to it. -type Inner = Arc>>; - -impl Clone for Shared { - fn clone(&self) -> Self { - Self(self.0.clone()) - } -} - -/// Message used when a [`Shared`] cell is empty. -const DB_LOST_MSG: &str = - "database was lost by an earlier failed or interrupted operation; restart to recover"; - -impl Shared { - /// Create a cell holding `db`, identified by `label` in lock traces. - pub fn new(label: &'static str, db: DB) -> Self { - Self(Arc::new(TracedAsyncRwLock::new(label, Some(db)))) - } - - /// Acquire shared read access to the database. - /// - /// The lock is write-preferring: once a writer is queued, new readers wait - /// behind it. Holding a guard across an await that acquires this cell - /// again therefore deadlocks once a writer arrives in between. - /// - /// # Panics - /// - /// Panics if the database was lost by an earlier failed or interrupted mutation. - pub async fn read(&self) -> ReadGuard<'_, DB> { - ReadGuard(AsyncRwLockReadGuard::map(self.0.read().await, |db| { - db.as_ref().expect(DB_LOST_MSG) - })) - } - - /// Take the database out for a by-value mutation. - /// - /// The returned [`WriteSlot`] holds the cell locked and empty until - /// [`WriteSlot::put`] restores the database. Dropping the slot without a put - /// leaves the database lost. - /// - /// # Panics - /// - /// Panics if the database was lost by an earlier failed or interrupted mutation. - pub async fn write(&self) -> (WriteSlot<'_, DB>, DB) { - let mut guard = self.0.write().await; - let db = guard.take().expect(DB_LOST_MSG); - (WriteSlot(guard), db) - } - - async fn read_locked(&self) -> ReadLocked<'_, DB> { - ReadLocked { - database: self.0.read().await, - shared: self, - } - } - - #[cfg(test)] - async fn new_batch_for_test(&self) -> >::Unmerkleized - where - DB: ManagedDb, - { - let database = self.read_locked().await; - DB::new_batch(database.batch_context()) - } -} - -/// Read-only access to a database managed by [`Stateful`](super::Stateful). -/// -/// Unlike [`Shared`], this handle cannot acquire a write slot or construct and -/// finalize batches. Applications receive readers in -/// [`Application::finalized`](super::Application::finalized) so observing -/// finalized state cannot invalidate concurrent speculative batches. -/// -/// ```compile_fail -/// use commonware_glue::stateful::db::Reader; -/// -/// async fn mutate(reader: Reader) { -/// let _ = reader.write().await; -/// } -/// ``` -pub struct Reader(Shared); - -impl Reader { - /// Acquire shared read access to the database. - /// - /// The guard follows the same write-preferring lock discipline as - /// [`Shared::read`]. - pub async fn read(&self) -> ReadGuard<'_, DB> { - self.0.read().await - } -} - -/// Shared read access to a [`Shared`] database. -pub struct ReadGuard<'a, DB>(AsyncRwLockReadGuard<'a, DB>); - -impl Deref for ReadGuard<'_, DB> { - type Target = DB; - - fn deref(&self) -> &DB { - &self.0 - } -} - -/// An exclusively locked [`Shared`] cell whose database has been taken out. -pub struct WriteSlot<'a, DB>(AsyncRwLockWriteGuard<'a, Option>); - -impl WriteSlot<'_, DB> { - /// Restore the database, making it visible to other tasks again. - pub fn put(mut self, db: DB) { - *self.0 = Some(db); - } -} - -/// Origin-bound read access used to construct a database batch. -struct ReadLocked<'a, DB> { - database: AsyncRwLockReadGuard<'a, Option>, - shared: &'a Shared, -} - -impl ReadLocked<'_, DB> { - fn batch_context(&self) -> BatchContext<'_, DB> { - BatchContext { - database: self.database.as_ref().expect(DB_LOST_MSG), - shared: Shared::clone(self.shared), - } - } -} - -/// Origin-bound database access for synchronous batch construction. -/// -/// Only [`DatabaseSet`] can create this capability. Its borrow prevents a -/// [`ManagedDb`] implementation from retaining the set's read lock in the -/// returned batch. -pub struct BatchContext<'a, DB> { - database: &'a DB, - shared: Shared, -} - -impl<'a, DB> BatchContext<'a, DB> { - /// Split the capability into applied state and its matching shared handle. - pub fn into_parts(self) -> (&'a DB, Shared) { - (self.database, self.shared) - } -} - -impl Source for Shared -where - DB: Send + Sync + 'static, - Inner: Source, -{ - type Family = as Source>::Family; - type Digest = as Source>::Digest; - type Op = as Source>::Op; - type Error = as Source>::Error; - - async fn serve( - &self, - request: Request, - ) -> Result<(Response, FeedbackTx), Self::Error> { - self.0.serve(request).await - } -} - /// Mutable batch state before merkleization. /// /// Concrete types provide key-value operations (`get`, `write`, `set`, /// `append`, etc.) as inherent methods; the generic wrapper only needs -/// [`merkleize`](Self::merkleize). +/// [`merkleize`](Self::merkleize). Batches carry a [`Reader`] to the +/// database they were created from, so every operation reads the right +/// database and no operation can delay a mutation by more than one call. pub trait Unmerkleized: Sized + Send { /// The merkleized batch produced by [`merkleize`](Self::merkleize). type Merkleized: Merkleized; @@ -310,12 +159,16 @@ pub trait Unmerkleized: Sized + Send { /// /// The application uses [`root`](Self::root) in block headers, and the wrapper /// later finalizes this batch. -pub trait Merkleized: Sized + Send + Sync { +/// +/// Implementations are handles over shared batch state, so [`Clone`] is cheap. +/// The wrapper relies on it to keep a block forkable while that same block is +/// being applied. +pub trait Merkleized: Clone + Sized + Send + Sync { /// The digest type used for the state root. type Digest: Digest; /// The unmerkleized batch type produced by [`new_batch`](Self::new_batch). - type Unmerkleized: Unmerkleized; + type Unmerkleized: Send; /// The canonical state root committed in block headers. fn root(&self) -> Self::Digest; @@ -332,8 +185,8 @@ pub trait Merkleized: Sized + Send + Sync { /// Implementations create new batches from applied state and apply finalized /// batches back to storage, deferring each batch's flush to a returned handle. /// -/// [`new_batch`](Self::new_batch) consumes origin-bound read access so batch -/// types can snapshot applied state and retain the matching [`Shared`] handle. +/// Batches carry a [`Reader`] to their database. Reads acquire a short +/// lease per call and fall back from pending batch state to applied state. /// /// `E` is a trait generic (not an associated type), so one database type can /// work across runtimes that satisfy the bounds. @@ -343,18 +196,18 @@ pub trait Merkleized: Sized + Send + Sync { /// Mutating methods take the database by value and return it on success. If a mutating /// method returns an error, or its future is dropped before it finishes, the database is /// gone: state that was not yet durable is discarded, but everything already on disk stays -/// recoverable. +/// recoverable. The cell is left poisoned, so later reads park +/// rather than observe a database that is missing. pub trait ManagedDb: Send + Sync + Sized { /// An in-progress batch of mutations that has not yet been merkleized. - type Unmerkleized: Unmerkleized; + type Unmerkleized: Unmerkleized; /// A batch whose root has been computed but has not yet been applied to /// the underlying database. /// /// Constrained so that [`Merkleized::new_batch`] produces the same /// [`Unmerkleized`] type as [`ManagedDb::new_batch`](Self::new_batch). - /// Cloning must preserve the same sealed branch state and should be cheap. - type Merkleized: Clone + Merkleized; + type Merkleized: Merkleized; /// The error type returned by fallible operations. type Error: Debug + Send; @@ -367,7 +220,9 @@ pub trait ManagedDb: Send + Sync + Sized { /// Typically a database-specific state commitment plus the operation range needed to reach it. type SyncTarget: Clone + PartialEq + Send + Sync; - /// Owned immutable snapshot of applied state. + /// Owned immutable snapshot of the applied operation log and its proof + /// state, for serving state sync. Not a queryable key-value view -- only the + /// live database answers key reads. type Snapshot: Clone + Send + Sync + 'static; /// Construct a new database from its configuration. @@ -381,13 +236,12 @@ pub trait ManagedDb: Send + Sync + Sized { /// This must match [`sync_target`](Self::sync_target) after opening an empty partition. fn initial_sync_target() -> Self::SyncTarget; - /// Create a new unmerkleized batch rooted at the read-locked database's - /// applied state. + /// Create a new unmerkleized batch rooted at the database's applied + /// state. /// - /// This method must return without retaining `database`, releasing its read - /// lock before the batch performs any lazy read-through work. Batch types - /// can retain the matching handle returned by [`BatchContext::into_parts`]. - fn new_batch(database: BatchContext<'_, Self>) -> Self::Unmerkleized; + /// The batch keeps `reader` and leases the database through it on every + /// read, so it stays valid across applies of compatible batches. + fn new_batch(reader: Reader) -> impl Future + Send; /// Return true if a merkleized batch matches a sync target. fn matches_sync_target(batch: &Self::Merkleized, target: &Self::SyncTarget) -> bool; @@ -515,44 +369,37 @@ impl Barrier { } } -/// A collection of individually locked [`ManagedDb`] instances. -/// -/// Each database is wrapped in [`Shared`], so the set is cheap to -/// clone and each database can be shared without a global lock. -/// Multi-database mutations must not hold one member's writer while waiting -/// to acquire another. Readers may span members in the opposite order. +/// A collection of [`ManagedDb`] instances. /// -/// `E` is a trait generic (not an associated type), so one set type can work -/// across runtimes that satisfy the bounds. -pub trait DatabaseSet: Clone + Send + Sync + 'static { - /// Tuple of [`ManagedDb::Unmerkleized`] for every database in the set. +/// Every method treats a database error as fatal and panics. Deferred flush +/// failures surface later, through [`Barrier`]. +pub trait DatabaseSet: Send + Sync + Sized + 'static { + /// One [`ManagedDb::Unmerkleized`] per database, scalar under [`Single`] and a tuple + /// for tuple sets. type Unmerkleized: Send; - /// Tuple of [`ManagedDb::Merkleized`] for every database in the set. - /// Cloning must preserve the same sealed branch state and should be cheap. + /// One [`ManagedDb::Merkleized`] per database, shaped like [`Self::Unmerkleized`]. + /// + /// [`Clone`] is cheap (see [`Merkleized`]) and the wrapper uses it to keep a + /// block forkable while that block is being applied. type Merkleized: Clone + Send + Sync; - /// Read-only handles for observing the applied database state. + /// One [`Reader`] per database, shaped like [`Self::Unmerkleized`]. /// - /// Implementations must not expose mutation capabilities through this - /// type. In particular, readers must not construct, finalize, prune, or - /// rewind database batches. - type Readers: Send; + /// Readers are cloned into batches, and hooks that read applied state + /// directly acquire leases through them. + type Readers: Clone + Send + Sync + 'static; /// One [`ManagedDb::Snapshot`] per database, shaped like [`Self::Unmerkleized`]. type Snapshots: Send + Sync + 'static; - /// Configuration needed to construct every database in the set. - /// - /// - Single database sets use that database's [`ManagedDb::Config`]. - /// - Multi-database tuple sets use a tuple of per-database configs - /// `(Db1::Config, Db2::Config, ...)`. + /// Configuration needed to construct every database in the set -- the database's + /// [`ManagedDb::Config`] under [`Single`], a tuple of per-database configs for + /// tuple sets. type Config: Send; - /// Per-database sync targets extracted from a finalized block. - /// - /// For a single-database set this is one target. For multi-database sets it is a tuple of - /// targets, one per database. + /// Per-database sync targets extracted from a finalized block, shaped like + /// [`Self::Config`]. type SyncTargets: Clone + PartialEq + Send + Sync; /// Construct the database set from its configuration. @@ -561,73 +408,164 @@ pub trait DatabaseSet: Clone + Send + Sync + 'static { /// Return the sync targets produced by a newly initialized database set. fn initial_sync_targets() -> Self::SyncTargets; + /// Readers over the set's databases. + fn readers(&self) -> Self::Readers; + /// Create unmerkleized batches from each database's applied state. /// - /// Implementations must release every read lock before the returned - /// batches perform lazy reads. - fn new_batches(&self) -> impl Future + Send; + /// Takes readers rather than `&self` because verification jobs build + /// batches without holding the set. Methods that do take the set are the + /// ones only its owner calls. + fn new_batches(readers: &Self::Readers) -> impl Future + Send; /// Create child unmerkleized batches from a pending merkleized parent. /// - /// No lock is needed; reads come from the in-memory merkleized state. + /// Reads come from the in-memory merkleized state. fn fork_batches(parent: &Self::Merkleized) -> Self::Unmerkleized; - /// Return true if merkleized batches match the sync targets. + /// Return true if merkleized batches match the given sync targets. fn matches_sync_targets(batches: &Self::Merkleized, targets: &Self::SyncTargets) -> bool; - /// Return read-only handles for every database in the set. - fn readers(&self) -> Self::Readers; + /// Return sync targets for the set's current applied state. + /// + /// Applied state may be ahead of durable state while a flush is pending. + fn committed_targets(&self) -> impl Future + Send; /// Apply each merkleized batch's changeset to its underlying database, capture each /// database's snapshot, and begin persisting them. /// /// Returns once every database reflects its batch. Every returned /// [`Barrier`] must be awaited (see [`Barrier`]). - /// - /// Cancelling the future mid-flight loses the databases whose mutations - /// were in progress (see [Shared]); every later access panics. fn finalize( - &self, + self, batches: Self::Merkleized, - ) -> impl Future + Send; + ) -> impl Future + Send; /// Capture a snapshot of every database's current applied state. /// /// A snapshot can include state that is not yet durably persisted. - /// - /// Cancelling the future mid-flight loses the databases whose mutations - /// were in progress (see [Shared]); every later access panics. - fn snapshot(&self) -> impl Future + Send; + fn snapshot(self) -> impl Future + Send; - /// Prune each database to the provided per-database targets. - /// - /// The finalized state represented by `targets` must already be durable. Barriers for later - /// finalized state may still be pending, so implementations must coordinate pruning with those - /// in-flight writes. Pruning effects must be durable before this call returns. - /// - /// Cancelling the future mid-flight loses the databases whose mutations - /// were in progress (see [Shared]); every later access panics. - fn prune(&self, targets: &Self::SyncTargets) -> impl Future + Send; - - /// Return sync targets for the set's current applied state. - fn committed_targets(&self) -> impl Future + Send; + /// Prune each database to the provided per-database targets (see + /// [`ManagedDb::prune`] for the durability contract). + fn prune(self, targets: &Self::SyncTargets) -> impl Future + Send; /// Rewind the set to the provided per-database targets. - /// - /// Rewind failures are fatal for startup recovery and therefore panic. - /// - /// Cancelling the future mid-flight loses the databases whose mutations - /// were in progress (see [Shared]); every later access panics. - fn rewind_to_targets(&self, targets: Self::SyncTargets) -> impl Future + Send; + fn rewind_to_targets(self, targets: Self::SyncTargets) -> impl Future + Send; } /// The snapshot a log-backed QMDB database produces, shared for serving. pub type LogSnapshot = Arc::Reader, H>>; +/// The unmerkleized batches of a [`DatabaseSet`]. +pub type UnmerkleizedOf = >::Unmerkleized; + +/// The merkleized batches of a [`DatabaseSet`]. +pub type MerkleizedOf = >::Merkleized; + /// The snapshot set a [`DatabaseSet`] captures for publication. pub type SnapshotsOf = >::Snapshots; +/// The sync targets of a [`DatabaseSet`]. +pub type SyncTargetsOf = >::SyncTargets; + +/// The readers of a [`DatabaseSet`]. +pub type ReadersOf = >::Readers; + +/// A one-database set. +/// +/// The database lives in a cell. The set holds the sole [`Writer`] and derives +/// [`Reader`]s from it. +pub struct Single { + writer: Writer, +} + +impl From for Single { + fn from(database: T) -> Self { + Self { + writer: Writer::new(database), + } + } +} + +impl Single { + /// A reader over the set's database. + /// + /// Same reader as [`DatabaseSet::readers`], reachable without naming the + /// runtime the set is used with. + pub fn reader(&self) -> Reader { + self.writer.reader() + } +} + +impl DatabaseSet for Single +where + E: Send + Sync + Metrics, + T: ManagedDb + 'static, +{ + type Unmerkleized = T::Unmerkleized; + type Merkleized = T::Merkleized; + type Readers = Reader; + type Snapshots = T::Snapshot; + type Config = T::Config; + type SyncTargets = T::SyncTarget; + + async fn init(context: E, config: Self::Config) -> Self { + match T::init(context.child("db"), config).await { + Ok(database) => Self::from(database), + Err(err) => panic!( + "database init failed (type {}): {err:?}", + core::any::type_name::(), + ), + } + } + + fn initial_sync_targets() -> Self::SyncTargets { + T::initial_sync_target() + } + + fn readers(&self) -> Self::Readers { + self.reader() + } + + async fn new_batches(readers: &Self::Readers) -> Self::Unmerkleized { + T::new_batch(readers.clone()).await + } + + fn fork_batches(parent: &Self::Merkleized) -> Self::Unmerkleized { + parent.new_batch() + } + + fn matches_sync_targets(batches: &Self::Merkleized, targets: &Self::SyncTargets) -> bool { + T::matches_sync_target(batches, targets) + } + + async fn committed_targets(&self) -> Self::SyncTargets { + self.reader().read().await.sync_target() + } + + async fn finalize(self, batches: Self::Merkleized) -> (Self, Self::Snapshots, Barrier) { + let (member, snapshot, sync) = finalize_or_panic(self, batches, None).await; + let barrier = Barrier { + syncs: vec![(core::any::type_name::(), None, sync)], + }; + (member, snapshot, barrier) + } + + async fn snapshot(self) -> (Self, Self::Snapshots) { + snapshot_or_panic(self, None).await + } + + async fn prune(self, targets: &Self::SyncTargets) -> Self { + prune_or_panic(self, targets, None).await + } + + async fn rewind_to_targets(self, targets: Self::SyncTargets) -> Self { + rewind_or_panic(self, targets, None).await + } +} + /// Parameters for a one-time state-sync pass. #[derive(Clone, Copy, Debug)] pub struct SyncEngineConfig { @@ -762,72 +700,7 @@ where ) -> impl Future), Self::Error>> + Send; } -/// Implement [`DatabaseSet`] for a single [`ManagedDb`] behind a lock. -impl + 'static> DatabaseSet for Shared { - type Unmerkleized = T::Unmerkleized; - type Merkleized = T::Merkleized; - type Readers = Reader; - type Snapshots = T::Snapshot; - type Config = T::Config; - type SyncTargets = T::SyncTarget; - - async fn init(context: E, config: Self::Config) -> Self { - let db = T::init(context, config) - .await - .expect("database init failed"); - Self::new("stateful.db", db) - } - - fn initial_sync_targets() -> Self::SyncTargets { - T::initial_sync_target() - } - - async fn new_batches(&self) -> Self::Unmerkleized { - let database = self.read_locked().await; - T::new_batch(database.batch_context()) - } - - fn fork_batches(parent: &Self::Merkleized) -> Self::Unmerkleized { - parent.new_batch() - } - - fn matches_sync_targets(batches: &Self::Merkleized, targets: &Self::SyncTargets) -> bool { - T::matches_sync_target(batches, targets) - } - - fn readers(&self) -> Self::Readers { - Reader(self.clone()) - } - - async fn finalize(&self, batches: Self::Merkleized) -> (Self::Snapshots, Barrier) { - let (snapshot, handle) = finalize_shared_or_panic::(self, batches, None).await; - ( - snapshot, - Barrier { - syncs: vec![(core::any::type_name::(), None, handle)], - }, - ) - } - - async fn snapshot(&self) -> Self::Snapshots { - snapshot_shared_or_panic::(self, None).await - } - - async fn prune(&self, target: &Self::SyncTargets) { - prune_shared_or_panic::(self, target, None).await; - } - - async fn committed_targets(&self) -> Self::SyncTargets { - let database = self.read().await; - T::sync_target(&database) - } - - async fn rewind_to_targets(&self, target: Self::SyncTargets) { - rewind_shared_or_panic::(self, target, None).await; - } -} - -impl StateSyncSet for Shared +impl StateSyncSet for Single where E: Send + Sync + Metrics, T: StateSyncDb + 'static, @@ -934,7 +807,7 @@ where T::sync_target(&database) == converged_target, "state sync database target does not match the coordinator target", ); - Ok((Self::new("stateful.db", database), converged_anchor)) + Ok((Self::from(database), converged_anchor)) } } @@ -988,12 +861,11 @@ where target_tx.send_lossy(new_target).await } -/// Implement [`DatabaseSet`] for a tuple of individually-locked -/// [`ManagedDb`] instances. +/// Implement [`DatabaseSet`] for a tuple of single-database sets. macro_rules! impl_database_set { ($($T:ident : $idx:tt),+) => { impl + 'static),+> DatabaseSet - for ($(Shared<$T>,)+) + for ($(Single<$T>,)+) { type Unmerkleized = ($($T::Unmerkleized,)+); type Merkleized = ($($T::Merkleized,)+); @@ -1003,9 +875,10 @@ macro_rules! impl_database_set { type SyncTargets = ($($T::SyncTarget,)+); async fn init(context: E, config: Self::Config) -> Self { - let result = join!($( + join!($( async { - let db = $T::init( + Single::from( + $T::init( context.child(concat!("db_", stringify!($idx))), config.$idx, ) @@ -1016,20 +889,24 @@ macro_rules! impl_database_set { ", type ", stringify!($T), ")", - )); - Shared::new(concat!("stateful.db.", stringify!($idx)), db) + )), + ) }, - )+); - result + )+) } fn initial_sync_targets() -> Self::SyncTargets { ($($T::initial_sync_target(),)+) } - async fn new_batches(&self) -> Self::Unmerkleized { - let databases = join!($(self.$idx.read_locked(),)+); - ($($T::new_batch(databases.$idx.batch_context()),)+) + fn readers(&self) -> Self::Readers { + ($(self.$idx.reader(),)+) + } + + async fn new_batches(readers: &Self::Readers) -> Self::Unmerkleized { + join!($( + $T::new_batch(readers.$idx.clone()), + )+) } fn fork_batches(parent: &Self::Merkleized) -> Self::Unmerkleized { @@ -1040,70 +917,52 @@ macro_rules! impl_database_set { $($T::matches_sync_target(&batches.$idx, &targets.$idx))&&+ } - fn readers(&self) -> Self::Readers { - ($(Reader(self.$idx.clone()),)+) + async fn committed_targets(&self) -> Self::SyncTargets { + join!($( + async { self.$idx.reader().read().await.sync_target() }, + )+) } - async fn finalize( - &self, - batches: Self::Merkleized, - ) -> (Self::Snapshots, Barrier) { - // Each member completes its own write-lock lifecycle. Holding - // a partial tuple of writers can deadlock cross-database reads. - let results = join!($(finalize_shared_or_panic::( - &self.$idx, - batches.$idx, - Some($idx), - ),)+); - let snapshots = ($(results.$idx.0,)+); + async fn finalize(self, batches: Self::Merkleized) -> (Self, Self::Snapshots, Barrier) { + // Every database captures at its own apply boundary inside this call, so the + // captured snapshots form one capture. + let results = join!($( + finalize_or_panic(self.$idx, batches.$idx, Some($idx)), + )+); let barrier = Barrier { - syncs: vec![$(( - core::any::type_name::<$T>(), - Some($idx), - results.$idx.1, - ),)+], + syncs: vec![$( + (core::any::type_name::<$T>(), Some($idx), results.$idx.2), + )+], }; - (snapshots, barrier) + ( + ($(results.$idx.0,)+), + ($(results.$idx.1,)+), + barrier, + ) } - async fn snapshot(&self) -> Self::Snapshots { - join!($(snapshot_shared_or_panic::( - &self.$idx, - Some($idx), - ),)+) + async fn snapshot(self) -> (Self, Self::Snapshots) { + let results = join!($( + snapshot_or_panic(self.$idx, Some($idx)), + )+); + (($(results.$idx.0,)+), ($(results.$idx.1,)+)) } - async fn prune( - &self, - targets: &Self::SyncTargets, - ) { - join!($(prune_shared_or_panic::( - &self.$idx, - &targets.$idx, - Some($idx), - ),)+); + async fn prune(self, targets: &Self::SyncTargets) -> Self { + join!($( + prune_or_panic(self.$idx, &targets.$idx, Some($idx)), + )+) } - async fn committed_targets(&self) -> Self::SyncTargets { - let databases = join!($(self.$idx.read(),)+); - ($($T::sync_target(&databases.$idx),)+) - } - - async fn rewind_to_targets( - &self, - targets: Self::SyncTargets, - ) { - join!($(rewind_shared_or_panic::( - &self.$idx, - targets.$idx, - Some($idx), - ),)+); + async fn rewind_to_targets(self, targets: Self::SyncTargets) -> Self { + join!($( + rewind_or_panic(self.$idx, targets.$idx, Some($idx)), + )+) } } }; } -impl_database_set!(DB1: 0); impl_database_set!(DB1: 0, DB2: 1); impl_database_set!(DB1: 0, DB2: 1, DB3: 2); impl_database_set!(DB1: 0, DB2: 1, DB3: 2, DB4: 3); @@ -1150,7 +1009,7 @@ struct CoordinatorSyncSenders { macro_rules! impl_state_sync_set { ($($T:ident : $R:ident : $idx:tt),+) => { - impl StateSyncSet for ($(Shared<$T>,)+) + impl StateSyncSet for ($(Single<$T>,)+) where E: Send + Sync + Spawner + Metrics + 'static, D: Digest + 'static, @@ -1397,20 +1256,13 @@ macro_rules! impl_state_sync_set { } }; let (sync_result, _) = join!(sync, forward_reached); - let result = sync_result - .map(|database| { - Shared::new( - concat!("stateful.db.", stringify!($idx)), - database, - ) - }) - .map_err(|err| { - format!( - "state sync failed (index {}, db {}): {err:?}", - $idx, - core::any::type_name::<$T>(), - ) - }); + let result = sync_result.map_err(|err| { + format!( + "state sync failed (index {}, db {}): {err:?}", + $idx, + core::any::type_name::<$T>(), + ) + }); if let Err(err) = &result { let mut first = first_db_error.lock(); if first.is_none() { @@ -1441,13 +1293,11 @@ macro_rules! impl_state_sync_set { return Err(err); } - let synced = ($(synced.$idx?,)+); + let synced = ($(Single::from(synced.$idx?),)+); let Some((converged_anchor, converged_targets)) = converged_anchor else { return Err("state sync coordinator did not report a converged anchor".into()); }; - let committed_targets = - >::committed_targets(&synced).await; - if committed_targets != converged_targets { + if >::committed_targets(&synced).await != converged_targets { return Err( "state sync database targets do not match the coordinator target set" .into(), @@ -1837,125 +1687,116 @@ where result } +#[tracing::instrument(name = "stateful.db.snapshot_or_panic", level = "info", skip_all, fields(index = index))] +async fn snapshot_or_panic>( + member: Single, + index: Option, +) -> (Single, T::Snapshot) { + let Single { writer } = member; + let (writer, snapshot) = writer + .mutate(|database| async move { + match database.snapshot().await { + Ok(result) => result, + Err(err) => { + let index = index.map_or(String::new(), |i| format!("index {i}, ")); + panic!( + "database snapshot capture failed ({index}type {}): {err:?}", + core::any::type_name::(), + ); + } + } + }) + .await; + (Single { writer }, snapshot) +} + #[tracing::instrument(name = "stateful.db.finalize_or_panic", level = "info", skip_all, fields(index = index))] async fn finalize_or_panic>( - database: T, + member: Single, batch: T::Merkleized, index: Option, -) -> (T, T::Snapshot, Handle<()>) { +) -> (Single, T::Snapshot, Handle<()>) { // Mutable finalize failures are fatal by design because the batch may already have been // applied to other databases in the same set, leaving partially applied state. - match database.finalize(batch).await { - Ok(result) => result, - Err(err) => { - let index = index.map_or(String::new(), |i| format!("index {i}, ")); - panic!( - "database finalize failed ({index}type {}): {err:?}", - core::any::type_name::(), - ); - } - } -} - -async fn finalize_shared_or_panic>( - shared: &Shared, - batch: T::Merkleized, - index: Option, -) -> (T::Snapshot, Handle<()>) { - let (slot, database) = shared.write().await; - let (database, snapshot, handle) = finalize_or_panic(database, batch, index).await; - slot.put(database); - (snapshot, handle) -} - -#[tracing::instrument(name = "stateful.db.snapshot_or_panic", level = "info", skip_all, fields(index = index))] -async fn snapshot_shared_or_panic>( - shared: &Shared, - index: Option, -) -> T::Snapshot { - let (slot, database) = shared.write().await; - let (database, snapshot) = match database.snapshot().await { - Ok(result) => result, - Err(err) => { - let index = index.map_or(String::new(), |i| format!("index {i}, ")); - panic!( - "database snapshot capture failed ({index}type {}): {err:?}", - core::any::type_name::(), - ); - } - }; - slot.put(database); - snapshot -} - -async fn prune_shared_or_panic>( - shared: &Shared, - target: &T::SyncTarget, - index: Option, -) { - let (slot, database) = shared.write().await; - slot.put(prune_or_panic(database, target, index).await); -} - -async fn rewind_shared_or_panic>( - shared: &Shared, - target: T::SyncTarget, - index: Option, -) { - let (slot, database) = shared.write().await; - if T::sync_target(&database) == target { - slot.put(database); - return; - } - slot.put(rewind_or_panic(database, target, index).await); + let Single { writer } = member; + let (writer, (snapshot, sync)) = writer + .mutate(|database| async move { + match database.finalize(batch).await { + Ok((database, snapshot, sync)) => (database, (snapshot, sync)), + Err(err) => { + let index = index.map_or(String::new(), |i| format!("index {i}, ")); + panic!( + "database finalize failed ({index}type {}): {err:?}", + core::any::type_name::(), + ); + } + } + }) + .await; + (Single { writer }, snapshot, sync) } #[tracing::instrument(name = "stateful.db.rewind_or_panic", level = "info", skip_all, fields(index = index))] async fn rewind_or_panic>( - database: T, + member: Single, target: T::SyncTarget, index: Option, -) -> T { - // Mutable rewind failures are fatal by design because the database handle +) -> Single { + // Mutable rewind failures are fatal by design because the database // may be internally diverged after a failed rewind. - match database.rewind_to_target(target).await { - Ok(database) => database, - Err(err) => { - let index = index.map_or(String::new(), |i| format!("index {i}, ")); - panic!( - "database rewind failed ({index}type {}): {err:?}", - core::any::type_name::(), - ); - } - } + let Single { writer } = member; + let (writer, ()) = writer + .mutate(|database| async move { + if T::sync_target(&database) == target { + return (database, ()); + } + match database.rewind_to_target(target).await { + Ok(database) => (database, ()), + Err(err) => { + let index = index.map_or(String::new(), |i| format!("index {i}, ")); + panic!( + "database rewind failed ({index}type {}): {err:?}", + core::any::type_name::(), + ); + } + } + }) + .await; + Single { writer } } #[tracing::instrument(name = "stateful.db.prune_or_panic", level = "info", skip_all, fields(index = index))] async fn prune_or_panic>( - database: T, + member: Single, target: &T::SyncTarget, index: Option, -) -> T { +) -> Single { // Prune failures are fatal because pruning may already have discarded part // of the retained history before the error surfaced. - match database.prune(target).await { - Ok(database) => database, - Err(err) => { - let index = index.map_or(String::new(), |i| format!("index {i}, ")); - panic!( - "database prune failed ({index}type {}): {err:?}", - core::any::type_name::(), - ); - } - } + let Single { writer } = member; + let (writer, ()) = writer + .mutate(|database| async move { + match database.prune(target).await { + Ok(database) => (database, ()), + Err(err) => { + let index = index.map_or(String::new(), |i| format!("index {i}, ")); + panic!( + "database prune failed ({index}type {}): {err:?}", + core::any::type_name::(), + ); + } + } + }) + .await; + Single { writer } } #[cfg(test)] mod tests { use super::{ - Anchor, Barrier, BatchContext, CoordinatorAction, CoordinatorState, DatabaseSet, - MAX_CHANNEL_DRAIN_PER_TICK, ManagedDb, Shared, StateSyncDb, StateSyncSet, SyncEngineConfig, - TipUpdate, drain_single_tip_updates, + Anchor, Barrier, CoordinatorAction, CoordinatorState, DatabaseSet, + MAX_CHANNEL_DRAIN_PER_TICK, ManagedDb, Reader, Single, StateSyncDb, StateSyncSet, + SyncEngineConfig, TipUpdate, Writer, drain_single_tip_updates, split, }; use crate::stateful::tests::mocks::{TestMerkleized, TestUnmerkleized, anchor as mock_anchor}; use commonware_cryptography::sha256; @@ -1980,12 +1821,12 @@ mod tests { }; mod managed_db_lifecycle { - use super::{ManagedDb, Shared}; + use super::{ManagedDb, Writer, split}; use crate::stateful::db::Unmerkleized; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_parallel::Sequential; use commonware_runtime::{ - Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, + Handle, Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, }; use commonware_storage::{ journal::contiguous::{ @@ -2296,26 +2137,39 @@ mod tests { } } + /// Finalize `batch` through the cell, returning the snapshot and flush handle. + async fn finalize>( + writer: Writer, + batch: T::Merkleized, + ) -> (Writer, T::Snapshot, Handle<()>) { + let (writer, (snapshot, sync)) = writer + .mutate(|db| async move { + let (db, snapshot, sync) = T::finalize(db, batch).await.unwrap_or_else(|err| { + panic!("finalize failed: {err:?}"); + }); + (db, (snapshot, sync)) + }) + .await; + (writer, snapshot, sync) + } + async fn assert_initial_sync_target_and_finalize(context: Context, config: T::Config) where T: ManagedDb + 'static, - T::Unmerkleized: Unmerkleized, ::Error: Debug, T::SyncTarget: Debug, { let initial = T::initial_sync_target(); let db = T::init(context, config).await.unwrap(); assert_eq!(initial, db.sync_target()); - let db = Shared::new("test", db); - let batch = db - .new_batch_for_test::() + let (writer, reader) = split(db); + let batch = T::new_batch(reader) .await .merkleize() .await .expect("empty batch must merkleize"); - let (slot, database) = db.write().await; - let (database, _snapshot, sync) = T::finalize(database, batch).await.unwrap(); - slot.put(database); + let (_writer, snapshot, sync) = finalize(writer, batch).await; + drop(snapshot); sync.await.expect("empty batch finalize flush failed"); } @@ -2366,7 +2220,6 @@ mod tests { #[case] config: fn(&Context, &str) -> T::Config, ) where T: ManagedDb + 'static, - T::Unmerkleized: Unmerkleized, ::Error: Debug, T::SyncTarget: Debug, { @@ -2375,6 +2228,104 @@ mod tests { assert_initial_sync_target_and_finalize::(context.child("db"), config).await; }); } + + async fn assert_rewind_restores_finalized_target(context: Context, config: T::Config) + where + T: ManagedDb + 'static, + ::Error: Debug, + T::SyncTarget: Debug, + { + let db = T::init(context, config).await.unwrap(); + let (writer, reader) = split(db); + let batch = T::new_batch(reader.clone()) + .await + .merkleize() + .await + .expect("first batch must merkleize"); + let (writer, snapshot, sync) = finalize(writer, batch).await; + drop(snapshot); + sync.await.expect("first finalize flush failed"); + let target = reader.read().await.sync_target(); + + let batch = T::new_batch(reader.clone()) + .await + .merkleize() + .await + .expect("second batch must merkleize"); + let (writer, snapshot, sync) = finalize(writer, batch).await; + drop(snapshot); + sync.await.expect("second finalize flush failed"); + + let (_writer, ()) = writer + .mutate(|db| { + let target = target.clone(); + async move { + let db = db + .rewind_to_target(target) + .await + .unwrap_or_else(|_| panic!("rewind to finalized target failed")); + (db, ()) + } + }) + .await; + assert_eq!(reader.read().await.sync_target(), target); + } + + #[rstest] + #[case::any_fixed(PhantomData::, any_fixed_config)] + #[case::any_variable(PhantomData::, any_variable_config)] + #[case::current_unordered_fixed( + PhantomData::, + current_fixed_config + )] + #[case::current_ordered_fixed( + PhantomData::, + current_fixed_config + )] + #[case::current_unordered_variable( + PhantomData::, + current_variable_config + )] + #[case::current_ordered_variable( + PhantomData::, + current_variable_config + )] + #[case::immutable_fixed(PhantomData::, immutable_fixed_config)] + #[case::immutable_variable( + PhantomData::, + immutable_variable_config + )] + #[case::immutable_compact_fixed( + PhantomData::, + immutable_compact_fixed_config + )] + #[case::immutable_compact_variable( + PhantomData::, + immutable_compact_variable_config + )] + #[case::keyless_fixed(PhantomData::, keyless_fixed_config)] + #[case::keyless_variable(PhantomData::, keyless_variable_config)] + #[case::keyless_compact_fixed( + PhantomData::, + keyless_compact_fixed_config + )] + #[case::keyless_compact_variable( + PhantomData::, + keyless_compact_variable_config + )] + fn rewind_restores_earlier_finalized_sync_target( + #[case] _db: PhantomData, + #[case] config: fn(&Context, &str) -> T::Config, + ) where + T: ManagedDb + 'static, + ::Error: Debug, + T::SyncTarget: Debug, + { + deterministic::Runner::default().start(|context| async move { + let config = config(&context, "db"); + assert_rewind_restores_finalized_target::(context.child("db"), config).await; + }); + } } macro_rules! ready_finalize { @@ -2418,7 +2369,7 @@ mod tests { Ok(Self) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2455,7 +2406,7 @@ mod tests { unreachable!("CountingRewindDb is constructed directly in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2494,7 +2445,7 @@ mod tests { Ok(Self { prune_count }) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2604,7 +2555,7 @@ mod tests { Ok(Self) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2626,37 +2577,75 @@ mod tests { } } + struct FailingSnapshotDb; + + impl ManagedDb for FailingSnapshotDb { + type Unmerkleized = TestUnmerkleized; + type Merkleized = TestMerkleized; + type Error = TestFinalizeError; + type Config = (); + type SyncTarget = (); + type Snapshot = (); + + async fn snapshot(self) -> Result<(Self, Self::Snapshot), Self::Error> { + Err(TestFinalizeError) + } + + fn initial_sync_target() -> Self::SyncTarget {} + + async fn init(_context: E, _config: Self::Config) -> Result { + Ok(Self) + } + + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { + TestUnmerkleized + } + + fn matches_sync_target(_batch: &Self::Merkleized, _target: &Self::SyncTarget) -> bool { + true + } + + async fn finalize( + self, + _batch: Self::Merkleized, + ) -> Result<(Self, Self::Snapshot, Handle<()>), Self::Error> { + Ok((self, (), Handle::ready(Ok(())))) + } + + fn sync_target(&self) -> Self::SyncTarget {} + + async fn rewind_to_target(self, _target: Self::SyncTarget) -> Result { + Ok(self) + } + } + #[test] fn tuple_rewind_to_targets_skips_already_aligned_databases() { deterministic::Runner::default().start(|_context| async move { - type DbSet = (Shared, Shared); + type RewindPair = (Single, Single); - let left = Shared::new( - "test", - CountingRewindDb { - current_target: 2, - rewind_count: 0, - }, - ); - let right = Shared::new( - "test", - CountingRewindDb { - current_target: 1, - rewind_count: 0, - }, - ); - let databases: DbSet = (left.clone(), right.clone()); + let left = CountingRewindDb { + current_target: 2, + rewind_count: 0, + }; + let right = CountingRewindDb { + current_target: 1, + rewind_count: 0, + }; - >::rewind_to_targets(&databases, (1, 1)) - .await; + let databases: RewindPair = (Single::from(left), Single::from(right)); + let databases = >::rewind_to_targets( + databases, + (1, 1), + ) + .await; - let left = left.read().await; - assert_eq!(left.current_target, 1); - assert_eq!(left.rewind_count, 1); + let (left, right) = (databases.0.reader(), databases.1.reader()); + assert_eq!(left.read().await.current_target, 1); + assert_eq!(left.read().await.rewind_count, 1); - let right = right.read().await; - assert_eq!(right.current_target, 1); - assert_eq!(right.rewind_count, 0); + assert_eq!(right.read().await.current_target, 1); + assert_eq!(right.read().await.rewind_count, 0); }); } @@ -2664,15 +2653,11 @@ mod tests { fn database_set_prune_calls_managed_db_prune() { deterministic::Runner::default().start(|_context| async move { let prune_count = Arc::new(AtomicUsize::new(0)); - let database = Shared::new( - "test", - PruneCountingDb { - prune_count: prune_count.clone(), - }, - ); + let database = Single::from(PruneCountingDb { + prune_count: prune_count.clone(), + }); - as DatabaseSet>::prune(&database, &()) - .await; + let _database = DatabaseSet::::prune(database, &()).await; assert_eq!(prune_count.load(Ordering::SeqCst), 1); }); @@ -2696,7 +2681,7 @@ mod tests { unreachable!("BlockingFinalizeDb is constructed directly in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2744,7 +2729,7 @@ mod tests { unreachable!("SlowSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2787,7 +2772,7 @@ mod tests { ) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2826,7 +2811,7 @@ mod tests { unreachable!("FastSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2865,7 +2850,7 @@ mod tests { unreachable!("FailingStateSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2904,7 +2889,7 @@ mod tests { unreachable!("MismatchedTargetSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2943,7 +2928,7 @@ mod tests { unreachable!("ImmediateStateSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -2982,7 +2967,7 @@ mod tests { unreachable!("FinishClosedSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -3021,7 +3006,7 @@ mod tests { unreachable!("ObservedSlowSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -3060,7 +3045,7 @@ mod tests { unreachable!("ObservedFastSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -3103,7 +3088,7 @@ mod tests { ) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -3251,7 +3236,7 @@ mod tests { unreachable!("StaleReachedSyncDb is only constructed through state sync in tests") } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -3690,85 +3675,6 @@ mod tests { } } - #[test] - fn tuple_new_batches_queues_reads_concurrently() { - deterministic::Runner::default().start(|_context| async move { - let db1 = Shared::new("test", TestDb); - let db2 = Shared::new("test", TestDb); - let databases = (db1.clone(), db2.clone()); - - let (slot1, taken1) = db1.write().await; - let (slot2, taken2) = db2.write().await; - - let new_batches = <(Shared, Shared) as DatabaseSet< - deterministic::Context, - >>::new_batches(&databases); - pin_mut!(new_batches); - assert!(new_batches.as_mut().now_or_never().is_none()); - - slot2.put(taken2); - { - let writer2_again = db2.write(); - pin_mut!(writer2_again); - assert!( - writer2_again.as_mut().now_or_never().is_none(), - "tuple new_batches should queue reads for all databases concurrently" - ); - } - - slot1.put(taken1); - let _ = new_batches.await; - }); - } - - #[test] - fn new_batches_releases_read_lock_before_returning() { - deterministic::Runner::default().start(|_context| async move { - let database = Shared::new("test", TestDb); - let _ = as DatabaseSet>::new_batches(&database) - .await; - - let writer = database.write(); - pin_mut!(writer); - assert!( - writer.as_mut().now_or_never().is_some(), - "batch construction must release its read lock before returning", - ); - }); - } - - #[test] - fn tuple_finalize_does_not_hold_ready_writer_while_waiting_for_reader() { - deterministic::Runner::default().start(|_context| async move { - type DbSet = (Shared, Shared); - - let db1 = Shared::new("test", TestDb); - let db2 = Shared::new("test", TestDb); - let databases = (db1.clone(), db2.clone()); - let reader1 = db1.read().await; - - let finalize = async { - >::finalize( - &databases, - (TestMerkleized, TestMerkleized), - ) - .await - }; - pin_mut!(finalize); - assert!(finalize.as_mut().now_or_never().is_none()); - - let reader2 = db2.read(); - pin_mut!(reader2); - assert!( - reader2.as_mut().now_or_never().is_some(), - "finalization must not hold one writer while waiting for another database's reader", - ); - - drop(reader1); - assert!(finalize.await.1.durable().await); - }); - } - #[test] fn tuple_finalize_runs_databases_in_parallel() { deterministic::Runner::default().start(|_context| async move { @@ -3778,14 +3684,14 @@ mod tests { let (release2_tx, release2_rx) = oneshot::channel(); let databases = ( - Shared::new("test", BlockingFinalizeDb::new(started1_tx, release1_rx)), - Shared::new("test", BlockingFinalizeDb::new(started2_tx, release2_rx)), + Single::from(BlockingFinalizeDb::new(started1_tx, release1_rx)), + Single::from(BlockingFinalizeDb::new(started2_tx, release2_rx)), ); - let finalize = - <(Shared, Shared) as DatabaseSet< - deterministic::Context, - >>::finalize(&databases, (TestMerkleized, TestMerkleized)); + let finalize = DatabaseSet::::finalize( + databases, + (TestMerkleized, TestMerkleized), + ); pin_mut!(finalize); assert!(finalize.as_mut().now_or_never().is_none()); @@ -3801,7 +3707,8 @@ mod tests { let _ = release1_tx.send(()); let _ = release2_tx.send(()); - assert!(finalize.await.1.durable().await); + let (_, _, barrier) = finalize.await; + assert!(barrier.durable().await); }); } @@ -3811,17 +3718,26 @@ mod tests { )] fn tuple_finalize_panic_identifies_failing_database() { deterministic::Runner::default().start(|_context| async move { - let databases = ( - Shared::new("test", TestDb), - Shared::new("test", FailingFinalizeDb), - ); - let _ = <(Shared, Shared) as DatabaseSet< - deterministic::Context, - >>::finalize(&databases, (TestMerkleized, TestMerkleized)) + let databases = (Single::from(TestDb), Single::from(FailingFinalizeDb)); + let _ = DatabaseSet::::finalize( + databases, + (TestMerkleized, TestMerkleized), + ) .await; }); } + #[test] + #[should_panic( + expected = "database snapshot capture failed (index 1, type commonware_glue::stateful::db::tests::FailingSnapshotDb)" + )] + fn tuple_snapshot_panic_identifies_failing_database() { + deterministic::Runner::default().start(|_context| async move { + let databases = (Single::from(TestDb), Single::from(FailingSnapshotDb)); + let _ = DatabaseSet::::snapshot(databases).await; + }); + } + #[test] #[should_panic( expected = "database finalize flush failed (index 1, type commonware_glue::stateful::db::tests::TestDb)" @@ -3957,7 +3873,7 @@ mod tests { let sync = context.child("single_state_sync_closed_tip_updates").spawn( move |context| async move { - as StateSyncSet< + as StateSyncSet< deterministic::Context, Arc, sha256::Digest, @@ -3996,7 +3912,7 @@ mod tests { let (mut tip_tx, tip_rx) = ring::channel(NonZeroUsize::new(1).unwrap()); let _ = tip_tx.send(TipUpdate::new(anchor(1), 1u64)).await; - let result = as StateSyncSet< + let result = as StateSyncSet< deterministic::Context, (), sha256::Digest, @@ -4033,7 +3949,7 @@ mod tests { let sync = context .child("single_state_sync_ignores_backward_tip_updates") .spawn(move |context| async move { - as StateSyncSet< + as StateSyncSet< deterministic::Context, SlowSyncController, sha256::Digest, @@ -4061,7 +3977,7 @@ mod tests { drop(tip_tx); let (database, converged_anchor) = sync.await.expect("sync task should complete"); - let final_target = database.read().await.final_target; + let final_target = database.reader().read().await.final_target; assert_eq!( final_target, 2, "single-db sync target must never move backward" @@ -4083,7 +3999,7 @@ mod tests { let sync = context.child("single_state_sync_noop_target_update").spawn( move |context| async move { - as StateSyncSet< + as StateSyncSet< deterministic::Context, Arc, sha256::Digest, @@ -4119,7 +4035,7 @@ mod tests { drop(tip_tx); let (database, converged_anchor) = sync.await.expect("sync task should complete"); - assert_eq!(database.read().await.final_target, 7); + assert_eq!(database.reader().read().await.final_target, 7); assert_eq!(converged_anchor, anchor(9)); }); } @@ -4133,7 +4049,7 @@ mod tests { context .child("single_state_sync_stale_reached") .spawn(move |context| async move { - as StateSyncSet< + as StateSyncSet< deterministic::Context, (), sha256::Digest, @@ -4159,7 +4075,7 @@ mod tests { let _ = tip_tx.send(TipUpdate::new(anchor(2), 2)).await; let (database, converged_anchor) = sync.await.expect("sync task should complete"); - let final_target = database.read().await.final_target; + let final_target = database.reader().read().await.final_target; assert_eq!( final_target, 2, "single-db sync must not finish on a stale reached target", @@ -4184,7 +4100,7 @@ mod tests { let sync = context .child("tuple_state_sync") .spawn(move |context| async move { - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, (Arc, Arc), sha256::Digest, @@ -4216,8 +4132,8 @@ mod tests { drop(tip_tx); let (synced, converged_anchor) = sync.await.expect("sync task should complete"); - let slow_target = synced.0.read().await.final_target; - let fast_target = synced.1.read().await.final_target; + let slow_target = synced.0.reader().read().await.final_target; + let fast_target = synced.1.reader().read().await.final_target; assert_eq!( slow_target, fast_target, @@ -4243,7 +4159,7 @@ mod tests { let sync = context .child("tuple_state_sync_ignores_backward_tip_updates") .spawn(move |context| async move { - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, (Arc, Arc), sha256::Digest, @@ -4277,8 +4193,8 @@ mod tests { slow_release.store(true, Ordering::SeqCst); let (synced, converged_anchor) = sync.await.expect("sync task should complete"); - let slow_target = synced.0.read().await.final_target; - let fast_target = synced.1.read().await.final_target; + let slow_target = synced.0.reader().read().await.final_target; + let fast_target = synced.1.reader().read().await.final_target; assert_eq!( slow_target, 2, "slow database target must never move backward" @@ -4301,7 +4217,7 @@ mod tests { let (_tip_tx, tip_rx) = ring::channel(NonZeroUsize::new(1).unwrap()); let fast_done = Arc::new(AtomicBool::new(false)); - let result = <(Shared, Shared) as StateSyncSet< + let result = <(Single, Single) as StateSyncSet< deterministic::Context, ((), Arc), sha256::Digest, @@ -4339,7 +4255,7 @@ mod tests { let (_tip_tx, tip_rx) = ring::channel(NonZeroUsize::new(1).unwrap()); let result = - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, ((), ()), sha256::Digest, @@ -4381,7 +4297,7 @@ mod tests { let (_tip_tx, tip_rx) = ring::channel(NonZeroUsize::new(1).unwrap()); let release = Arc::new(AtomicBool::new(true)); - let result = <(Shared, Shared) as StateSyncSet< + let result = <(Single, Single) as StateSyncSet< deterministic::Context, (Arc, ()), sha256::Digest, @@ -4423,7 +4339,7 @@ mod tests { let (_tip_tx, tip_rx) = ring::channel(NonZeroUsize::new(1).unwrap()); let result = - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, ((), ()), sha256::Digest, @@ -4450,11 +4366,11 @@ mod tests { }; assert!( err.contains("state sync failed (index 1, db"), - "error should include failing database index, got: {err}", + "error should include failing database index: {err}" ); assert!( err.contains("FailingStateSyncDb"), - "error should include failing database type, got: {err}", + "error should include failing database type: {err}" ); }); } @@ -4554,8 +4470,8 @@ mod tests { let sync = context.child("tuple_state_sync_algorithm").spawn( move |context| async move { <( - Shared, - Shared, + Single, + Single, ) as StateSyncSet< deterministic::Context, (SlowSyncController, FastSyncObserver), @@ -4590,8 +4506,8 @@ mod tests { drop(tip_tx); let (synced, converged_anchor) = sync.await.expect("sync task should complete"); - let slow_target = synced.0.read().await.final_target; - let fast_target = synced.1.read().await.final_target; + let slow_target = synced.0.reader().read().await.final_target; + let fast_target = synced.1.reader().read().await.final_target; assert_eq!( slow_target, fast_target, @@ -4625,7 +4541,7 @@ mod tests { update_count: fast_update_count.clone(), }; move |context| async move { - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, (Arc, FastSyncObserver), sha256::Digest, @@ -4657,8 +4573,8 @@ mod tests { slow_release.store(true, Ordering::SeqCst); let (synced, converged_anchor) = sync.await.expect("sync task should complete"); - let slow_target = synced.0.read().await.final_target; - let fast_target = synced.1.read().await.final_target; + let slow_target = synced.0.reader().read().await.final_target; + let fast_target = synced.1.reader().read().await.final_target; assert_eq!(slow_target, target); assert_eq!(fast_target, target); @@ -4688,7 +4604,7 @@ mod tests { update_count: fast_update_count.clone(), }; move |context| async move { - <(Shared, Shared) as StateSyncSet< + <(Single, Single) as StateSyncSet< deterministic::Context, (Arc, FastSyncObserver), sha256::Digest, @@ -4722,8 +4638,8 @@ mod tests { drop(tip_tx); let (synced, converged_anchor) = sync.await.expect("sync task should complete"); - let slow_target = synced.0.read().await.final_target; - let fast_target = synced.1.read().await.final_target; + let slow_target = synced.0.reader().read().await.final_target; + let fast_target = synced.1.reader().read().await.final_target; assert_eq!(slow_target, 9); assert_eq!(fast_target, 7); diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index bf7ff0f62ef..b0bc9c363aa 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -6,7 +6,7 @@ //! bookkeeping: //! //! 1. Before each `propose` or `verify`, the actor forks unmerkleized batches -//! from the parent block's pending state (or from committed database state +//! from the parent block's pending state (or from applied database state //! if the parent has been finalized). //! 2. The application executes against those batches and returns merkleized //! results, which the actor stores as a new pending tip keyed by the @@ -96,9 +96,11 @@ use commonware_consensus::{CertifiableBlock, Epochable, Viewable, marshal::ancestry::Ancestry}; use commonware_cryptography::certificate::Scheme; use commonware_runtime::{Clock, Metrics, Spawner}; -use db::DatabaseSet; +use commonware_storage::{merkle::Family, qmdb}; +use db::{DatabaseSet, MerkleizedOf, ReadersOf, UnmerkleizedOf}; use rand_core::Rng; use std::future::Future; +use thiserror::Error; mod actor; pub use actor::{Config, Mailbox, PruneConfig, Stateful, SyncPlan}; @@ -109,6 +111,33 @@ pub mod probe; #[cfg(test)] mod tests; +/// Why a batch operation failed during block execution. +/// +/// Implementations of [`Application`] propagate storage errors from batch operations +/// with `?` and never interpret them. The wrapper is the only layer that knows what +/// each case means for the block being executed. +#[derive(Debug, Error)] +pub enum ExecutionError { + /// Applied state left the executing block's branch because a competing block was + /// finalized mid-execution and the batches refused their next read. The wrapper + /// re-checks the block against the new canonical state. + #[error("stale execution: a competing block was finalized")] + Stale, + /// Any other storage failure. Storage errors are unrecoverable, so the + /// wrapper panics on this everywhere. Only `Ok(None)` declines a proposal. + #[error("storage failure: {0}")] + Fatal(String), +} + +impl From> for ExecutionError { + fn from(err: qmdb::Error) -> Self { + match err { + qmdb::Error::StaleRead => Self::Stale, + err => Self::Fatal(err.to_string()), + } + } +} + /// The output of a successful [`Application::propose`] call. pub struct Proposed, E: Rng + Spawner + Metrics + Clock> { /// The block built by the application. @@ -141,14 +170,19 @@ pub struct Input { /// return [`DatabaseSet::Merkleized`] batches after execution. The surrounding /// wrapper handles persistence: storing merkleized batches as pending tips on /// the block tree and applying changesets to the underlying databases on -/// finalization. +/// finalization. Every execution method reads through `batches`, which is the +/// only database access an implementor is given. A batch overlays speculative +/// ancestor state and falls back to applied state for anything it does not +/// cover, so it is always the complete view for its branch. /// /// [`Stateful`] may freely clone the application and invoke its methods -/// concurrently. Implementors should treat `Application` as a stateless, -/// deterministic state machine: given the same method inputs and database -/// state, every clone must produce the same state-transition result. Mutable -/// state that affects those results must live in the database batches provided -/// to proposal, verification, and replay methods. +/// concurrently. Any method may run on a fresh clone that is discarded +/// afterward, so cloning must be cheap and state an implementor wants to keep +/// must live behind shared handles. Implementors should treat `Application` as +/// a stateless, deterministic state machine. Given the same method inputs and +/// database state, every clone must produce the same state-transition result. +/// Mutable state that affects those results must live in the database batches +/// provided to proposal, verification, and replay methods. pub trait Application: Clone + Send + 'static where E: Rng + Spawner + Metrics + Clock, @@ -212,7 +246,7 @@ where /// result is cached as pending state. If the implementor produces a /// block with mismatched targets, this function will panic. /// - /// Applications using [`qmdb::current`](commonware_storage::qmdb::current) + /// Applications using [`qmdb::current`] /// must still ensure the proposed block commits to the merkleized batch's /// canonical root. The wrapper's sync-target check only verifies the ops /// root and operation range used by replay sync. @@ -220,13 +254,16 @@ where /// This future may be cancelled by consensus if the caller drops its /// response receiver. Implementations should be cancellation-safe: dropping /// and retrying must not violate invariants or lose durable progress. + /// + /// Storage errors from batch operations are propagated as [`ExecutionError`], + /// never interpreted. The wrapper declines the proposal on any error. fn propose( &mut self, context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, input: Input, - ) -> impl Future>> + Send; + ) -> impl Future>, ExecutionError>> + Send; /// Verify a block received from a peer, relative to its ancestry. /// @@ -237,15 +274,14 @@ where /// stable verdict. Return [`None`] only when the block is permanently /// invalid for the supplied context, ancestry, and batches. If validity may /// still change as additional information becomes available, continue - /// waiting instead of returning [`None`]. - /// - /// Validity is relative to those inputs: finalizing a competing branch - /// later does not retroactively change a completed verdict. + /// waiting instead of returning [`None`]. In other words, to abstain from + /// voting, do not resolve this future yet. Abstaining is not represented by + /// a special return value. /// - /// In other words, to abstain from voting, do not resolve this future yet. - /// Keep it pending until the implementation can either prove the block - /// valid, prove it invalid, or the consensus engine cancels the request. - /// Abstaining is not represented by a special return value. + /// Validity is relative to the supplied inputs. Finalizing a competing + /// branch later does not retroactively change a completed verdict. A + /// verdict reached after that finalization is reported as invalid instead, + /// because its branch is no longer reachable. /// /// Verification must reject any block whose execution result does not /// match the block's committed state (for example, a state root mismatch). @@ -254,31 +290,30 @@ where /// this by checking that any returned merkleized state matches the block /// before it is cached as pending state. /// - /// Applications using [`qmdb::current`](commonware_storage::qmdb::current) + /// Applications using [`qmdb::current`] /// must still reject blocks whose committed canonical root differs from the /// merkleized batch root. The wrapper's sync-target check only verifies the /// ops root and operation range used by replay sync. /// - /// This future is scoped to its caller. Stateful may also cancel and retry - /// it before finalization or pruning. Cancellation and retry must not - /// violate invariants or lose durable progress. + /// This future is scoped to its caller. Dropping the response cancels only + /// this request. The wrapper never cancels it, so a batch operation running + /// when a finalized block is applied waits for that apply and then + /// continues. /// - /// Verification may overlap finalization while its batches remain valid. - /// Stateful retries or rejects requests that cannot safely overlap it. - /// Read through the provided batches without holding the database set's - /// locks. Batches are branch-scoped views rather than historical - /// snapshots. Retained ancestor overlays preserve same-branch state, and - /// unresolved reads answer from committed state only while applied state - /// advances along the batch's own branch. Once a competing branch is - /// applied, reads refuse with a `StaleRead` error instead of consulting state - /// the branch never accounted for (see [`db::Shared::read`] for guard - /// discipline). + /// `batches` is a branch-scoped view, not a historical snapshot. Retained + /// ancestor overlays preserve same-branch state, while unresolved reads fall + /// through to the batch's own database. Once a block from a competing branch + /// is finalized, every batch operation refuses with a stale error instead of + /// answering across branches. Implementations propagate storage errors with + /// `?` as [`ExecutionError`] and never interpret them. On + /// [`ExecutionError::Stale`] the wrapper re-checks the block against the new + /// canonical state and retries or answers from there. fn verify( &mut self, context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> impl Future>::Merkleized>> + Send; + batches: UnmerkleizedOf, + ) -> impl Future>, ExecutionError>> + Send; /// Apply a previously certified block to reconstruct its merkleized state. /// @@ -292,20 +327,20 @@ where /// replay result during finalization and cannot re-check block-specific /// commitments generically. /// - /// This future may be cancelled if its originating request is dropped, or - /// cancelled and retried before finalization or pruning. Cancellation and - /// retry must not violate invariants or lose durable progress. - /// - /// # Panics + /// This future may be cancelled if its originating request is dropped. The + /// wrapper itself never cancels it. Cancellation must not violate + /// invariants or lose durable progress. /// - /// Implementations should panic if execution fails, as this indicates - /// data corruption or non-determinism. + /// Storage errors from batch operations are propagated as [`ExecutionError`], + /// never interpreted (see [`verify`](Self::verify)). The wrapper re-checks + /// canonical state when a verification replay goes stale and panics when the + /// failure is impossible on a correct node (the finalize path). fn apply( &mut self, context: (E, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> impl Future>::Merkleized> + Send; + batches: UnmerkleizedOf, + ) -> impl Future, ExecutionError>> + Send; /// Observe a finalized block after it is reflected in the database set. /// @@ -322,10 +357,10 @@ where /// reported or applied during handoff. Applications must derive synchronized state from the /// database set rather than rely on receiving every peer-state-sync finalization here. /// - /// This hook receives read-only database handles and may overlap verification - /// of blocks built on the newly finalized block or one of its retained - /// descendants. Result-affecting mutations must be made through normal block - /// execution, not from this observer. + /// This hook receives readers over the set. It runs after the block is + /// applied, and the block's marshal acknowledgement waits for it, so a slow + /// implementation stalls finalization. Result-affecting mutations must be + /// made through normal block execution, not from this observer. /// /// For blocks that are reported, this is an at-least-once notification inherited from /// marshal's reporter stream: a crash after this hook runs but before the block's flush and @@ -338,7 +373,7 @@ where &mut self, _context: (E, Self::Context), _block: &Self::Block, - _readers: >::Readers, + _readers: ReadersOf, ) -> impl Future + Send { async {} } diff --git a/glue/src/stateful/tests/mocks.rs b/glue/src/stateful/tests/mocks.rs index e73b6895a84..7feb49e7019 100644 --- a/glue/src/stateful/tests/mocks.rs +++ b/glue/src/stateful/tests/mocks.rs @@ -1,6 +1,9 @@ use crate::stateful::{ - Application, Input, Proposed, - db::{BatchContext, DatabaseSet, ManagedDb, Merkleized, Shared, Unmerkleized}, + Application, ExecutionError, Input, Proposed, + db::{ + DatabaseSet, ManagedDb, Merkleized, MerkleizedOf, Reader, Single, Unmerkleized, + UnmerkleizedOf, Writer, + }, }; use commonware_codec::{EncodeSize, Error as CodecError, Read, ReadExt as _, Write}; use commonware_consensus::{ @@ -12,11 +15,11 @@ use commonware_consensus::{ use commonware_cryptography::{ Digest as _, Digestible, Signer as _, ed25519, sha256::Digest as Sha256Digest, }; -use commonware_runtime::{Buf, BufMut, Error as RuntimeError, Handle}; +use commonware_runtime::{Buf, BufMut, Error as RuntimeError, Handle, deterministic}; use commonware_utils::{channel::oneshot, sync::Mutex}; use std::{convert::Infallible, sync::Arc}; -pub(crate) type TestDatabases = Shared; +pub(crate) type TestDatabases = Single; pub(crate) type TestScheme = scheme_mocks::Scheme; pub(crate) type TestVariant = Standard; @@ -51,39 +54,12 @@ impl Merkleized for TestMerkleized { /// Completes one parked flush when released by the test. pub(crate) type FlushRelease = oneshot::Sender>; -/// Signals that pruning has started, then blocks it until the test releases it. -struct PruneGate { - started: oneshot::Sender<()>, - release: oneshot::Receiver<()>, -} - /// Shared observer for a gated [`TestDb`]: parked flush releases and recorded /// prune targets. #[derive(Clone, Default)] pub(crate) struct FlushControl { pub(crate) flushes: Arc>>, pub(crate) pruned: Arc>>, - prune_gate: Arc>>, -} - -impl FlushControl { - /// Gates the next prune. The receiver reports entry, and sending on the - /// returned sender lets pruning continue. Only one gate may be active. - pub(crate) fn gate_prune(&self) -> (oneshot::Receiver<()>, oneshot::Sender<()>) { - let (started, started_rx) = oneshot::channel(); - let (release, release_rx) = oneshot::channel(); - assert!( - self.prune_gate - .lock() - .replace(PruneGate { - started, - release: release_rx, - }) - .is_none(), - "prune gate already installed", - ); - (started_rx, release) - } } #[derive(Default)] @@ -133,7 +109,7 @@ impl ManagedDb for TestDb { Ok(Self::default()) } - fn new_batch(_database: BatchContext<'_, Self>) -> Self::Unmerkleized { + async fn new_batch(_reader: Reader) -> Self::Unmerkleized { TestUnmerkleized } @@ -162,11 +138,6 @@ impl ManagedDb for TestDb { async fn prune(self, target: &Self::SyncTarget) -> Result { if let Some(control) = &self.control { - let gate = control.prune_gate.lock().take(); - if let Some(mut gate) = gate { - gate.started.send(()).expect("test must await prune"); - let _ = (&mut gate.release).await; - } control.pruned.lock().push(*target); } Ok(self) @@ -307,33 +278,52 @@ impl< &mut self, _context: (E, Self::Context), _ancestry: impl Ancestry, - _batches: >::Unmerkleized, + _batches: UnmerkleizedOf, _input: Input, - ) -> Option> { - None + ) -> Result>, ExecutionError> { + Ok(None) } async fn verify( &mut self, _context: (E, Self::Context), _ancestry: impl Ancestry, - _batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { - None + _batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> { + Ok(None) } async fn apply( &mut self, _context: (E, Self::Context), _block: &Self::Block, - _batches: >::Unmerkleized, - ) -> >::Merkleized { - TestMerkleized + _batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { + Ok(TestMerkleized) } } pub(crate) fn test_databases() -> TestDatabases { - Shared::new("test", TestDb::default()) + TestDb::default().into() +} + +/// Finalize `batch` through the cell, returning the snapshot and flush handle. +/// +/// Tests that drive [`ManagedDb`] directly split their database and use this +/// instead of the set layer. +pub(crate) async fn finalize>( + writer: Writer, + batch: D::Merkleized, +) -> (Writer, D::Snapshot, Handle<()>) { + let (writer, (snapshot, sync)) = writer + .mutate(|db| async move { + let (db, snapshot, sync) = D::finalize(db, batch) + .await + .unwrap_or_else(|err| panic!("finalize failed: {err:?}")); + (db, (snapshot, sync)) + }) + .await; + (writer, snapshot, sync) } pub(crate) fn anchor(height: u64, digest_byte: u8) -> crate::stateful::db::Anchor { diff --git a/glue/src/stateful/tests/mod.rs b/glue/src/stateful/tests/mod.rs index 3a5cc3cff9f..10078adddc5 100644 --- a/glue/src/stateful/tests/mod.rs +++ b/glue/src/stateful/tests/mod.rs @@ -18,9 +18,9 @@ use crate::{ property::Property, }, stateful::{ - Application, Config as StatefulConfig, Input, Proposed, PruneConfig, + Application, Config as StatefulConfig, ExecutionError, Input, Proposed, Stateful as StatefulActor, SyncPlan, - db::{DatabaseSet, Merkleized as _, Publisher, SyncEngineConfig}, + db::{DatabaseSet, Merkleized as _, Publisher, ReadersOf, SyncEngineConfig}, }, }; use commonware_actor::Feedback; @@ -47,6 +47,7 @@ use commonware_runtime::{ }; use commonware_storage::{ archive::prunable, + journal::contiguous::Contiguous as _, mmr, qmdb::{ any::unordered::fixed, @@ -68,11 +69,28 @@ mod common; pub(crate) mod fixtures; pub(crate) mod mocks; mod multi_db_app; +mod ownership; mod properties; mod single_db_app; const NUM_VALIDATORS: u32 = 5; +/// Storage errors never masquerade as shutdown. Only a refused stale read maps +/// to Stale, and every other storage failure is fatal. +#[test] +fn storage_errors_map_to_fatal() { + use commonware_runtime::Error as RuntimeError; + use commonware_storage::{journal, qmdb}; + + let stale: ExecutionError = qmdb::Error::::StaleRead.into(); + assert!(matches!(stale, ExecutionError::Stale)); + let direct: ExecutionError = qmdb::Error::::Runtime(RuntimeError::Closed).into(); + assert!(matches!(direct, ExecutionError::Fatal(_))); + let nested: ExecutionError = + qmdb::Error::::Journal(journal::Error::Runtime(RuntimeError::Aborted)).into(); + assert!(matches!(nested, ExecutionError::Fatal(_))); +} + fn delay_first(participants: &[P], view: u64) -> Crash

{ Crash::DelayRound { participants: vec![participants[0].clone()], @@ -948,19 +966,22 @@ impl Application for GatedMultiApp { ancestry: impl Ancestry, batches: >::Unmerkleized, input: Input, - ) -> Option> { - let proposed = >::propose( + ) -> Result>, ExecutionError> { + let Some(proposed) = >::propose( &mut self.inner, context, ancestry, batches, input, ) - .await?; - Some(Proposed { + .await? + else { + return Ok(None); + }; + Ok(Some(Proposed { block: proposed.block, merkleized: proposed.merkleized, - }) + })) } async fn verify( @@ -968,7 +989,10 @@ impl Application for GatedMultiApp { context: (deterministic::Context, Self::Context), ancestry: impl Ancestry, batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + ) -> Result< + Option<>::Merkleized>, + ExecutionError, + > { let gate = self.verify_gates.lock().pop_front(); if let Some(mut gate) = gate { let _ = gate.started.send(()); @@ -988,7 +1012,8 @@ impl Application for GatedMultiApp { context: (deterministic::Context, Self::Context), block: &Self::Block, batches: >::Unmerkleized, - ) -> >::Merkleized { + ) -> Result<>::Merkleized, ExecutionError> + { >::apply( &mut self.inner, context, @@ -1002,7 +1027,7 @@ impl Application for GatedMultiApp { &mut self, context: (deterministic::Context, Self::Context), block: &Self::Block, - readers: >::Readers, + readers: ReadersOf, ) { >::finalized( &mut self.inner, @@ -1037,7 +1062,7 @@ async fn build_chain(context: &deterministic::Context, blocks: u64) -> (Block, V .await; let mut batches = as DatabaseSet< deterministic::Context, - >>::new_batches(&databases) + >>::new_batches(&databases.readers()) .await; let mut parent = genesis.clone(); let mut chain = Vec::with_capacity(blocks as usize); @@ -1048,7 +1073,7 @@ async fn build_chain(context: &deterministic::Context, blocks: u64) -> (Block, V for height in 1..=blocks { let height = Height::new(height); - let merkleized = App::execute(height, batches).await; + let merkleized = App::execute(height, batches).await.unwrap(); let bounds = merkleized.bounds(); let block = Block { context: Context { @@ -1092,7 +1117,7 @@ async fn build_multi_chain( .await; let mut batches = as DatabaseSet< deterministic::Context, - >>::new_batches(&databases) + >>::new_batches(&databases.readers()) .await; let mut parent = genesis.clone(); let mut chain = Vec::with_capacity(blocks as usize); @@ -1100,7 +1125,7 @@ async fn build_multi_chain( for height in 1..=blocks { let height = Height::new(height); - let (merkleized_a, merkleized_b) = MultiApp::execute(height, batches).await; + let (merkleized_a, merkleized_b) = MultiApp::execute(height, batches).await.unwrap(); let bounds_a = merkleized_a.bounds(); let bounds_b = merkleized_b.bounds(); let block = MultiBlock { @@ -1212,7 +1237,6 @@ fn out_of_order_certifications_complete_on_qmdb() { }, ); let stateful_actor = stateful.start(); - let _databases = stateful_mailbox.subscribe_databases().await; for block in &blocks { assert!(marshal.verified(block.context.round, block.clone()).await); @@ -1322,7 +1346,7 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { }; let plan = SyncPlan::init(&context, "certify-multi-qmdb-stateful".to_string()).await; let publication_context = context.child("publication"); - let (snapshot_publisher, _snapshot_subscriber) = Publisher::new(&publication_context); + let (snapshot_publisher, snapshot_subscriber) = Publisher::new(&publication_context); let (stateful, stateful_mailbox) = StatefulActor::init( context.child("stateful"), StatefulConfig { @@ -1345,7 +1369,6 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { }, ); let stateful_actor = stateful.start(); - let databases = stateful_mailbox.subscribe_databases().await; for block in &blocks { assert!(marshal.verified(block.context.round, block.clone()).await); @@ -1420,9 +1443,11 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { finalize_started .await .expect("first multi-QMDB finalization should reach the application gate"); + // The descendant verifications are untouched by the apply. Each is still + // waiting in the application, holding the gate it was given. assert!( verify_releases.iter().all(|release| !release.is_closed()), - "the first finalization should retain descendant verifications", + "an apply must not disturb a running verification", ); // A queued finalization is not active until the current one completes. @@ -1435,10 +1460,12 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { finalizations.push(waiter); } context.sleep(Duration::from_millis(10)).await; - assert!( - verify_releases.iter().all(|release| !release.is_closed()), - "queued finalization quiesced work before the current finalization completed", - ); + for waiter in &mut finalizations { + assert!( + futures::poll!(waiter).is_pending(), + "a queued finalization must wait for the active one", + ); + } finalize_release .send(()) .expect("first multi-QMDB finalization should remain active"); @@ -1453,10 +1480,10 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { panic!("multi-QMDB finalizations did not become durable"); }, } + // Released, the verifications finish on the attempts they were already + // running when the finalizations landed. for release in verify_releases { - release - .send(()) - .expect("compatible verification should remain active across finalization"); + let _ = release.send(()); } for (index, certification) in certifications { select! { @@ -1489,289 +1516,17 @@ fn overlapping_finalizations_complete_on_multi_qmdb() { }, } - let committed = as DatabaseSet< - deterministic::Context, - >>::committed_targets(&databases) - .await; - let expected = - >::sync_targets(&blocks[5]); - assert_eq!(committed.0, expected.0, "full QMDB target diverged"); - assert_eq!(committed.1, expected.1, "compact QMDB target diverged"); - - stateful_actor.abort(); - marshal_actor.abort(); - let _ = stateful_actor.await; - let _ = marshal_actor.await; - }); -} - -#[test] -fn pruning_quiesces_and_retries_verification_on_real_qmdbs() { - deterministic::Runner::timed(Duration::from_secs(10)).start(|context| async move { - let (genesis, blocks) = build_multi_chain(&context, 5).await; - let page_cache = CacheRef::from_pooler(&context, PAGE_SIZE, PAGE_CACHE_SIZE); - let mut signing_context = context.child("signing"); - let fixture = scheme_mocks::fixture( - &mut signing_context, - b"_COMMONWARE_GLUE_MULTI_QMDB_PRUNE_OVERLAP", - 1, - ); - let provider = ConstantProvider::new(fixture.schemes[0].clone()); - let finalizations_by_height = prunable::Archive::init( - context.child("finalizations_by_height"), - archive_config( - "prune-overlap-multi-qmdb-marshal", - "finalizations", - page_cache.clone(), - (), - ), - ) - .await - .expect("failed to initialize finalizations archive"); - let finalized_blocks = prunable::Archive::init( - context.child("finalized_blocks"), - archive_config( - "prune-overlap-multi-qmdb-marshal", - "blocks", - page_cache.clone(), - (), - ), - ) - .await - .expect("failed to initialize blocks archive"); - let (marshal_actor, marshal, floor) = - MarshalActor::<_, Standard, _, _, _, _, _>::init( - context.child("marshal"), - finalizations_by_height, - finalized_blocks, - marshal::Config { - provider, - epocher: FixedEpocher::new(EPOCH_LENGTH), - start: marshal::Start::Genesis(genesis.clone()), - partition_prefix: "prune-overlap-multi-qmdb-marshal".to_string(), - mailbox_size: NZUsize!(8), - view_retention: ViewDelta::new(10), - prunable_items_per_section: NZU64!(10), - page_cache: page_cache.clone(), - replay_buffer: IO_BUFFER_SIZE, - key_write_buffer: IO_BUFFER_SIZE, - value_write_buffer: IO_BUFFER_SIZE, - block_codec_config: (), - max_repair: NZUsize!(10), - max_pending_acks: NZUsize!(1), - strategy: Sequential, - }, - ) - .await; - let (resolver_receiver, _resolver_handler) = - handler::init(context.child("marshal_resolver"), NZUsize!(8)); - let marshal_actor = marshal_actor.start_unbuffered( - NoopMultiMarshalApplication, - (resolver_receiver, fixtures::IgnoreResolver), - ); - - let verify_gates = Arc::new(Mutex::new(VecDeque::new())); - let finalize_gate = Arc::new(Mutex::new(None)); - let application = GatedMultiApp { - inner: MultiApp::new(genesis), - verify_gates: verify_gates.clone(), - finalize_gate: finalize_gate.clone(), - }; - let plan = SyncPlan::init(&context, "prune-overlap-multi-qmdb-stateful".to_string()).await; - let publication_context = context.child("publication"); - let (snapshot_publisher, _snapshot_subscriber) = Publisher::new(&publication_context); - let (stateful, stateful_mailbox) = StatefulActor::init( - context.child("stateful"), - StatefulConfig { - application, - db_config: multi_qmdb_config("prune-overlap-multi-qmdb-stateful", page_cache), - provider: (), - marshal: (marshal.clone(), floor), - mailbox_size: NZUsize!(1), - plan, - resolvers: (NoopQmdbResolver, NoopCompactQmdbResolver), - sync_config: SyncEngineConfig { - fetch_batch_size: NZU64!(1), - apply_batch_size: NZU64!(1), - max_outstanding_requests: 1, - update_channel_size: NZUsize!(1), - max_retained_roots: 1, - }, - // The first prune runs at block 4 and targets block 3's floor, - // which crosses the full QMDB's first journal blob. - prune_config: Some(PruneConfig { - maintenance_interval: NZUsize!(1), - retained_marshal_blocks: 2, - retained_qmdb_blocks: 0, - }), - snapshot_publisher, - }, - ); - let stateful_actor = stateful.start(); - let databases = stateful_mailbox.subscribe_databases().await; - - for block in &blocks { - assert!(marshal.verified(block.context.round, block.clone()).await); - } - - let mut deferred = Deferred::new( - context.child("deferred"), - stateful_mailbox, - marshal, - FixedEpocher::new(EPOCH_LENGTH), - ); - - // Keep the first four batches available so block 5 reaches application - // verification without owning ancestor replay. - for block in &blocks[..4] { - let certification = deferred.certify(block.context.round, block.digest()).await; - assert!( - certification - .await - .expect("priming certification result missing"), - ); - } - - let finalized_tip = &blocks[3]; - let _ = deferred.report(marshal::Update::Tip( - finalized_tip.context.round, - finalized_tip.height, - finalized_tip.digest(), - )); - let mut reporter = deferred; - for block in &blocks[..3] { - let (acknowledgement, waiter) = Exact::handle(); - let _ = reporter.report(marshal::Update::Block( - Arc::new(block.clone()), - acknowledgement, - )); - select! { - result = waiter => result.expect("priming finalization should be durable"), - _ = context.sleep(Duration::from_secs(2)) => { - panic!("priming finalization did not become durable"); - }, - } - } - - let expected_floor = *blocks[2].range_a.start(); - assert!( - expected_floor > mmr::Location::new(0), - "the prune target must discard real QMDB history", - ); - - let (first_gate, first_started, mut first_release) = application_gate(); - let (retry_gate, mut retry_started, retry_release) = application_gate(); - verify_gates.lock().extend([first_gate, retry_gate]); - let (gate, finalize_started, finalize_release) = application_gate(); - assert!( - finalize_gate.lock().replace(gate).is_none(), - "finalization gate already installed", - ); - - let block = &blocks[4]; - let mut certification = reporter.certify(block.context.round, block.digest()).await; - first_started - .await - .expect("verification should start before pruning"); - assert!( - futures::poll!(&mut certification).is_pending(), - "verification completed before pruning", - ); - - let (acknowledgement, finalized) = Exact::handle(); - let _ = reporter.report(marshal::Update::Block( - Arc::new(blocks[3].clone()), - acknowledgement, - )); - finalize_started - .await - .expect("block 4 finalization should reach the application gate"); - assert!( - !first_release.is_closed(), - "same-branch finalization should retain verification", - ); - - // Hold the full QMDB reader after finalization applies block 4. Pruning - // can quiesce verification, but cannot delete history or requeue it - // until this guard is released. - let full_database = databases.0.read().await; - let before_prune = full_database.bounds(); - assert_eq!( - before_prune.start, - mmr::Location::new(0), - "QMDB pruned before the configured retention window filled", - ); - finalize_release - .send(()) - .expect("block 4 finalization should remain active"); - - select! { - _ = first_release.closed() => {}, - _ = context.sleep(Duration::from_secs(2)) => { - panic!("pruning did not quiesce the active verification"); - }, - } - assert_eq!( - full_database.bounds(), - before_prune, - "QMDB history changed while its reader was held", - ); - assert!( - futures::poll!(&mut certification).is_pending(), - "quiesced verification completed before retry", - ); - assert!( - futures::poll!(&mut retry_started).is_pending(), - "verification restarted before physical pruning completed", - ); - drop(full_database); - - select! { - result = &mut retry_started => { - result.expect("verification should restart after pruning"); - }, - _ = context.sleep(Duration::from_secs(2)) => { - panic!("verification did not restart after pruning"); - }, + // The set is owned by the actor, so assert through the published + // snapshots, which are what a peer can actually observe. The final + // publication must reflect the last finalized block's state. + let expected_end = blocks[5].range_a.end(); + while !snapshot_subscriber + .latest() + .is_some_and(|(a, _)| a.bounds().end == expected_end) + { + context.sleep(Duration::from_millis(10)).await; } - let after_prune = databases.0.read().await.bounds(); - assert!( - after_prune.start > before_prune.start, - "verification restarted before the full QMDB discarded history", - ); - assert!( - after_prune.start <= expected_floor, - "full QMDB pruned past the requested floor", - ); - assert_eq!( - after_prune.end, before_prune.end, - "pruning changed the full QMDB tip", - ); - retry_release - .send(()) - .expect("retried verification should remain active"); - select! { - result = certification => { - assert!(result.expect("retried certification result missing")); - }, - _ = context.sleep(Duration::from_secs(2)) => { - panic!("retried verification did not complete"); - }, - } - finalized - .await - .expect("block 4 finalization should become durable"); - - let committed = as DatabaseSet< - deterministic::Context, - >>::committed_targets(&databases) - .await; - let expected = - >::sync_targets(&blocks[3]); - assert_eq!(committed.0, expected.0, "full QMDB target diverged"); - assert_eq!(committed.1, expected.1, "compact QMDB target diverged"); - stateful_actor.abort(); marshal_actor.abort(); let _ = stateful_actor.await; diff --git a/glue/src/stateful/tests/multi_db_app.rs b/glue/src/stateful/tests/multi_db_app.rs index d54093b0d04..5a1d5e97419 100644 --- a/glue/src/stateful/tests/multi_db_app.rs +++ b/glue/src/stateful/tests/multi_db_app.rs @@ -5,11 +5,11 @@ use crate::{ reporter::MonitorReporter, }, stateful::{ - Application, Config as StatefulConfig, Input, Proposed, PruneConfig, + Application, Config as StatefulConfig, ExecutionError, Input, Proposed, PruneConfig, Stateful as StatefulActor, SyncPlan, db::{ - DatabaseSet, Merkleized as _, Shared, SnapshotsOf, SyncEngineConfig, Unmerkleized as _, - p2p as qmdb_resolver, + DatabaseSet, Merkleized as _, MerkleizedOf, Publisher, Single, SnapshotsOf, + SyncEngineConfig, Unmerkleized as _, UnmerkleizedOf, p2p as qmdb_resolver, }, probe::{Config as ProbeConfig, Probe}, }, @@ -74,12 +74,8 @@ type QmdbA = pub(super) type QmdbB = immutable::fixed::CompactDb; -/// A single QMDB database behind a lock. -type DbA = Shared>; -type DbB = Shared>; - /// A full and a compact QMDB as a tuple. -pub(crate) type MultiDatabaseSet = (DbA, DbB); +pub(crate) type MultiDatabaseSet = (Single>, Single>); /// The set's published snapshots. type MultiSnapshot = SnapshotsOf, E>; @@ -249,14 +245,8 @@ impl App { /// Execute a block against two databases. pub(super) async fn execute( height: Height, - batches: ( - as DatabaseSet>::Unmerkleized, - as DatabaseSet>::Unmerkleized, - ), - ) -> ( - as DatabaseSet>::Merkleized, - as DatabaseSet>::Merkleized, - ) { + batches: UnmerkleizedOf, E>, + ) -> Result, E>, ExecutionError> { let (mut batch_a, batch_b) = batches; // DB-A: increment counter and write a height marker, mirroring the single-db app's @@ -264,8 +254,7 @@ impl App { let counter = Sha256::hash(&[b"counter"]); let current: u64 = batch_a .get(&counter) - .await - .unwrap() + .await? .map_or(0, |v| digest_to_u64(&v)); batch_a = batch_a.write(counter, Some(u64_to_digest(current + 1))); batch_a = batch_a.write( @@ -279,9 +268,9 @@ impl App { u64_to_digest(height.get()), ); - let merkleized_a = batch_a.merkleize().await.unwrap(); - let merkleized_b = batch_b.merkleize().await.unwrap(); - (merkleized_a, merkleized_b) + let merkleized_a = batch_a.merkleize().await?; + let merkleized_b = batch_b.merkleize().await?; + Ok((merkleized_a, merkleized_b)) } } @@ -301,13 +290,15 @@ impl Application for App { &mut self, context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { let mut ancestry = Box::pin(ancestry); - let parent = ancestry.next().await?; + let Some(parent) = ancestry.next().await else { + return Ok(None); + }; let height = Height::new(parent.height().get() + 1); - let (merkleized_a, merkleized_b) = Self::execute(height, batches).await; + let (merkleized_a, merkleized_b) = Self::execute(height, batches).await?; let bounds_a = merkleized_a.bounds(); let bounds_b = merkleized_b.bounds(); let block = Block { @@ -319,21 +310,23 @@ impl Application for App { root_b: merkleized_b.root(), range_b: non_empty_range!(bounds_b.inactivity_floor, bounds_b.tip.size), }; - Some(Proposed { + Ok(Some(Proposed { block, merkleized: (merkleized_a, merkleized_b), - }) + })) } async fn verify( &mut self, _context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> { let mut ancestry = Box::pin(ancestry); - let tip = ancestry.next().await?; - let (merkleized_a, merkleized_b) = Self::execute(tip.height(), batches).await; + let Some(tip) = ancestry.next().await else { + return Ok(None); + }; + let (merkleized_a, merkleized_b) = Self::execute(tip.height(), batches).await?; let bounds_a = merkleized_a.bounds(); let bounds_b = merkleized_b.bounds(); let matches_a = merkleized_a.root() == tip.root_a @@ -341,17 +334,17 @@ impl Application for App { let matches_b = merkleized_b.root() == tip.root_b && non_empty_range!(bounds_b.inactivity_floor, bounds_b.tip.size) == tip.range_b; if !matches_a || !matches_b { - return None; + return Ok(None); } - Some((merkleized_a, merkleized_b)) + Ok(Some((merkleized_a, merkleized_b))) } async fn apply( &mut self, _context: (E, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> >::Merkleized { + batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { Self::execute(block.height(), batches).await } @@ -594,7 +587,7 @@ impl EngineDefinition for MultiDbEngine { // database), each serving from its own reader. let publication_context = context.child("publication"); let (snapshot_publisher, snapshot_subscriber) = - crate::stateful::db::Publisher::>::new(&publication_context); + Publisher::>::new(&publication_context); let snapshot_subscriber_a = snapshot_subscriber.view(|snapshots| &snapshots.0); let snapshot_subscriber_b = snapshot_subscriber.view(|snapshots| &snapshots.1); let (qmdb_resolver_actor_a, qmdb_sync_resolver_a) = qmdb_resolver::Actor::new( diff --git a/glue/src/stateful/tests/ownership.rs b/glue/src/stateful/tests/ownership.rs new file mode 100644 index 00000000000..b8db65ffab9 --- /dev/null +++ b/glue/src/stateful/tests/ownership.rs @@ -0,0 +1,128 @@ +//! Causal ownership tests for the by-value database set. Readers and the writer +//! share nothing that can block either side. +//! +//! - a parked flush holds back neither the writer nor publication +//! ([`parked_flush_never_delays_the_writer`]) +//! - a serve holding a published snapshot across parked I/O cannot delay the +//! writer, and its snapshot stays frozen while publication moves on +//! ([`parked_serve_never_delays_the_writer`]) +//! +//! The deterministic runtime advances time only at quiescence, so `blocked_on` +//! resolving to its timeout proves the probed future could not progress at any +//! scheduling point. + +use super::mocks::{FlushControl, TestDb, TestMerkleized}; +use crate::stateful::db::{Barrier, DatabaseSet, Publisher, Single}; +use commonware_consensus::types::Height; +use commonware_macros::test_traced; +use commonware_runtime::{Clock, Runner as _, Spawner as _, Supervisor as _, deterministic}; +use commonware_utils::channel::oneshot; +use std::time::Duration; + +/// How long `blocked_on` waits before declaring the probed future blocked. +const BLOCKED: Duration = Duration::from_secs(1); + +/// A parked single-member set plus the flush controls driving it. +fn parked_set() -> (Single, FlushControl) { + let control = FlushControl::default(); + (Single::from(TestDb::gated(control.clone())), control) +} + +/// Finalize an empty batch, pinning the set's environment to the +/// deterministic runtime ([`TestDb`] works in any environment). +async fn finalize(set: Single) -> (Single, u64, Barrier) { + DatabaseSet::::finalize(set, TestMerkleized).await +} + +/// Await `future` against a deterministic timeout, `Ok` if it completed and +/// `Err(future)` if the runtime reached quiescence without it progressing. +async fn blocked_on(context: &deterministic::Context, future: F) -> Result +where + F: std::future::Future + Unpin, +{ + let mut future = future; + commonware_macros::select! { + result = &mut future => Ok(result), + _ = context.sleep(BLOCKED) => Err(future), + } +} + +/// Release the oldest parked flush. +fn release(control: &FlushControl) { + control.flushes.lock().remove(0).send(Ok(())).unwrap(); +} + +/// A parked flush holds back neither the writer nor publication. +#[test_traced] +fn parked_flush_never_delays_the_writer() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let (set, control) = parked_set(); + let (mut publisher, reader) = Publisher::new(&context); + + // The first snapshots publish at apply while their flush is parked. + let (set, snapshot, first) = finalize(set).await; + publisher.publish(Height::new(1), snapshot); + assert_eq!(reader.latest(), Some(1)); + + // The parked flush does not hold the writer back. + let next = context.child("finalize").spawn(move |_| finalize(set)); + let (_, snapshot, second) = blocked_on(&context, next) + .await + .unwrap_or_else(|_| panic!("a parked flush delayed the writer")) + .unwrap(); + publisher.publish(Height::new(2), snapshot); + assert_eq!(reader.latest(), Some(2)); + + // Flushes still resolve durability, in order. + release(&control); + assert!(first.durable().await); + release(&control); + assert!(second.durable().await); + }); +} + +/// A serve holding a published snapshot across parked I/O cannot delay the +/// writer, and the held snapshot stays frozen while publication moves on. +#[test_traced] +fn parked_serve_never_delays_the_writer() { + let executor = deterministic::Runner::default(); + executor.start(|context| async move { + let (set, control) = parked_set(); + let (mut publisher, reader) = Publisher::new(&context); + let (set, snapshot, barrier) = finalize(set).await; + publisher.publish(Height::new(1), snapshot); + release(&control); + assert!(barrier.durable().await); + + // A serve takes the published snapshot and parks mid-assembly. + let served = reader.latest().unwrap(); + let (io_done, io_gate) = oneshot::channel(); + let serve = context.child("serve").spawn(move |_| async move { + let _ = io_gate.await; + served + }); + + // The parked serve shares nothing with the writer, so the next + // finalize and publication proceed without delay. + let next = context.child("finalize").spawn(move |_| finalize(set)); + let (_, snapshot, barrier) = blocked_on(&context, next) + .await + .unwrap_or_else(|_| panic!("a parked serve delayed the writer")) + .unwrap(); + publisher.publish(Height::new(2), snapshot); + release(&control); + assert!(barrier.durable().await); + + // The serve completes against its captured snapshot while the reader + // already serves the newer snapshots. + io_done.send(()).unwrap(); + let held = serve.await.unwrap(); + assert_eq!(held, 1, "the held snapshot never moved"); + assert_eq!( + reader.latest(), + Some(2), + "publication moved on while the serve was parked" + ); + }); +} diff --git a/glue/src/stateful/tests/single_db_app.rs b/glue/src/stateful/tests/single_db_app.rs index df38d1a0803..ddcbf9a3f00 100644 --- a/glue/src/stateful/tests/single_db_app.rs +++ b/glue/src/stateful/tests/single_db_app.rs @@ -5,11 +5,11 @@ use crate::{ reporter::MonitorReporter, }, stateful::{ - Application, Config as StatefulConfig, Input, Proposed, PruneConfig, + Application, Config as StatefulConfig, ExecutionError, Input, Proposed, PruneConfig, Stateful as StatefulActor, SyncPlan, db::{ - DatabaseSet, Merkleized as _, Shared, SyncEngineConfig, Unmerkleized as _, - p2p as qmdb_resolver, + DatabaseSet, Merkleized as _, MerkleizedOf, Publisher, Single, SyncEngineConfig, + Unmerkleized as _, UnmerkleizedOf, p2p as qmdb_resolver, }, probe::{Config as ProbeConfig, Probe}, }, @@ -65,7 +65,7 @@ use std::{collections::BTreeMap, sync::Arc, time::Duration}; pub(super) type Qmdb = fixed::Db; -pub(crate) type SingleDatabaseSet = Shared>; +pub(crate) type SingleDatabaseSet = Single>; /// Builds the QMDB configuration used by single-database tests. pub(super) fn qmdb_config(prefix: &str, page_cache: CacheRef) -> FixedConfig { @@ -193,20 +193,19 @@ impl App { /// Execute a block: increment "counter" and write `height -> height_val`. pub(super) async fn execute( height: Height, - mut batches: as DatabaseSet>::Unmerkleized, - ) -> as DatabaseSet>::Merkleized { + mut batches: UnmerkleizedOf, E>, + ) -> Result, E>, ExecutionError> { let counter = Sha256::hash(&[b"counter"]); let current: u64 = batches .get(&counter) - .await - .unwrap() + .await? .map_or(0, |v| digest_to_u64(&v)); batches = batches.write(counter, Some(u64_to_digest(current + 1))); batches = batches.write( Sha256::hash(&[&height.get().to_be_bytes()]), Some(u64_to_digest(height.get())), ); - batches.merkleize().await.unwrap() + Ok(batches.merkleize().await?) } } @@ -226,13 +225,15 @@ impl Application for App { &mut self, context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, + batches: UnmerkleizedOf, _input: Input, - ) -> Option> { + ) -> Result>, ExecutionError> { let mut ancestry = Box::pin(ancestry); - let parent = ancestry.next().await?; + let Some(parent) = ancestry.next().await else { + return Ok(None); + }; let height = Height::new(parent.height().get() + 1); - let merkleized = Self::execute(height, batches).await; + let merkleized = Self::execute(height, batches).await?; let bounds = merkleized.bounds(); let block = Block { context: context.1.clone(), @@ -241,33 +242,35 @@ impl Application for App { state_root: merkleized.root(), range: non_empty_range!(bounds.inactivity_floor, bounds.tip.size), }; - Some(Proposed { block, merkleized }) + Ok(Some(Proposed { block, merkleized })) } async fn verify( &mut self, _context: (E, Self::Context), ancestry: impl Ancestry, - batches: >::Unmerkleized, - ) -> Option<>::Merkleized> { + batches: UnmerkleizedOf, + ) -> Result>, ExecutionError> { let mut ancestry = Box::pin(ancestry); - let tip = ancestry.next().await?; - let merkleized = Self::execute(tip.height(), batches).await; + let Some(tip) = ancestry.next().await else { + return Ok(None); + }; + let merkleized = Self::execute(tip.height(), batches).await?; let bounds = merkleized.bounds(); if merkleized.root() != tip.state_root || non_empty_range!(bounds.inactivity_floor, bounds.tip.size) != tip.range { - return None; + return Ok(None); } - Some(merkleized) + Ok(Some(merkleized)) } async fn apply( &mut self, _context: (E, Self::Context), block: &Self::Block, - batches: >::Unmerkleized, - ) -> >::Merkleized { + batches: UnmerkleizedOf, + ) -> Result, ExecutionError> { Self::execute(block.height(), batches).await } @@ -488,8 +491,7 @@ impl EngineDefinition for SingleDbEngine { // Snapshot publication channel and the QMDB state-sync resolver serving from it. let publication_context = context.child("publication"); - let (snapshot_publisher, snapshot_subscriber) = - crate::stateful::db::Publisher::new(&publication_context); + let (snapshot_publisher, snapshot_subscriber) = Publisher::new(&publication_context); let (qmdb_resolver_actor, qmdb_sync_resolver) = qmdb_resolver::Actor::new( context.child("qmdb_resolver"), qmdb_resolver::Config { From 79ff453847874cb6143a7d9ac0a23992e9d26536 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 13:11:10 -0400 Subject: [PATCH 02/14] [glue] Close the database cell when the writer drops The writer-outlives-readers rule was documentation only. Dropping the writer now latches the cell closed, and later leases park instead of answering from a database that can never advance, mirroring the poisoned case. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/db/cell.rs | 67 +++++++++++++++++++++++++++++------- glue/src/stateful/db/mod.rs | 9 +++-- 2 files changed, 59 insertions(+), 17 deletions(-) diff --git a/glue/src/stateful/db/cell.rs b/glue/src/stateful/db/cell.rs index 2a2f2dc69ac..2abd72a1883 100644 --- a/glue/src/stateful/db/cell.rs +++ b/glue/src/stateful/db/cell.rs @@ -24,14 +24,20 @@ //! //! Two rules keep callers out of trouble. The lock is not reentrant, so never //! hold a lease while acquiring another, on any cell, because a mutation queued -//! between the two would deadlock both. And a [`Writer`] must outlive the -//! readers from the same cell. Dropping the writer does not drop the database, -//! so readers would otherwise go on answering from a database that can never -//! advance. +//! between the two would deadlock both. And a [`Writer`] should outlive the +//! readers from the same cell. Dropping the writer closes the cell, and later +//! leases park instead of answering from a database that can never advance. use commonware_utils::sync::{AsyncRwLockReadGuard, TracedAsyncRwLock}; use futures::future; -use std::{future::Future, ops::Deref, sync::Arc}; +use std::{ + future::Future, + ops::Deref, + sync::{ + Arc, + atomic::{AtomicBool, Ordering}, + }, +}; enum State { Live(T), @@ -41,11 +47,22 @@ enum State { struct Cell { state: TracedAsyncRwLock>, + /// Latched when the writer drops. A one-way flag, so a relaxed load + /// suffices, and a lease that races the drop is indistinguishable from + /// one granted a moment earlier. + closed: AtomicBool, } impl Cell { async fn read(&self) -> ReadGuard<'_, T> { let guard = self.state.read().await; + if self.closed.load(Ordering::Relaxed) { + // The writer is gone, so the database can never advance. Serving + // would hand out frozen state. Park like the poisoned case. + drop(guard); + tracing::error!("database cell closed; parking reader"); + return future::pending().await; + } match AsyncRwLockReadGuard::try_map(guard, |state| match state { State::Live(db) => Some(db), State::Poisoned => None, @@ -75,15 +92,22 @@ pub fn split(db: T) -> (Writer, Reader) { /// /// This is the only value with [`mutate`](Self::mutate), and it is deliberately /// not [`Clone`], so at most one exists per cell. It does not own the database. -/// Dropping it leaves readers on a database that can no longer advance, which is -/// why it must outlive the readers taken from the same cell. +/// Dropping it closes the cell, since the database can never advance again, and +/// later leases park instead of serving frozen state. pub struct Writer(Arc>); +impl Drop for Writer { + fn drop(&mut self) { + self.0.closed.store(true, Ordering::Relaxed); + } +} + impl Writer { /// Wrap `db` in a fresh cell, returning its sole mutation authority. pub fn new(db: T) -> Self { Self(Arc::new(Cell { state: TracedAsyncRwLock::new("database_cell", State::Live(db)), + closed: AtomicBool::new(false), })) } @@ -103,8 +127,8 @@ impl Writer { /// takes the writer with it, and if the database was already taken out it /// also poisons the cell, so a second mutation of a poisoned cell is /// unreachable rather than merely documented. Leases taken afterward park - /// forever. (Dropped while still queued for the lock, the cell stays live - /// but can never advance again -- the same wedge as dropping the writer.) + /// forever. (Dropped while still queued for the lock, the writer's own drop + /// closes the cell the same way.) pub async fn mutate(self, mutation: F) -> (Self, R) where F: FnOnce(T) -> Fut, @@ -183,7 +207,7 @@ mod tests { } context.sleep(Duration::from_millis(5)).await; - writer + let (_writer, ()) = writer .mutate(|db| async move { assert_eq!(db, 0); (db + 1, ()) @@ -206,13 +230,14 @@ mod tests { let (acquired_tx, acquired) = oneshot::channel::<()>(); let mutation_task = context.child("mutation").spawn(move |_| async move { - writer + let (writer, ()) = writer .mutate(|db| async move { let _ = acquired_tx.send(()); let _ = release.await; (db + 1, ()) }) .await; + writer }); acquired.await.expect("mutation must start"); @@ -224,11 +249,29 @@ mod tests { ); release_tx.send(()).expect("mutation is waiting"); - mutation_task.await.expect("mutation completes"); + let _writer = mutation_task.await.expect("mutation completes"); assert_eq!(*read.await, 1, "the lease sees the mutated state"); }); } + /// Dropping the writer closes the cell. Later leases park instead of + /// answering from a database that can never advance. + #[test] + fn dropped_writer_parks_readers() { + deterministic::Runner::default().start(|_context| async move { + let (writer, reader) = split(0u64); + assert_eq!(*reader.read().await, 0); + drop(writer); + + let read = reader.read(); + futures::pin_mut!(read); + assert!( + read.as_mut().now_or_never().is_none(), + "a lease after the writer drops must park, not serve frozen state", + ); + }); + } + /// Dropping a mutation mid-flight poisons the cell, and takes the writer /// with it. Later leases park forever instead of observing missing state. /// A second mutation is unrepresentable, so there is nothing to assert. diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index c3421dd8738..13e6c253af6 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -31,11 +31,10 @@ //! //! Two invariants keep this sound. A batch handed to [`ManagedDb::finalize`] //! must not read through its own reader, because that call runs while the write -//! side is held. And a [`Writer`] must outlive the readers from the same cell. -//! Dropping the writer does not drop the database, so readers would otherwise -//! go on answering from a database that can never advance. Both invariants -//! hold structurally today, because the set holds the [`Writer`] and every -//! reader lives in a batch the set outlives. +//! side is held. And a [`Writer`] should outlive the readers from the same +//! cell. Dropping the writer closes the cell, and later leases park instead of +//! answering from a database that can never advance. The set holds the +//! [`Writer`], and every reader lives in a batch the set outlives. //! //! # State Sync //! From fb28054974e4eb0932adacd28521d04ab907ca0c Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 13:11:19 -0400 Subject: [PATCH 03/14] [glue] State the actual propose and verify error contracts The Fatal doc claimed only Ok(None) declines a proposal and the propose doc claimed every error declines. Neither was true. Spell out the matrix, declines on Ok(None), Shutdown, and Stale, a panic on Fatal, and pin verify's verdict boundary to the applied anchor move. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/mod.rs | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index b0bc9c363aa..0895a559a61 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -124,7 +124,7 @@ pub enum ExecutionError { #[error("stale execution: a competing block was finalized")] Stale, /// Any other storage failure. Storage errors are unrecoverable, so the - /// wrapper panics on this everywhere. Only `Ok(None)` declines a proposal. + /// wrapper panics on this everywhere. #[error("storage failure: {0}")] Fatal(String), } @@ -256,7 +256,9 @@ where /// and retrying must not violate invariants or lose durable progress. /// /// Storage errors from batch operations are propagated as [`ExecutionError`], - /// never interpreted. The wrapper declines the proposal on any error. + /// never interpreted. The wrapper declines the proposal on `Ok(None)` and + /// [`Stale`](ExecutionError::Stale), and panics on + /// [`Fatal`](ExecutionError::Fatal). fn propose( &mut self, context: (E, Self::Context), @@ -278,10 +280,12 @@ where /// voting, do not resolve this future yet. Abstaining is not represented by /// a special return value. /// - /// Validity is relative to the supplied inputs. Finalizing a competing - /// branch later does not retroactively change a completed verdict. A - /// verdict reached after that finalization is reported as invalid instead, - /// because its branch is no longer reachable. + /// Validity is relative to the supplied inputs, and the boundary is the + /// applied anchor move. A verdict completed before the anchor moves stays + /// valid even when a competing branch finalizes afterward. A verdict that + /// completes after the anchor moved is answered from the new canonical + /// chain instead. True when the block itself became canonical, and false + /// when its branch is no longer reachable. /// /// Verification must reject any block whose execution result does not /// match the block's committed state (for example, a state root mismatch). From b55a408aa916fefd639edb3f345beefd336eae79 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 13:15:03 -0400 Subject: [PATCH 04/14] [glue] Exit the actor when shutdown interrupts execution An application still parked inside apply when cooperative shutdown began used to park the finalize future forever. The park sat inside the processing loop's step, so the loop never released its own stop signal and runtime shutdown deadlocked on it. The stop signal now covers the application awaits: the processing loop selects it around the finalize drive and the proposal drive, and the sync handoff does the same around its applies. On stop the actor drops the in-flight future and exits without acknowledging the block, so marshal redelivers it after a restart. Mailbox::verify parks on a lost response channel instead of panicking with a message blaming the actor, which also covers verifications the exiting actor drops. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/actor/core/mailbox.rs | 10 +- glue/src/stateful/actor/core/processing.rs | 214 ++++++++++++++++++++- glue/src/stateful/actor/core/syncing.rs | 31 ++- 3 files changed, 240 insertions(+), 15 deletions(-) diff --git a/glue/src/stateful/actor/core/mailbox.rs b/glue/src/stateful/actor/core/mailbox.rs index daba9b332e1..9b998052adf 100644 --- a/glue/src/stateful/actor/core/mailbox.rs +++ b/glue/src/stateful/actor/core/mailbox.rs @@ -227,9 +227,13 @@ where ancestry: BoxedAncestry::new(ancestry), verification: Verification { response }, }); - receiver - .await - .expect("stateful actor dropped during verify") + match receiver.await { + Ok(valid) => valid, + // The actor exited or discarded the request while shutting down. + // Never fabricate a verdict. Park until the caller loses interest + // and drops this future. + Err(_) => std::future::pending().await, + } } } diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index 7210dd5699f..a39e8582e5a 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -50,7 +50,7 @@ use futures::{ }; use rand_core::Rng; use std::{collections::BTreeSet, sync::mpsc::TryRecvError}; -use tracing::{Instrument as _, debug, info_span}; +use tracing::{Instrument as _, debug, info_span, warn}; /// A single unit of work for the processing loop: either a mailbox message to /// handle or a deferred prune to run while the mailbox is idle. @@ -235,6 +235,10 @@ where loop { if receive_messages { select! { + _ = &mut shutdown => { + debug!("shutdown signal received, stopping processing"); + return; + }, _ = &mut proposal => break, message = self.mailbox.recv() => match message { Some(Message::Verify { @@ -264,6 +268,10 @@ where } } else { select! { + _ = &mut shutdown => { + debug!("shutdown signal received, stopping processing"); + return; + }, _ = &mut proposal => break, _ = verifications.next_completed() => {}, } @@ -307,12 +315,26 @@ where // The apply owns mutation. Live verification jobs pause at // their next batch operation and resume afterward, so they - // keep being polled throughout. + // keep being polled throughout. The stop signal covers the + // await: an application parked mid-apply cannot wedge + // shutdown, and exiting just drops un-applied batches. The + // block stays unacknowledged, and marshal redelivers it + // after a restart. let applied; - (processor, applied) = verifications - .drive(processor.finalize(self.context.as_present(), block.as_ref())) - .instrument(process.clone()) - .await; + select! { + _ = &mut shutdown => { + warn!( + height = block.height().get(), + "exiting mid-finalize on shutdown" + ); + return; + }, + driven = verifications + .drive(processor.finalize(self.context.as_present(), block.as_ref())) + .instrument(process.clone()) => { + (processor, applied) = driven; + }, + } // Keep the publication bookkeeping under the same span. let _span = process.entered(); @@ -2966,6 +2988,186 @@ mod tests { }); } + /// An application still parked inside execution when shutdown begins. + #[derive(Clone)] + struct ParkedApp { + /// Signals entry into `verify` or `apply`. + started: Arc>>>, + } + + impl Application for ParkedApp { + type SigningScheme = TestScheme; + type Context = >::Context; + type Block = TestBlock; + type Databases = TestDatabases; + type Provider = (); + type Input = (); + + fn sync_targets(block: &Self::Block) -> u64 { + block.height().get() + } + + async fn genesis(&mut self) -> Self::Block { + panic!("shutdown application genesis is not used") + } + + async fn propose( + &mut self, + _context: (deterministic::Context, Self::Context), + _ancestry: impl Ancestry, + _batches: TestUnmerkleized, + _input: Input, + ) -> Result>, ExecutionError> { + panic!("shutdown application propose is not used") + } + + async fn verify( + &mut self, + _context: (deterministic::Context, Self::Context), + mut ancestry: impl Ancestry, + _batches: TestUnmerkleized, + ) -> Result, ExecutionError> { + let _ = ancestry.next().await; + if let Some(started) = self.started.lock().take() { + let _ = started.send(()); + } + std::future::pending().await + } + + async fn apply( + &mut self, + _context: (deterministic::Context, Self::Context), + _block: &Self::Block, + _batches: TestUnmerkleized, + ) -> Result { + if let Some(started) = self.started.lock().take() { + let _ = started.send(()); + } + std::future::pending().await + } + } + + /// A spawned parked application's mailbox, execution entry signal, marshal + /// guard, and actor handle. + type SpawnedParkedApplication = ( + Mailbox, + oneshot::Receiver<()>, + Box, + Handle<()>, + ); + + fn spawn_parked_application( + context: &deterministic::Context, + marshal: fixtures::MarshalFixture, + ) -> SpawnedParkedApplication { + let (started_tx, started) = oneshot::channel(); + let processor = Processor::new( + ParkedApp { + started: Arc::new(Mutex::new(Some(started_tx))), + }, + test_databases(), + anchor(0, 0), + StatefulMetrics::new(context), + None, + ); + let (sender, receiver) = actor_mailbox::new(context.child("mailbox"), NZUsize!(8)); + let publication_context = context.child("publication"); + let (publisher, _subscriber) = Publisher::new(&publication_context); + let processing = Processing { + context: ContextCell::new(context.child("processing")), + mailbox: receiver, + provider: (), + marshal: marshal.mailbox, + snapshot_publisher: publisher, + skip_finalized_until: None, + }; + let actor = context + .child("loop") + .spawn(move |_| processing.start(processor, Vec::new())); + (Mailbox::new(sender), started, marshal.guards, actor) + } + + /// The stop signal interrupting a finalize replay parked in the application + /// exits the actor loop instead of wedging it. The block stays + /// unacknowledged so marshal redelivers it after a restart. + #[test] + fn shutdown_interrupts_a_parked_finalize() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let mut signing = context.child("signing"); + let scheme = scheme_mocks::fixture(&mut signing, b"shutdown-app", 1).schemes[0].clone(); + let marshal = fixtures::marshal_fixture( + context.child("marshal"), + "finalize-shutdown", + scheme, + None, + NZUsize!(1), + false, + ) + .await; + let (mut mailbox, started, guards, actor) = spawn_parked_application(&context, marshal); + + let genesis = TestBlock::new(0, 0); + let block = TestBlock::child(&genesis, 1); + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(block), acknowledgement)); + started.await.expect("finalize replay should start"); + + let stopper = context.child("stopper"); + context.child("stop").spawn(|_| async move { + stopper.stop(0, None).await.expect("runtime should stop"); + }); + assert!( + waiter.await.is_err(), + "an interrupted finalize must leave the block unacknowledged", + ); + actor.await.expect("the actor should exit cleanly"); + drop(guards); + }); + } + + /// A verification still in flight at shutdown never resolves for the + /// caller and never panics it, before or after the actor exits. + #[test] + fn verify_caller_parks_across_shutdown() { + deterministic::Runner::default().start(|context| async move { + let mut signing = context.child("signing"); + let scheme = scheme_mocks::fixture(&mut signing, b"shutdown-app", 1).schemes[0].clone(); + let marshal = fixtures::marshal_fixture( + context.child("marshal"), + "verify-shutdown", + scheme, + None, + NZUsize!(1), + false, + ) + .await; + let (mut mailbox, started, guards, actor) = spawn_parked_application(&context, marshal); + + let genesis = TestBlock::new(0, 0); + let block = TestBlock::child(&genesis, 1); + let mut verify = Box::pin(mailbox.verify( + (context.child("verify"), block.context()), + ancestry::from_iter([Arc::new(block), Arc::new(genesis)]), + )); + assert!(poll!(&mut verify).is_pending()); + started.await.expect("verification should start"); + + let stopper = context.child("stopper"); + context.child("stop").spawn(|_| async move { + stopper.stop(0, None).await.expect("runtime should stop"); + }); + actor.await.expect("the actor should exit cleanly"); + for _ in 0..64 { + assert!( + poll!(&mut verify).is_pending(), + "an unanswered verify must park its caller", + ); + context.sleep(Duration::from_millis(1)).await; + } + drop(guards); + }); + } + #[test] fn skip_finalized_block_skips_through_target_height() { let mut skip_until = Some(Height::new(3)); diff --git a/glue/src/stateful/actor/core/syncing.rs b/glue/src/stateful/actor/core/syncing.rs index 0b06c4efc6b..ecfe9c1e880 100644 --- a/glue/src/stateful/actor/core/syncing.rs +++ b/glue/src/stateful/actor/core/syncing.rs @@ -38,7 +38,7 @@ use commonware_utils::{ }; use rand_core::Rng; use std::{collections::VecDeque, sync::Arc}; -use tracing::{Instrument as _, debug, error, info_span}; +use tracing::{Instrument as _, debug, error, info_span, warn}; /// Finalized work needed to transition from syncing to processing. enum FinalizedHandoff { @@ -348,6 +348,10 @@ where let mut pending_prune = None; + // One signal for the whole handoff. Re-creating it per block would + // record an extra auditor event on the deterministic runtime each time. + let mut shutdown = context.stopped(); + for handoff in handoffs { match handoff { FinalizedHandoff::Covered(block, acknowledgement) @@ -358,15 +362,30 @@ where acknowledgement.acknowledge(); } FinalizedHandoff::Apply(block, acknowledgement) => { + // The stop signal covers the apply await, so an application + // parked mid-apply cannot wedge shutdown. The block stays + // unacknowledged, and marshal redelivers it after a restart. let applied; - (processor, applied) = processor - .finalize(context.as_present(), block.as_ref()) - .await; - let Applied { + select! { + _ = &mut shutdown => { + warn!( + height = block.height().get(), + "exiting mid-handoff on shutdown" + ); + return; + }, + driven = processor.finalize(context.as_present(), block.as_ref()) => { + (processor, applied) = driven; + }, + } + let Some(Applied { snapshots, barrier, prune, - } = applied.expect("sync handoff block cannot be a duplicate"); + }) = applied + else { + panic!("sync handoff block cannot be a duplicate") + }; // The processing loop's flush pool does not exist yet, so observe the // deferred flush inline. Keep state-sync metadata in progress until every From fe38dd26b1e0f62f323f7ce2038e8fb6c7b8dbb8 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 13:20:04 -0400 Subject: [PATCH 05/14] [glue] Keep verdicts branch-relative when caching is refused A verification that completed after a finalization swept its branch answered false, even though the execution matched the block's commitments on the supplied ancestry. Validity is relative to the supplied inputs, so a refused cache discards the verified state without changing the verdict, which now stays true. This also drops the classifier round trip through marshal on the refusal path, which could park forever on incomplete ancestry. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/actor/core/processing.rs | 93 +++++++++++++++++-- glue/src/stateful/actor/processor/verifier.rs | 9 +- glue/src/stateful/mod.rs | 9 +- 3 files changed, 95 insertions(+), 16 deletions(-) diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index a39e8582e5a..d73f791d142 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -682,6 +682,21 @@ mod tests { false, ) .await; + spawn_gated_application_over(context, app, marshal) + } + + /// Spawn the gated application's processing loop over a caller-supplied + /// marshal fixture. + fn spawn_gated_application_over( + context: &deterministic::Context, + app: GatedApp, + marshal: fixtures::MarshalFixture, + ) -> ( + Mailbox, + Subscriber>, + Box, + Handle<()>, + ) { let processor = Processor::new( app, test_databases(), @@ -901,6 +916,68 @@ mod tests { }); } + /// A valid verification overtaken by its own descendant's finalization + /// still answers true. + #[test] + fn overtaken_verification_of_finalized_block_answers_true() { + deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { + let (gate, started, release) = application_gate(); + let app = GatedApp { + verify_gates: Arc::new(Mutex::new(VecDeque::from([gate]))), + proposal_gate: Arc::new(Mutex::new(None)), + verify_valid: true, + stale_verifies: Arc::new(Mutex::new(0)), + observed_contexts: Arc::default(), + }; + let genesis = TestBlock::new(0, 0); + let block = TestBlock::child(&genesis, 1); + let child = TestBlock::child(&block, 2); + let mut signing = context.child("signing"); + let scheme = + scheme_mocks::fixture(&mut signing, b"gated-application", 1).schemes[0].clone(); + let marshal = fixtures::marshal_fixture_with_finalized_block( + context.child("marshal"), + "overtaken-canonical", + scheme, + &block, + NZUsize!(1), + true, + ) + .await; + let (mut mailbox, _subscriber, marshal_guards, actor) = + spawn_gated_application_over(&context, app, marshal); + + let block_context = block.context(); + let mut verifier = mailbox.clone(); + let mut verify = Box::pin(verifier.verify( + (context.child("verify"), block_context), + ancestry::from_iter([Arc::new(block.clone()), Arc::new(genesis)]), + )); + assert!(poll!(&mut verify).is_pending()); + started.await.expect("verification should start"); + + // The candidate and then its child finalize while the verification + // is parked, moving the anchor past the candidate's height. + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(block), acknowledgement)); + waiter + .await + .expect("candidate finalization should be acknowledged"); + let (acknowledgement, waiter) = Exact::handle(); + let _ = mailbox.report(Update::Block(Arc::new(child), acknowledgement)); + waiter + .await + .expect("child finalization should be acknowledged"); + + // The resumed execution succeeds with no stale read to surface, and + // the refused cache does not change the verdict. + release.send(()).expect("verification should remain active"); + assert!(verify.await); + actor.abort(); + drop(marshal_guards); + }); + } + /// A verification that goes stale because a competing block finalized is /// answered from the canonical chain as false. #[test] @@ -1421,7 +1498,7 @@ mod tests { } #[test] - fn finalization_refuses_incompatible_verification_result() { + fn finalized_away_fork_verification_answers_true() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let (fork_gate, fork_started, fork_release) = application_gate(); let (child_gate, child_started, child_release) = application_gate(); @@ -1477,14 +1554,14 @@ mod tests { } // The losing child runs to completion. Its parent is gone from the - // pending set, so caching its result is refused and the verdict is - // false. + // pending set, so caching its result is refused, and the verdict + // is unchanged. child_release .send(()) .expect("the attempt should still be live after the apply"); select! { valid = &mut verify_child => { - assert!(!valid, "verification on a finalized-away fork must fail"); + assert!(valid, "a branch-valid verification answers true on a finalized-away fork"); }, _ = context.sleep(Duration::from_millis(100)) => { panic!("incompatible verification did not resolve"); @@ -1494,8 +1571,10 @@ mod tests { }); } + /// A verification whose fork was swept by a finalization still answers its + /// branch-relative verdict. #[test] - fn finalization_rejects_deep_incompatible_verification() { + fn pruned_deep_fork_verification_answers_true() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { let (parent_gate, parent_started, parent_release) = application_gate(); let (child_gate, child_started, child_release) = application_gate(); @@ -1575,8 +1654,8 @@ mod tests { actor.abort(); assert_eq!( result, - Some(false), - "verification on a pruned deep fork must resolve false", + Some(true), + "a branch-valid verification answers true even after its fork is pruned", ); }); } diff --git a/glue/src/stateful/actor/processor/verifier.rs b/glue/src/stateful/actor/processor/verifier.rs index 76dc44f135c..e46c1053c99 100644 --- a/glue/src/stateful/actor/processor/verifier.rs +++ b/glue/src/stateful/actor/processor/verifier.rs @@ -457,16 +457,19 @@ where ); return Attempt::Done(VerificationResult::Decided(false)); } + // Caching is retention, not part of the verdict. The execution matched + // the block's commitments on its own branch, and a finalization + // discarding the entry does not change that answer. if !self .execution .cache_pending(block_digest, parent.digest, round, merkleized) { - warn!( + debug!( parent_digest = ?parent.digest, ?block_digest, - "verification result became incompatible before caching" + "verified state not cached, overtaken by finalization" ); - return Attempt::Done(VerificationResult::Decided(false)); + return Attempt::Done(VerificationResult::Decided(true)); } self.execution.update_pending_metric(); drop(block); diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index 0895a559a61..d3514cc953d 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -280,12 +280,9 @@ where /// voting, do not resolve this future yet. Abstaining is not represented by /// a special return value. /// - /// Validity is relative to the supplied inputs, and the boundary is the - /// applied anchor move. A verdict completed before the anchor moves stays - /// valid even when a competing branch finalizes afterward. A verdict that - /// completes after the anchor moved is answered from the new canonical - /// chain instead. True when the block itself became canonical, and false - /// when its branch is no longer reachable. + /// Validity is relative to the supplied inputs. A completed verdict stays + /// valid even when a competing branch finalizes: the wrapper may discard + /// the verified state instead of caching it, but the answer is unchanged. /// /// Verification must reject any block whose execution result does not /// match the block's committed state (for example, a state root mismatch). From dbf3b8bee1ba29badfb0f94b24ba02b5b498b378 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 14:03:25 -0400 Subject: [PATCH 06/14] [glue] Polish three actor comments Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/db/mod.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index 13e6c253af6..ffb16da91d6 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -923,8 +923,8 @@ macro_rules! impl_database_set { } async fn finalize(self, batches: Self::Merkleized) -> (Self, Self::Snapshots, Barrier) { - // Every database captures at its own apply boundary inside this call, so the - // captured snapshots form one capture. + // Every database captures at its own apply boundary inside this call, so + // the snapshots together describe the set's one post-apply state. let results = join!($( finalize_or_panic(self.$idx, batches.$idx, Some($idx)), )+); From 6dbc023c187f9562fc1441a98b778c3c9313cb7d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 14:10:19 -0400 Subject: [PATCH 07/14] [glue] Say read guard, not lease The cell hands out read guards. One word for the concept, matching the type's name. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/db/any.rs | 2 +- glue/src/stateful/db/cell.rs | 62 +++++++++++----------- glue/src/stateful/db/current.rs | 2 +- glue/src/stateful/db/immutable/standard.rs | 2 +- glue/src/stateful/db/keyless/standard.rs | 2 +- glue/src/stateful/db/mod.rs | 10 ++-- 6 files changed, 40 insertions(+), 40 deletions(-) diff --git a/glue/src/stateful/db/any.rs b/glue/src/stateful/db/any.rs index ceaaba31979..6d65e0c2588 100644 --- a/glue/src/stateful/db/any.rs +++ b/glue/src/stateful/db/any.rs @@ -2,7 +2,7 @@ //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for //! read-through to applied state. The wrapper types here hold a [`Reader`] -//! to their database and lease it for each such call, so a batch stays usable +//! to their database and take read access through it for each such call, so a batch stays usable //! across applies of compatible batches and never delays a mutation by more //! than one storage call. diff --git a/glue/src/stateful/db/cell.rs b/glue/src/stateful/db/cell.rs index 2abd72a1883..5f2b1dbc6a5 100644 --- a/glue/src/stateful/db/cell.rs +++ b/glue/src/stateful/db/cell.rs @@ -2,31 +2,31 @@ //! //! [`split`] wraps a database and returns two capabilities over it. The //! [`Writer`] is unique and runs consuming mutations. The [`Reader`] is -//! freely cloned into batches, and grants short leases that cover exactly one -//! storage call. +//! freely cloned into batches, and hands out short read guards, each covering +//! exactly one storage call. //! //! Neither value owns the database. The cell does, and both capabilities keep //! it alive. What distinguishes them is what they permit. //! //! The cell is a tokio read-write lock, whose documented policy is fair and -//! write-preferring, so a waiting mutation blocks later leases and cannot be -//! starved, while leases already granted finish first. The write side covers +//! write-preferring, so a waiting mutation blocks new read guards and cannot be +//! starved, while guards already granted finish first. The write side covers //! the whole take-and-restore of a mutation, so a reader can never observe the //! database missing. A mutation that is interrupted mid-flight leaves the cell //! poisoned, and the only thing that interrupts one here is the owning task -//! being torn down, so later leases park forever and are dropped along with +//! being torn down, so later reads park forever and are dropped along with //! the tasks holding them. //! -//! A lease guarantees the database is present and unchanging for one call, not -//! that the caller's batch is still current. A batch operation under a lease +//! A read guard guarantees the database is present and unchanging for one call, +//! not that the caller's batch is still current. A batch operation holding one //! can still refuse because a competing batch was applied (see //! [`commonware_storage::qmdb::Error::StaleRead`]). //! //! Two rules keep callers out of trouble. The lock is not reentrant, so never -//! hold a lease while acquiring another, on any cell, because a mutation queued +//! hold a read guard while acquiring another, on any cell, because a mutation queued //! between the two would deadlock both. And a [`Writer`] should outlive the //! readers from the same cell. Dropping the writer closes the cell, and later -//! leases park instead of answering from a database that can never advance. +//! reads park instead of answering from a database that can never advance. use commonware_utils::sync::{AsyncRwLockReadGuard, TracedAsyncRwLock}; use futures::future; @@ -48,8 +48,8 @@ enum State { struct Cell { state: TracedAsyncRwLock>, /// Latched when the writer drops. A one-way flag, so a relaxed load - /// suffices, and a lease that races the drop is indistinguishable from - /// one granted a moment earlier. + /// suffices, and a read guard that races the drop is indistinguishable + /// from one granted a moment earlier. closed: AtomicBool, } @@ -67,7 +67,7 @@ impl Cell { State::Live(db) => Some(db), State::Poisoned => None, }) { - Ok(lease) => ReadGuard(lease), + Ok(guard) => ReadGuard(guard), Err(guard) => { // Poisoning is only reachable during writer teardown. Park until // this task is dropped with the rest of the actor, and the trace @@ -93,7 +93,7 @@ pub fn split(db: T) -> (Writer, Reader) { /// This is the only value with [`mutate`](Self::mutate), and it is deliberately /// not [`Clone`], so at most one exists per cell. It does not own the database. /// Dropping it closes the cell, since the database can never advance again, and -/// later leases park instead of serving frozen state. +/// later reads park instead of serving frozen state. pub struct Writer(Arc>); impl Drop for Writer { @@ -118,15 +118,15 @@ impl Writer { /// Run one consuming mutation to completion, returning the capability. /// - /// New leases queue behind the mutation and leases already granted finish - /// first, so this waits at most one storage call before starting. + /// New read guards queue behind the mutation and guards already granted + /// finish first, so this waits at most one storage call before starting. /// /// Consume/produce at both levels. `mutation` takes the database by value and /// must produce it back, the contract mutable storage operations already use. /// This method does the same with the capability. An interrupted mutation /// takes the writer with it, and if the database was already taken out it /// also poisons the cell, so a second mutation of a poisoned cell is - /// unreachable rather than merely documented. Leases taken afterward park + /// unreachable rather than merely documented. Reads taken afterward park /// forever. (Dropped while still queued for the lock, the writer's own drop /// closes the cell the same way.) pub async fn mutate(self, mutation: F) -> (Self, R) @@ -155,13 +155,13 @@ impl Clone for Reader { } impl Reader { - /// Acquire a read lease. + /// Acquire a read guard. pub async fn read(&self) -> ReadGuard<'_, T> { self.0.read().await } } -/// A short read lease. Must cover exactly one storage call, never an +/// A short read guard. Must cover exactly one storage call, never an /// application await, so a waiting mutation is delayed by at most one call. pub struct ReadGuard<'a, T>(AsyncRwLockReadGuard<'a, T>); @@ -181,8 +181,8 @@ mod tests { use futures::FutureExt as _; use std::time::Duration; - /// A waiting mutation cannot be starved by a stream of short read leases, - /// and no lease ever observes taken-out state. + /// A waiting mutation cannot be starved by a stream of short reads, and + /// no read ever observes taken-out state. #[test] fn mutation_is_not_starved_by_read_storm() { deterministic::Runner::default().start(|context| async move { @@ -194,10 +194,10 @@ mod tests { workers.push(context.child(worker).spawn(move |ctx| async move { loop { { - let lease = reader.read().await; + let guard = reader.read().await; // Every observation is a full, live value. - assert!(*lease == 0 || *lease == 1); - if *lease == 1 { + assert!(*guard == 0 || *guard == 1); + if *guard == 1 { return; } } @@ -220,10 +220,10 @@ mod tests { }); } - /// A lease waits out an in-flight mutation and then sees the mutated + /// A read waits out an in-flight mutation and then sees the mutated /// state, never a gap. #[test] - fn leases_wait_out_a_parked_mutation() { + fn reads_wait_out_a_parked_mutation() { deterministic::Runner::default().start(|context| async move { let (writer, reader) = split(0u64); let (release_tx, release) = oneshot::channel::<()>(); @@ -245,16 +245,16 @@ mod tests { futures::pin_mut!(read); assert!( read.as_mut().now_or_never().is_none(), - "a lease must wait while a mutation holds the cell", + "a read must wait while a mutation holds the cell", ); release_tx.send(()).expect("mutation is waiting"); let _writer = mutation_task.await.expect("mutation completes"); - assert_eq!(*read.await, 1, "the lease sees the mutated state"); + assert_eq!(*read.await, 1, "the read sees the mutated state"); }); } - /// Dropping the writer closes the cell. Later leases park instead of + /// Dropping the writer closes the cell. Later reads park instead of /// answering from a database that can never advance. #[test] fn dropped_writer_parks_readers() { @@ -267,13 +267,13 @@ mod tests { futures::pin_mut!(read); assert!( read.as_mut().now_or_never().is_none(), - "a lease after the writer drops must park, not serve frozen state", + "a read after the writer drops must park, not serve frozen state", ); }); } /// Dropping a mutation mid-flight poisons the cell, and takes the writer - /// with it. Later leases park forever instead of observing missing state. + /// with it. Later reads park forever instead of observing missing state. /// A second mutation is unrepresentable, so there is nothing to assert. #[test] fn interrupted_mutation_poisons() { @@ -297,7 +297,7 @@ mod tests { futures::pin_mut!(read); assert!( read.as_mut().now_or_never().is_none(), - "a lease after poisoning must park, not observe a gap", + "a read after poisoning must park, not observe a gap", ); }); } diff --git a/glue/src/stateful/db/current.rs b/glue/src/stateful/db/current.rs index 997e3bac239..9df7b1488e3 100644 --- a/glue/src/stateful/db/current.rs +++ b/glue/src/stateful/db/current.rs @@ -2,7 +2,7 @@ //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for //! read-through to applied state. The wrapper types here hold a [`Reader`] -//! to their database and lease it for each such call, so a batch stays usable +//! to their database and take read access through it for each such call, so a batch stays usable //! across applies of compatible batches and never delays a mutation by more //! than one storage call. diff --git a/glue/src/stateful/db/immutable/standard.rs b/glue/src/stateful/db/immutable/standard.rs index 1d45913d5e7..75d0b06d93d 100644 --- a/glue/src/stateful/db/immutable/standard.rs +++ b/glue/src/stateful/db/immutable/standard.rs @@ -2,7 +2,7 @@ //! [`immutable`](commonware_storage::qmdb::immutable) databases. //! //! Immutable databases support adding new keyed values but not updates or -//! deletions. Keyed batch reads lease the database through the batch's +//! deletions. Keyed batch reads access the database through the batch's //! [`Reader`] because the immutable proof snapshot carries no keyed //! index. diff --git a/glue/src/stateful/db/keyless/standard.rs b/glue/src/stateful/db/keyless/standard.rs index e1d9a729c24..9dc93f86961 100644 --- a/glue/src/stateful/db/keyless/standard.rs +++ b/glue/src/stateful/db/keyless/standard.rs @@ -2,7 +2,7 @@ //! [`keyless`](commonware_storage::qmdb::keyless) databases. //! //! Keyless databases are append-only. Operations are addressed by -//! [`Location`] rather than by key. Positional batch reads lease the +//! [`Location`] rather than by key. Positional batch reads access the //! database through the batch's [`Reader`]. use crate::stateful::db::{ diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index ffb16da91d6..eb7df7b1068 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -23,7 +23,7 @@ //! mutation is uniquely permitted. Batches hold a [`Reader`], which is freely //! cloned and grants a [`ReadGuard`] covering exactly one storage call. //! -//! Because a lease never spans application code, a mutation waits at most one +//! Because a read guard never spans application code, a mutation waits at most one //! storage call to start, and work holding a reader is never cancelled, so it //! pauses at its next read and resumes once the mutation completes. A mutation //! that is interrupted leaves the cell poisoned, which is reachable only while @@ -32,7 +32,7 @@ //! Two invariants keep this sound. A batch handed to [`ManagedDb::finalize`] //! must not read through its own reader, because that call runs while the write //! side is held. And a [`Writer`] should outlive the readers from the same -//! cell. Dropping the writer closes the cell, and later leases park instead of +//! cell. Dropping the writer closes the cell, and later reads park instead of //! answering from a database that can never advance. The set holds the //! [`Writer`], and every reader lives in a batch the set outlives. //! @@ -185,7 +185,7 @@ pub trait Merkleized: Clone + Sized + Send + Sync { /// batches back to storage, deferring each batch's flush to a returned handle. /// /// Batches carry a [`Reader`] to their database. Reads acquire a short -/// lease per call and fall back from pending batch state to applied state. +/// guard per call and fall back from pending batch state to applied state. /// /// `E` is a trait generic (not an associated type), so one database type can /// work across runtimes that satisfy the bounds. @@ -238,7 +238,7 @@ pub trait ManagedDb: Send + Sync + Sized { /// Create a new unmerkleized batch rooted at the database's applied /// state. /// - /// The batch keeps `reader` and leases the database through it on every + /// The batch keeps `reader` and takes read access through it on every /// read, so it stays valid across applies of compatible batches. fn new_batch(reader: Reader) -> impl Future + Send; @@ -386,7 +386,7 @@ pub trait DatabaseSet: Send + Sync + Sized + 'static { /// One [`Reader`] per database, shaped like [`Self::Unmerkleized`]. /// /// Readers are cloned into batches, and hooks that read applied state - /// directly acquire leases through them. + /// directly acquire read guards through them. type Readers: Clone + Send + Sync + 'static; /// One [`ManagedDb::Snapshot`] per database, shaped like [`Self::Unmerkleized`]. From cd4f64accc4ca4a4c64b1be04b135089c72ff81f Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 14:40:40 -0400 Subject: [PATCH 08/14] [glue] Name the gated-application spawn tuple CI's newer clippy flags the sync helper's return type as too complex. A shared alias satisfies it and names the tuple for both spawn helpers. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/actor/core/processing.rs | 23 +++++++++++----------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index d73f791d142..feef71d59d6 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -660,16 +660,20 @@ mod tests { ) } - async fn spawn_gated_application( - context: &deterministic::Context, - prefix: &str, - app: GatedApp, - ) -> ( + /// A spawned gated application's mailbox, snapshot subscriber, marshal + /// guard, and actor handle. + type GatedApplication = ( Mailbox, Subscriber>, Box, Handle<()>, - ) { + ); + + async fn spawn_gated_application( + context: &deterministic::Context, + prefix: &str, + app: GatedApp, + ) -> GatedApplication { let mut signing = context.child("signing"); let scheme = scheme_mocks::fixture(&mut signing, b"gated-application", 1).schemes[0].clone(); @@ -691,12 +695,7 @@ mod tests { context: &deterministic::Context, app: GatedApp, marshal: fixtures::MarshalFixture, - ) -> ( - Mailbox, - Subscriber>, - Box, - Handle<()>, - ) { + ) -> GatedApplication { let processor = Processor::new( app, test_databases(), From a1b72d2580d86b2a5b658470015ad9a4f804f40d Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 14:45:15 -0400 Subject: [PATCH 09/14] [glue] Review polish for the owned databases Scope the reentrancy and never-cancels claims to what actually holds, name every reader holder in the set doc, unify a duplicated panic message and test helper, and tidy two log lines and a syncer comment. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/actor/core/processing.rs | 5 ++-- glue/src/stateful/actor/processor/mod.rs | 2 +- glue/src/stateful/actor/syncer/mod.rs | 7 ++--- glue/src/stateful/db/cell.rs | 9 ++++--- glue/src/stateful/db/mod.rs | 30 ++++++---------------- glue/src/stateful/mod.rs | 12 ++++----- 6 files changed, 25 insertions(+), 40 deletions(-) diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index feef71d59d6..1e7d4b2b154 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -1,8 +1,9 @@ //! The post-sync processing loop of the stateful actor. //! //! The loop owns the database set and is the only thing that mutates it. -//! Verification jobs hold readers instead, so they are never cancelled. A -//! job that is mid-read when a finalized block arrives finishes that read, the +//! Verification jobs hold readers instead, so nothing short of actor shutdown +//! cancels one. A job that is mid-read when a finalized block arrives finishes +//! that read, the //! apply runs, and the job continues against the state it installs. A job on //! the losing side of that apply is refused at its next batch operation //! ([`ExecutionError::Stale`](crate::stateful::ExecutionError::Stale)) and diff --git a/glue/src/stateful/actor/processor/mod.rs b/glue/src/stateful/actor/processor/mod.rs index 73de9cc4874..538fa555fa8 100644 --- a/glue/src/stateful/actor/processor/mod.rs +++ b/glue/src/stateful/actor/processor/mod.rs @@ -2335,7 +2335,7 @@ mod tests { .finalize(harness.context_cell.as_present(), &winner), ); select! { - _ = &mut finalize => panic!("finalize completed before its hook returned"), + _ = &mut finalize => panic!("finalize completed before its finalized hook returned"), result = &mut started => result.expect("finalized hook should start"), } diff --git a/glue/src/stateful/actor/syncer/mod.rs b/glue/src/stateful/actor/syncer/mod.rs index 20000714dca..c3e93f8d728 100644 --- a/glue/src/stateful/actor/syncer/mod.rs +++ b/glue/src/stateful/actor/syncer/mod.rs @@ -411,11 +411,8 @@ where let mut databases = A::Databases::init(context.child("db_set"), db_config).await; let processed_targets = A::sync_targets(&floor_block); - // In the case that the applied targets do not match the marshal floor, we may - // have suffered a crash that left the set in an inconsistent state. In this case, - // we attempt to repair by rewinding the databases back to the marshal floor. If - // the rewind fails to produce a consistent state, we must crash. This can occur - // if the databases were corrupted or pruned too aggressively. + // Applied targets off the marshal floor mean a crash left the set + // inconsistent. Rewinding to the floor repairs it or panics. if databases.committed_targets().await != processed_targets { databases = databases.rewind_to_targets(processed_targets.clone()).await; assert!( diff --git a/glue/src/stateful/db/cell.rs b/glue/src/stateful/db/cell.rs index 5f2b1dbc6a5..0176c304c99 100644 --- a/glue/src/stateful/db/cell.rs +++ b/glue/src/stateful/db/cell.rs @@ -23,8 +23,9 @@ //! [`commonware_storage::qmdb::Error::StaleRead`]). //! //! Two rules keep callers out of trouble. The lock is not reentrant, so never -//! hold a read guard while acquiring another, on any cell, because a mutation queued -//! between the two would deadlock both. And a [`Writer`] should outlive the +//! hold a read guard while acquiring another. On the same cell a mutation queued +//! between the two deadlocks both, and across cells the same habit invites +//! order-inversion deadlocks. And a [`Writer`] should outlive the //! readers from the same cell. Dropping the writer closes the cell, and later //! reads park instead of answering from a database that can never advance. @@ -60,7 +61,7 @@ impl Cell { // The writer is gone, so the database can never advance. Serving // would hand out frozen state. Park like the poisoned case. drop(guard); - tracing::error!("database cell closed; parking reader"); + tracing::error!("database cell closed, parking reader"); return future::pending().await; } match AsyncRwLockReadGuard::try_map(guard, |state| match state { @@ -73,7 +74,7 @@ impl Cell { // this task is dropped with the rest of the actor, and the trace // separates that from a bug if the process outlives the cell. drop(guard); - tracing::error!("database cell poisoned; parking reader"); + tracing::error!("database cell poisoned, parking reader"); future::pending().await } } diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index eb7df7b1068..528166d4202 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -24,7 +24,7 @@ //! cloned and grants a [`ReadGuard`] covering exactly one storage call. //! //! Because a read guard never spans application code, a mutation waits at most one -//! storage call to start, and work holding a reader is never cancelled, so it +//! storage call to start, and no mutation ever cancels work holding a reader, so it //! pauses at its next read and resumes once the mutation completes. A mutation //! that is interrupted leaves the cell poisoned, which is reachable only while //! the mutating task is being torn down. @@ -34,7 +34,9 @@ //! side is held. And a [`Writer`] should outlive the readers from the same //! cell. Dropping the writer closes the cell, and later reads park instead of //! answering from a database that can never advance. The set holds the -//! [`Writer`], and every reader lives in a batch the set outlives. +//! [`Writer`]; readers live in batches, in verification jobs, and in the +//! [`Application::finalized`](crate::stateful::Application::finalized) hook, +//! which may keep one past the actor's exit, where its reads park. //! //! # State Sync //! @@ -1795,7 +1797,7 @@ mod tests { use super::{ Anchor, Barrier, CoordinatorAction, CoordinatorState, DatabaseSet, MAX_CHANNEL_DRAIN_PER_TICK, ManagedDb, Reader, Single, StateSyncDb, StateSyncSet, - SyncEngineConfig, TipUpdate, Writer, drain_single_tip_updates, split, + SyncEngineConfig, TipUpdate, drain_single_tip_updates, split, }; use crate::stateful::tests::mocks::{TestMerkleized, TestUnmerkleized, anchor as mock_anchor}; use commonware_cryptography::sha256; @@ -1820,12 +1822,12 @@ mod tests { }; mod managed_db_lifecycle { - use super::{ManagedDb, Writer, split}; - use crate::stateful::db::Unmerkleized; + use super::{ManagedDb, split}; + use crate::stateful::{db::Unmerkleized, tests::mocks::finalize}; use commonware_cryptography::{Sha256, sha256::Digest}; use commonware_parallel::Sequential; use commonware_runtime::{ - Handle, Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, + Runner as _, Supervisor as _, buffer::paged::CacheRef, deterministic, }; use commonware_storage::{ journal::contiguous::{ @@ -2136,22 +2138,6 @@ mod tests { } } - /// Finalize `batch` through the cell, returning the snapshot and flush handle. - async fn finalize>( - writer: Writer, - batch: T::Merkleized, - ) -> (Writer, T::Snapshot, Handle<()>) { - let (writer, (snapshot, sync)) = writer - .mutate(|db| async move { - let (db, snapshot, sync) = T::finalize(db, batch).await.unwrap_or_else(|err| { - panic!("finalize failed: {err:?}"); - }); - (db, (snapshot, sync)) - }) - .await; - (writer, snapshot, sync) - } - async fn assert_initial_sync_target_and_finalize(context: Context, config: T::Config) where T: ManagedDb + 'static, diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index d3514cc953d..a4f129c74e1 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -297,9 +297,9 @@ where /// ops root and operation range used by replay sync. /// /// This future is scoped to its caller. Dropping the response cancels only - /// this request. The wrapper never cancels it, so a batch operation running - /// when a finalized block is applied waits for that apply and then - /// continues. + /// this request. The wrapper never cancels it while the actor runs, so a + /// batch operation running when a finalized block is applied waits for that + /// apply and then continues. Actor shutdown drops it with everything else. /// /// `batches` is a branch-scoped view, not a historical snapshot. Retained /// ancestor overlays preserve same-branch state, while unresolved reads fall @@ -328,9 +328,9 @@ where /// replay result during finalization and cannot re-check block-specific /// commitments generically. /// - /// This future may be cancelled if its originating request is dropped. The - /// wrapper itself never cancels it. Cancellation must not violate - /// invariants or lose durable progress. + /// This future may be cancelled if its originating request is dropped or + /// the actor shuts down; the wrapper never cancels it while the actor runs. + /// Cancellation must not violate invariants or lose durable progress. /// /// Storage errors from batch operations are propagated as [`ExecutionError`], /// never interpreted (see [`verify`](Self::verify)). The wrapper re-checks From 559cc14bb37a2df4bdc9763cff6bddc3bbf5e886 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 15:09:29 -0400 Subject: [PATCH 10/14] [glue] Call the flush handle a handle again The finalize paths renamed the flush handle binding to sync for no reason. Restore the name the callers and docs use. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01Q92PhqGoHbWCGHCsBJ7EBU --- glue/src/stateful/db/any.rs | 8 ++++---- glue/src/stateful/db/current.rs | 16 ++++++++-------- glue/src/stateful/db/immutable/compact.rs | 8 ++++---- glue/src/stateful/db/immutable/standard.rs | 8 ++++---- glue/src/stateful/db/keyless/compact.rs | 8 ++++---- glue/src/stateful/db/keyless/standard.rs | 8 ++++---- glue/src/stateful/db/mod.rs | 22 +++++++++++----------- glue/src/stateful/tests/mocks.rs | 8 ++++---- 8 files changed, 43 insertions(+), 43 deletions(-) diff --git a/glue/src/stateful/db/any.rs b/glue/src/stateful/db/any.rs index 6d65e0c2588..0d43dd25264 100644 --- a/glue/src/stateful/db/any.rs +++ b/glue/src/stateful/db/any.rs @@ -551,9 +551,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -671,9 +671,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/current.rs b/glue/src/stateful/db/current.rs index 9df7b1488e3..688303ece61 100644 --- a/glue/src/stateful/db/current.rs +++ b/glue/src/stateful/db/current.rs @@ -551,9 +551,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -664,9 +664,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -859,9 +859,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -981,9 +981,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/immutable/compact.rs b/glue/src/stateful/db/immutable/compact.rs index 528d1bed2df..2e89ad86046 100644 --- a/glue/src/stateful/db/immutable/compact.rs +++ b/glue/src/stateful/db/immutable/compact.rs @@ -263,9 +263,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, sync)) + Ok((db, snapshot, handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -345,9 +345,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, sync)) + Ok((db, snapshot, handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/immutable/standard.rs b/glue/src/stateful/db/immutable/standard.rs index 75d0b06d93d..fe34e6de0b2 100644 --- a/glue/src/stateful/db/immutable/standard.rs +++ b/glue/src/stateful/db/immutable/standard.rs @@ -335,9 +335,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -441,9 +441,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/keyless/compact.rs b/glue/src/stateful/db/keyless/compact.rs index b5a4b0babc3..b74173b1423 100644 --- a/glue/src/stateful/db/keyless/compact.rs +++ b/glue/src/stateful/db/keyless/compact.rs @@ -252,9 +252,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, sync)) + Ok((db, snapshot, handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -333,9 +333,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner)?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let snapshot = Self::snapshot(&db); - Ok((db, snapshot, sync)) + Ok((db, snapshot, handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/keyless/standard.rs b/glue/src/stateful/db/keyless/standard.rs index 9dc93f86961..12a89d2136e 100644 --- a/glue/src/stateful/db/keyless/standard.rs +++ b/glue/src/stateful/db/keyless/standard.rs @@ -301,9 +301,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { @@ -400,9 +400,9 @@ where batch: Self::Merkleized, ) -> Result<(Self, Self::Snapshot, Handle<()>), Error> { let (db, _) = self.apply_batch(batch.inner).await?; - let (db, sync) = db.start_sync().await?; + let (db, handle) = db.start_sync().await?; let (db, snapshot) = db.snapshot().await?; - Ok((db, Arc::new(snapshot), sync)) + Ok((db, Arc::new(snapshot), handle)) } async fn snapshot(self) -> Result<(Self, Self::Snapshot), Error> { diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index 528166d4202..da5b7da1512 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -547,9 +547,9 @@ where } async fn finalize(self, batches: Self::Merkleized) -> (Self, Self::Snapshots, Barrier) { - let (member, snapshot, sync) = finalize_or_panic(self, batches, None).await; + let (member, snapshot, handle) = finalize_or_panic(self, batches, None).await; let barrier = Barrier { - syncs: vec![(core::any::type_name::(), None, sync)], + syncs: vec![(core::any::type_name::(), None, handle)], }; (member, snapshot, barrier) } @@ -1720,10 +1720,10 @@ async fn finalize_or_panic>( // Mutable finalize failures are fatal by design because the batch may already have been // applied to other databases in the same set, leaving partially applied state. let Single { writer } = member; - let (writer, (snapshot, sync)) = writer + let (writer, (snapshot, handle)) = writer .mutate(|database| async move { match database.finalize(batch).await { - Ok((database, snapshot, sync)) => (database, (snapshot, sync)), + Ok((database, snapshot, handle)) => (database, (snapshot, handle)), Err(err) => { let index = index.map_or(String::new(), |i| format!("index {i}, ")); panic!( @@ -1734,7 +1734,7 @@ async fn finalize_or_panic>( } }) .await; - (Single { writer }, snapshot, sync) + (Single { writer }, snapshot, handle) } #[tracing::instrument(name = "stateful.db.rewind_or_panic", level = "info", skip_all, fields(index = index))] @@ -2153,9 +2153,9 @@ mod tests { .merkleize() .await .expect("empty batch must merkleize"); - let (_writer, snapshot, sync) = finalize(writer, batch).await; + let (_writer, snapshot, handle) = finalize(writer, batch).await; drop(snapshot); - sync.await.expect("empty batch finalize flush failed"); + handle.await.expect("empty batch finalize flush failed"); } #[rstest] @@ -2227,9 +2227,9 @@ mod tests { .merkleize() .await .expect("first batch must merkleize"); - let (writer, snapshot, sync) = finalize(writer, batch).await; + let (writer, snapshot, handle) = finalize(writer, batch).await; drop(snapshot); - sync.await.expect("first finalize flush failed"); + handle.await.expect("first finalize flush failed"); let target = reader.read().await.sync_target(); let batch = T::new_batch(reader.clone()) @@ -2237,9 +2237,9 @@ mod tests { .merkleize() .await .expect("second batch must merkleize"); - let (writer, snapshot, sync) = finalize(writer, batch).await; + let (writer, snapshot, handle) = finalize(writer, batch).await; drop(snapshot); - sync.await.expect("second finalize flush failed"); + handle.await.expect("second finalize flush failed"); let (_writer, ()) = writer .mutate(|db| { diff --git a/glue/src/stateful/tests/mocks.rs b/glue/src/stateful/tests/mocks.rs index 7feb49e7019..2af2f87524d 100644 --- a/glue/src/stateful/tests/mocks.rs +++ b/glue/src/stateful/tests/mocks.rs @@ -315,15 +315,15 @@ pub(crate) async fn finalize>( writer: Writer, batch: D::Merkleized, ) -> (Writer, D::Snapshot, Handle<()>) { - let (writer, (snapshot, sync)) = writer + let (writer, (snapshot, handle)) = writer .mutate(|db| async move { - let (db, snapshot, sync) = D::finalize(db, batch) + let (db, snapshot, handle) = D::finalize(db, batch) .await .unwrap_or_else(|err| panic!("finalize failed: {err:?}")); - (db, (snapshot, sync)) + (db, (snapshot, handle)) }) .await; - (writer, snapshot, sync) + (writer, snapshot, handle) } pub(crate) fn anchor(height: u64, digest_byte: u8) -> crate::stateful::db::Anchor { From fc1b98b6f74b643149ffd38b8451288630ae775c Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 20:26:36 -0400 Subject: [PATCH 11/14] cleanup --- glue/src/stateful/actor/core/mailbox.rs | 5 +- glue/src/stateful/actor/core/processing.rs | 61 ++++------- glue/src/stateful/actor/core/syncing.rs | 5 +- glue/src/stateful/actor/core/verifications.rs | 4 +- glue/src/stateful/actor/processor/mod.rs | 101 +++++++++--------- glue/src/stateful/actor/processor/verifier.rs | 20 ++-- glue/src/stateful/actor/syncer/actor.rs | 14 +-- glue/src/stateful/actor/syncer/mod.rs | 8 +- glue/src/stateful/db/any.rs | 4 +- glue/src/stateful/db/cell.rs | 92 +++++----------- glue/src/stateful/db/current.rs | 4 +- glue/src/stateful/db/mod.rs | 29 ++--- glue/src/stateful/mod.rs | 20 ++-- glue/src/stateful/tests/mod.rs | 4 +- glue/src/stateful/tests/ownership.rs | 2 +- 15 files changed, 139 insertions(+), 234 deletions(-) diff --git a/glue/src/stateful/actor/core/mailbox.rs b/glue/src/stateful/actor/core/mailbox.rs index 9b998052adf..f71cb4a9370 100644 --- a/glue/src/stateful/actor/core/mailbox.rs +++ b/glue/src/stateful/actor/core/mailbox.rs @@ -229,9 +229,8 @@ where }); match receiver.await { Ok(valid) => valid, - // The actor exited or discarded the request while shutting down. - // Never fabricate a verdict. Park until the caller loses interest - // and drops this future. + // The actor exited without answering. Never fabricate a verdict. + // Park until this future is dropped. Err(_) => std::future::pending().await, } } diff --git a/glue/src/stateful/actor/core/processing.rs b/glue/src/stateful/actor/core/processing.rs index 1e7d4b2b154..0c47ec01d8d 100644 --- a/glue/src/stateful/actor/core/processing.rs +++ b/glue/src/stateful/actor/core/processing.rs @@ -1,25 +1,17 @@ //! The post-sync processing loop of the stateful actor. //! //! The loop owns the database set and is the only thing that mutates it. -//! Verification jobs hold readers instead, so nothing short of actor shutdown -//! cancels one. A job that is mid-read when a finalized block arrives finishes -//! that read, the -//! apply runs, and the job continues against the state it installs. A job on -//! the losing side of that apply is refused at its next batch operation +//! Verification jobs hold readers, so an apply never cancels one: a job +//! mid-read finishes that read and continues against the new state. A job on +//! the losing side of the apply is refused at its next batch operation //! ([`ExecutionError::Stale`](crate::stateful::ExecutionError::Stale)) and //! answered from the canonical chain. //! -//! Each finalized block is applied to the databases, its flush deferred to a -//! pool, and a snapshot of the applied state published for serving as soon as -//! the apply completes. Served state may run ahead of disk, which is safe -//! because peers verify what they fetch against a finalized root. The block is -//! acknowledged to marshal only once its flush is durable, so marshal's floor -//! never gets ahead of disk. -//! -//! Pruning is maintenance, run only while the mailbox is idle. A prune waits -//! until the pruned range is durable, prunes, and publishes fresh snapshots -//! right away, since the served snapshots pin the pruned storage (see -//! [`Publisher`]). +//! Each finalized block is applied, its flush deferred to a pool, and its +//! snapshots published at apply (see [`Publisher`]). The block is +//! acknowledged to marshal only once its flush is durable. Pruning runs +//! while the mailbox is idle, waits until the pruned range is durable, and +//! publishes fresh snapshots right away. use crate::stateful::{ Application, Input, @@ -142,8 +134,7 @@ where } // Publish completed verdicts before admitting another message, so - // continuous mailbox traffic cannot starve them under the biased - // `select!`. + // mailbox traffic cannot starve them. verifications.complete_ready(); // A message deferred by an active proposal is the FIFO barrier @@ -314,13 +305,10 @@ where continue; } - // The apply owns mutation. Live verification jobs pause at - // their next batch operation and resume afterward, so they - // keep being polled throughout. The stop signal covers the - // await: an application parked mid-apply cannot wedge - // shutdown, and exiting just drops un-applied batches. The - // block stays unacknowledged, and marshal redelivers it - // after a restart. + // Verification jobs keep running during the apply, + // pausing at their next batch read. Exiting on stop drops + // the un-applied batches, and marshal redelivers the + // unacknowledged block after restart. let applied; select! { _ = &mut shutdown => { @@ -1020,9 +1008,7 @@ mod tests { } /// A stale attempt whose candidate is still above the new anchor re-executes - /// against the post-finalization state and completes with a verdict. (The - /// staleness is injected through the mock, while the storage-refused shape is - /// pinned by `fork_refuses_inside_the_finalize_window`.) + /// against the post-finalization state and completes with a verdict. #[test] fn stale_verification_reexecutes_and_answers_true() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { @@ -2045,9 +2031,7 @@ mod tests { } /// A valid descendant whose parent replay crosses the parent's own - /// finalization is retried against the new anchor, not answered false. The - /// interrupted attempt resolves as stale or invalid ancestry, and either way - /// the verifier re-runs and forks the candidate from applied state. + /// finalization is retried against the new anchor, not answered false. #[test] fn parent_finalized_during_replay_retries_and_answers_true() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { @@ -2108,9 +2092,9 @@ mod tests { assert!(poll!(&mut verify).is_pending()); apply_started.await.expect("parent replay should start"); - // The PARENT finalizes while the replay is parked, moving the anchor - // past the walk this attempt started from. The candidate remains a - // valid, unfinalized descendant of the new anchor. + // The parent finalizes while the replay is parked, moving the + // anchor past the walk this attempt started from. The candidate + // remains a valid, unfinalized descendant of the new anchor. let (acknowledgement, waiter) = Exact::handle(); let _ = mailbox.report(Update::Block(Arc::new(parent), acknowledgement)); waiter @@ -3166,9 +3150,8 @@ mod tests { (Mailbox::new(sender), started, marshal.guards, actor) } - /// The stop signal interrupting a finalize replay parked in the application - /// exits the actor loop instead of wedging it. The block stays - /// unacknowledged so marshal redelivers it after a restart. + /// A stop mid-replay exits the actor loop. The block stays unacknowledged + /// so marshal redelivers it after a restart. #[test] fn shutdown_interrupts_a_parked_finalize() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { @@ -3204,8 +3187,8 @@ mod tests { }); } - /// A verification still in flight at shutdown never resolves for the - /// caller and never panics it, before or after the actor exits. + /// A verification in flight at shutdown never resolves for its caller, + /// before or after the actor exits. #[test] fn verify_caller_parks_across_shutdown() { deterministic::Runner::default().start(|context| async move { diff --git a/glue/src/stateful/actor/core/syncing.rs b/glue/src/stateful/actor/core/syncing.rs index ecfe9c1e880..ba01a906c1e 100644 --- a/glue/src/stateful/actor/core/syncing.rs +++ b/glue/src/stateful/actor/core/syncing.rs @@ -362,9 +362,8 @@ where acknowledgement.acknowledge(); } FinalizedHandoff::Apply(block, acknowledgement) => { - // The stop signal covers the apply await, so an application - // parked mid-apply cannot wedge shutdown. The block stays - // unacknowledged, and marshal redelivers it after a restart. + // Exiting on stop leaves the block unacknowledged, and + // marshal redelivers it after a restart. let applied; select! { _ = &mut shutdown => { diff --git a/glue/src/stateful/actor/core/verifications.rs b/glue/src/stateful/actor/core/verifications.rs index 10aeac10d48..305dce3fe40 100644 --- a/glue/src/stateful/actor/core/verifications.rs +++ b/glue/src/stateful/actor/core/verifications.rs @@ -32,8 +32,8 @@ where /// Owns independently-polled verification jobs. /// -/// A job runs to its own conclusion. The only thing that ends one early is its -/// caller leaving. +/// A job runs to its own conclusion. The only thing that ends one early is +/// its request future being dropped. pub(super) struct Handler { marshal: MarshalMailbox, jobs: Pool<(Verification, VerificationResult)>, diff --git a/glue/src/stateful/actor/processor/mod.rs b/glue/src/stateful/actor/processor/mod.rs index 538fa555fa8..3a4402d8c29 100644 --- a/glue/src/stateful/actor/processor/mod.rs +++ b/glue/src/stateful/actor/processor/mod.rs @@ -83,7 +83,7 @@ struct ReplayFlight { pub(in crate::stateful::actor) enum VerificationResult { /// A verdict to return to the caller. Decided(bool), - /// The caller left, so there is nothing left to answer. + /// The request future was dropped, so there is nothing left to answer. Cancelled, } @@ -156,10 +156,7 @@ where compatible } -/// Read capability and speculative state, shared by every verification job. -/// -/// Deliberately carries no authority to mutate the database set. Jobs holding -/// one are therefore `'static` and can outlive any number of applies. +/// Readers and speculative state shared by every verification job. struct Execution where E: Rng + Spawner + Metrics + Clock, @@ -201,12 +198,6 @@ impl Default for ReplayFlights { } impl ReplayFlights { - /// Whether no replay is in flight. - #[cfg(test)] - fn is_empty(&self) -> bool { - self.entries.lock().is_empty() - } - fn waiter(&self, digest: D, flight: &mut ReplayFlight) -> ReplayWaiter { let (sender, completion) = oneshot::channel(); let waiters = &mut flight.waiters; @@ -573,23 +564,6 @@ where self } - #[cfg(test)] - fn readers(&self) -> ReadersOf { - self.execution.readers.clone() - } - - #[cfg(test)] - fn cache_pending( - &self, - digest: PendingDigest, - parent: PendingDigest, - round: Round, - merkleized: PendingBatches, - ) -> bool { - self.execution - .cache_pending(digest, parent, round, merkleized) - } - #[cfg(test)] fn last_processed(&self) -> Anchor> { self.execution.last_processed() @@ -675,15 +649,12 @@ where // Marshal finalization is ordered. A pending miss means we can replay // this block on top of finalized state. // - // The entry stays in the pending map until the retention sweep below. - // Verification jobs run throughout this call, and one that forks from - // this block must find it rather than rebuild it on top of itself. - // - // Safety contract. Replayed `Application::apply` output must match the - // block commitments previously enforced by `Application::verify`. - // Every path from here must reach `advance_to_finalized` (which closes - // the window) or take the actor down, because a stranded window parks every - // later verification forever. + // The entry stays in the pending map until the retention sweep below, + // so a job forking from this block finds it instead of rebuilding it + // on top of itself. Replayed `apply` output must match the block's + // commitments, and every path from here must reach + // `advance_to_finalized` or take the actor down -- a stranded window + // parks every later verification forever. self.execution.state.lock().finalizing = true; let batch = match self.execution.pending_batch(&digest) { Some(merkleized) => merkleized, @@ -823,9 +794,8 @@ where ); return; } - // Unreachable, since the actor admits no finalization while a - // proposal runs (it becomes the FIFO barrier), so nothing can - // go stale. Decline loudly rather than hide a broken barrier. + // Unreachable, since the actor admits no finalization while + // a proposal runs (it becomes the FIFO barrier). Err(PrepareBatchesError::Stale) => { warn!(?parent_digest, "proposal went stale during prepare_batches"); debug_assert!(false, "no finalization can interleave a proposal"); @@ -930,9 +900,8 @@ where return true; } - // Verification runs across finalizations, so a verdict can land against - // a newer anchor and pending set than the one it started from. This is - // where such a result is refused because its branch is no longer reachable. + // A verdict can land after the anchor moved past its branch. This is + // where it is refused. let compatible = round > state.last_processed.round && (parent == state.last_processed.digest || state.pending.contains_key(&parent)); if !compatible { @@ -1003,10 +972,9 @@ where /// Wait until no finalization is mid-flight and the anchor differs from `seen`. /// - /// Returns immediately when that already holds. Used by stale verification - /// attempts, whose staleness proves a finalization opened its window, and that - /// finalization either moves the anchor or takes the actor down, so parking - /// here (instead of retrying immediately) cannot outlive it. + /// Returns immediately when that already holds. A stale attempt parks + /// here until the in-flight finalization moves the anchor or takes the + /// actor down. async fn anchor_past(&self, seen: &Anchor>) { loop { let waiter = { @@ -1370,9 +1338,37 @@ where #[cfg(test)] mod tests { use super::{ - Applied, PrepareBatchesError, Processor, Prune, Pruning, ReplayClaim, ReplayFlights, - fetch_ancestor, + Applied, Clock, Metrics, PendingBatches, PendingDigest, PrepareBatchesError, Processor, + Prune, Pruning, ReplayClaim, ReplayFlights, Rng, Spawner, fetch_ancestor, }; + + impl ReplayFlights { + /// Whether no replay is in flight. + fn is_empty(&self) -> bool { + self.entries.lock().is_empty() + } + } + + impl Processor + where + E: Rng + Spawner + Metrics + Clock, + A: Application, + { + fn readers(&self) -> ReadersOf { + self.execution.readers.clone() + } + + fn cache_pending( + &self, + digest: PendingDigest, + parent: PendingDigest, + round: Round, + merkleized: PendingBatches, + ) -> bool { + self.execution + .cache_pending(digest, parent, round, merkleized) + } + } use crate::stateful::{ Application, ExecutionError, Input, Proposed, PruneConfig, actor::metrics::Metrics as StatefulMetrics, @@ -2310,10 +2306,9 @@ mod tests { }); } - /// A verification job holds an [`Execution`] and keeps running while a - /// block is applied. The block being finalized must stay reachable as a - /// parent for that whole window, otherwise a job that forks from it - /// mid-apply rebuilds it on top of itself and rejects a valid descendant. + /// The block being finalized stays reachable as a parent while it + /// applies, so a job forking from it mid-apply does not rebuild it on top + /// of itself. #[test] fn finalized_block_stays_forkable_while_it_applies() { deterministic::Runner::timed(Duration::from_secs(5)).start(|context| async move { diff --git a/glue/src/stateful/actor/processor/verifier.rs b/glue/src/stateful/actor/processor/verifier.rs index e46c1053c99..e6880bfb68c 100644 --- a/glue/src/stateful/actor/processor/verifier.rs +++ b/glue/src/stateful/actor/processor/verifier.rs @@ -66,9 +66,6 @@ where } /// Executes one independently-polled verification request. -/// -/// Carries only read capability and speculative state, so a job outlives any -/// number of applies. Its batch operations pause while one is running. pub(in crate::stateful::actor) struct Verifier where E: Rng + Spawner + Metrics + Clock, @@ -139,13 +136,10 @@ where // Each iteration classifies the candidate against the canonical chain, // then executes it. A stale or invalid-looking attempt means a - // finalization landed while this one ran, and re-classifying answers - // correctly whether the finalized block was the candidate itself, an - // ancestor, or a competitor. The loop is bounded because each retry - // consumes an anchor move, and classification decides outright once the - // anchor reaches the candidate's height. Each attempt consumes its own - // ancestry clone, so a retry starts from the same position after the - // candidate. + // finalization landed mid-attempt, and re-classifying answers correctly + // whether the finalized block was the candidate, an ancestor, or a + // competitor. Each retry consumes an anchor move, so the loop is + // bounded. loop { let seen = self.execution.last_processed(); @@ -279,7 +273,7 @@ where ); // Incomplete ancestry is not an invalid verdict. Keep the job - // parked until its caller leaves. + // parked until its request future is dropped. verification.cancelled().await; ProcessedBlock::Cancelled } @@ -311,8 +305,8 @@ where "verification request waiting on incomplete parent ancestry" ); - // As with incomplete candidate ancestry, only the caller - // leaving should release this pending request. + // As with incomplete candidate ancestry, only dropping the + // request future should release this pending request. verification.cancelled().await; return Err(PrepareFailure::Cancelled); } diff --git a/glue/src/stateful/actor/syncer/actor.rs b/glue/src/stateful/actor/syncer/actor.rs index 1b19c6a68f9..b4ec2cba1bf 100644 --- a/glue/src/stateful/actor/syncer/actor.rs +++ b/glue/src/stateful/actor/syncer/actor.rs @@ -121,9 +121,8 @@ where } pub async fn run(mut self) { - // Everything before the select loop runs outside shutdown handling, so a - // stop during this await tears the task down crash-style, which the - // durable InProgress metadata makes recoverable. + // A stop during this await tears the task down as a crash would. + // The durable InProgress metadata makes that recoverable. let (marshal, floor) = &self.marshal; let resolved_floor = resolve_state_sync_floor::(marshal, *floor, &self.finalization).await; @@ -716,12 +715,13 @@ mod tests { }); } - /// A tip update stranded in the ring buffer by sync completion must resolve through the - /// caller's retry with the completed artifact, not wedge its observation forever. + /// A tip update stranded in the ring buffer by sync completion resolves + /// through the caller's retry with the completed artifact instead of + /// parking its observation forever. #[test] fn stranded_tip_update_resolves_to_artifact() { deterministic::Runner::timed(Duration::from_secs(10)).start(|mut context| async move { - let fixture = scheme_mocks::fixture(&mut context, b"syncer-wedge", 1); + let fixture = scheme_mocks::fixture(&mut context, b"syncer-stranded-update", 1); let block = TestBlock::new(0, 0); let finalization = fixtures::finalization(&fixture, 0, Sha256::fill(0)); let MarshalFixture { @@ -730,7 +730,7 @@ mod tests { guards: _guards, } = fixtures::marshal_fixture( context.child("marshal"), - "syncer-wedge", + "syncer-stranded-update", fixture.schemes[0].clone(), Some((&block, finalization.clone())), NZUsize!(1), diff --git a/glue/src/stateful/actor/syncer/mod.rs b/glue/src/stateful/actor/syncer/mod.rs index c3e93f8d728..3ab2cfbd25a 100644 --- a/glue/src/stateful/actor/syncer/mod.rs +++ b/glue/src/stateful/actor/syncer/mod.rs @@ -363,10 +363,10 @@ where /// /// If the databases are found to be inconsistent with the marshal floor, this /// function will attempt to repair by rewinding the databases which are ahead. If the -/// databases are entirely inconsistent, this function will panic. That covers a crash -/// between marshal installing a sync floor and the sync metadata recording it, so the -/// operator's sync request must persist across such a restart so startup re-enters the -/// sync path instead of asking a fresh database set to reach the installed floor. +/// databases are entirely inconsistent, this function panics. That covers a crash +/// between marshal installing a sync floor and the sync metadata recording it. The +/// operator's sync request must survive that restart, so startup re-enters the sync +/// path instead of asking a fresh database set to reach the installed floor. pub(crate) async fn init_databases_from_marshal( context: &E, marshal: &MarshalMailbox, diff --git a/glue/src/stateful/db/any.rs b/glue/src/stateful/db/any.rs index 0d43dd25264..12fc08ec66c 100644 --- a/glue/src/stateful/db/any.rs +++ b/glue/src/stateful/db/any.rs @@ -2,9 +2,7 @@ //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for //! read-through to applied state. The wrapper types here hold a [`Reader`] -//! to their database and take read access through it for each such call, so a batch stays usable -//! across applies of compatible batches and never delays a mutation by more -//! than one storage call. +//! to their database and take a read guard per such call. use crate::stateful::db::{ LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, diff --git a/glue/src/stateful/db/cell.rs b/glue/src/stateful/db/cell.rs index 0176c304c99..a4f8e988832 100644 --- a/glue/src/stateful/db/cell.rs +++ b/glue/src/stateful/db/cell.rs @@ -1,33 +1,13 @@ -//! Separates read access to a live database from the authority to mutate it. +//! Splits access to a live database: a unique [`Writer`] runs consuming +//! mutations, and cloneable [`Reader`]s hand out short read guards, one per +//! storage call. //! -//! [`split`] wraps a database and returns two capabilities over it. The -//! [`Writer`] is unique and runs consuming mutations. The [`Reader`] is -//! freely cloned into batches, and hands out short read guards, each covering -//! exactly one storage call. -//! -//! Neither value owns the database. The cell does, and both capabilities keep -//! it alive. What distinguishes them is what they permit. -//! -//! The cell is a tokio read-write lock, whose documented policy is fair and -//! write-preferring, so a waiting mutation blocks new read guards and cannot be -//! starved, while guards already granted finish first. The write side covers -//! the whole take-and-restore of a mutation, so a reader can never observe the -//! database missing. A mutation that is interrupted mid-flight leaves the cell -//! poisoned, and the only thing that interrupts one here is the owning task -//! being torn down, so later reads park forever and are dropped along with -//! the tasks holding them. -//! -//! A read guard guarantees the database is present and unchanging for one call, -//! not that the caller's batch is still current. A batch operation holding one -//! can still refuse because a competing batch was applied (see -//! [`commonware_storage::qmdb::Error::StaleRead`]). -//! -//! Two rules keep callers out of trouble. The lock is not reentrant, so never -//! hold a read guard while acquiring another. On the same cell a mutation queued -//! between the two deadlocks both, and across cells the same habit invites -//! order-inversion deadlocks. And a [`Writer`] should outlive the -//! readers from the same cell. Dropping the writer closes the cell, and later -//! reads park instead of answering from a database that can never advance. +//! The cell is a fair, write-preferring read-write lock: a waiting mutation +//! blocks new read guards but cannot be starved, and guards already granted +//! finish first. A guard proves the database is present for one call, not +//! that the caller's batch is still current (see +//! [`commonware_storage::qmdb::Error::StaleRead`]). The lock is not +//! reentrant: never hold two read guards at once. use commonware_utils::sync::{AsyncRwLockReadGuard, TracedAsyncRwLock}; use futures::future; @@ -48,9 +28,7 @@ enum State { struct Cell { state: TracedAsyncRwLock>, - /// Latched when the writer drops. A one-way flag, so a relaxed load - /// suffices, and a read guard that races the drop is indistinguishable - /// from one granted a moment earlier. + /// Set when the writer drops. One-way, so a relaxed load suffices. closed: AtomicBool, } @@ -58,8 +36,7 @@ impl Cell { async fn read(&self) -> ReadGuard<'_, T> { let guard = self.state.read().await; if self.closed.load(Ordering::Relaxed) { - // The writer is gone, so the database can never advance. Serving - // would hand out frozen state. Park like the poisoned case. + // The writer is gone. Park rather than serve frozen state. drop(guard); tracing::error!("database cell closed, parking reader"); return future::pending().await; @@ -70,9 +47,8 @@ impl Cell { }) { Ok(guard) => ReadGuard(guard), Err(guard) => { - // Poisoning is only reachable during writer teardown. Park until - // this task is dropped with the rest of the actor, and the trace - // separates that from a bug if the process outlives the cell. + // Poisoning only happens during actor teardown. Park until + // this task is dropped with it. drop(guard); tracing::error!("database cell poisoned, parking reader"); future::pending().await @@ -81,20 +57,17 @@ impl Cell { } } -/// Split access to `db` into the sole mutation authority and a cloneable -/// read capability. +/// Split `db` into its unique [`Writer`] and a cloneable [`Reader`]. pub fn split(db: T) -> (Writer, Reader) { let writer = Writer::new(db); let reader = writer.reader(); (writer, reader) } -/// The unique capability to mutate the database behind a cell. +/// The unique handle that mutates the database behind a cell. /// -/// This is the only value with [`mutate`](Self::mutate), and it is deliberately -/// not [`Clone`], so at most one exists per cell. It does not own the database. -/// Dropping it closes the cell, since the database can never advance again, and -/// later reads park instead of serving frozen state. +/// Not [`Clone`], so at most one exists. Dropping it closes the cell, and +/// later reads park rather than serve frozen state. pub struct Writer(Arc>); impl Drop for Writer { @@ -104,7 +77,7 @@ impl Drop for Writer { } impl Writer { - /// Wrap `db` in a fresh cell, returning its sole mutation authority. + /// Put `db` in a fresh cell and return its writer. pub fn new(db: T) -> Self { Self(Arc::new(Cell { state: TracedAsyncRwLock::new("database_cell", State::Live(db)), @@ -112,24 +85,16 @@ impl Writer { })) } - /// A read capability over the writer's cell. + /// A reader over the writer's cell. pub fn reader(&self) -> Reader { Reader(self.0.clone()) } - /// Run one consuming mutation to completion, returning the capability. - /// - /// New read guards queue behind the mutation and guards already granted - /// finish first, so this waits at most one storage call before starting. + /// Run one consuming mutation to completion, returning the writer. /// - /// Consume/produce at both levels. `mutation` takes the database by value and - /// must produce it back, the contract mutable storage operations already use. - /// This method does the same with the capability. An interrupted mutation - /// takes the writer with it, and if the database was already taken out it - /// also poisons the cell, so a second mutation of a poisoned cell is - /// unreachable rather than merely documented. Reads taken afterward park - /// forever. (Dropped while still queued for the lock, the writer's own drop - /// closes the cell the same way.) + /// Waits at most one storage call to start, since new read guards queue + /// behind it. Dropping the future mid-flight poisons the cell, and later + /// reads park. pub async fn mutate(self, mutation: F) -> (Self, R) where F: FnOnce(T) -> Fut, @@ -146,7 +111,7 @@ impl Writer { } } -/// A cloneable read capability over the database behind a cell. +/// A cloneable read handle over the database behind a cell. pub struct Reader(Arc>); impl Clone for Reader { @@ -196,7 +161,6 @@ mod tests { loop { { let guard = reader.read().await; - // Every observation is a full, live value. assert!(*guard == 0 || *guard == 1); if *guard == 1 { return; @@ -255,8 +219,7 @@ mod tests { }); } - /// Dropping the writer closes the cell. Later reads park instead of - /// answering from a database that can never advance. + /// Dropping the writer closes the cell, and later reads park. #[test] fn dropped_writer_parks_readers() { deterministic::Runner::default().start(|_context| async move { @@ -273,9 +236,8 @@ mod tests { }); } - /// Dropping a mutation mid-flight poisons the cell, and takes the writer - /// with it. Later reads park forever instead of observing missing state. - /// A second mutation is unrepresentable, so there is nothing to assert. + /// Dropping a mutation mid-flight poisons the cell, and later reads park. + /// The type rules out a second mutation, so there is nothing to assert. #[test] fn interrupted_mutation_poisons() { deterministic::Runner::default().start(|_context| async move { diff --git a/glue/src/stateful/db/current.rs b/glue/src/stateful/db/current.rs index 688303ece61..5914b94eae9 100644 --- a/glue/src/stateful/db/current.rs +++ b/glue/src/stateful/db/current.rs @@ -2,9 +2,7 @@ //! //! The QMDB batch API passes `&db` to `get()` and `merkleize()` for //! read-through to applied state. The wrapper types here hold a [`Reader`] -//! to their database and take read access through it for each such call, so a batch stays usable -//! across applies of compatible batches and never delays a mutation by more -//! than one storage call. +//! to their database and take a read guard per such call. use crate::stateful::db::{ LogSnapshot, ManagedDb, Merkleized as MerkleizedTrait, Reader, StateSyncDb, SyncEngineConfig, diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index da5b7da1512..aa4eb0ea282 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -18,25 +18,10 @@ //! //! # Read access and mutation //! -//! Each database is split ([`split`]) into two capabilities over one shared -//! cell. The set holds the only [`Writer`], which is not [`Clone`], so -//! mutation is uniquely permitted. Batches hold a [`Reader`], which is freely -//! cloned and grants a [`ReadGuard`] covering exactly one storage call. -//! -//! Because a read guard never spans application code, a mutation waits at most one -//! storage call to start, and no mutation ever cancels work holding a reader, so it -//! pauses at its next read and resumes once the mutation completes. A mutation -//! that is interrupted leaves the cell poisoned, which is reachable only while -//! the mutating task is being torn down. -//! -//! Two invariants keep this sound. A batch handed to [`ManagedDb::finalize`] -//! must not read through its own reader, because that call runs while the write -//! side is held. And a [`Writer`] should outlive the readers from the same -//! cell. Dropping the writer closes the cell, and later reads park instead of -//! answering from a database that can never advance. The set holds the -//! [`Writer`]; readers live in batches, in verification jobs, and in the -//! [`Application::finalized`](crate::stateful::Application::finalized) hook, -//! which may keep one past the actor's exit, where its reads park. +//! Each database is split ([`split`]) into the set's sole [`Writer`] and +//! cloneable [`Reader`]s. Batches hold a reader and take a [`ReadGuard`] per +//! storage call. A batch handed to [`ManagedDb::finalize`] must not read +//! through its own reader, because that call holds the write side. //! //! # State Sync //! @@ -142,8 +127,7 @@ pub use snapshot::{Publisher, Subscriber}; /// Concrete types provide key-value operations (`get`, `write`, `set`, /// `append`, etc.) as inherent methods; the generic wrapper only needs /// [`merkleize`](Self::merkleize). Batches carry a [`Reader`] to the -/// database they were created from, so every operation reads the right -/// database and no operation can delay a mutation by more than one call. +/// database they were created from. pub trait Unmerkleized: Sized + Send { /// The merkleized batch produced by [`merkleize`](Self::merkleize). type Merkleized: Merkleized; @@ -197,8 +181,7 @@ pub trait Merkleized: Clone + Sized + Send + Sync { /// Mutating methods take the database by value and return it on success. If a mutating /// method returns an error, or its future is dropped before it finishes, the database is /// gone: state that was not yet durable is discarded, but everything already on disk stays -/// recoverable. The cell is left poisoned, so later reads park -/// rather than observe a database that is missing. +/// recoverable. The cell is left poisoned and later reads park. pub trait ManagedDb: Send + Sync + Sized { /// An in-progress batch of mutations that has not yet been merkleized. type Unmerkleized: Unmerkleized; diff --git a/glue/src/stateful/mod.rs b/glue/src/stateful/mod.rs index a4f129c74e1..b0b12927032 100644 --- a/glue/src/stateful/mod.rs +++ b/glue/src/stateful/mod.rs @@ -118,13 +118,11 @@ mod tests; /// each case means for the block being executed. #[derive(Debug, Error)] pub enum ExecutionError { - /// Applied state left the executing block's branch because a competing block was - /// finalized mid-execution and the batches refused their next read. The wrapper - /// re-checks the block against the new canonical state. + /// A competing finalization invalidated the batch's reads mid-execution. + /// The wrapper re-checks the block against the new canonical state. #[error("stale execution: a competing block was finalized")] Stale, - /// Any other storage failure. Storage errors are unrecoverable, so the - /// wrapper panics on this everywhere. + /// Any other storage failure. The wrapper panics. #[error("storage failure: {0}")] Fatal(String), } @@ -301,14 +299,10 @@ where /// batch operation running when a finalized block is applied waits for that /// apply and then continues. Actor shutdown drops it with everything else. /// - /// `batches` is a branch-scoped view, not a historical snapshot. Retained - /// ancestor overlays preserve same-branch state, while unresolved reads fall - /// through to the batch's own database. Once a block from a competing branch - /// is finalized, every batch operation refuses with a stale error instead of - /// answering across branches. Implementations propagate storage errors with - /// `?` as [`ExecutionError`] and never interpret them. On - /// [`ExecutionError::Stale`] the wrapper re-checks the block against the new - /// canonical state and retries or answers from there. + /// Once a block from a competing branch is finalized, every batch + /// operation refuses with [`ExecutionError::Stale`]. The wrapper then + /// re-checks the block against the new canonical state and retries or + /// answers from it. fn verify( &mut self, context: (E, Self::Context), diff --git a/glue/src/stateful/tests/mod.rs b/glue/src/stateful/tests/mod.rs index 10078adddc5..e0f8203dd30 100644 --- a/glue/src/stateful/tests/mod.rs +++ b/glue/src/stateful/tests/mod.rs @@ -75,8 +75,8 @@ mod single_db_app; const NUM_VALIDATORS: u32 = 5; -/// Storage errors never masquerade as shutdown. Only a refused stale read maps -/// to Stale, and every other storage failure is fatal. +/// Only a refused stale read maps to Stale. Every other storage failure is +/// fatal. #[test] fn storage_errors_map_to_fatal() { use commonware_runtime::Error as RuntimeError; diff --git a/glue/src/stateful/tests/ownership.rs b/glue/src/stateful/tests/ownership.rs index b8db65ffab9..11117d33387 100644 --- a/glue/src/stateful/tests/ownership.rs +++ b/glue/src/stateful/tests/ownership.rs @@ -1,4 +1,4 @@ -//! Causal ownership tests for the by-value database set. Readers and the writer +//! Ownership tests for the by-value database set. Readers and the writer //! share nothing that can block either side. //! //! - a parked flush holds back neither the writer nor publication From c2d44dfa2115983aaa4b34bfca972e405247a705 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 21:46:26 -0400 Subject: [PATCH 12/14] cleanup --- glue/src/stateful/db/mod.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index aa4eb0ea282..ea76e5a82a8 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -181,7 +181,7 @@ pub trait Merkleized: Clone + Sized + Send + Sync { /// Mutating methods take the database by value and return it on success. If a mutating /// method returns an error, or its future is dropped before it finishes, the database is /// gone: state that was not yet durable is discarded, but everything already on disk stays -/// recoverable. The cell is left poisoned and later reads park. +/// recoverable. pub trait ManagedDb: Send + Sync + Sized { /// An in-progress batch of mutations that has not yet been merkleized. type Unmerkleized: Unmerkleized; @@ -204,9 +204,7 @@ pub trait ManagedDb: Send + Sync + Sized { /// Typically a database-specific state commitment plus the operation range needed to reach it. type SyncTarget: Clone + PartialEq + Send + Sync; - /// Owned immutable snapshot of the applied operation log and its proof - /// state, for serving state sync. Not a queryable key-value view -- only the - /// live database answers key reads. + /// Owned immutable snapshot of applied state. type Snapshot: Clone + Send + Sync + 'static; /// Construct a new database from its configuration. @@ -362,19 +360,18 @@ pub trait DatabaseSet: Send + Sync + Sized + 'static { /// for tuple sets. type Unmerkleized: Send; - /// One [`ManagedDb::Merkleized`] per database, shaped like [`Self::Unmerkleized`]. + /// One [`ManagedDb::Merkleized`] per database. /// /// [`Clone`] is cheap (see [`Merkleized`]) and the wrapper uses it to keep a /// block forkable while that block is being applied. type Merkleized: Clone + Send + Sync; - /// One [`Reader`] per database, shaped like [`Self::Unmerkleized`]. + /// One [`Reader`] per database. /// - /// Readers are cloned into batches, and hooks that read applied state - /// directly acquire read guards through them. + /// Cloned into batches and into hooks that read applied state. type Readers: Clone + Send + Sync + 'static; - /// One [`ManagedDb::Snapshot`] per database, shaped like [`Self::Unmerkleized`]. + /// One [`ManagedDb::Snapshot`] per database. type Snapshots: Send + Sync + 'static; /// Configuration needed to construct every database in the set -- the database's @@ -382,8 +379,7 @@ pub trait DatabaseSet: Send + Sync + Sized + 'static { /// tuple sets. type Config: Send; - /// Per-database sync targets extracted from a finalized block, shaped like - /// [`Self::Config`]. + /// Per-database sync targets extracted from a finalized block. type SyncTargets: Clone + PartialEq + Send + Sync; /// Construct the database set from its configuration. From 3c44a863b11855379f4bfdd34da59fc821a6002e Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 21:48:06 -0400 Subject: [PATCH 13/14] cleanup --- glue/src/stateful/db/any.rs | 62 +++++++++--------- glue/src/stateful/db/current.rs | 74 +++++++++++----------- glue/src/stateful/db/immutable/compact.rs | 24 +++---- glue/src/stateful/db/immutable/standard.rs | 32 +++++----- glue/src/stateful/db/keyless/compact.rs | 24 +++---- glue/src/stateful/db/keyless/standard.rs | 32 +++++----- glue/src/stateful/db/mod.rs | 4 +- 7 files changed, 126 insertions(+), 126 deletions(-) diff --git a/glue/src/stateful/db/any.rs b/glue/src/stateful/db/any.rs index 12fc08ec66c..a585c5dbfc5 100644 --- a/glue/src/stateful/db/any.rs +++ b/glue/src/stateful/db/any.rs @@ -62,7 +62,7 @@ where Operation: Codec, { batch: UnmerkleizedBatch, - reader: Reader>, + db: Reader>, metadata: Option, } @@ -83,7 +83,7 @@ where Operation: Codec, { staged: Staged, - reader: Reader>, + db: Reader>, metadata: Option, } @@ -108,7 +108,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get(key, &db).await } @@ -116,7 +116,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get_many(keys, &db).await } @@ -129,18 +129,18 @@ where ) -> Result<(Vec>, AnyStaged), Error> { let Self { batch, - reader, + db, metadata, } = self; let (values, staged) = { - let guard = reader.read().await; + let guard = db.read().await; batch.stage(keys, &guard).await? }; Ok(( values, AnyStaged { staged, - reader, + db, metadata, }, )) @@ -166,7 +166,7 @@ where Operation: Codec, { inner: Arc>, - reader: Reader>, + db: Reader>, } impl Clone for AnyMerkleized @@ -183,7 +183,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -255,11 +255,11 @@ where ) -> Result<(Range, Vec>, Self), Error> { let Self { staged, - reader, + db, metadata, } = self; let (range, values, staged) = { - let guard = reader.read().await; + let guard = db.read().await; staged.expand(keys, &guard).await? }; Ok(( @@ -267,7 +267,7 @@ where values, Self { staged, - reader, + db, metadata, }, )) @@ -307,14 +307,14 @@ where ) -> Result, S>, Error> { let Self { staged, - reader, + db, metadata, } = self; let inner = { - let guard = reader.read().await; + let guard = db.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(AnyMerkleized { inner, reader }) + Ok(AnyMerkleized { inner, db }) } } @@ -351,14 +351,14 @@ where ) -> Result, S>, Error> { let Self { staged, - reader, + db, metadata, } = self; let inner = { - let guard = reader.read().await; + let guard = db.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(AnyMerkleized { inner, reader }) + Ok(AnyMerkleized { inner, db }) } } @@ -376,7 +376,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get(key, &db).await } @@ -384,7 +384,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get_many(keys, &db).await } } @@ -407,11 +407,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(AnyMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -434,11 +434,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(AnyMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -465,7 +465,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { AnyUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, } } @@ -529,11 +529,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); AnyUnmerkleized { batch, - reader, + db: database, metadata: None, } } @@ -649,11 +649,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); AnyUnmerkleized { batch, - reader, + db: database, metadata: None, } } diff --git a/glue/src/stateful/db/current.rs b/glue/src/stateful/db/current.rs index 5914b94eae9..7256373f492 100644 --- a/glue/src/stateful/db/current.rs +++ b/glue/src/stateful/db/current.rs @@ -59,7 +59,7 @@ where Operation: Codec, { batch: UnmerkleizedBatch, - reader: Reader>, + db: Reader>, metadata: Option, } @@ -80,7 +80,7 @@ where Operation: Codec, { staged: Staged, - reader: Reader>, + db: Reader>, metadata: Option, } @@ -105,7 +105,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get(key, &db).await } @@ -113,7 +113,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get_many(keys, &db).await } @@ -126,18 +126,18 @@ where ) -> Result<(Vec>, CurrentStaged), Error> { let Self { batch, - reader, + db, metadata, } = self; let (values, staged) = { - let guard = reader.read().await; + let guard = db.read().await; batch.stage(keys, &guard).await? }; Ok(( values, CurrentStaged { staged, - reader, + db, metadata, }, )) @@ -163,7 +163,7 @@ where Operation: Codec, { inner: Arc>, - reader: Reader>, + db: Reader>, } impl Clone for CurrentMerkleized @@ -180,7 +180,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -252,11 +252,11 @@ where ) -> Result<(Range, Vec>, Self), Error> { let Self { staged, - reader, + db, metadata, } = self; let (range, values, staged) = { - let guard = reader.read().await; + let guard = db.read().await; staged.expand(keys, &guard).await? }; Ok(( @@ -264,7 +264,7 @@ where values, Self { staged, - reader, + db, metadata, }, )) @@ -305,14 +305,14 @@ where ) -> Result, N, S>, Error> { let Self { staged, - reader, + db, metadata, } = self; let inner = { - let guard = reader.read().await; + let guard = db.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(CurrentMerkleized { inner, reader }) + Ok(CurrentMerkleized { inner, db }) } } @@ -350,14 +350,14 @@ where ) -> Result, N, S>, Error> { let Self { staged, - reader, + db, metadata, } = self; let inner = { - let guard = reader.read().await; + let guard = db.read().await; staged.merkleize(updates, upserts, metadata, &guard).await? }; - Ok(CurrentMerkleized { inner, reader }) + Ok(CurrentMerkleized { inner, db }) } } @@ -375,7 +375,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &U::Key) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get(key, &db).await } @@ -383,7 +383,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&U::Key]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get_many(keys, &db).await } } @@ -406,11 +406,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(CurrentMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -433,11 +433,11 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self.batch.merkleize(&db, self.metadata).await?; Ok(CurrentMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -465,7 +465,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { CurrentUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, } } @@ -529,11 +529,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); CurrentUnmerkleized { batch, - reader, + db: database, metadata: None, } } @@ -642,11 +642,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); CurrentUnmerkleized { batch, - reader, + db: database, metadata: None, } } @@ -837,11 +837,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); CurrentUnmerkleized { batch, - reader, + db: database, metadata: None, } } @@ -959,11 +959,11 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { - let batch = reader.read().await.new_batch(); + async fn new_batch(database: Reader) -> Self::Unmerkleized { + let batch = database.read().await.new_batch(); CurrentUnmerkleized { batch, - reader, + db: database, metadata: None, } } diff --git a/glue/src/stateful/db/immutable/compact.rs b/glue/src/stateful/db/immutable/compact.rs index 2e89ad86046..c85bb2fe1a3 100644 --- a/glue/src/stateful/db/immutable/compact.rs +++ b/glue/src/stateful/db/immutable/compact.rs @@ -44,7 +44,7 @@ where S: Strategy, { batch: CompactUnmerkleizedBatch, - reader: Reader>, + db: Reader>, metadata: Option, inactivity_floor: Location, } @@ -113,7 +113,7 @@ where S: Strategy, { inner: Arc>, - reader: Reader>, + db: Reader>, } impl Clone for ImmutableUnjournaledMerkleized @@ -131,7 +131,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -172,14 +172,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(ImmutableUnjournaledMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -206,7 +206,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { ImmutableUnjournaledUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -241,14 +241,14 @@ where } } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; ImmutableUnjournaledUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } @@ -323,14 +323,14 @@ where } } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; ImmutableUnjournaledUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } diff --git a/glue/src/stateful/db/immutable/standard.rs b/glue/src/stateful/db/immutable/standard.rs index fe34e6de0b2..01900090a67 100644 --- a/glue/src/stateful/db/immutable/standard.rs +++ b/glue/src/stateful/db/immutable/standard.rs @@ -54,7 +54,7 @@ where Operation: EncodeShared, { batch: UnmerkleizedBatch, - reader: ImmutableDbHandle, + db: ImmutableDbHandle, metadata: Option, inactivity_floor: Location, } @@ -105,7 +105,7 @@ where /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &K) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get(key, &db).await } @@ -113,7 +113,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&K]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get_many(keys, &db).await } @@ -139,7 +139,7 @@ where Operation: EncodeShared, { inner: Arc>, - reader: ImmutableDbHandle, + db: ImmutableDbHandle, } impl Clone for ImmutableMerkleized @@ -157,7 +157,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -195,7 +195,7 @@ where { /// Read a value by key, falling back to applied state. pub async fn get(&self, key: &K) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get(key, &db).await } @@ -203,7 +203,7 @@ where /// /// Returns results in the same order as the input keys. pub async fn get_many(&self, keys: &[&K]) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get_many(keys, &db).await } } @@ -224,14 +224,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(ImmutableMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -258,7 +258,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { ImmutableUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -311,14 +311,14 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; ImmutableUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } @@ -417,14 +417,14 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; ImmutableUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } diff --git a/glue/src/stateful/db/keyless/compact.rs b/glue/src/stateful/db/keyless/compact.rs index b74173b1423..29af36456a4 100644 --- a/glue/src/stateful/db/keyless/compact.rs +++ b/glue/src/stateful/db/keyless/compact.rs @@ -42,7 +42,7 @@ where S: Strategy, { batch: CompactUnmerkleizedBatch, - reader: Reader>, + db: Reader>, metadata: Option, inactivity_floor: Location, } @@ -108,7 +108,7 @@ where S: Strategy, { inner: Arc>, - reader: Reader>, + db: Reader>, } impl Clone for KeylessUnjournaledMerkleized @@ -125,7 +125,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -163,14 +163,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(KeylessUnjournaledMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -196,7 +196,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { KeylessUnjournaledUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -230,14 +230,14 @@ where } } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; KeylessUnjournaledUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } @@ -311,14 +311,14 @@ where } } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; KeylessUnjournaledUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } diff --git a/glue/src/stateful/db/keyless/standard.rs b/glue/src/stateful/db/keyless/standard.rs index 12a89d2136e..e0dc8bd3742 100644 --- a/glue/src/stateful/db/keyless/standard.rs +++ b/glue/src/stateful/db/keyless/standard.rs @@ -46,7 +46,7 @@ where Operation: EncodeShared, { batch: UnmerkleizedBatch, - reader: Reader>, + db: Reader>, metadata: Option, inactivity_floor: Location, } @@ -93,7 +93,7 @@ where /// Read a value by location, falling back to applied state. pub async fn get(&self, location: Location) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get(location, &db).await } @@ -105,7 +105,7 @@ where &self, locations: &[Location], ) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.batch.get_many(locations, &db).await } @@ -129,7 +129,7 @@ where Operation: EncodeShared, { inner: Arc>, - reader: Reader>, + db: Reader>, } impl Clone for KeylessMerkleized @@ -145,7 +145,7 @@ where fn clone(&self) -> Self { Self { inner: Arc::clone(&self.inner), - reader: self.reader.clone(), + db: self.db.clone(), } } } @@ -179,7 +179,7 @@ where { /// Read a value by location, falling back to applied state. pub async fn get(&self, location: Location) -> Result, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get(location, &db).await } @@ -191,7 +191,7 @@ where &self, locations: &[Location], ) -> Result>, Error> { - let db = self.reader.read().await; + let db = self.db.read().await; self.inner.get_many(locations, &db).await } } @@ -210,14 +210,14 @@ where type Error = Error; async fn merkleize(self) -> Result> { - let db = self.reader.read().await; + let db = self.db.read().await; let merkleized = self .batch .merkleize(&db, self.metadata, self.inactivity_floor) .await?; Ok(KeylessMerkleized { inner: merkleized, - reader: self.reader.clone(), + db: self.db.clone(), }) } } @@ -242,7 +242,7 @@ where fn new_batch(&self) -> Self::Unmerkleized { KeylessUnmerkleized { batch: self.inner.new_batch::(), - reader: self.reader.clone(), + db: self.db.clone(), metadata: None, inactivity_floor: self.inner.bounds().inactivity_floor, } @@ -277,14 +277,14 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; KeylessUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } @@ -376,14 +376,14 @@ where ) } - async fn new_batch(reader: Reader) -> Self::Unmerkleized { + async fn new_batch(database: Reader) -> Self::Unmerkleized { let (batch, inactivity_floor) = { - let db = reader.read().await; + let db = database.read().await; (db.new_batch(), db.inactivity_floor_loc()) }; KeylessUnmerkleized { batch, - reader, + db: database, metadata: None, inactivity_floor, } diff --git a/glue/src/stateful/db/mod.rs b/glue/src/stateful/db/mod.rs index ea76e5a82a8..a284901928e 100644 --- a/glue/src/stateful/db/mod.rs +++ b/glue/src/stateful/db/mod.rs @@ -221,9 +221,9 @@ pub trait ManagedDb: Send + Sync + Sized { /// Create a new unmerkleized batch rooted at the database's applied /// state. /// - /// The batch keeps `reader` and takes read access through it on every + /// The batch keeps `database` and takes read access through it on every /// read, so it stays valid across applies of compatible batches. - fn new_batch(reader: Reader) -> impl Future + Send; + fn new_batch(database: Reader) -> impl Future + Send; /// Return true if a merkleized batch matches a sync target. fn matches_sync_target(batch: &Self::Merkleized, target: &Self::SyncTarget) -> bool; From ea467dff7c047e39eac9c77f0517f500f7ccd817 Mon Sep 17 00:00:00 2001 From: Dan Laine Date: Wed, 19 Aug 2026 22:04:18 -0400 Subject: [PATCH 14/14] cleanup --- glue/src/stateful/actor/core/verifications.rs | 3 -- glue/src/stateful/actor/processor/mod.rs | 32 +++++++++++++++++-- glue/src/stateful/actor/processor/verifier.rs | 31 +++++------------- 3 files changed, 38 insertions(+), 28 deletions(-) diff --git a/glue/src/stateful/actor/core/verifications.rs b/glue/src/stateful/actor/core/verifications.rs index 305dce3fe40..aa49d979da3 100644 --- a/glue/src/stateful/actor/core/verifications.rs +++ b/glue/src/stateful/actor/core/verifications.rs @@ -31,9 +31,6 @@ where } /// Owns independently-polled verification jobs. -/// -/// A job runs to its own conclusion. The only thing that ends one early is -/// its request future being dropped. pub(super) struct Handler { marshal: MarshalMailbox, jobs: Pool<(Verification, VerificationResult)>, diff --git a/glue/src/stateful/actor/processor/mod.rs b/glue/src/stateful/actor/processor/mod.rs index 3a4402d8c29..06fc1273cf8 100644 --- a/glue/src/stateful/actor/processor/mod.rs +++ b/glue/src/stateful/actor/processor/mod.rs @@ -20,6 +20,34 @@ //! //! - Maintenance -- [`Processor::prune`] runs due prunes and //! [`Processor::publish_snapshot`] publishes fresh snapshots afterwards. +//! +//! # How verification races finalization +//! +//! Finalization never waits for verification. When a block finalizes, the +//! actor applies it immediately. Verification jobs keep running through the +//! apply and are never cancelled, so a job can race any number of applies, +//! and the pieces here make that race safe: +//! +//! - Jobs share an [`Execution`]: readers over the databases plus, under one +//! lock, the speculative world-view -- the pending map of verified blocks, +//! the applied anchor, and the finalizing window below. +//! +//! - A job whose branch an apply invalidated is refused at its next read +//! ([`ExecutionError::Stale`], enforced by storage). The job waits for the +//! anchor to move past what it saw ([`Execution::anchor_past`]), then +//! re-classifies the candidate against the new canonical chain: the +//! candidate itself finalized means true, swept away means false, and still +//! open means execute again (the loop in `verifier::Verifier::run`). +//! +//! - An apply mutates the databases before the anchor moves. A fork taken +//! from the anchor in that window could mix pre- and post-apply databases, +//! so forks refuse while a finalization is mid-flight (the `finalizing` +//! flag), and the sweep, the anchor move, and the window close happen under +//! one lock ([`Execution::advance_to_finalized`]). +//! +//! - The finalized block stays in the pending map until that sweep, so a job +//! forking from it mid-apply finds it instead of rebuilding it on top of +//! itself. use crate::stateful::{ Application, ExecutionError, Input, Proposed, PruneConfig, @@ -79,7 +107,7 @@ struct ReplayFlight { vacant_slots: Vec, } -/// What one verification attempt concluded. +/// The verification's final answer. pub(in crate::stateful::actor) enum VerificationResult { /// A verdict to return to the caller. Decided(bool), @@ -337,7 +365,7 @@ enum PrepareBatchesError { Invalid, /// Parent ancestry ended before validity could be proven. Incomplete, - /// The attempt was cancelled while waiting. + /// The request future was dropped while waiting. Cancelled, /// A competing finalization landed mid-preparation. The caller re-checks /// against the new canonical state. diff --git a/glue/src/stateful/actor/processor/verifier.rs b/glue/src/stateful/actor/processor/verifier.rs index e6880bfb68c..f44435938ef 100644 --- a/glue/src/stateful/actor/processor/verifier.rs +++ b/glue/src/stateful/actor/processor/verifier.rs @@ -173,30 +173,15 @@ where { Ok(parent) => parent, Err(PrepareFailure::Invalid) => { - // A finalization that completed while this attempt waited can - // make valid ancestry look invalid (the parent swept below the - // new anchor), so classify once more before answering false. - match self - .check_processed(marshal.clone(), block.as_ref(), verification) - .await - { - ProcessedBlock::Accepted => { - timer.observe(context); - return VerificationResult::Decided(true); - } - ProcessedBlock::Cancelled => return VerificationResult::Cancelled, - ProcessedBlock::Rejected => return VerificationResult::Decided(false), - // The candidate still sits above the anchor. An anchor - // that moved during this attempt may have invalidated the - // walk itself, so retry against the new anchor. A stable - // anchor means the ancestry is genuinely invalid. - ProcessedBlock::Continue => { - if self.execution.last_processed().digest != seen.digest { - continue; - } - return VerificationResult::Decided(false); - } + // An anchor that moved during this attempt can make valid + // ancestry look invalid (the parent swept below the new + // anchor), so retry and let the loop's classification + // decide. A stable anchor means the ancestry is genuinely + // invalid. + if self.execution.last_processed().digest != seen.digest { + continue; } + return VerificationResult::Decided(false); } Err(PrepareFailure::Cancelled) => return VerificationResult::Cancelled, Err(PrepareFailure::Stale) => {