Phase 5: SimpleStorageRootDeriver + resolve_storage_roots; closes UnresolvedStorageRoot gap in PR3; restore ssz doc comments

runtime/builder/Cargo.toml

@@ -6,3 +6,4 @@ license.workspace = true
 
 [dependencies]
 kaysentinel-cse = { path = "../cse" }
+kaysentinel-hash = { path = "../hash" }

runtime/builder/src/lib.rs

@@ -27,3 +27,4 @@ pub use lifecycle::hydration::{
     BackendError, CertificateAssemblyError, DerivationError, HydratedAccountState,
     StorageDerivationError, StorageRootDeriver, assemble_certificates, hydrate_accounts,
 };
+pub use lifecycle::storage_root::{SimpleStorageRootDeriver, resolve_storage_roots};

runtime/builder/src/lifecycle/mod.rs

@@ -6,6 +6,7 @@ pub mod hydration;
 pub mod keys;
 pub mod resolve;
 pub mod semantic;
+pub mod storage_root;
 
 use std::collections::BTreeMap;
 

runtime/builder/src/lifecycle/storage_root.rs

@@ -0,0 +1,256 @@
+//! Storage root derivation — implements the `StorageRootDeriver` trait from
+//! `lifecycle::hydration`, and wires its output into already-built certificates.
+//!
+//! Deliberately self-contained (depends only on `kaysentinel-hash`, not on
+//! `runtime/ssz`): `runtime/ssz` already depends on `runtime/builder` (to encode
+//! `CanonicalAccountCertificate`), so the reverse dependency would create a cycle.
+//! This means the pairwise Merkle fold below is a small, independent
+//! re-implementation of the same zero-padding convention `runtime/ssz::merkleize`
+//! uses — not a reuse of it. If the two ever need to be unified, that has to
+//! happen by extracting a third, lower-level crate both depend on, not by either
+//! depending on the other.
+//!
+//! ## Design note: not an Ethereum-compatible storage root
+//!
+//! This derives a protocol-native commitment over an account's touched storage
+//! slots, using the project's own domain-separated BLAKE3 scheme
+//! (`Domain::TrieLeaf` / `Domain::TrieBranch`) — it does **not** attempt to
+//! reproduce Ethereum's real Merkle-Patricia Trie root (which requires Keccak256
+//! and RLP encoding, neither of which exist anywhere in this codebase).
+//!
+//! This is inferred from the available evidence, not confirmed by an explicit
+//! spec statement:
+//!   - `docs/emes_profile.md` §5.2 confirms the Geth tracer only ever captures
+//!     per-slot (prev, new) mutation pairs (`OnStorageChange`); it never captures
+//!     Geth's real per-account MPT root, so there is no authoritative Ethereum
+//!     root flowing through this pipeline to reproduce or compare against.
+//!   - `kaysentinel-hash`'s entire domain-separation scheme is built on BLAKE3,
+//!     structurally incompatible with Ethereum's Keccak/RLP-based MPT regardless
+//!     of design intent.
+//! If a future requirement demands real Ethereum MPT compatibility, this
+//! implementation must be replaced, not extended — see `StorageRootDeriver`'s
+//! trait boundary, which exists precisely so the rest of the builder doesn't
+//! need to care which algorithm produced the root.
+
+use std::collections::BTreeMap;
+
+use kaysentinel_hash::{Domain, derive_commitment};
+
+use crate::ir::timeline::CanonicalKey;
+use crate::ir::reduced::ReducedVariant;
+use crate::lifecycle::certificate::CanonicalAccountCertificate;
+use crate::lifecycle::hydration::{DerivationError, StorageRootDeriver};
+use crate::lifecycle::certificate::StorageRootState;
+
+/// The canonical "no storage touched" root: derived the same way as any other
+/// case (a zero-leaf Merkle fold), not a magic sentinel constant, so it's
+/// automatically consistent if the leaf/branch hashing ever changes.
+pub struct SimpleStorageRootDeriver;
+
+impl StorageRootDeriver for SimpleStorageRootDeriver {
+    /// No external base state is used — this derivation only covers slots
+    /// actually touched in the current transaction batch, not an account's full
+    /// historical storage. Real Ethereum-style trie derivation would need a
+    /// `BaseState` carrying the account's prior slots; this interim design
+    /// deliberately doesn't model that yet (see module-level design note).
+    type BaseState = ();
+
+    fn derive(
+        &self,
+        _base_state: &(),
+        terminal_slots: &BTreeMap<[u8; 32], [u8; 32]>,
+    ) -> Result<[u8; 32], DerivationError> {
+        let mut leaves: Vec<[u8; 32]> = terminal_slots
+            .iter() // BTreeMap iteration is key-sorted -> deterministic leaf order
+            .map(|(slot, value)| {
+                let mut canonical = Vec::with_capacity(64);
+                canonical.extend_from_slice(slot);
+                canonical.extend_from_slice(value);
+                derive_commitment(Domain::TrieLeaf, &canonical).to_bytes()
+            })
+            .collect();
+
+        if leaves.is_empty() {
+            leaves.push([0u8; 32]);
+        }
+
+        let mut size = 1usize;
+        while size < leaves.len() {
+            size <<= 1;
+        }
+        while leaves.len() < size {
+            leaves.push([0u8; 32]);
+        }
+
+        while leaves.len() > 1 {
+            let mut next = Vec::with_capacity(leaves.len() / 2);
+            for pair in leaves.chunks(2) {
+                let mut canonical = Vec::with_capacity(64);
+                canonical.extend_from_slice(&pair[0]);
+                canonical.extend_from_slice(&pair[1]);
+                next.push(derive_commitment(Domain::TrieBranch, &canonical).to_bytes());
+            }
+            leaves = next;
+        }
+
+        Ok(leaves[0])
+    }
+}
+
+/// Mutates every certificate's `storage_root` in place, deriving it from the
+/// account's `CanonicalKey::Storage` entries in `state_table`. Deliberately
+/// separate from `build_canonical_certificates`/`hydrate_accounts` (an
+/// additive post-processing pass, not a signature change to either) so
+/// existing call sites — including the already-pushed
+/// `runtime/builder/tests/conformance.rs` — keep working unmodified.
+///
+/// `CanonicalKey::Transient` entries are excluded on purpose: EIP-1153
+/// transient storage is explicitly ephemeral (cleared at transaction end) and
+/// is not part of an account's persistent storage root in real Ethereum
+/// semantics either.
+pub fn resolve_storage_roots<D: StorageRootDeriver<BaseState = ()>>(
+    certificates: &mut BTreeMap<[u8; 20], CanonicalAccountCertificate>,
+    state_table: &BTreeMap<CanonicalKey, ReducedVariant>,
+    deriver: &D,
+) -> Result<(), DerivationError> {
+    let mut slots_by_address: BTreeMap<[u8; 20], BTreeMap<[u8; 32], [u8; 32]>> = BTreeMap::new();
+
+    for (key, variant) in state_table {
+        if let (CanonicalKey::Storage { address, slot }, ReducedVariant::Storage(t)) = (key, variant) {
+            slots_by_address.entry(*address).or_default().insert(*slot, *t.terminal());
+        }
+    }
+
+    for (address, cert) in certificates.iter_mut() {
+        let slots = slots_by_address.get(address).cloned().unwrap_or_default();
+        let root = deriver.derive(&(), &slots)?;
+        cert.storage_root = StorageRootState::Verified(root);
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use crate::lifecycle::certificate::VerifiedGeneration;
+    use crate::lifecycle::keys::GenerationKey;
+
+    fn sample_cert(address: [u8; 20]) -> CanonicalAccountCertificate {
+        CanonicalAccountCertificate {
+            address,
+            generation: VerifiedGeneration {
+                key: GenerationKey { address, generation_id: 0 },
+                state_table_proof_root: [0u8; 32],
+            },
+            nonce: 0,
+            balance: [0u8; 32],
+            code_hash: [0u8; 32],
+            storage_root: StorageRootState::AwaitingDerivation,
+        }
+    }
+
+    #[test]
+    fn empty_storage_derives_deterministic_root() {
+        let deriver = SimpleStorageRootDeriver;
+        let empty = BTreeMap::new();
+        let root_a = deriver.derive(&(), &empty).unwrap();
+        let root_b = deriver.derive(&(), &empty).unwrap();
+        assert_eq!(root_a, root_b);
+    }
+
+    #[test]
+    fn nonempty_storage_differs_from_empty() {
+        let deriver = SimpleStorageRootDeriver;
+        let empty = BTreeMap::new();
+        let mut one_slot = BTreeMap::new();
+        one_slot.insert([1u8; 32], [2u8; 32]);
+
+        let empty_root = deriver.derive(&(), &empty).unwrap();
+        let nonempty_root = deriver.derive(&(), &one_slot).unwrap();
+        assert_ne!(empty_root, nonempty_root);
+    }
+
+    #[test]
+    fn different_slot_values_produce_different_roots() {
+        let deriver = SimpleStorageRootDeriver;
+        let mut slots_a = BTreeMap::new();
+        slots_a.insert([1u8; 32], [2u8; 32]);
+        let mut slots_b = BTreeMap::new();
+        slots_b.insert([1u8; 32], [3u8; 32]);
+
+        assert_ne!(deriver.derive(&(), &slots_a).unwrap(), deriver.derive(&(), &slots_b).unwrap());
+    }
+
+    #[test]
+    fn root_is_independent_of_insertion_order() {
+        let deriver = SimpleStorageRootDeriver;
+        let mut slots_a = BTreeMap::new();
+        slots_a.insert([1u8; 32], [0xAAu8; 32]);
+        slots_a.insert([2u8; 32], [0xBBu8; 32]);
+
+        let mut slots_b = BTreeMap::new();
+        slots_b.insert([2u8; 32], [0xBBu8; 32]);
+        slots_b.insert([1u8; 32], [0xAAu8; 32]);
+
+        // Both are BTreeMaps, so iteration order is identical regardless of
+        // insertion order -- this asserts that fact holds through derivation.
+        assert_eq!(deriver.derive(&(), &slots_a).unwrap(), deriver.derive(&(), &slots_b).unwrap());
+    }
+
+    #[test]
+    fn resolve_storage_roots_moves_every_certificate_off_awaiting_derivation() {
+        let addr_a = [0x01u8; 20];
+        let addr_b = [0x02u8; 20]; // has no storage entries at all
+
+        let mut certs = BTreeMap::new();
+        certs.insert(addr_a, sample_cert(addr_a));
+        certs.insert(addr_b, sample_cert(addr_b));
+
+        let mut state_table = BTreeMap::new();
+        state_table.insert(
+            CanonicalKey::Storage { address: addr_a, slot: [7u8; 32] },
+            ReducedVariant::Storage(crate::traits::ReducedTransition::new(
+                [0u8; 32],
+                [9u8; 32],
+                crate::ir::timeline::Provenance::new(),
+            )),
+        );
+
+        resolve_storage_roots(&mut certs, &state_table, &SimpleStorageRootDeriver).unwrap();
+
+        for cert in certs.values() {
+            assert!(matches!(cert.storage_root, StorageRootState::Verified(_)));
+        }
+        // addr_b had zero storage entries but still gets a real (empty) root,
+        // not a leftover AwaitingDerivation.
+        assert_ne!(certs[&addr_b].storage_root, StorageRootState::AwaitingDerivation);
+    }
+
+    #[test]
+    fn transient_storage_is_excluded_from_the_root() {
+        let addr = [0x03u8; 20];
+        let mut certs = BTreeMap::new();
+        certs.insert(addr, sample_cert(addr));
+
+        let mut state_table = BTreeMap::new();
+        state_table.insert(
+            CanonicalKey::Transient { address: addr, slot: [1u8; 32] },
+            ReducedVariant::Transient(crate::traits::ReducedTransition::new(
+                [0u8; 32],
+                [0xFFu8; 32],
+                crate::ir::timeline::Provenance::new(),
+            )),
+        );
+
+        resolve_storage_roots(&mut certs, &state_table, &SimpleStorageRootDeriver).unwrap();
+
+        // No CanonicalKey::Storage entries exist for this address (only
+        // Transient), so the root must equal the empty-storage root.
+        let empty_root = SimpleStorageRootDeriver.derive(&(), &BTreeMap::new()).unwrap();
+        match certs[&addr].storage_root {
+            StorageRootState::Verified(root) => assert_eq!(root, empty_root),
+            StorageRootState::AwaitingDerivation => panic!("expected Verified"),
+        }
+    }
+}

runtime/ssz/src/certificate.rs

@@ -220,6 +220,67 @@ mod tests {
         );
     }
 
+    /// The actual payoff of this phase: proves that a certificate produced by
+    /// the real pipeline, once its storage root is resolved, no longer hits
+    /// `UnresolvedStorageRoot` — closing the gap flagged when PR3 first shipped
+    /// (every real certificate hit that error, since nothing ever produced
+    /// `Verified` before this phase existed).
+    #[test]
+    fn full_vertical_slice_resolved_certificate_commits_successfully() {
+        use kaysentinel_builder::{SimpleStorageRootDeriver, resolve_storage_roots};
+        use kaysentinel_builder::lifecycle::certificate::{
+            AccountSnapshotSource, GenerationResolutionError, ResolvedAccountSnapshot, VerifiedGeneration,
+            build_canonical_certificates,
+        };
+        use kaysentinel_builder::lifecycle::keys::GenerationKey;
+        use kaysentinel_builder::ir::timeline::CanonicalKey;
+        use kaysentinel_builder::ir::reduced::ReducedVariant;
+
+        struct NullSnapshotSource;
+        impl AccountSnapshotSource for NullSnapshotSource {
+            fn resolve_identity_generation(&self, address: [u8; 20]) -> Result<VerifiedGeneration, GenerationResolutionError> {
+                Ok(VerifiedGeneration { key: GenerationKey { address, generation_id: 0 }, state_table_proof_root: [0u8; 32] })
+            }
+            fn recover_account_snapshot(&self, _address: [u8; 20]) -> Result<ResolvedAccountSnapshot, GenerationResolutionError> {
+                Ok(ResolvedAccountSnapshot { nonce: 0, balance: [0u8; 32], code_hash: [0u8; 32] })
+            }
+        }
+
+        let address = [0x55u8; 20];
+        let mut state_table = BTreeMap::new();
+        state_table.insert(
+            CanonicalKey::Storage { address, slot: [1u8; 32] },
+            ReducedVariant::Storage(kaysentinel_builder::ReducedTransition::new(
+                [0u8; 32],
+                [2u8; 32],
+                kaysentinel_builder::Provenance::new(),
+            )),
+        );
+
+        let generations = vec![kaysentinel_builder::lifecycle::canonical::CanonicalGeneration {
+            key: GenerationKey { address, generation_id: 0 },
+            begin: kaysentinel_builder::ir::timeline::TraceProvenance::default(),
+            end: None,
+        }];
+
+        // Before this phase: build_canonical_certificates alone always leaves
+        // storage_root == AwaitingDerivation, and every commit_root() call
+        // below would fail with UnresolvedStorageRoot.
+        let mut certs = build_canonical_certificates(&state_table, &generations, &NullSnapshotSource).unwrap();
+        assert!(certs.values().next().unwrap().commit_root().is_err());
+
+        // After: resolve_storage_roots closes the gap.
+        resolve_storage_roots(&mut certs, &state_table, &SimpleStorageRootDeriver).unwrap();
+
+        let cert = certs.get(&address).unwrap();
+        let encoded = cert.ssz_encode().unwrap();
+        let commitment = cert.commit_root().unwrap();
+
+        assert_eq!(encoded.len(), 180);
+        // Deterministic: re-deriving the same certificate produces the same commitment.
+        assert_eq!(commitment, cert.commit_root().unwrap());
+    }
+
     fn hex_encode(bytes: &[u8]) -> String {
         bytes.iter().map(|b| format!("{:02x}", b)).collect()
     }

runtime/ssz/src/lib.rs

@@ -39,6 +39,9 @@ pub struct CanonicalEventSummary {
 impl CanonicalEventSummary {
     pub const ENCODED_LEN: usize = 8 + 8 + 20 + 32;
 
+    /// Phase 3 (Write) for a fixed-size container: fields are simply concatenated
+    /// in declared order — there's no header/offset machinery needed since nothing
+    /// in this container is variable-length.
     pub fn ssz_encode(&self) -> Vec<u8> {
         let mut out = Vec::with_capacity(Self::ENCODED_LEN);
         out.extend_from_slice(&self.block_timestamp.to_le_bytes());
@@ -49,6 +52,11 @@ impl CanonicalEventSummary {
     }
 }
 
+/// `EpochStateSnapshot` — variable-size container: a fixed `epoch_index`, followed
+/// by a 4-byte offset pointing at the `event_summaries` list payload. Since
+/// `CanonicalEventSummary` is itself fixed-size, the list body is a plain
+/// concatenation of encoded elements — SSZ only needs one offset for the whole
+/// list field, not one per element.
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct EpochStateSnapshot {
     pub epoch_index: u64,
@@ -56,6 +64,8 @@ pub struct EpochStateSnapshot {
 }
 
 impl EpochStateSnapshot {
+    /// Bytes 0-7: epoch_index. Bytes 8-11: 4-byte offset to the variable payload
+    /// (always 12, since the fixed header is exactly 8 + 4 bytes).
     const FIXED_HEADER_LEN: usize = 8 + 4;
 
     pub fn ssz_encode(&self) -> Result<Vec<u8>, SszError> {
@@ -78,6 +88,31 @@ impl EpochStateSnapshot {
     }
 }
 
+// ============================================================================
+// Merkle Profile MP1
+// ============================================================================
+//
+// Two things in the source spec were ambiguous or inconsistent with the standard
+// SSZ convention, resolved here explicitly rather than guessed silently:
+//
+// 1. "If the number of leaf chunks is odd, the final leaf node is duplicated or
+//    handled strictly according to the deterministic tree packing standard" reads
+//    as offering two different, uncommitted-to alternatives. This implementation
+//    uses the real, standard SSZ approach: pad with zero chunks up to the next
+//    power of two, not duplication. Duplication (as used by some other Merkle
+//    schemes) creates second-preimage weaknesses that zero-padding to a fixed
+//    known width avoids.
+// 2. The formula "H = BLAKE3(Left || Right)" is captioned "domain-separated" but
+//    the formula itself shows no domain prefix. To actually be domain-separated
+//    (and consistent with `kaysentinel-hash`'s own domain-separation invariant:
+//    HashInput := DomainBytes || CanonicalBytes), this implementation hashes
+//    `Domain::SsrRoot bytes || Left || Right` via `derive_commitment`, reusing
+//    `Domain::SsrRoot` since the hash crate's domain registry already documents
+//    it for exactly this purpose ("Serialization Record roots").
+
+/// Splits a canonical byte stream into 32-byte chunks, right-padding the final
+/// chunk with zero octets if needed. An empty input produces a single zero chunk
+/// (there must always be at least one leaf to Merkleize).
 pub fn chunkify(bytes: &[u8]) -> Vec<[u8; 32]> {
     if bytes.is_empty() {
         return vec![[0u8; 32]];
@@ -101,6 +136,8 @@ fn hash_pair(left: &[u8; 32], right: &[u8; 32]) -> [u8; 32] {
     derive_commitment(Domain::SsrRoot, &canonical).to_bytes()
 }
 
+/// Builds the deterministic binary Merkle tree over a chunk sequence and returns
+/// its root, per Merkle Profile MP1.
 pub fn merkleize(chunks: &[[u8; 32]]) -> [u8; 32] {
     let mut layer: Vec<[u8; 32]> = if chunks.is_empty() { vec![[0u8; 32]] } else { chunks.to_vec() };
 
@@ -122,6 +159,8 @@ pub fn merkleize(chunks: &[[u8; 32]]) -> [u8; 32] {
     layer[0]
 }
 
+/// Convenience: the full Layer 2 -> Layer 3 pipeline (chunkify + merkleize) applied
+/// directly to an object's SSZ-encoded bytes.
 pub fn ssr_root(serialized_bytes: &[u8]) -> [u8; 32] {
     merkleize(&chunkify(serialized_bytes))
 }
@@ -146,6 +185,12 @@ mod tests {
         assert_eq!(encoded.len(), 68);
     }
 
+    /// Regression pin for VEC-SSR-RC1-001 (single-element EpochStateSnapshot). The
+    /// source spec's own claimed hex for this vector was wrong — decoding its
+    /// `block_timestamp` field back out gives 171711, not the 171717 stated in the
+    /// same vector's input object. This pins the value this implementation
+    /// actually computes for the stated input, verified independently in Python
+    /// before being hardcoded here.
     #[test]
     fn epoch_state_snapshot_matches_correctly_computed_vector() {
         let snapshot = EpochStateSnapshot { epoch_index: 5, event_summaries: vec![sample_summary()] };
@@ -170,7 +215,7 @@ mod tests {
 
     #[test]
     fn chunkify_pads_final_chunk_with_zeros() {
-        let bytes = vec![1u8, 2, 3];
+        let bytes = vec![1u8, 2, 3]; // 3 bytes, one short chunk
         let chunks = chunkify(&bytes);
         assert_eq!(chunks.len(), 1);
         assert_eq!(&chunks[0][..3], &[1, 2, 3]);
@@ -179,7 +224,7 @@ mod tests {
 
     #[test]
     fn chunkify_exact_multiple_has_no_padding_chunk() {
-        let bytes = vec![7u8; 64];
+        let bytes = vec![7u8; 64]; // exactly 2 chunks
         let chunks = chunkify(&bytes);
         assert_eq!(chunks.len(), 2);
     }
@@ -199,11 +244,16 @@ mod tests {
 
     #[test]
     fn merkleize_single_chunk_equals_the_chunk_itself() {
+        // A single-leaf tree's root is just that leaf — no hashing needed, since
+        // there's nothing to combine it with.
         let chunks = chunkify(&[9u8; 10]);
         assert_eq!(chunks.len(), 1);
         assert_eq!(merkleize(&chunks), chunks[0]);
     }
 
+    /// Real, computed root for VEC-SSR-RC1-001's correctly-encoded bytes — printed
+    /// first via `cargo test -- --nocapture`, then pinned, exactly like the
+    /// hash crate's vectors. Not copied from the (incorrect) source doc.
     #[test]
     fn ssr_root_for_vec_ssr_rc1_001_is_pinned() {
         let snapshot = EpochStateSnapshot { epoch_index: 5, event_summaries: vec![sample_summary()] };