zebra_state/service/finalized_state/disk_format/
block.rs

1//! Block and transaction serialization formats for finalized data.
2//!
3//! # Correctness
4//!
5//! [`crate::constants::state_database_format_version_in_code()`] must be incremented
6//! each time the database format (column, serialization, etc) changes.
7
8use zebra_chain::{
9    block::{self, Height},
10    serialization::{ZcashDeserializeInto, ZcashSerialize},
11    transaction::{self, Transaction},
12};
13
14use crate::service::finalized_state::disk_format::{
15    expand_zero_be_bytes, truncate_zero_be_bytes, FromDisk, IntoDisk,
16};
17
18#[cfg(any(test, feature = "proptest-impl"))]
19use proptest_derive::Arbitrary;
20#[cfg(any(test, feature = "proptest-impl"))]
21use serde::{Deserialize, Serialize};
22
23/// The maximum value of an on-disk serialized [`Height`].
24///
25/// This allows us to store [`OutputLocation`](crate::OutputLocation)s in
26/// 8 bytes, which makes database searches more efficient.
27///
28/// # Consensus
29///
30/// This maximum height supports on-disk storage of blocks until around 2050.
31///
32/// Since Zebra only stores fully verified blocks on disk, blocks with heights
33/// larger than this height are rejected before reaching the database.
34/// (It would take decades to generate a valid chain this high.)
35pub const MAX_ON_DISK_HEIGHT: Height = Height((1 << (HEIGHT_DISK_BYTES * 8)) - 1);
36
37/// [`Height`]s are stored as 3 bytes on disk.
38///
39/// This reduces database size and increases lookup performance.
40pub const HEIGHT_DISK_BYTES: usize = 3;
41
42/// [`TransactionIndex`]es are stored as 2 bytes on disk.
43///
44/// This reduces database size and increases lookup performance.
45pub const TX_INDEX_DISK_BYTES: usize = 2;
46
47/// [`TransactionLocation`]s are stored as a 3 byte height and a 2 byte transaction index.
48///
49/// This reduces database size and increases lookup performance.
50pub const TRANSACTION_LOCATION_DISK_BYTES: usize = HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES;
51
52// Block and transaction types
53
54/// A transaction's index in its block.
55///
56/// # Consensus
57///
58/// A 2-byte index supports on-disk storage of transactions in blocks up to ~5 MB.
59/// (The current maximum block size is 2 MB.)
60///
61/// Since Zebra only stores fully verified blocks on disk,
62/// blocks larger than this size are rejected before reaching the database.
63///
64/// (The maximum transaction count is tested by the large generated block serialization tests.)
65#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
66#[cfg_attr(
67    any(test, feature = "proptest-impl"),
68    derive(Arbitrary, Default, Serialize, Deserialize)
69)]
70pub struct TransactionIndex(pub(super) u16);
71
72impl TransactionIndex {
73    /// Creates a transaction index from the inner type.
74    pub fn from_index(transaction_index: u16) -> TransactionIndex {
75        TransactionIndex(transaction_index)
76    }
77
78    /// Returns this index as the inner type.
79    pub fn index(&self) -> u16 {
80        self.0
81    }
82
83    /// Creates a transaction index from a `usize`.
84    pub fn from_usize(transaction_index: usize) -> TransactionIndex {
85        TransactionIndex(
86            transaction_index
87                .try_into()
88                .expect("the maximum valid index fits in the inner type"),
89        )
90    }
91
92    /// Returns this index as a `usize`.
93    pub fn as_usize(&self) -> usize {
94        self.0.into()
95    }
96
97    /// Creates a transaction index from a `u64`.
98    pub fn from_u64(transaction_index: u64) -> TransactionIndex {
99        TransactionIndex(
100            transaction_index
101                .try_into()
102                .expect("the maximum valid index fits in the inner type"),
103        )
104    }
105
106    /// Returns this index as a `u64`.
107    #[allow(dead_code)]
108    pub fn as_u64(&self) -> u64 {
109        self.0.into()
110    }
111
112    /// The minimum value of a transaction index.
113    ///
114    /// This value corresponds to the coinbase transaction.
115    pub const MIN: Self = Self(u16::MIN);
116
117    /// The maximum value of a transaction index.
118    ///
119    /// This value corresponds to the highest possible transaction index.
120    pub const MAX: Self = Self(u16::MAX);
121}
122
123/// A transaction's location in the chain, by block height and transaction index.
124///
125/// This provides a chain-order list of transactions.
126#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
127#[cfg_attr(
128    any(test, feature = "proptest-impl"),
129    derive(Arbitrary, Default, Serialize, Deserialize)
130)]
131pub struct TransactionLocation {
132    /// The block height of the transaction.
133    pub height: Height,
134
135    /// The index of the transaction in its block.
136    pub index: TransactionIndex,
137}
138
139impl TransactionLocation {
140    /// Creates a transaction location from a block height and transaction index.
141    pub fn from_parts(height: Height, index: TransactionIndex) -> TransactionLocation {
142        TransactionLocation { height, index }
143    }
144
145    /// Creates a transaction location from a block height and transaction index.
146    pub fn from_index(height: Height, transaction_index: u16) -> TransactionLocation {
147        TransactionLocation {
148            height,
149            index: TransactionIndex::from_index(transaction_index),
150        }
151    }
152
153    /// Creates a transaction location from a block height and `usize` transaction index.
154    pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation {
155        TransactionLocation {
156            height,
157            index: TransactionIndex::from_usize(transaction_index),
158        }
159    }
160
161    /// Creates a transaction location from a block height and `u64` transaction index.
162    pub fn from_u64(height: Height, transaction_index: u64) -> TransactionLocation {
163        TransactionLocation {
164            height,
165            index: TransactionIndex::from_u64(transaction_index),
166        }
167    }
168
169    /// The minimum value of a transaction location.
170    ///
171    /// This value corresponds to the genesis coinbase transaction.
172    pub const MIN: Self = Self {
173        height: Height::MIN,
174        index: TransactionIndex::MIN,
175    };
176
177    /// The maximum value of a transaction location.
178    ///
179    /// This value corresponds to the last transaction in the highest possible block.
180    pub const MAX: Self = Self {
181        height: Height::MAX,
182        index: TransactionIndex::MAX,
183    };
184
185    /// The minimum value of a transaction location for `height`.
186    ///
187    /// This value is the coinbase transaction.
188    pub const fn min_for_height(height: Height) -> Self {
189        Self {
190            height,
191            index: TransactionIndex::MIN,
192        }
193    }
194
195    /// The maximum value of a transaction location for `height`.
196    ///
197    /// This value can be a valid entry, but it won't fit in a 2MB block.
198    pub const fn max_for_height(height: Height) -> Self {
199        Self {
200            height,
201            index: TransactionIndex::MAX,
202        }
203    }
204}
205
206// Block and transaction trait impls
207
208impl IntoDisk for block::Header {
209    type Bytes = Vec<u8>;
210
211    fn as_bytes(&self) -> Self::Bytes {
212        self.zcash_serialize_to_vec()
213            .expect("serialization to vec doesn't fail")
214    }
215}
216
217impl FromDisk for block::Header {
218    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
219        bytes
220            .as_ref()
221            .zcash_deserialize_into()
222            .expect("deserialization format should match the serialization format used by IntoDisk")
223    }
224}
225
226impl IntoDisk for Height {
227    /// Consensus: see the note at [`MAX_ON_DISK_HEIGHT`].
228    type Bytes = [u8; HEIGHT_DISK_BYTES];
229
230    fn as_bytes(&self) -> Self::Bytes {
231        let mem_bytes = self.0.to_be_bytes();
232
233        let disk_bytes = truncate_zero_be_bytes(&mem_bytes, HEIGHT_DISK_BYTES);
234
235        match disk_bytes {
236            Some(b) => b.try_into().unwrap(),
237
238            // # Security
239            //
240            // The RPC method or state query was given a block height that is ridiculously high.
241            // But to save space in database indexes, we don't support heights 2^24 and above.
242            //
243            // Instead we return the biggest valid database Height to the lookup code.
244            // So RPC methods and queued block checks will return an error or None.
245            //
246            // At the current block production rate, these heights can't be inserted into the
247            // database until at least 2050. (Blocks are verified in strict height order.)
248            None => truncate_zero_be_bytes(&MAX_ON_DISK_HEIGHT.0.to_be_bytes(), HEIGHT_DISK_BYTES)
249                .expect("max on disk height is valid")
250                .try_into()
251                .unwrap(),
252        }
253    }
254}
255
256impl FromDisk for Height {
257    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
258        let mem_len = u32::BITS / 8;
259        let mem_len = mem_len.try_into().unwrap();
260
261        let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len);
262        let mem_bytes = mem_bytes.try_into().unwrap();
263        Height(u32::from_be_bytes(mem_bytes))
264    }
265}
266
267impl IntoDisk for block::Hash {
268    type Bytes = [u8; 32];
269
270    fn as_bytes(&self) -> Self::Bytes {
271        self.0
272    }
273}
274
275impl FromDisk for block::Hash {
276    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
277        let array = bytes.as_ref().try_into().unwrap();
278        Self(array)
279    }
280}
281
282// Transaction trait impls
283
284impl IntoDisk for Transaction {
285    type Bytes = Vec<u8>;
286
287    fn as_bytes(&self) -> Self::Bytes {
288        self.zcash_serialize_to_vec()
289            .expect("serialization to vec doesn't fail")
290    }
291}
292
293impl FromDisk for Transaction {
294    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
295        let bytes = bytes.as_ref();
296
297        // TODO: skip cryptography verification during transaction deserialization from storage,
298        //       or do it in a rayon thread (ideally in parallel with other transactions)
299        bytes
300            .as_ref()
301            .zcash_deserialize_into()
302            .expect("deserialization format should match the serialization format used by IntoDisk")
303    }
304}
305
306/// TransactionIndex is only serialized as part of TransactionLocation
307impl IntoDisk for TransactionIndex {
308    type Bytes = [u8; TX_INDEX_DISK_BYTES];
309
310    fn as_bytes(&self) -> Self::Bytes {
311        self.index().to_be_bytes()
312    }
313}
314
315impl FromDisk for TransactionIndex {
316    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
317        let disk_bytes = disk_bytes.as_ref().try_into().unwrap();
318
319        TransactionIndex::from_index(u16::from_be_bytes(disk_bytes))
320    }
321}
322
323impl IntoDisk for TransactionLocation {
324    type Bytes = [u8; TRANSACTION_LOCATION_DISK_BYTES];
325
326    fn as_bytes(&self) -> Self::Bytes {
327        let height_bytes = self.height.as_bytes().to_vec();
328        let index_bytes = self.index.as_bytes().to_vec();
329
330        [height_bytes, index_bytes].concat().try_into().unwrap()
331    }
332}
333
334impl FromDisk for Option<TransactionLocation> {
335    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
336        if disk_bytes.as_ref().len() == TRANSACTION_LOCATION_DISK_BYTES {
337            Some(TransactionLocation::from_bytes(disk_bytes))
338        } else {
339            None
340        }
341    }
342}
343
344impl FromDisk for TransactionLocation {
345    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
346        let (height_bytes, index_bytes) = disk_bytes.as_ref().split_at(HEIGHT_DISK_BYTES);
347
348        let height = Height::from_bytes(height_bytes);
349        let index = TransactionIndex::from_bytes(index_bytes);
350
351        TransactionLocation { height, index }
352    }
353}
354
355impl IntoDisk for transaction::Hash {
356    type Bytes = [u8; 32];
357
358    fn as_bytes(&self) -> Self::Bytes {
359        self.0
360    }
361}
362
363impl FromDisk for transaction::Hash {
364    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
365        transaction::Hash(disk_bytes.as_ref().try_into().unwrap())
366    }
367}
zebra_state/service/finalized_state/disk_format/block.rs

zebra_state/service/finalized_state/disk_format/
block.rs
