zebra_state/service/finalized_state/disk_format/
block.rs

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
//! Block and transaction serialization formats for finalized data.
//!
//! # Correctness
//!
//! [`crate::constants::state_database_format_version_in_code()`] must be incremented
//! each time the database format (column, serialization, etc) changes.

use zebra_chain::{
    block::{self, Height},
    serialization::{ZcashDeserializeInto, ZcashSerialize},
    transaction::{self, Transaction},
};

use crate::service::finalized_state::disk_format::{
    expand_zero_be_bytes, truncate_zero_be_bytes, FromDisk, IntoDisk,
};

#[cfg(any(test, feature = "proptest-impl"))]
use proptest_derive::Arbitrary;
#[cfg(any(test, feature = "proptest-impl"))]
use serde::{Deserialize, Serialize};

/// The maximum value of an on-disk serialized [`Height`].
///
/// This allows us to store [`OutputLocation`](crate::OutputLocation)s in
/// 8 bytes, which makes database searches more efficient.
///
/// # Consensus
///
/// This maximum height supports on-disk storage of blocks until around 2050.
///
/// Since Zebra only stores fully verified blocks on disk, blocks with heights
/// larger than this height are rejected before reaching the database.
/// (It would take decades to generate a valid chain this high.)
pub const MAX_ON_DISK_HEIGHT: Height = Height((1 << (HEIGHT_DISK_BYTES * 8)) - 1);

/// [`Height`]s are stored as 3 bytes on disk.
///
/// This reduces database size and increases lookup performance.
pub const HEIGHT_DISK_BYTES: usize = 3;

/// [`TransactionIndex`]es are stored as 2 bytes on disk.
///
/// This reduces database size and increases lookup performance.
pub const TX_INDEX_DISK_BYTES: usize = 2;

/// [`TransactionLocation`]s are stored as a 3 byte height and a 2 byte transaction index.
///
/// This reduces database size and increases lookup performance.
pub const TRANSACTION_LOCATION_DISK_BYTES: usize = HEIGHT_DISK_BYTES + TX_INDEX_DISK_BYTES;

// Block and transaction types

/// A transaction's index in its block.
///
/// # Consensus
///
/// A 2-byte index supports on-disk storage of transactions in blocks up to ~5 MB.
/// (The current maximum block size is 2 MB.)
///
/// Since Zebra only stores fully verified blocks on disk,
/// blocks larger than this size are rejected before reaching the database.
///
/// (The maximum transaction count is tested by the large generated block serialization tests.)
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
#[cfg_attr(
    any(test, feature = "proptest-impl"),
    derive(Arbitrary, Default, Serialize, Deserialize)
)]
pub struct TransactionIndex(pub(super) u16);

impl TransactionIndex {
    /// Creates a transaction index from the inner type.
    pub fn from_index(transaction_index: u16) -> TransactionIndex {
        TransactionIndex(transaction_index)
    }

    /// Returns this index as the inner type.
    pub fn index(&self) -> u16 {
        self.0
    }

    /// Creates a transaction index from a `usize`.
    pub fn from_usize(transaction_index: usize) -> TransactionIndex {
        TransactionIndex(
            transaction_index
                .try_into()
                .expect("the maximum valid index fits in the inner type"),
        )
    }

    /// Returns this index as a `usize`.
    pub fn as_usize(&self) -> usize {
        self.0.into()
    }

    /// Creates a transaction index from a `u64`.
    pub fn from_u64(transaction_index: u64) -> TransactionIndex {
        TransactionIndex(
            transaction_index
                .try_into()
                .expect("the maximum valid index fits in the inner type"),
        )
    }

    /// Returns this index as a `u64`.
    #[allow(dead_code)]
    pub fn as_u64(&self) -> u64 {
        self.0.into()
    }

    /// The minimum value of a transaction index.
    ///
    /// This value corresponds to the coinbase transaction.
    pub const MIN: Self = Self(u16::MIN);

    /// The maximum value of a transaction index.
    ///
    /// This value corresponds to the highest possible transaction index.
    pub const MAX: Self = Self(u16::MAX);
}

/// A transaction's location in the chain, by block height and transaction index.
///
/// This provides a chain-order list of transactions.
#[derive(Copy, Clone, Debug, Eq, PartialEq, Ord, PartialOrd, Hash)]
#[cfg_attr(
    any(test, feature = "proptest-impl"),
    derive(Arbitrary, Default, Serialize, Deserialize)
)]
pub struct TransactionLocation {
    /// The block height of the transaction.
    pub height: Height,

    /// The index of the transaction in its block.
    pub index: TransactionIndex,
}

impl TransactionLocation {
    /// Creates a transaction location from a block height and transaction index.
    pub fn from_parts(height: Height, index: TransactionIndex) -> TransactionLocation {
        TransactionLocation { height, index }
    }

    /// Creates a transaction location from a block height and transaction index.
    pub fn from_index(height: Height, transaction_index: u16) -> TransactionLocation {
        TransactionLocation {
            height,
            index: TransactionIndex::from_index(transaction_index),
        }
    }

    /// Creates a transaction location from a block height and `usize` transaction index.
    pub fn from_usize(height: Height, transaction_index: usize) -> TransactionLocation {
        TransactionLocation {
            height,
            index: TransactionIndex::from_usize(transaction_index),
        }
    }

    /// Creates a transaction location from a block height and `u64` transaction index.
    pub fn from_u64(height: Height, transaction_index: u64) -> TransactionLocation {
        TransactionLocation {
            height,
            index: TransactionIndex::from_u64(transaction_index),
        }
    }

    /// The minimum value of a transaction location.
    ///
    /// This value corresponds to the genesis coinbase transaction.
    pub const MIN: Self = Self {
        height: Height::MIN,
        index: TransactionIndex::MIN,
    };

    /// The maximum value of a transaction location.
    ///
    /// This value corresponds to the last transaction in the highest possible block.
    pub const MAX: Self = Self {
        height: Height::MAX,
        index: TransactionIndex::MAX,
    };

    /// The minimum value of a transaction location for `height`.
    ///
    /// This value is the coinbase transaction.
    pub const fn min_for_height(height: Height) -> Self {
        Self {
            height,
            index: TransactionIndex::MIN,
        }
    }

    /// The maximum value of a transaction location for `height`.
    ///
    /// This value can be a valid entry, but it won't fit in a 2MB block.
    pub const fn max_for_height(height: Height) -> Self {
        Self {
            height,
            index: TransactionIndex::MAX,
        }
    }
}

// Block and transaction trait impls

impl IntoDisk for block::Header {
    type Bytes = Vec<u8>;

    fn as_bytes(&self) -> Self::Bytes {
        self.zcash_serialize_to_vec()
            .expect("serialization to vec doesn't fail")
    }
}

impl FromDisk for block::Header {
    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
        bytes
            .as_ref()
            .zcash_deserialize_into()
            .expect("deserialization format should match the serialization format used by IntoDisk")
    }
}

impl IntoDisk for Height {
    /// Consensus: see the note at [`MAX_ON_DISK_HEIGHT`].
    type Bytes = [u8; HEIGHT_DISK_BYTES];

    fn as_bytes(&self) -> Self::Bytes {
        let mem_bytes = self.0.to_be_bytes();

        let disk_bytes = truncate_zero_be_bytes(&mem_bytes, HEIGHT_DISK_BYTES);

        match disk_bytes {
            Some(b) => b.try_into().unwrap(),

            // # Security
            //
            // The RPC method or state query was given a block height that is ridiculously high.
            // But to save space in database indexes, we don't support heights 2^24 and above.
            //
            // Instead we return the biggest valid database Height to the lookup code.
            // So RPC methods and queued block checks will return an error or None.
            //
            // At the current block production rate, these heights can't be inserted into the
            // database until at least 2050. (Blocks are verified in strict height order.)
            None => truncate_zero_be_bytes(&MAX_ON_DISK_HEIGHT.0.to_be_bytes(), HEIGHT_DISK_BYTES)
                .expect("max on disk height is valid")
                .try_into()
                .unwrap(),
        }
    }
}

impl FromDisk for Height {
    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
        let mem_len = u32::BITS / 8;
        let mem_len = mem_len.try_into().unwrap();

        let mem_bytes = expand_zero_be_bytes(disk_bytes.as_ref(), mem_len);
        let mem_bytes = mem_bytes.try_into().unwrap();
        Height(u32::from_be_bytes(mem_bytes))
    }
}

impl IntoDisk for block::Hash {
    type Bytes = [u8; 32];

    fn as_bytes(&self) -> Self::Bytes {
        self.0
    }
}

impl FromDisk for block::Hash {
    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
        let array = bytes.as_ref().try_into().unwrap();
        Self(array)
    }
}

// Transaction trait impls

impl IntoDisk for Transaction {
    type Bytes = Vec<u8>;

    fn as_bytes(&self) -> Self::Bytes {
        self.zcash_serialize_to_vec()
            .expect("serialization to vec doesn't fail")
    }
}

impl FromDisk for Transaction {
    fn from_bytes(bytes: impl AsRef<[u8]>) -> Self {
        let bytes = bytes.as_ref();

        // TODO: skip cryptography verification during transaction deserialization from storage,
        //       or do it in a rayon thread (ideally in parallel with other transactions)
        bytes
            .as_ref()
            .zcash_deserialize_into()
            .expect("deserialization format should match the serialization format used by IntoDisk")
    }
}

/// TransactionIndex is only serialized as part of TransactionLocation
impl IntoDisk for TransactionIndex {
    type Bytes = [u8; TX_INDEX_DISK_BYTES];

    fn as_bytes(&self) -> Self::Bytes {
        self.index().to_be_bytes()
    }
}

impl FromDisk for TransactionIndex {
    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
        let disk_bytes = disk_bytes.as_ref().try_into().unwrap();

        TransactionIndex::from_index(u16::from_be_bytes(disk_bytes))
    }
}

impl IntoDisk for TransactionLocation {
    type Bytes = [u8; TRANSACTION_LOCATION_DISK_BYTES];

    fn as_bytes(&self) -> Self::Bytes {
        let height_bytes = self.height.as_bytes().to_vec();
        let index_bytes = self.index.as_bytes().to_vec();

        [height_bytes, index_bytes].concat().try_into().unwrap()
    }
}

impl FromDisk for TransactionLocation {
    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
        let (height_bytes, index_bytes) = disk_bytes.as_ref().split_at(HEIGHT_DISK_BYTES);

        let height = Height::from_bytes(height_bytes);
        let index = TransactionIndex::from_bytes(index_bytes);

        TransactionLocation { height, index }
    }
}

impl IntoDisk for transaction::Hash {
    type Bytes = [u8; 32];

    fn as_bytes(&self) -> Self::Bytes {
        self.0
    }
}

impl FromDisk for transaction::Hash {
    fn from_bytes(disk_bytes: impl AsRef<[u8]>) -> Self {
        transaction::Hash(disk_bytes.as_ref().try_into().unwrap())
    }
}