zebra_state/config.rs
1//! Cached state configuration for Zebra.
2
3use std::{
4 fs::{self, canonicalize, remove_dir_all, DirEntry, ReadDir},
5 io::ErrorKind,
6 path::{Path, PathBuf},
7 time::Duration,
8};
9
10use semver::Version;
11use serde::{Deserialize, Serialize};
12use tokio::task::{spawn_blocking, JoinHandle};
13use tracing::Span;
14
15use zebra_chain::{common::default_cache_dir, parameters::Network};
16
17use crate::{
18 constants::{DATABASE_FORMAT_VERSION_FILE_NAME, STATE_DATABASE_KIND},
19 service::finalized_state::restorable_db_versions,
20 state_database_format_version_in_code, BoxError,
21};
22
23/// Configuration for the state service.
24#[derive(Clone, Debug, Eq, PartialEq, Deserialize, Serialize)]
25#[serde(deny_unknown_fields, default)]
26pub struct Config {
27 /// The root directory for storing cached block data.
28 ///
29 /// If you change this directory, you might also want to change `network.cache_dir`.
30 ///
31 /// This cache stores permanent blockchain state that can be replicated from
32 /// the network, including the best chain, blocks, the UTXO set, and other indexes.
33 /// Any state that can be rolled back is only stored in memory.
34 ///
35 /// The `zebra-state` cache does *not* include any private data, such as wallet data.
36 ///
37 /// You can delete the entire cached state directory, but it will impact your node's
38 /// readiness and network usage. If you do, Zebra will re-sync from genesis the next
39 /// time it is launched.
40 ///
41 /// The default directory is platform dependent, based on
42 /// [`dirs::cache_dir()`](https://docs.rs/dirs/3.0.1/dirs/fn.cache_dir.html):
43 ///
44 /// |Platform | Value | Example |
45 /// | ------- | ----------------------------------------------- | ------------------------------------ |
46 /// | Linux | `$XDG_CACHE_HOME/zebra` or `$HOME/.cache/zebra` | `/home/alice/.cache/zebra` |
47 /// | macOS | `$HOME/Library/Caches/zebra` | `/Users/Alice/Library/Caches/zebra` |
48 /// | Windows | `{FOLDERID_LocalAppData}\zebra` | `C:\Users\Alice\AppData\Local\zebra` |
49 /// | Other | `std::env::current_dir()/cache/zebra` | `/cache/zebra` |
50 ///
51 /// # Security
52 ///
53 /// If you are running Zebra with elevated permissions ("root"), create the
54 /// directory for this file before running Zebra, and make sure the Zebra user
55 /// account has exclusive access to that directory, and other users can't modify
56 /// its parent directories.
57 ///
58 /// # Implementation Details
59 ///
60 /// Each state format version and network has a separate state.
61 /// These states are stored in `state/vN/mainnet` and `state/vN/testnet` subdirectories,
62 /// underneath the `cache_dir` path, where `N` is the state format version.
63 ///
64 /// When Zebra's state format changes, it creates a new state subdirectory for that version,
65 /// and re-syncs from genesis.
66 ///
67 /// Old state versions are automatically deleted at startup. You can also manually delete old
68 /// state versions.
69 pub cache_dir: PathBuf,
70
71 /// Whether to use an ephemeral database.
72 ///
73 /// Ephemeral databases are stored in a temporary directory created using [`tempfile::tempdir()`].
74 /// They are deleted when Zebra exits successfully.
75 /// (If Zebra panics or crashes, the ephemeral database won't be deleted.)
76 ///
77 /// Set to `false` by default. If this is set to `true`, [`cache_dir`] is ignored.
78 ///
79 /// Ephemeral directories are created in the [`std::env::temp_dir()`].
80 /// Zebra names each directory after the state version and network, for example: `zebra-state-v21-mainnet-XnyGnE`.
81 ///
82 /// [`cache_dir`]: struct.Config.html#structfield.cache_dir
83 pub ephemeral: bool,
84
85 /// Whether to cache non-finalized blocks on disk to be restored when Zebra restarts.
86 ///
87 /// Set to `true` by default. If this is set to `false`, Zebra will irrecoverably drop
88 /// non-finalized blocks when the process exits and will have to re-download them from
89 /// the network when it restarts, if those blocks are still available in the network.
90 ///
91 /// Note: The non-finalized state will be written to a backup cache once per 5 seconds at most.
92 /// If blocks are added to the non-finalized state more frequently, the backup may not reflect
93 /// Zebra's last non-finalized state before it shut down.
94 pub should_backup_non_finalized_state: bool,
95
96 /// Whether to delete the old database directories when present.
97 ///
98 /// Set to `true` by default. If this is set to `false`,
99 /// no check for old database versions will be made and nothing will be
100 /// deleted.
101 pub delete_old_database: bool,
102
103 // Debug configs
104 //
105 /// Commit blocks to the finalized state up to this height, then exit Zebra.
106 ///
107 /// Set to `None` by default: Zebra continues syncing indefinitely.
108 pub debug_stop_at_height: Option<u32>,
109
110 /// While Zebra is running, check state validity this often.
111 ///
112 /// Set to `None` by default: Zebra only checks state format validity on startup and shutdown.
113 #[serde(with = "humantime_serde")]
114 pub debug_validity_check_interval: Option<Duration>,
115
116 // Elasticsearch configs
117 //
118 #[cfg(feature = "elasticsearch")]
119 /// The elasticsearch database url.
120 pub elasticsearch_url: String,
121
122 #[cfg(feature = "elasticsearch")]
123 /// The elasticsearch database username.
124 pub elasticsearch_username: String,
125
126 #[cfg(feature = "elasticsearch")]
127 /// The elasticsearch database password.
128 pub elasticsearch_password: String,
129}
130
131fn gen_temp_path(prefix: &str) -> PathBuf {
132 tempfile::Builder::new()
133 .prefix(prefix)
134 .tempdir()
135 .expect("temporary directory is created successfully")
136 .keep()
137}
138
139impl Config {
140 /// Returns the path for the database, based on the kind, major version and network.
141 /// Each incompatible database format or network gets its own unique path.
142 pub fn db_path(
143 &self,
144 db_kind: impl AsRef<str>,
145 major_version: u64,
146 network: &Network,
147 ) -> PathBuf {
148 let db_kind = db_kind.as_ref();
149 let major_version = format!("v{major_version}");
150 let net_dir = network.lowercase_name();
151
152 if self.ephemeral {
153 gen_temp_path(&format!("zebra-{db_kind}-{major_version}-{net_dir}-"))
154 } else {
155 self.cache_dir
156 .join(db_kind)
157 .join(major_version)
158 .join(net_dir)
159 }
160 }
161
162 /// Returns the path for the non-finalized state backup directory, based on the network.
163 /// Non-finalized state backup files are encoded in the network protocol format and remain
164 /// valid across db format upgrades.
165 pub fn non_finalized_state_backup_dir(&self, network: &Network) -> Option<PathBuf> {
166 if self.ephemeral || !self.should_backup_non_finalized_state {
167 // Ephemeral databases are intended to be irrecoverable across restarts and don't
168 // require a backup for the non-finalized state.
169 return None;
170 }
171
172 let net_dir = network.lowercase_name();
173 Some(self.cache_dir.join("non_finalized_state").join(net_dir))
174 }
175
176 /// Returns the path for the database format minor/patch version file,
177 /// based on the kind, major version and network.
178 pub fn version_file_path(
179 &self,
180 db_kind: impl AsRef<str>,
181 major_version: u64,
182 network: &Network,
183 ) -> PathBuf {
184 let mut version_path = self.db_path(db_kind, major_version, network);
185
186 version_path.push(DATABASE_FORMAT_VERSION_FILE_NAME);
187
188 version_path
189 }
190
191 /// Returns a config for a temporary database that is deleted when it is dropped.
192 pub fn ephemeral() -> Config {
193 Config {
194 ephemeral: true,
195 ..Config::default()
196 }
197 }
198}
199
200impl Default for Config {
201 fn default() -> Self {
202 Self {
203 cache_dir: default_cache_dir(),
204 ephemeral: false,
205 should_backup_non_finalized_state: true,
206 delete_old_database: true,
207 debug_stop_at_height: None,
208 debug_validity_check_interval: None,
209 #[cfg(feature = "elasticsearch")]
210 elasticsearch_url: "https://localhost:9200".to_string(),
211 #[cfg(feature = "elasticsearch")]
212 elasticsearch_username: "elastic".to_string(),
213 #[cfg(feature = "elasticsearch")]
214 elasticsearch_password: "".to_string(),
215 }
216 }
217}
218
219// Cleaning up old database versions
220// TODO: put this in a different module?
221
222/// Spawns a task that checks if there are old state database folders,
223/// and deletes them from the filesystem.
224///
225/// See `check_and_delete_old_databases()` for details.
226pub fn check_and_delete_old_state_databases(config: &Config, network: &Network) -> JoinHandle<()> {
227 check_and_delete_old_databases(
228 config,
229 STATE_DATABASE_KIND,
230 state_database_format_version_in_code().major,
231 network,
232 )
233}
234
235/// Spawns a task that checks if there are old database folders,
236/// and deletes them from the filesystem.
237///
238/// Iterate over the files and directories in the databases folder and delete if:
239/// - The `db_kind` directory exists.
240/// - The entry in `db_kind` is a directory.
241/// - The directory name has a prefix `v`.
242/// - The directory name without the prefix can be parsed as an unsigned number.
243/// - The parsed number is lower than the `major_version`.
244///
245/// The network is used to generate the path, then ignored.
246/// If `config` is an ephemeral database, no databases are deleted.
247///
248/// # Panics
249///
250/// If the path doesn't match the expected `db_kind/major_version/network` format.
251pub fn check_and_delete_old_databases(
252 config: &Config,
253 db_kind: impl AsRef<str>,
254 major_version: u64,
255 network: &Network,
256) -> JoinHandle<()> {
257 let current_span = Span::current();
258 let config = config.clone();
259 let db_kind = db_kind.as_ref().to_string();
260 let network = network.clone();
261
262 spawn_blocking(move || {
263 current_span.in_scope(|| {
264 delete_old_databases(config, db_kind, major_version, &network);
265 info!("finished old database version cleanup task");
266 })
267 })
268}
269
270/// Check if there are old database folders and delete them from the filesystem.
271///
272/// See [`check_and_delete_old_databases`] for details.
273fn delete_old_databases(config: Config, db_kind: String, major_version: u64, network: &Network) {
274 if config.ephemeral || !config.delete_old_database {
275 return;
276 }
277
278 info!(db_kind, "checking for old database versions");
279
280 let mut db_path = config.db_path(&db_kind, major_version, network);
281 // Check and remove the network path.
282 assert_eq!(
283 db_path.file_name(),
284 Some(network.lowercase_name().as_ref()),
285 "unexpected database network path structure"
286 );
287 assert!(db_path.pop());
288
289 // Check and remove the major version path, we'll iterate over them all below.
290 assert_eq!(
291 db_path.file_name(),
292 Some(format!("v{major_version}").as_ref()),
293 "unexpected database version path structure"
294 );
295 assert!(db_path.pop());
296
297 // Check for the correct database kind to iterate within.
298 assert_eq!(
299 db_path.file_name(),
300 Some(db_kind.as_ref()),
301 "unexpected database kind path structure"
302 );
303
304 if let Some(db_kind_dir) = read_dir(&db_path) {
305 for entry in db_kind_dir.flatten() {
306 let deleted_db = check_and_delete_database(&config, major_version, &entry);
307
308 if let Some(deleted_db) = deleted_db {
309 info!(?deleted_db, "deleted outdated {db_kind} database directory");
310 }
311 }
312 }
313}
314
315/// Return a `ReadDir` for `dir`, after checking that `dir` exists and can be read.
316///
317/// Returns `None` if any operation fails.
318fn read_dir(dir: &Path) -> Option<ReadDir> {
319 if dir.exists() {
320 if let Ok(read_dir) = dir.read_dir() {
321 return Some(read_dir);
322 }
323 }
324 None
325}
326
327/// Check if `entry` is an old database directory, and delete it from the filesystem.
328/// See [`check_and_delete_old_databases`] for details.
329///
330/// If the directory was deleted, returns its path.
331fn check_and_delete_database(
332 config: &Config,
333 major_version: u64,
334 entry: &DirEntry,
335) -> Option<PathBuf> {
336 let dir_name = parse_dir_name(entry)?;
337 let dir_major_version = parse_major_version(&dir_name)?;
338
339 if dir_major_version >= major_version {
340 return None;
341 }
342
343 // Don't delete databases that can be reused.
344 if restorable_db_versions()
345 .iter()
346 .map(|v| v - 1)
347 .any(|v| v == dir_major_version)
348 {
349 return None;
350 }
351
352 let outdated_path = entry.path();
353
354 // # Correctness
355 //
356 // Check that the path we're about to delete is inside the cache directory.
357 // If the user has symlinked the outdated state directory to a non-cache directory,
358 // we don't want to delete it, because it might contain other files.
359 //
360 // We don't attempt to guard against malicious symlinks created by attackers
361 // (TOCTOU attacks). Zebra should not be run with elevated privileges.
362 let cache_path = canonicalize(&config.cache_dir).ok()?;
363 let outdated_path = canonicalize(outdated_path).ok()?;
364
365 if !outdated_path.starts_with(&cache_path) {
366 info!(
367 skipped_path = ?outdated_path,
368 ?cache_path,
369 "skipped cleanup of outdated state directory: state is outside cache directory",
370 );
371
372 return None;
373 }
374
375 remove_dir_all(&outdated_path).ok().map(|()| outdated_path)
376}
377
378/// Check if `entry` is a directory with a valid UTF-8 name.
379/// (State directory names are guaranteed to be UTF-8.)
380///
381/// Returns `None` if any operation fails.
382fn parse_dir_name(entry: &DirEntry) -> Option<String> {
383 if let Ok(file_type) = entry.file_type() {
384 if file_type.is_dir() {
385 if let Ok(dir_name) = entry.file_name().into_string() {
386 return Some(dir_name);
387 }
388 }
389 }
390 None
391}
392
393/// Parse the database major version number from `dir_name`.
394///
395/// Returns `None` if parsing fails, or the directory name is not in the expected format.
396fn parse_major_version(dir_name: &str) -> Option<u64> {
397 dir_name
398 .strip_prefix('v')
399 .and_then(|version| version.parse().ok())
400}
401
402// TODO: move these to the format upgrade module
403
404/// Returns the full semantic version of the on-disk state database, based on its config and network.
405pub fn state_database_format_version_on_disk(
406 config: &Config,
407 network: &Network,
408) -> Result<Option<Version>, BoxError> {
409 database_format_version_on_disk(
410 config,
411 STATE_DATABASE_KIND,
412 state_database_format_version_in_code().major,
413 network,
414 )
415}
416
417/// Returns the full semantic version of the on-disk database, based on its config, kind, major version,
418/// and network.
419///
420/// Typically, the version is read from a version text file.
421///
422/// If there is an existing on-disk database, but no version file,
423/// returns `Ok(Some(major_version.0.0))`.
424/// (This happens even if the database directory was just newly created.)
425///
426/// If there is no existing on-disk database, returns `Ok(None)`.
427///
428/// This is the format of the data on disk, the version
429/// implemented by the running Zebra code can be different.
430pub fn database_format_version_on_disk(
431 config: &Config,
432 db_kind: impl AsRef<str>,
433 major_version: u64,
434 network: &Network,
435) -> Result<Option<Version>, BoxError> {
436 let version_path = config.version_file_path(&db_kind, major_version, network);
437 let db_path = config.db_path(db_kind, major_version, network);
438
439 database_format_version_at_path(&version_path, &db_path, major_version)
440}
441
442/// Returns the full semantic version of the on-disk database at `version_path`.
443///
444/// See [`database_format_version_on_disk()`] for details.
445pub(crate) fn database_format_version_at_path(
446 version_path: &Path,
447 db_path: &Path,
448 major_version: u64,
449) -> Result<Option<Version>, BoxError> {
450 let disk_version_file = match fs::read_to_string(version_path) {
451 Ok(version) => Some(version),
452 Err(e) if e.kind() == ErrorKind::NotFound => {
453 // If the version file doesn't exist, don't guess the version yet.
454 None
455 }
456 Err(e) => Err(e)?,
457 };
458
459 // The database has a version file on disk
460 if let Some(version) = disk_version_file {
461 return Ok(Some(
462 version
463 .parse()
464 // Try to parse the previous format of the disk version file if it cannot be parsed as a `Version` directly.
465 .or_else(|err| {
466 format!("{major_version}.{version}")
467 .parse()
468 .map_err(|err2| format!("failed to parse format version: {err}, {err2}"))
469 })?,
470 ));
471 }
472
473 // There's no version file on disk, so we need to guess the version
474 // based on the database content
475 match fs::metadata(db_path) {
476 // But there is a database on disk, so it has the current major version with no upgrades.
477 // If the database directory was just newly created, we also return this version.
478 Ok(_metadata) => Ok(Some(Version::new(major_version, 0, 0))),
479
480 // There's no version file and no database on disk, so it's a new database.
481 // It will be created with the current version,
482 // but temporarily return the default version above until the version file is written.
483 Err(e) if e.kind() == ErrorKind::NotFound => Ok(None),
484
485 Err(e) => Err(e)?,
486 }
487}
488
489// Hide this destructive method from the public API, except in tests.
490#[allow(unused_imports)]
491pub(crate) use hidden::{
492 write_database_format_version_to_disk, write_state_database_format_version_to_disk,
493};
494
495pub(crate) mod hidden {
496 #![allow(dead_code)]
497
498 use zebra_chain::common::atomic_write;
499
500 use super::*;
501
502 /// Writes `changed_version` to the on-disk state database after the format is changed.
503 /// (Or a new database is created.)
504 ///
505 /// See `write_database_format_version_to_disk()` for details.
506 pub fn write_state_database_format_version_to_disk(
507 config: &Config,
508 changed_version: &Version,
509 network: &Network,
510 ) -> Result<(), BoxError> {
511 write_database_format_version_to_disk(
512 config,
513 STATE_DATABASE_KIND,
514 state_database_format_version_in_code().major,
515 changed_version,
516 network,
517 )
518 }
519
520 /// Writes `changed_version` to the on-disk database after the format is changed.
521 /// (Or a new database is created.)
522 ///
523 /// The database path is based on its kind, `major_version_in_code`, and network.
524 ///
525 /// # Correctness
526 ///
527 /// This should only be called:
528 /// - after each format upgrade is complete,
529 /// - when creating a new database, or
530 /// - when an older Zebra version opens a newer database.
531 ///
532 /// # Concurrency
533 ///
534 /// This must only be called while RocksDB has an open database for `config`.
535 /// Otherwise, multiple Zebra processes could write the version at the same time,
536 /// corrupting the file.
537 pub fn write_database_format_version_to_disk(
538 config: &Config,
539 db_kind: impl AsRef<str>,
540 major_version_in_code: u64,
541 changed_version: &Version,
542 network: &Network,
543 ) -> Result<(), BoxError> {
544 // Write the version file atomically so the cache is not corrupted if Zebra shuts down or
545 // crashes.
546 atomic_write(
547 config.version_file_path(db_kind, major_version_in_code, network),
548 changed_version.to_string().as_bytes(),
549 )??;
550
551 Ok(())
552 }
553}