Write bank hash components to file when node diverges

Steven Czabaniuk · Steven Czabaniuk · commit b9cd4cb27df9 · 2023-07-26T10:53:50.000-05:00
Also allow the data to be written from ledger-tool in order to create a
file to compare against. The generated files are human-readable (JSON)
and diff-friendly.
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/core/src/replay_stage.rs b/core/src/replay_stage.rs
@@ -1500,6 +1500,7 @@ impl ReplayStage {
                     let bank = w_bank_forks
                         .remove(*slot)
                         .expect("BankForks should not have been purged yet");
+                    bank.write_hash_details_file();
                     ((*slot, bank.bank_id()), bank)
                 })
                 .unzip()
diff --git a/ledger-tool/src/main.rs b/ledger-tool/src/main.rs
@@ -1654,6 +1654,13 @@ fn main() {
                     .takes_value(false)
                     .help("After verifying the ledger, print some information about the account stores"),
             )
+            .arg(
+                Arg::with_name("write_bank_file")
+                    .long("write-bank-file")
+                    .takes_value(false)
+                    .help("After verifying the ledger, write a file with that contains the \
+                        information that went into computing the final bank's bank hash"),
+            )
         ).subcommand(
             SubCommand::with_name("graph")
             .about("Create a Graphviz rendering of the ledger")
@@ -2616,6 +2623,7 @@ fn main() {
                     ..ProcessOptions::default()
                 };
                 let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
+                let write_bank_file = arg_matches.is_present("write_bank_file");
                 let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
                 info!("genesis hash: {}", genesis_config.hash());
 
@@ -2641,6 +2649,10 @@ fn main() {
                     let working_bank = bank_forks.read().unwrap().working_bank();
                     working_bank.print_accounts_stats();
                 }
+                if write_bank_file {
+                    let working_bank = bank_forks.read().unwrap().working_bank();
+                    working_bank.write_hash_details_file();
+                }
                 exit_signal.store(true, Ordering::Relaxed);
                 system_monitor_service.join().unwrap();
             }
diff --git a/programs/sbf/Cargo.lock b/programs/sbf/Cargo.lock
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
@@ -44,6 +44,7 @@ rayon = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
 serde_derive = { workspace = true }
+serde_json = { workspace = true }
 siphasher = { workspace = true }
 solana-address-lookup-table-program = { workspace = true }
 solana-bpf-loader-program = { workspace = true }
@@ -62,6 +63,7 @@ solana-rayon-threadlimit = { workspace = true }
 solana-sdk = { workspace = true }
 solana-stake-program = { workspace = true }
 solana-system-program = { workspace = true }
+solana-version = { workspace = true }
 solana-vote-program = { workspace = true }
 solana-zk-token-proof-program = { workspace = true }
 solana-zk-token-sdk = { workspace = true }
diff --git a/runtime/src/accounts_db.rs b/runtime/src/accounts_db.rs
@@ -1015,6 +1015,16 @@ struct CleanKeyTimings {
     dirty_ancient_stores: usize,
 }
 
+/// Fundamental details of an account
+#[derive(Clone, Copy, Debug, Default, Deserialize, PartialEq, Serialize)]
+pub struct AccountDetails {
+    pub pubkey: Pubkey,
+    pub lamports: u64,
+    pub rent_epoch: Epoch,
+    pub executable: bool,
+    pub hash: Hash,
+}
+
 /// Persistent storage structure holding the accounts
 #[derive(Debug)]
 pub struct AccountStorageEntry {
@@ -2372,8 +2382,8 @@ impl AccountsDb {
         let (base_working_path, accounts_hash_cache_path, temp_accounts_hash_cache_path) =
             match base_working_path {
                 Some(base_working_path) => {
-                    let accounts_hash_cache_path = base_working_path
-                        .join(Self::ACCOUNTS_HASH_CACHE_DIR);
+                    let accounts_hash_cache_path =
+                        base_working_path.join(Self::ACCOUNTS_HASH_CACHE_DIR);
                     (base_working_path, accounts_hash_cache_path, None)
                 }
                 None => {
@@ -2383,8 +2393,8 @@ impl AccountsDb {
                         .unwrap()
                         .path()
                         .to_path_buf();
-                    let accounts_hash_cache_path = base_working_path
-                        .join(Self::ACCOUNTS_HASH_CACHE_DIR);
+                    let accounts_hash_cache_path =
+                        base_working_path.join(Self::ACCOUNTS_HASH_CACHE_DIR);
                     (
                         base_working_path,
                         accounts_hash_cache_path,
@@ -7822,6 +7832,37 @@ impl AccountsDb {
         (hashes, scan.as_us(), accumulate)
     }
 
+    /// Return details on all of the accounts for the given `slot`, sorted by pubkey.
+    pub(crate) fn get_account_details_for_slot(&self, slot: Slot) -> Vec<AccountDetails> {
+        let account_info_func = |loaded_account: &LoadedAccount| -> AccountDetails {
+            AccountDetails {
+                pubkey: *loaded_account.pubkey(),
+                lamports: loaded_account.lamports(),
+                rent_epoch: loaded_account.rent_epoch(),
+                executable: loaded_account.executable(),
+                hash: loaded_account.loaded_hash(),
+            }
+        };
+
+        type ScanResult = ScanStorageResult<AccountDetails, Mutex<Vec<AccountDetails>>>;
+        let scan_result: ScanResult = self.scan_account_storage(
+            slot,
+            |loaded_account: LoadedAccount| {
+                // Cache only has one version per key, don't need to worry about versioning
+                Some(account_info_func(&loaded_account))
+            },
+            |accum: &Mutex<Vec<AccountDetails>>, loaded_account: LoadedAccount| {
+                let account_info = account_info_func(&loaded_account);
+                accum.lock().unwrap().push(account_info);
+            },
+        );
+
+        match scan_result {
+            ScanStorageResult::Cached(cached_result) => cached_result,
+            ScanStorageResult::Stored(stored_result) => stored_result.into_inner().unwrap(),
+        }
+    }
+
     /// Calculate accounts delta hash for `slot`
     ///
     /// As part of calculating the accounts delta hash, get a list of accounts modified this slot
diff --git a/runtime/src/bank.rs b/runtime/src/bank.rs
@@ -96,6 +96,7 @@ use {
         iter::{IntoParallelIterator, IntoParallelRefIterator, ParallelIterator},
         ThreadPool, ThreadPoolBuilder,
     },
+    serde_json::json,
     solana_bpf_loader_program::syscalls::create_program_runtime_environment,
     solana_cost_model::cost_tracker::CostTracker,
     solana_measure::{measure, measure::Measure, measure_us},
@@ -8164,6 +8165,66 @@ impl Bank {
         }
         false
     }
+
+    /// Output the components that comprise bank hash
+    // TODO: maybe give ledger-tool a unique path so it can bypass path.exists() check?
+    pub fn write_hash_details_file(&self) {
+        let slot = self.slot();
+        // Only allow generating (and thus comparing) results on frozen (complete) banks
+        if !self.is_frozen() {
+            warn!("Cannot write bank hash details for non-frozen bank {slot}");
+            return;
+        }
+
+        let hash = self.hash();
+        let file_name = format!("{slot}.{hash}");
+        let parent_dir = self
+            .rc
+            .accounts
+            .accounts_db
+            .get_base_working_path()
+            .join("bank_hash_details");
+        let path = parent_dir.join(file_name);
+        // A file with the same name implies the same hash for this slot. Skip
+        // rewriting a duplicate file in this scenario
+        if path.exists() {
+            return;
+        }
+        info!("writing details of bank {} to {}", slot, path.display());
+
+        // This bank is frozen; as a result, we know that the state has been
+        // hashed which means the delta hash is Some(). So, .unwrap() is safe
+        let accounts_delta_hash = self
+            .rc
+            .accounts
+            .accounts_db
+            .get_accounts_delta_hash(slot)
+            .unwrap()
+            .0;
+        let mut account_details = self
+            .rc
+            .accounts
+            .accounts_db
+            .get_account_details_for_slot(slot);
+        account_details.sort_by_key(|details| details.pubkey);
+
+        let output = json!({
+            "version": solana_version::version!(),
+            "slot": slot,
+            "hash": hash,
+            "parent_hash": self.parent_hash(),
+            "accounts_delta_hash": accounts_delta_hash,
+            "signature_count": self.signature_count(),
+            "last_blockhash": self.last_blockhash(),
+            "accounts": account_details
+        });
+
+        // std::fs::write may fail (depending on platform) if the full directory
+        // path does not exist. So, call std::fs_create_dir_all first.
+        // https://doc.rust-lang.org/std/fs/fn.write.html
+        _ = std::fs::create_dir_all(parent_dir);
+        _ = std::fs::write(path, serde_json::to_vec_pretty(&output).unwrap());
+    }
 }
 
 /// Compute how much an account has changed size.  This function is useful when the data size delta
