v1.16: Add ability to output Bank hash details (backport of #32632) (#34257)

* Add ability to output components that go into Bank hash (#32632)

When a consensus divergance occurs, the current workflow involves a
handful of manual steps to hone in on the offending slot and
transaction. This process isn't overly difficult to execute; however, it
is tedious and currently involves creating and parsing logs.

This change introduces functionality to output a debug file that
contains the components go into the bank hash. The file can be generated
in two ways:
- Via solana-validator when the node realizes it has diverged
- Via solana-ledger-tool verify by passing a flag

When a divergance occurs now, the steps to debug would be:
- Grab the file from the node that diverged
- Generate a file for the same slot with ledger-tool with a known good
  version
- Diff the files, they are pretty-printed json

(cherry picked from commit 6bbf514)

# Conflicts:
#	Cargo.lock
#	ledger-tool/src/args.rs
#	ledger-tool/src/main.rs
#	programs/sbf/Cargo.lock
#	runtime/Cargo.toml
#	runtime/src/accounts_db.rs
#	validator/src/main.rs

* Merge conflict

* Reorder base_wotking with accounts_hash to match other branches

---------

Co-authored-by: steviez <[email protected]>

Author: mergify[bot]

Cargo.lock

@@ -59,7 +59,7 @@ use {
     solana_rpc_client_api::response::SlotUpdate,
     solana_runtime::{
         accounts_background_service::AbsRequestSender,
-        bank::{Bank, NewBankOptions},
+        bank::{bank_hash_details, Bank, NewBankOptions},
         bank_forks::{BankForks, MAX_ROOT_DISTANCE_FOR_VOTE_ONLY},
         commitment::BlockCommitmentCache,
         prioritization_fee_cache::PrioritizationFeeCache,
@@ -1502,6 +1502,7 @@ impl ReplayStage {
                     let bank = w_bank_forks
                         .remove(*slot)
                         .expect("BankForks should not have been purged yet");
+                    let _ = bank_hash_details::write_bank_hash_details_file(&bank);
                     ((*slot, bank.bank_id()), bank)
                 })
                 .unzip()

core/src/replay_stage.rs

@@ -57,6 +57,7 @@ pub fn get_accounts_db_config(
 
     AccountsDbConfig {
         index: Some(accounts_index_config),
+        base_working_path: Some(ledger_path.to_path_buf()),
         accounts_hash_cache_path: Some(
             ledger_path.join(AccountsDb::DEFAULT_ACCOUNTS_HASH_CACHE_DIR),
         ),

ledger-tool/src/args.rs

@@ -47,7 +47,7 @@ use {
         accounts::Accounts,
         accounts_db::CalcAccountsHashDataSource,
         accounts_index::ScanConfig,
-        bank::{Bank, RewardCalculationEvent, TotalAccountsStats},
+        bank::{bank_hash_details, Bank, RewardCalculationEvent, TotalAccountsStats},
         bank_forks::BankForks,
         cost_model::CostModel,
         cost_tracker::CostTracker,
@@ -1629,6 +1629,14 @@ fn main() {
                     .takes_value(false)
                     .help("After verifying the ledger, print some information about the account stores"),
             )
+            .arg(
+                Arg::with_name("write_bank_file")
+                    .long("write-bank-file")
+                    .takes_value(false)
+                    .help("After verifying the ledger, write a file that contains the information \
+                        that went into computing the completed bank's bank hash. The file will be \
+                        written within <LEDGER_DIR>/bank_hash_details/"),
+            )
         ).subcommand(
             SubCommand::with_name("graph")
             .about("Create a Graphviz rendering of the ledger")
@@ -2592,6 +2600,7 @@ fn main() {
                     ..ProcessOptions::default()
                 };
                 let print_accounts_stats = arg_matches.is_present("print_accounts_stats");
+                let write_bank_file = arg_matches.is_present("write_bank_file");
                 let genesis_config = open_genesis_config_by(&ledger_path, arg_matches);
                 info!("genesis hash: {}", genesis_config.hash());
 
@@ -2617,6 +2626,10 @@ fn main() {
                     let working_bank = bank_forks.read().unwrap().working_bank();
                     working_bank.print_accounts_stats();
                 }
+                if write_bank_file {
+                    let working_bank = bank_forks.read().unwrap().working_bank();
+                    let _ = bank_hash_details::write_bank_hash_details_file(&working_bank);
+                }
                 exit_signal.store(true, Ordering::Relaxed);
                 system_monitor_service.join().unwrap();
             }

ledger-tool/src/main.rs

@@ -11,6 +11,7 @@ edition = { workspace = true }
 
 [dependencies]
 arrayref = { workspace = true }
+base64 = { workspace = true }
 bincode = { workspace = true }
 blake3 = { workspace = true }
 bv = { workspace = true, features = ["serde"] }
@@ -43,6 +44,7 @@ rayon = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true, features = ["rc"] }
 serde_derive = { workspace = true }
+serde_json = { workspace = true }
 solana-address-lookup-table-program = { workspace = true }
 solana-bpf-loader-program = { workspace = true }
 solana-bucket-map = { workspace = true }
@@ -59,6 +61,7 @@ solana-rayon-threadlimit = { workspace = true }
 solana-sdk = { workspace = true }
 solana-stake-program = { workspace = true }
 solana-system-program = { workspace = true }
+solana-version = { workspace = true }
 solana-vote-program = { workspace = true }
 solana-zk-token-proof-program = { workspace = true }
 solana-zk-token-sdk = { workspace = true }

programs/sbf/Cargo.lock

@@ -469,6 +469,7 @@ pub(crate) struct ShrinkCollect<'a, T: ShrinkCollectRefs<'a>> {
 
 pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
     index: Some(ACCOUNTS_INDEX_CONFIG_FOR_TESTING),
+    base_working_path: None,
     accounts_hash_cache_path: None,
     filler_accounts_config: FillerAccountsConfig::const_default(),
     write_cache_limit_bytes: None,
@@ -480,6 +481,7 @@ pub const ACCOUNTS_DB_CONFIG_FOR_TESTING: AccountsDbConfig = AccountsDbConfig {
 };
 pub const ACCOUNTS_DB_CONFIG_FOR_BENCHMARKS: AccountsDbConfig = AccountsDbConfig {
     index: Some(ACCOUNTS_INDEX_CONFIG_FOR_BENCHMARKS),
+    base_working_path: None,
     accounts_hash_cache_path: None,
     filler_accounts_config: FillerAccountsConfig::const_default(),
     write_cache_limit_bytes: None,
@@ -539,6 +541,8 @@ const ANCIENT_APPEND_VEC_DEFAULT_OFFSET: Option<i64> = Some(-10_000);
 #[derive(Debug, Default, Clone)]
 pub struct AccountsDbConfig {
     pub index: Option<AccountsIndexConfig>,
+    /// Base directory for various necessary files
+    pub base_working_path: Option<PathBuf>,
     pub accounts_hash_cache_path: Option<PathBuf>,
     pub filler_accounts_config: FillerAccountsConfig,
     pub write_cache_limit_bytes: Option<u64>,
@@ -1395,12 +1399,14 @@ pub struct AccountsDb {
     /// Set of storage paths to pick from
     pub(crate) paths: Vec<PathBuf>,
 
-    accounts_hash_cache_path: PathBuf,
-
+    /// Base directory for various necessary files
+    base_working_path: PathBuf,
     // used by tests
     // holds this until we are dropped
     #[allow(dead_code)]
-    temp_accounts_hash_cache_path: Option<TempDir>,
+    base_working_temp_dir: Option<TempDir>,
+    /// Directory for account hash calculations, within base_working_path
+    accounts_hash_cache_path: PathBuf,
 
     pub shrink_paths: RwLock<Option<Vec<PathBuf>>>,
 
@@ -2347,29 +2353,47 @@ impl<'a> AppendVecScan for ScanState<'a> {
     }
 }
 
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub struct PubkeyHashAccount {
+    pub pubkey: Pubkey,
+    pub hash: Hash,
+    pub account: AccountSharedData,
+}
+
 impl AccountsDb {
     pub const DEFAULT_ACCOUNTS_HASH_CACHE_DIR: &str = "accounts_hash_cache";
 
     pub fn default_for_tests() -> Self {
-        Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None)
+        Self::default_with_accounts_index(AccountInfoAccountsIndex::default_for_tests(), None, None)
     }
 
     fn default_with_accounts_index(
         accounts_index: AccountInfoAccountsIndex,
+        base_working_path: Option<PathBuf>,
         accounts_hash_cache_path: Option<PathBuf>,
     ) -> Self {
         let num_threads = get_thread_count();
         const MAX_READ_ONLY_CACHE_DATA_SIZE: usize = 400_000_000; // 400M bytes
 
-        let (accounts_hash_cache_path, temp_accounts_hash_cache_path) =
-            if let Some(accounts_hash_cache_path) = accounts_hash_cache_path {
-                (accounts_hash_cache_path, None)
+        let (base_working_path, base_working_temp_dir) =
+            if let Some(base_working_path) = base_working_path {
+                (base_working_path, None)
             } else {
-                let temp_dir = TempDir::new().expect("new tempdir");
-                let cache_path = temp_dir.path().to_path_buf();
-                (cache_path, Some(temp_dir))
+                let temp_base_working_dir = TempDir::new().unwrap();
+                let base_working_path = temp_base_working_dir.path().to_path_buf();
+                (base_working_path, Some(temp_base_working_dir))
             };
 
+        let accounts_hash_cache_path = accounts_hash_cache_path.unwrap_or_else(|| {
+            let accounts_hash_cache_path =
+                base_working_path.join(Self::DEFAULT_ACCOUNTS_HASH_CACHE_DIR);
+            if !accounts_hash_cache_path.exists() {
+                std::fs::create_dir(&accounts_hash_cache_path)
+                    .expect("create accounts hash cache dir");
+            }
+            accounts_hash_cache_path
+        });
+
         let mut bank_hash_stats = HashMap::new();
         bank_hash_stats.insert(0, BankHashStats::default());
 
@@ -2398,8 +2422,9 @@ impl AccountsDb {
             write_cache_limit_bytes: None,
             write_version: AtomicU64::new(0),
             paths: vec![],
+            base_working_path,
+            base_working_temp_dir,
             accounts_hash_cache_path,
-            temp_accounts_hash_cache_path,
             shrink_paths: RwLock::new(None),
             temp_paths: None,
             file_size: DEFAULT_FILE_SIZE,
@@ -2477,6 +2502,9 @@ impl AccountsDb {
             accounts_db_config.as_mut().and_then(|x| x.index.take()),
             exit,
         );
+        let base_working_path = accounts_db_config
+            .as_ref()
+            .and_then(|config| config.base_working_path.clone());
         let accounts_hash_cache_path = accounts_db_config
             .as_ref()
             .and_then(|config| config.accounts_hash_cache_path.clone());
@@ -2534,7 +2562,11 @@ impl AccountsDb {
                 .and_then(|x| x.write_cache_limit_bytes),
             partitioned_epoch_rewards_config,
             exhaustively_verify_refcounts,
-            ..Self::default_with_accounts_index(accounts_index, accounts_hash_cache_path)
+            ..Self::default_with_accounts_index(
+                accounts_index,
+                base_working_path,
+                accounts_hash_cache_path,
+            )
         };
         if paths_is_empty {
             // Create a temporary set of accounts directories, used primarily
@@ -2581,6 +2613,11 @@ impl AccountsDb {
         self.file_size
     }
 
+    /// Get the base working directory
+    pub fn get_base_working_path(&self) -> PathBuf {
+        self.base_working_path.clone()
+    }
+
     pub fn new_single_for_tests() -> Self {
         AccountsDb::new_for_tests(Vec::new(), &ClusterType::Development)
     }
@@ -7775,6 +7812,42 @@ impl AccountsDb {
         (hashes, scan.as_us(), accumulate)
     }
 
+    /// Return all of the accounts for a given slot
+    pub fn get_pubkey_hash_account_for_slot(&self, slot: Slot) -> Vec<PubkeyHashAccount> {
+        type ScanResult =
+            ScanStorageResult<PubkeyHashAccount, DashMap<Pubkey, (Hash, AccountSharedData)>>;
+        let scan_result: ScanResult = self.scan_account_storage(
+            slot,
+            |loaded_account: LoadedAccount| {
+                // Cache only has one version per key, don't need to worry about versioning
+                Some(PubkeyHashAccount {
+                    pubkey: *loaded_account.pubkey(),
+                    hash: loaded_account.loaded_hash(),
+                    account: loaded_account.take_account(),
+                })
+            },
+            |accum: &DashMap<Pubkey, (Hash, AccountSharedData)>, loaded_account: LoadedAccount| {
+                // Storage may have duplicates so only keep the latest version for each key
+                accum.insert(
+                    *loaded_account.pubkey(),
+                    (loaded_account.loaded_hash(), loaded_account.take_account()),
+                );
+            },
+        );
+
+        match scan_result {
+            ScanStorageResult::Cached(cached_result) => cached_result,
+            ScanStorageResult::Stored(stored_result) => stored_result
+                .into_iter()
+                .map(|(pubkey, (hash, account))| PubkeyHashAccount {
+                    pubkey,
+                    hash,
+                    account,
+                })
+                .collect(),
+        }
+    }
+
     /// Calculate accounts delta hash for `slot`
     ///
     /// As part of calculating the accounts delta hash, get a list of accounts modified this slot

runtime/Cargo.toml

@@ -200,6 +200,7 @@ struct VerifyAccountsHashConfig {
 }
 
 mod address_lookup_table;
+pub mod bank_hash_details;
 mod builtin_programs;
 mod metrics;
 mod sysvar_cache;