feat: ICP-ledger: FI-1440: Implement V4 for ICP ledger - migrate balances to stable structures (dfinity#3314)

This version of the ledger migrates the balances stored by the ledger
from heap to stable memory stable structures. This allows the ledger to
store more data (heap memory is limited to 4GB) and reduces the number
of instructions used during the upgrades - there is no need to serialize
and deserialize the balances stored in heap memory. Since we are no
longer limited by the heap size, the logic for trimming balances and
related init/upgrade arguments (`maximum_number_of_accounts`,
`accounts_overflow_trim_quantity`) were removed. The migration to stable
structures is performed during the upgrade and, if 250B instruction
limit is reached, using timer invocations after the upgrade. If timer
invocations are used by the migration, the ledger is unavailable for
transfers and approvals until migration is finished. The migration
status can be checked by calling the `is_ledger_ready` endpoint. After
migration is finished, it is no longer possible to downgrade the ledger
to the previous version .

---------

Co-authored-by: Mathias Björkqvist <[email protected]>
Co-authored-by: mraszyk <[email protected]>

Author: 3 people

Cargo.lock

@@ -216,7 +216,6 @@ members = [
     "rs/nns/handlers/root/interface",
     "rs/nns/identity",
     "rs/nns/init",
-    "rs/nns/inspector",
     "rs/nns/integration_tests",
     "rs/nns/nns-ui",
     "rs/nns/test_utils",

Cargo.toml

@@ -17,6 +17,8 @@ canisters(
         "registry": "registry-canister.wasm.gz",
         "governance": "governance-canister.wasm.gz",
         "ledger": "ledger-canister_notify-method.wasm.gz",
+        "ledger_v1": "ledger-canister_notify-method.wasm.gz",
+        "ledger_v2": "ledger-canister_notify-method.wasm.gz",
         "archive": "ledger-archive-node-canister.wasm.gz",
         "index": "ic-icp-index-canister.wasm.gz",
         "root": "root-canister.wasm.gz",
@@ -51,6 +53,8 @@ canisters(
         "registry": "mainnet_nns_registry_canister",
         "governance": "mainnet_nns_governance_canister",
         "ledger": "mainnet_icp_ledger_canister",
+        "ledger_v1": "mainnet_icp_ledger_canister-v1",
+        "ledger_v2": "mainnet_icp_ledger_canister-v2",
         "archive": "mainnet_icp_ledger-archive-node-canister",
         "index": "mainnet_icp_index_canister",
         "root": "mainnet_nns_root-canister",

WORKSPACE.bazel

@@ -79,6 +79,14 @@
     "rev": "7c6309cb5bec7ab28ed657ac7672af08a59fc1ba",
     "sha256": "a9ed1cb9dda555e0fc1038825eb7b3a6b366f17aa4b88575184c7537e864e551"
   },
+  "ledger_v1": {
+    "rev": "6dcfafb491092704d374317d9a72a7ad2475d7c9",
+    "sha256": "4fe38a91a3130e9d8b39e3413ae3b3f46c40d3fbd507df1b6092f962d970a7ea"
+  },
+  "ledger_v2": {
+    "rev": "dac2f36f96d7549d82fa8e3c714979255ce57afd",
+    "sha256": "50c05fd687883fe788c0bb91996de358d8f856ba56088c6ff47767ea853001d7"
+  },
   "lifeline": {
     "rev": "b5192581ccd35b67fe5a1f795ead9cbcd25956d6",
     "sha256": "8c8eb285de53ca5609abd7dc41ba3ec8eeb67708b81469311fd670e6738d7d0a"

mainnet-canister-revisions.json

@@ -65,8 +65,8 @@ CANISTERS_MAX_SIZE_COMPRESSED_E5_BYTES = {
     "ic-icrc1-ledger-u256.wasm.gz": "7",
     # Size when constraint addded: 841_234 bytes
     "ledger-canister.wasm.gz": "9",
-    # Size when constraint addded: 847_186 bytes
-    "ledger-canister_notify-method.wasm.gz": "9",
+    # Size when constraint addded: 906_940 bytes
+    "ledger-canister_notify-method.wasm.gz": "10",
 
     # -- XC team --
     # Size when constraint addded: 447_593 bytes

publish/canisters/BUILD.bazel

@@ -4,22 +4,18 @@ use ic_canister_log::{log, Sink};
 use ic_ledger_core::approvals::{
     AllowanceTable, AllowancesData, ApproveError, InsufficientAllowance,
 };
-use ic_ledger_core::tokens::Zero;
 use serde::{Deserialize, Serialize};
 use std::collections::{BTreeMap, VecDeque};
 use std::ops::Range;
 use std::time::Duration;
 
 use crate::archive::{ArchivingGuardError, FailedToArchiveBlocks, LedgerArchivingGuard};
-use ic_ledger_core::balances::{BalanceError, Balances, BalancesStore, InspectableBalancesStore};
+use ic_ledger_core::balances::{BalanceError, Balances, BalancesStore};
 use ic_ledger_core::block::{BlockIndex, BlockType, EncodedBlock, FeeCollector};
 use ic_ledger_core::timestamp::TimeStamp;
 use ic_ledger_core::tokens::TokensType;
 use ic_ledger_hash_of::HashOf;
 
-/// The memo to use for balances burned and approvals reset to 0 during trimming
-const TRIMMED_MEMO: u64 = u64::MAX;
-
 #[derive(Debug, Deserialize, Serialize)]
 pub struct TransactionInfo<TransactionType> {
     pub block_timestamp: TimeStamp,
@@ -158,13 +154,6 @@ pub trait LedgerData: LedgerContext {
     /// The maximum number of transactions that we attempt to purge in one go.
     fn max_transactions_to_purge(&self) -> usize;
 
-    /// The maximum size of the balances map.
-    fn max_number_of_accounts(&self) -> usize;
-
-    /// How many accounts with lowest balances to purge when the number of accounts exceeds
-    /// [LedgerData::max_number_of_accounts].
-    fn accounts_overflow_trim_quantity(&self) -> usize;
-
     // Token configuration
 
     /// Token name (e.g., Bitcoin).
@@ -208,30 +197,12 @@ pub enum TransferError<Tokens> {
 const APPROVE_PRUNE_LIMIT: usize = 100;
 
 /// Adds a new block with the specified transaction to the ledger.
-/// Trim balances if necessary.
 pub fn apply_transaction<L>(
     ledger: &mut L,
     transaction: L::Transaction,
     now: TimeStamp,
     effective_fee: L::Tokens,
 ) -> Result<(BlockIndex, HashOf<EncodedBlock>), TransferError<L::Tokens>>
-where
-    L: LedgerData,
-    L::BalancesStore: InspectableBalancesStore,
-{
-    let result = apply_transaction_no_trimming(ledger, transaction, now, effective_fee);
-    trim_balances(ledger, now);
-    result
-}
-
-/// Adds a new block with the specified transaction to the ledger.
-/// Do not perform any balance trimming.
-pub fn apply_transaction_no_trimming<L>(
-    ledger: &mut L,
-    transaction: L::Transaction,
-    now: TimeStamp,
-    effective_fee: L::Tokens,
-) -> Result<(BlockIndex, HashOf<EncodedBlock>), TransferError<L::Tokens>>
 where
     L: LedgerData,
 {
@@ -322,44 +293,6 @@ where
     Ok((height, ledger.blockchain().last_hash.unwrap()))
 }
 
-/// Trim balances. Can be used e.g. if the ledger is low on heap memory.
-fn trim_balances<L>(ledger: &mut L, now: TimeStamp)
-where
-    L: LedgerData,
-    L::BalancesStore: InspectableBalancesStore,
-{
-    let effective_max_number_of_accounts =
-        ledger.max_number_of_accounts() + ledger.accounts_overflow_trim_quantity() - 1;
-
-    let to_trim = if ledger.balances().store.len() > effective_max_number_of_accounts {
-        select_accounts_to_trim(ledger)
-    } else {
-        vec![]
-    };
-
-    for (balance, account) in to_trim {
-        let burn_tx = L::Transaction::burn(account, None, balance, Some(now), Some(TRIMMED_MEMO));
-
-        burn_tx
-            .apply(ledger, now, L::Tokens::zero())
-            .expect("failed to burn funds that must have existed");
-
-        let parent_hash = ledger.blockchain().last_hash;
-        let fee_collector = ledger.fee_collector().cloned();
-
-        ledger
-            .blockchain_mut()
-            .add_block(L::Block::from_transaction(
-                parent_hash,
-                burn_tx,
-                now,
-                L::Tokens::zero(),
-                fee_collector,
-            ))
-            .unwrap();
-    }
-}
-
 /// Finds the archive canister that contains the block with the specified height.
 pub fn find_block_in_archive<L: LedgerData>(ledger: &L, block_height: u64) -> Option<CanisterId> {
     let index = ledger
@@ -453,39 +386,6 @@ pub fn purge_old_transactions<L: LedgerData>(ledger: &mut L, now: TimeStamp) ->
     num_tx_purged
 }
 
-// Find the specified number of accounts with lowest balances so that their
-// balances can be reclaimed.
-fn select_accounts_to_trim<L>(ledger: &L) -> Vec<(L::Tokens, L::AccountId)>
-where
-    L: LedgerData,
-    L::BalancesStore: InspectableBalancesStore<Tokens = L::Tokens>,
-    L::Tokens: TokensType,
-{
-    let mut to_trim: std::collections::BinaryHeap<(L::Tokens, L::AccountId)> =
-        std::collections::BinaryHeap::new();
-
-    let num_accounts = ledger.accounts_overflow_trim_quantity();
-    let mut iter = ledger.balances().store.iter();
-
-    // Accumulate up to `trim_quantity` accounts
-    for (account, balance) in iter.by_ref().take(num_accounts) {
-        to_trim.push((balance.clone(), account.clone()));
-    }
-
-    for (account, balance) in iter {
-        // If any account's balance is lower than the maximum in our set,
-        // include that account, and remove the current maximum
-        if let Some((greatest_balance, _)) = to_trim.peek() {
-            if balance < greatest_balance {
-                to_trim.push((balance.clone(), account.clone()));
-                to_trim.pop();
-            }
-        }
-    }
-
-    to_trim.into_vec()
-}
-
 /// Asynchronously archives a suffix of the locally available blockchain.
 ///
 /// NOTE: only one archiving task can run at each point in time.

rs/ledger_suite/common/ledger_canister_core/src/ledger.rs

@@ -19,13 +19,6 @@ pub trait BalancesStore {
         F: FnMut(Option<&Self::Tokens>) -> Result<Self::Tokens, E>;
 }
 
-#[allow(clippy::len_without_is_empty)]
-pub trait InspectableBalancesStore: BalancesStore {
-    fn iter(&self) -> Box<dyn Iterator<Item = (&Self::AccountId, &Self::Tokens)> + '_>;
-
-    fn len(&self) -> usize;
-}
-
 impl<AccountId, Tokens> BalancesStore for BTreeMap<AccountId, Tokens>
 where
     AccountId: Eq + Clone + std::cmp::Ord,
@@ -63,20 +56,6 @@ where
     }
 }
 
-impl<AccountId, Tokens> InspectableBalancesStore for BTreeMap<AccountId, Tokens>
-where
-    AccountId: Eq + Clone + std::cmp::Ord,
-    Tokens: TokensType,
-{
-    fn len(&self) -> usize {
-        self.len()
-    }
-
-    fn iter(&self) -> Box<dyn Iterator<Item = (&Self::AccountId, &Self::Tokens)> + '_> {
-        Box::new(self.iter())
-    }
-}
-
 /// An error returned by `Balances` if the debit operation fails.
 #[derive(Debug)]
 pub enum BalanceError<Tokens> {

rs/ledger_suite/common/ledger_core/src/balances.rs

@@ -1,10 +1,15 @@
 use candid::{CandidType, Nat};
+use ic_stable_structures::{storable::Bound, Storable};
 use minicbor::{Decode, Encode};
 use num_traits::{Bounded, ToPrimitive};
 use serde::{de::DeserializeOwned, Deserialize, Serialize};
+use std::borrow::Cow;
 use std::fmt;
 use std::fmt::Debug;
 
+#[cfg(test)]
+mod tests;
+
 /// Performs addition that returns `None` instead of wrapping around on
 /// overflow.
 ///
@@ -302,3 +307,20 @@ impl From<Tokens> for Nat {
         Nat::from(value.e8s)
     }
 }
+
+impl Storable for Tokens {
+    fn to_bytes(&self) -> Cow<[u8]> {
+        Cow::Owned(self.e8s.to_le_bytes().to_vec())
+    }
+
+    fn from_bytes(bytes: Cow<[u8]>) -> Self {
+        Self {
+            e8s: u64::from_le_bytes(bytes.into_owned().as_slice().try_into().unwrap()),
+        }
+    }
+
+    const BOUND: Bound = Bound::Bounded {
+        max_size: 8,
+        is_fixed_size: true,
+    };
+}

rs/ledger_suite/common/ledger_core/src/tokens.rs

@@ -0,0 +1,12 @@
+use crate::tokens::Tokens;
+use ic_stable_structures::Storable;
+use proptest::prelude::{any, prop_assert_eq, proptest};
+
+#[test]
+fn tokens_serialization() {
+    proptest!(|(e8s in any::<u64>())| {
+        let tokens = Tokens { e8s };
+        let new_tokens = Tokens::from_bytes(tokens.to_bytes());
+        prop_assert_eq!(new_tokens, tokens);
+    })
+}

rs/ledger_suite/common/ledger_core/src/tokens/tests.rs

@@ -302,8 +302,6 @@ type InitArgs = record {
     token_symbol: opt text;
     token_name: opt text;
     feature_flags : opt FeatureFlags;
-    maximum_number_of_accounts : opt nat64;
-    accounts_overflow_trim_quantity: opt nat64;
 };
 
 type Icrc1BlockIndex = nat;