feat: enable testnet4 support in the bitcoin adapter (dfinity#3267)

This involved:
- changing the seeds, 
- upgrading the bitcoin crate (see base branch)
- finally, fixing a bug in the header validation, a fix which is
required in the bitcoin canister too (and even on the bitcoin crate if
we want to use their helpers): For testnet networks, the difficulty
temporarily drops to the minimum if no block has been found in 20
minutes. [Block
30239](https://mempool.space/testnet4/block/00000000eaf8e0ea253d833614892aed70c55e5dc4b4d6709dd6420b8284debb)
is an example of such block, and it (coincidentally) occurs right before
a "difficulty change", once every 2016 blocks. With the previous
implementation, this leads to the difficulty to be dropped massively (to
the minimum) on the next difficulty batch, because the difficulty of an
"epoch" block (in this case block 30240) depends directly on its
previous block (block 30239, which had the "temporarily" drop in
difficulty). The fix involved making the new difficulty depend on the
difficulty of the previous "epoch block", which does not have a drop in
difficulty.

---------

Co-authored-by: IDX GitHub Automation <[email protected]>

Author: mihailjianu1

Cargo.lock

@@ -69,11 +69,9 @@ if [ "${SOCKS_FILE}" != "" -a -e "${SOCKS_FILE}" ]; then
     read_socks_proxy "${SOCKS_FILE}"
 fi
 
-BITCOIN_NETWORK='"testnet"'
-DNS_SEEDS='"testnet-seed.bitcoin.jonasschnelli.ch",
-            "seed.tbtc.petertodd.org",
-            "seed.testnet.bitcoin.sprovoost.nl",
-            "testnet-seed.bluematt.me"'
+BITCOIN_NETWORK='"testnet4"'
+DNS_SEEDS='"seed.testnet4.bitcoin.sprovoost.nl",
+            "seed.testnet4.wiz.biz"'
 
 if [ "$MAINNET" = true ]; then
     BITCOIN_NETWORK='"bitcoin"'

ic-os/components/ic/ic-btc-adapter/generate-btc-adapter-config.sh

@@ -62,9 +62,11 @@ pub fn address_limits(network: Network) -> (usize, usize) {
     match network {
         Network::Bitcoin => (500, 2000),
         Network::Testnet => (100, 1000),
+        //TODO(mihailjianu): revisit these values
+        Network::Testnet4 => (100, 1000),
         Network::Signet => (1, 1),
         Network::Regtest => (1, 1),
-        other => unreachable!("Unsupported network: {:?}", other),
+        _ => (1, 1),
     }
 }
 
@@ -74,6 +76,7 @@ impl Config {
         match self.network {
             Network::Bitcoin => 8333,
             Network::Testnet => 18333,
+            Network::Testnet4 => 48333,
             _ => 8333,
         }
     }

rs/bitcoin/adapter/src/config.rs

@@ -237,7 +237,7 @@ fn get_next_headers(
 fn are_multiple_blocks_allowed(network: Network, anchor_height: BlockHeight) -> bool {
     match network {
         Network::Bitcoin => anchor_height <= MAINNET_MAX_MULTI_BLOCK_ANCHOR_HEIGHT,
-        Network::Testnet | Network::Signet | Network::Regtest => true,
+        Network::Testnet | Network::Signet | Network::Regtest | Network::Testnet4 => true,
         other => unreachable!("Unsupported network: {:?}", other),
     }
 }

rs/bitcoin/adapter/src/get_successors_handler.rs

@@ -14,6 +14,7 @@ DEV_DEPENDENCIES = [
     "@crate_index//:csv",
     "@crate_index//:hex",
     "@crate_index//:proptest",
+    "@crate_index//:rstest",
 ]
 
 MACRO_DEV_DEPENDENCIES = []

rs/bitcoin/validation/BUILD.bazel

@@ -15,3 +15,4 @@ hex = { workspace = true }
 [dev-dependencies]
 csv = "1.1"
 proptest = "0.9.4"
+rstest = { workspace = true }

rs/bitcoin/validation/Cargo.toml

@@ -57,11 +57,18 @@ const TESTNET: &[(BlockHeight, &str)] = &[
     (546, "000000002a936ca763904c3c35fce2f3556c559c0214345d31b1bcebf76acb70")
 ];
 
+/// Bitcoin testnet checkpoints
+#[rustfmt::skip]
+const TESTNET4: &[(BlockHeight, &str)] = &[
+    (64000, "000000000deb369dca3115f66e208733066f44c8cc177edd4b5f86869e6355b5")
+];
+
 /// Returns the maximum difficulty target depending on the network
 pub fn max_target(network: &Network) -> Target {
     match network {
         Network::Bitcoin => Target::MAX_ATTAINABLE_MAINNET,
         Network::Testnet => Target::MAX_ATTAINABLE_TESTNET,
+        Network::Testnet4 => Target::MAX_ATTAINABLE_TESTNET,
         Network::Regtest => Target::MAX_ATTAINABLE_REGTEST,
         Network::Signet => Target::MAX_ATTAINABLE_SIGNET,
         &other => unreachable!("Unsupported network: {:?}", other),
@@ -72,7 +79,7 @@ pub fn max_target(network: &Network) -> Target {
 /// readjusted in the network after a fixed time interval.
 pub fn no_pow_retargeting(network: &Network) -> bool {
     match network {
-        Network::Bitcoin | Network::Testnet | Network::Signet => false,
+        Network::Bitcoin | Network::Testnet | Network::Signet | Network::Testnet4 => false,
         Network::Regtest => true,
         &other => unreachable!("Unsupported network: {:?}", other),
     }
@@ -83,6 +90,7 @@ pub fn pow_limit_bits(network: &Network) -> CompactTarget {
     CompactTarget::from_consensus(match network {
         Network::Bitcoin => 0x1d00ffff,
         Network::Testnet => 0x1d00ffff,
+        Network::Testnet4 => 0x1d00ffff,
         Network::Regtest => 0x207fffff,
         Network::Signet => 0x1e0377ae,
         &other => unreachable!("Unsupported network: {:?}", other),
@@ -94,7 +102,7 @@ pub fn checkpoints(network: &Network) -> HashMap<BlockHeight, BlockHash> {
     let points = match network {
         Network::Bitcoin => BITCOIN,
         Network::Testnet => TESTNET,
-        Network::Testnet4 => &[],
+        Network::Testnet4 => TESTNET4,
         Network::Signet => &[],
         Network::Regtest => &[],
         _ => &[],
@@ -114,7 +122,7 @@ pub fn latest_checkpoint_height(network: &Network, current_height: BlockHeight)
     let points = match network {
         Network::Bitcoin => BITCOIN,
         Network::Testnet => TESTNET,
-        Network::Testnet4 => &[],
+        Network::Testnet4 => TESTNET4,
         Network::Signet => &[],
         Network::Regtest => &[],
         _ => &[],

rs/bitcoin/validation/src/constants.rs

@@ -91,7 +91,11 @@ pub fn validate_header(
     if let Err(err) = header.validate_pow(Target::from_compact(compact_target)) {
         match err {
             ValidationError::BadProofOfWork => println!("bad proof of work"),
-            ValidationError::BadTarget => println!("bad target"),
+            ValidationError::BadTarget => println!(
+                "bad target {:?}, {:?}",
+                Target::from_compact(compact_target),
+                header.target()
+            ),
             _ => {}
         };
         return Err(ValidateHeaderError::InvalidPoWForComputedTarget);
@@ -164,7 +168,7 @@ fn get_next_compact_target(
     timestamp: u32,
 ) -> CompactTarget {
     match network {
-        Network::Testnet | Network::Regtest => {
+        Network::Testnet | Network::Regtest | Network::Testnet4 => {
             if (prev_height + 1) % DIFFICULTY_ADJUSTMENT_INTERVAL != 0 {
                 // This if statements is reached only for Regtest and Testnet networks
                 // Here is the quote from "https://en.bitcoin.it/wiki/Testnet"
@@ -206,8 +210,9 @@ fn find_next_difficulty_in_chain(
 ) -> CompactTarget {
     // This is the maximum difficulty target for the network
     let pow_limit_bits = pow_limit_bits(network);
+
     match network {
-        Network::Testnet | Network::Regtest => {
+        Network::Testnet | Network::Regtest | Network::Testnet4 => {
             let mut current_header = *prev_header;
             let mut current_height = prev_height;
             let mut current_hash = current_header.block_hash();
@@ -282,7 +287,20 @@ fn compute_next_difficulty(
     let actual_interval =
         std::cmp::max((prev_header.time as i64) - (last_adjustment_time as i64), 0) as u64;
 
-    CompactTarget::from_next_work_required(prev_header.bits, actual_interval, *network)
+    //TODO: ideally from_next_work_required works by itself
+    // On Testnet networks, prev_header.bits could be different than last_adjustment_header.bits
+    // if prev_header took more than 20 minutes to be created.
+    // Testnet3 (mistakenly) uses the temporary difficulty drop of prev_header to calculate
+    // the difficulty of th next epoch; this results in the whole epoch having a very low difficulty,
+    // and therefore likely blockstorms.
+    // Testnet4 uses the last_adjustment_header.bits to calculate the next epoch's difficulty, making it
+    // more stable.
+    //TODO(mihailjianu): add a test for testnet4.
+    let previous_difficulty = match network {
+        Network::Testnet4 => last_adjustment_header.bits,
+        _ => prev_header.bits,
+    };
+    CompactTarget::from_next_work_required(previous_difficulty, actual_interval, *network)
 }
 
 #[cfg(test)]
@@ -295,6 +313,8 @@ mod test {
     };
     use csv::Reader;
 
+    use rstest::rstest;
+
     use super::*;
     use crate::constants::test::{
         MAINNET_HEADER_586656, MAINNET_HEADER_705600, MAINNET_HEADER_705601, MAINNET_HEADER_705602,
@@ -559,9 +579,49 @@ mod test {
     }
 
     #[test]
-    fn test_compute_next_difficulty_for_backdated_blocks() {
+    fn test_compute_next_difficulty_for_temporary_difficulty_drops_testnet4() {
+        // Arrange
+        let network = Network::Testnet4;
+        let chain_length = DIFFICULTY_ADJUSTMENT_INTERVAL - 1; // To trigger the difficulty adjustment.
+        let genesis_difficulty = CompactTarget::from_consensus(473956288);
+
+        // Create the genesis header and initialize the header store with 2014 blocks
+        let genesis_header = genesis_header(genesis_difficulty);
+        let mut store = SimpleHeaderStore::new(genesis_header, 0);
+        let mut last_header = genesis_header;
+        for _ in 1..(chain_length - 1) {
+            let new_header = BlockHeader {
+                prev_blockhash: last_header.block_hash(),
+                time: last_header.time + 1,
+                ..last_header
+            };
+            store.add(new_header);
+            last_header = new_header;
+        }
+        // Add the last header in the epoch, which has the lowest difficulty, or highest possible target.
+        // This can happen if the block is created more than 20 minutes after the previous block.
+        let last_header_in_epoch = BlockHeader {
+            prev_blockhash: last_header.block_hash(),
+            time: last_header.time + 1,
+            bits: max_target(&network).to_compact_lossy(),
+            ..last_header
+        };
+        store.add(last_header_in_epoch);
+
+        // Act.
+        let difficulty =
+            compute_next_difficulty(&network, &store, &last_header_in_epoch, chain_length);
+
+        // Assert.
+        // Note: testnet3 would produce 473956288, as it depends on the previous header's difficulty.
+        assert_eq!(difficulty, CompactTarget::from_consensus(470810608));
+    }
+
+    #[rstest]
+    #[case(Network::Testnet)]
+    #[case(Network::Testnet4)]
+    fn test_compute_next_difficulty_for_backdated_blocks(#[case] network: Network) {
         // Arrange: Set up the test network and parameters
-        let network = Network::Testnet;
         let chain_length = DIFFICULTY_ADJUSTMENT_INTERVAL - 1; // To trigger the difficulty adjustment.
         let genesis_difficulty = CompactTarget::from_consensus(486604799);