Merge remote-tracking branch 'origin/unstable' into tree-states-hot-rebase

Author: michaelsproul

.github/workflows/local-testnet.yml

@@ -67,6 +67,7 @@ jobs:
         working-directory: scripts/local_testnet
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-local-testnet
@@ -125,6 +126,7 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-success
@@ -160,13 +162,56 @@ jobs:
         working-directory: scripts/tests
 
       - name: Upload logs artifact
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: logs-doppelganger-protection-failure
           path: |
             scripts/local_testnet/logs
           retention-days: 3
 
+  # Tests checkpoint syncing to a live network (current fork) and a running devnet (usually next scheduled fork)
+  checkpoint-sync-test:
+    name: checkpoint-sync-test-${{ matrix.network }}
+    runs-on: ubuntu-latest
+    needs: dockerfile-ubuntu
+    if: contains(github.event.pull_request.labels.*.name, 'syncing')
+    continue-on-error: true
+    strategy:
+      matrix:
+        network: [sepolia, devnet]
+    steps:
+      - uses: actions/checkout@v4
+
+      - name: Install Kurtosis
+        run: |
+          echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
+          sudo apt update
+          sudo apt install -y kurtosis-cli
+          kurtosis analytics disable
+
+      - name: Download Docker image artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: lighthouse-docker
+          path: .
+
+      - name: Load Docker image
+        run: docker load -i lighthouse-docker.tar
+
+      - name: Run the checkpoint sync test script
+        run: |
+          ./checkpoint-sync.sh "sync-${{ matrix.network }}" "checkpoint-sync-config-${{ matrix.network }}.yaml"
+        working-directory: scripts/tests
+
+      - name: Upload logs artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: logs-checkpoint-sync-${{ matrix.network }}
+          path: |
+            scripts/local_testnet/logs
+          retention-days: 3
 
   # This job succeeds ONLY IF all others succeed. It is used by the merge queue to determine whether
   # a PR is safe to merge. New jobs should be added here.
@@ -182,4 +227,6 @@ jobs:
     steps:
       - uses: actions/checkout@v4
       - name: Check that success job is dependent on all others
-        run: ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success
+        run: |
+          exclude_jobs='checkpoint-sync-test'
+          ./scripts/ci/check-success-job.sh ./.github/workflows/local-testnet.yml local-testnet-success "$exclude_jobs" 

Cargo.lock

@@ -218,6 +218,9 @@ run-state-transition-tests:
 # Downloads and runs the EF test vectors.
 test-ef: make-ef-tests run-ef-tests
 
+# Downloads and runs the nightly EF test vectors.
+test-ef-nightly: make-ef-tests-nightly run-ef-tests
+
 # Downloads and runs the EF test vectors with nextest.
 nextest-ef: make-ef-tests nextest-run-ef-tests
 
@@ -278,6 +281,10 @@ lint-full:
 make-ef-tests:
 	make -C $(EF_TESTS)
 
+# Download/extract the nightly EF test vectors.
+make-ef-tests-nightly:
+	CONSENSUS_SPECS_TEST_VERSION=nightly make -C $(EF_TESTS)
+
 # Verifies that crates compile with fuzzing features enabled
 arbitrary-fuzz:
 	cargo check -p state_processing --features arbitrary-fuzz,$(TEST_FEATURES)

Makefile

@@ -32,7 +32,7 @@ pub fn cli_app() -> Command {
         .about(
             "Imports one or more EIP-2335 passwords into a Lighthouse VC directory, \
             requesting passwords interactively. The directory flag provides a convenient \
-            method for importing a directory of keys generated by the eth2-deposit-cli \
+            method for importing a directory of keys generated by the ethstaker-deposit-cli \
             Python utility.",
         )
         .arg(

account_manager/src/validator/import.rs

@@ -181,7 +181,7 @@ pub fn compute_proposer_duties_from_head<T: BeaconChainTypes>(
     ensure_state_is_in_epoch(&mut state, head_state_root, request_epoch, &chain.spec)?;
 
     let indices = state
-        .get_beacon_proposer_indices(&chain.spec)
+        .get_beacon_proposer_indices(request_epoch, &chain.spec)
         .map_err(BeaconChainError::from)?;
 
     let dependent_root = state

beacon_node/beacon_chain/src/beacon_proposer_cache.rs

@@ -523,7 +523,8 @@ pub fn validate_blob_sidecar_for_gossip<T: BeaconChainTypes, O: ObservationStrat
             &chain.spec,
         )?;
 
-        let proposers = state.get_beacon_proposer_indices(&chain.spec)?;
+        let epoch = state.current_epoch();
+        let proposers = state.get_beacon_proposer_indices(epoch, &chain.spec)?;
         let proposer_index = *proposers
             .get(blob_slot.as_usize() % T::EthSpec::slots_per_epoch() as usize)
             .ok_or_else(|| BeaconChainError::NoProposerForSlot(blob_slot))?;

beacon_node/beacon_chain/src/blob_verification.rs

@@ -962,7 +962,8 @@ impl<T: BeaconChainTypes> GossipVerifiedBlock<T> {
                 &chain.spec,
             )?;
 
-            let proposers = state.get_beacon_proposer_indices(&chain.spec)?;
+            let epoch = state.current_epoch();
+            let proposers = state.get_beacon_proposer_indices(epoch, &chain.spec)?;
             let proposer_index = *proposers
                 .get(block.slot().as_usize() % T::EthSpec::slots_per_epoch() as usize)
                 .ok_or_else(|| BeaconChainError::NoProposerForSlot(block.slot()))?;

beacon_node/beacon_chain/src/block_verification.rs

@@ -494,24 +494,36 @@ where
 
         // Verify that blobs (if provided) match the block.
         if let Some(blobs) = &weak_subj_blobs {
-            let commitments = weak_subj_block
-                .message()
-                .body()
-                .blob_kzg_commitments()
-                .map_err(|e| format!("Blobs provided but block does not reference them: {e:?}"))?;
-            if blobs.len() != commitments.len() {
-                return Err(format!(
-                    "Wrong number of blobs, expected: {}, got: {}",
-                    commitments.len(),
-                    blobs.len()
-                ));
-            }
-            if commitments
-                .iter()
-                .zip(blobs.iter())
-                .any(|(commitment, blob)| *commitment != blob.kzg_commitment)
-            {
-                return Err("Checkpoint blob does not match block commitment".into());
+            let fulu_enabled = weak_subj_block.fork_name_unchecked().fulu_enabled();
+            if fulu_enabled && blobs.is_empty() {
+                // Blobs expected for this block, but the checkpoint server is not able to serve them.
+                // This is expected from Fulu, as only supernodes are able to serve blobs.
+                // We can consider using backfill to retrieve the data columns from the p2p network,
+                // but we can ignore this fow now until we have validator custody backfill
+                // implemented as we'll likely be able to reuse the logic.
+                // https://github.com/sigp/lighthouse/issues/6837
+            } else {
+                let commitments = weak_subj_block
+                    .message()
+                    .body()
+                    .blob_kzg_commitments()
+                    .map_err(|e| {
+                        format!("Blobs provided but block does not reference them: {e:?}")
+                    })?;
+                if blobs.len() != commitments.len() {
+                    return Err(format!(
+                        "Wrong number of blobs, expected: {}, got: {}",
+                        commitments.len(),
+                        blobs.len()
+                    ));
+                }
+                if commitments
+                    .iter()
+                    .zip(blobs.iter())
+                    .any(|(commitment, blob)| *commitment != blob.kzg_commitment)
+                {
+                    return Err("Checkpoint blob does not match block commitment".into());
+                }
             }
         }
 

beacon_node/beacon_chain/src/builder.rs

@@ -669,7 +669,8 @@ fn verify_proposer_and_signature<T: BeaconChainTypes>(
             &chain.spec,
         )?;
 
-        let proposers = state.get_beacon_proposer_indices(&chain.spec)?;
+        let epoch = state.current_epoch();
+        let proposers = state.get_beacon_proposer_indices(epoch, &chain.spec)?;
         // Prime the proposer shuffling cache with the newly-learned value.
         Ok::<_, GossipDataColumnError>(EpochBlockProposers {
             epoch: column_epoch,

beacon_node/beacon_chain/src/data_column_verification.rs

@@ -187,9 +187,9 @@ pub fn blobs_to_data_column_sidecars<E: EthSpec>(
         .collect::<Vec<_>>();
 
     // NOTE: assumes blob sidecars are ordered by index
-    let blob_cells_and_proofs_vec = blobs
+    let zipped: Vec<_> = blobs.iter().zip(proof_chunks).collect();
+    let blob_cells_and_proofs_vec = zipped
         .into_par_iter()
-        .zip(proof_chunks.into_par_iter())
         .map(|(blob, proofs)| {
             let blob = blob
                 .as_ref()