Skip to content

Commit d6e08da

Browse files
tbenrtbenr
authored
Merge branch 'master' into update-map-failover-validator-api
2 parents a6161e4 + 110ce91 commit d6e08da

File tree

94 files changed

+994
-634
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

94 files changed

+994
-634
lines changed

‎CHANGELOG.md

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,10 @@
1010

1111
### Additions and Improvements
1212
- Clean up old beacon states when switching from ARCHIVE to PRUNE or MINIMAL data storage mode
13+
- Upgrade to jvm-libp2p 1.2.1 which brings:
14+
- message publishing over gossipsub improvements (addresses `Failed to publish * because no peers were available on the required gossip topic`)
15+
- IDONTWANT control message usage improvements
1316

1417
### Bug Fixes
1518
- Fixed a block production issue for Validator Client (24.10.0 to 24.10.2 teku VC), where required headers were not provided for JSON payloads. Default SSZ block production was unaffected.
16-
- Block production now uses json data (more like 24.8.0 did than 24.10) if the Eth-Consensus-version header is absent.
19+
- Block production now uses json data (more like 24.8.0 did than 24.10) if the Eth-Consensus-version header is absent.

‎beacon/sync/src/testFixtures/java/tech/pegasys/teku/beacon/sync/SyncingNodeManager.java

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,7 @@ public class SyncingNodeManager {
8484
private final BlockGossipChannel blockGossipChannel;
8585

8686
private SyncingNodeManager(
87+
final AsyncRunner asyncRunner,
8788
final EventChannels eventChannels,
8889
final RecentChainData recentChainData,
8990
final BeaconChainUtil chainUtil,
@@ -94,7 +95,7 @@ private SyncingNodeManager(
9495
this.chainUtil = chainUtil;
9596
this.eth2P2PNetwork = eth2P2PNetwork;
9697
this.syncService = syncService;
97-
this.blockGossipChannel = eventChannels.getPublisher(BlockGossipChannel.class);
98+
this.blockGossipChannel = eventChannels.getPublisher(BlockGossipChannel.class, asyncRunner);
9899
}
99100

100101
@SuppressWarnings("FutureReturnValueIgnored")
@@ -215,7 +216,7 @@ public static SyncingNodeManager create(
215216
syncService.start().join();
216217

217218
return new SyncingNodeManager(
218-
eventChannels, recentChainData, chainUtil, eth2P2PNetwork, syncService);
219+
asyncRunner, eventChannels, recentChainData, chainUtil, eth2P2PNetwork, syncService);
219220
}
220221

221222
public SafeFuture<Peer> connect(final SyncingNodeManager peer) {
@@ -250,6 +251,6 @@ public void setSlot(final UInt64 slot) {
250251
}
251252

252253
public void gossipBlock(final SignedBeaconBlock block) {
253-
blockGossipChannel.publishBlock(block);
254+
blockGossipChannel.publishBlock(block).ifExceptionGetsHereRaiseABug();
254255
}
255256
}

‎beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactory.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -465,7 +465,7 @@ public Function<SignedBlockContainer, List<BlobSidecar>> createBlobSidecarsSelec
465465
// the blobs and the proofs wouldn't be part of the BlockContainer.
466466
final BuilderPayloadOrFallbackData builderPayloadOrFallbackData =
467467
executionLayerBlockProductionManager
468-
.getCachedUnblindedPayload(block.getSlotAndBlockRoot())
468+
.getCachedUnblindedPayload(slot)
469469
.orElseThrow(
470470
() ->
471471
new IllegalStateException(

‎beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandler.java

Lines changed: 41 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,6 @@
3434
import java.util.HashSet;
3535
import java.util.List;
3636
import java.util.Map;
37-
import java.util.Objects;
3837
import java.util.Optional;
3938
import java.util.Set;
4039
import org.apache.logging.log4j.LogManager;
@@ -122,8 +121,8 @@ public class ValidatorApiHandler implements ValidatorApiChannel {
122121
*/
123122
private static final int DUTY_EPOCH_TOLERANCE = 1;
124123

125-
private final Map<UInt64, Bytes32> createdBlockRootsBySlotCache =
126-
LimitedMap.createSynchronizedLRU(2);
124+
private final Map<UInt64, SafeFuture<Optional<BlockContainerAndMetaData>>>
125+
localBlockProductionBySlotCache = LimitedMap.createSynchronizedLRU(2);
127126

128127
private final BlockProductionAndPublishingPerformanceFactory
129128
blockProductionAndPublishingPerformanceFactory;
@@ -330,12 +329,35 @@ public SafeFuture<Optional<Map<BLSPublicKey, ValidatorStatus>>> getValidatorStat
330329
StateValidatorData::getStatus))));
331330
}
332331

332+
/**
333+
* Block would be produced only once per slot. Any additional calls to this method for the same
334+
* slot would return the same {@link SafeFuture} as the first one. The only exception is when the
335+
* block production fails. In this case, the next call would attempt to produce the block again.
336+
*/
333337
@Override
334338
public SafeFuture<Optional<BlockContainerAndMetaData>> createUnsignedBlock(
335339
final UInt64 slot,
336340
final BLSSignature randaoReveal,
337341
final Optional<Bytes32> graffiti,
338342
final Optional<UInt64> requestedBuilderBoostFactor) {
343+
return localBlockProductionBySlotCache
344+
.computeIfAbsent(
345+
slot,
346+
__ ->
347+
createUnsignedBlockInternal(
348+
slot, randaoReveal, graffiti, requestedBuilderBoostFactor))
349+
.whenException(
350+
__ -> {
351+
// allow further block production attempts for this slot
352+
localBlockProductionBySlotCache.remove(slot);
353+
});
354+
}
355+
356+
public SafeFuture<Optional<BlockContainerAndMetaData>> createUnsignedBlockInternal(
357+
final UInt64 slot,
358+
final BLSSignature randaoReveal,
359+
final Optional<Bytes32> graffiti,
360+
final Optional<UInt64> requestedBuilderBoostFactor) {
339361
LOG.info("Creating unsigned block for slot {}", slot);
340362
performanceTracker.reportBlockProductionAttempt(spec.computeEpochAtSlot(slot));
341363
if (isSyncActive()) {
@@ -389,12 +411,7 @@ private SafeFuture<Optional<BlockContainerAndMetaData>> createBlock(
389411
graffiti,
390412
requestedBuilderBoostFactor,
391413
blockProductionPerformance)
392-
.thenApply(
393-
block -> {
394-
final Bytes32 blockRoot = block.blockContainer().getBlock().getRoot();
395-
createdBlockRootsBySlotCache.put(slot, blockRoot);
396-
return Optional.of(block);
397-
});
414+
.thenApply(Optional::of);
398415
}
399416

400417
@Override
@@ -866,11 +883,21 @@ private List<ProposerDuty> getProposalSlotsForEpoch(final BeaconState state, fin
866883
return proposerSlots;
867884
}
868885

869-
private boolean isLocallyCreatedBlock(final SignedBlockContainer blockContainer) {
870-
final Bytes32 blockRoot = blockContainer.getSignedBlock().getMessage().getRoot();
871-
final Bytes32 locallyCreatedBlockRoot =
872-
createdBlockRootsBySlotCache.get(blockContainer.getSlot());
873-
return Objects.equals(blockRoot, locallyCreatedBlockRoot);
886+
private boolean isLocallyCreatedBlock(final SignedBlockContainer signedBlockContainer) {
887+
final SafeFuture<Optional<BlockContainerAndMetaData>> localBlockProduction =
888+
localBlockProductionBySlotCache.get(signedBlockContainer.getSlot());
889+
if (localBlockProduction == null || !localBlockProduction.isCompletedNormally()) {
890+
return false;
891+
}
892+
return localBlockProduction
893+
.getImmediately()
894+
.map(
895+
blockContainerAndMetaData ->
896+
blockContainerAndMetaData
897+
.blockContainer()
898+
.getRoot()
899+
.equals(signedBlockContainer.getRoot()))
900+
.orElse(false);
874901
}
875902

876903
@Override

‎beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherDeneb.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,8 +64,8 @@ void publishBlockAndBlobSidecars(
6464
final SignedBeaconBlock block,
6565
final List<BlobSidecar> blobSidecars,
6666
final BlockPublishingPerformance blockPublishingPerformance) {
67-
blockGossipChannel.publishBlock(block);
68-
blobSidecarGossipChannel.publishBlobSidecars(blobSidecars);
67+
blockGossipChannel.publishBlock(block).ifExceptionGetsHereRaiseABug();
68+
blobSidecarGossipChannel.publishBlobSidecars(blobSidecars).ifExceptionGetsHereRaiseABug();
6969
blockPublishingPerformance.blockAndBlobSidecarsPublishingInitiated();
7070
}
7171
}

‎beacon/validator/src/main/java/tech/pegasys/teku/validator/coordinator/publisher/BlockPublisherPhase0.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ void publishBlockAndBlobSidecars(
5353
final SignedBeaconBlock block,
5454
final List<BlobSidecar> blobSidecars,
5555
final BlockPublishingPerformance blockPublishingPerformance) {
56-
blockGossipChannel.publishBlock(block);
56+
blockGossipChannel.publishBlock(block).ifExceptionGetsHereRaiseABug();
5757
blockPublishingPerformance.blockPublishingInitiated();
5858
}
5959
}

‎beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/AbstractBlockFactoryTest.java

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -374,8 +374,7 @@ protected BlockAndBlobSidecars createBlockAndBlobSidecars(
374374
}
375375

376376
// simulate caching of the builder payload
377-
when(executionLayer.getCachedUnblindedPayload(
378-
signedBlockContainer.getSignedBlock().getSlotAndBlockRoot()))
377+
when(executionLayer.getCachedUnblindedPayload(signedBlockContainer.getSlot()))
379378
.thenReturn(builderPayload.map(BuilderPayloadOrFallbackData::create));
380379

381380
final List<BlobSidecar> blobSidecars =

‎beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockFactoryDenebTest.java

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,7 +156,7 @@ void shouldCreateValidBlobSidecarsForBlindedBlock() {
156156
final SignedBlockContainer block = blockAndBlobSidecars.block();
157157
final List<BlobSidecar> blobSidecars = blockAndBlobSidecars.blobSidecars();
158158

159-
verify(executionLayer).getCachedUnblindedPayload(block.getSlotAndBlockRoot());
159+
verify(executionLayer).getCachedUnblindedPayload(block.getSlot());
160160

161161
final SszList<SszKZGCommitment> expectedCommitments =
162162
block.getSignedBlock().getMessage().getBody().getOptionalBlobKzgCommitments().orElseThrow();

‎beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/BlockOperationSelectorFactoryTest.java

Lines changed: 10 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -50,7 +50,6 @@
5050
import tech.pegasys.teku.spec.datastructures.blocks.BeaconBlock;
5151
import tech.pegasys.teku.spec.datastructures.blocks.Eth1Data;
5252
import tech.pegasys.teku.spec.datastructures.blocks.SignedBeaconBlock;
53-
import tech.pegasys.teku.spec.datastructures.blocks.SlotAndBlockRoot;
5453
import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBody;
5554
import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBodyBuilder;
5655
import tech.pegasys.teku.spec.datastructures.blocks.blockbody.BeaconBlockBodySchema;
@@ -818,7 +817,7 @@ void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleCommitmentsRootIsNotConsi
818817
dataStructureUtil.randomBuilderBlobsBundle(3);
819818

820819
prepareCachedBuilderPayload(
821-
signedBlindedBeaconBlock.getSlotAndBlockRoot(),
820+
signedBlindedBeaconBlock.getSlot(),
822821
dataStructureUtil.randomExecutionPayload(),
823822
blobsBundle);
824823

@@ -843,7 +842,7 @@ void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleProofsIsNotConsistent() {
843842
when(blobsBundle.getBlobs()).thenReturn(dataStructureUtil.randomSszBlobs(2));
844843

845844
prepareCachedBuilderPayload(
846-
signedBlindedBeaconBlock.getSlotAndBlockRoot(),
845+
signedBlindedBeaconBlock.getSlot(),
847846
dataStructureUtil.randomExecutionPayload(),
848847
blobsBundle);
849848

@@ -868,7 +867,7 @@ void shouldFailCreatingBlobSidecarsIfBuilderBlobsBundleBlobsIsNotConsistent() {
868867
when(blobsBundle.getProofs()).thenReturn(dataStructureUtil.randomSszKZGProofs(2));
869868

870869
prepareCachedBuilderPayload(
871-
signedBlindedBeaconBlock.getSlotAndBlockRoot(),
870+
signedBlindedBeaconBlock.getSlot(),
872871
dataStructureUtil.randomExecutionPayload(),
873872
blobsBundle);
874873

@@ -902,14 +901,10 @@ void shouldCreateBlobSidecarsForBlindedBlock(final boolean useLocalFallback) {
902901
.toList(),
903902
blobsBundle.getProofs().stream().map(SszKZGProof::getKZGProof).toList(),
904903
blobsBundle.getBlobs().stream().toList());
905-
prepareCachedFallbackData(
906-
signedBlindedBeaconBlock.getSlotAndBlockRoot(),
907-
executionPayload,
908-
localFallbackBlobsBundle);
904+
prepareCachedFallbackData(slot, executionPayload, localFallbackBlobsBundle);
909905
} else {
910906

911-
prepareCachedBuilderPayload(
912-
signedBlindedBeaconBlock.getSlotAndBlockRoot(), executionPayload, blobsBundle);
907+
prepareCachedBuilderPayload(slot, executionPayload, blobsBundle);
913908
}
914909

915910
final List<BlobSidecar> blobSidecars =
@@ -1286,23 +1281,20 @@ private void prepareCachedPayloadHeaderWithFallbackResult(
12861281
}
12871282

12881283
private void prepareCachedBuilderPayload(
1289-
final SlotAndBlockRoot slotAndBlockRoot,
1284+
final UInt64 slot,
12901285
final ExecutionPayload executionPayload,
12911286
final tech.pegasys.teku.spec.datastructures.builder.BlobsBundle blobsBundle) {
12921287
final BuilderPayload builderPayload =
1293-
SchemaDefinitionsDeneb.required(
1294-
spec.atSlot(slotAndBlockRoot.getSlot()).getSchemaDefinitions())
1288+
SchemaDefinitionsDeneb.required(spec.atSlot(slot).getSchemaDefinitions())
12951289
.getExecutionPayloadAndBlobsBundleSchema()
12961290
.create(executionPayload, blobsBundle);
1297-
when(executionLayer.getCachedUnblindedPayload(slotAndBlockRoot))
1291+
when(executionLayer.getCachedUnblindedPayload(slot))
12981292
.thenReturn(Optional.of(BuilderPayloadOrFallbackData.create(builderPayload)));
12991293
}
13001294

13011295
private void prepareCachedFallbackData(
1302-
final SlotAndBlockRoot slotAndBlockRoot,
1303-
final ExecutionPayload executionPayload,
1304-
final BlobsBundle blobsBundle) {
1305-
when(executionLayer.getCachedUnblindedPayload(slotAndBlockRoot))
1296+
final UInt64 slot, final ExecutionPayload executionPayload, final BlobsBundle blobsBundle) {
1297+
when(executionLayer.getCachedUnblindedPayload(slot))
13061298
.thenReturn(
13071299
Optional.of(
13081300
BuilderPayloadOrFallbackData.create(

‎beacon/validator/src/test/java/tech/pegasys/teku/validator/coordinator/ValidatorApiHandlerTest.java

Lines changed: 59 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import static org.mockito.Mockito.inOrder;
2525
import static org.mockito.Mockito.mock;
2626
import static org.mockito.Mockito.never;
27+
import static org.mockito.Mockito.times;
2728
import static org.mockito.Mockito.verify;
2829
import static org.mockito.Mockito.verifyNoInteractions;
2930
import static org.mockito.Mockito.verifyNoMoreInteractions;
@@ -194,6 +195,9 @@ public void setUp() {
194195
this.epochStartSlot = spec.computeStartSlotAtEpoch(EPOCH);
195196
this.previousEpochStartSlot = spec.computeStartSlotAtEpoch(PREVIOUS_EPOCH);
196197
this.dataStructureUtil = new DataStructureUtil(spec);
198+
when(blockGossipChannel.publishBlock(any())).thenReturn(SafeFuture.COMPLETE);
199+
when(blobSidecarGossipChannel.publishBlobSidecar(any())).thenReturn(SafeFuture.COMPLETE);
200+
when(blobSidecarGossipChannel.publishBlobSidecars(any())).thenReturn(SafeFuture.COMPLETE);
197201
when(dutyMetrics.getValidatorDutyMetric())
198202
.thenReturn(ValidatorDutyMetricUtils.createValidatorDutyMetric(new StubMetricsSystem()));
199203
this.validatorApiHandler =
@@ -543,10 +547,20 @@ public void createUnsignedBlock_shouldCreateBlock() {
543547
// even if passing a non-empty requestedBlinded and requestedBuilderBoostFactor isn't a valid
544548
// combination,
545549
// we still want to check that all parameters are passed down the line to the block factory
546-
final SafeFuture<Optional<BlockContainerAndMetaData>> result =
550+
SafeFuture<Optional<BlockContainerAndMetaData>> result =
551+
validatorApiHandler.createUnsignedBlock(
552+
newSlot, randaoReveal, Optional.empty(), Optional.of(ONE));
553+
554+
assertThat(result).isCompletedWithValue(Optional.of(blockContainerAndMetaData));
555+
556+
// further calls in the same slot should return the same block
557+
result =
547558
validatorApiHandler.createUnsignedBlock(
548559
newSlot, randaoReveal, Optional.empty(), Optional.of(ONE));
549560

561+
assertThat(result).isCompletedWithValue(Optional.of(blockContainerAndMetaData));
562+
563+
// only produced once
550564
verify(blockFactory)
551565
.createUnsignedBlock(
552566
blockSlotState,
@@ -555,7 +569,51 @@ public void createUnsignedBlock_shouldCreateBlock() {
555569
Optional.empty(),
556570
Optional.of(ONE),
557571
BlockProductionPerformance.NOOP);
572+
}
573+
574+
@Test
575+
public void createUnsignedBlock_shouldAllowProducingBlockTwiceIfFirstAttemptFailed() {
576+
final UInt64 newSlot = UInt64.valueOf(25);
577+
final BeaconState blockSlotState = dataStructureUtil.randomBeaconState(newSlot);
578+
final BLSSignature randaoReveal = dataStructureUtil.randomSignature();
579+
final BlockContainerAndMetaData blockContainerAndMetaData =
580+
dataStructureUtil.randomBlockContainerAndMetaData(newSlot);
581+
582+
when(chainDataClient.getStateForBlockProduction(newSlot, false))
583+
.thenReturn(SafeFuture.completedFuture(Optional.of(blockSlotState)));
584+
when(blockFactory.createUnsignedBlock(
585+
blockSlotState,
586+
newSlot,
587+
randaoReveal,
588+
Optional.empty(),
589+
Optional.of(ONE),
590+
BlockProductionPerformance.NOOP))
591+
.thenThrow(new IllegalStateException("oopsy"))
592+
.thenReturn(SafeFuture.completedFuture(blockContainerAndMetaData));
593+
594+
// first call should fail
595+
SafeFuture<Optional<BlockContainerAndMetaData>> result =
596+
validatorApiHandler.createUnsignedBlock(
597+
newSlot, randaoReveal, Optional.empty(), Optional.of(ONE));
598+
599+
assertThat(result).isCompletedExceptionally();
600+
601+
// second call in the same slot should succeed and return the block
602+
result =
603+
validatorApiHandler.createUnsignedBlock(
604+
newSlot, randaoReveal, Optional.empty(), Optional.of(ONE));
605+
558606
assertThat(result).isCompletedWithValue(Optional.of(blockContainerAndMetaData));
607+
608+
// attempted to produce twice
609+
verify(blockFactory, times(2))
610+
.createUnsignedBlock(
611+
blockSlotState,
612+
newSlot,
613+
randaoReveal,
614+
Optional.empty(),
615+
Optional.of(ONE),
616+
BlockProductionPerformance.NOOP);
559617
}
560618

561619
@Test

0 commit comments

Comments
 (0)