simulators/ethereum/engine: Invalid Terminal Block due to Invalid Execution Tests (#626)

* simulators/ethereum/engine: Multiple bootnodes on client start

* simulator/ethereum/engine: Refactor helper, add block modifiers

* simulator/ethereum/engine: Invalid terminal block gossip tests

* simulators/ethereum/engine: block distant future test

* simulators/ethereum/engine: update readme

Author: marioevz

simulators/ethereum/engine/README.md

@@ -372,13 +372,50 @@ Test cases using multiple Proof of Work chains to test the client's behavior whe
    - Blocks `PoW4` through `PoW8` are all siblings and all reach TTD
    - Verify that `PoW Receiver` receives exactly 7 gossiped new blocks (`PoW2` through `PoW8`)
 
+- Terminal blocks are gossiped (Common Ancestor Depth 5)
+   - Clients `C1`, `PoW Producer` and `PoW Receiver` start with `G <- PoW1`
+   - `PoW Producer` and `PoW Receiver` are connected only to `C1`, not between each other
+   - `PoW Producer` continues mining chains `PoW1 <- PoW2 <- PoW3 <- PoW4 <- PoW5 <- PoW6` and `PoW1 <- PoW2' <- PoW3' <- PoW4' <- PoW5' <- PoW6'`
+   - Blocks `PoW2` and `PoW2'` have same parent `PoW1`, but have different hashes (e.g. different `ethash` seal)
+   - Blocks `PoW6` and `PoW6'` reach TTD
+   - Verify that `PoW Receiver` receives exactly 10 gossiped new blocks (`PoW2` through `PoW6` and `PoW2'` through `PoW6'`)
+
 - Build Payload After Multiple Terminal blocks via gossip
    - Clients `C1` and `PoW Producer` start with `G <- PoW1`
    - `PoW Producer` continues mining chains `PoW1 <- PoW2 <- PoW3`, `PoW1 <- PoW2 <- PoW4`, ..., `PoW1 <- PoW2 <- PoW7`
    - Blocks `PoW3` through `PoW7` are all siblings and all reach TTD
    - Send `newPayload(P1)` where `P1.parentHash == PoW7` to `C1`
    - Verify that `C1` immediately validates `P1` and returns `VALID`
 
+- Build Payload After Multiple Terminal blocks via gossip (Common Ancestor Depth 5)
+   - Clients `C1` and `PoW Producer` start with `G <- PoW1`
+   - `PoW Producer` continues mining chains `PoW1 <- PoW2 <- PoW3 <- PoW4 <- PoW5 <- PoW6` and `PoW1 <- PoW2' <- PoW3' <- PoW4' <- PoW5' <- PoW6'`
+   - Blocks `PoW2` and `PoW2'` have same parent `PoW1`, but have different hashes (e.g. different `ethash` seal)
+   - Blocks `PoW6` and `PoW6'` reach TTD
+   - Send `newPayload(P1)` where `P1.parentHash == PoW6'` to `C1`
+   - Verify that `C1` immediately validates `P1` and returns `VALID`
+
+- Transition on an Invalid Terminal Execution - Difficulty
+   - Clients `C1` and `PoW Producer` start with `G <- PoW1`
+   - `PoW Producer` mines and gossips `PoW2`
+   - `PoW2` has a `difficulty` value which reaches `TTD`, but it's not the correct expected value for `PoW2` according to `ethash` consensus rules
+   - Send `newPayload(P1)` where `P1.parentHash == PoW2` to `C1`
+   - Verify that `C1` does not follow `PoS` chain built on top of `P1` and its head still points to `PoW1`
+
+- Transition on an Invalid Terminal Execution - Distant Future
+   - Clients `C1` and `PoW Producer` start with `G <- PoW1`
+   - `PoW Producer` mines and gossips `PoW2`
+   - `PoW2` has a correct `difficulty` value and reaches `TTD`, but its `timestamp` value is 60 seconds into the future
+   - Send `newPayload(P1)` where `P1.parentHash == PoW2` to `C1`
+   - Verify that `C1` does not follow `PoS` chain built on top of `P1` and its head still points to `PoW1`
+
+- Transition on an Invalid Terminal Execution - Sealed MixHash/Nonce
+   - Clients `C1` and `PoW Producer` start with `G <- PoW1`
+   - `PoW Producer` mines and gossips `PoW2`
+   - `PoW2` has a correct `difficulty` value and reaches `TTD`, but its `mixHash`/`nonce` values are incorrect according to `ethash` consensus rules
+   - Send `newPayload(P1)` where `P1.parentHash == PoW2` to `C1`
+   - Verify that `C1` does not follow `PoS` chain built on top of `P1` and its head still points to `PoW1`
+
 
 ## Engine API Authentication (JWT) Tests:
 - No time drift, correct secret:  

simulators/ethereum/engine/client/engine.go

@@ -57,7 +57,7 @@ type EngineClient interface {
 }
 
 type EngineStarter interface {
-	StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClient EngineClient) (EngineClient, error)
+	StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...EngineClient) (EngineClient, error)
 }
 
 var (

simulators/ethereum/engine/client/hive_rpc/hive_rpc.go

@@ -7,6 +7,7 @@ import (
 	"math/big"
 	"net"
 	"net/http"
+	"strings"
 	"time"
 
 	"github.com/ethereum/go-ethereum"
@@ -33,7 +34,7 @@ type HiveRPCEngineStarter struct {
 	JWTSecret               []byte
 }
 
-func (s HiveRPCEngineStarter) StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClient client.EngineClient) (client.EngineClient, error) {
+func (s HiveRPCEngineStarter) StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...client.EngineClient) (client.EngineClient, error) {
 	var (
 		clientType = s.ClientType
 		enginePort = s.EnginePort
@@ -78,12 +79,19 @@ func (s HiveRPCEngineStarter) StartClient(T *hivesim.T, testContext context.Cont
 		ttdInt := helper.CalculateRealTTD(ClientFiles["/genesis.json"], ttd.Int64())
 		ClientParams = ClientParams.Set("HIVE_TERMINAL_TOTAL_DIFFICULTY", fmt.Sprintf("%d", ttdInt))
 	}
-	if bootClient != nil {
-		enode, err := bootClient.EnodeURL()
-		if err != nil {
-			return nil, fmt.Errorf("Unable to obtain bootnode: %v", err)
+	if bootClients != nil && len(bootClients) > 0 {
+		var (
+			enodes = make([]string, len(bootClients))
+			err    error
+		)
+		for i, bootClient := range bootClients {
+			enodes[i], err = bootClient.EnodeURL()
+			if err != nil {
+				return nil, fmt.Errorf("Unable to obtain bootnode: %v", err)
+			}
 		}
-		ClientParams = ClientParams.Set("HIVE_BOOTNODE", enode)
+		enodeString := strings.Join(enodes, ",")
+		ClientParams = ClientParams.Set("HIVE_BOOTNODE", enodeString)
 	}
 
 	// Start the client and create the engine client object

simulators/ethereum/engine/client/node/node.go

@@ -45,6 +45,10 @@ type GethNodeTestConfiguration struct {
 	// The node mines proof of work blocks
 	PoWMiner          bool
 	PoWMinerEtherBase common.Address
+	DisableGossiping  bool
+
+	// Block Modifier
+	BlockModifier helper.BlockModifier
 
 	// In PoW production mode, produce many terminal blocks which shall be gossiped
 	TerminalBlockSiblingCount *big.Int
@@ -81,15 +85,14 @@ var (
 	DefaultTerminalBlockSiblingDepth = big.NewInt(1)
 )
 
-func (s GethNodeEngineStarter) StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClient client.EngineClient) (client.EngineClient, error) {
-	return s.StartGethNode(T, testContext, ClientParams, ClientFiles, bootClient)
+func (s GethNodeEngineStarter) StartClient(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...client.EngineClient) (client.EngineClient, error) {
+	return s.StartGethNode(T, testContext, ClientParams, ClientFiles, bootClients...)
 }
 
-func (s GethNodeEngineStarter) StartGethNode(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClient client.EngineClient) (*GethNode, error) {
+func (s GethNodeEngineStarter) StartGethNode(T *hivesim.T, testContext context.Context, ClientParams hivesim.Params, ClientFiles hivesim.Params, bootClients ...client.EngineClient) (*GethNode, error) {
 	var (
-		ttd      = s.TerminalTotalDifficulty
-		bootnode string
-		err      error
+		ttd = s.TerminalTotalDifficulty
+		err error
 	)
 	genesisPath, ok := ClientFiles["/genesis.json"]
 	if !ok {
@@ -111,10 +114,14 @@ func (s GethNodeEngineStarter) StartGethNode(T *hivesim.T, testContext context.C
 	}
 	genesis.Config.TerminalTotalDifficulty = ttd
 
-	if bootClient != nil {
-		bootnode, err = bootClient.EnodeURL()
-		if err != nil {
-			return nil, fmt.Errorf("Unable to obtain bootnode: %v", err)
+	var enodes []string
+	if bootClients != nil && len(bootClients) > 0 {
+		enodes = make([]string, len(bootClients))
+		for i, bootClient := range bootClients {
+			enodes[i], err = bootClient.EnodeURL()
+			if err != nil {
+				return nil, fmt.Errorf("Unable to obtain bootnode: %v", err)
+			}
 		}
 	}
 
@@ -138,7 +145,7 @@ func (s GethNodeEngineStarter) StartGethNode(T *hivesim.T, testContext context.C
 		s.Config.TerminalBlockSiblingDepth = DefaultTerminalBlockSiblingDepth
 	}
 
-	g, err := newNode(s.Config, bootnode, &genesis)
+	g, err := newNode(s.Config, enodes, &genesis)
 	if err != nil {
 		return nil, err
 	}
@@ -166,7 +173,6 @@ type GethNode struct {
 	mustHeadBlock *types.Block
 
 	datadir    string
-	bootnode   string
 	genesis    *core.Genesis
 	ttd        *big.Int
 	api        *ethcatalyst.ConsensusAPI
@@ -190,14 +196,14 @@ type GethNode struct {
 	config GethNodeTestConfiguration
 }
 
-func newNode(config GethNodeTestConfiguration, bootnode string, genesis *core.Genesis) (*GethNode, error) {
+func newNode(config GethNodeTestConfiguration, bootnodes []string, genesis *core.Genesis) (*GethNode, error) {
 	// Define the basic configurations for the Ethereum node
 	datadir, _ := os.MkdirTemp("", "")
 
-	return restart(config, bootnode, datadir, genesis)
+	return restart(config, bootnodes, datadir, genesis)
 }
 
-func restart(startConfig GethNodeTestConfiguration, bootnode, datadir string, genesis *core.Genesis) (*GethNode, error) {
+func restart(startConfig GethNodeTestConfiguration, bootnodes []string, datadir string, genesis *core.Genesis) (*GethNode, error) {
 	if startConfig.Name == "" {
 		startConfig.Name = "Modified Geth Module"
 	}
@@ -247,16 +253,20 @@ func restart(startConfig GethNodeTestConfiguration, bootnode, datadir string, ge
 		time.Sleep(250 * time.Millisecond)
 	}
 	// Connect the node to the bootnode
-	node := enode.MustParse(bootnode)
-	stack.Server().AddPeer(node)
+	if bootnodes != nil && len(bootnodes) > 0 {
+		for _, bootnode := range bootnodes {
+			node := enode.MustParse(bootnode)
+			stack.Server().AddTrustedPeer(node)
+			stack.Server().AddPeer(node)
+		}
+	}
 
 	stack.Server().EnableMsgEvents = true
 
 	g := &GethNode{
 		node:         stack,
 		eth:          ethBackend,
 		datadir:      datadir,
-		bootnode:     bootnode,
 		genesis:      genesis,
 		ttd:          genesis.Config.TerminalTotalDifficulty,
 		api:          ethcatalyst.NewConsensusAPI(ethBackend),
@@ -392,6 +402,14 @@ func (n *GethNode) PoWMiningLoop() {
 		// Wait until the previous block is succesfully propagated
 		<-time.After(time.Millisecond * 200)
 
+		// Modify the block before sealing
+		if n.config.BlockModifier != nil {
+			b, err = n.config.BlockModifier.ModifyUnsealedBlock(b)
+			if err != nil {
+				panic(err)
+			}
+		}
+
 		// Seal the next block to broadcast
 		rChan := make(chan *types.Block, 0)
 		stopChan := make(chan struct{})
@@ -401,8 +419,21 @@ func (n *GethNode) PoWMiningLoop() {
 			if b == nil {
 				panic(fmt.Errorf("no block got sealed"))
 			}
+			// Modify the sealed block if necessary
+			if n.config.BlockModifier != nil {
+				sealVerifier := func(h *types.Header) bool {
+					return n.powEngine.VerifyHeader(n.eth.BlockChain(), h, true) == nil
+				}
+				b, err = n.config.BlockModifier.ModifySealedBlock(sealVerifier, b)
+				if err != nil {
+					panic(err)
+				}
+			}
+
 			// Broadcast
-			n.eth.EventMux().Post(core.NewMinedBlockEvent{Block: b})
+			if !n.config.DisableGossiping {
+				n.eth.EventMux().Post(core.NewMinedBlockEvent{Block: b})
+			}
 
 			// Check whether the block was a terminal block
 			if t, td, err := n.isBlockTerminal(b); err == nil {

simulators/ethereum/engine/helper/block.go

@@ -0,0 +1,72 @@
+package helper
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/big"
+	"math/rand"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+)
+
+type BlockModifier interface {
+	ModifyUnsealedBlock(*types.Block) (*types.Block, error)
+	ModifySealedBlock(func(*types.Header) bool, *types.Block) (*types.Block, error)
+}
+
+type PoWBlockModifier struct {
+	Difficulty           *big.Int
+	RandomStateRoot      bool
+	InvalidSealedMixHash bool
+	InvalidSealedNonce   bool
+	TimeSecondsInFuture  uint64
+}
+
+func (m PoWBlockModifier) ModifyUnsealedBlock(baseBlock *types.Block) (*types.Block, error) {
+	modifiedHeader := types.CopyHeader(baseBlock.Header())
+
+	if m.Difficulty != nil {
+		modifiedHeader.Difficulty = m.Difficulty
+	}
+	if m.RandomStateRoot {
+		rand.Read(modifiedHeader.Root[:])
+	}
+	if m.TimeSecondsInFuture > 0 {
+		modifiedHeader.Time += m.TimeSecondsInFuture
+	}
+
+	modifiedBlock := types.NewBlockWithHeader(modifiedHeader)
+	modifiedBlock = modifiedBlock.WithBody(baseBlock.Transactions(), baseBlock.Uncles())
+
+	js, _ := json.MarshalIndent(modifiedBlock.Header(), "", "  ")
+	fmt.Printf("DEBUG: Modified unsealed block with hash %v:\n%s\n", modifiedBlock.Hash(), js)
+
+	return modifiedBlock, nil
+}
+
+func (m PoWBlockModifier) ModifySealedBlock(f func(*types.Header) bool, baseBlock *types.Block) (*types.Block, error) {
+	modifiedHeader := types.CopyHeader(baseBlock.Header())
+
+	if m.InvalidSealedMixHash {
+		modifiedHeader.MixDigest = common.Hash{}
+		for f(modifiedHeader) {
+			// Increase the hash until it's not valid
+			modifiedHeader.MixDigest[len(modifiedHeader.MixDigest[:])]++
+		}
+	}
+	if m.InvalidSealedNonce {
+		modifiedHeader.Nonce = types.BlockNonce{}
+		for f(modifiedHeader) {
+			// Increase the nonce until it's not valid
+			modifiedHeader.Nonce[len(modifiedHeader.Nonce[:])]++
+		}
+	}
+
+	modifiedBlock := types.NewBlockWithHeader(modifiedHeader)
+	modifiedBlock = modifiedBlock.WithBody(baseBlock.Transactions(), baseBlock.Uncles())
+
+	js, _ := json.MarshalIndent(modifiedBlock.Header(), "", "  ")
+	fmt.Printf("DEBUG: Modified sealed block with hash %v:\n%s\n", modifiedBlock.Hash(), js)
+	return modifiedBlock, nil
+}