Merge pull request #3553 from algoidan/create-v31-version

## Summary

Create an update path that would enable the following features:
1. Batch Verification (#3031)
2. State Proof Keys (#2990)
3. TEAL 6 (#3397)
4. Expired Participation Keys (#2924)
5. Fix rewards calculation bug (#3403)

## Test Plan

Unit tests added.

Author: tsachiherman

config/consensus.go

@@ -1057,9 +1057,31 @@ func initConsensusProtocols() {
 	// v29 can be upgraded to v30, with an update delay of 7 days ( see calculation above )
 	v29.ApprovedUpgrades[protocol.ConsensusV30] = 140000
 
+	v31 := v30
+	v31.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
+	v31.EnableBatchVerification = true
+	v31.RewardsCalculationFix = true
+	v31.MaxProposedExpiredOnlineAccounts = 32
+
+	// Enable TEAL 6 / AVM 1.1
+	v31.LogicSigVersion = 6
+	v31.EnableInnerTransactionPooling = true
+	v31.IsolateClearState = true
+
+	// stat proof key registration
+	v31.EnableStateProofKeyregCheck = true
+
+	// Maximum validity period for key registration, to prevent generating too many StateProof keys
+	v31.MaxKeyregValidPeriod = 256*(1<<16) - 1
+
+	Consensus[protocol.ConsensusV31] = v31
+
+	// v30 can be upgraded to v31, with an update delay of 7 days ( see calculation above )
+	v30.ApprovedUpgrades[protocol.ConsensusV31] = 140000
+
 	// ConsensusFuture is used to test features that are implemented
 	// but not yet released in a production protocol version.
-	vFuture := v30
+	vFuture := v31
 	vFuture.ApprovedUpgrades = map[protocol.ConsensusVersion]uint64{}
 
 	// FilterTimeout for period 0 should take a new optimized, configured value, need to revisit this later
@@ -1072,23 +1094,6 @@ func initConsensusProtocols() {
 	vFuture.CompactCertWeightThreshold = (1 << 32) * 30 / 100
 	vFuture.CompactCertSecKQ = 128
 
-	// Enable TEAL 6 / AVM 1.1
-	vFuture.LogicSigVersion = 6
-	vFuture.EnableInnerTransactionPooling = true
-	vFuture.IsolateClearState = true
-
-	vFuture.MaxProposedExpiredOnlineAccounts = 32
-
-	vFuture.EnableBatchVerification = true
-
-	vFuture.RewardsCalculationFix = true
-
-	// stat proof key registration
-	vFuture.EnableStateProofKeyregCheck = true
-
-	// Maximum validity period for key registration, to prevent generating too many StateProof keys
-	vFuture.MaxKeyregValidPeriod = 256*(1<<16) - 1
-
 	Consensus[protocol.ConsensusFuture] = vFuture
 }
 

data/account/participation.go

@@ -222,7 +222,7 @@ func FillDBWithParticipationKeys(store db.Accessor, address basics.Address, firs
 
 	// TODO: change to ConsensusCurrentVersion when updated
 	interval := config.Consensus[protocol.ConsensusFuture].CompactCertRounds
-	maxValidPeriod := config.Consensus[protocol.ConsensusFuture].MaxKeyregValidPeriod
+	maxValidPeriod := config.Consensus[protocol.ConsensusCurrentVersion].MaxKeyregValidPeriod
 
 	if maxValidPeriod != 0 && uint64(lastValid-firstValid) > maxValidPeriod {
 		return PersistedParticipation{}, fmt.Errorf("the validity period for mss is too large: the limit is %d", maxValidPeriod)

data/account/participation_test.go

@@ -462,9 +462,8 @@ func TestKeyregValidityOverLimit(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	a := require.New(t)
 
-	// TODO: change to ConsensusCurrentVersion when updated
-	maxValidPeriod := config.Consensus[protocol.ConsensusFuture].MaxKeyregValidPeriod
-	dilution := config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution
+	maxValidPeriod := config.Consensus[protocol.ConsensusCurrentVersion].MaxKeyregValidPeriod
+	dilution := config.Consensus[protocol.ConsensusCurrentVersion].DefaultKeyDilution
 
 	var address basics.Address
 	crypto.RandBytes(address[:])
@@ -481,8 +480,7 @@ func TestFillDBWithParticipationKeys(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	a := require.New(t)
 
-	// TODO: change to ConsensusCurrentVersion when updated
-	dilution := config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution
+	dilution := config.Consensus[protocol.ConsensusCurrentVersion].DefaultKeyDilution
 
 	var address basics.Address
 	crypto.RandBytes(address[:])
@@ -499,19 +497,19 @@ func TestKeyregValidityPeriod(t *testing.T) {
 	partitiontest.PartitionTest(t)
 	a := require.New(t)
 
-	// TODO: change to ConsensusCurrentVersion when updated
-	// setup patched version
-	version := config.Consensus[protocol.ConsensusFuture]
-	oldValue := config.Consensus[protocol.ConsensusFuture].MaxKeyregValidPeriod
+	// Patch the global consensus variable since FillDBWithParticipationKeys uses is to check the validity period
+	// this allows us to reduce the runtime of the test while checking the logic of FillDBWithParticipationKeys
+	version := config.Consensus[protocol.ConsensusCurrentVersion]
+	oldValue := config.Consensus[protocol.ConsensusCurrentVersion].MaxKeyregValidPeriod
 	version.MaxKeyregValidPeriod = 256*(1<<4) - 1
-	config.Consensus[protocol.ConsensusFuture] = version
+	config.Consensus[protocol.ConsensusCurrentVersion] = version
 	defer func() {
 		version.MaxKeyregValidPeriod = oldValue
-		config.Consensus[protocol.ConsensusFuture] = version
+		config.Consensus[protocol.ConsensusCurrentVersion] = version
 	}()
 
-	maxValidPeriod := config.Consensus[protocol.ConsensusFuture].MaxKeyregValidPeriod
-	dilution := config.Consensus[protocol.ConsensusFuture].DefaultKeyDilution
+	maxValidPeriod := config.Consensus[protocol.ConsensusCurrentVersion].MaxKeyregValidPeriod
+	dilution := config.Consensus[protocol.ConsensusCurrentVersion].DefaultKeyDilution
 
 	var address basics.Address
 

gen/generate.go

@@ -270,7 +270,7 @@ func generateGenesisFiles(outDir string, protoVersion protocol.ConsensusVersion,
 				data.VoteFirstValid = part.FirstValid
 				data.VoteLastValid = part.LastValid
 				data.VoteKeyDilution = part.KeyDilution
-				if config.Consensus[protocol.ConsensusFuture].EnableStateProofKeyregCheck {
+				if protoParams.EnableStateProofKeyregCheck {
 					data.StateProofID = *part.StateProofVerifier()
 				}
 			}

ledger/accountdb_test.go

@@ -290,7 +290,7 @@ func TestAccountDBRound(t *testing.T) {
 }
 
 func TestAccountStorageWithStateProofID(t *testing.T) {
-	proto := config.Consensus[protocol.ConsensusFuture]
+	proto := config.Consensus[protocol.ConsensusCurrentVersion]
 
 	dbs, _ := dbOpenTest(t, true)
 	setDbLogging(t, dbs)

ledger/apply/keyreg_test.go

@@ -188,21 +188,21 @@ func TestStateProofPKKeyReg(t *testing.T) {
 	secretParticipation := keypair()
 
 	tx := createTestTxn(t, src, secretParticipation, vrfSecrets)
-	mockBal := makeMockBalances(protocol.ConsensusCurrentVersion)
+	mockBal := makeMockBalances(protocol.ConsensusV30)
 	err := Keyreg(tx.KeyregTxnFields, tx.Header, mockBal, transactions.SpecialAddresses{FeeSink: feeSink}, nil, basics.Round(0))
 	require.NoError(t, err)
 
 	acct, err := mockBal.Get(tx.Src(), false)
 	require.NoError(t, err)
-	require.Equal(t, acct.StateProofID.IsEmpty(), true)
+	require.True(t, acct.StateProofID.IsEmpty())
 
-	mockBal = makeMockBalances(protocol.ConsensusFuture)
+	mockBal = makeMockBalances(protocol.ConsensusCurrentVersion)
 	err = Keyreg(tx.KeyregTxnFields, tx.Header, mockBal, transactions.SpecialAddresses{FeeSink: feeSink}, nil, basics.Round(0))
 	require.NoError(t, err)
 
 	acct, err = mockBal.Get(tx.Src(), false)
 	require.NoError(t, err)
-	require.Equal(t, acct.StateProofID.IsEmpty(), false)
+	require.False(t, acct.StateProofID.IsEmpty())
 }
 
 func createTestTxn(t *testing.T, src basics.Address, secretParticipation *crypto.SignatureSecrets, vrfSecrets *crypto.VRFSecrets) transactions.Transaction {

protocol/consensus.go

@@ -164,6 +164,13 @@ const ConsensusV30 = ConsensusVersion(
 	"https://github.com/algorandfoundation/specs/tree/bc36005dbd776e6d1eaf0c560619bb183215645c",
 )
 
+// ConsensusV31 enables the batch verification for ed25519 signatures, Fix reward calculation issue, introduces the ability
+// to force an expired participation offline, enables TEAL 6 ( AVM 1.1 ) and add support for creating
+// state proof keys.
+const ConsensusV31 = ConsensusVersion(
+	"https://github.com/algorandfoundation/specs/tree/85e6db1fdbdef00aa232c75199e10dc5fe9498f6",
+)
+
 // ConsensusFuture is a protocol that should not appear in any production
 // network, but is used to test features before they are released.
 const ConsensusFuture = ConsensusVersion(
@@ -176,7 +183,7 @@ const ConsensusFuture = ConsensusVersion(
 
 // ConsensusCurrentVersion is the latest version and should be used
 // when a specific version is not provided.
-const ConsensusCurrentVersion = ConsensusV30
+const ConsensusCurrentVersion = ConsensusV31
 
 // Error is used to indicate that an unsupported protocol has been detected.
 type Error ConsensusVersion

test/e2e-go/features/participation/onlineOfflineParticipation_test.go

@@ -284,6 +284,7 @@ func TestAccountGoesOnlineForShortPeriod(t *testing.T) {
 
 	// we try to register online with a period in which we don't have stateproof keys
 	partKeyFirstValid := uint64(1)
+	// TODO: Change consensus version when compact certs are deployed
 	partKeyLastValid := config.Consensus[protocol.ConsensusFuture].CompactCertRounds - 1
 	partkeyResponse, _, err := client.GenParticipationKeys(newAccount, partKeyFirstValid, partKeyLastValid, 1000)
 	a.NoError(err, "rest client should be able to add participation key to new account")

test/e2e-go/features/participation/participationExpiration_test.go

@@ -76,7 +76,7 @@ func testExpirationAccounts(t *testing.T, fixture *fixtures.RestClientFixture, f
 		_, currentRound := fixture.GetBalanceAndRound(richAccount)
 		// account adds part key
 		partKeyFirstValid := uint64(0)
-		partKeyValidityPeriod := uint64(150)
+		partKeyValidityPeriod := uint64(150) // TODO: maybe increase?
 		partKeyLastValid = currentRound + partKeyValidityPeriod
 		partkeyResponse, _, err := sClient.GenParticipationKeys(sAccount, partKeyFirstValid, partKeyLastValid, 0)
 		a.NoError(err)

test/e2e-go/restAPI/restClient_test.go

@@ -553,6 +553,9 @@ func TestAccountParticipationInfo(t *testing.T) {
 	firstRound := basics.Round(params.LastRound + 1)
 	lastRound := basics.Round(params.LastRound + 1000)
 	dilution := uint64(100)
+	var stateproof merklesignature.Verifier
+	stateproof[0] = 1 // change some byte so the stateproof is not considered empty (required since consensus v31)
+
 	randomVotePKStr := randomString(32)
 	var votePK crypto.OneTimeSignatureVerifier
 	copy(votePK[:], []byte(randomVotePKStr))
@@ -576,6 +579,7 @@ func TestAccountParticipationInfo(t *testing.T) {
 			VoteKeyDilution: dilution,
 			VoteFirst:       firstRound,
 			VoteLast:        lastRound,
+			StateProofPK:    stateproof,
 		},
 	}
 	txID, err := testClient.SignAndBroadcastTransaction(wh, nil, tx)
@@ -590,6 +594,7 @@ func TestAccountParticipationInfo(t *testing.T) {
 	a.Equal(uint64(firstRound), account.Participation.VoteFirst, "API must print correct first participation round")
 	a.Equal(uint64(lastRound), account.Participation.VoteLast, "API must print correct last participation round")
 	a.Equal(dilution, account.Participation.VoteKeyDilution, "API must print correct key dilution")
+	// TODO: should we update the v1 API to support state proof? Currently it does not return this field.
 }
 
 func TestSupply(t *testing.T) {
@@ -1063,9 +1068,6 @@ func TestStateProofInParticipationInfo(t *testing.T) {
 	var localFixture fixtures.RestClientFixture
 
 	proto := config.Consensus[protocol.ConsensusCurrentVersion]
-	// TODO: remove these 2 lines when CurrentVersion contains them already
-	proto.EnableStateProofKeyregCheck = true
-	proto.MaxKeyregValidPeriod = config.Consensus[protocol.ConsensusFuture].MaxKeyregValidPeriod
 	localFixture.SetConsensus(config.ConsensusProtocols{protocol.ConsensusCurrentVersion: proto})
 
 	localFixture.Setup(t, filepath.Join("nettemplates", "TwoNodes50Each.json"))
@@ -1168,22 +1170,7 @@ func TestNilStateProofInParticipationInfo(t *testing.T) {
 	a := require.New(fixtures.SynchronizedTest(t))
 	var localFixture fixtures.RestClientFixture
 
-	// currently, the genesis creator uses the EnableStateProofKeyregCheck flag on the future
-	// version to write a statproof to the genesis file.
-	// we want to create a gensis file without state proof.
-	// + need to revert this change if other tests use that
-	tmp := config.Consensus[protocol.ConsensusFuture]
-	tmp.EnableStateProofKeyregCheck = false
-	config.Consensus[protocol.ConsensusFuture] = tmp
-
-	defer func() {
-		tmp := config.Consensus[protocol.ConsensusFuture]
-		tmp.EnableStateProofKeyregCheck = true
-		config.Consensus[protocol.ConsensusFuture] = tmp
-	}()
-
-	localFixture.SetConsensus(config.Consensus)
-	localFixture.Setup(t, filepath.Join("nettemplates", "TwoNodes50Each.json"))
+	localFixture.Setup(t, filepath.Join("nettemplates", "TwoNodes50EachV30.json"))
 	defer localFixture.Shutdown()
 
 	testClient := localFixture.LibGoalClient
@@ -1226,7 +1213,6 @@ func TestNilStateProofInParticipationInfo(t *testing.T) {
 			VotePK:           votePK,
 			SelectionPK:      selPK,
 			VoteFirst:        firstRound,
-			StateProofPK:     merklesignature.Verifier{},
 			VoteLast:         lastRound,
 			VoteKeyDilution:  dilution,
 			Nonparticipation: false,