Merge branch 'master' into create-v31-version

Author: tsachiherman

.dockerignore

@@ -29,6 +29,7 @@ cmd/updater/updater
 tmp/dev_pkg
 tmp/out
 tmp/node_pkgs
+tmp/go-cache
 
 # Ignore vim backup and swap files
 *~

THANKS.md

@@ -5,10 +5,12 @@ A big thank you to everyone who has contributed to the `go-algorand` codebase.
 
 ### External Contributors
 - aybehrouz
+- fionnachan
 - jeapostrophe
 - jecassis
 - jsign
 - mxmauro
+- Olshansk
 - qkniep
 - RomitKumar
 - scnale

agreement/pseudonode.go

@@ -33,8 +33,9 @@ import (
 
 // TODO put these in config
 const (
-	pseudonodeVerificationBacklog   = 32
-	maxPseudonodeOutputWaitDuration = 2 * time.Second
+	pseudonodeVerificationBacklog       = 32
+	maxPseudonodeOutputWaitDuration     = 2 * time.Second
+	votingKeysLoggingDurationThreashold = 200 * time.Millisecond
 )
 
 var errPseudonodeBacklogFull = fmt.Errorf("pseudonode input channel is full")
@@ -215,9 +216,18 @@ func (n *asyncPseudonode) loadRoundParticipationKeys(voteRound basics.Round) []a
 	}
 	balanceRound := balanceRound(voteRound, cparams)
 
+	// measure the time it takes to acquire the voting keys.
+	beforeVotingKeysTime := time.Now()
+
 	// otherwise, we want to load the participation keys.
 	n.participationKeys = n.keys.VotingKeys(voteRound, balanceRound)
 	n.participationKeysRound = voteRound
+
+	votingKeysDuration := time.Since(beforeVotingKeysTime)
+	if votingKeysDuration > votingKeysLoggingDurationThreashold {
+		n.log.Warnf("asyncPseudonode: acquiring the %d voting keys for round %d took %v", len(n.participationKeys), voteRound, votingKeysDuration)
+	}
+
 	return n.participationKeys
 }
 

cmd/catchpointdump/commands.go

@@ -98,7 +98,6 @@ func main() {
 	}
 
 	if err := rootCmd.Execute(); err != nil {
-		fmt.Println(err)
 		os.Exit(1)
 	}
 }

cmd/catchpointdump/database.go

@@ -58,7 +58,7 @@ var databaseCmd = &cobra.Command{
 			}
 			defer outFile.Close()
 		}
-		err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile)
+		err = printAccountsDatabase(ledgerTrackerFilename, ledger.CatchpointFileHeader{}, outFile, nil)
 		if err != nil {
 			reportErrorf("Unable to print account database : %v", err)
 		}

cmd/catchpointdump/file.go

@@ -19,21 +19,21 @@ package main
 import (
 	"archive/tar"
 	"bufio"
-	"bytes"
 	"context"
 	"database/sql"
 	"encoding/json"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 	"strings"
 	"time"
 
 	"github.com/spf13/cobra"
 
+	cmdutil "github.com/algorand/go-algorand/cmd/util"
 	"github.com/algorand/go-algorand/config"
 	"github.com/algorand/go-algorand/data/basics"
+	"github.com/algorand/go-algorand/data/bookkeeping"
 	"github.com/algorand/go-algorand/ledger"
 	"github.com/algorand/go-algorand/ledger/ledgercore"
 	"github.com/algorand/go-algorand/logging"
@@ -43,10 +43,12 @@ import (
 
 var tarFile string
 var outFileName string
+var excludedFields *cmdutil.CobraStringSliceValue = cmdutil.MakeCobraStringSliceValue(nil, []string{"version", "catchpoint"})
 
 func init() {
 	fileCmd.Flags().StringVarP(&tarFile, "tar", "t", "", "Specify the tar file to process")
 	fileCmd.Flags().StringVarP(&outFileName, "output", "o", "", "Specify an outfile for the dump ( i.e. tracker.dump.txt )")
+	fileCmd.Flags().VarP(excludedFields, "exclude-fields", "e", "List of fields to exclude from the dump: ["+excludedFields.AllowedString()+"]")
 }
 
 var fileCmd = &cobra.Command{
@@ -59,11 +61,23 @@ var fileCmd = &cobra.Command{
 			cmd.HelpFunc()(cmd, args)
 			return
 		}
-		tarFileBytes, err := ioutil.ReadFile(tarFile)
-		if err != nil || len(tarFileBytes) == 0 {
-			reportErrorf("Unable to read '%s' : %v", tarFile, err)
+		stats, err := os.Stat(tarFile)
+		if err != nil {
+			reportErrorf("Unable to stat '%s' : %v", tarFile, err)
+		}
+		tarSize := stats.Size()
+		if tarSize == 0 {
+			reportErrorf("Empty file '%s' : %v", tarFile, err)
+		}
+		// TODO: store CurrentProtocol in catchpoint file header.
+		// As a temporary workaround use a current protocol version.
+		genesisInitState := ledgercore.InitState{
+			Block: bookkeeping.Block{BlockHeader: bookkeeping.BlockHeader{
+				UpgradeState: bookkeeping.UpgradeState{
+					CurrentProtocol: protocol.ConsensusCurrentVersion,
+				},
+			}},
 		}
-		genesisInitState := ledgercore.InitState{}
 		cfg := config.GetDefaultLocal()
 		l, err := ledger.OpenLedger(logging.Base(), "./ledger", false, genesisInitState, cfg)
 		if err != nil {
@@ -84,7 +98,14 @@ var fileCmd = &cobra.Command{
 			reportErrorf("Unable to initialize catchup database : %v", err)
 		}
 		var fileHeader ledger.CatchpointFileHeader
-		fileHeader, err = loadCatchpointIntoDatabase(context.Background(), catchupAccessor, tarFileBytes)
+
+		reader, err := os.Open(tarFile)
+		if err != nil {
+			reportErrorf("Unable to read '%s' : %v", tarFile, err)
+		}
+		defer reader.Close()
+
+		fileHeader, err = loadCatchpointIntoDatabase(context.Background(), catchupAccessor, reader, tarSize)
 		if err != nil {
 			reportErrorf("Unable to load catchpoint file into in-memory database : %v", err)
 		}
@@ -98,7 +119,7 @@ var fileCmd = &cobra.Command{
 			defer outFile.Close()
 		}
 
-		err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile)
+		err = printAccountsDatabase("./ledger.tracker.sqlite", fileHeader, outFile, excludedFields.GetSlice())
 		if err != nil {
 			reportErrorf("Unable to print account database : %v", err)
 		}
@@ -115,15 +136,14 @@ func printLoadCatchpointProgressLine(progress int, barLength int, dld int64) {
 	fmt.Printf(escapeCursorUp+escapeDeleteLine+outString+" %s\n", formatSize(dld))
 }
 
-func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.CatchpointCatchupAccessor, fileBytes []byte) (fileHeader ledger.CatchpointFileHeader, err error) {
+func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.CatchpointCatchupAccessor, tarFile io.Reader, tarSize int64) (fileHeader ledger.CatchpointFileHeader, err error) {
 	fmt.Printf("\n")
 	printLoadCatchpointProgressLine(0, 50, 0)
 	lastProgressUpdate := time.Now()
 	progress := uint64(0)
 	defer printLoadCatchpointProgressLine(0, 0, 0)
 
-	reader := bytes.NewReader(fileBytes)
-	tarReader := tar.NewReader(reader)
+	tarReader := tar.NewReader(tarFile)
 	var downloadProgress ledger.CatchpointCatchupAccessorProgress
 	for {
 		header, err := tarReader.Next()
@@ -158,9 +178,9 @@ func loadCatchpointIntoDatabase(ctx context.Context, catchupAccessor ledger.Catc
 			// we already know it's valid, since we validated that above.
 			protocol.Decode(balancesBlockBytes, &fileHeader)
 		}
-		if time.Now().Sub(lastProgressUpdate) > 50*time.Millisecond && len(fileBytes) > 0 {
+		if time.Since(lastProgressUpdate) > 50*time.Millisecond && tarSize > 0 {
 			lastProgressUpdate = time.Now()
-			printLoadCatchpointProgressLine(int(float64(progress)*50.0/float64(len(fileBytes))), 50, int64(progress))
+			printLoadCatchpointProgressLine(int(float64(progress)*50.0/float64(tarSize)), 50, int64(progress))
 		}
 	}
 }
@@ -178,7 +198,7 @@ func printDumpingCatchpointProgressLine(progress int, barLength int, dld int64)
 	fmt.Printf(escapeCursorUp + escapeDeleteLine + outString + "\n")
 }
 
-func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File) error {
+func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFileHeader, outFile *os.File, excludeFields []string) error {
 	lastProgressUpdate := time.Now()
 	progress := uint64(0)
 	defer printDumpingCatchpointProgressLine(0, 0, 0)
@@ -191,14 +211,54 @@ func printAccountsDatabase(databaseName string, fileHeader ledger.CatchpointFile
 		return err
 	}
 	if fileHeader.Version != 0 {
-		fmt.Fprintf(fileWriter, "Version: %d\nBalances Round: %d\nBlock Round: %d\nBlock Header Digest: %s\nCatchpoint: %s\nTotal Accounts: %d\nTotal Chunks: %d\n",
+		var headerFields = []string{
+			"Version: %d",
+			"Balances Round: %d",
+			"Block Round: %d",
+			"Block Header Digest: %s",
+			"Catchpoint: %s",
+			"Total Accounts: %d",
+			"Total Chunks: %d",
+		}
+		var headerValues = []interface{}{
 			fileHeader.Version,
 			fileHeader.BalancesRound,
 			fileHeader.BlocksRound,
 			fileHeader.BlockHeaderDigest.String(),
 			fileHeader.Catchpoint,
 			fileHeader.TotalAccounts,
-			fileHeader.TotalChunks)
+			fileHeader.TotalChunks,
+		}
+		// safety check
+		if len(headerFields) != len(headerValues) {
+			return fmt.Errorf("printing failed: header formatting mismatch")
+		}
+
+		var actualFields []string
+		var actualValues []interface{}
+		if len(excludeFields) == 0 {
+			actualFields = headerFields
+			actualValues = headerValues
+		} else {
+			actualFields = make([]string, 0, len(headerFields)-len(excludeFields))
+			actualValues = make([]interface{}, 0, len(headerFields)-len(excludeFields))
+			for i, field := range headerFields {
+				lower := strings.ToLower(field)
+				excluded := false
+				for _, filter := range excludeFields {
+					if strings.HasPrefix(lower, filter) {
+						excluded = true
+						break
+					}
+				}
+				if !excluded {
+					actualFields = append(actualFields, field)
+					actualValues = append(actualValues, headerValues[i])
+				}
+			}
+		}
+
+		fmt.Fprintf(fileWriter, strings.Join(actualFields, "\n")+"\n", actualValues...)
 
 		totals := fileHeader.Totals
 		fmt.Fprintf(fileWriter, "AccountTotals - Online Money: %d\nAccountTotals - Online RewardUnits : %d\nAccountTotals - Offline Money: %d\nAccountTotals - Offline RewardUnits : %d\nAccountTotals - Not Participating Money: %d\nAccountTotals - Not Participating Money RewardUnits: %d\nAccountTotals - Rewards Level: %d\n",