chore(trie): proof code refactoring and fixes (#2604)

- New package `lib/trie/proof` containing:
  - `lib/trie/proof.go` (refactored)
  - `internal/trie/record` files (then removed)
  - `lib/trie/database.go` (moved since it had nothing to do with database)
- Proof `Verify` function:
  - ⚠️ returns an error when root is not found in proof encoded nodes
  - ⚠️ fix `buildTrie` to detect root node for roots with encoding smaller than 32 bytes
  - ⚠️ Removes child with no matching hash from proof trie
  - ⚠️ Only decode root node, and then lazy decode nodes. **This is to allow for 'values' (not node encodings) to be together node encodings with v1 generated proofs**.
  - Only verify for a single key and value instead of pairs (no usage at all)
  - Produce ordered (from root to leaf) proof encoded nodes
  - Returns only an error and no boolean (was unneeded before)
- Proof `Generate` function:
  - ⚠️ returns an error when one key is not found in trie when generating proof
  - Remove `internal/trie/recorder` code (now unneeded)
  - Generate encoded nodes ordered from root to leaf (to reduce verify hash computations)
  - Do not add inlined nodes
- Add full coverage unit tests and remove older tests
- Minor changes
  - Better error wrapping
  - Database arguments uses a smaller locally defined `Database` interface in `proof` and `trie` packages
  - rename `proofEncodedNodes` to `encodedProofNodes`
  - rename `decProofs` to `encodedProofNodes`

Author: qdm12

dot/state/storage.go

@@ -14,6 +14,7 @@ import (
 	"github.com/ChainSafe/gossamer/lib/common"
 	rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/trie"
+	"github.com/ChainSafe/gossamer/lib/trie/proof"
 )
 
 // storagePrefix storage key prefix.
@@ -301,6 +302,7 @@ func (s *StorageState) LoadCodeHash(hash *common.Hash) (common.Hash, error) {
 }
 
 // GenerateTrieProof returns the proofs related to the keys on the state root trie
-func (s *StorageState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) ([][]byte, error) {
-	return trie.GenerateProof(stateRoot[:], keys, s.db)
+func (s *StorageState) GenerateTrieProof(stateRoot common.Hash, keys [][]byte) (
+	encodedProofNodes [][]byte, err error) {
+	return proof.Generate(stateRoot[:], keys, s.db)
 }

internal/trie/node/children.go

@@ -30,3 +30,13 @@ func (n *Node) NumChildren() (count int) {
 	}
 	return count
 }
+
+// HasChild returns true if the node has at least one child.
+func (n *Node) HasChild() (has bool) {
+	for _, child := range n.Children {
+		if child != nil {
+			return true
+		}
+	}
+	return false
+}

internal/trie/node/children_test.go

@@ -118,3 +118,42 @@ func Test_Node_NumChildren(t *testing.T) {
 		})
 	}
 }
+
+func Test_Node_HasChild(t *testing.T) {
+	t.Parallel()
+
+	testCases := map[string]struct {
+		node Node
+		has  bool
+	}{
+		"no child": {},
+		"one child at index 0": {
+			node: Node{
+				Children: []*Node{
+					{},
+				},
+			},
+			has: true,
+		},
+		"one child at index 1": {
+			node: Node{
+				Children: []*Node{
+					nil,
+					{},
+				},
+			},
+			has: true,
+		},
+	}
+
+	for name, testCase := range testCases {
+		testCase := testCase
+		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
+			has := testCase.node.HasChild()
+
+			assert.Equal(t, testCase.has, has)
+		})
+	}
+}

internal/trie/record/node.go

@@ -121,6 +121,7 @@ import (
 	rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/transaction"
 	"github.com/ChainSafe/gossamer/lib/trie"
+	"github.com/ChainSafe/gossamer/lib/trie/proof"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 
 	wasm "github.com/wasmerio/go-ext-wasm/wasmer"
@@ -886,8 +887,8 @@ func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, rootSpan
 	instanceContext := wasm.IntoInstanceContext(context)
 
 	toDecProofs := asMemorySlice(instanceContext, proofSpan)
-	var decProofs [][]byte
-	err := scale.Unmarshal(toDecProofs, &decProofs)
+	var encodedProofNodes [][]byte
+	err := scale.Unmarshal(toDecProofs, &encodedProofNodes)
 	if err != nil {
 		logger.Errorf("[ext_trie_blake2_256_verify_proof_version_1]: %s", err)
 		return C.int32_t(0)
@@ -899,18 +900,13 @@ func ext_trie_blake2_256_verify_proof_version_1(context unsafe.Pointer, rootSpan
 	mem := instanceContext.Memory().Data()
 	trieRoot := mem[rootSpan : rootSpan+32]
 
-	exists, err := trie.VerifyProof(decProofs, trieRoot, []trie.Pair{{Key: key, Value: value}})
+	err = proof.Verify(encodedProofNodes, trieRoot, key, value)
 	if err != nil {
 		logger.Errorf("[ext_trie_blake2_256_verify_proof_version_1]: %s", err)
 		return C.int32_t(0)
 	}
 
-	var result C.int32_t = 0
-	if exists {
-		result = 1
-	}
-
-	return result
+	return C.int32_t(1)
 }
 
 //export ext_misc_print_hex_version_1

internal/trie/record/recorder.go

@@ -22,6 +22,7 @@ import (
 	"github.com/ChainSafe/gossamer/lib/runtime"
 	"github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/trie"
+	"github.com/ChainSafe/gossamer/lib/trie/proof"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -1801,7 +1802,7 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) {
 	root := hash.ToBytes()
 	otherRoot := otherHash.ToBytes()
 
-	proof, err := trie.GenerateProof(root, keys, memdb)
+	allProofs, err := proof.Generate(root, keys, memdb)
 	require.NoError(t, err)
 
 	testcases := map[string]struct {
@@ -1810,17 +1811,17 @@ func Test_ext_trie_blake2_256_verify_proof_version_1(t *testing.T) {
 		expect           bool
 	}{
 		"Proof should be true": {
-			root: root, key: []byte("do"), proof: proof, value: []byte("verb"), expect: true},
+			root: root, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: true},
 		"Root empty, proof should be false": {
-			root: []byte{}, key: []byte("do"), proof: proof, value: []byte("verb"), expect: false},
+			root: []byte{}, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false},
 		"Other root, proof should be false": {
-			root: otherRoot, key: []byte("do"), proof: proof, value: []byte("verb"), expect: false},
+			root: otherRoot, key: []byte("do"), proof: allProofs, value: []byte("verb"), expect: false},
 		"Value empty, proof should be true": {
-			root: root, key: []byte("do"), proof: proof, value: nil, expect: true},
+			root: root, key: []byte("do"), proof: allProofs, value: nil, expect: true},
 		"Unknow key, proof should be false": {
-			root: root, key: []byte("unknow"), proof: proof, value: nil, expect: false},
+			root: root, key: []byte("unknow"), proof: allProofs, value: nil, expect: false},
 		"Key and value unknow, proof should be false": {
-			root: root, key: []byte("unknow"), proof: proof, value: []byte("unknow"), expect: false},
+			root: root, key: []byte("unknow"), proof: allProofs, value: []byte("unknow"), expect: false},
 		"Empty proof, should be false": {
 			root: root, key: []byte("do"), proof: [][]byte{}, value: nil, expect: false},
 	}