feat: implement ext_offchain_index_set_version_1 for wasmer runtime (#1739)

* implement ext_offchain_index_set_version_1

* add storage to local

* add baseDB to runtime context

* add init base db for tests

* add base db to node storage in tests

* add base db to node storage for sync/test_helpers

* use baseDB instead of creating new db

* lint issues

* address deep source issues

* fix lint issues

* add db check, rename var

* add init BaseDB to createRuntimeStorage

Author: edwardmack

dot/core/test_helpers.go

@@ -18,19 +18,21 @@ package core
 
 import (
 	"io/ioutil"
+	"path/filepath"
 	"testing"
 
+	coremocks "github.com/ChainSafe/gossamer/dot/core/mocks"
 	"github.com/ChainSafe/gossamer/dot/network"
 	"github.com/ChainSafe/gossamer/dot/state"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/crypto/sr25519"
 	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/keystore"
+	"github.com/ChainSafe/gossamer/lib/runtime"
 	rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
+	"github.com/ChainSafe/gossamer/lib/utils"
 	log "github.com/ChainSafe/log15"
-
-	coremocks "github.com/ChainSafe/gossamer/dot/core/mocks"
 	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
 )
@@ -105,6 +107,17 @@ func NewTestService(t *testing.T, cfg *Config) *Service {
 		rtCfg.CodeHash, err = cfg.StorageState.LoadCodeHash(nil)
 		require.NoError(t, err)
 
+		nodeStorage := runtime.NodeStorage{}
+
+		if stateSrvc != nil {
+			nodeStorage.BaseDB = stateSrvc.Base
+		} else {
+			nodeStorage.BaseDB, err = utils.SetupDatabase(filepath.Join(testDatadirPath, "offline_storage"), false)
+			require.NoError(t, err)
+		}
+
+		rtCfg.NodeStorage = nodeStorage
+
 		cfg.Runtime, err = wasmer.NewRuntimeFromGenesis(gen, rtCfg)
 		require.NoError(t, err)
 	}

dot/services.go

@@ -99,6 +99,7 @@ func createRuntimeStorage(st *state.Service) (*runtime.NodeStorage, error) {
 	return &runtime.NodeStorage{
 		LocalStorage:      localStorage,
 		PersistentStorage: chaindb.NewTable(st.DB(), "offlinestorage"),
+		BaseDB:            st.Base,
 	}, nil
 }
 

dot/state/base.go

@@ -125,6 +125,16 @@ func (s *BaseState) LoadCodeSubstitutedBlockHash() common.Hash {
 	return common.NewHash(hash)
 }
 
+// Put stores key/value pair in database
+func (s *BaseState) Put(key, value []byte) error {
+	return s.db.Put(key, value)
+}
+
+// Get retrieves value by key from database
+func (s *BaseState) Get(key []byte) ([]byte, error) {
+	return s.db.Get(key)
+}
+
 func (s *BaseState) storeSkipToEpoch(epoch uint64) error {
 	buf := make([]byte, 8)
 	binary.LittleEndian.PutUint64(buf, epoch)

dot/sync/test_helpers.go

@@ -19,12 +19,12 @@ package sync
 import (
 	"io/ioutil"
 	"math/big"
+	"path/filepath"
 	"testing"
 	"time"
 
-	"github.com/stretchr/testify/mock"
-
 	"github.com/ChainSafe/gossamer/dot/state"
+	syncmocks "github.com/ChainSafe/gossamer/dot/sync/mocks"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/lib/babe"
 	"github.com/ChainSafe/gossamer/lib/common"
@@ -34,11 +34,11 @@ import (
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	"github.com/ChainSafe/gossamer/lib/transaction"
 	"github.com/ChainSafe/gossamer/lib/trie"
+	"github.com/ChainSafe/gossamer/lib/utils"
 	"github.com/ChainSafe/gossamer/pkg/scale"
 	log "github.com/ChainSafe/log15"
+	"github.com/stretchr/testify/mock"
 	"github.com/stretchr/testify/require"
-
-	syncmocks "github.com/ChainSafe/gossamer/dot/sync/mocks"
 )
 
 // NewMockFinalityGadget create and return sync FinalityGadget interface mock
@@ -100,6 +100,16 @@ func NewTestSyncer(t *testing.T, usePolkadotGenesis bool) *Service {
 		rtCfg.CodeHash, err = cfg.StorageState.LoadCodeHash(nil)
 		require.NoError(t, err)
 
+		nodeStorage := runtime.NodeStorage{}
+		if stateSrvc != nil {
+			nodeStorage.BaseDB = stateSrvc.Base
+		} else {
+			nodeStorage.BaseDB, err = utils.SetupDatabase(filepath.Join(testDatadirPath, "offline_storage"), false)
+			require.NoError(t, err)
+		}
+
+		rtCfg.NodeStorage = nodeStorage
+
 		instance, err := wasmer.NewRuntimeFromGenesis(gen, rtCfg) //nolint
 		require.NoError(t, err)
 		cfg.Runtime = instance

lib/babe/babe_test.go

@@ -21,24 +21,25 @@ import (
 	"io/ioutil"
 	"math/big"
 	"os"
+	"path/filepath"
 	"testing"
 	"time"
 
 	"github.com/ChainSafe/gossamer/dot/core"
 	"github.com/ChainSafe/gossamer/dot/state"
 	"github.com/ChainSafe/gossamer/dot/types"
+	"github.com/ChainSafe/gossamer/lib/babe/mocks"
 	"github.com/ChainSafe/gossamer/lib/common"
 	"github.com/ChainSafe/gossamer/lib/crypto/sr25519"
 	"github.com/ChainSafe/gossamer/lib/genesis"
 	"github.com/ChainSafe/gossamer/lib/runtime"
 	rtstorage "github.com/ChainSafe/gossamer/lib/runtime/storage"
 	"github.com/ChainSafe/gossamer/lib/runtime/wasmer"
 	"github.com/ChainSafe/gossamer/lib/trie"
+	"github.com/ChainSafe/gossamer/lib/utils"
 	log "github.com/ChainSafe/log15"
-	"github.com/stretchr/testify/require"
-
-	"github.com/ChainSafe/gossamer/lib/babe/mocks"
 	mock "github.com/stretchr/testify/mock"
+	"github.com/stretchr/testify/require"
 )
 
 var (
@@ -100,15 +101,17 @@ func createTestService(t *testing.T, cfg *ServiceConfig) *Service {
 		cfg.TransactionState = state.NewTransactionState()
 	}
 
+	testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") //nolint
+
+	var dbSrv *state.Service
 	if cfg.BlockState == nil || cfg.StorageState == nil || cfg.EpochState == nil {
-		testDatadirPath, err := ioutil.TempDir("/tmp", "test-datadir-*") //nolint
 		require.NoError(t, err)
 
 		config := state.Config{
 			Path:     testDatadirPath,
 			LogLevel: log.LvlInfo,
 		}
-		dbSrv := state.NewService(config)
+		dbSrv = state.NewService(config)
 		dbSrv.UseMemDB()
 
 		if cfg.EpochLength > 0 {
@@ -140,6 +143,16 @@ func createTestService(t *testing.T, cfg *ServiceConfig) *Service {
 		rtCfg.CodeHash, err = storageState.LoadCodeHash(nil)
 		require.NoError(t, err)
 
+		nodeStorage := runtime.NodeStorage{}
+		if dbSrv != nil {
+			nodeStorage.BaseDB = dbSrv.Base
+		} else {
+			nodeStorage.BaseDB, err = utils.SetupDatabase(filepath.Join(testDatadirPath, "offline_storage"), false)
+			require.NoError(t, err)
+		}
+
+		rtCfg.NodeStorage = nodeStorage
+
 		cfg.Runtime, err = wasmer.NewRuntimeFromGenesis(gen, rtCfg)
 		require.NoError(t, err)
 	}

lib/runtime/types.go

@@ -35,6 +35,7 @@ const NodeStorageTypeLocal NodeStorageType = 2
 type NodeStorage struct {
 	LocalStorage      BasicStorage
 	PersistentStorage BasicStorage
+	BaseDB            BasicStorage
 }
 
 // SetLocal persists a key and value into LOCAL node storage

lib/runtime/wasmer/exports_test.go

@@ -265,6 +265,9 @@ func TestNodeRuntime_ValidateTransaction(t *testing.T) {
 	cfg := &Config{}
 	cfg.Storage = genState
 	cfg.LogLvl = 4
+	nodeStorage := runtime.NodeStorage{}
+	nodeStorage.BaseDB = runtime.NewInMemoryDB(t)
+	cfg.NodeStorage = nodeStorage
 
 	rt, err := NewRuntimeFromGenesis(gen, cfg)
 	require.NoError(t, err)

lib/runtime/wasmer/imports.go

@@ -133,7 +133,7 @@ import (
 )
 
 //export ext_logging_log_version_1
-func ext_logging_log_version_1(context unsafe.Pointer, level C.int32_t, targetData C.int64_t, msgData C.int64_t) {
+func ext_logging_log_version_1(context unsafe.Pointer, level C.int32_t, targetData, msgData C.int64_t) {
 	logger.Trace("[ext_logging_log_version_1] executing...")
 	instanceContext := wasm.IntoInstanceContext(context)
 
@@ -319,7 +319,7 @@ func ext_crypto_ed25519_public_keys_version_1(context unsafe.Pointer, keyTypeID
 }
 
 //export ext_crypto_ed25519_sign_version_1
-func ext_crypto_ed25519_sign_version_1(context unsafe.Pointer, keyTypeID C.int32_t, key C.int32_t, msg C.int64_t) C.int64_t {
+func ext_crypto_ed25519_sign_version_1(context unsafe.Pointer, keyTypeID, key C.int32_t, msg C.int64_t) C.int64_t {
 	logger.Debug("[ext_crypto_ed25519_sign_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
@@ -872,7 +872,7 @@ func ext_misc_print_hex_version_1(context unsafe.Pointer, dataSpan C.int64_t) {
 }
 
 //export ext_misc_print_num_version_1
-func ext_misc_print_num_version_1(context unsafe.Pointer, data C.int64_t) {
+func ext_misc_print_num_version_1(_ unsafe.Pointer, data C.int64_t) {
 	logger.Trace("[ext_misc_print_num_version_1] executing...")
 
 	logger.Debug("[ext_misc_print_num_version_1]", "num", fmt.Sprintf("%d", int64(data)))
@@ -932,7 +932,7 @@ func ext_misc_runtime_version_version_1(context unsafe.Pointer, dataSpan C.int64
 }
 
 //export ext_default_child_storage_read_version_1
-func ext_default_child_storage_read_version_1(context unsafe.Pointer, childStorageKey C.int64_t, key C.int64_t, valueOut C.int64_t, offset C.int32_t) C.int64_t {
+func ext_default_child_storage_read_version_1(context unsafe.Pointer, childStorageKey, key, valueOut C.int64_t, offset C.int32_t) C.int64_t {
 	logger.Debug("[ext_default_child_storage_read_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
@@ -979,7 +979,7 @@ func ext_default_child_storage_clear_version_1(context unsafe.Pointer, childStor
 }
 
 //export ext_default_child_storage_clear_prefix_version_1
-func ext_default_child_storage_clear_prefix_version_1(context unsafe.Pointer, childStorageKey C.int64_t, prefixSpan C.int64_t) {
+func ext_default_child_storage_clear_prefix_version_1(context unsafe.Pointer, childStorageKey, prefixSpan C.int64_t) {
 	logger.Debug("[ext_default_child_storage_clear_prefix_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
@@ -996,7 +996,7 @@ func ext_default_child_storage_clear_prefix_version_1(context unsafe.Pointer, ch
 }
 
 //export ext_default_child_storage_exists_version_1
-func ext_default_child_storage_exists_version_1(context unsafe.Pointer, childStorageKey C.int64_t, key C.int64_t) C.int32_t {
+func ext_default_child_storage_exists_version_1(context unsafe.Pointer, childStorageKey, key C.int64_t) C.int32_t {
 	logger.Debug("[ext_default_child_storage_exists_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
@@ -1036,7 +1036,7 @@ func ext_default_child_storage_get_version_1(context unsafe.Pointer, childStorag
 }
 
 //export ext_default_child_storage_next_key_version_1
-func ext_default_child_storage_next_key_version_1(context unsafe.Pointer, childStorageKey C.int64_t, key C.int64_t) C.int64_t {
+func ext_default_child_storage_next_key_version_1(context unsafe.Pointer, childStorageKey, key C.int64_t) C.int64_t {
 	logger.Debug("[ext_default_child_storage_next_key_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)
@@ -1348,7 +1348,18 @@ func ext_hashing_twox_64_version_1(context unsafe.Pointer, dataSpan C.int64_t) C
 //export ext_offchain_index_set_version_1
 func ext_offchain_index_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int64_t) {
 	logger.Trace("[ext_offchain_index_set_version_1] executing...")
-	logger.Warn("[ext_offchain_index_set_version_1] unimplemented")
+	instanceContext := wasm.IntoInstanceContext(context)
+	runtimeCtx := instanceContext.Data().(*runtime.Context)
+
+	storageKey := asMemorySlice(instanceContext, keySpan)
+	newValue := asMemorySlice(instanceContext, valueSpan)
+	cp := make([]byte, len(newValue))
+	copy(cp, newValue)
+
+	err := runtimeCtx.NodeStorage.BaseDB.Put(storageKey, cp)
+	if err != nil {
+		logger.Error("[ext_offchain_index_set_version_1] failed to set value in raw storage", "error", err)
+	}
 }
 
 //export ext_offchain_local_storage_clear_version_1
@@ -1573,14 +1584,14 @@ func storageAppend(storage runtime.Storage, key, valueToAppend []byte) error {
 		nextLength = big.NewInt(0).Add(currLength, big.NewInt(1))
 	}
 
-	lengthEnc, err := scale.Encode(nextLength)
+	finalVal, err := scale.Encode(nextLength)
 	if err != nil {
 		logger.Trace("[ext_storage_append_version_1] failed to encode new length", "error", err)
 		return err
 	}
 
 	// append new length prefix to start of items array
-	finalVal := append(lengthEnc, valueRes...)
+	finalVal = append(finalVal, valueRes...)
 	logger.Debug("[ext_storage_append_version_1]", "resulting value", fmt.Sprintf("0x%x", finalVal))
 	storage.Set(key, finalVal)
 	return nil
@@ -1794,7 +1805,7 @@ func ext_storage_root_version_1(context unsafe.Pointer) C.int64_t {
 }
 
 //export ext_storage_set_version_1
-func ext_storage_set_version_1(context unsafe.Pointer, keySpan C.int64_t, valueSpan C.int64_t) {
+func ext_storage_set_version_1(context unsafe.Pointer, keySpan, valueSpan C.int64_t) {
 	logger.Trace("[ext_storage_set_version_1] executing...")
 
 	instanceContext := wasm.IntoInstanceContext(context)

lib/runtime/wasmer/imports_test.go

@@ -409,6 +409,27 @@ func Test_ext_storage_set_version_1(t *testing.T) {
 	require.Equal(t, testvalue, val)
 }
 
+func Test_ext_offline_index_set_version_1(t *testing.T) {
+	// TODO this currently fails with error could nat find exported function, determine how else to test this
+	t.Skip()
+	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
+
+	testkey := []byte("noot")
+	testvalue := []byte("washere")
+
+	encKey, err := scale.Encode(testkey)
+	require.NoError(t, err)
+	encValue, err := scale.Encode(testvalue)
+	require.NoError(t, err)
+
+	_, err = inst.Exec("rtm_ext_offline_index_set_version_1", append(encKey, encValue...))
+	require.NoError(t, err)
+
+	val, err := inst.ctx.NodeStorage.PersistentStorage.Get(testkey)
+	require.NoError(t, err)
+	require.Equal(t, testvalue, val)
+}
+
 func Test_ext_crypto_ed25519_generate_version_1(t *testing.T) {
 	inst := NewTestInstance(t, runtime.HOST_API_TEST_RUNTIME)
 

lib/runtime/wasmer/test_helpers.go

@@ -71,6 +71,7 @@ func setupConfig(t *testing.T, targetRuntime string, tt *trie.Trie, lvl log.Lvl,
 	ns := runtime.NodeStorage{
 		LocalStorage:      runtime.NewInMemoryDB(t),
 		PersistentStorage: runtime.NewInMemoryDB(t), // we're using a local storage here since this is a test runtime
+		BaseDB:            runtime.NewInMemoryDB(t), // we're using a local storage here since this is a test runtime
 	}
 	cfg := &Config{
 		Imports: ImportsNodeRuntime,