Skip to content

Commit 31700c6

Browse files
arijitADtimwu20
arijitAD
authored and
timwu20
committed
feat(dot/state): implement online pruning of historical state tries (ChainSafe#1596)
1 parent 9e054fc commit 31700c6

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

52 files changed

+1030
-255
lines changed

‎chain/dev/defaults.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ var (
3939
// DefaultLvl is the default log level
4040
DefaultLvl = log.LvlInfo
4141

42+
// DefaultPruningMode is the default pruning mode
43+
DefaultPruningMode = "archive"
44+
// DefaultRetainBlocks is the default retained blocks
45+
DefaultRetainBlocks = int64(512)
46+
4247
// InitConfig
4348

4449
// DefaultGenesis is the default genesis configuration path

‎chain/gssmr/defaults.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ var (
3939
// DefaultLvl is the default log level
4040
DefaultLvl = log.LvlInfo
4141

42+
// DefaultPruningMode is the default pruning mode
43+
DefaultPruningMode = "archive"
44+
// DefaultRetainBlocks is the default retained blocks
45+
DefaultRetainBlocks = int64(512)
46+
4247
// InitConfig
4348

4449
// DefaultGenesis is the default genesis configuration path

‎chain/kusama/defaults.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,11 @@ var (
3939
// DefaultLvl is the default log level
4040
DefaultLvl = log.LvlInfo
4141

42+
// DefaultPruningMode is the default pruning mode
43+
DefaultPruningMode = "archive"
44+
// DefaultRetainBlocks is the default retained blocks
45+
DefaultRetainBlocks = int64(512)
46+
4247
// InitConfig
4348

4449
// DefaultGenesis is the default genesis configuration path

‎chain/polkadot/defaults.go

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,11 @@ var (
3636
// DefaultLvl is the default log level
3737
DefaultLvl = log.LvlInfo
3838

39+
// DefaultPruningMode is the default pruning mode
40+
DefaultPruningMode = "archive"
41+
// DefaultRetainBlocks is the default pruning mode
42+
DefaultRetainBlocks = int64(512)
43+
3944
// InitConfig
4045

4146
// DefaultGenesis is the default genesis configuration path

‎cmd/gossamer/config.go

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,10 +21,12 @@ import (
2121
"strconv"
2222
"strings"
2323

24+
"github.com/ChainSafe/gossamer/chain/dev"
2425
"github.com/ChainSafe/gossamer/chain/gssmr"
2526
"github.com/ChainSafe/gossamer/dot"
2627
ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
2728
"github.com/ChainSafe/gossamer/dot/state"
29+
"github.com/ChainSafe/gossamer/dot/state/pruner"
2830
"github.com/ChainSafe/gossamer/dot/types"
2931
"github.com/ChainSafe/gossamer/lib/common"
3032
"github.com/ChainSafe/gossamer/lib/genesis"
@@ -174,6 +176,14 @@ func createInitConfig(ctx *cli.Context) (*dot.Config, error) {
174176
return nil, err
175177
}
176178

179+
if !cfg.Global.Pruning.IsValid() {
180+
return nil, fmt.Errorf("--%s must be either %s or %s", PruningFlag.Name, pruner.Full, pruner.Archive)
181+
}
182+
183+
if cfg.Global.RetainBlocks < dev.DefaultRetainBlocks {
184+
return nil, fmt.Errorf("--%s cannot be less than %d", RetainBlockNumberFlag.Name, dev.DefaultRetainBlocks)
185+
}
186+
177187
// set log config
178188
err = setLogConfig(ctx, tomlCfg, &cfg.Global, &cfg.Log)
179189
if err != nil {
@@ -443,6 +453,9 @@ func setDotGlobalConfigFromToml(tomlCfg *ctoml.Config, cfg *dot.GlobalConfig) {
443453
}
444454

445455
cfg.MetricsPort = tomlCfg.Global.MetricsPort
456+
457+
cfg.RetainBlocks = tomlCfg.Global.RetainBlocks
458+
cfg.Pruning = pruner.Mode(tomlCfg.Global.Pruning)
446459
}
447460
}
448461

@@ -472,6 +485,8 @@ func setDotGlobalConfigFromFlags(ctx *cli.Context, cfg *dot.GlobalConfig) {
472485
cfg.MetricsPort = uint32(metricsPort)
473486
}
474487

488+
cfg.RetainBlocks = ctx.Int64(RetainBlockNumberFlag.Name)
489+
cfg.Pruning = pruner.Mode(ctx.String(PruningFlag.Name))
475490
cfg.NoTelemetry = ctx.Bool("no-telemetry")
476491
}
477492

‎cmd/gossamer/config_test.go

Lines changed: 11 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"io/ioutil"
2121
"testing"
2222

23+
"github.com/ChainSafe/gossamer/chain/dev"
2324
"github.com/ChainSafe/gossamer/chain/gssmr"
2425
"github.com/ChainSafe/gossamer/dot"
2526
"github.com/ChainSafe/gossamer/dot/state"
@@ -45,26 +46,26 @@ func TestConfigFromChainFlag(t *testing.T) {
4546
}{
4647
{
4748
"Test gossamer --chain gssmr",
48-
[]string{"chain", "name"},
49-
[]interface{}{"gssmr", dot.GssmrConfig().Global.Name},
49+
[]string{"chain", "name", "pruning", "retain-blocks"},
50+
[]interface{}{"gssmr", dot.GssmrConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
5051
dot.GssmrConfig(),
5152
},
5253
{
5354
"Test gossamer --chain kusama",
54-
[]string{"chain", "name"},
55-
[]interface{}{"kusama", dot.KusamaConfig().Global.Name},
55+
[]string{"chain", "name", "pruning", "retain-blocks"},
56+
[]interface{}{"kusama", dot.KusamaConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
5657
dot.KusamaConfig(),
5758
},
5859
{
5960
"Test gossamer --chain polkadot",
60-
[]string{"chain", "name"},
61-
[]interface{}{"polkadot", dot.PolkadotConfig().Global.Name},
61+
[]string{"chain", "name", "pruning", "retain-blocks"},
62+
[]interface{}{"polkadot", dot.PolkadotConfig().Global.Name, gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
6263
dot.PolkadotConfig(),
6364
},
6465
{
6566
"Test gossamer --chain dev",
66-
[]string{"chain", "name"},
67-
[]interface{}{"dev", dot.DevConfig().Global.Name},
67+
[]string{"chain", "name", "pruning", "retain-blocks"},
68+
[]interface{}{"dev", dot.DevConfig().Global.Name, dev.DefaultPruningMode, dev.DefaultRetainBlocks},
6869
dot.DevConfig(),
6970
},
7071
}
@@ -100,8 +101,8 @@ func TestInitConfigFromFlags(t *testing.T) {
100101
}{
101102
{
102103
"Test gossamer --genesis",
103-
[]string{"config", "genesis"},
104-
[]interface{}{testCfgFile.Name(), "test_genesis"},
104+
[]string{"config", "genesis", "pruning", "retain-blocks"},
105+
[]interface{}{testCfgFile.Name(), "test_genesis", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
105106
dot.InitConfig{
106107
Genesis: "test_genesis",
107108
},

‎cmd/gossamer/export.go

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,11 +79,13 @@ func dotConfigToToml(dcfg *dot.Config) *ctoml.Config {
7979
cfg := &ctoml.Config{}
8080

8181
cfg.Global = ctoml.GlobalConfig{
82-
Name: dcfg.Global.Name,
83-
ID: dcfg.Global.ID,
84-
BasePath: dcfg.Global.BasePath,
85-
LogLvl: dcfg.Global.LogLvl.String(),
86-
MetricsPort: dcfg.Global.MetricsPort,
82+
Name: dcfg.Global.Name,
83+
ID: dcfg.Global.ID,
84+
BasePath: dcfg.Global.BasePath,
85+
LogLvl: dcfg.Global.LogLvl.String(),
86+
MetricsPort: dcfg.Global.MetricsPort,
87+
RetainBlocks: dcfg.Global.RetainBlocks,
88+
Pruning: string(dcfg.Global.Pruning),
8789
}
8890

8991
cfg.Log = ctoml.LogConfig{

‎cmd/gossamer/export_test.go

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"io/ioutil"
2121
"testing"
2222

23+
"github.com/ChainSafe/gossamer/chain/gssmr"
2324
"github.com/ChainSafe/gossamer/dot"
2425
"github.com/ChainSafe/gossamer/lib/utils"
2526

@@ -91,8 +92,8 @@ func TestExportCommand(t *testing.T) {
9192
},
9293
{
9394
"Test gossamer export --config --genesis --bootnodes --log --force",
94-
[]string{"config", "genesis", "bootnodes", "name", "force"},
95-
[]interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true"},
95+
[]string{"config", "genesis", "bootnodes", "name", "force", "pruning", "retain-blocks"},
96+
[]interface{}{testConfig, genFile.Name(), testBootnode, "Gossamer", "true", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
9697
&dot.Config{
9798
Global: testCfg.Global,
9899
Init: dot.InitConfig{
@@ -122,8 +123,8 @@ func TestExportCommand(t *testing.T) {
122123
},
123124
{
124125
"Test gossamer export --config --genesis --protocol --log --force",
125-
[]string{"config", "genesis", "protocol", "force", "name"},
126-
[]interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer"},
126+
[]string{"config", "genesis", "protocol", "force", "name", "pruning", "retain-blocks"},
127+
[]interface{}{testConfig, genFile.Name(), testProtocol, "true", "Gossamer", gssmr.DefaultPruningMode, gssmr.DefaultRetainBlocks},
127128
&dot.Config{
128129
Global: testCfg.Global,
129130
Init: dot.InitConfig{

‎cmd/gossamer/flags.go

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package main
1818

1919
import (
20+
"github.com/ChainSafe/gossamer/chain/dev"
2021
log "github.com/ChainSafe/log15"
2122
"github.com/urfave/cli"
2223
)
@@ -280,10 +281,18 @@ var (
280281
}
281282

282283
// RetainBlockNumberFlag retain number of block from latest block while pruning, valid for the use with prune-state subcommand
283-
RetainBlockNumberFlag = cli.IntFlag{
284+
RetainBlockNumberFlag = cli.Int64Flag{
284285
Name: "retain-blocks",
285286
Usage: "Retain number of block from latest block while pruning",
286-
Value: 256,
287+
Value: dev.DefaultRetainBlocks,
288+
}
289+
290+
// PruningFlag triggers the online pruning of historical state tries. It's either full or archive. To enable pruning the value
291+
// should be set to `full`.
292+
PruningFlag = cli.StringFlag{
293+
Name: "pruning",
294+
Usage: `State trie online pruning ("full", "archive")`,
295+
Value: dev.DefaultPruningMode,
287296
}
288297
)
289298

@@ -301,7 +310,6 @@ var (
301310
RewindFlag,
302311
DBPathFlag,
303312
BloomFilterSizeFlag,
304-
RetainBlockNumberFlag,
305313
}
306314

307315
// StartupFlags are flags that are valid for use with the root command and the export subcommand
@@ -346,6 +354,8 @@ var (
346354
InitFlags = append([]cli.Flag{
347355
ForceFlag,
348356
GenesisFlag,
357+
PruningFlag,
358+
RetainBlockNumberFlag,
349359
}, GlobalFlags...)
350360

351361
BuildSpecFlags = append([]cli.Flag{

‎cmd/gossamer/flags_test.go

Lines changed: 13 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ import (
2020
"io/ioutil"
2121
"testing"
2222

23+
"github.com/ChainSafe/gossamer/chain/dev"
2324
"github.com/ChainSafe/gossamer/dot"
2425
"github.com/ChainSafe/gossamer/lib/utils"
2526

@@ -43,24 +44,24 @@ func TestFixFlagOrder(t *testing.T) {
4344
values []interface{}
4445
}{
4546
{
46-
"Test gossamer --config --genesis --log --force",
47-
[]string{"config", "genesis", "log", "force"},
48-
[]interface{}{testConfig.Name(), genFile.Name(), "trace", true},
47+
"Test gossamer --config --genesis --log --force --pruning --retain-blocks",
48+
[]string{"config", "genesis", "log", "force", "pruning", "retain-blocks"},
49+
[]interface{}{testConfig.Name(), genFile.Name(), "trace", true, dev.DefaultPruningMode, dev.DefaultRetainBlocks},
4950
},
5051
{
51-
"Test gossamer --config --genesis --force --log",
52-
[]string{"config", "genesis", "force", "log"},
53-
[]interface{}{testConfig.Name(), genFile.Name(), true, "trace"},
52+
"Test gossamer --config --genesis --force --log --pruning --retain-blocks",
53+
[]string{"config", "genesis", "force", "log", "pruning", "retain-blocks"},
54+
[]interface{}{testConfig.Name(), genFile.Name(), true, "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
5455
},
5556
{
56-
"Test gossamer --config --force --genesis --log",
57-
[]string{"config", "force", "genesis", "log"},
58-
[]interface{}{testConfig.Name(), true, genFile.Name(), "trace"},
57+
"Test gossamer --config --force --genesis --log ---pruning --retain-blocks",
58+
[]string{"config", "force", "genesis", "log", "pruning", "retain-blocks"},
59+
[]interface{}{testConfig.Name(), true, genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
5960
},
6061
{
61-
"Test gossamer --force --config --genesis --log",
62-
[]string{"force", "config", "genesis", "log"},
63-
[]interface{}{true, testConfig.Name(), genFile.Name(), "trace"},
62+
"Test gossamer --force --config --genesis --log --pruning --retain-blocks",
63+
[]string{"force", "config", "genesis", "log", "pruning", "retain-blocks"},
64+
[]interface{}{true, testConfig.Name(), genFile.Name(), "trace", dev.DefaultPruningMode, dev.DefaultRetainBlocks},
6465
},
6566
}
6667

0 commit comments

Comments
 (0)