Skip to content

Commit c395d22

Browse files
ibuildthecloudibuildthecloud
committed
Improve dqlite error reporting and start failure modes
1 parent 5d820e3 commit c395d22

File tree

6 files changed

+74
-22
lines changed

6 files changed

+74
-22
lines changed

‎pkg/drivers/dqlite/dqlite.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,28 +68,28 @@ func New(ctx context.Context, datasourceName string) (server.Backend, error) {
6868
if opts.peerFile != "" {
6969
nodeStore, err = client.DefaultNodeStore(opts.peerFile)
7070
if err != nil {
71-
return nil, err
71+
return nil, errors.Wrap(err, "opening peerfile")
7272
}
7373
} else {
7474
nodeStore = client.NewInmemNodeStore()
7575
}
7676

7777
if err := AddPeers(ctx, nodeStore, opts.peers...); err != nil {
78-
return nil, err
78+
return nil, errors.Wrap(err, "add peers")
7979
}
8080

8181
d, err := driver.New(nodeStore,
8282
driver.WithLogFunc(Logger),
8383
driver.WithContext(ctx),
8484
driver.WithDialFunc(Dialer))
8585
if err != nil {
86-
return nil, err
86+
return nil, errors.Wrap(err, "new dqlite driver")
8787
}
8888

8989
sql.Register("dqlite", d)
90-
backend, generic, err := sqlite.NewVariant("dqlite", opts.dsn)
90+
backend, generic, err := sqlite.NewVariant(ctx, "dqlite", opts.dsn)
9191
if err != nil {
92-
return nil, err
92+
return nil, errors.Wrap(err, "sqlite client")
9393
}
9494
if err := migrate(ctx, generic.DB); err != nil {
9595
return nil, errors.Wrap(err, "failed to migrate DB from sqlite")

‎pkg/drivers/generic/generic.go‎

Lines changed: 32 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -128,12 +128,42 @@ func (d *Generic) Migrate(ctx context.Context) {
128128
}
129129
}
130130

131-
func Open(driverName, dataSourceName string, paramCharacter string, numbered bool) (*Generic, error) {
131+
func openAndTest(driverName, dataSourceName string) (*sql.DB, error) {
132132
db, err := sql.Open(driverName, dataSourceName)
133133
if err != nil {
134134
return nil, err
135135
}
136136

137+
for i := 0; i < 3; i++ {
138+
if err := db.Ping(); err != nil {
139+
db.Close()
140+
return nil, err
141+
}
142+
}
143+
144+
return db, nil
145+
}
146+
147+
func Open(ctx context.Context, driverName, dataSourceName string, paramCharacter string, numbered bool) (*Generic, error) {
148+
var (
149+
db *sql.DB
150+
err error
151+
)
152+
153+
for i := 0; i < 300; i++ {
154+
db, err = openAndTest(driverName, dataSourceName)
155+
if err == nil {
156+
break
157+
}
158+
159+
logrus.Errorf("failed to ping connection: %v", err)
160+
select {
161+
case <-ctx.Done():
162+
return nil, ctx.Err()
163+
case <-time.After(time.Second):
164+
}
165+
}
166+
137167
return &Generic{
138168
DB: db,
139169

@@ -178,7 +208,7 @@ func Open(driverName, dataSourceName string, paramCharacter string, numbered boo
178208

179209
FillSQL: q(`INSERT INTO kine(id, name, created, deleted, create_revision, prev_revision, lease, value, old_value)
180210
values(?, ?, ?, ?, ?, ?, ?, ?, ?)`, paramCharacter, numbered),
181-
}, nil
211+
}, err
182212
}
183213

184214
func (d *Generic) query(ctx context.Context, sql string, args ...interface{}) (*sql.Rows, error) {

‎pkg/drivers/mysql/mysql.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ var (
3939
createDB = "create database if not exists "
4040
)
4141

42-
func New(dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
42+
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
4343
tlsConfig, err := tlsInfo.ClientConfig()
4444
if err != nil {
4545
return nil, err
@@ -58,7 +58,7 @@ func New(dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
5858
return nil, err
5959
}
6060

61-
dialect, err := generic.Open("mysql", parsedDSN, "?", false)
61+
dialect, err := generic.Open(ctx, "mysql", parsedDSN, "?", false)
6262
if err != nil {
6363
return nil, err
6464
}

‎pkg/drivers/pgsql/pgsql.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ var (
4040
createDB = "create database "
4141
)
4242

43-
func New(dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
43+
func New(ctx context.Context, dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
4444
parsedDSN, err := prepareDSN(dataSourceName, tlsInfo)
4545
if err != nil {
4646
return nil, err
@@ -50,7 +50,7 @@ func New(dataSourceName string, tlsInfo tls.Config) (server.Backend, error) {
5050
return nil, err
5151
}
5252

53-
dialect, err := generic.Open("postgres", parsedDSN, "$", true)
53+
dialect, err := generic.Open(ctx, "postgres", parsedDSN, "$", true)
5454
if err != nil {
5555
return nil, err
5656
}

‎pkg/drivers/sqlite/sqlite.go‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,12 +4,15 @@ import (
44
"context"
55
"database/sql"
66
"os"
7+
"time"
78

89
"github.com/mattn/go-sqlite3"
10+
"github.com/pkg/errors"
911
"github.com/rancher/kine/pkg/drivers/generic"
1012
"github.com/rancher/kine/pkg/logstructured"
1113
"github.com/rancher/kine/pkg/logstructured/sqllog"
1214
"github.com/rancher/kine/pkg/server"
15+
"github.com/sirupsen/logrus"
1316

1417
// sqlite db driver
1518
_ "github.com/mattn/go-sqlite3"
@@ -34,20 +37,20 @@ var (
3437
}
3538
)
3639

37-
func New(dataSourceName string) (server.Backend, error) {
38-
backend, _, err := NewVariant("sqlite3", dataSourceName)
40+
func New(ctx context.Context, dataSourceName string) (server.Backend, error) {
41+
backend, _, err := NewVariant(ctx, "sqlite3", dataSourceName)
3942
return backend, err
4043
}
4144

42-
func NewVariant(driverName, dataSourceName string) (server.Backend, *generic.Generic, error) {
45+
func NewVariant(ctx context.Context, driverName, dataSourceName string) (server.Backend, *generic.Generic, error) {
4346
if dataSourceName == "" {
4447
if err := os.MkdirAll("./db", 0700); err != nil {
4548
return nil, nil, err
4649
}
4750
dataSourceName = "./db/state.db?_journal=WAL&cache=shared"
4851
}
4952

50-
dialect, err := generic.Open(driverName, dataSourceName, "?", false)
53+
dialect, err := generic.Open(ctx, driverName, dataSourceName, "?", false)
5154
if err != nil {
5255
return nil, nil, err
5356
}
@@ -59,9 +62,27 @@ func NewVariant(driverName, dataSourceName string) (server.Backend, *generic.Gen
5962
return err
6063
}
6164

62-
if err := setup(dialect.DB); err != nil {
63-
return nil, nil, err
65+
// this is the first SQL that will be executed on a new DB conn so
66+
// loop on failure here because in the case of dqlite it could still be initializing
67+
for i := 0; i < 300; i++ {
68+
err = setup(dialect.DB)
69+
if err == nil {
70+
break
71+
}
72+
logrus.Errorf("failed to setup db: %v", err)
73+
select {
74+
case <-ctx.Done():
75+
return nil, nil, ctx.Err()
76+
case <-time.After(time.Second):
77+
}
78+
time.Sleep(time.Second)
79+
}
80+
if err != nil {
81+
return nil, nil, errors.Wrap(err, "setup db")
6482
}
83+
//if err := setup(dialect.DB); err != nil {
84+
// return nil, nil, errors.Wrap(err, "setup db")
85+
//}
6586

6687
dialect.Migrate(context.Background())
6788
return logstructured.New(sqllog.New(dialect)), dialect, nil

‎pkg/endpoint/endpoint.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"os"
88
"strings"
99

10+
"github.com/pkg/errors"
1011
"github.com/rancher/kine/pkg/drivers/dqlite"
1112
"github.com/rancher/kine/pkg/drivers/mysql"
1213
"github.com/rancher/kine/pkg/drivers/pgsql"
@@ -52,11 +53,11 @@ func Listen(ctx context.Context, config Config) (ETCDConfig, error) {
5253

5354
leaderelect, backend, err := getKineStorageBackend(ctx, driver, dsn, config)
5455
if err != nil {
55-
return ETCDConfig{}, err
56+
return ETCDConfig{}, errors.Wrap(err, "building kine")
5657
}
5758

5859
if err := backend.Start(ctx); err != nil {
59-
return ETCDConfig{}, err
60+
return ETCDConfig{}, errors.Wrap(err, "starting kine backend")
6061
}
6162

6263
listen := config.Listener
@@ -123,13 +124,13 @@ func getKineStorageBackend(ctx context.Context, driver, dsn string, cfg Config)
123124
switch driver {
124125
case SQLiteBackend:
125126
leaderElect = false
126-
backend, err = sqlite.New(dsn)
127+
backend, err = sqlite.New(ctx, dsn)
127128
case DQLiteBackend:
128129
backend, err = dqlite.New(ctx, dsn)
129130
case PostgresBackend:
130-
backend, err = pgsql.New(dsn, cfg.Config)
131+
backend, err = pgsql.New(ctx, dsn, cfg.Config)
131132
case MySQLBackend:
132-
backend, err = mysql.New(dsn, cfg.Config)
133+
backend, err = mysql.New(ctx, dsn, cfg.Config)
133134
default:
134135
return false, nil, fmt.Errorf("storage backend is not defined")
135136
}

0 commit comments

Comments
 (0)