Skip to content

Commit 1de10fa

Browse files
sitoleCode42Cate
sitole
authored and
Code42Cate
committed
Send sandbox routing data to shared Redis storage via catalog impl (#1298)
1 parent 02d6a4e commit 1de10fa

File tree

24 files changed

+133
-153
lines changed

24 files changed

+133
-153
lines changed

‎packages/api/go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ require (
4848
github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843
4949
github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c
5050
github.com/jackc/pgx/v5 v5.7.4
51-
github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359
51+
github.com/jellydator/ttlcache/v3 v3.4.0
5252
github.com/launchdarkly/go-sdk-common/v3 v3.3.0
5353
github.com/miekg/dns v1.1.63
5454
github.com/oapi-codegen/gin-middleware v1.0.2

‎packages/api/go.sum‎

Lines changed: 2 additions & 2 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/api/internal/dns/server.go‎

Lines changed: 24 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@ import (
1616
"go.uber.org/zap"
1717

1818
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
19-
"github.com/e2b-dev/infra/packages/shared/pkg/smap"
19+
e2bcatalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog"
2020
)
2121

2222
const (
@@ -30,11 +30,9 @@ const defaultRoutingIP = "127.0.0.1"
3030
const cachedDnsPrefix = "sandbox.dns."
3131

3232
type DNS struct {
33-
srv *resolver.Server
34-
33+
srv *resolver.Server
3534
redisCache *cache.Cache
36-
37-
local *smap.Map[string]
35+
catalog e2bcatalog.SandboxesCatalog
3836

3937
closer struct {
4038
once sync.Once
@@ -48,60 +46,60 @@ func New(ctx context.Context, redisClient redis.UniversalClient) *DNS {
4846

4947
if redisClient != nil && !reflect.ValueOf(redisClient).IsNil() {
5048
d.redisCache = cache.New(&cache.Options{Redis: redisClient, LocalCache: cache.NewTinyLFU(10_000, time.Hour)})
49+
d.catalog = e2bcatalog.NewRedisSandboxesCatalog(redisClient)
5150
} else {
52-
d.local = smap.New[string]()
51+
d.catalog = e2bcatalog.NewMemorySandboxesCatalog()
5352
}
5453

5554
return d
5655
}
5756

58-
func (d *DNS) Add(ctx context.Context, sandboxID, ip string) {
59-
switch {
60-
case d.redisCache != nil:
57+
func (d *DNS) Add(ctx context.Context, sandboxID string, info e2bcatalog.SandboxInfo) {
58+
if d.redisCache != nil {
6159
err := d.redisCache.Set(&cache.Item{
6260
Ctx: ctx,
6361
TTL: redisTTL,
6462
Key: d.cacheKey(sandboxID),
65-
Value: ip,
63+
Value: info.OrchestratorIP,
6664
})
6765
if err != nil {
6866
zap.L().Error("error adding DNS item to redis cache", zap.Error(err), logger.WithSandboxID(sandboxID))
6967
}
70-
case d.local != nil:
71-
d.local.Insert(sandboxID, ip)
68+
}
69+
70+
lifetime := time.Duration(info.SandboxMaxLengthInHours) * time.Hour
71+
err := d.catalog.StoreSandbox(ctx, sandboxID, &info, lifetime)
72+
if err != nil {
73+
zap.L().Error("error adding routing record to catalog", zap.Error(err), logger.WithSandboxID(sandboxID))
7274
}
7375
}
7476

75-
func (d *DNS) Remove(ctx context.Context, sandboxID, ip string) {
76-
switch {
77-
case d.redisCache != nil:
77+
func (d *DNS) Remove(ctx context.Context, sandboxID, executionID string) {
78+
if d.redisCache != nil {
7879
if err := d.redisCache.Delete(ctx, d.cacheKey(sandboxID)); err != nil {
7980
zap.L().Debug("removing item from DNS cache", zap.Error(err), logger.WithSandboxID(sandboxID))
8081
}
81-
case d.local != nil:
82-
d.local.RemoveCb(d.cacheKey(sandboxID), func(k string, v string, ok bool) bool { return v == ip })
82+
}
83+
84+
err := d.catalog.DeleteSandbox(ctx, sandboxID, executionID)
85+
if err != nil {
86+
zap.L().Error("error removing routing record from catalog", zap.Error(err), logger.WithSandboxID(sandboxID))
8387
}
8488
}
8589

8690
func (d *DNS) Get(ctx context.Context, sandboxID string) net.IP {
8791
var res string
88-
switch {
89-
case d.redisCache != nil:
92+
93+
if d.redisCache != nil {
9094
if err := d.redisCache.Get(ctx, d.cacheKey(sandboxID), &res); err != nil && !errors.Is(err, cache.ErrCacheMiss) {
9195
zap.L().Error("resolving item from redisCache DNS cache", logger.WithSandboxID(sandboxID), zap.Error(err))
9296
}
93-
case d.local != nil:
94-
var ok bool
95-
res, ok = d.local.Get(d.cacheKey(sandboxID))
96-
if !ok {
97-
zap.L().Debug("item not found in local DNS cache", logger.WithSandboxID(sandboxID))
98-
}
9997
}
10098

10199
addr := net.ParseIP(res)
102100
if addr == nil {
103101
if res != "" {
104-
zap.L().Error("malformed address in cache", zap.Bool("local", d.local != nil), zap.String("addr", res))
102+
zap.L().Error("malformed address in cache", zap.Bool("mocked", d.redisCache == nil), zap.String("addr", res))
105103
}
106104

107105
addr = net.ParseIP(defaultRoutingIP)
@@ -160,8 +158,6 @@ func (d *DNS) handleDNSRequest(ctx context.Context, w resolver.ResponseWriter, r
160158

161159
var errOnStartup = errors.New("failed to start DNS server")
162160

163-
func CheckErrOnStartup(err error) bool { return errors.Is(err, errOnStartup) }
164-
165161
func (d *DNS) Start(ctx context.Context, address string, port uint16) {
166162
if d.srv != nil {
167163
return

‎packages/api/internal/orchestrator/delete_instance.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -78,7 +78,7 @@ func (o *Orchestrator) removeSandboxFromNode(ctx context.Context, sbx sandbox.Sa
7878
// Remove the sandbox resources after the sandbox is deleted
7979
defer node.RemoveSandbox(sbx)
8080

81-
o.dns.Remove(ctx, sbx.SandboxID, node.IPAddress)
81+
o.dns.Remove(ctx, sbx.SandboxID, sbx.ExecutionID)
8282

8383
sbxlogger.I(sbx).Debug("Removing sandbox",
8484
zap.Bool("auto_pause", sbx.AutoPause),

‎packages/api/internal/orchestrator/lifecycle.go‎

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,13 @@ package orchestrator
22

33
import (
44
"context"
5+
"time"
56

67
"go.uber.org/zap"
78

89
"github.com/e2b-dev/infra/packages/api/internal/sandbox"
910
"github.com/e2b-dev/infra/packages/shared/pkg/logger"
11+
e2bcatalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog"
1012
)
1113

1214
func (o *Orchestrator) observeTeamSandbox(ctx context.Context, sandbox sandbox.Sandbox, created bool) {
@@ -20,6 +22,15 @@ func (o *Orchestrator) addToNode(ctx context.Context, sandbox sandbox.Sandbox, _
2022
} else {
2123
node.AddSandbox(sandbox)
2224

23-
o.dns.Add(ctx, sandbox.SandboxID, node.IPAddress)
25+
info := e2bcatalog.SandboxInfo{
26+
OrchestratorID: node.Metadata().ServiceInstanceID,
27+
OrchestratorIP: node.IPAddress,
28+
ExecutionID: sandbox.ExecutionID,
29+
30+
SandboxStartedAt: sandbox.StartTime,
31+
SandboxMaxLengthInHours: int64(sandbox.MaxInstanceLength / time.Hour),
32+
}
33+
34+
o.dns.Add(ctx, sandbox.SandboxID, info)
2435
}
2536
}

‎packages/client-proxy/go.mod‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,9 @@ require (
2727
github.com/gin-contrib/size v1.0.2
2828
github.com/gin-contrib/zap v1.1.5
2929
github.com/gin-gonic/gin v1.10.1
30-
github.com/go-redsync/redsync/v4 v4.13.0
3130
github.com/google/uuid v1.6.0
3231
github.com/grafana/loki v0.0.0-20250609195516-7b805ba7c843
3332
github.com/hashicorp/nomad/api v0.0.0-20231208134655-099ee06a607c
34-
github.com/jellydator/ttlcache/v3 v3.3.1-0.20250207140243-aefc35918359
3533
github.com/miekg/dns v1.1.63
3634
github.com/oapi-codegen/gin-middleware v1.0.2
3735
github.com/redis/go-redis/v9 v9.12.1
@@ -135,6 +133,7 @@ require (
135133
github.com/hashicorp/serf v0.10.1 // indirect
136134
github.com/huandu/xstrings v1.4.0 // indirect
137135
github.com/imdario/mergo v0.3.16 // indirect
136+
github.com/jellydator/ttlcache/v3 v3.4.0 // indirect
138137
github.com/jmespath/go-jmespath v0.4.0 // indirect
139138
github.com/josharian/intern v1.0.0 // indirect
140139
github.com/jpillora/backoff v1.0.0 // indirect

‎packages/client-proxy/go.sum‎

Lines changed: 2 additions & 14 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/client-proxy/internal/edge-pass-through/events.go‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import (
1010
"google.golang.org/grpc/metadata"
1111
"google.golang.org/grpc/status"
1212

13-
"github.com/e2b-dev/infra/packages/proxy/internal/edge/sandboxes"
1413
"github.com/e2b-dev/infra/packages/shared/pkg/edge"
14+
catalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog"
1515
)
1616

1717
func (s *NodePassThroughServer) eventsHandler(ctx context.Context, md metadata.MD) (func(error), error) {
@@ -41,12 +41,19 @@ func (s *NodePassThroughServer) catalogCreateEventHandler(ctx context.Context, m
4141
return nil, err
4242
}
4343

44+
o, ok := s.nodes.GetOrchestrator(c.OrchestratorID)
45+
if !ok {
46+
return nil, status.Errorf(codes.InvalidArgument, "orchestrator %s not found", c.OrchestratorID)
47+
}
48+
4449
err = s.catalog.StoreSandbox(
4550
ctx,
4651
c.SandboxID,
47-
&sandboxes.SandboxInfo{
48-
OrchestratorID: c.OrchestratorID,
49-
ExecutionID: c.ExecutionID,
52+
&catalog.SandboxInfo{
53+
OrchestratorID: c.OrchestratorID,
54+
OrchestratorIP: o.GetInfo().IP,
55+
ExecutionID: c.ExecutionID,
56+
5057
SandboxStartedAt: c.SandboxStartTime,
5158
SandboxMaxLengthInHours: c.SandboxMaxLengthInHours,
5259
},

‎packages/client-proxy/internal/edge-pass-through/proxy.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -14,14 +14,14 @@ import (
1414
"github.com/e2b-dev/infra/packages/proxy/internal/edge/authorization"
1515
e2binfo "github.com/e2b-dev/infra/packages/proxy/internal/edge/info"
1616
e2borchestrators "github.com/e2b-dev/infra/packages/proxy/internal/edge/pool"
17-
"github.com/e2b-dev/infra/packages/proxy/internal/edge/sandboxes"
1817
"github.com/e2b-dev/infra/packages/shared/pkg/consts"
1918
api "github.com/e2b-dev/infra/packages/shared/pkg/http/edge"
19+
catalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog"
2020
)
2121

2222
type NodePassThroughServer struct {
2323
nodes *e2borchestrators.OrchestratorsPool
24-
catalog sandboxes.SandboxesCatalog
24+
catalog catalog.SandboxesCatalog
2525

2626
info *e2binfo.ServiceInfo
2727

@@ -39,7 +39,7 @@ func NewNodePassThroughServer(
3939
nodes *e2borchestrators.OrchestratorsPool,
4040
info *e2binfo.ServiceInfo,
4141
authorization authorization.AuthorizationService,
42-
catalog sandboxes.SandboxesCatalog,
42+
catalog catalog.SandboxesCatalog,
4343
) *grpc.Server {
4444
nodePassThrough := &NodePassThroughServer{
4545
authorization: authorization,

‎packages/client-proxy/internal/edge/handlers/sandbox-catalog-create-entry.go‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,9 +8,9 @@ import (
88
"github.com/gin-gonic/gin"
99
"go.uber.org/zap"
1010

11-
"github.com/e2b-dev/infra/packages/proxy/internal/edge/sandboxes"
1211
"github.com/e2b-dev/infra/packages/shared/pkg/http/edge"
1312
l "github.com/e2b-dev/infra/packages/shared/pkg/logger"
13+
catalog "github.com/e2b-dev/infra/packages/shared/pkg/sandbox-catalog"
1414
"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
1515
)
1616

@@ -20,16 +20,24 @@ func (a *APIStore) V1SandboxCatalogCreate(c *gin.Context) {
2020
body, err := parseBody[api.V1SandboxCatalogCreateJSONRequestBody](ctx, c)
2121
if err != nil {
2222
a.sendAPIStoreError(c, http.StatusBadRequest, fmt.Sprintf("Error when parsing request: %s", err))
23-
telemetry.ReportCriticalError(ctx, "error when parsing request", err)
23+
telemetry.ReportError(ctx, "error when parsing request", err)
2424
return
2525
}
2626

2727
_, span := tracer.Start(ctx, "create-sandbox-catalog-entry-handler")
2828
defer span.End()
2929

30+
o, ok := a.orchestratorPool.GetOrchestrator(body.OrchestratorID)
31+
if !ok {
32+
a.sendAPIStoreError(c, http.StatusBadRequest, "Orchestrator not found")
33+
telemetry.ReportError(ctx, "orchestrator not found", nil)
34+
return
35+
}
36+
3037
sbxMaxLifetime := time.Duration(body.SandboxMaxLength) * time.Hour
31-
sbxInfo := &sandboxes.SandboxInfo{
38+
sbxInfo := &catalog.SandboxInfo{
3239
OrchestratorID: body.OrchestratorID,
40+
OrchestratorIP: o.GetInfo().IP,
3341
ExecutionID: body.ExecutionID,
3442

3543
SandboxMaxLengthInHours: body.SandboxMaxLength,

0 commit comments

Comments
 (0)