internal/core/adt: add generation tracking to prevent stale context usage

This change adds a generation counter to OpContext and nodeContext to
detect when stale nodeContexts are being used across different evaluation
contexts. Each OpContext gets a unique generation ID, and nodeContexts
inherit this generation when created.

This helps identify bugs where nodeContexts from previous evaluations
are incorrectly reused, which can lead to evaluation errors and
inconsistent results.

Down the line we will use this to invalidate
closedness info in conjuncts.

Signed-off-by: Marcel van Lohuizen <[email protected]>
Change-Id: I860b10df46c3b35c6317dcd686ebac8cc061ed46
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1218834
TryBot-Result: CUEcueckoo <[email protected]>
Unity-Result: CUE porcuepine <[email protected]>
Reviewed-by: Daniel Martí <[email protected]>

Author: mpvl

cmd/cue/cmd/testdata/script/stats.txtar

@@ -54,6 +54,7 @@ contents overwritten
         "ConjunctInfos": 6,
         "MaxConjunctInfos": 2,
         "MaxReqSets": 0,
+        "GenerationMismatch": 0,
         "Freed": 6,
         "Reused": 0,
         "Allocs": 6,
@@ -66,19 +67,20 @@ contents overwritten
 }
 -- out/stats.cue --
 CUE: {
-	EvalVersion:      3
-	Unifications:     4
-	Disjuncts:        2
-	Notifications:    0
-	Conjuncts:        8
-	NumCloseIDs:      2
-	ConjunctInfos:    6
-	MaxConjunctInfos: 2
-	MaxReqSets:       0
-	Freed:            6
-	Reused:           0
-	Allocs:           6
-	Retained:         0
+	EvalVersion:        3
+	Unifications:       4
+	Disjuncts:          2
+	Notifications:      0
+	Conjuncts:          8
+	NumCloseIDs:        2
+	ConjunctInfos:      6
+	MaxConjunctInfos:   2
+	MaxReqSets:         0
+	GenerationMismatch: 0
+	Freed:              6
+	Reused:             0
+	Allocs:             6
+	Retained:           0
 }
 Go: {
 	AllocBytes:   300456
@@ -95,6 +97,7 @@ CUE:
   ConjunctInfos: 6
   MaxConjunctInfos: 2
   MaxReqSets: 0
+  GenerationMismatch: 0
   Freed: 6
   Reused: 0
   Allocs: 6
@@ -114,6 +117,7 @@ Go:
         "ConjunctInfos": 6,
         "MaxConjunctInfos": 2,
         "MaxReqSets": 0,
+        "GenerationMismatch": 0,
         "Freed": 6,
         "Reused": 0,
         "Allocs": 6,

cue/stats/stats.go

@@ -75,6 +75,14 @@ type Counts struct {
 	MaxConjunctInfos int64 // Maximum number of conjunct infos in a node
 	MaxReqSets       int64 // Number of replace IDs processed
 
+	// Exception counters
+	//
+	// These counters track exceptional conditions that occur during evaluation.
+
+	// GenerationMismatch indicates the number of times a node was unified
+	// with a different generation than the one it was created in.
+	GenerationMismatch int64 // Number of exceptional unification cases
+
 	// Buffer counters
 	//
 	// Each unification and disjunct operation is associated with an object
@@ -113,6 +121,8 @@ func (c *Counts) Add(other Counts) {
 	c.Disjuncts += other.Disjuncts
 	c.Notifications += other.Notifications
 
+	c.GenerationMismatch += other.GenerationMismatch
+
 	c.NumCloseIDs += other.NumCloseIDs
 	c.ConjunctInfos += other.ConjunctInfos
 	if other.MaxConjunctInfos > c.MaxConjunctInfos {
@@ -133,6 +143,7 @@ func (c Counts) Since(start Counts) Counts {
 	c.Conjuncts -= start.Conjuncts
 	c.Disjuncts -= start.Disjuncts
 	c.Notifications -= start.Notifications
+	c.GenerationMismatch -= start.GenerationMismatch
 	c.NumCloseIDs -= start.NumCloseIDs
 
 	c.ConjunctInfos -= start.ConjunctInfos
@@ -166,7 +177,9 @@ Retain: {{.Retained}}
 Unifications: {{.Unifications}}
 Conjuncts:    {{.Conjuncts}}
 Disjuncts:    {{.Disjuncts}}{{if .Notifications}}
-Notifications: {{.Notifications}}{{end}}{{if .NumCloseIDs}}
+Notifications: {{.Notifications}}{{end}}{{if .GenerationMismatch}}
+
+GenerationMismatch: {{.GenerationMismatch}}{{end}}{{if .NumCloseIDs}}
 
 NumCloseIDs: {{.NumCloseIDs}}{{end}}{{if or (ge .MaxReqSets 150) (ge .MaxConjunctInfos 8)}}
 

internal/core/adt/closed.go

@@ -309,7 +309,7 @@ type closeStats struct {
 	// the other fields of this closeStats value are only valid if generation
 	// is equal to the generation in OpContext. This allows for lazy
 	// initialization of closeStats.
-	generation int
+	generation uint64
 
 	// These counts keep track of how many required child nodes need to be
 	// completed before this node is accepted.
@@ -352,7 +352,7 @@ func Accept(ctx *OpContext, n *Vertex, f Feature) (found, required bool) {
 	if ctx.isDevVersion() {
 		return n.accept(ctx, f), true
 	}
-	ctx.generation++
+	ctx.opID++
 	ctx.todo = nil
 
 	var optionalTypes OptionalType
@@ -488,8 +488,8 @@ func getScratch(ctx *OpContext, s *closeInfo) *closeStats {
 		m[s] = x
 	}
 
-	if x.generation != ctx.generation {
-		*x = closeStats{generation: ctx.generation}
+	if x.generation != ctx.opID {
+		*x = closeStats{generation: ctx.opID}
 	}
 
 	return x

internal/core/adt/context.go

@@ -18,6 +18,7 @@ import (
 	"fmt"
 	"reflect"
 	"regexp"
+	"sync/atomic"
 
 	"github.com/cockroachdb/apd/v3"
 	"golang.org/x/text/encoding/unicode"
@@ -59,13 +60,16 @@ type Config struct {
 	Format func(Runtime, Node) string
 }
 
+var contextGeneration atomic.Uint64
+
 // New creates an operation context.
 func New(v *Vertex, cfg *Config) *OpContext {
 	if cfg.Runtime == nil {
 		panic("nil Runtime")
 	}
 
 	ctx := &OpContext{
+		opID:        contextGeneration.Add(1),
 		Runtime:     cfg.Runtime,
 		Format:      cfg.Format,
 		vertex:      v,
@@ -76,6 +80,9 @@ func New(v *Vertex, cfg *Config) *OpContext {
 	if v != nil {
 		ctx.e = &Environment{Up: nil, Vertex: v}
 	}
+	if ctx.LogEval > 0 {
+		ctx.Logf(v, "New context at opID %d", ctx.opID)
+	}
 	return ctx
 }
 
@@ -142,9 +149,9 @@ type OpContext struct {
 	// TODO(perf): have two generations: one for each pass of the closedness
 	// algorithm, so that the results of the first pass can be reused for all
 	// features of a node.
-	generation int
-	closed     map[*closeInfo]*closeStats
-	todo       *closeStats
+	opID   uint64
+	closed map[*closeInfo]*closeStats
+	todo   *closeStats
 
 	// evalDepth indicates the current depth of evaluation. It is used to
 	// detect structural cycles and their severity.s

internal/core/adt/eval.go

@@ -1013,6 +1013,10 @@ type arcKey struct {
 type nodeContext struct {
 	nextFree *nodeContext
 
+	// opID is assigned the opID of the OpContext upon creation.
+	// This allows checking that we are not using stale nodeContexts.
+	opID uint64
+
 	// refCount:
 	// evalv2: keeps track of all current usages of the node, such that the
 	//    node can be freed when the counter reaches zero.
@@ -1406,6 +1410,7 @@ func (c *OpContext) newNodeContext(node *Vertex) *nodeContext {
 		}
 	}
 
+	n.opID = c.opID
 	n.scheduler.node = n
 	n.underlying = node
 	if p := node.Parent; p != nil && p.state != nil {

internal/core/adt/unify.go

@@ -145,6 +145,10 @@ func (v *Vertex) unify(c *OpContext, needs condition, mode runMode, checkTypos b
 		return false
 	}
 
+	if n := v.state; n != nil && n.ctx.opID != c.opID {
+		c.stats.GenerationMismatch++
+	}
+
 	// Note that the state of a node can be removed before the node is.
 	// This happens with the close builtin, for instance.
 	// See TestFromAPI in pkg export.