Merge pull request #20279 from ahrtr/20250704_verify_watch

[release-3.5] Add verification to verify the watch response have a bigger revision than startRev

Author: ahrtr

server/mvcc/watchable_store.go

@@ -15,6 +15,7 @@
 package mvcc
 
 import (
+	"fmt"
 	"sync"
 	"time"
 
@@ -24,6 +25,7 @@ import (
 	"go.etcd.io/etcd/server/v3/lease"
 	"go.etcd.io/etcd/server/v3/mvcc/backend"
 	"go.etcd.io/etcd/server/v3/mvcc/buckets"
+	"go.etcd.io/etcd/server/v3/verify"
 
 	"go.uber.org/zap"
 )
@@ -571,6 +573,16 @@ func (w *watcher) send(wr WatchResponse) bool {
 		wr.Events = ne
 	}
 
+	if verify.VerifyEnabled() {
+		if w.startRev > 0 {
+			for _, ev := range wr.Events {
+				if ev.Kv.ModRevision < w.startRev {
+					panic(fmt.Sprintf("Event.ModRevision(%d) is less than the w.startRev(%d) for watchID: %d", ev.Kv.ModRevision, w.startRev, w.id))
+				}
+			}
+		}
+	}
+
 	// if all events are filtered out, we should send nothing.
 	if !progressEvent && len(wr.Events) == 0 {
 		return true