Skip to content

Commit c55aebf

Browse files
aakash070aakash070
authored
add support for handling unknown expression ids in test result (#1183)
1 parent 746d711 commit c55aebf

File tree

5 files changed

+124
-14
lines changed

5 files changed

+124
-14
lines changed

‎tools/celtest/test_runner.go‎

Lines changed: 82 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ import (
2121
"os"
2222
"path/filepath"
2323
"reflect"
24+
"slices"
2425
"strings"
2526
"testing"
2627

@@ -133,7 +134,7 @@ func TriggerTests(t *testing.T, testRunnerOpts ...TestRunnerOption) {
133134
if err != nil {
134135
t.Fatalf("error creating test runner: %v", err)
135136
}
136-
programs, err := tr.Programs(t)
137+
programs, err := tr.Programs(t, tr.testProgramOptions...)
137138
if err != nil {
138139
t.Fatalf("error creating programs: %v", err)
139140
}
@@ -275,6 +276,7 @@ func DefaultTestSuiteParser(path string) TestRunnerOption {
275276
// - Test Suite File Path: The path to the test suite file.
276277
// - File Descriptor Set Path: The path to the file descriptor set file.
277278
// - test Suite Parser: A parser for a test suite file serialized in Textproto/YAML format.
279+
// - test Program Options: A list of options to be used when creating the CEL programs.
278280
//
279281
// The TestRunner provides the following methods:
280282
// - Programs: Creates a list of CEL programs from the input expressions.
@@ -286,6 +288,7 @@ type TestRunner struct {
286288
TestSuiteFilePath string
287289
FileDescriptorSetPath string
288290
testSuiteParser TestSuiteParser
291+
testProgramOptions []cel.ProgramOption
289292
}
290293

291294
// Test represents a single test case to be executed. It encompasses the following:
@@ -295,12 +298,12 @@ type TestRunner struct {
295298
// returns a TestResult.
296299
type Test struct {
297300
name string
298-
input interpreter.Activation
301+
input interpreter.PartialActivation
299302
resultMatcher func(ref.Val, error) TestResult
300303
}
301304

302305
// NewTest creates a new Test with the provided name, input and result matcher.
303-
func NewTest(name string, input interpreter.Activation, resultMatcher func(ref.Val, error) TestResult) *Test {
306+
func NewTest(name string, input interpreter.PartialActivation, resultMatcher func(ref.Val, error) TestResult) *Test {
304307
return &Test{
305308
name: name,
306309
input: input,
@@ -417,6 +420,17 @@ func fileDescriptorSet(path string) (*descpb.FileDescriptorSet, error) {
417420
return fds, nil
418421
}
419422

423+
// PartialEvalProgramOption returns a TestRunnerOption which enables partial evaluation for the CEL
424+
// program by setting the OptPartialEval eval option.
425+
//
426+
// Note: The test setup uses env.PartialVars() for creating PartialActivation.
427+
func PartialEvalProgramOption() TestRunnerOption {
428+
return func(tr *TestRunner) (*TestRunner, error) {
429+
tr.testProgramOptions = append(tr.testProgramOptions, cel.EvalOptions(cel.OptPartialEval))
430+
return tr, nil
431+
}
432+
}
433+
420434
// Program represents the result of creating CEL programs for the configured expressions in the
421435
// test runner. It encompasses the following:
422436
// - CELProgram - the evaluable CEL program
@@ -461,6 +475,8 @@ func (tr *TestRunner) Programs(t *testing.T, opts ...cel.ProgramOption) ([]Progr
461475

462476
// Tests creates a list of tests from the test suite file and test suite parser configured in the
463477
// test runner.
478+
//
479+
// Note: The test setup uses env.PartialVars() for creating PartialActivation.
464480
func (tr *TestRunner) Tests(t *testing.T) ([]*Test, error) {
465481
if tr.Compiler == nil {
466482
return nil, fmt.Errorf("compiler is not set")
@@ -507,13 +523,14 @@ func (tr *TestRunner) createTestsFromTextproto(t *testing.T, testSuite *conforma
507523
return tests, nil
508524
}
509525

510-
func (tr *TestRunner) createTestInputFromPB(t *testing.T, testCase *conformancepb.TestCase) (interpreter.Activation, error) {
526+
func (tr *TestRunner) createTestInputFromPB(t *testing.T, testCase *conformancepb.TestCase) (interpreter.PartialActivation, error) {
511527
t.Helper()
512528
input := map[string]any{}
513529
e, err := tr.CreateEnv()
514530
if err != nil {
515531
return nil, err
516532
}
533+
var activation interpreter.Activation
517534
if testCase.GetInputContext() != nil {
518535
if len(testCase.GetInput()) != 0 {
519536
return nil, fmt.Errorf("only one of input and input_context can be provided at a time")
@@ -529,15 +546,22 @@ func (tr *TestRunner) createTestInputFromPB(t *testing.T, testCase *conformancep
529546
if err != nil {
530547
return nil, fmt.Errorf("context variable is not a valid proto: %w", err)
531548
}
532-
return cel.ContextProtoVars(ctx.(proto.Message))
549+
activation, err = cel.ContextProtoVars(ctx.(proto.Message))
550+
if err != nil {
551+
return nil, fmt.Errorf("cel.ContextProtoVars() failed: %w", err)
552+
}
533553
case *conformancepb.InputContext_ContextMessage:
534554
refVal := e.CELTypeAdapter().NativeToValue(testInput.ContextMessage)
535555
ctx, err := refVal.ConvertToNative(reflect.TypeOf((*proto.Message)(nil)).Elem())
536556
if err != nil {
537557
return nil, fmt.Errorf("context variable is not a valid proto: %w", err)
538558
}
539-
return cel.ContextProtoVars(ctx.(proto.Message))
559+
activation, err = cel.ContextProtoVars(ctx.(proto.Message))
560+
if err != nil {
561+
return nil, fmt.Errorf("cel.ContextProtoVars() failed: %w", err)
562+
}
540563
}
564+
return e.PartialVars(activation)
541565
}
542566
for k, v := range testCase.GetInput() {
543567
switch v.GetKind().(type) {
@@ -553,7 +577,11 @@ func (tr *TestRunner) createTestInputFromPB(t *testing.T, testCase *conformancep
553577
}
554578
}
555579
}
556-
return interpreter.NewActivation(input)
580+
activation, err = interpreter.NewActivation(input)
581+
if err != nil {
582+
return nil, fmt.Errorf("interpreter.NewActivation(%q) failed: %w", input, err)
583+
}
584+
return e.PartialVars(activation)
557585
}
558586

559587
func (tr *TestRunner) createResultMatcherFromPB(t *testing.T, testCase *conformancepb.TestCase) (func(ref.Val, error) TestResult, error) {
@@ -627,11 +655,34 @@ func (tr *TestRunner) createResultMatcherFromPB(t *testing.T, testCase *conforma
627655
return failureResult
628656
}, nil
629657
case *conformancepb.TestOutput_Unknown:
630-
// TODO: to implement
658+
// Validate that all expected unknown expression ids are returned by the evaluation result.
659+
return func(out ref.Val, err error) TestResult {
660+
expectedUnknownIDs := testOutput.Unknown.GetExprs()
661+
if err == nil && types.IsUnknown(out) {
662+
actualUnknownIDs := out.Value().(*types.Unknown).IDs()
663+
return compareUnknownIDs(expectedUnknownIDs, actualUnknownIDs)
664+
}
665+
return TestResult{Success: false, Wanted: fmt.Sprintf("unknown value %v", expectedUnknownIDs), Error: err}
666+
}, nil
631667
}
632668
return nil, nil
633669
}
634670

671+
func compareUnknownIDs(expectedUnknownIDs, actualUnknownIDs []int64) TestResult {
672+
sortOption := cmp.Transformer("Sort", func(in []int64) []int64 {
673+
out := append([]int64{}, in...)
674+
slices.Sort(out)
675+
return out
676+
})
677+
if diff := cmp.Diff(expectedUnknownIDs, actualUnknownIDs, sortOption); diff != "" {
678+
return TestResult{
679+
Success: false,
680+
Wanted: fmt.Sprintf("unknown value %v", expectedUnknownIDs),
681+
Error: fmt.Errorf("mismatched test output with diff (-got +want):\n%s", diff)}
682+
}
683+
return TestResult{Success: true}
684+
}
685+
635686
func refValueToExprValue(refVal ref.Val) (*exprpb.ExprValue, error) {
636687
if types.IsUnknown(refVal) {
637688
return &exprpb.ExprValue{
@@ -704,8 +755,13 @@ func (tr *TestRunner) createTestsFromYAML(t *testing.T, testSuite *test.Suite) (
704755
return tests, nil
705756
}
706757

707-
func (tr *TestRunner) createTestInput(t *testing.T, testCase *test.Case) (interpreter.Activation, error) {
758+
func (tr *TestRunner) createTestInput(t *testing.T, testCase *test.Case) (interpreter.PartialActivation, error) {
708759
t.Helper()
760+
e, err := tr.CreateEnv()
761+
if err != nil {
762+
return nil, err
763+
}
764+
var activation interpreter.Activation
709765
if testCase.InputContext != nil && testCase.InputContext.ContextExpr != "" {
710766
if len(testCase.Input) != 0 {
711767
return nil, fmt.Errorf("only one of input and input_context can be provided at a time")
@@ -719,7 +775,11 @@ func (tr *TestRunner) createTestInput(t *testing.T, testCase *test.Case) (interp
719775
if err != nil {
720776
return nil, fmt.Errorf("context variable is not a valid proto: %w", err)
721777
}
722-
return cel.ContextProtoVars(ctx.(proto.Message))
778+
activation, err = cel.ContextProtoVars(ctx.(proto.Message))
779+
if err != nil {
780+
return nil, fmt.Errorf("cel.ContextProtoVars() failed: %w", err)
781+
}
782+
return e.PartialVars(activation)
723783
}
724784
input := map[string]any{}
725785
for k, v := range testCase.Input {
@@ -733,7 +793,11 @@ func (tr *TestRunner) createTestInput(t *testing.T, testCase *test.Case) (interp
733793
}
734794
input[k] = v.Value
735795
}
736-
return interpreter.NewActivation(input)
796+
activation, err = interpreter.NewActivation(input)
797+
if err != nil {
798+
return nil, fmt.Errorf("interpreter.NewActivation(%q) failed: %w", input, err)
799+
}
800+
return e.PartialVars(activation)
737801
}
738802

739803
func (tr *TestRunner) createResultMatcher(t *testing.T, testOutput *test.Output) (func(ref.Val, error) TestResult, error) {
@@ -793,7 +857,13 @@ func (tr *TestRunner) createResultMatcher(t *testing.T, testOutput *test.Output)
793857
}, nil
794858
}
795859
if testOutput.UnknownSet != nil {
796-
// TODO: to implement
860+
return func(out ref.Val, err error) TestResult {
861+
if err == nil && types.IsUnknown(out) {
862+
unknownIDs := out.Value().(*types.Unknown).IDs()
863+
return compareUnknownIDs(testOutput.UnknownSet, unknownIDs)
864+
}
865+
return TestResult{Success: false, Wanted: fmt.Sprintf("unknown value %v", testOutput.UnknownSet), Error: err}
866+
}, nil
797867
}
798868
return nil, nil
799869
}

‎tools/celtest/test_runner_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func setupTests() []*testCase {
6767
},
6868
{
6969
name: "raw expression test",
70-
celExpression: "i + fn(j) == 42",
70+
celExpression: "a || i + fn(j) == 42",
7171
testSuitePath: "testdata/raw_expr_tests.yaml",
7272
configPath: "testdata/config.yaml",
7373
opts: []any{fnEnvOption()},
@@ -204,6 +204,7 @@ func TestTriggerTests(t *testing.T) {
204204
DefaultTestSuiteParser(tc.testSuitePath),
205205
AddFileDescriptorSet(tc.fileDescriptorSetPath),
206206
TestExpression(tc.celExpression),
207+
PartialEvalProgramOption(),
207208
)
208209
TriggerTests(t, testOpts...)
209210
})

‎tools/celtest/testdata/config.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,3 +18,5 @@ variables:
1818
type_name: "int"
1919
- name: "j"
2020
type_name: "int"
21+
- name: "a"
22+
type_name: "bool"

‎tools/celtest/testdata/raw_expr.cel‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1 +1 @@
1-
i + fn(j) == 42
1+
a || i + fn(j) == 42

‎tools/celtest/testdata/raw_expr_tests.yaml‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ section:
2222
value: 21
2323
j:
2424
value: 42
25+
a:
26+
value: false
2527
output:
2628
value: true
2729
- name: "false"
@@ -30,5 +32,40 @@ section:
3032
value: 22
3133
j:
3234
value: 42
35+
a:
36+
value: false
3337
output:
3438
value: false
39+
- name: "true a"
40+
input:
41+
a:
42+
value: true
43+
output:
44+
value: true
45+
- name: "unknown"
46+
tests:
47+
- name: "unknown i"
48+
input:
49+
j:
50+
value: 42
51+
a:
52+
value: false
53+
output:
54+
unknown_set:
55+
- 2
56+
- name: "unknown a and j"
57+
input:
58+
i:
59+
value: 21
60+
output:
61+
unknown_set:
62+
- 1
63+
- 5
64+
- name: "unknown a and i"
65+
input:
66+
j:
67+
value: 42
68+
output:
69+
unknown_set:
70+
- 1
71+
- 2

0 commit comments

Comments
 (0)