Skip to content

Commit 0534ff6

Browse files
XuPeng-SHCabbage74
XuPeng-SH
authored and
Cabbage74
committed
enhance expr format (matrixorigin#22089)
If the ColPos and ColName in pushdown's ColExpr do not match, panic. If ColName is empty, preferentially use ColPos Approved by: @aunjgr, @gouhongshen
1 parent a2b221f commit 0534ff6

File tree

5 files changed

+135
-47
lines changed

5 files changed

+135
-47
lines changed

‎pkg/sql/plan/utils.go

Lines changed: 47 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@ import (
4646
"github.com/matrixorigin/matrixone/pkg/sql/util"
4747
"github.com/matrixorigin/matrixone/pkg/stage"
4848
"github.com/matrixorigin/matrixone/pkg/stage/stageutil"
49+
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
4950
"github.com/matrixorigin/matrixone/pkg/vm/process"
5051
)
5152

@@ -1939,47 +1940,84 @@ func PkColByTableDef(tblDef *plan.TableDef) *plan.ColDef {
19391940
return pkCol
19401941
}
19411942

1942-
func FormatExprs(exprs []*plan.Expr) string {
1943+
type FormatOption struct {
1944+
ExpandVec bool
1945+
ExpandVecMaxLen int
1946+
1947+
// <=0 means no limit
1948+
MaxDepth int
1949+
}
1950+
1951+
func FormatExprs(exprs []*plan.Expr, option FormatOption) string {
1952+
return FormatExprsInConsole(exprs, option)
1953+
}
1954+
1955+
func FormatExpr(expr *plan.Expr, option FormatOption) string {
1956+
return FormatExprInConsole(expr, option)
1957+
}
1958+
1959+
func FormatExprsInConsole(exprs []*plan.Expr, option FormatOption) string {
19431960
var w bytes.Buffer
19441961
for _, expr := range exprs {
1945-
w.WriteString(FormatExpr(expr))
1962+
w.WriteString(FormatExpr(expr, option))
19461963
w.WriteByte('\n')
19471964
}
19481965
return w.String()
19491966
}
19501967

1951-
func FormatExpr(expr *plan.Expr) string {
1968+
func FormatExprInConsole(expr *plan.Expr, option FormatOption) string {
19521969
var w bytes.Buffer
1953-
doFormatExpr(expr, &w, 0)
1970+
doFormatExprInConsole(expr, &w, 0, option)
19541971
return w.String()
19551972
}
19561973

1957-
func doFormatExpr(expr *plan.Expr, out *bytes.Buffer, depth int) {
1974+
func doFormatExprInConsole(expr *plan.Expr, out *bytes.Buffer, depth int, option FormatOption) {
19581975
out.WriteByte('\n')
19591976
prefix := strings.Repeat("\t", depth)
1977+
if depth >= option.MaxDepth && option.MaxDepth > 0 {
1978+
out.WriteString(fmt.Sprintf("%s...", prefix))
1979+
return
1980+
}
19601981
switch t := expr.Expr.(type) {
19611982
case *plan.Expr_Col:
1962-
out.WriteString(fmt.Sprintf("%sExpr_Col(%s)", prefix, t.Col.Name))
1983+
out.WriteString(fmt.Sprintf("%sExpr_Col(%s.%d)", prefix, t.Col.Name, t.Col.ColPos))
19631984
case *plan.Expr_Lit:
19641985
out.WriteString(fmt.Sprintf("%sExpr_C(%s)", prefix, t.Lit.String()))
19651986
case *plan.Expr_F:
19661987
out.WriteString(fmt.Sprintf("%sExpr_F(\n%s\tFunc[\"%s\"](nargs=%d)", prefix, prefix, t.F.Func.ObjName, len(t.F.Args)))
19671988
for _, arg := range t.F.Args {
1968-
doFormatExpr(arg, out, depth+1)
1989+
doFormatExprInConsole(arg, out, depth+1, option)
19691990
}
19701991
out.WriteString(fmt.Sprintf("\n%s)", prefix))
19711992
case *plan.Expr_P:
19721993
out.WriteString(fmt.Sprintf("%sExpr_P(%d)", prefix, t.P.Pos))
19731994
case *plan.Expr_T:
19741995
out.WriteString(fmt.Sprintf("%sExpr_T(%s)", prefix, t.T.String()))
19751996
case *plan.Expr_Vec:
1976-
out.WriteString(fmt.Sprintf("%sExpr_Vec(len=%d)", prefix, t.Vec.Len))
1997+
if option.ExpandVec {
1998+
expandVecMaxLen := option.ExpandVecMaxLen
1999+
if expandVecMaxLen <= 0 {
2000+
expandVecMaxLen = 1
2001+
}
2002+
var (
2003+
vecStr string
2004+
vec vector.Vector
2005+
)
2006+
if err := vec.UnmarshalBinary(t.Vec.Data); err != nil {
2007+
vecStr = fmt.Sprintf("error: %s", err.Error())
2008+
} else {
2009+
vecStr = common.MoVectorToString(&vec, expandVecMaxLen)
2010+
}
2011+
out.WriteString(fmt.Sprintf("%sExpr_Vec(%s)", prefix, vecStr))
2012+
} else {
2013+
out.WriteString(fmt.Sprintf("%sExpr_Vec(len=%d)", prefix, t.Vec.Len))
2014+
}
19772015
case *plan.Expr_Fold:
19782016
out.WriteString(fmt.Sprintf("%sExpr_Fold(id=%d)", prefix, t.Fold.Id))
19792017
case *plan.Expr_List:
19802018
out.WriteString(fmt.Sprintf("%sExpr_List(len=%d)", prefix, len(t.List.List)))
19812019
for _, arg := range t.List.List {
1982-
doFormatExpr(arg, out, depth+1)
2020+
doFormatExprInConsole(arg, out, depth+1, option)
19832021
}
19842022
default:
19852023
out.WriteString(fmt.Sprintf("%sExpr_Unknown(%s)", prefix, expr.String()))

‎pkg/vm/engine/disttae/txn_table.go

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -715,7 +715,13 @@ func (tbl *txnTable) doRanges(ctx context.Context, rangesParam engine.RangesPara
715715
logutil.Info(
716716
"TXN-FILTER-RANGE-LOG",
717717
zap.String("name", tbl.tableDef.Name),
718-
zap.String("exprs", plan2.FormatExprs(rangesParam.BlockFilters)),
718+
zap.String("exprs", plan2.FormatExprs(
719+
rangesParam.BlockFilters, plan2.FormatOption{
720+
ExpandVec: true,
721+
ExpandVecMaxLen: 2,
722+
MaxDepth: 5,
723+
},
724+
)),
719725
zap.Uint64("tbl-id", tbl.tableId),
720726
zap.String("txn", tbl.db.op.Txn().DebugString()),
721727
zap.Int("blocks", blocks.Len()),
@@ -839,7 +845,14 @@ func (tbl *txnTable) rangesOnePart(
839845
logutil.Info(
840846
"SLOW-RANGES:",
841847
zap.String("table", tbl.tableDef.Name),
842-
zap.String("exprs", plan2.FormatExprs(rangesParam.BlockFilters)),
848+
zap.String("exprs", plan2.FormatExprs(
849+
rangesParam.BlockFilters,
850+
plan2.FormatOption{
851+
ExpandVec: true,
852+
ExpandVecMaxLen: 2,
853+
MaxDepth: 5,
854+
},
855+
)),
843856
)
844857
}
845858

‎pkg/vm/engine/readutil/expr_filter.go

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -428,7 +428,7 @@ func CompileFilterExpr(
428428
canCompile = false
429429
return
430430
}
431-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
431+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
432432
_, isSorted := isSortedKey(colDef)
433433
if isSorted {
434434
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -462,7 +462,7 @@ func CompileFilterExpr(
462462
canCompile = false
463463
return
464464
}
465-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
465+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
466466
_, isSorted := isSortedKey(colDef)
467467
if isSorted {
468468
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -501,7 +501,7 @@ func CompileFilterExpr(
501501
canCompile = false
502502
return
503503
}
504-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
504+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
505505
_, isSorted := isSortedKey(colDef)
506506
if isSorted {
507507
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -540,7 +540,7 @@ func CompileFilterExpr(
540540
canCompile = false
541541
return
542542
}
543-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
543+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
544544
_, isSorted := isSortedKey(colDef)
545545
if isSorted {
546546
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -574,7 +574,7 @@ func CompileFilterExpr(
574574
canCompile = false
575575
return
576576
}
577-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
577+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
578578
isPK, isSorted := isSortedKey(colDef)
579579
if isSorted {
580580
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -608,7 +608,7 @@ func CompileFilterExpr(
608608
canCompile = false
609609
return
610610
}
611-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
611+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
612612
_, isSorted := isSortedKey(colDef)
613613
if isSorted {
614614
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -641,7 +641,7 @@ func CompileFilterExpr(
641641
canCompile = false
642642
return
643643
}
644-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
644+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
645645
_, isSorted := isSortedKey(colDef)
646646
if isSorted {
647647
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -675,7 +675,7 @@ func CompileFilterExpr(
675675
}
676676
vec := vector.NewVec(types.T_any.ToType())
677677
_ = vec.UnmarshalBinary(val)
678-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
678+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
679679
isPK, isSorted := isSortedKey(colDef)
680680
if isSorted {
681681
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -712,7 +712,7 @@ func CompileFilterExpr(
712712
canCompile = false
713713
return
714714
}
715-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
715+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
716716
fastFilterOp = nil
717717
loadOp = loadMetadataOnlyOpFactory(fs)
718718
seqNum := colDef.Seqnum
@@ -733,7 +733,7 @@ func CompileFilterExpr(
733733
canCompile = false
734734
return
735735
}
736-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
736+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
737737
fastFilterOp = nil
738738
loadOp = loadMetadataOnlyOpFactory(fs)
739739
seqNum := colDef.Seqnum
@@ -755,7 +755,7 @@ func CompileFilterExpr(
755755
}
756756
vec := vector.NewVec(types.T_any.ToType())
757757
_ = vec.UnmarshalBinary(val)
758-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
758+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
759759
isPK, isSorted := isSortedKey(colDef)
760760
if isSorted {
761761
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {
@@ -809,7 +809,7 @@ func CompileFilterExpr(
809809
canCompile = false
810810
return
811811
}
812-
colDef := getColDefByName(colExpr.Col.Name, tableDef)
812+
colDef := getColDefByName(expr, colExpr.Col.Name, colExpr.Col.ColPos, tableDef)
813813
isPK, isSorted := isSortedKey(colDef)
814814
if isSorted {
815815
fastFilterOp = func(obj *objectio.ObjectStats) (bool, error) {

‎pkg/vm/engine/readutil/expr_util.go

Lines changed: 51 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,8 @@ import (
1818
"context"
1919
"strings"
2020

21+
"go.uber.org/zap"
22+
2123
"github.com/matrixorigin/matrixone/pkg/common/mpool"
2224
"github.com/matrixorigin/matrixone/pkg/container/batch"
2325
"github.com/matrixorigin/matrixone/pkg/container/types"
@@ -28,6 +30,7 @@ import (
2830
plan2 "github.com/matrixorigin/matrixone/pkg/sql/plan"
2931
"github.com/matrixorigin/matrixone/pkg/sql/plan/function"
3032
"github.com/matrixorigin/matrixone/pkg/sql/plan/rule"
33+
"github.com/matrixorigin/matrixone/pkg/vm/engine/tae/common"
3134
"github.com/matrixorigin/matrixone/pkg/vm/process"
3235
)
3336

@@ -59,7 +62,7 @@ func ConstructInExpr(
5962
)
6063
}
6164

62-
func getColDefByName(name string, tableDef *plan.TableDef) *plan.ColDef {
65+
func getColDefByName(expr *plan.Expr, name string, colPos int32, tableDef *plan.TableDef) *plan.ColDef {
6366
idx := strings.Index(name, ".")
6467
var pos int32
6568
if idx >= 0 {
@@ -68,6 +71,17 @@ func getColDefByName(name string, tableDef *plan.TableDef) *plan.ColDef {
6871
} else {
6972
pos = tableDef.Name2ColIndex[name]
7073
}
74+
common.DoIfDebugEnabled(func() {
75+
if name != tableDef.Cols[colPos].Name {
76+
logutil.Error(
77+
"Bad-ColExpr",
78+
zap.String("col-name", name),
79+
zap.Int32("col-actual-pos", colPos),
80+
zap.Int32("col-expected-pos", pos),
81+
zap.String("col-expr", plan2.FormatExpr(expr, plan2.FormatOption{})),
82+
)
83+
}
84+
})
7185
return tableDef.Cols[pos]
7286
}
7387

@@ -78,6 +92,7 @@ func compPkCol(colName string, pkName string) bool {
7892
}
7993

8094
func evalValue(
95+
expr *plan.Expr,
8196
exprImpl *plan.Expr_F,
8297
tblDef *plan.TableDef,
8398
isVec bool,
@@ -97,22 +112,44 @@ func evalValue(
97112
if !ok {
98113
return false, 0, nil
99114
}
100-
if !compPkCol(col.Col.Name, pkName) {
115+
116+
colName := col.Col.Name
117+
118+
common.DoIfDebugEnabled(func() {
119+
if colName == "" {
120+
logutil.Error(
121+
"Bad-ColExpr",
122+
zap.String("col-name", colName),
123+
zap.String("pk-name", pkName),
124+
zap.String("col-expr", plan2.FormatExpr(expr, plan2.FormatOption{})),
125+
)
126+
}
127+
})
128+
if !compPkCol(colName, pkName) {
101129
return false, 0, nil
102130
}
103131

104-
var colPos int32
105-
if col.Col.Name == "" {
106-
colPos = col.Col.ColPos
107-
logutil.Warnf("colExpr.Col.Name is empty")
132+
var (
133+
colPos int32
134+
idx = strings.Index(colName, ".")
135+
)
136+
if idx == -1 {
137+
colPos = tblDef.Name2ColIndex[colName]
108138
} else {
109-
idx := strings.Index(col.Col.Name, ".")
110-
if idx == -1 {
111-
colPos = tblDef.Name2ColIndex[col.Col.Name]
112-
} else {
113-
colPos = tblDef.Name2ColIndex[col.Col.Name[idx+1:]]
139+
colPos = tblDef.Name2ColIndex[colName[idx+1:]]
140+
}
141+
142+
common.DoIfDebugEnabled(func() {
143+
if colPos != col.Col.ColPos {
144+
logutil.Error(
145+
"Bad-ColExpr",
146+
zap.String("col-name", colName),
147+
zap.Int32("col-actual-pos", col.Col.ColPos),
148+
zap.Int32("col-expected-pos", colPos),
149+
zap.String("col-expr", plan2.FormatExpr(expr, plan2.FormatOption{})),
150+
)
114151
}
115-
}
152+
})
116153

117154
if isVec {
118155
return true, types.T(tblDef.Cols[colPos].Typ.Id), [][]byte{val}
@@ -185,7 +222,7 @@ func getConstBytesFromExpr(exprs []*plan.Expr) ([][]byte, bool) {
185222
vals[idx] = nil
186223
vals[idx] = append(vals[idx], fExpr.Fold.Data...)
187224
} else {
188-
logutil.Warnf("const folded val expr is not a fold expr: %s\n", plan2.FormatExpr(exprs[idx]))
225+
logutil.Warnf("const folded val expr is not a fold expr: %s\n", plan2.FormatExpr(exprs[idx], plan2.FormatOption{}))
189226
return nil, false
190227
}
191228
}
@@ -234,7 +271,7 @@ func mustColVecValueFromBinaryFuncExpr(expr *plan.Expr_F) (*plan.Expr_Col, []byt
234271
return colExpr, fExpr.Fold.Data, ok
235272
}
236273

237-
logutil.Warnf("const folded val expr is not a vec expr: %s\n", plan2.FormatExpr(valExpr))
274+
logutil.Warnf("const folded val expr is not a vec expr: %s\n", plan2.FormatExpr(valExpr, plan2.FormatOption{}))
238275
return nil, nil, false
239276
}
240277

0 commit comments

Comments
 (0)