Skip to content

Commit 9c0f594

Browse files
xiaocai2333xiaocai2333
authored
feat: [cherry-pick]The expression supports filling elements through templates (#37058)
issue: #36672 master pr: #37033 milvus-proto pr: milvus-io/milvus-proto#332 Signed-off-by: Cai Zhang <[email protected]>
1 parent 4cb5b2c commit 9c0f594

30 files changed

+2733
-1371
lines changed

‎go.mod

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ require (
2626
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
2727
github.com/klauspost/compress v1.17.9
2828
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
29-
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13
29+
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b
3030
github.com/minio/minio-go/v7 v7.0.73
3131
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
3232
github.com/prometheus/client_golang v1.14.0

‎go.sum

+2
Original file line numberDiff line numberDiff line change
@@ -610,6 +610,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
610610
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
611611
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13 h1:79fAk5rUFNl7A5RhYUlxcZTHAY31lygULC6QUvpBrUo=
612612
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.13/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
613+
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b h1:Q4f3a6Y+HZFEadEx3pUXTZmBwrS3QRvkEHWNd8fjDtI=
614+
github.com/milvus-io/milvus-proto/go-api/v2 v2.4.14-0.20241025033521-13fc3509ee0b/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
613615
github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70 h1:Z+sp64fmAOxAG7mU0dfVOXvAXlwRB0c8a96rIM5HevI=
614616
github.com/milvus-io/milvus-storage/go v0.0.0-20231227072638-ebd0b8e56d70/go.mod h1:GPETMcTZq1gLY1WA6Na5kiNAKnq8SEMMiVKUZrM3sho=
615617
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=

‎internal/parser/planparserv2/Plan.g4

+11-8
Original file line numberDiff line numberDiff line change
@@ -5,19 +5,20 @@ expr:
55
| FloatingConstant # Floating
66
| BooleanConstant # Boolean
77
| StringLiteral # String
8-
| Identifier # Identifier
8+
| (Identifier|Meta) # Identifier
99
| JSONIdentifier # JSONIdentifier
10+
| LBRACE Identifier RBRACE # TemplateVariable
1011
| '(' expr ')' # Parens
1112
| '[' expr (',' expr)* ','? ']' # Array
13+
| EmptyArray # EmptyArray
1214
| expr LIKE StringLiteral # Like
1315
| expr POW expr # Power
1416
| op = (ADD | SUB | BNOT | NOT) expr # Unary
1517
// | '(' typeName ')' expr # Cast
1618
| expr op = (MUL | DIV | MOD) expr # MulDivMod
1719
| expr op = (ADD | SUB) expr # AddSub
1820
| expr op = (SHL | SHR) expr # Shift
19-
| expr op = (IN | NIN) ('[' expr (',' expr)* ','? ']') # Term
20-
| expr op = (IN | NIN) EmptyTerm # EmptyTerm
21+
| expr op = NOT? IN expr # Term
2122
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
2223
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
2324
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
@@ -42,6 +43,8 @@ expr:
4243
// INT64: 'int64';
4344
// FLOAT: 'float';
4445
// DOUBLE: 'double';
46+
LBRACE: '{';
47+
RBRACE: '}';
4548

4649
LT: '<';
4750
LE: '<=';
@@ -71,9 +74,8 @@ OR: '||' | 'or';
7174
BNOT: '~';
7275
NOT: '!' | 'not';
7376

74-
IN: 'in';
75-
NIN: 'not in';
76-
EmptyTerm: '[' (Whitespace | Newline)* ']';
77+
IN: 'in' | 'IN';
78+
EmptyArray: '[' (Whitespace | Newline)* ']';
7779

7880
JSONContains: 'json_contains' | 'JSON_CONTAINS';
7981
JSONContainsAll: 'json_contains_all' | 'JSON_CONTAINS_ALL';
@@ -96,10 +98,11 @@ FloatingConstant:
9698
DecimalFloatingConstant
9799
| HexadecimalFloatingConstant;
98100

99-
Identifier: Nondigit (Nondigit | Digit)* | '$meta';
101+
Identifier: Nondigit (Nondigit | Digit)*;
102+
Meta: '$meta';
100103

101104
StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
102-
JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
105+
JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;
103106

104107
fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
105108

‎internal/parser/planparserv2/check_identical_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -17,9 +17,9 @@ func TestCheckIdentical(t *testing.T) {
1717
exprStr1 := `not (((Int64Field > 0) and (FloatField <= 20.0)) or ((Int32Field in [1, 2, 3]) and (VarCharField < "str")))`
1818
exprStr2 := `Int32Field in [1, 2, 3]`
1919

20-
expr1, err := ParseExpr(helper, exprStr1)
20+
expr1, err := ParseExpr(helper, exprStr1, nil)
2121
assert.NoError(t, err)
22-
expr2, err := ParseExpr(helper, exprStr2)
22+
expr2, err := ParseExpr(helper, exprStr2, nil)
2323
assert.NoError(t, err)
2424

2525
assert.True(t, CheckPredicatesIdentical(expr1, expr1))

‎internal/parser/planparserv2/convert_field_data_to_generic_value.go

+73
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,73 @@
1+
package planparserv2
2+
3+
import (
4+
"fmt"
5+
6+
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
7+
"github.com/milvus-io/milvus/internal/proto/planpb"
8+
)
9+
10+
func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
11+
if templateValue == nil {
12+
return nil, fmt.Errorf("expression template variable value is nil, template name: {%s}", templateName)
13+
}
14+
switch templateValue.GetType() {
15+
case schemapb.DataType_Bool:
16+
return &planpb.GenericValue{
17+
Val: &planpb.GenericValue_BoolVal{
18+
BoolVal: templateValue.GetBoolVal(),
19+
},
20+
}, nil
21+
case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
22+
return &planpb.GenericValue{
23+
Val: &planpb.GenericValue_Int64Val{
24+
Int64Val: templateValue.GetInt64Val(),
25+
},
26+
}, nil
27+
case schemapb.DataType_Float, schemapb.DataType_Double:
28+
return &planpb.GenericValue{
29+
Val: &planpb.GenericValue_FloatVal{
30+
FloatVal: templateValue.GetFloatVal(),
31+
},
32+
}, nil
33+
case schemapb.DataType_String, schemapb.DataType_VarChar:
34+
return &planpb.GenericValue{
35+
Val: &planpb.GenericValue_StringVal{
36+
StringVal: templateValue.GetStringVal(),
37+
},
38+
}, nil
39+
case schemapb.DataType_Array:
40+
elements := templateValue.GetArrayVal().GetArray()
41+
arrayValues := make([]*planpb.GenericValue, len(elements))
42+
for i, element := range elements {
43+
arrayElement, err := ConvertToGenericValue(templateName, element)
44+
if err != nil {
45+
return nil, err
46+
}
47+
arrayValues[i] = arrayElement
48+
}
49+
return &planpb.GenericValue{
50+
Val: &planpb.GenericValue_ArrayVal{
51+
ArrayVal: &planpb.Array{
52+
Array: arrayValues,
53+
SameType: templateValue.GetArrayVal().GetSameType(),
54+
ElementType: templateValue.GetArrayVal().GetElementType(),
55+
},
56+
},
57+
}, nil
58+
default:
59+
return nil, fmt.Errorf("expression elements can only be scalars")
60+
}
61+
}
62+
63+
func UnmarshalExpressionValues(input map[string]*schemapb.TemplateValue) (map[string]*planpb.GenericValue, error) {
64+
result := make(map[string]*planpb.GenericValue, len(input))
65+
for name, value := range input {
66+
rv, err := ConvertToGenericValue(name, value)
67+
if err != nil {
68+
return nil, err
69+
}
70+
result[name] = rv
71+
}
72+
return result, nil
73+
}

0 commit comments

Comments
 (0)