feat: [2.5]The expression supports filling elements through templates (#37280)

issue: #36672

master pr: #37033

milvus-proto pr: milvus-io/milvus-proto#332

Signed-off-by: Cai Zhang <[email protected]>

Author: xiaocai2333

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
 	github.com/klauspost/compress v1.17.9
 	github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
-	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34
+	github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7
 	github.com/minio/minio-go/v7 v7.0.73
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/prometheus/client_golang v1.14.0

go.sum

@@ -629,6 +629,8 @@ github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZz
 github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34 h1:Fwxpg98128gfWRbQ1A3PMP9o2IfYZk7RSEy8rcoCWDA=
 github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20240930043709-0c23514e4c34/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7 h1:HwAitQk+V59QdYUwwVVYHTujd4QZrebg2Cc2hmcjhAg=
+github.com/milvus-io/milvus-proto/go-api/v2 v2.3.4-0.20241025031121-4d5c88b00cf7/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
 github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
 github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
 github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=

internal/parser/planparserv2/Plan.g4

@@ -5,8 +5,9 @@ expr:
 	| FloatingConstant										                     # Floating
 	| BooleanConstant										                     # Boolean
 	| StringLiteral											                     # String
-	| Identifier											                     # Identifier
+	| (Identifier|Meta)           			      							     # Identifier
 	| JSONIdentifier                                                             # JSONIdentifier
+	| LBRACE Identifier RBRACE                                                   # TemplateVariable
 	| '(' expr ')'											                     # Parens
 	| '[' expr (',' expr)* ','? ']'                                              # Array
 	| EmptyArray                                                                 # EmptyArray
@@ -19,7 +20,6 @@ expr:
 	| expr op = (ADD | SUB) expr							                     # AddSub
 	| expr op = (SHL | SHR) expr							                     # Shift
 	| expr op = NOT? IN expr                                                     # Term
-//	| EmptyTerm                                                                  # EmptyTerm
 	| (JSONContains | ArrayContains)'('expr',' expr')'                           # JSONContains
 	| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')'                     # JSONContainsAll
 	| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')'                     # JSONContainsAny
@@ -45,6 +45,8 @@ expr:
 // INT64: 'int64';
 // FLOAT: 'float';
 // DOUBLE: 'double';
+LBRACE: '{';
+RBRACE: '}';
 
 LT: '<';
 LE: '<=';
@@ -99,10 +101,11 @@ FloatingConstant:
 	DecimalFloatingConstant
 	| HexadecimalFloatingConstant;
 
-Identifier: Nondigit (Nondigit | Digit)* | '$meta';
+Identifier: Nondigit (Nondigit | Digit)*;
+Meta: '$meta';
 
 StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
-JSONIdentifier: Identifier('[' (StringLiteral | DecimalConstant) ']')+;
+JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;
 
 fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
 

internal/parser/planparserv2/check_identical_test.go

@@ -26,9 +26,9 @@ func TestCheckIdentical(t *testing.T) {
 		exprStr1 := exprStr1Arr[i]
 		exprStr2 := exprStr2Arr[i]
 
-		expr1, err := ParseExpr(helper, exprStr1)
+		expr1, err := ParseExpr(helper, exprStr1, nil)
 		assert.NoError(t, err)
-		expr2, err := ParseExpr(helper, exprStr2)
+		expr2, err := ParseExpr(helper, exprStr2, nil)
 		assert.NoError(t, err)
 
 		assert.True(t, CheckPredicatesIdentical(expr1, expr1))

internal/parser/planparserv2/convert_field_data_to_generic_value.go

@@ -0,0 +1,73 @@
+package planparserv2
+
+import (
+	"fmt"
+
+	"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
+	"github.com/milvus-io/milvus/internal/proto/planpb"
+)
+
+func ConvertToGenericValue(templateName string, templateValue *schemapb.TemplateValue) (*planpb.GenericValue, error) {
+	if templateValue == nil {
+		return nil, fmt.Errorf("expression template variable value is nil, template name: {%s}", templateName)
+	}
+	switch templateValue.GetType() {
+	case schemapb.DataType_Bool:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_BoolVal{
+				BoolVal: templateValue.GetBoolVal(),
+			},
+		}, nil
+	case schemapb.DataType_Int8, schemapb.DataType_Int16, schemapb.DataType_Int32, schemapb.DataType_Int64:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_Int64Val{
+				Int64Val: templateValue.GetInt64Val(),
+			},
+		}, nil
+	case schemapb.DataType_Float, schemapb.DataType_Double:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_FloatVal{
+				FloatVal: templateValue.GetFloatVal(),
+			},
+		}, nil
+	case schemapb.DataType_String, schemapb.DataType_VarChar:
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_StringVal{
+				StringVal: templateValue.GetStringVal(),
+			},
+		}, nil
+	case schemapb.DataType_Array:
+		elements := templateValue.GetArrayVal().GetArray()
+		arrayValues := make([]*planpb.GenericValue, len(elements))
+		for i, element := range elements {
+			arrayElement, err := ConvertToGenericValue(templateName, element)
+			if err != nil {
+				return nil, err
+			}
+			arrayValues[i] = arrayElement
+		}
+		return &planpb.GenericValue{
+			Val: &planpb.GenericValue_ArrayVal{
+				ArrayVal: &planpb.Array{
+					Array:       arrayValues,
+					SameType:    templateValue.GetArrayVal().GetSameType(),
+					ElementType: templateValue.GetArrayVal().GetElementType(),
+				},
+			},
+		}, nil
+	default:
+		return nil, fmt.Errorf("expression elements can only be scalars")
+	}
+}
+
+func UnmarshalExpressionValues(input map[string]*schemapb.TemplateValue) (map[string]*planpb.GenericValue, error) {
+	result := make(map[string]*planpb.GenericValue, len(input))
+	for name, value := range input {
+		rv, err := ConvertToGenericValue(name, value)
+		if err != nil {
+			return nil, err
+		}
+		result[name] = rv
+	}
+	return result, nil
+}