From 98927422d22577d44314933f58a62ebb2c1d3471 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 29 Jan 2026 14:22:35 +0100 Subject: [PATCH] [MEVD] Introduce FilterTranslatorBase and remove duplication --- .../AzureAISearchFilterTranslator.cs | 210 +-------- .../VectorData/Common/SqlFilterTranslator.cs | 206 +-------- .../CosmosMongoFilterTranslator.cs | 146 +------ .../CosmosNoSqlFilterTranslator.cs | 196 +-------- .../MongoDB/MongoFilterTranslator.cs | 145 +------ .../Pinecone/PineconeFilterTranslator.cs | 144 +------ .../Qdrant/QdrantFilterTranslator.cs | 185 +------- .../VectorData/Redis/RedisFilterTranslator.cs | 193 +-------- .../Filter/FilterPreprocessingOptions.cs | 22 + .../Filter/FilterTranslationPreprocessor.cs | 182 -------- .../Filter/FilterTranslatorBase.cs | 399 ++++++++++++++++++ .../Weaviate/WeaviateFilterTranslator.cs | 194 +-------- 12 files changed, 517 insertions(+), 1705 deletions(-) create mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs delete mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslationPreprocessor.cs create mode 100644 dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs index b84f9a497654..d149a7f96e1d 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs @@ -2,9 +2,6 @@ using System; using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using System.Text; @@ -13,26 +10,17 @@ namespace Microsoft.SemanticKernel.Connectors.AzureAISearch; -internal class AzureAISearchFilterTranslator -{ - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; +#pragma warning disable MEVD9001 // Experimental: filter translation base types +internal class AzureAISearchFilterTranslator : FilterTranslatorBase +{ private readonly StringBuilder _filter = new(); private static readonly char[] s_searchInDefaultDelimiter = [' ', ',']; internal string Translate(LambdaExpression lambdaExpression, CollectionModel model) { - Debug.Assert(this._filter.Length == 0); - - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); this.Translate(preprocessedExpression); @@ -161,52 +149,25 @@ private void TranslateMember(MemberExpression memberExpression) private void TranslateMethodCall(MethodCallExpression methodCall) { - switch (methodCall) + // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") + if (this.TryBindProperty(methodCall, out var property)) { - // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") - case MethodCallExpression when this.TryBindProperty(methodCall, out var property): - // OData identifiers cannot be escaped; storage names are validated during model building. - this._filter.Append(property.StorageName); - return; - - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable): - this.TranslateContains(source, item); - return; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - this.TranslateContains(source, item); - return; + // OData identifiers cannot be escaped; storage names are validated during model building. + this._filter.Append(property.StorageName); + return; + } - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source): + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): this.TranslateContains(source, item); return; // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) - case { Method.Name: nameof(Enumerable.Any), Arguments: [var source, LambdaExpression lambda] } any + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any when any.Method.DeclaringType == typeof(Enumerable): - this.TranslateAny(source, lambda); + this.TranslateAny(anySource, lambda); return; default: @@ -254,35 +215,12 @@ private void TranslateAny(Expression source, LambdaExpression lambda) // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) // Translates to: Field/any(t: search.in(t, 'value1, value2, value3')) if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression { Method.Name: "Contains" } containsCall) + || lambda.Body is not MethodCallExpression { Method.Name: "Contains" } containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any"), - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -390,65 +328,6 @@ private void GenerateSearchInValues(IEnumerable values) return result; } - private static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } - private void TranslateUnary(UnaryExpression unary) { switch (unary.NodeType) @@ -485,57 +364,4 @@ private void TranslateUnary(UnaryExpression unary) throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } } diff --git a/dotnet/src/VectorData/Common/SqlFilterTranslator.cs b/dotnet/src/VectorData/Common/SqlFilterTranslator.cs index 794a24ad7af9..0652d7675049 100644 --- a/dotnet/src/VectorData/Common/SqlFilterTranslator.cs +++ b/dotnet/src/VectorData/Common/SqlFilterTranslator.cs @@ -1,9 +1,7 @@ // Copyright (c) Microsoft. All rights reserved. using System; -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using System.Text; @@ -14,23 +12,20 @@ namespace Microsoft.SemanticKernel.Connectors; #pragma warning disable MEVD9001 // Microsoft.Extensions.VectorData experimental connector-facing APIs -internal abstract class SqlFilterTranslator +internal abstract class SqlFilterTranslator : FilterTranslatorBase { - private readonly CollectionModel _model; - private readonly LambdaExpression _lambdaExpression; - private readonly ParameterExpression _recordParameter; protected readonly StringBuilder _sql; + private readonly Expression _preprocessedExpression; internal SqlFilterTranslator( CollectionModel model, LambdaExpression lambdaExpression, StringBuilder? sql = null) { - this._model = model; - this._lambdaExpression = lambdaExpression; Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; this._sql = sql ?? new(); + + this._preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true }); } internal StringBuilder Clause => this._sql; @@ -42,10 +37,7 @@ internal void Translate(bool appendWhere) this._sql.Append("WHERE "); } - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = true }; - var preprocessedExpression = preprocessor.Preprocess(this._lambdaExpression.Body); - - this.Translate(preprocessedExpression, isSearchCondition: true); + this.Translate(this._preprocessedExpression, isSearchCondition: true); } protected void Translate(Expression? node, bool isSearchCondition = false) @@ -212,51 +204,24 @@ protected virtual void GenerateColumn(PropertyModel property, bool isSearchCondi private void TranslateMethodCall(MethodCallExpression methodCall, bool isSearchCondition = false) { - switch (methodCall) + // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") + if (this.TryBindProperty(methodCall, out var property)) { - // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") - case MethodCallExpression when this.TryBindProperty(methodCall, out var property): - this.GenerateColumn(property, isSearchCondition); - return; - - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable): - this.TranslateContains(source, item); - return; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - this.TranslateContains(source, item); - return; + this.GenerateColumn(property, isSearchCondition); + return; + } - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source): + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): this.TranslateContains(source, item); return; // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) - case { Method.Name: nameof(Enumerable.Any), Arguments: [var source, LambdaExpression lambda] } any + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any when any.Method.DeclaringType == typeof(Enumerable): - this.TranslateAny(source, lambda); + this.TranslateAny(anySource, lambda); return; default: @@ -319,35 +284,12 @@ private void TranslateAny(Expression source, LambdaExpression lambda) // We only support the pattern: r.ArrayColumn.Any(x => values.Contains(x)) // where 'values' is an inline array, captured array, or captured list. if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression containsCall) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -427,116 +369,4 @@ private void TranslateUnary(UnaryExpression unary, bool isSearchCondition) throw new NotSupportedException("Unsupported unary expression node type: " + unary.NodeType); } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - protected static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } diff --git a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoFilterTranslator.cs b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoFilterTranslator.cs index e2124150c502..5fca84cbe3d3 100644 --- a/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoFilterTranslator.cs +++ b/dotnet/src/VectorData/CosmosMongoDB/CosmosMongoFilterTranslator.cs @@ -2,9 +2,7 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using Microsoft.Extensions.VectorData.ProviderServices; @@ -13,22 +11,15 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable MEVD9001 // Experimental: filter translation base types + // MongoDB query reference: https://www.mongodb.com/docs/manual/reference/operator/query // Information specific to vector search pre-filter: https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter -internal class CosmosMongoFilterTranslator +internal class CosmosMongoFilterTranslator : FilterTranslatorBase { - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; - internal BsonDocument Translate(LambdaExpression lambdaExpression, CollectionModel model) { - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); return this.Translate(preprocessedExpression); } @@ -161,84 +152,12 @@ private BsonDocument TranslateMethodCall(MethodCallExpression methodCall) { return methodCall switch { - // Enumerable.Contains() - { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable) - => this.TranslateContains(source, item), - - // List.Contains() - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => this.TranslateContains(source, item), - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source) + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) => this.TranslateContains(source, item), _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") }; - - static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - if (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null - } convert - && convert.Type == typeof(object[])) - { - result = convert.Operand; - return true; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } private BsonDocument TranslateContains(Expression source, Expression item) @@ -289,57 +208,4 @@ BsonDocument ProcessInlineEnumerable(IEnumerable elements, Expression item) }; } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } } diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs index c18d7b043267..2849d41dd3cf 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs @@ -3,8 +3,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Globalization; using System.Linq; using System.Linq.Expressions; @@ -14,25 +12,16 @@ namespace Microsoft.SemanticKernel.Connectors.CosmosNoSql; -internal class CosmosNoSqlFilterTranslator -{ - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; +#pragma warning disable MEVD9001 // Experimental: filter translation base types +internal class CosmosNoSqlFilterTranslator : FilterTranslatorBase +{ private readonly Dictionary _parameters = []; private readonly StringBuilder _sql = new(); internal (string WhereClause, Dictionary Parameters) Translate(LambdaExpression lambdaExpression, CollectionModel model) { - Debug.Assert(this._sql.Length == 0); - - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = true }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions { SupportsParameterization = true }); this.Translate(preprocessedExpression); @@ -203,44 +192,17 @@ private void TranslateNewArray(NewArrayExpression newArray) private void TranslateMethodCall(MethodCallExpression methodCall) { - switch (methodCall) + // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") + if (this.TryBindProperty(methodCall, out var property)) { - // Dictionary access for dynamic mapping (r => r["SomeString"] == "foo") - case MethodCallExpression when this.TryBindProperty(methodCall, out var property): - this.GeneratePropertyAccess(property); - return; - - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable): - this.TranslateContains(source, item); - return; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - this.TranslateContains(source, item); - return; + this.GeneratePropertyAccess(property); + return; + } - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source): + switch (methodCall) + { + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): this.TranslateContains(source, item); return; @@ -273,35 +235,12 @@ private void TranslateAny(Expression source, LambdaExpression lambda) // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) // Translates to: EXISTS(SELECT VALUE t FROM t IN c["Field"] WHERE ARRAY_CONTAINS(@values, t)) if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression containsCall) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -412,109 +351,4 @@ protected virtual void GeneratePropertyAccess(PropertyModel property) .Append("[\"") .Append(property.StorageName.Replace(@"\", @"\\").Replace("\"", "\\\"")) .Append("\"]"); - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - Type? convertedClrType = null; - - if (expression is UnaryExpression { NodeType: ExpressionType.Convert } unary) - { - expression = unary.Operand; - convertedClrType = unary.Type; - } - - var modelName = expression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - if (convertedClrType is not null && convertedClrType != property.Type) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convertedClrType.Name}', but its configured type is '{property.Type.Name}'."); - } - - return true; - } - - private static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } diff --git a/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs b/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs index ed05f18e8295..7c05f90639b2 100644 --- a/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs +++ b/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs @@ -2,9 +2,7 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using Microsoft.Extensions.VectorData.ProviderServices; @@ -13,22 +11,15 @@ namespace Microsoft.SemanticKernel.Connectors.MongoDB; +#pragma warning disable MEVD9001 // Experimental: filter translation base types + // MongoDB query reference: https://www.mongodb.com/docs/manual/reference/operator/query // Information specific to vector search pre-filter: https://www.mongodb.com/docs/atlas/atlas-vector-search/vector-search-stage/#atlas-vector-search-pre-filter -internal class MongoFilterTranslator +internal class MongoFilterTranslator : FilterTranslatorBase { - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; - internal BsonDocument Translate(LambdaExpression lambdaExpression, CollectionModel model) { - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); return this.Translate(preprocessedExpression); } @@ -167,83 +158,12 @@ private BsonDocument TranslateMethodCall(MethodCallExpression methodCall) { return methodCall switch { - // Enumerable.Contains() - { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable) - => this.TranslateContains(source, item), - - // List.Contains() - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>) => this.TranslateContains(source, item), - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source) + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) => this.TranslateContains(source, item), _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") }; - - static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - if (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null - } convert - && convert.Type == typeof(object[])) - { - result = convert.Operand; - return true; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } private BsonDocument TranslateContains(Expression source, Expression item) @@ -293,57 +213,4 @@ BsonDocument ProcessInlineEnumerable(IEnumerable elements, Expression item) }; } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } } diff --git a/dotnet/src/VectorData/Pinecone/PineconeFilterTranslator.cs b/dotnet/src/VectorData/Pinecone/PineconeFilterTranslator.cs index cb969b1238aa..0fe0875e0b81 100644 --- a/dotnet/src/VectorData/Pinecone/PineconeFilterTranslator.cs +++ b/dotnet/src/VectorData/Pinecone/PineconeFilterTranslator.cs @@ -4,7 +4,6 @@ using System.Collections; using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using Microsoft.Extensions.VectorData.ProviderServices; @@ -13,23 +12,17 @@ namespace Microsoft.SemanticKernel.Connectors.Pinecone; +#pragma warning disable MEVD9001 // Experimental: filter translation base types + // This class is a modification of MongoDBFilterTranslator that uses the same query language // (https://docs.pinecone.io/guides/data/understanding-metadata#metadata-query-language), // with the difference of representing everything as Metadata rather than BsonDocument. // For representing collections of any kinds, we use List, // as we sometimes need to extend the collection (with for example another condition). -internal class PineconeFilterTranslator +internal class PineconeFilterTranslator : FilterTranslatorBase { - private Extensions.VectorData.ProviderServices.CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; - internal Metadata? Translate(LambdaExpression lambdaExpression, Extensions.VectorData.ProviderServices.CollectionModel model) { - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - // Pinecone doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), // nor is 'x && true'. @@ -38,8 +31,7 @@ internal class PineconeFilterTranslator return null; } - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); return this.Translate(preprocessedExpression); } @@ -169,83 +161,12 @@ private Metadata TranslateMethodCall(MethodCallExpression methodCall) { return methodCall switch { - // Enumerable.Contains() - { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable) - => this.TranslateContains(source, item), - - // List.Contains() - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>) => this.TranslateContains(source, item), - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source) + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) => this.TranslateContains(source, item), _ => throw new NotSupportedException($"Unsupported method call: {methodCall.Method.DeclaringType?.Name}.{methodCall.Method.Name}") }; - - static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - if (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null - } convert - && convert.Type == typeof(object[])) - { - result = convert.Operand; - return true; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } private Metadata TranslateContains(Expression source, Expression item) @@ -296,59 +217,6 @@ Metadata ProcessInlineEnumerable(IEnumerable elements, Expression item) } } - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - private static MetadataValue? ToMetadata(object? value) => value is null ? null : PineconeFieldMapping.ConvertToMetadataValue(value); diff --git a/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs b/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs index 25a1b25109fa..0e13bd0d8801 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs @@ -3,7 +3,6 @@ using System; using System.Collections; using System.Collections.Generic; -using System.Diagnostics; using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; @@ -17,21 +16,14 @@ namespace Microsoft.SemanticKernel.Connectors.Qdrant; +#pragma warning disable MEVD9001 // Experimental: filter translation base types + // https://qdrant.tech/documentation/concepts/filtering -internal class QdrantFilterTranslator +internal class QdrantFilterTranslator : FilterTranslatorBase { - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; - internal Filter Translate(LambdaExpression lambdaExpression, CollectionModel model) { - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); return this.Translate(preprocessedExpression); } @@ -273,34 +265,8 @@ private Filter TranslateMethodCall(MethodCallExpression methodCall) { return methodCall switch { - // Enumerable.Contains() - { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable) - => this.TranslateContains(source, item), - - // List.Contains() - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => this.TranslateContains(source, item), - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source) + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + _ when TryMatchContains(methodCall, out var source, out var item) => this.TranslateContains(source, item), // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) @@ -391,35 +357,12 @@ private Filter TranslateAny(Expression source, LambdaExpression lambda) { // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression containsCall) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -489,116 +432,4 @@ when declaringType.GetGenericTypeDefinition() == typeof(List<>) return result; } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - private static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } diff --git a/dotnet/src/VectorData/Redis/RedisFilterTranslator.cs b/dotnet/src/VectorData/Redis/RedisFilterTranslator.cs index bdbf5d825348..20f1fad54fb0 100644 --- a/dotnet/src/VectorData/Redis/RedisFilterTranslator.cs +++ b/dotnet/src/VectorData/Redis/RedisFilterTranslator.cs @@ -2,9 +2,6 @@ using System; using System.Collections; -using System.Collections.Generic; -using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using System.Text; @@ -13,21 +10,14 @@ namespace Microsoft.SemanticKernel.Connectors.Redis; -internal class RedisFilterTranslator +#pragma warning disable MEVD9001 // Experimental: filter translation base types + +internal class RedisFilterTranslator : FilterTranslatorBase { - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; private readonly StringBuilder _filter = new(); internal string Translate(LambdaExpression lambdaExpression, CollectionModel model) { - Debug.Assert(this._filter.Length == 0); - - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - // Redis doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), // nor is 'x && true'. @@ -36,8 +26,7 @@ internal string Translate(LambdaExpression lambdaExpression, CollectionModel mod return "*"; } - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); this.Translate(preprocessedExpression); return this._filter.ToString(); @@ -111,7 +100,7 @@ bool TryProcessEqualityComparison(Expression first, Expression second) if (this.TryBindProperty(first, out var property) && second is ConstantExpression { Value: var constantValue }) { // Numeric negation has a special syntax (!=), for the rest we nest in a NOT - if (binary.NodeType is ExpressionType.NotEqual && constantValue is not int or long or float or double) + if (binary.NodeType is ExpressionType.NotEqual && constantValue is not (int or long or float or double)) { this.TranslateNot(Expression.Equal(first, second)); return true; @@ -163,37 +152,8 @@ private void TranslateMethodCall(MethodCallExpression methodCall) { switch (methodCall) { - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable): - this.TranslateContains(source, item); - return; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - this.TranslateContains(source, item); - return; - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source): + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): this.TranslateContains(source, item); return; @@ -235,35 +195,12 @@ private void TranslateAny(Expression source, LambdaExpression lambda) // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) // Translates to: @Field:{value1 | value2 | value3} if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression containsCall) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -322,116 +259,4 @@ when declaringType.GetGenericTypeDefinition() == typeof(List<>) return result; } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - private static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } } diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs new file mode 100644 index 000000000000..4b0cc8f07e2b --- /dev/null +++ b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterPreprocessingOptions.cs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; + +/// +/// Options for filter expression preprocessing. +/// This is an internal support type meant for use by connectors only and not by applications. +/// +[Experimental("MEVD9001")] +public class FilterPreprocessingOptions +{ + /// + /// Whether the connector supports parameterization. + /// + /// + /// If , the visitor will inline captured variables and constant member accesses as simple constant nodes. + /// If , these will instead be replaced with nodes. + /// + public bool SupportsParameterization { get; init; } +} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslationPreprocessor.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslationPreprocessor.cs deleted file mode 100644 index 301dde49866f..000000000000 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslationPreprocessor.cs +++ /dev/null @@ -1,182 +0,0 @@ -// Copyright (c) Microsoft. All rights reserved. - -using System; -using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq.Expressions; -using System.Reflection; - -namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; - -/// -/// A processor for user-provided filter expressions which performs various common transformations before actual translation takes place. -/// This is an internal support type meant for use by connectors only and not by applications. -/// -[Experimental("MEVD9001")] -public class FilterTranslationPreprocessor : ExpressionVisitor -{ - private List? _parameterNames; - - /// - /// Whether the connector supports parameterization. - /// - /// - /// If , the visitor will inline captured variables and constant member accesses as simple constant nodes. - /// If , these will instead be replaced with nodes. - /// - public required bool SupportsParameterization { get; init; } - - /// - /// Preprocesses the filter expression before translation. - /// - public Expression Preprocess(Expression node) - { - if (this.SupportsParameterization) - { - this._parameterNames = []; - } - - return this.Visit(node); - } - - /// - protected override Expression VisitMember(MemberExpression node) - { - var visited = (MemberExpression)base.VisitMember(node); - - // This identifies field and property access over constants, which can be evaluated immediately. - // This covers captured variables, since those are actually member accesses over compiled-generated closure types: - // var x = 8; - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x }); - // - // This also covers member variables: - // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x }); - // ... as "this" here is represented by a ConstantExpression node in the tree. - // - // Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders - // from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those - // connectors, and centralizes the pattern matching in a single centralized place). - // For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't - // even need to be aware of it. - - // Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for - // static members (expression is null). - object? baseValue; - switch (visited.Expression) - { - // Member access over constant (i.e. instance members) - case ConstantExpression { Value: var v }: - baseValue = v; - break; - - // Member constant over null (i.e. static members) - case null: - baseValue = null; - break; - - // Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something) - case QueryParameterExpression p: - baseValue = p.Value; - - // The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names. - this._parameterNames!.Remove(p.Name); - break; - - default: - return visited; - } - - object? evaluatedValue; - - var memberInfo = visited.Member; - - switch (memberInfo) - { - case FieldInfo fieldInfo: - evaluatedValue = fieldInfo.GetValue(baseValue); - break; - - case PropertyInfo { GetMethod.IsStatic: false } propertyInfo when baseValue is null: - throw new InvalidOperationException($"Cannot access member '{propertyInfo.Name}' on null object."); - - case PropertyInfo propertyInfo: - evaluatedValue = propertyInfo.GetValue(baseValue); - break; - default: - return visited; - } - - // Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly), - if (!this.SupportsParameterization) - { - return Expression.Constant(evaluatedValue, visited.Type); - } - - // Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter). - - // TODO: Share the same parameter when it references the same captured value - - // Make sure parameter names are unique. - var origName = memberInfo.Name; - var name = origName; - for (var i = 0; this._parameterNames!.Contains(name); i++) - { - name = $"{origName}_{i}"; - } - this._parameterNames.Add(name); - - return new QueryParameterExpression(name, evaluatedValue, visited.Type); - } - - /// - protected override Expression VisitNew(NewExpression node) - { - var visited = (NewExpression)base.VisitNew(node); - - // Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression. - // This is particularly useful for converting inline instantiation of DateTime and DateTimeOffset to constants, which can then be easily translated. - switch (visited.Constructor) - { - case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime): - var constantArguments = new object?[visited.Arguments.Count]; - - // We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1). - // If an argument isn't a constant (e.g. new DateTimeOffset(..., TimeSpan.FromHours(2))), we fall back to trying the LINQ interpreter - // as a general-purpose expression evaluator - but note that this is considerably slower. - for (var i = 0; i < visited.Arguments.Count; i++) - { - if (visited.Arguments[i] is ConstantExpression constantArgument) - { - constantArguments[i] = constantArgument.Value; - } - else - { - // There's a non-constant argument - try the LINQ interpreter. -#pragma warning disable CA1031 // Do not catch general exception types - try - { - var evaluated = Expression.Lambda>(Expression.Convert(visited, typeof(object))) -#if NET - .Compile(preferInterpretation: true) -#else - .Compile() -#endif - .Invoke(); - - return Expression.Constant(evaluated, constructor.DeclaringType); - } - catch - { - return visited; - } -#pragma warning restore CA1031 - } - } - - var constantValue = constructor.Invoke(constantArguments); - return Expression.Constant(constantValue, constructor.DeclaringType); - } - - return visited; - } -} diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs new file mode 100644 index 000000000000..bc5a2f50f6c9 --- /dev/null +++ b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs @@ -0,0 +1,399 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Linq.Expressions; +using System.Reflection; + +namespace Microsoft.Extensions.VectorData.ProviderServices.Filter; + +/// +/// Base class for filter translators used by vector data connectors. +/// Provides common functionality for preprocessing filter expressions and matching common patterns. +/// This is an internal support type meant for use by connectors only and not by applications. +/// +[Experimental("MEVD9001")] +public abstract class FilterTranslatorBase +{ + /// + /// The collection model for the current translation operation. + /// + protected CollectionModel Model { get; private set; } = null!; + + /// + /// The parameter expression representing the record in the filter lambda. + /// + protected ParameterExpression RecordParameter { get; private set; } = null!; + + /// + /// Preprocesses the filter expression before translation. + /// Sets and , runs the preprocessing visitor, + /// and returns the preprocessed expression. + /// + /// The filter lambda expression to preprocess. + /// The collection model containing property information. + /// Options controlling the preprocessing behavior. + /// The preprocessed expression ready for translation. + protected Expression PreprocessFilter(LambdaExpression lambdaExpression, CollectionModel model, FilterPreprocessingOptions options) + { + this.Model = model; + this.RecordParameter = lambdaExpression.Parameters[0]; + + var preprocessor = new FilterTranslationPreprocessor(options.SupportsParameterization); + return preprocessor.Preprocess(lambdaExpression.Body); + } + + /// + /// Tries to match a Contains method call expression and extract the source collection and item expressions. + /// + /// The method call expression to match. + /// When successful, the source collection expression. + /// When successful, the item expression being searched for. + /// if the expression is a recognized Contains pattern; otherwise, . + protected static bool TryMatchContains( + MethodCallExpression methodCall, + [NotNullWhen(true)] out Expression? source, + [NotNullWhen(true)] out Expression? item) + { + switch (methodCall) + { + // Enumerable.Contains() + case { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var itm] } + when methodCall.Method.DeclaringType == typeof(Enumerable): + source = src; + item = itm; + return true; + + // List.Contains() + case + { + Method: + { + Name: nameof(Enumerable.Contains), + DeclaringType: { IsGenericType: true } declaringType + }, + Object: Expression src, + Arguments: [var itm] + } when declaringType.GetGenericTypeDefinition() == typeof(List<>): + source = src; + item = itm; + return true; + + // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); + // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). + // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. + // See https://github.com/dotnet/runtime/issues/109757 for more context. + // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when + // it's null. + case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var itm, ..] } + when methodCall.Method.DeclaringType == typeof(MemoryExtensions) + && (methodCall.Arguments.Count is 2 + || (methodCall.Arguments.Count is 3 && methodCall.Arguments[2] is ConstantExpression { Value: null })) + && TryUnwrapSpanImplicitCast(spanArg, out var src): + source = src; + item = itm; + return true; + + default: + source = null; + item = null; + return false; + } + } + + /// + /// Tries to bind an expression to a property in the collection model. + /// + /// The expression to bind. + /// When successful, the property model that was bound. + /// if the expression was successfully bound to a property; otherwise, . + protected virtual bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? propertyModel) + { + var unwrappedExpression = expression; + while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) + { + unwrappedExpression = convert.Operand; + } + + var modelName = unwrappedExpression switch + { + // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) + MemberExpression memberExpression when memberExpression.Expression == this.RecordParameter + => memberExpression.Member.Name, + + // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) + MethodCallExpression + { + Method: { Name: "get_Item", DeclaringType: var declaringType }, + Arguments: [ConstantExpression { Value: string keyName }] + } methodCall when methodCall.Object == this.RecordParameter && declaringType == typeof(Dictionary) + => keyName, + + _ => null + }; + + if (modelName is null) + { + propertyModel = null; + return false; + } + + if (!this.Model.PropertyMap.TryGetValue(modelName, out propertyModel)) + { + throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); + } + + // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type + var unwrappedPropertyType = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type; + unwrappedExpression = expression; + while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) + { + var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; + if (convertType != unwrappedPropertyType && convertType != typeof(object)) + { + throw new InvalidCastException($"Property '{propertyModel.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{propertyModel.Type.Name}'."); + } + + unwrappedExpression = convert.Operand; + } + + return true; + } + + /// + /// Tries to unwrap an implicit cast to Span or ReadOnlySpan that may be present in expressions + /// when C# 14's first-class span support causes MemoryExtensions methods to be resolved. + /// + /// The expression to unwrap. + /// When successful, the unwrapped expression. + /// if a span implicit cast was unwrapped; otherwise, . + protected static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) + { + // Different versions of the compiler seem to generate slightly different expression tree representations for this + // implicit cast: + var (unwrapped, castDeclaringType) = expression switch + { + UnaryExpression + { + NodeType: ExpressionType.Convert, + Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, + Operand: var operand + } => (operand, implicitCastDeclaringType), + + MethodCallExpression + { + Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, + Arguments: [var firstArgument] + } => (firstArgument, implicitCastDeclaringType), + + // After the preprocessor runs, the Convert node may have Method: null because the visitor + // recreates the UnaryExpression with a different operand type (QueryParameterExpression). + // Handle this case by checking if the target type is Span or ReadOnlySpan. + UnaryExpression + { + NodeType: ExpressionType.Convert, + Method: null, + Type: { IsGenericType: true } targetType, + Operand: var operand + } when targetType.GetGenericTypeDefinition() is var gtd + && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) + => (operand, targetType), + + _ => (null, null) + }; + + // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. + // Also handle cases where the preprocessor adds a Convert node back to the array type. + while (unwrapped is UnaryExpression + { + NodeType: ExpressionType.Convert, + Method: null, + Operand: var innerOperand + }) + { + unwrapped = innerOperand; + } + + if (unwrapped is not null + && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition + && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) + { + result = unwrapped; + return true; + } + + result = null; + return false; + } + + #region FilterTranslationPreprocessor + + /// + /// A processor for user-provided filter expressions which performs various common transformations before actual translation takes place. + /// + private sealed class FilterTranslationPreprocessor : ExpressionVisitor + { + private readonly bool _supportsParameterization; + private List? _parameterNames; + + internal FilterTranslationPreprocessor(bool supportsParameterization) + { + this._supportsParameterization = supportsParameterization; + } + + internal Expression Preprocess(Expression node) + { + if (this._supportsParameterization) + { + this._parameterNames = []; + } + + return this.Visit(node); + } + + /// + protected override Expression VisitMember(MemberExpression node) + { + var visited = (MemberExpression)base.VisitMember(node); + + // This identifies field and property access over constants, which can be evaluated immediately. + // This covers captured variables, since those are actually member accesses over compiled-generated closure types: + // var x = 8; + // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == x }); + // + // This also covers member variables: + // _ = await collection.SearchAsync(vector, top: 3, new() { Filter = r => r.Int == this._x }); + // ... as "this" here is represented by a ConstantExpression node in the tree. + // + // Some databases - mostly relational ones - support out-of-band parameters which can be referenced via placeholders + // from the query itself. For those databases, we transform the member access to QueryParameterExpression (this simplifies things for those + // connectors, and centralizes the pattern matching in a single centralized place). + // For databases which don't support parameters, we simply inline the evaluated member access as a constant in the tree, so that translators don't + // even need to be aware of it. + + // Evaluate the MemberExpression to get the actual value, either for instance members (expression is a ConstantExpression) or for + // static members (expression is null). + object? baseValue; + switch (visited.Expression) + { + // Member access over constant (i.e. instance members) + case ConstantExpression { Value: var v }: + baseValue = v; + break; + + // Member constant over null (i.e. static members) + case null: + baseValue = null; + break; + + // Member constant over something that has already been parameterized (i.e. nested member access, e.g. r=> r.Int == this.SomeWrapper.Something) + case QueryParameterExpression p: + baseValue = p.Value; + + // The previous parameter is getting replaced by the new one we're creating here, so remove its name from the list of parameter names. + this._parameterNames!.Remove(p.Name); + break; + + default: + return visited; + } + + object? evaluatedValue; + + var memberInfo = visited.Member; + + switch (memberInfo) + { + case FieldInfo fieldInfo: + evaluatedValue = fieldInfo.GetValue(baseValue); + break; + + case PropertyInfo { GetMethod.IsStatic: false } propertyInfo when baseValue is null: + throw new InvalidOperationException($"Cannot access member '{propertyInfo.Name}' on null object."); + + case PropertyInfo propertyInfo: + evaluatedValue = propertyInfo.GetValue(baseValue); + break; + default: + return visited; + } + + // Inline the evaluated value (if the connector doesn't support parameterization, or if the field is readonly), + if (!this._supportsParameterization) + { + return Expression.Constant(evaluatedValue, visited.Type); + } + + // Otherwise, transform the node to a QueryParameterExpression which the connector will then translate to a parameter (e.g. SqlParameter). + + // TODO: Share the same parameter when it references the same captured value + + // Make sure parameter names are unique. + var origName = memberInfo.Name; + var name = origName; + for (var i = 0; this._parameterNames!.Contains(name); i++) + { + name = $"{origName}_{i}"; + } + this._parameterNames.Add(name); + + return new QueryParameterExpression(name, evaluatedValue, visited.Type); + } + + /// + protected override Expression VisitNew(NewExpression node) + { + var visited = (NewExpression)base.VisitNew(node); + + // Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression. + // This is particularly useful for converting inline instantiation of DateTime and DateTimeOffset to constants, which can then be easily translated. + switch (visited.Constructor) + { + case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime): + var constantArguments = new object?[visited.Arguments.Count]; + + // We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1). + // If an argument isn't a constant (e.g. new DateTimeOffset(..., TimeSpan.FromHours(2))), we fall back to trying the LINQ interpreter + // as a general-purpose expression evaluator - but note that this is considerably slower. + for (var i = 0; i < visited.Arguments.Count; i++) + { + if (visited.Arguments[i] is ConstantExpression constantArgument) + { + constantArguments[i] = constantArgument.Value; + } + else + { + // There's a non-constant argument - try the LINQ interpreter. +#pragma warning disable CA1031 // Do not catch general exception types + try + { + var evaluated = Expression.Lambda>(Expression.Convert(visited, typeof(object))) +#if NET + .Compile(preferInterpretation: true) +#else + .Compile() +#endif + .Invoke(); + + return Expression.Constant(evaluated, constructor.DeclaringType); + } + catch + { + return visited; + } +#pragma warning restore CA1031 + } + } + + var constantValue = constructor.Invoke(constantArguments); + return Expression.Constant(constantValue, constructor.DeclaringType); + } + + return visited; + } + } + + #endregion FilterTranslationPreprocessor +} diff --git a/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs b/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs index f91393e05455..5efa580044dd 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs @@ -2,9 +2,7 @@ using System; using System.Collections; -using System.Collections.Generic; using System.Diagnostics; -using System.Diagnostics.CodeAnalysis; using System.Linq; using System.Linq.Expressions; using System.Text; @@ -14,22 +12,15 @@ namespace Microsoft.SemanticKernel.Connectors.Weaviate; +#pragma warning disable MEVD9001 // Experimental: filter translation base types + // https://weaviate.io/developers/weaviate/api/graphql/filters#filter-structure -internal class WeaviateFilterTranslator +internal class WeaviateFilterTranslator : FilterTranslatorBase { - private CollectionModel _model = null!; - private ParameterExpression _recordParameter = null!; private readonly StringBuilder _filter = new(); internal string? Translate(LambdaExpression lambdaExpression, CollectionModel model) { - Debug.Assert(this._filter.Length == 0); - - this._model = model; - - Debug.Assert(lambdaExpression.Parameters.Count == 1); - this._recordParameter = lambdaExpression.Parameters[0]; - // Weaviate doesn't seem to have a native way of expressing "always true" filters; since this scenario is important for fetching // all records (via GetAsync with filter), we special-case and support it here. Note that false isn't supported (useless), // nor is 'x && true'. @@ -38,8 +29,7 @@ internal class WeaviateFilterTranslator return null; } - var preprocessor = new FilterTranslationPreprocessor { SupportsParameterization = false }; - var preprocessedExpression = preprocessor.Preprocess(lambdaExpression.Body); + var preprocessedExpression = this.PreprocessFilter(lambdaExpression, model, new FilterPreprocessingOptions()); this.Translate(preprocessedExpression); return this._filter.ToString(); @@ -202,44 +192,15 @@ private void TranslateMethodCall(MethodCallExpression methodCall) { switch (methodCall) { - // Enumerable.Contains() - case { Method.Name: nameof(Enumerable.Contains), Arguments: [var source, var item] } contains - when contains.Method.DeclaringType == typeof(Enumerable): - this.TranslateContains(source, item); - return; - - // List.Contains() - case - { - Method: - { - Name: nameof(Enumerable.Contains), - DeclaringType: { IsGenericType: true } declaringType - }, - Object: Expression source, - Arguments: [var item] - } when declaringType.GetGenericTypeDefinition() == typeof(List<>): - this.TranslateContains(source, item); - return; - - // C# 14 made changes to overload resolution to prefer Span-based overloads when those exist ("first-class spans"); - // this makes MemoryExtensions.Contains() be resolved rather than Enumerable.Contains() (see above). - // MemoryExtensions.Contains() also accepts a Span argument for the source, adding an implicit cast we need to remove. - // See https://github.com/dotnet/runtime/issues/109757 for more context. - // Note that MemoryExtensions.Contains has an optional 3rd ComparisonType parameter; we only match when - // it's null. - case { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } contains - when contains.Method.DeclaringType == typeof(MemoryExtensions) - && (contains.Arguments.Count is 2 - || (contains.Arguments.Count is 3 && contains.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var source): + // Enumerable.Contains(), List.Contains(), MemoryExtensions.Contains() + case var _ when TryMatchContains(methodCall, out var source, out var item): this.TranslateContains(source, item); return; // Enumerable.Any() with a Contains predicate (r => r.Strings.Any(s => array.Contains(s))) - case { Method.Name: nameof(Enumerable.Any), Arguments: [var source, LambdaExpression lambda] } any + case { Method.Name: nameof(Enumerable.Any), Arguments: [var anySource, LambdaExpression lambda] } any when any.Method.DeclaringType == typeof(Enumerable): - this.TranslateAny(source, lambda); + this.TranslateAny(anySource, lambda); return; default: @@ -274,35 +235,12 @@ private void TranslateAny(Expression source, LambdaExpression lambda) // We only support the pattern: r.ArrayField.Any(x => values.Contains(x)) // Translates to: { path: ["Field"], operator: ContainsAny, valueText: ["value1", "value2"] } if (!this.TryBindProperty(source, out var property) - || lambda.Body is not MethodCallExpression containsCall) + || lambda.Body is not MethodCallExpression containsCall + || !TryMatchContains(containsCall, out var valuesExpression, out var itemExpression)) { throw new NotSupportedException("Unsupported method call: Enumerable.Any"); } - // Match Enumerable.Contains(source, item), List.Contains(item), or MemoryExtensions.Contains - var (valuesExpression, itemExpression) = containsCall switch - { - // Enumerable.Contains(source, item) - { Method.Name: nameof(Enumerable.Contains), Arguments: [var src, var item] } - when containsCall.Method.DeclaringType == typeof(Enumerable) - => (src, item), - - // List.Contains(item) - { Method: { Name: nameof(Enumerable.Contains), DeclaringType: { IsGenericType: true } declaringType }, Object: Expression src, Arguments: [var item] } - when declaringType.GetGenericTypeDefinition() == typeof(List<>) - => (src, item), - - // MemoryExtensions.Contains (C# 14 first-class spans) - { Method.Name: nameof(MemoryExtensions.Contains), Arguments: [var spanArg, var item, ..] } - when containsCall.Method.DeclaringType == typeof(MemoryExtensions) - && (containsCall.Arguments.Count is 2 - || (containsCall.Arguments.Count is 3 && containsCall.Arguments[2] is ConstantExpression { Value: null })) - && TryUnwrapSpanImplicitCast(spanArg, out var unwrappedSource) - => (unwrappedSource, item), - - _ => throw new NotSupportedException("Unsupported method call: Enumerable.Any") - }; - // Verify that the item is the lambda parameter if (itemExpression != lambda.Parameters[0]) { @@ -361,116 +299,4 @@ when declaringType.GetGenericTypeDefinition() == typeof(List<>) return result; } } - - private bool TryBindProperty(Expression expression, [NotNullWhen(true)] out PropertyModel? property) - { - var unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - unwrappedExpression = convert.Operand; - } - - var modelName = unwrappedExpression switch - { - // Regular member access for strongly-typed POCO binding (e.g. r => r.SomeInt == 8) - MemberExpression memberExpression when memberExpression.Expression == this._recordParameter - => memberExpression.Member.Name, - - // Dictionary lookup for weakly-typed dynamic binding (e.g. r => r["SomeInt"] == 8) - MethodCallExpression - { - Method: { Name: "get_Item", DeclaringType: var declaringType }, - Arguments: [ConstantExpression { Value: string keyName }] - } methodCall when methodCall.Object == this._recordParameter && declaringType == typeof(Dictionary) - => keyName, - - _ => null - }; - - if (modelName is null) - { - property = null; - return false; - } - - if (!this._model.PropertyMap.TryGetValue(modelName, out property)) - { - throw new InvalidOperationException($"Property name '{modelName}' provided as part of the filter clause is not a valid property name."); - } - - // Now that we have the property, go over all wrapping Convert nodes again to ensure that they're compatible with the property type - var unwrappedPropertyType = Nullable.GetUnderlyingType(property.Type) ?? property.Type; - unwrappedExpression = expression; - while (unwrappedExpression is UnaryExpression { NodeType: ExpressionType.Convert } convert) - { - var convertType = Nullable.GetUnderlyingType(convert.Type) ?? convert.Type; - if (convertType != unwrappedPropertyType && convertType != typeof(object)) - { - throw new InvalidCastException($"Property '{property.ModelName}' is being cast to type '{convert.Type.Name}', but its configured type is '{property.Type.Name}'."); - } - - unwrappedExpression = convert.Operand; - } - - return true; - } - - private static bool TryUnwrapSpanImplicitCast(Expression expression, [NotNullWhen(true)] out Expression? result) - { - // Different versions of the compiler seem to generate slightly different expression tree representations for this - // implicit cast: - var (unwrapped, castDeclaringType) = expression switch - { - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Operand: var operand - } => (operand, implicitCastDeclaringType), - - MethodCallExpression - { - Method: { Name: "op_Implicit", DeclaringType: { IsGenericType: true } implicitCastDeclaringType }, - Arguments: [var firstArgument] - } => (firstArgument, implicitCastDeclaringType), - - // After the preprocessor runs, the Convert node may have Method: null because the visitor - // recreates the UnaryExpression with a different operand type (QueryParameterExpression). - // Handle this case by checking if the target type is Span or ReadOnlySpan. - UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Type: { IsGenericType: true } targetType, - Operand: var operand - } when targetType.GetGenericTypeDefinition() is var gtd - && (gtd == typeof(Span<>) || gtd == typeof(ReadOnlySpan<>)) - => (operand, targetType), - - _ => (null, null) - }; - - // For the dynamic case, there's a Convert node representing an up-cast to object[]; unwrap that too. - // Also handle cases where the preprocessor adds a Convert node back to the array type. - while (unwrapped is UnaryExpression - { - NodeType: ExpressionType.Convert, - Method: null, - Operand: var innerOperand - }) - { - unwrapped = innerOperand; - } - - if (unwrapped is not null - && castDeclaringType?.GetGenericTypeDefinition() is var genericTypeDefinition - && (genericTypeDefinition == typeof(Span<>) || genericTypeDefinition == typeof(ReadOnlySpan<>))) - { - result = unwrapped; - return true; - } - - result = null; - return false; - } }