From 72216e7effb08c5bdf9080a4a0673640b47ad896 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 19 Feb 2026 15:31:20 +0100 Subject: [PATCH 1/2] [MEVD] Support DateTime/DateTimeOffset/DateOnly/TimeOnly across providers Closes #11286 Closes #11086 --- .../Memory/MongoDB/BsonValueFactory.cs | 13 +- .../Memory/MongoDB/MongoDynamicMapper.cs | 12 +- .../Memory/MongoDB/MongoModelBuilder.cs | 13 +- .../AzureAISearchCollectionCreateMapping.cs | 10 ++ .../AzureAISearchDynamicMapper.cs | 40 +++++- .../AzureAISearchFilterTranslator.cs | 8 ++ .../AzureAISearch/AzureAISearchMapper.cs | 44 ++++++ .../AzureAISearchModelBuilder.cs | 10 +- .../CosmosNoSqlFilterTranslator.cs | 16 +++ .../CosmosNoSql/CosmosNoSqlModelBuilder.cs | 10 +- .../MongoDB/MongoFilterTranslator.cs | 6 +- .../PgVector/PostgresFilterTranslator.cs | 10 ++ .../src/VectorData/PgVector/PostgresMapper.cs | 16 +++ .../PgVector/PostgresModelBuilder.cs | 4 + .../PgVector/PostgresPropertyMapping.cs | 9 ++ .../Qdrant/QdrantCollectionCreateMapping.cs | 11 +- .../VectorData/Qdrant/QdrantFieldMapping.cs | 126 ++++++++++-------- .../Qdrant/QdrantFilterTranslator.cs | 30 +++++ .../VectorData/Qdrant/QdrantModelBuilder.cs | 10 +- .../SqliteVec/SqliteFilterTranslator.cs | 14 ++ .../src/VectorData/SqliteVec/SqliteMapper.cs | 6 + .../SqliteVec/SqliteModelBuilder.cs | 15 ++- .../SqliteVec/SqlitePropertyMapping.cs | 8 ++ .../Filter/FilterTranslatorBase.cs | 8 +- .../Converters/WeaviateDateOnlyConverter.cs | 40 ++++++ .../Converters/WeaviateDateTimeConverter.cs | 42 ++++++ .../WeaviateNullableDateOnlyConverter.cs | 47 +++++++ .../WeaviateNullableDateTimeConverter.cs | 50 +++++++ .../WeaviateCollectionCreateMapping.cs | 3 + .../VectorData/Weaviate/WeaviateConstants.cs | 8 +- .../Weaviate/WeaviateFilterTranslator.cs | 8 +- .../Weaviate/WeaviateModelBuilder.cs | 9 +- .../TypeTests/AzureAISearchDataTypeTests.cs | 38 +++--- .../TypeTests/CosmosMongoDataTypeTests.cs | 10 +- .../TypeTests/CosmosNoSqlDataTypeTests.cs | 2 - .../TypeTests/MongoDataTypeTests.cs | 20 ++- .../TypeTests/PostgresDataTypeTests.cs | 4 - .../TypeTests/QdrantDataTypeTests.cs | 2 - .../TypeTests/SqliteDataTypeTests.cs | 6 - .../TypeTests/WeaviateDataTypeTests.cs | 2 - 40 files changed, 624 insertions(+), 116 deletions(-) create mode 100644 dotnet/src/VectorData/Weaviate/Converters/WeaviateDateOnlyConverter.cs create mode 100644 dotnet/src/VectorData/Weaviate/Converters/WeaviateDateTimeConverter.cs create mode 100644 dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs create mode 100644 dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs index 6787d9e07166..244149d61319 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/BsonValueFactory.cs @@ -21,10 +21,15 @@ public static BsonValue Create(object? value) => value switch { null => BsonNull.Value, - Guid guid => new BsonBinaryData(guid, GuidRepresentation.Standard), - object[] array => new BsonArray(Array.ConvertAll(array, Create)), - Array array => new BsonArray(array), - IEnumerable enumerable => new BsonArray(enumerable.Select(Create)), + Guid v => new BsonBinaryData(v, GuidRepresentation.Standard), + DateTimeOffset v => new BsonDateTime(v.UtcDateTime), +#if NET + DateOnly v => new BsonDateTime(v.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc)), +#endif + object[] v => new BsonArray(Array.ConvertAll(v, Create)), + Array v => new BsonArray(v), + IEnumerable v => new BsonArray(v.Select(Create)), + _ => BsonValue.Create(value) }; } diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs index 4ee72ae2ac8f..791de1ea231c 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoDynamicMapper.cs @@ -160,7 +160,7 @@ Embedding e Type t when t == typeof(int?) => value.AsNullableInt32, Type t when t == typeof(long) => value.AsInt64, Type t when t == typeof(long?) => value.AsNullableInt64, - Type t when t == typeof(float) => ((float)value.AsDouble), + Type t when t == typeof(float) => (float)value.AsDouble, Type t when t == typeof(float?) => ((float?)value.AsNullableDouble), Type t when t == typeof(double) => value.AsDouble, Type t when t == typeof(double?) => value.AsNullableDouble, @@ -168,6 +168,16 @@ Embedding e Type t when t == typeof(decimal?) => value.AsNullableDecimal, Type t when t == typeof(DateTime) => value.ToUniversalTime(), Type t when t == typeof(DateTime?) => value.ToNullableUniversalTime(), + Type t when t == typeof(DateTimeOffset) => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero), + Type t when t == typeof(DateTimeOffset?) => value.ToNullableUniversalTime() is DateTime dateTime + ? new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero) + : null, +#if NET + Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.ToUniversalTime()), + Type t when t == typeof(DateOnly?) => value.ToNullableUniversalTime() is DateTime dateTime + ? DateOnly.FromDateTime(dateTime) + : null, +#endif _ => (object?)null }; diff --git a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs index c8926258bd3c..dde3980815c1 100644 --- a/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs +++ b/dotnet/src/InternalUtilities/connectors/Memory/MongoDB/MongoModelBuilder.cs @@ -58,7 +58,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types"; + supportedTypes = "string, int, long, double, float, bool, decimal, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; if (Nullable.GetUnderlyingType(type) is Type underlyingType) { @@ -77,7 +81,12 @@ static bool IsValid(Type type) type == typeof(float) || type == typeof(double) || type == typeof(decimal) || - type == typeof(DateTime); + type == typeof(DateTime) || + type == typeof(DateTimeOffset) || +#if NET + type == typeof(DateOnly) || +#endif + false; } protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs index d39019a729ad..c21053e15dda 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchCollectionCreateMapping.cs @@ -128,7 +128,11 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType) // Half is also listed by the SDK, but currently not supported. Type t when t == typeof(float) => SearchFieldDataType.Double, Type t when t == typeof(double) => SearchFieldDataType.Double, + Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset, Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset, +#if NET + Type t when t == typeof(DateOnly) => SearchFieldDataType.DateTimeOffset, +#endif Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String), Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.String), @@ -142,8 +146,14 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType) Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double), Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double), Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.Double), + Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), +#if NET + Type t when t == typeof(DateOnly[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), + Type t when t == typeof(List) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset), +#endif _ => throw new NotSupportedException($"Data type '{propertyType}' for {nameof(VectorStoreDataProperty)} is not supported by the Azure AI Search VectorStore.") }; diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs index 6691a0a6c611..5b8024e5505d 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchDynamicMapper.cs @@ -41,7 +41,7 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue)) { jsonObject[dataProperty.StorageName] = dataValue is not null ? - JsonSerializer.SerializeToNode(dataValue, dataProperty.Type, jsonSerializerOptions) : + JsonSerializer.SerializeToNode(ConvertToStorageValue(dataValue), GetStorageType(dataProperty.Type), jsonSerializerOptions) : null; } } @@ -183,6 +183,10 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode Type t when t == typeof(DateTime?) => value.GetValue(), Type t when t == typeof(DateTimeOffset) => value.GetValue(), Type t when t == typeof(DateTimeOffset?) => value.GetValue(), +#if NET + Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.GetValue().DateTime), + Type t when t == typeof(DateOnly?) => (DateOnly?)DateOnly.FromDateTime(value.GetValue().DateTime), +#endif _ => (object?)null }; @@ -242,4 +246,38 @@ public JsonObject MapFromDataToStorageModel(Dictionary dataMode throw new UnreachableException($"Unsupported property type '{propertyType.Name}'."); } + + /// + /// Converts a value to its storage representation for Azure AI Search. + /// Azure AI Search only supports for date/time types, and always internally + /// converts to UTC for storage. + /// + /// + /// Gets the storage type for a given property type. Azure AI Search stores DateTime and DateOnly as DateTimeOffset. + /// + private static Type GetStorageType(Type propertyType) + { + var underlying = Nullable.GetUnderlyingType(propertyType) ?? propertyType; + + if (underlying == typeof(DateTime) +#if NET + || underlying == typeof(DateOnly) +#endif + ) + { + return propertyType == underlying ? typeof(DateTimeOffset) : typeof(DateTimeOffset?); + } + + return propertyType; + } + + private static object ConvertToStorageValue(object value) + => value switch + { + DateTime dateTime => new DateTimeOffset(dateTime, TimeSpan.Zero), +#if NET + DateOnly dateOnly => new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero), +#endif + _ => value + }; } diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs index d149a7f96e1d..18b58eba855c 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchFilterTranslator.cs @@ -119,9 +119,17 @@ private void GenerateLiteral(object? value) this._filter.Append('\'').Append(g.ToString()).Append('\''); return; + case DateTime d: + this._filter.Append(new DateTimeOffset(d, TimeSpan.Zero).ToString("o")); + return; case DateTimeOffset d: this._filter.Append(d.ToString("o")); return; +#if NET + case DateOnly d: + this._filter.Append(new DateTimeOffset(d.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero).ToString("o")); + return; +#endif case Array: throw new NotImplementedException(); diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs index a4ac12b405cf..88b554845552 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs @@ -20,6 +20,29 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, var jsonObject = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject(); + // Azure AI Search only supports Edm.DateTimeOffset for date/time types. + // DateTime properties may be serialized by STJ without timezone info (when Kind=Unspecified), + // which Azure AI Search rejects. Convert them to DateTimeOffset (UTC). + // DateOnly properties are serialized as "yyyy-MM-dd", which also isn't accepted. + // Convert them to full DateTimeOffset representation (midnight UTC). + foreach (var dataProperty in model.DataProperties) + { + var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type; + + if (propertyType == typeof(DateTime) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null) + { + var dateTime = dtNode.Deserialize(jsonSerializerOptions); + jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), TimeSpan.Zero)); + } +#if NET + else if (propertyType == typeof(DateOnly) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null) + { + var dateOnly = dateNode.Deserialize(jsonSerializerOptions); + jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)); + } +#endif + } + // Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above. // Also, for Embedding properties we also need to overwrite with a simple array (since Embedding gets serialized as a complex object). for (var i = 0; i < model.VectorProperties.Count; i++) @@ -64,6 +87,27 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex, public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors) { + // Azure AI Search stores DateTime properties as Edm.DateTimeOffset. + // Convert them back to DateTime so STJ can deserialize them into the correct type. + // DateOnly properties also need to be converted back from DateTimeOffset. + foreach (var dataProperty in model.DataProperties) + { + var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type; + + if (propertyType == typeof(DateTime) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null) + { + var dateTimeOffset = dtNode.Deserialize(jsonSerializerOptions); + storageModel[dataProperty.StorageName] = JsonValue.Create(dateTimeOffset.UtcDateTime); + } +#if NET + else if (propertyType == typeof(DateOnly) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null) + { + var dateTimeOffset = dateNode.Deserialize(jsonSerializerOptions); + storageModel[dataProperty.StorageName] = JsonValue.Create(DateOnly.FromDateTime(dateTimeOffset.DateTime)); + } +#endif + } + if (includeVectors) { foreach (var vectorProperty in model.VectorProperties) diff --git a/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs b/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs index 31f90d77dd3c..7c6abd8c087a 100644 --- a/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs +++ b/dotnet/src/VectorData/AzureAISearch/AzureAISearchModelBuilder.cs @@ -40,7 +40,11 @@ protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false) internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types"; + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; if (Nullable.GetUnderlyingType(type) is Type underlyingType) { @@ -58,6 +62,10 @@ static bool IsValid(Type type) type == typeof(double) || type == typeof(float) || type == typeof(bool) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif type == typeof(DateTimeOffset); } diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs index 2849d41dd3cf..63f173a8cf45 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlFilterTranslator.cs @@ -136,6 +136,22 @@ private void TranslateConstant(object? value) .Append("Z\""); return; + case DateTime v: + this._sql + .Append('"') + .Append(v.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)) + .Append('"'); + return; + +#if NET + case DateOnly v: + this._sql + .Append('"') + .Append(v.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)) + .Append('"'); + return; +#endif + case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)): this._sql.Append('['); diff --git a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs index fcbd60877571..e70511c16f0a 100644 --- a/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs +++ b/dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs @@ -36,7 +36,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types"; + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; if (Nullable.GetUnderlyingType(type) is Type underlyingType) { @@ -54,6 +58,10 @@ static bool IsValid(Type type) type == typeof(long) || type == typeof(float) || type == typeof(double) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif type == typeof(DateTimeOffset); } diff --git a/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs b/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs index 7c05f90639b2..9f32e3d0f4be 100644 --- a/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs +++ b/dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs @@ -69,7 +69,11 @@ private BsonDocument GenerateEqualityComparison(PropertyModel property, object? throw new NotSupportedException("MongoDB does not support null checks in vector search pre-filters"); } - if (value is DateTime or decimal or IList) + if (value is DateTime or DateTimeOffset or decimal or IList +#if NET + or DateOnly +#endif + ) { // Operand type is not supported for $vectorSearch: date/decimal throw new NotSupportedException($"MongoDB does not support type {value.GetType().Name} in vector search pre-filters."); diff --git a/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs b/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs index 198cde9e97a5..eb6d38f9ddb8 100644 --- a/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs +++ b/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs @@ -55,6 +55,16 @@ protected override void TranslateConstant(object? value, bool isSearchCondition) this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\''); return; +#if NET + case DateOnly dateOnly: + this._sql.Append('\'').Append(dateOnly.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)).Append('\''); + return; + + case TimeOnly timeOnly: + this._sql.Append('\'').Append(timeOnly.ToString("HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture)).Append('\''); + return; +#endif + // Array constants (ARRAY[1, 2, 3]) case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)): this._sql.Append("ARRAY["); diff --git a/dotnet/src/VectorData/PgVector/PostgresMapper.cs b/dotnet/src/VectorData/PgVector/PostgresMapper.cs index d85f87177d54..57d4c2b56a18 100644 --- a/dotnet/src/VectorData/PgVector/PostgresMapper.cs +++ b/dotnet/src/VectorData/PgVector/PostgresMapper.cs @@ -156,6 +156,14 @@ private static void PopulateProperty(PropertyModel property, NpgsqlDataReader re case var t when t == typeof(DateTimeOffset): property.SetValue(record, reader.GetFieldValue(ordinal)); return; +#if NET + case var t when t == typeof(DateOnly): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; + case var t when t == typeof(TimeOnly): + property.SetValue(record, reader.GetFieldValue(ordinal)); + return; +#endif case var t when t == typeof(byte[]): property.SetValue(record, reader.GetFieldValue(ordinal)); return; @@ -203,6 +211,14 @@ private static void PopulateProperty(PropertyModel property, NpgsqlDataReader re case Type t when t == typeof(List): property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); return; +#if NET + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; + case Type t when t == typeof(List): + property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); + return; +#endif case Type t when t == typeof(List): property.SetValueAsObject(record, reader.GetFieldValue>(ordinal)); return; diff --git a/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs b/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs index 5b87dbe02d04..16769d699570 100644 --- a/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs +++ b/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs @@ -65,6 +65,10 @@ static bool IsValid(Type type) type == typeof(byte[]) || type == typeof(DateTime) || type == typeof(DateTimeOffset) || +#if NET + type == typeof(DateOnly) || + type == typeof(TimeOnly) || +#endif type == typeof(Guid); } diff --git a/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs b/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs index b1111b1c0282..3143b2f3f06c 100644 --- a/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs +++ b/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs @@ -61,6 +61,11 @@ internal static class PostgresPropertyMapping Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp, Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz, +#if NET + Type t when t == typeof(DateOnly) => NpgsqlDbType.Date, + Type t when t == typeof(TimeOnly) => NpgsqlDbType.Time, +#endif + _ => null }; @@ -84,6 +89,10 @@ static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName) Type t when t == typeof(byte[]) => "BYTEA", Type t when t == typeof(DateTime) => "TIMESTAMPTZ", Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ", +#if NET + Type t when t == typeof(DateOnly) => "DATE", + Type t when t == typeof(TimeOnly) => "TIME", +#endif Type t when t == typeof(Guid) => "UUID", _ => null }; diff --git a/dotnet/src/VectorData/Qdrant/QdrantCollectionCreateMapping.cs b/dotnet/src/VectorData/Qdrant/QdrantCollectionCreateMapping.cs index 4cceebc2e156..c03145ce8fb5 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantCollectionCreateMapping.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantCollectionCreateMapping.cs @@ -41,11 +41,18 @@ internal static class QdrantCollectionCreateMapping { typeof(decimal?), PayloadSchemaType.Float }, { typeof(string), PayloadSchemaType.Keyword }, - { typeof(DateTimeOffset), PayloadSchemaType.Datetime }, { typeof(bool), PayloadSchemaType.Bool }, + { typeof(bool?), PayloadSchemaType.Bool }, + { typeof(DateTime), PayloadSchemaType.Datetime }, + { typeof(DateTimeOffset), PayloadSchemaType.Datetime }, + { typeof(DateTime?), PayloadSchemaType.Datetime }, { typeof(DateTimeOffset?), PayloadSchemaType.Datetime }, - { typeof(bool?), PayloadSchemaType.Bool }, + +#if NET + { typeof(DateOnly), PayloadSchemaType.Datetime }, + { typeof(DateOnly?), PayloadSchemaType.Datetime }, +#endif }; /// diff --git a/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs b/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs index d3aa55609dbc..16150765e4a4 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs @@ -37,7 +37,15 @@ internal static class QdrantFieldMapping => targetType == typeof(int) ? (object)(int)payloadValue.IntegerValue : (object)payloadValue.IntegerValue, Value.KindOneofCase.StringValue when targetType == typeof(DateTimeOffset) - => DeserializeDateTimeOffset(payloadValue.StringValue), + => DateTimeOffset.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + + Value.KindOneofCase.StringValue when targetType == typeof(DateTime) + => DateTime.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind), + +#if NET + Value.KindOneofCase.StringValue when targetType == typeof(DateOnly) + => DateOnly.Parse(payloadValue.StringValue, CultureInfo.InvariantCulture), +#endif Value.KindOneofCase.StringValue => payloadValue.StringValue, @@ -85,14 +93,24 @@ internal static class QdrantFieldMapping => payloadValue.ListValue.Values.Select(v => v.BoolValue).ToArray(), Type t when t == typeof(List) - => payloadValue.ListValue.Values.Select(v => DeserializeDateTimeOffset(v.StringValue)).ToList(), + => payloadValue.ListValue.Values.Select(v => DateTimeOffset.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToList(), Type t when t == typeof(DateTimeOffset[]) - => payloadValue.ListValue.Values.Select(v => DeserializeDateTimeOffset(v.StringValue)).ToArray(), + => payloadValue.ListValue.Values.Select(v => DateTimeOffset.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToArray(), + + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => DateTime.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToList(), + Type t when t == typeof(DateTime[]) + => payloadValue.ListValue.Values.Select(v => DateTime.Parse(v.StringValue, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind)).ToArray(), + +#if NET + Type t when t == typeof(List) + => payloadValue.ListValue.Values.Select(v => DateOnly.Parse(v.StringValue, CultureInfo.InvariantCulture)).ToList(), + Type t when t == typeof(DateOnly[]) + => payloadValue.ListValue.Values.Select(v => DateOnly.Parse(v.StringValue, CultureInfo.InvariantCulture)).ToArray(), +#endif _ => throw new UnreachableException($"Unsupported collection type {targetType.Name}"), }; - - static DateTimeOffset DeserializeDateTimeOffset(string s) => DateTimeOffset.Parse(s, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind); } /// @@ -104,57 +122,59 @@ internal static class QdrantFieldMapping public static Value ConvertToGrpcFieldValue(object? sourceValue) { var value = new Value(); - if (sourceValue is null) - { - value.NullValue = NullValue.NullValue; - } - else if (sourceValue is int intValue) - { - value.IntegerValue = intValue; - } - else if (sourceValue is long longValue) - { - value.IntegerValue = longValue; - } - else if (sourceValue is string stringValue) - { - value.StringValue = stringValue; - } - else if (sourceValue is float floatValue) - { - value.DoubleValue = floatValue; - } - else if (sourceValue is double doubleValue) - { - value.DoubleValue = doubleValue; - } - else if (sourceValue is bool boolValue) - { - value.BoolValue = boolValue; - } - else if (sourceValue is DateTimeOffset dateTimeOffsetValue) - { - value.StringValue = dateTimeOffsetValue.ToString("O"); - } - else if (sourceValue is IEnumerable or - IEnumerable or - IEnumerable or - IEnumerable or - IEnumerable or - IEnumerable or - IEnumerable or - IEnumerable) + switch (sourceValue) { - var listValue = sourceValue as IEnumerable; - value.ListValue = new ListValue(); - foreach (var item in listValue!) + case null: + value.NullValue = NullValue.NullValue; + break; + case int intValue: + value.IntegerValue = intValue; + break; + case long longValue: + value.IntegerValue = longValue; + break; + case string stringValue: + value.StringValue = stringValue; + break; + case float floatValue: + value.DoubleValue = floatValue; + break; + case double doubleValue: + value.DoubleValue = doubleValue; + break; + case bool boolValue: + value.BoolValue = boolValue; + break; + case DateTimeOffset dateTimeOffsetValue: + value.StringValue = dateTimeOffsetValue.ToString("O"); + break; + case DateTime dateTimeValue: + value.StringValue = dateTimeValue.ToString("O"); + break; + case DateOnly dateOnlyValue: + value.StringValue = dateOnlyValue.ToString("O"); + break; + case IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable or + IEnumerable: { - value.ListValue.Values.Add(ConvertToGrpcFieldValue(item)); + var listValue = sourceValue as IEnumerable; + value.ListValue = new ListValue(); + foreach (var item in listValue!) + { + value.ListValue.Values.Add(ConvertToGrpcFieldValue(item)); + } + + break; } - } - else - { - throw new InvalidOperationException($"Unsupported source value type {sourceValue?.GetType().FullName}."); + + default: + throw new InvalidOperationException($"Unsupported source value type {sourceValue?.GetType().FullName}."); } return value; diff --git a/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs b/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs index 0e13bd0d8801..aeebd5b72dce 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantFilterTranslator.cs @@ -81,7 +81,11 @@ private Filter GenerateEqual(string propertyStorageName, object? value, bool neg int v => new Match { Integer = v }, long v => new Match { Integer = v }, bool v => new Match { Boolean = v }, + DateTime v => new Match { Keyword = v.ToString("o") }, DateTimeOffset v => new Match { Keyword = v.ToString("o") }, +#if NET + DateOnly v => new Match { Keyword = v.ToString("O") }, +#endif _ => throw new NotSupportedException($"Unsupported filter value type '{value.GetType().Name}'.") } @@ -135,6 +139,32 @@ bool TryProcessComparison(Expression first, Expression second, [NotNullWhen(true } }, + DateTime v => new FieldCondition + { + Key = property.StorageName, + DatetimeRange = new DatetimeRange + { + Gt = comparison.NodeType == ExpressionType.GreaterThan ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Gte = comparison.NodeType == ExpressionType.GreaterThanOrEqual ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Lt = comparison.NodeType == ExpressionType.LessThan ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null, + Lte = comparison.NodeType == ExpressionType.LessThanOrEqual ? Timestamp.FromDateTime(DateTime.SpecifyKind(v, DateTimeKind.Utc)) : null + } + }, + +#if NET + DateOnly v => new FieldCondition + { + Key = property.StorageName, + DatetimeRange = new DatetimeRange + { + Gt = comparison.NodeType == ExpressionType.GreaterThan ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Gte = comparison.NodeType == ExpressionType.GreaterThanOrEqual ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Lt = comparison.NodeType == ExpressionType.LessThan ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null, + Lte = comparison.NodeType == ExpressionType.LessThanOrEqual ? Timestamp.FromDateTimeOffset(new DateTimeOffset(v.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero)) : null + } + }, +#endif + _ => throw new NotSupportedException($"Can't perform comparison on type '{constantValue?.GetType().Name}'") } }); diff --git a/dotnet/src/VectorData/Qdrant/QdrantModelBuilder.cs b/dotnet/src/VectorData/Qdrant/QdrantModelBuilder.cs index a290c303b85e..771829cc6028 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantModelBuilder.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantModelBuilder.cs @@ -33,7 +33,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types"; + supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " or arrays/lists of these types"; if (Nullable.GetUnderlyingType(type) is Type underlyingType) { @@ -51,6 +55,10 @@ static bool IsValid(Type type) type == typeof(double) || type == typeof(float) || type == typeof(bool) || + type == typeof(DateTime) || +#if NET + type == typeof(DateOnly) || +#endif type == typeof(DateTimeOffset); } diff --git a/dotnet/src/VectorData/SqliteVec/SqliteFilterTranslator.cs b/dotnet/src/VectorData/SqliteVec/SqliteFilterTranslator.cs index c33fc31b080a..3baed486eead 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteFilterTranslator.cs +++ b/dotnet/src/VectorData/SqliteVec/SqliteFilterTranslator.cs @@ -27,6 +27,20 @@ protected override void TranslateConstant(object? value, bool isSearchCondition) // Microsoft.Data.Sqlite writes GUIDs as upper-case strings, align our constant formatting with that. this._sql.Append('\'').Append(g.ToString().ToUpperInvariant()).Append('\''); break; + case DateTime dateTime: + this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFF", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + break; + case DateTimeOffset dateTimeOffset: + this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-dd HH:mm:ss.FFFFFFFzzz", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + break; +#if NET + case DateOnly dateOnly: + this._sql.Append('\'').Append(dateOnly.ToString("yyyy-MM-dd", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + break; + case TimeOnly timeOnly: + this._sql.Append('\'').Append(timeOnly.ToString("HH:mm:ss.FFFFFFF", System.Globalization.CultureInfo.InvariantCulture)).Append('\''); + break; +#endif default: base.TranslateConstant(value, isSearchCondition); break; diff --git a/dotnet/src/VectorData/SqliteVec/SqliteMapper.cs b/dotnet/src/VectorData/SqliteVec/SqliteMapper.cs index 2e34082ba47d..9b4da455b682 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteMapper.cs +++ b/dotnet/src/VectorData/SqliteVec/SqliteMapper.cs @@ -97,6 +97,12 @@ public TRecord MapFromStorageToDataModel(DbDataReader reader, bool includeVector Type t when t == typeof(double) => reader.GetDouble(ordinal), Type t when t == typeof(string) => reader.GetString(ordinal), Type t when t == typeof(Guid) => reader.GetGuid(ordinal), + Type t when t == typeof(DateTime) => reader.GetDateTime(ordinal), + Type t when t == typeof(DateTimeOffset) => reader.GetFieldValue(ordinal), +#if NET + Type t when t == typeof(DateOnly) => reader.GetFieldValue(ordinal), + Type t when t == typeof(TimeOnly) => reader.GetFieldValue(ordinal), +#endif Type t when t == typeof(byte[]) => (byte[])reader[ordinal], Type t when t == typeof(ReadOnlyMemory) => (byte[])reader[ordinal], Type t when t == typeof(Embedding) => (byte[])reader[ordinal], diff --git a/dotnet/src/VectorData/SqliteVec/SqliteModelBuilder.cs b/dotnet/src/VectorData/SqliteVec/SqliteModelBuilder.cs index 3f2935fe68ab..8855c54d26e1 100644 --- a/dotnet/src/VectorData/SqliteVec/SqliteModelBuilder.cs +++ b/dotnet/src/VectorData/SqliteVec/SqliteModelBuilder.cs @@ -33,7 +33,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "int, long, short, string, bool, float, double, byte[], Guid"; + supportedTypes = "int, long, short, string, bool, float, double, byte[], Guid, DateTime, DateTimeOffset" +#if NET + + ", DateOnly, TimeOnly" +#endif + ; if (Nullable.GetUnderlyingType(type) is Type underlyingType) { @@ -48,7 +52,14 @@ protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] || type == typeof(float) || type == typeof(double) || type == typeof(byte[]) - || type == typeof(Guid); + || type == typeof(Guid) + || type == typeof(DateTime) + || type == typeof(DateTimeOffset) +#if NET + || type == typeof(DateOnly) + || type == typeof(TimeOnly) +#endif + ; } protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) diff --git a/dotnet/src/VectorData/SqliteVec/SqlitePropertyMapping.cs b/dotnet/src/VectorData/SqliteVec/SqlitePropertyMapping.cs index 20f745e1799f..404b43f0f238 100644 --- a/dotnet/src/VectorData/SqliteVec/SqlitePropertyMapping.cs +++ b/dotnet/src/VectorData/SqliteVec/SqlitePropertyMapping.cs @@ -113,6 +113,14 @@ private static string GetStorageDataPropertyType(PropertyModel property) // Guid type - represent as TEXT Type t when t == typeof(Guid) || t == typeof(Guid?) => "TEXT", + // Date/time types - represent as TEXT (ISO 8601) + Type t when t == typeof(DateTime) || t == typeof(DateTime?) => "TEXT", + Type t when t == typeof(DateTimeOffset) || t == typeof(DateTimeOffset?) => "TEXT", +#if NET + Type t when t == typeof(DateOnly) || t == typeof(DateOnly?) => "TEXT", + Type t when t == typeof(TimeOnly) || t == typeof(TimeOnly?) => "TEXT", +#endif + // Byte array (BLOB) Type t when t == typeof(byte[]) => "BLOB", diff --git a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs index bc5a2f50f6c9..ad847b34b1d1 100644 --- a/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs +++ b/dotnet/src/VectorData/VectorData.Abstractions/ProviderServices/Filter/FilterTranslatorBase.cs @@ -348,10 +348,14 @@ protected override Expression VisitNew(NewExpression node) var visited = (NewExpression)base.VisitNew(node); // Recognize certain well-known constructors where we can evaluate immediately, converting the NewExpression to a ConstantExpression. - // This is particularly useful for converting inline instantiation of DateTime and DateTimeOffset to constants, which can then be easily translated. + // This is particularly useful for converting inline instantiation of DateTime, DateTimeOffset, DateOnly, and TimeOnly to constants, which can then be easily translated. switch (visited.Constructor) { - case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime): + case ConstructorInfo constructor when constructor.DeclaringType == typeof(DateTimeOffset) || constructor.DeclaringType == typeof(DateTime) +#if NET + || constructor.DeclaringType == typeof(DateOnly) || constructor.DeclaringType == typeof(TimeOnly) +#endif + : var constantArguments = new object?[visited.Arguments.Count]; // We first do a fast path to check if all arguments are constants; this catches the common case of e.g. new DateTime(2023, 10, 1). diff --git a/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateOnlyConverter.cs b/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateOnlyConverter.cs new file mode 100644 index 000000000000..1e0deb1cffc5 --- /dev/null +++ b/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateOnlyConverter.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. + +#if NET + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts to RFC 3339 formatted string for Weaviate. +/// DateOnly values are stored as midnight UTC. +/// +internal sealed class WeaviateDateOnlyConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffZ"; + + public override DateOnly Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + var dateTimeOffset = DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + return DateOnly.FromDateTime(dateTimeOffset.DateTime); + } + + public override void Write(Utf8JsonWriter writer, DateOnly value, JsonSerializerOptions options) + { + var dateTimeOffset = new DateTimeOffset(value.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + writer.WriteStringValue(dateTimeOffset.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} + +#endif diff --git a/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateTimeConverter.cs b/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateTimeConverter.cs new file mode 100644 index 000000000000..d52ace646307 --- /dev/null +++ b/dotnet/src/VectorData/Weaviate/Converters/WeaviateDateTimeConverter.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts to RFC 3339 formatted string for Weaviate. +/// +internal sealed class WeaviateDateTimeConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTime Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return default; + } + + // Parse as DateTimeOffset to properly handle timezone, then convert to UTC DateTime. + // Weaviate may return the timestamp in a different timezone than it was stored in. + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture).UtcDateTime; + } + + public override void Write(Utf8JsonWriter writer, DateTime value, JsonSerializerOptions options) + { + // When DateTime.Kind is Unspecified, the 'K' format specifier produces an empty string (no timezone), + // which violates RFC 3339. Treat Unspecified as UTC so 'K' produces 'Z'. + if (value.Kind == DateTimeKind.Unspecified) + { + value = DateTime.SpecifyKind(value, DateTimeKind.Utc); + } + + writer.WriteStringValue(value.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } +} diff --git a/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs b/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs new file mode 100644 index 000000000000..3284046dda28 --- /dev/null +++ b/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateOnlyConverter.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. + +#if NET + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts nullable to RFC 3339 formatted string for Weaviate. +/// DateOnly values are stored as midnight UTC. +/// +internal sealed class WeaviateNullableDateOnlyConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffZ"; + + public override DateOnly? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + var dateTimeOffset = DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture); + return DateOnly.FromDateTime(dateTimeOffset.DateTime); + } + + public override void Write(Utf8JsonWriter writer, DateOnly? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + var dateTimeOffset = new DateTimeOffset(value.Value.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero); + writer.WriteStringValue(dateTimeOffset.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} + +#endif diff --git a/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs b/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs new file mode 100644 index 000000000000..c1ee47b2449d --- /dev/null +++ b/dotnet/src/VectorData/Weaviate/Converters/WeaviateNullableDateTimeConverter.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Globalization; +using System.Text.Json; +using System.Text.Json.Serialization; + +namespace Microsoft.SemanticKernel.Connectors.Weaviate; + +/// +/// Converts nullable to RFC 3339 formatted string for Weaviate. +/// +internal sealed class WeaviateNullableDateTimeConverter : JsonConverter +{ + private const string DateTimeFormat = "yyyy-MM-dd'T'HH:mm:ss.fffK"; + + public override DateTime? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + var dateString = reader.GetString(); + + if (string.IsNullOrWhiteSpace(dateString)) + { + return null; + } + + // Parse as DateTimeOffset to properly handle timezone, then convert to UTC DateTime. + // Weaviate may return the timestamp in a different timezone than it was stored in. + return DateTimeOffset.Parse(dateString, CultureInfo.InvariantCulture).UtcDateTime; + } + + public override void Write(Utf8JsonWriter writer, DateTime? value, JsonSerializerOptions options) + { + if (value.HasValue) + { + // When DateTime.Kind is Unspecified, the 'K' format specifier produces an empty string (no timezone), + // which violates RFC 3339. Treat Unspecified as UTC so 'K' produces 'Z'. + var v = value.Value; + if (v.Kind == DateTimeKind.Unspecified) + { + v = DateTime.SpecifyKind(v, DateTimeKind.Utc); + } + + writer.WriteStringValue(v.ToString(DateTimeFormat, CultureInfo.InvariantCulture)); + } + else + { + writer.WriteNullValue(); + } + } +} diff --git a/dotnet/src/VectorData/Weaviate/WeaviateCollectionCreateMapping.cs b/dotnet/src/VectorData/Weaviate/WeaviateCollectionCreateMapping.cs index 8f28a0eb060a..dc7c53977ffe 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateCollectionCreateMapping.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateCollectionCreateMapping.cs @@ -155,6 +155,9 @@ bool TryMapType(Type type, [NotNullWhen(true)] out string? mappedType) Type t when t == typeof(int) || t == typeof(long) || t == typeof(short) || t == typeof(byte) => "int", Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) => "number", Type t when t == typeof(DateTime) || t == typeof(DateTimeOffset) => "date", +#if NET + Type t when t == typeof(DateOnly) => "date", +#endif Type t when t == typeof(Guid) => "uuid", Type t when t == typeof(bool) => "boolean", diff --git a/dotnet/src/VectorData/Weaviate/WeaviateConstants.cs b/dotnet/src/VectorData/Weaviate/WeaviateConstants.cs index d7d190134a7e..084e1771b34e 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateConstants.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateConstants.cs @@ -43,7 +43,13 @@ internal sealed class WeaviateConstants Converters = { new WeaviateDateTimeOffsetConverter(), - new WeaviateNullableDateTimeOffsetConverter() + new WeaviateNullableDateTimeOffsetConverter(), + new WeaviateDateTimeConverter(), + new WeaviateNullableDateTimeConverter(), +#if NET + new WeaviateDateOnlyConverter(), + new WeaviateNullableDateOnlyConverter(), +#endif } }; } diff --git a/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs b/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs index 5efa580044dd..cf0a745aef33 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateFilterTranslator.cs @@ -175,15 +175,19 @@ private void GenerateEqualityComparison(string propertyStorageName, object? valu Type t when t == typeof(bool) => "valueBoolean", Type t when t == typeof(string) || t == typeof(Guid) => "valueText", Type t when t == typeof(float) || t == typeof(double) || t == typeof(decimal) => "valueNumber", + Type t when t == typeof(DateTime) => "valueDate", Type t when t == typeof(DateTimeOffset) => "valueDate", +#if NET + Type t when t == typeof(DateOnly) => "valueDate", +#endif _ => throw new NotSupportedException($"Unsupported value type {type.FullName} in filter.") }); this._filter.Append(": "); - // Value - this._filter.Append(JsonSerializer.Serialize(value)); + // Value — use Weaviate's JSON serializer options for proper date/time formatting + this._filter.Append(JsonSerializer.Serialize(value, value.GetType(), WeaviateConstants.s_jsonSerializerOptions)); this._filter.Append('}'); } diff --git a/dotnet/src/VectorData/Weaviate/WeaviateModelBuilder.cs b/dotnet/src/VectorData/Weaviate/WeaviateModelBuilder.cs index 9a1f429849de..89b208111d1a 100644 --- a/dotnet/src/VectorData/Weaviate/WeaviateModelBuilder.cs +++ b/dotnet/src/VectorData/Weaviate/WeaviateModelBuilder.cs @@ -34,7 +34,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty) protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes) { - supportedTypes = "string, bool, int, long, short, byte, float, double, decimal, DateTime, DateTimeOffset, Guid, or arrays/lists of these types"; + supportedTypes = "string, bool, int, long, short, byte, float, double, decimal, DateTime, DateTimeOffset," +#if NET + + " DateOnly," +#endif + + " Guid, or arrays/lists of these types"; return IsValid(type) || (type.IsArray && IsValid(type.GetElementType()!)) @@ -58,6 +62,9 @@ static bool IsValid(Type type) || type == typeof(decimal) || type == typeof(DateTime) || type == typeof(DateTimeOffset) +#if NET + || type == typeof(DateOnly) +#endif || type == typeof(Guid); } } diff --git a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs index 5793a35339f1..535f51632b29 100644 --- a/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs +++ b/dotnet/test/VectorData/AzureAISearch.ConformanceTests/TypeTests/AzureAISearchDataTypeTests.cs @@ -13,25 +13,7 @@ public class AzureAISearchDataTypeTests(AzureAISearchDataTypeTests.Fixture fixtu : DataTypeTests(fixture), IClassFixture { - public override Task Byte() => Task.CompletedTask; - public override Task Short() => Task.CompletedTask; - public override Task Decimal() => Task.CompletedTask; - - [ConditionalFact(Skip = "Guid not yet supported")] - public override Task Guid() => Task.CompletedTask; - - [ConditionalFact(Skip = "DateTime not yet supported")] - public override Task DateTime() => Task.CompletedTask; - - // [ConditionalFact(Skip = "DateTimeOffset not yet supported")] - // public override Task DateTimeOffset() => Task.CompletedTask; - - [ConditionalFact(Skip = "DateOnly not yet supported")] - public override Task DateOnly() => Task.CompletedTask; - - [ConditionalFact(Skip = "TimeOnly not yet supported")] - public override Task TimeOnly() => Task.CompletedTask; - + [ConditionalFact(Skip = "Issues around empty collection initialization")] public override Task String_array() => Task.CompletedTask; protected override object? GenerateEmptyProperty(VectorStoreProperty property) @@ -55,13 +37,22 @@ public override IList GetDataProperties() && p.Type != typeof(short) && p.Type != typeof(decimal) && p.Type != typeof(Guid) - && p.Type != typeof(DateTime) #if NET - && p.Type != typeof(DateOnly) && p.Type != typeof(TimeOnly) #endif ).ToList(); + public override Type[] UnsupportedDefaultTypes { get; } = + [ + typeof(byte), + typeof(short), + typeof(decimal), + typeof(Guid), +#if NET + typeof(TimeOnly) +#endif + ]; + public class AzureAISearchRecord : RecordBase { public int Int { get; set; } @@ -72,8 +63,13 @@ public class AzureAISearchRecord : RecordBase public string? String { get; set; } public bool Bool { get; set; } + public DateTime DateTime { get; set; } public DateTimeOffset DateTimeOffset { get; set; } +#if NET + public DateOnly DateOnly { get; set; } +#endif + public string[] StringArray { get; set; } = null!; public int? NullableInt { get; set; } diff --git a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs index c613b83234de..258a22f93028 100644 --- a/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs +++ b/dotnet/test/VectorData/CosmosMongoDB.ConformanceTests/TypeTests/CosmosMongoDataTypeTests.cs @@ -22,6 +22,14 @@ public override Task DateTime() new DateTime(2021, 2, 3, 13, 40, 55, DateTimeKind.Utc), instantiationExpression: () => new DateTime(2020, 1, 1, 12, 30, 45, DateTimeKind.Utc)); + // MongoDB stores DateTimeOffset as UTC BsonDateTime; the offset is lost on round-trip. + public override Task DateTimeOffset() + => this.Test( + "DateTimeOffset", + new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero), + new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.Zero), + instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero)); + public new class Fixture : DataTypeTests.DefaultRecord>.Fixture { public override TestStore TestStore => CosmosMongoTestStore.Instance; @@ -34,9 +42,7 @@ public override Task DateTime() typeof(byte), typeof(short), typeof(Guid), - typeof(DateTimeOffset), #if NET - typeof(DateOnly), typeof(TimeOnly) #endif ]; diff --git a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs index fa5f524cc452..cfd228cdac17 100644 --- a/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs +++ b/dotnet/test/VectorData/CosmosNoSql.ConformanceTests/TypeTests/CosmosNoSqlDataTypeTests.cs @@ -31,9 +31,7 @@ public override Task DateTimeOffset() typeof(short), typeof(decimal), typeof(Guid), - typeof(DateTime), #if NET - typeof(DateOnly), typeof(TimeOnly) #endif ]; diff --git a/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoDataTypeTests.cs b/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoDataTypeTests.cs index 8454e97b2747..e19c913b39c2 100644 --- a/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoDataTypeTests.cs +++ b/dotnet/test/VectorData/MongoDB.ConformanceTests/TypeTests/MongoDataTypeTests.cs @@ -23,6 +23,24 @@ public override Task DateTime() instantiationExpression: () => new DateTime(2020, 1, 1, 12, 30, 45), isFilterable: false); // Operand type is not supported for $vectorSearch: date + // MongoDB stores DateTimeOffset as UTC BsonDateTime; the offset is lost on round-trip. + public override Task DateTimeOffset() + => this.Test( + "DateTimeOffset", + new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero), + new DateTimeOffset(2021, 2, 3, 13, 40, 55, TimeSpan.Zero), + instantiationExpression: () => new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.Zero), + isFilterable: false); // Operand type is not supported for $vectorSearch: date + +#if NET + public override Task DateOnly() + => this.Test( + "DateOnly", + new DateOnly(2020, 1, 1), + new DateOnly(2021, 2, 3), + isFilterable: false); // Operand type is not supported for $vectorSearch: date +#endif + public override Task String_array() => this.Test( "StringArray", @@ -42,9 +60,7 @@ public override Task String_array() typeof(byte), typeof(short), typeof(Guid), - typeof(DateTimeOffset), #if NET - typeof(DateOnly), typeof(TimeOnly) #endif ]; diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs index 336104663fe3..43dfc06f0c61 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs @@ -40,10 +40,6 @@ public virtual Task DateTimeOffset_with_offset_zero() public override Type[] UnsupportedDefaultTypes { get; } = [ typeof(byte), -#if NET - typeof(DateOnly), - typeof(TimeOnly), -#endif ]; } } diff --git a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs index 5a3c85e296ca..fcd8f55566ef 100644 --- a/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs +++ b/dotnet/test/VectorData/Qdrant.ConformanceTests/TypeTests/QdrantDataTypeTests.cs @@ -39,9 +39,7 @@ public override Task String_array() typeof(short), typeof(decimal), typeof(Guid), - typeof(DateTime), #if NET - typeof(DateOnly), typeof(TimeOnly) #endif ]; diff --git a/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs b/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs index a33295b7ec15..b98b8616ad24 100644 --- a/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs +++ b/dotnet/test/VectorData/SqliteVec.ConformanceTests/TypeTests/SqliteDataTypeTests.cs @@ -18,13 +18,7 @@ public class SqliteDataTypeTests(SqliteDataTypeTests.Fixture fixture) [ typeof(byte), typeof(decimal), - typeof(DateTime), - typeof(DateTimeOffset), typeof(string[]), -#if NET - typeof(DateOnly), - typeof(TimeOnly) -#endif ]; } } diff --git a/dotnet/test/VectorData/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs b/dotnet/test/VectorData/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs index 094ac385e811..dc422713af4b 100644 --- a/dotnet/test/VectorData/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs +++ b/dotnet/test/VectorData/Weaviate.ConformanceTests/TypeTests/WeaviateDataTypeTests.cs @@ -26,9 +26,7 @@ public override Task String_array() public override Type[] UnsupportedDefaultTypes { get; } = [ - typeof(DateTime), #if NET - typeof(DateOnly), typeof(TimeOnly) #endif ]; From 3b3cbbf673411aafdcf8dc5e8c710aa43bf967cb Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Thu, 19 Feb 2026 17:29:54 +0100 Subject: [PATCH 2/2] Fix --- dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs b/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs index 16150765e4a4..68392b0a986f 100644 --- a/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs +++ b/dotnet/src/VectorData/Qdrant/QdrantFieldMapping.cs @@ -151,9 +151,11 @@ public static Value ConvertToGrpcFieldValue(object? sourceValue) case DateTime dateTimeValue: value.StringValue = dateTimeValue.ToString("O"); break; +#if NET case DateOnly dateOnlyValue: value.StringValue = dateOnlyValue.ToString("O"); break; +#endif case IEnumerable or IEnumerable or IEnumerable or