From 9d4009435eb77053ced4f229ffdc60378ef78f06 Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Fri, 23 Jan 2026 16:44:38 +0100 Subject: [PATCH] [MEVD] Map DateTime to timestamptz on PostgreSQL Closes #10641 --- .../VectorData/PgVector/PostgresCollection.cs | 2 +- .../PgVector/PostgresFilterTranslator.cs | 24 +++- .../PgVector/PostgresModelBuilder.cs | 17 +++ .../PgVector/PostgresPropertyExtensions.cs | 69 ++++++++++ .../PgVector/PostgresPropertyMapping.cs | 52 +++++--- .../VectorData/PgVector/PostgresSqlBuilder.cs | 29 +++- .../TypeTests/PostgresDataTypeTests.cs | 15 +++ .../PostgresFilterTranslatorTests.cs | 100 ++++++++++++++ .../PostgresPropertyExtensionsTests.cs | 125 ++++++++++++++++++ .../PostgresPropertyMappingTests.cs | 58 +++++++- .../PostgresSqlBuilderTests.cs | 33 ++++- 11 files changed, 489 insertions(+), 35 deletions(-) create mode 100644 dotnet/test/VectorData/PgVector.UnitTests/PostgresFilterTranslatorTests.cs create mode 100644 dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs diff --git a/dotnet/src/VectorData/PgVector/PostgresCollection.cs b/dotnet/src/VectorData/PgVector/PostgresCollection.cs index f9061387378b..14d9cef81f69 100644 --- a/dotnet/src/VectorData/PgVector/PostgresCollection.cs +++ b/dotnet/src/VectorData/PgVector/PostgresCollection.cs @@ -375,7 +375,7 @@ public override async Task DeleteAsync(IEnumerable keys, CancellationToken using var connection = await this._dataSource.OpenConnectionAsync(cancellationToken).ConfigureAwait(false); using var command = connection.CreateCommand(); - PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty.StorageName, listOfKeys); + PostgresSqlBuilder.BuildDeleteBatchCommand(command, this._schema, this.Name, this._model.KeyProperty, listOfKeys); await this.RunOperationAsync("DeleteBatch", () => command.ExecuteNonQueryAsync(cancellationToken)).ConfigureAwait(false); } diff --git a/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs b/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs index fedd12a80906..198cde9e97a5 100644 --- a/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs +++ b/dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs @@ -3,6 +3,7 @@ using System; using System.Collections; using System.Collections.Generic; +using System.Diagnostics; using System.Globalization; using System.Linq.Expressions; using System.Text; @@ -29,18 +30,29 @@ protected override void TranslateConstant(object? value, bool isSearchCondition) { switch (value) { - // TODO: This aligns with our mapping of DateTime to PG's timestamp (as opposed to timestamptz) - we probably want to - // change that to timestamptz (aligning with Npgsql and EF). See #10641. case DateTime dateTime: - this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\''); - return; + switch (dateTime.Kind) + { + case DateTimeKind.Utc: + this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\''); + return; + + case DateTimeKind.Unspecified: + case DateTimeKind.Local: + this._sql.Append('\'').Append(dateTime.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append('\''); + return; + + default: + throw new UnreachableException(); + } + case DateTimeOffset dateTimeOffset: if (dateTimeOffset.Offset != TimeSpan.Zero) { - throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL"); + throw new NotSupportedException("DateTimeOffset with non-zero offset is not supported with PostgreSQL. Use DateTimeOffset.UtcNow or convert to UTC."); } - this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture)).Append("Z'"); + this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\''); return; // Array constants (ARRAY[1, 2, 3]) diff --git a/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs b/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs index e6e54fdac8fa..5b87dbe02d04 100644 --- a/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs +++ b/dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs @@ -5,6 +5,7 @@ using System.Collections.Generic; using System.Diagnostics.CodeAnalysis; using Microsoft.Extensions.AI; +using Microsoft.Extensions.VectorData; using Microsoft.Extensions.VectorData.ProviderServices; using Pgvector; @@ -92,6 +93,22 @@ internal static bool IsVectorPropertyTypeValidCore(Type type, [NotNullWhen(false type == typeof(SparseVector); } + /// + protected override void ValidateProperty(PropertyModel propertyModel, VectorStoreCollectionDefinition? definition) + { + base.ValidateProperty(propertyModel, definition); + + if (propertyModel.IsTimestampWithoutTimezone()) + { + var type = Nullable.GetUnderlyingType(propertyModel.Type) ?? propertyModel.Type; + if (type != typeof(DateTime)) + { + throw new NotSupportedException( + $"Property '{propertyModel.ModelName}' has store type 'timestamp' configured, but this is only supported for DateTime properties. The property type is '{propertyModel.Type.Name}'."); + } + } + } + /// protected override Type? ResolveEmbeddingType( VectorPropertyModel vectorProperty, diff --git a/dotnet/src/VectorData/PgVector/PostgresPropertyExtensions.cs b/dotnet/src/VectorData/PgVector/PostgresPropertyExtensions.cs index e259aa181fa3..1ef1ef9071d7 100644 --- a/dotnet/src/VectorData/PgVector/PostgresPropertyExtensions.cs +++ b/dotnet/src/VectorData/PgVector/PostgresPropertyExtensions.cs @@ -1,5 +1,6 @@ // Copyright (c) Microsoft. All rights reserved. +using System; using Microsoft.Extensions.VectorData; using Microsoft.Extensions.VectorData.ProviderServices; @@ -11,6 +12,9 @@ namespace Microsoft.SemanticKernel.Connectors.PgVector; public static class PostgresPropertyExtensions { private const string FullTextSearchLanguageKey = "Postgres:FullTextSearchLanguage"; + private const string StoreTypeKey = "Postgres:StoreType"; + + #region Full-text search language /// /// Sets the PostgreSQL full-text search language for a data property. @@ -50,4 +54,69 @@ internal static string GetFullTextSearchLanguageOrDefault(this DataPropertyModel => property.ProviderAnnotations?.TryGetValue(FullTextSearchLanguageKey, out var value) == true && value is string language ? language : PostgresConstants.DefaultFullTextSearchLanguage; + + #endregion Full-text search language + + #region Store type + + /// + /// Sets the PostgreSQL store type for a property, overriding the default type mapping. + /// + /// The property to configure. + /// + /// The PostgreSQL type name. Currently, only "timestamp" and "timestamp without time zone" + /// are supported, and only on properties. This causes the property to be stored as + /// PostgreSQL timestamp (without time zone) instead of the default timestamptz. + /// + /// The same property instance for method chaining. + /// + /// + /// By default, .NET properties are mapped to PostgreSQL timestamptz (timestamp with time zone), + /// which requires UTC values. Use this method to map to timestamp (without time zone) instead, which stores + /// local/unspecified date-time values without time zone information. + /// + /// + /// When using timestamp, values with or + /// kind should be used. Values read back from the database will have + /// . + /// + /// + /// Thrown if the is not a supported value. + public static TProperty WithStoreType(this TProperty property, string storeType) + where TProperty : VectorStoreProperty + { + if (!IsTimestampStoreType(storeType)) + { + throw new NotSupportedException( + $"Store type '{storeType}' is not supported. Only 'timestamp' and 'timestamp without time zone' are supported."); + } + + property.ProviderAnnotations ??= []; + property.ProviderAnnotations[StoreTypeKey] = storeType; + return property; + } + + /// + /// Gets the PostgreSQL store type configured for a property. + /// + /// The property to read from. + /// The configured store type, or if not set. + public static string? GetStoreType(this VectorStoreProperty property) + => property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true + ? value as string + : null; + + /// + /// Gets whether the property model has been configured with a timestamp (without time zone) store type. + /// + internal static bool IsTimestampWithoutTimezone(this PropertyModel property) + => property.ProviderAnnotations?.TryGetValue(StoreTypeKey, out var value) == true + && value is string storeType + && IsTimestampStoreType(storeType); + + private static bool IsTimestampStoreType(string storeType) + => string.Equals(storeType, "timestamp", StringComparison.OrdinalIgnoreCase) + || string.Equals(storeType, "timestamp without time zone", StringComparison.OrdinalIgnoreCase); + + #endregion Store type } diff --git a/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs b/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs index 1515f332dd68..b1111b1c0282 100644 --- a/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs +++ b/dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs @@ -38,8 +38,11 @@ internal static class PostgresPropertyMapping var value => throw new NotSupportedException($"Mapping for type '{value.GetType().Name}' to a vector is not supported.") }; - public static NpgsqlDbType? GetNpgsqlDbType(Type propertyType) => - (Nullable.GetUnderlyingType(propertyType) ?? propertyType) switch + /// + /// Gets the NpgsqlDbType for a property, taking into account any store type annotation. + /// + internal static NpgsqlDbType? GetNpgsqlDbType(PropertyModel property) + => (Nullable.GetUnderlyingType(property.Type) ?? property.Type) switch { Type t when t == typeof(bool) => NpgsqlDbType.Boolean, Type t when t == typeof(short) => NpgsqlDbType.Smallint, @@ -50,19 +53,21 @@ internal static class PostgresPropertyMapping Type t when t == typeof(decimal) => NpgsqlDbType.Numeric, Type t when t == typeof(string) => NpgsqlDbType.Text, Type t when t == typeof(byte[]) => NpgsqlDbType.Bytea, - Type t when t == typeof(DateTime) => NpgsqlDbType.Timestamp, - Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz, Type t when t == typeof(Guid) => NpgsqlDbType.Uuid, + Type t when t == typeof(DateTimeOffset) => NpgsqlDbType.TimestampTz, + + // DateTime properties map to PG's "timestamp with time zone" (UTC timestamps) by default, aligning with Npgsql/EF/etc. + // Users can explicitly opt into "timestamp without time zone". + Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp, + Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz, _ => null }; /// - /// Maps a .NET type to a PostgreSQL type name. + /// Maps a .NET type to a PostgreSQL type name, taking into account any store type annotation on the property. /// - /// The .NET type. - /// Tuple of the the PostgreSQL type name and whether it can be NULL - public static (string PgType, bool IsNullable) GetPostgresTypeName(Type propertyType) + internal static (string PgType, bool IsNullable) GetPostgresTypeName(PropertyModel property) { static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName) { @@ -77,7 +82,7 @@ static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName) Type t when t == typeof(decimal) => "NUMERIC", Type t when t == typeof(string) => "TEXT", Type t when t == typeof(byte[]) => "BYTEA", - Type t when t == typeof(DateTime) => "TIMESTAMP", + Type t when t == typeof(DateTime) => "TIMESTAMPTZ", Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ", Type t when t == typeof(Guid) => "UUID", _ => null @@ -86,30 +91,43 @@ static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName) return typeName is not null; } + var propertyType = property.Type; + // TODO: Handle NRTs properly via NullabilityInfoContext + (string PgType, bool IsNullable) result; + if (TryGetBaseType(propertyType, out string? pgType)) { - return (pgType, !propertyType.IsValueType); + result = (pgType, !propertyType.IsValueType); } - // Handle nullable types (e.g. Nullable) - if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType + else if (Nullable.GetUnderlyingType(propertyType) is Type unwrappedType && TryGetBaseType(unwrappedType, out string? underlyingPgType)) { - return (underlyingPgType, true); + result = (underlyingPgType, true); } - // Handle collections - if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType)) + else if ((propertyType.IsArray && TryGetBaseType(propertyType.GetElementType()!, out string? elementPgType)) || (propertyType.IsGenericType && propertyType.GetGenericTypeDefinition() == typeof(List<>) && TryGetBaseType(propertyType.GetGenericArguments()[0], out elementPgType))) { - return (elementPgType + "[]", true); + result = (elementPgType + "[]", true); + } + else + { + throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store."); + } + + if (property.IsTimestampWithoutTimezone()) + { + // Replace TIMESTAMPTZ with TIMESTAMP in the PG type name. + // This handles both "TIMESTAMPTZ" and "TIMESTAMPTZ[]" cases. + result = ("TIMESTAMP", result.IsNullable); } - throw new NotSupportedException($"Type {propertyType.Name} is not supported by this store."); + return result; } /// diff --git a/dotnet/src/VectorData/PgVector/PostgresSqlBuilder.cs b/dotnet/src/VectorData/PgVector/PostgresSqlBuilder.cs index 2997a305d733..6ac787f26ad3 100644 --- a/dotnet/src/VectorData/PgVector/PostgresSqlBuilder.cs +++ b/dotnet/src/VectorData/PgVector/PostgresSqlBuilder.cs @@ -64,7 +64,7 @@ internal static string BuildCreateTableSql(string schema, string tableName, Coll createTableCommand.AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine(" ("); // Add the key column - var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty.Type).PgType; + var keyStoreType = PostgresPropertyMapping.GetPostgresTypeName(model.KeyProperty).PgType; createTableCommand.Append(" ").AppendIdentifier(keyName).Append(' ').Append(keyStoreType); if (model.KeyProperty.IsAutoGenerated) { @@ -90,7 +90,7 @@ internal static string BuildCreateTableSql(string schema, string tableName, Coll // Add the data columns foreach (var dataProperty in model.DataProperties) { - var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty.Type); + var dataPgTypeInfo = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); createTableCommand.Append(" ").AppendIdentifier(dataProperty.StorageName).Append(' ').Append(dataPgTypeInfo.PgType); if (!dataPgTypeInfo.IsNullable) { @@ -224,7 +224,22 @@ internal static bool BuildUpsertCommand( value = PostgresPropertyMapping.MapVectorForStorageModel(value); } - batchCommand.Parameters.Add(new() { Value = value ?? DBNull.Value }); + NpgsqlDbType? npgsqlDbType = null; + + if (property.Type == typeof(DateTime)) + { + npgsqlDbType = property.IsTimestampWithoutTimezone() + ? NpgsqlDbType.Timestamp + // DateTime properties map to PG's "timestamp with time zone" (timestamptz) by default. + // Npgsql would silently send non-UTC DateTimes as 'timestamp' (without timezone), and PG would + // implicitly cast to timestamptz using the session timezone, producing incorrect results. + // Explicitly set NpgsqlDbType.TimestampTz to ensure Npgsql rejects non-UTC DateTimes. + : NpgsqlDbType.TimestampTz; + } + + batchCommand.Parameters.Add(npgsqlDbType.HasValue + ? new() { Value = value ?? DBNull.Value, NpgsqlDbType = npgsqlDbType.Value } + : new() { Value = value ?? DBNull.Value }); } batch.BatchCommands.Add(batchCommand); @@ -357,7 +372,7 @@ internal static void BuildGetCommand(NpgsqlCommand command, string schema, internal static void BuildGetBatchCommand(NpgsqlCommand command, string schema, string tableName, CollectionModel model, List keys, bool includeVectors = false) where TKey : notnull { - NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty.Type) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}"); + NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(model.KeyProperty) ?? throw new UnreachableException($"Unsupported key type {model.KeyProperty.Type.Name}"); StringBuilder sql = new(); sql.Append("SELECT "); @@ -403,9 +418,9 @@ internal static void BuildDeleteCommand(NpgsqlCommand command, string sche } /// - internal static void BuildDeleteBatchCommand(NpgsqlCommand command, string schema, string tableName, string keyColumn, List keys) + internal static void BuildDeleteBatchCommand(NpgsqlCommand command, string schema, string tableName, KeyPropertyModel keyProperty, List keys) { - NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(typeof(TKey)) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}"); + NpgsqlDbType? keyType = PostgresPropertyMapping.GetNpgsqlDbType(keyProperty) ?? throw new ArgumentException($"Unsupported key type {typeof(TKey).Name}"); for (int i = 0; i < keys.Count; i++) { @@ -417,7 +432,7 @@ internal static void BuildDeleteBatchCommand(NpgsqlCommand command, string StringBuilder sql = new(); sql.Append("DELETE FROM ").AppendIdentifier(schema).Append('.').AppendIdentifier(tableName).AppendLine() - .Append("WHERE ").AppendIdentifier(keyColumn).Append(" = ANY($1);"); + .Append("WHERE ").AppendIdentifier(keyProperty.StorageName).Append(" = ANY($1);"); command.CommandText = sql.ToString(); Debug.Assert(command.Parameters.Count == 0); diff --git a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs index 8cb89a78fefe..336104663fe3 100644 --- a/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs +++ b/dotnet/test/VectorData/PgVector.ConformanceTests/TypeTests/PostgresDataTypeTests.cs @@ -10,8 +10,23 @@ namespace PgVector.ConformanceTests.TypeTests; public class PostgresDataTypeTests(PostgresDataTypeTests.Fixture fixture) : DataTypeTests.DefaultRecord>(fixture), IClassFixture { + // Npgsql maps DateTime to timestamptz by default, and so requires UTC DateTimes (the base test uses Unspecified). + public override Task DateTime() + => Assert.ThrowsAsync(base.DateTime); + + public virtual Task DateTime_utc() + => this.Test( + "DateTime", + System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc), + System.DateTime.SpecifyKind(new DateTime(2021, 2, 3, 13, 40, 55), DateTimeKind.Utc), + instantiationExpression: () => System.DateTime.SpecifyKind(new DateTime(2020, 1, 1, 12, 30, 45), DateTimeKind.Utc)); + // PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported. public override Task DateTimeOffset() + => Assert.ThrowsAsync(base.DateTimeOffset); + + // PostgreSQL does not support representing an offset, so only DateTimeOffsets with offset=0 are supported. + public virtual Task DateTimeOffset_with_offset_zero() => this.Test( "DateTimeOffset", new DateTimeOffset(2020, 1, 1, 12, 30, 45, TimeSpan.FromHours(0)), diff --git a/dotnet/test/VectorData/PgVector.UnitTests/PostgresFilterTranslatorTests.cs b/dotnet/test/VectorData/PgVector.UnitTests/PostgresFilterTranslatorTests.cs new file mode 100644 index 000000000000..7c1b4ff67570 --- /dev/null +++ b/dotnet/test/VectorData/PgVector.UnitTests/PostgresFilterTranslatorTests.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using System.Linq.Expressions; +using System.Text; +using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; +using Microsoft.SemanticKernel.Connectors.PgVector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public sealed class PostgresFilterTranslatorTests +{ + [Fact] + public void DateTime_Utc_FormattedWithZ() + { + // Inline new DateTime(..., DateTimeKind.Utc) in expression tree to exercise TranslateConstant + var sql = TranslateFilter(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45, DateTimeKind.Utc)); + Assert.Contains("'2024-06-15T10:30:45Z'", sql); + } + + [Fact] + public void DateTime_Unspecified_FormattedWithoutZ() + { + // Inline new DateTime(...) has Kind=Unspecified + var sql = TranslateFilter(r => r.Created == new DateTime(2024, 6, 15, 10, 30, 45)); + Assert.Contains("'2024-06-15T10:30:45'", sql); + Assert.DoesNotContain("Z'", sql); + } + + [Fact] + public void DateTimeOffset_Utc_FormattedWithZ() + { + // Use a ConstantExpression directly to bypass VisitNew issues with TimeSpan.Zero + var sql = TranslateFilterWithConstant( + nameof(TestRecord.CreatedOffset), + new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.Zero)); + Assert.Contains("'2024-06-15T10:30:45Z'", sql); + } + + [Fact] + public void DateTimeOffset_NonZeroOffset_Throws() + { + Assert.ThrowsAny(() => + TranslateFilterWithConstant( + nameof(TestRecord.CreatedOffset), + new DateTimeOffset(2024, 6, 15, 10, 30, 45, TimeSpan.FromHours(2)))); + } + + private static string TranslateFilter(Expression> filter) + { + var model = BuildModel(); + var sb = new StringBuilder(); + var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb); + translator.Translate(appendWhere: false); + return sb.ToString(); + } + + private static string TranslateFilterWithConstant(string propertyName, TValue value) + { + var model = BuildModel(); + var param = Expression.Parameter(typeof(TRecord), "r"); + var prop = Expression.Property(param, propertyName); + var constant = Expression.Constant(value, typeof(TValue)); + var body = Expression.Equal(prop, constant); + var filter = Expression.Lambda>(body, param); + + var sb = new StringBuilder(); + var translator = new PostgresFilterTranslator(model, filter, startParamIndex: 1, sb); + translator.Translate(appendWhere: false); + return sb.ToString(); + } + + private static CollectionModel BuildModel() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("Id", typeof(Guid)), + new VectorStoreDataProperty("Created", typeof(DateTime)), + new VectorStoreDataProperty("CreatedOffset", typeof(DateTimeOffset)), + new VectorStoreVectorProperty("Embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + return new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + } + +#pragma warning disable CA1812 // internal class that is apparently never instantiated. + private sealed record TestRecord + { + public Guid Id { get; set; } + public DateTime Created { get; set; } + public DateTimeOffset CreatedOffset { get; set; } + public ReadOnlyMemory Embedding { get; set; } + } +#pragma warning restore CA1812 +} diff --git a/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs b/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs new file mode 100644 index 000000000000..c56f7c9c8501 --- /dev/null +++ b/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyExtensionsTests.cs @@ -0,0 +1,125 @@ +// Copyright (c) Microsoft. All rights reserved. + +using System; +using Microsoft.Extensions.VectorData; +using Microsoft.SemanticKernel.Connectors.PgVector; +using Xunit; + +namespace SemanticKernel.Connectors.PgVector.UnitTests; + +public sealed class PostgresPropertyExtensionsTests +{ + [Theory] + [InlineData("timestamp")] + [InlineData("TIMESTAMP")] + [InlineData("timestamp without time zone")] + [InlineData("TIMESTAMP WITHOUT TIME ZONE")] + public void WithStoreType_Timestamp_SetsAnnotation(string storeType) + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + var result = property.WithStoreType(storeType); + + Assert.Same(property, result); + Assert.Equal(storeType, property.GetStoreType()); + } + + [Theory] + [InlineData("timestamptz")] + [InlineData("timestamp with time zone")] + [InlineData("integer")] + [InlineData("text")] + [InlineData("")] + public void WithStoreType_UnsupportedType_Throws(string storeType) + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + Assert.Throws(() => property.WithStoreType(storeType)); + } + + [Fact] + public void GetStoreType_NotSet_ReturnsNull() + { + var property = new VectorStoreDataProperty("test", typeof(DateTime)); + + Assert.Null(property.GetStoreType()); + } + + [Fact] + public void WithStoreType_OnlyAllowedOnDateTimeProperties() + { + // Setting the annotation on a non-DateTime property should succeed at the property level + // (validation happens in the model builder), but building the model should throw. + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("name", typeof(string)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + Assert.Throws(() => + new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null)); + } + + [Fact] + public void WithStoreType_AllowedOnDateTimeProperty() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("created", typeof(DateTime)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + // Should not throw + var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + Assert.NotNull(model); + } + + [Fact] + public void WithStoreType_AllowedOnKeyProperty() + { + var property = new VectorStoreKeyProperty("id", typeof(DateTime)).WithStoreType("timestamp"); + Assert.Equal("timestamp", property.GetStoreType()); + } + + [Fact] + public void WithStoreType_OnKeyProperty_NonDateTime_Throws() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + Assert.Throws(() => + new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null)); + } + + [Fact] + public void WithStoreType_AllowedOnNullableDateTimeProperty() + { + var definition = new VectorStoreCollectionDefinition + { + Properties = + [ + new VectorStoreKeyProperty("id", typeof(Guid)), + new VectorStoreDataProperty("created", typeof(DateTime?)).WithStoreType("timestamp"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + // Should not throw + var model = new PostgresModelBuilder().BuildDynamic(definition, defaultEmbeddingGenerator: null); + Assert.NotNull(model); + } +} diff --git a/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyMappingTests.cs b/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyMappingTests.cs index 61547fbb4e7d..64dafc41de81 100644 --- a/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyMappingTests.cs +++ b/dotnet/test/VectorData/PgVector.UnitTests/PostgresPropertyMappingTests.cs @@ -5,6 +5,7 @@ using Microsoft.Extensions.VectorData; using Microsoft.Extensions.VectorData.ProviderServices; using Microsoft.SemanticKernel.Connectors.PgVector; +using NpgsqlTypes; using Pgvector; using Xunit; @@ -50,14 +51,15 @@ public void GetPropertyValueReturnsCorrectValuesForLists() (typeof(List), "DOUBLE PRECISION[]"), (typeof(List), "TEXT[]"), (typeof(List), "BOOLEAN[]"), - (typeof(List), "TIMESTAMP[]"), + (typeof(List), "TIMESTAMPTZ[]"), (typeof(List), "UUID[]"), }; // Act & Assert foreach (var (type, expectedValue) in typesAndExpectedValues) { - var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(type); + var property = new DataPropertyModel("test", type); + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(property); Assert.Equal(expectedValue, pgType); } } @@ -81,7 +83,8 @@ public void GetPropertyValueReturnsCorrectNullableValue() // Act & Assert foreach (var (type, expectedValue) in typesAndExpectedValues) { - var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(type); + var property = new DataPropertyModel("test", type); + var (_, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(property); Assert.Equal(expectedValue, isNullable); } } @@ -142,4 +145,53 @@ public void GetVectorIndexInfoReturnsThrowsForInvalidDimensions(string indexKind // Act & Assert Assert.Throws(() => PostgresPropertyMapping.GetIndexInfo([vectorProperty])); } + + [Fact] + public void GetPostgresTypeName_DateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMP", pgType); + } + + [Fact] + public void GetPostgresTypeName_DateTime_WithoutAnnotation_ReturnsTimestampTz() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + + var (pgType, _) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMPTZ", pgType); + } + + [Fact] + public void GetPostgresTypeName_NullableDateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime?)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var (pgType, isNullable) = PostgresPropertyMapping.GetPostgresTypeName(dataProperty); + Assert.Equal("TIMESTAMP", pgType); + Assert.True(isNullable); + } + + [Fact] + public void GetNpgsqlDbType_DateTime_WithTimestampAnnotation_ReturnsTimestamp() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + dataProperty.ProviderAnnotations = new() { ["Postgres:StoreType"] = "timestamp" }; + + var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty); + Assert.Equal(NpgsqlDbType.Timestamp, dbType); + } + + [Fact] + public void GetNpgsqlDbType_DateTime_WithoutAnnotation_ReturnsTimestampTz() + { + var dataProperty = new DataPropertyModel("created", typeof(DateTime)); + + var dbType = PostgresPropertyMapping.GetNpgsqlDbType(dataProperty); + Assert.Equal(NpgsqlDbType.TimestampTz, dbType); + } } diff --git a/dotnet/test/VectorData/PgVector.UnitTests/PostgresSqlBuilderTests.cs b/dotnet/test/VectorData/PgVector.UnitTests/PostgresSqlBuilderTests.cs index 95d401b3c360..c5667361f928 100644 --- a/dotnet/test/VectorData/PgVector.UnitTests/PostgresSqlBuilderTests.cs +++ b/dotnet/test/VectorData/PgVector.UnitTests/PostgresSqlBuilderTests.cs @@ -4,6 +4,7 @@ using System.Collections.Generic; using System.Linq; using Microsoft.Extensions.VectorData; +using Microsoft.Extensions.VectorData.ProviderServices; using Microsoft.SemanticKernel.Connectors.PgVector; using Npgsql; using Xunit; @@ -75,6 +76,35 @@ public void TestBuildCreateTableCommand(bool ifNotExists) this._output.WriteLine(sql); } + [Fact] + public void TestBuildCreateTableCommand_WithTimestampStoreType() + { + var recordDefinition = new VectorStoreCollectionDefinition() + { + Properties = [ + new VectorStoreKeyProperty("id", typeof(long)), + new VectorStoreDataProperty("created_utc", typeof(DateTime)), + new VectorStoreDataProperty("created_local", typeof(DateTime)).WithStoreType("timestamp"), + new VectorStoreDataProperty("created_nullable", typeof(DateTime?)).WithStoreType("timestamp without time zone"), + new VectorStoreVectorProperty("embedding", typeof(ReadOnlyMemory), 10) + ] + }; + + var model = new PostgresModelBuilder().BuildDynamic(recordDefinition, defaultEmbeddingGenerator: null); + + var sql = PostgresSqlBuilder.BuildCreateTableSql("public", "testcollection", model, pgVersion: new Version(18, 0)); + + Assert.Contains("\"created_utc\" TIMESTAMPTZ NOT NULL", sql); + Assert.Contains("\"created_local\" TIMESTAMP NOT NULL", sql); + Assert.Contains("\"created_nullable\" TIMESTAMP", sql); + // Make sure it's TIMESTAMP (not TIMESTAMPTZ) for the nullable one + var idx = sql.IndexOf("\"created_nullable\"", StringComparison.Ordinal); + var fragment = sql.Substring(idx, sql.IndexOf('\n', idx) - idx); + Assert.DoesNotContain("TIMESTAMPTZ", fragment); + + this._output.WriteLine(sql); + } + [Theory] [InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, true)] [InlineData(IndexKind.Hnsw, DistanceFunction.EuclideanDistance, false)] @@ -373,7 +403,8 @@ public void TestBuildDeleteBatchCommand() // Act using var command = new NpgsqlCommand(); - PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", "id", keys); + var keyProperty = new KeyPropertyModel("id", typeof(long)); + PostgresSqlBuilder.BuildDeleteBatchCommand(command, "public", "testcollection", keyProperty, keys); // Assert Assert.Contains("DELETE", command.CommandText);