Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,15 @@ public static BsonValue Create(object? value)
=> value switch
{
null => BsonNull.Value,
Guid guid => new BsonBinaryData(guid, GuidRepresentation.Standard),
object[] array => new BsonArray(Array.ConvertAll(array, Create)),
Array array => new BsonArray(array),
IEnumerable<object> enumerable => new BsonArray(enumerable.Select(Create)),
Guid v => new BsonBinaryData(v, GuidRepresentation.Standard),
DateTimeOffset v => new BsonDateTime(v.UtcDateTime),
#if NET
DateOnly v => new BsonDateTime(v.ToDateTime(TimeOnly.MinValue, DateTimeKind.Utc)),
#endif
object[] v => new BsonArray(Array.ConvertAll(v, Create)),
Array v => new BsonArray(v),
IEnumerable<object> v => new BsonArray(v.Select(Create)),

_ => BsonValue.Create(value)
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -160,14 +160,24 @@ Embedding<float> e
Type t when t == typeof(int?) => value.AsNullableInt32,
Type t when t == typeof(long) => value.AsInt64,
Type t when t == typeof(long?) => value.AsNullableInt64,
Type t when t == typeof(float) => ((float)value.AsDouble),
Type t when t == typeof(float) => (float)value.AsDouble,
Type t when t == typeof(float?) => ((float?)value.AsNullableDouble),
Type t when t == typeof(double) => value.AsDouble,
Type t when t == typeof(double?) => value.AsNullableDouble,
Type t when t == typeof(decimal) => value.AsDecimal,
Type t when t == typeof(decimal?) => value.AsNullableDecimal,
Type t when t == typeof(DateTime) => value.ToUniversalTime(),
Type t when t == typeof(DateTime?) => value.ToNullableUniversalTime(),
Type t when t == typeof(DateTimeOffset) => new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero),
Type t when t == typeof(DateTimeOffset?) => value.ToNullableUniversalTime() is DateTime dateTime
? new DateTimeOffset(value.ToUniversalTime(), TimeSpan.Zero)
: null,
#if NET
Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.ToUniversalTime()),
Type t when t == typeof(DateOnly?) => value.ToNullableUniversalTime() is DateTime dateTime
? DateOnly.FromDateTime(dateTime)
: null,
#endif

_ => (object?)null
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)

protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
supportedTypes = "string, int, long, double, float, bool, decimal, DateTime, DateTimeOffset,"
#if NET
+ " DateOnly,"
#endif
+ " or arrays/lists of these types";

if (Nullable.GetUnderlyingType(type) is Type underlyingType)
{
Expand All @@ -77,7 +81,12 @@ static bool IsValid(Type type)
type == typeof(float) ||
type == typeof(double) ||
type == typeof(decimal) ||
type == typeof(DateTime);
type == typeof(DateTime) ||
type == typeof(DateTimeOffset) ||
#if NET
type == typeof(DateOnly) ||
#endif
false;
}

protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -128,7 +128,11 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
// Half is also listed by the SDK, but currently not supported.
Type t when t == typeof(float) => SearchFieldDataType.Double,
Type t when t == typeof(double) => SearchFieldDataType.Double,
Type t when t == typeof(DateTime) => SearchFieldDataType.DateTimeOffset,
Type t when t == typeof(DateTimeOffset) => SearchFieldDataType.DateTimeOffset,
#if NET
Type t when t == typeof(DateOnly) => SearchFieldDataType.DateTimeOffset,
#endif

Type t when t == typeof(string[]) => SearchFieldDataType.Collection(SearchFieldDataType.String),
Type t when t == typeof(List<string>) => SearchFieldDataType.Collection(SearchFieldDataType.String),
Expand All @@ -142,8 +146,14 @@ public static SearchFieldDataType GetSDKFieldDataType(Type propertyType)
Type t when t == typeof(List<float>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
Type t when t == typeof(double[]) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
Type t when t == typeof(List<double>) => SearchFieldDataType.Collection(SearchFieldDataType.Double),
Type t when t == typeof(DateTime[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
Type t when t == typeof(List<DateTime>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
Type t when t == typeof(DateTimeOffset[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
Type t when t == typeof(List<DateTimeOffset>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
#if NET
Type t when t == typeof(DateOnly[]) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
Type t when t == typeof(List<DateOnly>) => SearchFieldDataType.Collection(SearchFieldDataType.DateTimeOffset),
#endif

_ => throw new NotSupportedException($"Data type '{propertyType}' for {nameof(VectorStoreDataProperty)} is not supported by the Azure AI Search VectorStore.")
};
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode
if (dataModel.TryGetValue(dataProperty.ModelName, out var dataValue))
{
jsonObject[dataProperty.StorageName] = dataValue is not null ?
JsonSerializer.SerializeToNode(dataValue, dataProperty.Type, jsonSerializerOptions) :
JsonSerializer.SerializeToNode(ConvertToStorageValue(dataValue), GetStorageType(dataProperty.Type), jsonSerializerOptions) :
null;
}
}
Expand Down Expand Up @@ -183,6 +183,10 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode
Type t when t == typeof(DateTime?) => value.GetValue<DateTime?>(),
Type t when t == typeof(DateTimeOffset) => value.GetValue<DateTimeOffset>(),
Type t when t == typeof(DateTimeOffset?) => value.GetValue<DateTimeOffset?>(),
#if NET
Type t when t == typeof(DateOnly) => DateOnly.FromDateTime(value.GetValue<DateTimeOffset>().DateTime),
Type t when t == typeof(DateOnly?) => (DateOnly?)DateOnly.FromDateTime(value.GetValue<DateTimeOffset>().DateTime),
#endif

_ => (object?)null
};
Expand Down Expand Up @@ -242,4 +246,38 @@ public JsonObject MapFromDataToStorageModel(Dictionary<string, object?> dataMode

throw new UnreachableException($"Unsupported property type '{propertyType.Name}'.");
}

/// <summary>
/// Converts a value to its storage representation for Azure AI Search.
/// Azure AI Search only supports <see cref="DateTimeOffset"/> for date/time types, and always internally
/// converts to UTC for storage.
/// </summary>
/// <summary>
/// Gets the storage type for a given property type. Azure AI Search stores DateTime and DateOnly as DateTimeOffset.
/// </summary>
private static Type GetStorageType(Type propertyType)
{
var underlying = Nullable.GetUnderlyingType(propertyType) ?? propertyType;

if (underlying == typeof(DateTime)
#if NET
|| underlying == typeof(DateOnly)
#endif
)
{
return propertyType == underlying ? typeof(DateTimeOffset) : typeof(DateTimeOffset?);
}

return propertyType;
}

private static object ConvertToStorageValue(object value)
=> value switch
{
DateTime dateTime => new DateTimeOffset(dateTime, TimeSpan.Zero),
#if NET
DateOnly dateOnly => new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero),
#endif
_ => value
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -119,9 +119,17 @@ private void GenerateLiteral(object? value)
this._filter.Append('\'').Append(g.ToString()).Append('\'');
return;

case DateTime d:
this._filter.Append(new DateTimeOffset(d, TimeSpan.Zero).ToString("o"));
return;
case DateTimeOffset d:
this._filter.Append(d.ToString("o"));
return;
#if NET
case DateOnly d:
this._filter.Append(new DateTimeOffset(d.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero).ToString("o"));
return;
#endif

case Array:
throw new NotImplementedException();
Expand Down
44 changes: 44 additions & 0 deletions dotnet/src/VectorData/AzureAISearch/AzureAISearchMapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,29 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,

var jsonObject = JsonSerializer.SerializeToNode(dataModel, jsonSerializerOptions)!.AsObject();

// Azure AI Search only supports Edm.DateTimeOffset for date/time types.
// DateTime properties may be serialized by STJ without timezone info (when Kind=Unspecified),
// which Azure AI Search rejects. Convert them to DateTimeOffset (UTC).
// DateOnly properties are serialized as "yyyy-MM-dd", which also isn't accepted.
// Convert them to full DateTimeOffset representation (midnight UTC).
foreach (var dataProperty in model.DataProperties)
{
var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;

if (propertyType == typeof(DateTime) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
{
var dateTime = dtNode.Deserialize<DateTime>(jsonSerializerOptions);
jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(DateTime.SpecifyKind(dateTime, DateTimeKind.Utc), TimeSpan.Zero));
}
#if NET
else if (propertyType == typeof(DateOnly) && jsonObject.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
{
var dateOnly = dateNode.Deserialize<DateOnly>(jsonSerializerOptions);
jsonObject[dataProperty.StorageName] = JsonValue.Create(new DateTimeOffset(dateOnly.ToDateTime(TimeOnly.MinValue), TimeSpan.Zero));
}
#endif
}

// Go over the vector properties; inject any generated embeddings to overwrite the JSON serialized above.
// Also, for Embedding<T> properties we also need to overwrite with a simple array (since Embedding<T> gets serialized as a complex object).
for (var i = 0; i < model.VectorProperties.Count; i++)
Expand Down Expand Up @@ -64,6 +87,27 @@ public JsonObject MapFromDataToStorageModel(TRecord dataModel, int recordIndex,

public TRecord MapFromStorageToDataModel(JsonObject storageModel, bool includeVectors)
{
// Azure AI Search stores DateTime properties as Edm.DateTimeOffset.
// Convert them back to DateTime so STJ can deserialize them into the correct type.
// DateOnly properties also need to be converted back from DateTimeOffset.
foreach (var dataProperty in model.DataProperties)
{
var propertyType = Nullable.GetUnderlyingType(dataProperty.Type) ?? dataProperty.Type;

if (propertyType == typeof(DateTime) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dtNode) && dtNode is not null)
{
var dateTimeOffset = dtNode.Deserialize<DateTimeOffset>(jsonSerializerOptions);
storageModel[dataProperty.StorageName] = JsonValue.Create(dateTimeOffset.UtcDateTime);
}
#if NET
else if (propertyType == typeof(DateOnly) && storageModel.TryGetPropertyValue(dataProperty.StorageName, out var dateNode) && dateNode is not null)
{
var dateTimeOffset = dateNode.Deserialize<DateTimeOffset>(jsonSerializerOptions);
storageModel[dataProperty.StorageName] = JsonValue.Create(DateOnly.FromDateTime(dateTimeOffset.DateTime));
}
#endif
}

if (includeVectors)
{
foreach (var vectorProperty in model.VectorProperties)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,11 @@ protected override bool IsVectorPropertyTypeValid(Type type, [NotNullWhen(false)

internal static bool IsDataPropertyTypeValidCore(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset,"
#if NET
+ " DateOnly,"
#endif
+ " or arrays/lists of these types";

if (Nullable.GetUnderlyingType(type) is Type underlyingType)
{
Expand All @@ -58,6 +62,10 @@ static bool IsValid(Type type)
type == typeof(double) ||
type == typeof(float) ||
type == typeof(bool) ||
type == typeof(DateTime) ||
#if NET
type == typeof(DateOnly) ||
#endif
type == typeof(DateTimeOffset);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,22 @@ private void TranslateConstant(object? value)
.Append("Z\"");
return;

case DateTime v:
this._sql
.Append('"')
.Append(v.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFF", CultureInfo.InvariantCulture))
.Append('"');
return;

#if NET
case DateOnly v:
this._sql
.Append('"')
.Append(v.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture))
.Append('"');
return;
#endif

case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)):
this._sql.Append('[');

Expand Down
10 changes: 9 additions & 1 deletion dotnet/src/VectorData/CosmosNoSql/CosmosNoSqlModelBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,11 @@ protected override void ValidateKeyProperty(KeyPropertyModel keyProperty)

protected override bool IsDataPropertyTypeValid(Type type, [NotNullWhen(false)] out string? supportedTypes)
{
supportedTypes = "string, int, long, double, float, bool, DateTimeOffset, or arrays/lists of these types";
supportedTypes = "string, int, long, double, float, bool, DateTime, DateTimeOffset,"
#if NET
+ " DateOnly,"
#endif
+ " or arrays/lists of these types";

if (Nullable.GetUnderlyingType(type) is Type underlyingType)
{
Expand All @@ -54,6 +58,10 @@ static bool IsValid(Type type)
type == typeof(long) ||
type == typeof(float) ||
type == typeof(double) ||
type == typeof(DateTime) ||
#if NET
type == typeof(DateOnly) ||
#endif
type == typeof(DateTimeOffset);
}

Expand Down
6 changes: 5 additions & 1 deletion dotnet/src/VectorData/MongoDB/MongoFilterTranslator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,11 @@ private BsonDocument GenerateEqualityComparison(PropertyModel property, object?
throw new NotSupportedException("MongoDB does not support null checks in vector search pre-filters");
}

if (value is DateTime or decimal or IList)
if (value is DateTime or DateTimeOffset or decimal or IList
#if NET
or DateOnly
#endif
)
{
// Operand type is not supported for $vectorSearch: date/decimal
throw new NotSupportedException($"MongoDB does not support type {value.GetType().Name} in vector search pre-filters.");
Expand Down
10 changes: 10 additions & 0 deletions dotnet/src/VectorData/PgVector/PostgresFilterTranslator.cs
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,16 @@ protected override void TranslateConstant(object? value, bool isSearchCondition)
this._sql.Append('\'').Append(dateTimeOffset.ToString("yyyy-MM-ddTHH:mm:ss.FFFFFFZ", CultureInfo.InvariantCulture)).Append('\'');
return;

#if NET
case DateOnly dateOnly:
this._sql.Append('\'').Append(dateOnly.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture)).Append('\'');
return;

case TimeOnly timeOnly:
this._sql.Append('\'').Append(timeOnly.ToString("HH:mm:ss.FFFFFFF", CultureInfo.InvariantCulture)).Append('\'');
return;
#endif

// Array constants (ARRAY[1, 2, 3])
case IEnumerable v when v.GetType() is var type && (type.IsArray || type.IsGenericType && type.GetGenericTypeDefinition() == typeof(List<>)):
this._sql.Append("ARRAY[");
Expand Down
16 changes: 16 additions & 0 deletions dotnet/src/VectorData/PgVector/PostgresMapper.cs
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,14 @@ private static void PopulateProperty(PropertyModel property, NpgsqlDataReader re
case var t when t == typeof(DateTimeOffset):
property.SetValue(record, reader.GetFieldValue<DateTimeOffset>(ordinal));
return;
#if NET
case var t when t == typeof(DateOnly):
property.SetValue(record, reader.GetFieldValue<DateOnly>(ordinal));
return;
case var t when t == typeof(TimeOnly):
property.SetValue(record, reader.GetFieldValue<TimeOnly>(ordinal));
return;
#endif
case var t when t == typeof(byte[]):
property.SetValue(record, reader.GetFieldValue<byte[]>(ordinal));
return;
Expand Down Expand Up @@ -203,6 +211,14 @@ private static void PopulateProperty(PropertyModel property, NpgsqlDataReader re
case Type t when t == typeof(List<DateTimeOffset>):
property.SetValueAsObject(record, reader.GetFieldValue<List<DateTimeOffset>>(ordinal));
return;
#if NET
case Type t when t == typeof(List<DateOnly>):
property.SetValueAsObject(record, reader.GetFieldValue<List<DateOnly>>(ordinal));
return;
case Type t when t == typeof(List<TimeOnly>):
property.SetValueAsObject(record, reader.GetFieldValue<List<TimeOnly>>(ordinal));
return;
#endif
case Type t when t == typeof(List<byte[]>):
property.SetValueAsObject(record, reader.GetFieldValue<List<byte[]>>(ordinal));
return;
Expand Down
4 changes: 4 additions & 0 deletions dotnet/src/VectorData/PgVector/PostgresModelBuilder.cs
Original file line number Diff line number Diff line change
Expand Up @@ -65,6 +65,10 @@ static bool IsValid(Type type)
type == typeof(byte[]) ||
type == typeof(DateTime) ||
type == typeof(DateTimeOffset) ||
#if NET
type == typeof(DateOnly) ||
type == typeof(TimeOnly) ||
#endif
type == typeof(Guid);
}

Expand Down
9 changes: 9 additions & 0 deletions dotnet/src/VectorData/PgVector/PostgresPropertyMapping.cs
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,11 @@ internal static class PostgresPropertyMapping
Type t when t == typeof(DateTime) && property.IsTimestampWithoutTimezone() => NpgsqlDbType.Timestamp,
Type t when t == typeof(DateTime) => NpgsqlDbType.TimestampTz,

#if NET
Type t when t == typeof(DateOnly) => NpgsqlDbType.Date,
Type t when t == typeof(TimeOnly) => NpgsqlDbType.Time,
#endif

_ => null
};

Expand All @@ -84,6 +89,10 @@ static bool TryGetBaseType(Type type, [NotNullWhen(true)] out string? typeName)
Type t when t == typeof(byte[]) => "BYTEA",
Type t when t == typeof(DateTime) => "TIMESTAMPTZ",
Type t when t == typeof(DateTimeOffset) => "TIMESTAMPTZ",
#if NET
Type t when t == typeof(DateOnly) => "DATE",
Type t when t == typeof(TimeOnly) => "TIME",
#endif
Type t when t == typeof(Guid) => "UUID",
_ => null
};
Expand Down
Loading
Loading