From c732bd0ca70edaeccd304cbadcb0248114659865 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Fri, 17 Jan 2025 14:57:27 -0800 Subject: [PATCH 1/3] String and byte[] converters using segmented reads/writes --- .../src/System.Text.Json.csproj | 2 + .../Text/Json/Nodes/JsonValueOfTPrimitive.cs | 2 +- .../System/Text/Json/Reader/Utf8JsonReader.cs | 23 +- .../Json/Serialization/ConverterStrategy.cs | 6 +- .../Object/ObjectDefaultConverter.cs | 2 +- .../Converters/Value/ByteArrayConverter.cs | 178 ++++++++++++++- .../Value/ReadOnlyMemoryByteConverter.cs | 1 + .../Converters/Value/StringConverter.cs | 213 +++++++++++++++++- .../Text/Json/Serialization/JsonConverter.cs | 12 +- .../JsonConverterOfT.ReadCore.cs | 3 +- .../Json/Serialization/JsonConverterOfT.cs | 26 ++- .../JsonHybridResumableConverter.cs | 69 ++++++ .../JsonResumableConverterOfT.cs | 1 + .../Serialization/JsonSerializer.Helpers.cs | 3 + .../Metadata/JsonPropertyInfoOfT.cs | 2 +- .../Serialization/Metadata/JsonTypeInfo.cs | 3 +- .../Serialization/Metadata/StringTypeInfo.cs | 32 +++ .../Serialization/Array.WriteTests.cs | 1 + .../HybridResumableConverterTests.cs | 122 ++++++++++ .../Serialization/Stream.ReadTests.cs | 25 ++ .../System.Text.Json.Tests.csproj | 1 + 21 files changed, 699 insertions(+), 28 deletions(-) create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonHybridResumableConverter.cs create mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs create mode 100644 src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/HybridResumableConverterTests.cs diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index 0acc35d1987792..a5267f99382917 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -136,6 +136,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET + @@ -158,6 +159,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET + diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Nodes/JsonValueOfTPrimitive.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Nodes/JsonValueOfTPrimitive.cs index fce1d5fbf04cf4..4b31a3a2bbcc0b 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Nodes/JsonValueOfTPrimitive.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Nodes/JsonValueOfTPrimitive.cs @@ -18,7 +18,7 @@ internal sealed class JsonValuePrimitive : JsonValue public JsonValuePrimitive(TValue value, JsonConverter converter, JsonNodeOptions? options) : base(value, options) { Debug.Assert(TypeIsSupportedPrimitive, $"The type {typeof(TValue)} is not a supported primitive."); - Debug.Assert(converter is { IsInternalConverter: true, ConverterStrategy: ConverterStrategy.Value }); + Debug.Assert(converter is { IsInternalConverter: true, ConverterStrategy: ConverterStrategy.SimpleValue or ConverterStrategy.SegmentableValue }); _converter = converter; _valueKind = DetermineValueKind(value); diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs index c2f9dbcfa88a16..dd1c23d1b67325 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs @@ -46,6 +46,8 @@ public ref partial struct Utf8JsonReader private SequencePosition _currentPosition; private readonly ReadOnlySequence _sequence; + internal bool _hasPartialStringValue; + private readonly bool IsLastSpan => _isFinalBlock && (!_isMultiSegment || _isLastSegment); internal readonly ReadOnlySequence OriginalSequence => _sequence; @@ -1276,7 +1278,7 @@ private bool ConsumePropertyName() return true; } - private bool ConsumeString() + private bool ConsumeString(int offset = 0) { Debug.Assert(_buffer.Length >= _consumed + 1); Debug.Assert(_buffer[_consumed] == JsonConstants.Quote); @@ -1288,7 +1290,7 @@ private bool ConsumeString() // If the first found byte is a quote, we have reached an end of string, and // can avoid validation. // Otherwise, in the uncommon case, iterate one character at a time and validate. - int idx = localBuffer.IndexOfQuoteOrAnyControlOrBackSlash(); + int idx = localBuffer.Slice(offset).IndexOfQuoteOrAnyControlOrBackSlash() + offset; if (idx >= 0) { @@ -1300,6 +1302,7 @@ private bool ConsumeString() ValueIsEscaped = false; _tokenType = JsonTokenType.String; _consumed += idx + 2; + _hasPartialStringValue = false; return true; } else @@ -1314,10 +1317,19 @@ private bool ConsumeString() _bytePositionInLine += localBuffer.Length + 1; // Account for the start quote ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.EndOfStringNotFound); } + + ValueSpan = localBuffer; + ValueIsEscaped = false; + _hasPartialStringValue = true; return false; } } + internal bool ContinueConsumeString() + { + return ConsumeString(ValueSpan.Length); + } + // Found a backslash or control characters which are considered invalid within a string. // Search through the rest of the string one byte at a time. // https://tools.ietf.org/html/rfc8259#section-7 @@ -1367,7 +1379,8 @@ private bool ConsumeStringAndValidate(ReadOnlySpan data, int idx) else { // We found less than 4 hex digits. Check if there is more data to follow, otherwise throw. - idx = data.Length; + idx += 5; + Debug.Assert(idx > data.Length); break; } @@ -1390,6 +1403,9 @@ private bool ConsumeStringAndValidate(ReadOnlySpan data, int idx) } _lineNumber = prevLineNumber; _bytePositionInLine = prevLineBytePosition; + ValueSpan = idx > data.Length ? data.Slice(0, idx - 6) : data; + ValueIsEscaped = true; + _hasPartialStringValue = true; return false; } @@ -1399,6 +1415,7 @@ private bool ConsumeStringAndValidate(ReadOnlySpan data, int idx) ValueIsEscaped = true; _tokenType = JsonTokenType.String; _consumed += idx + 2; + _hasPartialStringValue = false; return true; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ConverterStrategy.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ConverterStrategy.cs index 67a4694d94edc1..4b0fdced89afae 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ConverterStrategy.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/ConverterStrategy.cs @@ -25,7 +25,11 @@ internal enum ConverterStrategy : byte /// /// Simple values or user-provided custom converters. /// - Value = 0x2, + SimpleValue = 0x2, + /// + /// Values that can participate in resumable serialization, for example by splitting the value and writing each split segment separately. + /// + SegmentableValue = 0x4, /// /// Enumerable collections except dictionaries. /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/ObjectDefaultConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/ObjectDefaultConverter.cs index 0b9bae378cf627..ef074993fda7db 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/ObjectDefaultConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Object/ObjectDefaultConverter.cs @@ -394,7 +394,7 @@ internal sealed override bool OnTryWrite( if (!jsonPropertyInfo.GetMemberAndWriteJson(obj!, ref state, writer)) { - Debug.Assert(jsonPropertyInfo.EffectiveConverter.ConverterStrategy != ConverterStrategy.Value); + Debug.Assert(jsonPropertyInfo.EffectiveConverter.ConverterStrategy != ConverterStrategy.SimpleValue); return false; } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs index 3f0f871c81941c..1a599d032ecdad 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs @@ -1,11 +1,16 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Buffers.Text; +using System.Collections.Generic; +using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text.Json.Schema; namespace System.Text.Json.Serialization.Converters { - internal sealed class ByteArrayConverter : JsonConverter + internal sealed class ByteArrayConverter : JsonHybridResumableConverter { public override byte[]? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { @@ -13,7 +18,6 @@ internal sealed class ByteArrayConverter : JsonConverter { return null; } - return reader.GetBytesFromBase64(); } @@ -29,6 +33,176 @@ public override void Write(Utf8JsonWriter writer, byte[]? value, JsonSerializerO } } + internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, scoped ref ReadStack state, out byte[]? value) + { + if (state.Current.ObjectState < StackFrameObjectState.CreatedObject) + { + // This is the first segment so it can't be the only/last segment since we are on the slow read path. + Debug.Assert(reader._hasPartialStringValue); + + state.Current.ObjectState = StackFrameObjectState.ReadElements; + + ReadSegment(ref reader, ref state); + + value = null; + return false; + } + + Debug.Assert(state.Current.ObjectState == StackFrameObjectState.ReadElements); + + bool consumedEntireString = reader.ContinueConsumeString(); + if (!consumedEntireString && reader.IsFinalBlock) + { + // TODO + throw new Exception(); + } + + ReadSegment(ref reader, ref state); + + if (consumedEntireString) + { + value = GetStringFromChunks(reader, state); + return true; + } + + value = null; + return false; + + static byte[] GetStringFromChunks(Utf8JsonReader reader, ReadStack state) + { + Debug.Assert(reader.TokenType == JsonTokenType.String); + + List>? chunks = (List>?)state.Current.ReturnValue!; + if (chunks == null) + { + // Nothing escaped, so just use the raw value. + return reader.ValueSpan.ToArray(); + } + + int totalSize = 0; + foreach (ArraySegment c in chunks) + { + totalSize += c.Count; + } + + byte[] ret = new byte[totalSize]; + int idx = 0; + foreach (ArraySegment c in chunks) + { + c.AsSpan().CopyTo(ret.AsSpan(idx, c.Count)); + idx += c.Count; + ArrayPool.Shared.Return(c.Array!); + } + + return ret; + } + } + + private static void ReadSegment(ref Utf8JsonReader reader, scoped ref ReadStack state) + { + ReadOnlySpan newSegment = reader.ValueSpan.Slice(state.Current.PropertyIndex); + + if (reader.ValueIsEscaped) + { + int idx = newSegment.IndexOf(JsonConstants.BackSlash); + if (idx >= 0) + { + ReadSegmentEscaped(ref reader, ref state, newSegment, idx); + return; + } + } + + state.Current.PropertyIndex = reader.ValueSpan.Length; + } + + private static void ReadSegmentEscaped(ref Utf8JsonReader reader, scoped ref ReadStack state, ReadOnlySpan newSegment, int indexOfFirstCharToEscape) + { + List>? chunks = (List>?)state.Current.ReturnValue; + if (chunks == null) + { + // First time we are encountering an escaped character. + chunks = new List>(); + state.Current.ReturnValue = chunks; + + // The chunk must include all the segments skipped so far. + indexOfFirstCharToEscape += state.Current.PropertyIndex; + newSegment = reader.ValueSpan; + } + + byte[] unescaped = ArrayPool.Shared.Rent(newSegment.Length); + JsonReaderHelper.Unescape(newSegment, unescaped, indexOfFirstCharToEscape, out int written); + state.Current.PropertyIndex = reader.ValueSpan.Length; + + chunks.Add(new ArraySegment(unescaped, 0, written)); + } + + internal override bool WriteWithoutStackFrame(Utf8JsonWriter writer, byte[]? value, JsonSerializerOptions options, ref WriteStack state) + { + if (value == null) + { + writer.WriteNullValue(); + return true; + } + else if (state.FlushThreshold == 0 || value.Length < state.FlushThreshold) + { + // Fast write for small strings. Note that previous unflushed data may still be in the + // writer but we can let the enclosing container handle the flushing in this case. + writer.WriteBase64StringValue(value); + return true; + } + + return WriteWithStackFrame(writer, value, options, ref state); + } + + internal override bool OnTryWrite(Utf8JsonWriter writer, byte[]? value, JsonSerializerOptions options, ref WriteStack state) + { + if (!state.Current.ProcessedStartToken) + { + state.Current.ProcessedStartToken = true; + if (ShouldFlush(ref state, writer)) + { + return false; + } + } + + Debug.Assert(value != null); + bool isFinal = GetNextWriteSegment(value, ref state, out int writeIndex, out int writeLength); + writer.WriteBase64StringSegment(value.AsSpan(writeIndex, writeLength), isFinal); + state.Current.EnumeratorIndex += writeLength; + + // We either wrote the entire input or hit the flush threshold. + Debug.Assert(ShouldFlush(ref state, writer) || isFinal); + + return isFinal; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool GetNextWriteSegment(byte[] value, ref WriteStack state, out int start, out int length) + { + Debug.Assert(state.Current.EnumeratorIndex >= 0); + Debug.Assert(state.Current.EnumeratorIndex < value.Length); + + int writeIndex = state.Current.EnumeratorIndex; + + // Write enough to guarantee a flush. Base64 encoding expands the data by 4/3, so we can write less and still hit the threshold, + // but we don't need to be exact because the threshold is set very conservatively. + int writeLength = state.FlushThreshold == 0 ? int.MaxValue : state.FlushThreshold + 1; + + // If the input isn't large enough to hit the flush threshold, write the entire input as the final segment. + bool isFinal = false; + int remainingInputBytes = value.Length - state.Current.EnumeratorIndex; + if (remainingInputBytes <= writeLength) + { + writeLength = remainingInputBytes; + isFinal = true; + } + + start = writeIndex; + length = writeLength; + + return isFinal; + } + internal override JsonSchema? GetSchema(JsonNumberHandling _) => new() { Type = JsonSchemaType.String }; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ReadOnlyMemoryByteConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ReadOnlyMemoryByteConverter.cs index 5139026fc4cae5..ddd62bf44baa5e 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ReadOnlyMemoryByteConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ReadOnlyMemoryByteConverter.cs @@ -6,6 +6,7 @@ namespace System.Text.Json.Serialization.Converters { + // TODO internal sealed class ReadOnlyMemoryByteConverter : JsonConverter> { public override bool HandleNull => true; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs index 8a290109e7e369..146fa3323d7107 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs @@ -1,30 +1,215 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; +using System.Collections.Generic; using System.Diagnostics; +using System.Runtime.CompilerServices; using System.Text.Json.Nodes; using System.Text.Json.Schema; +using System.Text.Json.Serialization.Metadata; namespace System.Text.Json.Serialization.Converters { - internal sealed class StringConverter : JsonPrimitiveConverter + internal sealed class StringConverter : JsonHybridResumableConverter { public override string? Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) { + if (reader.TokenType == JsonTokenType.Null) + { + return null; + } + return reader.GetString(); } + // When called without a stack (top level) public override void Write(Utf8JsonWriter writer, string? value, JsonSerializerOptions options) { - // For performance, lift up the writer implementation. + if (value is null) + { + writer.WriteNullValue(); + return; + } + + writer.WriteStringValue(value); + } + + internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, scoped ref ReadStack state, out string? value) + { + if (state.Current.ObjectState < StackFrameObjectState.CreatedObject) + { + // This is the first segment so it can't be the only/last segment since we are on the slow read path. + Debug.Assert(reader._hasPartialStringValue); + + state.Current.ObjectState = StackFrameObjectState.ReadElements; + state.Current.ReturnValue = new List>(); + + ReadSegment(ref reader, ref state); + + value = null; + return false; + } + + Debug.Assert(state.Current.ObjectState == StackFrameObjectState.ReadElements); + + bool consumedEntireString = reader.ContinueConsumeString(); + if (!consumedEntireString && reader.IsFinalBlock) + { + // TODO + throw new Exception(); + } + + ReadSegment(ref reader, ref state); + + if (consumedEntireString) + { + value = GetStringFromChunks(reader, state); + return true; + } + + value = null; + return false; + + static string GetStringFromChunks(Utf8JsonReader reader, ReadStack state) + { + Debug.Assert(reader.TokenType == JsonTokenType.String); + + List> chunks = (List>)state.Current.ReturnValue!; + + int totalSize = 0; + foreach (ArraySegment c in chunks) + { + totalSize += c.Count; + } + + // TODO skip zeroing + string ret = new string((char)0, totalSize); + unsafe + { + fixed (char* r = ret) + { + int idx = 0; + foreach (ArraySegment c in chunks) + { + c.AsSpan().CopyTo(new Span(r + idx, c.Count)); + idx += c.Count; + ArrayPool.Shared.Return(c.Array!); + } + } + } + + return ret; + } + } + + private static void ReadSegment(ref Utf8JsonReader reader, scoped ref ReadStack state) + { + ReadOnlySpan newSegment = reader.ValueSpan.Slice(state.Current.PropertyIndex); + + if (reader.ValueIsEscaped) + { + int idx = newSegment.IndexOf(JsonConstants.BackSlash); + if (idx >= 0) + { + ReadSegmentEscaped(ref reader, ref state, newSegment, idx); + return; + } + } + + ReadSegmentCore(ref reader, ref state, newSegment); + } + + private static void ReadSegmentEscaped(ref Utf8JsonReader reader, scoped ref ReadStack state, ReadOnlySpan newSegment, int indexOfFirstCharToEscape) + { + int additionalByteCount = reader.ValueSpan.Length - state.Current.PropertyIndex; + + byte[] unescaped = ArrayPool.Shared.Rent(additionalByteCount); + JsonReaderHelper.Unescape(newSegment, unescaped, indexOfFirstCharToEscape, out int writtenTemp); + newSegment = unescaped.AsSpan(0, writtenTemp); + + ReadSegmentCore(ref reader, ref state, newSegment); + + ArrayPool.Shared.Return(unescaped); + } + + private static void ReadSegmentCore(ref Utf8JsonReader reader, scoped ref ReadStack state, ReadOnlySpan newSegment) + { + int additionalByteCount = reader.ValueSpan.Length - state.Current.PropertyIndex; + char[] chunk = ArrayPool.Shared.Rent(additionalByteCount); + + int written = JsonReaderHelper.TranscodeHelper(newSegment, chunk); + state.Current.PropertyIndex = reader.ValueSpan.Length; + + List> chunks = (List>)state.Current.ReturnValue!; + chunks.Add(new ArraySegment(chunk, 0, written)); + } + + internal override bool WriteWithoutStackFrame(Utf8JsonWriter writer, string? value, JsonSerializerOptions options, ref WriteStack state) + { if (value == null) { writer.WriteNullValue(); + return true; } - else + else if (state.FlushThreshold == 0 || value.Length < state.FlushThreshold) { - writer.WriteStringValue(value.AsSpan()); + // Fast write for small strings. Note that previous unflushed data may still be in the + // writer but we can let the enclosing container handle the flushing in this case. + writer.WriteStringValue(value); + return true; } + + return WriteWithStackFrame(writer, value, options, ref state); + } + + // When called with a stack. + internal override bool OnTryWrite(Utf8JsonWriter writer, string? value, JsonSerializerOptions options, ref WriteStack state) + { + if (!state.Current.ProcessedStartToken) + { + state.Current.ProcessedStartToken = true; + if (ShouldFlush(ref state, writer)) + { + return false; + } + } + + bool isFinal = GetNextWriteSegment(value, ref state, out int writeIndex, out int writeLength); + writer.WriteStringValueSegment(value.AsSpan(writeIndex, writeLength), isFinal); + state.Current.EnumeratorIndex += writeLength; + + // We either wrote the entire input or hit the flush threshold. + Debug.Assert(ShouldFlush(ref state, writer) || isFinal); + + return isFinal; + } + + [MethodImpl(MethodImplOptions.AggressiveInlining)] + private static bool GetNextWriteSegment(ReadOnlySpan value, ref WriteStack state, out int start, out int length) + { + Debug.Assert(state.Current.EnumeratorIndex >= 0); + Debug.Assert(state.Current.EnumeratorIndex < value.Length); + + int writeIndex = state.Current.EnumeratorIndex; + + // Write enough to guarantee a flush. Base64 encoding expands the data by 4/3, so we can write less and still hit the threshold, + // but we don't need to be exact because the threshold is set very conservatively. + int writeLength = state.FlushThreshold == 0 ? int.MaxValue : state.FlushThreshold + 1; + + // If the input isn't large enough to hit the flush threshold, write the entire input as the final segment. + bool isFinal = false; + int remainingInputBytes = value.Length - state.Current.EnumeratorIndex; + if (remainingInputBytes <= writeLength) + { + writeLength = remainingInputBytes; + isFinal = true; + } + + start = writeIndex; + length = writeLength; + + return isFinal; } internal override string ReadAsPropertyNameCore(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) @@ -54,5 +239,25 @@ internal override void WriteAsPropertyNameCore(Utf8JsonWriter writer, string val } internal override JsonSchema? GetSchema(JsonNumberHandling _) => new() { Type = JsonSchemaType.String }; + + public sealed override void WriteAsPropertyName(Utf8JsonWriter writer, string? value, JsonSerializerOptions options) + { + if (value is null) + { + ThrowHelper.ThrowArgumentNullException(nameof(value)); + } + + WriteAsPropertyNameCore(writer, value, options, isWritingExtensionDataProperty: false); + } + + public sealed override string? ReadAsPropertyName(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + if (reader.TokenType != JsonTokenType.PropertyName) + { + ThrowHelper.ThrowInvalidOperationException_ExpectedPropertyName(reader.TokenType); + } + + return ReadAsPropertyNameCore(ref reader, typeToConvert, options); + } } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverter.cs index 1a0e007695dcfb..ac36f3d8f909b2 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverter.cs @@ -39,17 +39,15 @@ internal JsonConverter() internal ConverterStrategy ConverterStrategy { - get => _converterStrategy; + get => field; init { - CanUseDirectReadOrWrite = value == ConverterStrategy.Value && IsInternalConverter; - RequiresReadAhead = value == ConverterStrategy.Value; - _converterStrategy = value; + CanUseDirectReadOrWrite = value == ConverterStrategy.SimpleValue && IsInternalConverter; + RequiresReadAhead = value == ConverterStrategy.SimpleValue; + field = value; } } - private ConverterStrategy _converterStrategy; - /// /// Invoked by the base contructor to populate the initial value of the property. /// Used for declaring the default strategy for specific converter hierarchies without explicitly setting in a constructor. @@ -90,6 +88,8 @@ internal ConverterStrategy ConverterStrategy /// internal bool RequiresReadAhead { get; private protected set; } + internal bool CanConsumePartialReaderValue => ConverterStrategy == ConverterStrategy.SegmentableValue; + /// /// Whether the converter is a special root-level value streaming converter. /// diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.ReadCore.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.ReadCore.cs index 7240239b8647ba..1f7a4bce971e30 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.ReadCore.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.ReadCore.cs @@ -19,7 +19,8 @@ internal bool ReadCore( { // This is first call to the converter -- advance the reader // to the first JSON token and perform a read-ahead if necessary. - if (!reader.TryAdvanceWithOptionalReadAhead(RequiresReadAhead)) + if (!reader.TryAdvanceWithOptionalReadAhead(RequiresReadAhead) && + (!CanConsumePartialReaderValue || reader.IsFinalBlock)) // Partial non-final values can be processed by some converters { if (state.SupportContinuation && state.Current.ReturnValue is object result) { diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs index f03bbbee3cfde4..f9218ceba3ffb4 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs @@ -50,7 +50,7 @@ public override bool CanConvert(Type typeToConvert) return typeToConvert == typeof(T); } - private protected override ConverterStrategy GetDefaultConverterStrategy() => ConverterStrategy.Value; + private protected override ConverterStrategy GetDefaultConverterStrategy() => ConverterStrategy.SimpleValue; internal sealed override JsonTypeInfo CreateJsonTypeInfo(JsonSerializerOptions options) { @@ -160,7 +160,8 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali return true; } - if (ConverterStrategy == ConverterStrategy.Value) + if (ConverterStrategy == ConverterStrategy.SimpleValue || + ConverterStrategy == ConverterStrategy.SegmentableValue && !state.IsContinuation && !reader._hasPartialStringValue) { // A value converter should never be within a continuation. Debug.Assert(!state.IsContinuation); @@ -264,7 +265,7 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali state.Current.OriginalTokenType, state.Current.OriginalDepth, bytesConsumed: 0, - isValueConverter: false, + isValueConverter: ConverterStrategy == ConverterStrategy.SegmentableValue, ref reader); // No need to clear state.Current.* since a stack pop will occur. @@ -339,7 +340,7 @@ internal bool TryWrite(Utf8JsonWriter writer, in T? value, JsonSerializerOptions return true; } - if (ConverterStrategy == ConverterStrategy.Value) + if (ConverterStrategy == ConverterStrategy.SimpleValue) { Debug.Assert(!state.IsContinuation); @@ -360,9 +361,19 @@ internal bool TryWrite(Utf8JsonWriter writer, in T? value, JsonSerializerOptions Debug.Assert(IsInternalConverter); bool isContinuation = state.IsContinuation; + + if (ConverterStrategy == ConverterStrategy.SegmentableValue && !state.IsContinuation) + { + Debug.Assert(this is JsonHybridResumableConverter); + + JsonHybridResumableConverter jsonHybridConverter = (JsonHybridResumableConverter)this; + return jsonHybridConverter.WriteWithoutStackFrame(writer, value, options, ref state); + } + bool success; if ( + ConverterStrategy != ConverterStrategy.SegmentableValue && #if NET // Short-circuit the check against "is not null"; treated as a constant by recent versions of the JIT. !typeof(T).IsValueType && @@ -411,6 +422,7 @@ value is not null && // DEBUG: ensure push/pop operations preserve stack integrity JsonTypeInfo originalJsonTypeInfo = state.Current.JsonTypeInfo; #endif + state.Push(); Debug.Assert(Type == state.Current.JsonTypeInfo.Type); @@ -498,7 +510,7 @@ internal bool TryWriteDataExtensionProperty(Utf8JsonWriter writer, T value, Json internal void VerifyRead(JsonTokenType tokenType, int depth, long bytesConsumed, bool isValueConverter, ref Utf8JsonReader reader) { - Debug.Assert(isValueConverter == (ConverterStrategy == ConverterStrategy.Value)); + Debug.Assert(isValueConverter == JsonSerializer.IsValueConverterStrategy(ConverterStrategy)); switch (tokenType) { @@ -527,14 +539,14 @@ internal void VerifyRead(JsonTokenType tokenType, int depth, long bytesConsumed, break; case JsonTokenType.None: - Debug.Assert(IsRootLevelMultiContentStreamingConverter); + Debug.Assert(IsRootLevelMultiContentStreamingConverter || CanConsumePartialReaderValue); break; default: if (isValueConverter) { // A value converter should not make any reads. - if (reader.BytesConsumed != bytesConsumed) + if (reader.BytesConsumed != bytesConsumed && ConverterStrategy != ConverterStrategy.SegmentableValue) { ThrowHelper.ThrowJsonException_SerializationConverterRead(this); } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonHybridResumableConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonHybridResumableConverter.cs new file mode 100644 index 00000000000000..38e9b4830f3487 --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonHybridResumableConverter.cs @@ -0,0 +1,69 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System.Diagnostics; +using System.Text.Json.Serialization.Metadata; + +namespace System.Text.Json.Serialization +{ + /// + /// Base class for converters that are able to resume after reading or writing to a buffer. + /// Writes initially start as non-resumable but are able to upgrade to resumable. + /// This is used when the Stream-based serialization APIs are used. + /// + /// + internal abstract class JsonHybridResumableConverter : JsonConverter + { + public sealed override bool HandleNull => false; + + private protected sealed override ConverterStrategy GetDefaultConverterStrategy() => ConverterStrategy.SegmentableValue; + + /// + /// Writes the value without a stack frame. If the conversion should be done with resumption then + /// should be called from within this method to push/pop a stack frame. This allows opt-in resumption based on the value. + /// + internal abstract bool WriteWithoutStackFrame(Utf8JsonWriter writer, T? value, JsonSerializerOptions options, ref WriteStack state); + + /// + /// Pushes a stack frame, calls and + /// pops the frame. + /// + /// + /// + /// + /// + /// + private protected bool WriteWithStackFrame(Utf8JsonWriter writer, T value, JsonSerializerOptions options, ref WriteStack state) + { +#if DEBUG + // DEBUG: ensure push/pop operations preserve stack integrity + JsonTypeInfo originalJsonTypeInfo = state.Current.JsonTypeInfo; +#endif + + state.Push(); + +#if DEBUG + // For performance, only perform validation on internal converters on debug builds. + Debug.Assert(state.Current.OriginalDepth == 0); + state.Current.OriginalDepth = writer.CurrentDepth; +#endif + + bool success = OnTryWrite(writer, value, options, ref state); + +#if DEBUG + if (success) + { + VerifyWrite(state.Current.OriginalDepth, writer); + } +#endif + + state.Pop(success); + +#if DEBUG + Debug.Assert(ReferenceEquals(originalJsonTypeInfo, state.Current.JsonTypeInfo)); +#endif + + return success; + } + } +} diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs index 81bceda8d8404c..1e5113f119a216 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonResumableConverterOfT.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; using System.Text.Json.Serialization.Metadata; namespace System.Text.Json.Serialization diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Helpers.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Helpers.cs index 0b3301fcc0a125..3b07be60790b25 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Helpers.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonSerializer.Helpers.cs @@ -142,5 +142,8 @@ static void ThrowUnableToCastValue(object? value) return (T?)value; } + + internal static bool IsValueConverterStrategy(ConverterStrategy strategy) => + (strategy & (ConverterStrategy.SimpleValue | ConverterStrategy.SegmentableValue)) != 0; } } diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonPropertyInfoOfT.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonPropertyInfoOfT.cs index ed293cc4cb9ff2..4f023f29e961a5 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonPropertyInfoOfT.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonPropertyInfoOfT.cs @@ -182,7 +182,7 @@ internal override bool GetMemberAndWriteJson(object obj, ref WriteStack state, U value is not null && !state.IsContinuation && // .NET types that are serialized as JSON primitive values don't need to be tracked for cycle detection e.g: string. - EffectiveConverter.ConverterStrategy != ConverterStrategy.Value && + !JsonSerializer.IsValueConverterStrategy(EffectiveConverter.ConverterStrategy) && state.ReferenceResolver.ContainsReferenceForCycleDetection(value)) { // If a reference cycle is detected, treat value as null. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs index bd3e3d9241f857..325e954091a53f 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/JsonTypeInfo.cs @@ -1352,7 +1352,8 @@ private static JsonTypeInfoKind GetTypeInfoKind(Type type, JsonConverter convert switch (converter.ConverterStrategy) { - case ConverterStrategy.Value: return JsonTypeInfoKind.None; + case ConverterStrategy.SimpleValue: return JsonTypeInfoKind.None; + case ConverterStrategy.SegmentableValue: return JsonTypeInfoKind.None; case ConverterStrategy.Object: return JsonTypeInfoKind.Object; case ConverterStrategy.Enumerable: return JsonTypeInfoKind.Enumerable; case ConverterStrategy.Dictionary: return JsonTypeInfoKind.Dictionary; diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs new file mode 100644 index 00000000000000..30de531ecdc6ec --- /dev/null +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs @@ -0,0 +1,32 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.IO; +using System.IO.Pipelines; +using System.Text; +using System.Threading; +using System.Threading.Tasks; + +namespace System.Text.Json.Serialization.Metadata +{ + //internal class StringTypeInfo : JsonTypeInfo + //{ + // internal static readonly StringTypeInfo Instance = new StringTypeInfo(); + + // private StringTypeInfo() { } + + // internal override object? DeserializeAsObject(ref Utf8JsonReader reader, ref ReadStack state) => throw new NotImplementedException(); + // internal override object? DeserializeAsObject(Stream utf8Json) => throw new NotImplementedException(); + // internal override ValueTask DeserializeAsObjectAsync(Stream utf8Json, CancellationToken cancellationToken) => throw new NotImplementedException(); + // internal override void SerializeAsObject(Utf8JsonWriter writer, object? rootValue) => throw new NotImplementedException(); + // internal override void SerializeAsObject(Stream utf8Json, object? rootValue) => throw new NotImplementedException(); + // internal override Task SerializeAsObjectAsync(PipeWriter pipeWriter, object? rootValue, int flushThreshold, CancellationToken cancellationToken) => throw new NotImplementedException(); + // internal override Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken) => throw new NotImplementedException(); + // internal override Task SerializeAsObjectAsync(PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken) => throw new NotImplementedException(); + // private protected override JsonPropertyInfo CreateJsonPropertyInfo(JsonTypeInfo declaringTypeInfo, Type? declaringType, JsonSerializerOptions options) => throw new NotImplementedException(); + // private protected override JsonPropertyInfo CreatePropertyInfoForTypeInfo() => throw new NotImplementedException(); + // private protected override void SetCreateObject(Delegate? createObject) => throw new NotImplementedException(); + //} +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs index 909611ca2c7323..afe8da0cf6a73f 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs @@ -1,6 +1,7 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Buffers; using System.Collections.Generic; using Xunit; diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/HybridResumableConverterTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/HybridResumableConverterTests.cs new file mode 100644 index 00000000000000..7013e7a6a2f3b4 --- /dev/null +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/HybridResumableConverterTests.cs @@ -0,0 +1,122 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using Xunit; +using System.Threading.Tasks; +using System.IO.Pipelines; +using System.Buffers; +using System.Collections.Generic; + +namespace System.Text.Json.Serialization.Tests +{ + public class HybridResumableConverterTests + { + internal class InstrumentedMemoryPool : MemoryPool + { + public List RequestedBufferSizes = new(); + public int CumulativeAllocatedBytes = 0; + public int CurrentAllocatedBytes = 0; + public int PeakAllocatedBytes = 0; + + public override int MaxBufferSize => int.MaxValue; + + public override IMemoryOwner Rent(int minBufferSize = -1) + { + RequestedBufferSizes.Add(minBufferSize); + IMemoryOwner memory = MemoryPool.Shared.Rent(minBufferSize); + CurrentAllocatedBytes += memory.Memory.Length; + PeakAllocatedBytes = Math.Max(PeakAllocatedBytes, CurrentAllocatedBytes); + CumulativeAllocatedBytes += memory.Memory.Length; + return new IntrumentedOwner(this, memory); + } + + private void Return(IntrumentedOwner owner) + { + CurrentAllocatedBytes -= owner._memory.Memory.Length; + owner._memory.Dispose(); + } + + protected override void Dispose(bool disposing) { } + + private class IntrumentedOwner : IMemoryOwner + { + public readonly InstrumentedMemoryPool _parent; + public readonly IMemoryOwner _memory; + + public IntrumentedOwner(InstrumentedMemoryPool parent, IMemoryOwner memory) + { + _parent = parent; + _memory = memory; + } + + public Memory Memory => _memory.Memory; + public void Dispose() => _parent.Return(this); + } + } + + [Fact] + public static async Task WriteByteArraySegmentedAsync() + { + // We need to create a large enough value that will trigger the segmented writing logic. + // The threshold in the code is 90% of 4 * MinimumSegmentSize, so we need a write to exceed that. + // We also provide the buffer pool to validate that the requested buffers are less than the total write size. + int threshold = (int)(0.9 * 4 * PipeOptions.Default.MinimumSegmentSize); + var pool = new InstrumentedMemoryPool(); + var pipe = new Pipe(new PipeOptions(pool)); + + var consumerFunc = async () => + { + PipeReader reader = pipe.Reader; + ReadResult result; + while (!(result = await reader.ReadAsync()).IsCompleted) reader.AdvanceTo(result.Buffer.End); + await reader.CompleteAsync(); + }; + + Task consumer = Task.Run(consumerFunc); + + // Exceed the threshold by a large amount. + int writeSize = 64 * threshold; + await JsonSerializer.SerializeAsync(pipe.Writer, new byte[writeSize]); + await pipe.Writer.CompleteAsync(); + await consumer; + + // Ensure all requested buffer sizes are capped. Note the threshold is just a heuristic, so it is possible that the threshold + // will be far exceeded in practice. We just want to ensure that the requested buffer sizes have a constant upper bound. + Assert.All(pool.RequestedBufferSizes, size => Assert.InRange(size, 0, 16 * threshold)); + Assert.InRange(pool.PeakAllocatedBytes, 0, 16 * threshold); + } + + // TODO move to string test class + [Fact] + public static async Task WriteStringSegmentedAsync() + { + // We need to create a large enough value that will trigger the segmented writing logic. + // The threshold in the code is 90% of 4 * MinimumSegmentSize, so we need a write to exceed that. + // We also provide the buffer pool to validate that the requested buffers are less than the total write size. + int threshold = (int)(0.9 * 4 * PipeOptions.Default.MinimumSegmentSize); + var pool = new InstrumentedMemoryPool(); + var pipe = new Pipe(new PipeOptions(pool)); + + var consumerFunc = async () => + { + PipeReader reader = pipe.Reader; + ReadResult result; + while (!(result = await reader.ReadAsync()).IsCompleted) reader.AdvanceTo(result.Buffer.End); + await reader.CompleteAsync(); + }; + + Task consumer = Task.Run(consumerFunc); + + // Exceed the threshold by a large amount. + int writeSize = 64 * threshold; + await JsonSerializer.SerializeAsync(pipe.Writer, new string('a', writeSize)); + await pipe.Writer.CompleteAsync(); + await consumer; + + // Ensure all requested buffer sizes are capped. Note the threshold is just a heuristic, so it is possible that the threshold + // will be far exceeded in practice. We just want to ensure that the requested buffer sizes have a constant upper bound. + Assert.All(pool.RequestedBufferSizes, size => Assert.InRange(size, 0, 16 * threshold)); + Assert.InRange(pool.PeakAllocatedBytes, 0, 16 * threshold); + } + } +} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Stream.ReadTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Stream.ReadTests.cs index e296ee97c8d068..ac082eb161d45e 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Stream.ReadTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Stream.ReadTests.cs @@ -66,6 +66,31 @@ public async Task ReadPrimitivesAsync() } } + [Fact] + public async Task ReadLongStringAsync() + { + string str = new string('a', 1_000_000); + byte[] bytes = Encoding.UTF8.GetBytes(str); + bytes[0] = bytes[bytes.Length - 1] = (byte)'"'; + + using (MemoryStream stream = new MemoryStream(bytes)) + { + JsonSerializerOptions options = new JsonSerializerOptions + { + DefaultBufferSize = 1 + }; + + string actual = await Serializer.DeserializeWrapper(stream, options); + Assert.Equal(str +#if NET + .AsSpan(2) +#else + .Substring(2) +#endif + , actual); + } + } + [Fact] public async Task ReadPrimitivesWithTrailingTriviaAsync() { diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index b70e73d3b83254..d14c68d7378585 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -173,6 +173,7 @@ + From 672672f013150d6afb6c089ec7eaed29930fe7d5 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Tue, 4 Feb 2025 07:34:12 -0800 Subject: [PATCH 2/3] cleanup --- .../Serialization/Metadata/StringTypeInfo.cs | 32 ------------------- .../Serialization/Array.WriteTests.cs | 1 - 2 files changed, 33 deletions(-) delete mode 100644 src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs deleted file mode 100644 index 30de531ecdc6ec..00000000000000 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Metadata/StringTypeInfo.cs +++ /dev/null @@ -1,32 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.IO; -using System.IO.Pipelines; -using System.Text; -using System.Threading; -using System.Threading.Tasks; - -namespace System.Text.Json.Serialization.Metadata -{ - //internal class StringTypeInfo : JsonTypeInfo - //{ - // internal static readonly StringTypeInfo Instance = new StringTypeInfo(); - - // private StringTypeInfo() { } - - // internal override object? DeserializeAsObject(ref Utf8JsonReader reader, ref ReadStack state) => throw new NotImplementedException(); - // internal override object? DeserializeAsObject(Stream utf8Json) => throw new NotImplementedException(); - // internal override ValueTask DeserializeAsObjectAsync(Stream utf8Json, CancellationToken cancellationToken) => throw new NotImplementedException(); - // internal override void SerializeAsObject(Utf8JsonWriter writer, object? rootValue) => throw new NotImplementedException(); - // internal override void SerializeAsObject(Stream utf8Json, object? rootValue) => throw new NotImplementedException(); - // internal override Task SerializeAsObjectAsync(PipeWriter pipeWriter, object? rootValue, int flushThreshold, CancellationToken cancellationToken) => throw new NotImplementedException(); - // internal override Task SerializeAsObjectAsync(Stream utf8Json, object? rootValue, CancellationToken cancellationToken) => throw new NotImplementedException(); - // internal override Task SerializeAsObjectAsync(PipeWriter utf8Json, object? rootValue, CancellationToken cancellationToken) => throw new NotImplementedException(); - // private protected override JsonPropertyInfo CreateJsonPropertyInfo(JsonTypeInfo declaringTypeInfo, Type? declaringType, JsonSerializerOptions options) => throw new NotImplementedException(); - // private protected override JsonPropertyInfo CreatePropertyInfoForTypeInfo() => throw new NotImplementedException(); - // private protected override void SetCreateObject(Delegate? createObject) => throw new NotImplementedException(); - //} -} diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs index afe8da0cf6a73f..909611ca2c7323 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/Serialization/Array.WriteTests.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Buffers; using System.Collections.Generic; using Xunit; From 9adb45a12e64b2e0ac2d4445c9aefdf0325ed9e6 Mon Sep 17 00:00:00 2001 From: Pranav Senthilnathan Date: Tue, 15 Apr 2025 13:18:27 -0700 Subject: [PATCH 3/3] move logic from TryRead to converter --- .../System.Text.Json/src/System.Text.Json.csproj | 1 - .../Converters/Value/ByteArrayConverter.cs | 7 +++++++ .../Serialization/Converters/Value/StringConverter.cs | 6 ++++++ .../Text/Json/Serialization/JsonConverterOfT.cs | 11 +++++------ 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/src/libraries/System.Text.Json/src/System.Text.Json.csproj b/src/libraries/System.Text.Json/src/System.Text.Json.csproj index a5267f99382917..fbe483e1e7a6fd 100644 --- a/src/libraries/System.Text.Json/src/System.Text.Json.csproj +++ b/src/libraries/System.Text.Json/src/System.Text.Json.csproj @@ -159,7 +159,6 @@ The System.Text.Json library is built-in as part of the shared framework in .NET - diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs index 1a599d032ecdad..b356b0f6817cbc 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/ByteArrayConverter.cs @@ -18,6 +18,7 @@ internal sealed class ByteArrayConverter : JsonHybridResumableConverter { return null; } + return reader.GetBytesFromBase64(); } @@ -35,6 +36,12 @@ public override void Write(Utf8JsonWriter writer, byte[]? value, JsonSerializerO internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, scoped ref ReadStack state, out byte[]? value) { + if (!reader._hasPartialStringValue && state.Current.ObjectState == StackFrameObjectState.None) + { + value = reader.GetBytesFromBase64(); + return true; + } + if (state.Current.ObjectState < StackFrameObjectState.CreatedObject) { // This is the first segment so it can't be the only/last segment since we are on the slow read path. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs index 146fa3323d7107..0796d4d0b9d8f0 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/Converters/Value/StringConverter.cs @@ -37,6 +37,12 @@ public override void Write(Utf8JsonWriter writer, string? value, JsonSerializerO internal override bool OnTryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options, scoped ref ReadStack state, out string? value) { + if (!reader._hasPartialStringValue && state.Current.ObjectState == StackFrameObjectState.None) + { + value = reader.GetString(); + return true; + } + if (state.Current.ObjectState < StackFrameObjectState.CreatedObject) { // This is the first segment so it can't be the only/last segment since we are on the slow read path. diff --git a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs index f9218ceba3ffb4..4c85ddcbccb0e6 100644 --- a/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs +++ b/src/libraries/System.Text.Json/src/System/Text/Json/Serialization/JsonConverterOfT.cs @@ -160,8 +160,7 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali return true; } - if (ConverterStrategy == ConverterStrategy.SimpleValue || - ConverterStrategy == ConverterStrategy.SegmentableValue && !state.IsContinuation && !reader._hasPartialStringValue) + if (ConverterStrategy == ConverterStrategy.SimpleValue) { // A value converter should never be within a continuation. Debug.Assert(!state.IsContinuation); @@ -265,7 +264,7 @@ internal bool TryRead(ref Utf8JsonReader reader, Type typeToConvert, JsonSeriali state.Current.OriginalTokenType, state.Current.OriginalDepth, bytesConsumed: 0, - isValueConverter: ConverterStrategy == ConverterStrategy.SegmentableValue, + isValueConverter: false, ref reader); // No need to clear state.Current.* since a stack pop will occur. @@ -510,7 +509,7 @@ internal bool TryWriteDataExtensionProperty(Utf8JsonWriter writer, T value, Json internal void VerifyRead(JsonTokenType tokenType, int depth, long bytesConsumed, bool isValueConverter, ref Utf8JsonReader reader) { - Debug.Assert(isValueConverter == JsonSerializer.IsValueConverterStrategy(ConverterStrategy)); + Debug.Assert(isValueConverter == (ConverterStrategy == ConverterStrategy.SimpleValue)); switch (tokenType) { @@ -546,7 +545,7 @@ internal void VerifyRead(JsonTokenType tokenType, int depth, long bytesConsumed, if (isValueConverter) { // A value converter should not make any reads. - if (reader.BytesConsumed != bytesConsumed && ConverterStrategy != ConverterStrategy.SegmentableValue) + if (reader.BytesConsumed != bytesConsumed) { ThrowHelper.ThrowJsonException_SerializationConverterRead(this); } @@ -555,7 +554,7 @@ internal void VerifyRead(JsonTokenType tokenType, int depth, long bytesConsumed, { // A non-value converter (object or collection) should always have Start and End tokens // unless it is polymorphic or supports null value reads. - if (!CanBePolymorphic && !(HandleNullOnRead && tokenType == JsonTokenType.Null)) + if (!CanBePolymorphic && !(HandleNullOnRead && tokenType == JsonTokenType.Null) && tokenType != JsonTokenType.String) { ThrowHelper.ThrowJsonException_SerializationConverterRead(this); }