Add Utf8JsonReader.AllowMultipleValues and related APIs.

src/libraries/System.Text.Json/ref/System.Text.Json.cs

@@ -187,6 +187,7 @@ public partial struct JsonReaderOptions
     {
         private int _dummyPrimitive;
         public bool AllowTrailingCommas { readonly get { throw null; } set { } }
+        public bool AllowMultipleValues { readonly get { throw null; } set { } }
         public System.Text.Json.JsonCommentHandling CommentHandling { readonly get { throw null; } set { } }
         public int MaxDepth { readonly get { throw null; } set { } }
     }
@@ -247,7 +248,11 @@ public static partial class JsonSerializer
         public static System.Threading.Tasks.ValueTask<object?> DeserializeAsync(System.IO.Stream utf8Json, System.Type returnType, System.Text.Json.Serialization.JsonSerializerContext context, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
         [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]
+        public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, bool topLevelValues, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
+        [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
+        [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]
         public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.JsonSerializerOptions? options = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
+        public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.Serialization.Metadata.JsonTypeInfo<TValue> jsonTypeInfo, bool topLevelValues, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         public static System.Collections.Generic.IAsyncEnumerable<TValue?> DeserializeAsyncEnumerable<TValue>(System.IO.Stream utf8Json, System.Text.Json.Serialization.Metadata.JsonTypeInfo<TValue> jsonTypeInfo, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken)) { throw null; }
         [System.Diagnostics.CodeAnalysis.RequiresDynamicCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed and might need runtime code generation. Use System.Text.Json source generation for native AOT applications.")]
         [System.Diagnostics.CodeAnalysis.RequiresUnreferencedCodeAttribute("JSON serialization and deserialization might require types that cannot be statically analyzed. Use the overload that takes a JsonTypeInfo or JsonSerializerContext, or make sure all of the required types are preserved.")]

src/libraries/System.Text.Json/src/System.Text.Json.csproj

@@ -124,6 +124,7 @@ The System.Text.Json library is built-in as part of the shared framework in .NET
     <Compile Include="System\Text\Json\Serialization\Converters\Collection\ImmutableEnumerableOfTConverterWithReflection.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Collection\ReadOnlyMemoryConverter.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Collection\MemoryConverterFactory.cs" />
+    <Compile Include="System\Text\Json\Serialization\Converters\Collection\RootLevelListConverter.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Collection\StackOrQueueConverterWithReflection.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\JsonMetadataServicesConverter.cs" />
     <Compile Include="System\Text\Json\Serialization\Converters\Object\ObjectWithParameterizedConstructorConverter.Large.Reflection.cs" />

src/libraries/System.Text.Json/src/System/Text/Json/JsonHelpers.cs

@@ -37,43 +37,72 @@ public static bool TryAdvanceWithOptionalReadAhead(this scoped ref Utf8JsonReade
             // No read-ahead necessary if we're at the final block of JSON data.
             bool readAhead = requiresReadAhead && !reader.IsFinalBlock;
             return readAhead ? TryAdvanceWithReadAhead(ref reader) : reader.Read();
+        }
 
-            // The read-ahead method is not inlined
-            static bool TryAdvanceWithReadAhead(scoped ref Utf8JsonReader reader)
+        /// <summary>
+        /// Attempts to read ahead to the next root-level JSON value, if it exists.
+        /// </summary>
+        public static bool TryAdvanceToNextRootLevelValueWithOptionalReadAhead(this scoped ref Utf8JsonReader reader, bool requiresReadAhead, out bool isAtEndOfStream)
+        {
+            Debug.Assert(reader.AllowMultipleValues, "only supported by readers that support multiple values.");
+            Debug.Assert(reader.CurrentDepth == 0, "should only invoked for top-level values.");
+
+            Utf8JsonReader checkpoint = reader;
+            if (!reader.Read())
             {
-                // When we're reading ahead we always have to save the state
-                // as we don't know if the next token is a start object or array.
-                Utf8JsonReader restore = reader;
+                // If the reader didn't return any tokens and it's the final block,
+                // then there are no other JSON values to be read.
+                isAtEndOfStream = reader.IsFinalBlock;
+                reader = checkpoint;
+                return false;
+            }
 
-                if (!reader.Read())
-                {
-                    return false;
-                }
+            // We found another JSON value, read ahead accordingly.
+            isAtEndOfStream = false;
+            if (requiresReadAhead && !reader.IsFinalBlock)
+            {
+                // Perform full read-ahead to ensure the full JSON value has been buffered.
+                reader = checkpoint;
+                return TryAdvanceWithReadAhead(ref reader);
+            }
 
-                // Perform the actual read-ahead.
-                JsonTokenType tokenType = reader.TokenType;
-                if (tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray)
+            return true;
+        }
+
+        private static bool TryAdvanceWithReadAhead(scoped ref Utf8JsonReader reader)
+        {
+            // When we're reading ahead we always have to save the state
+            // as we don't know if the next token is a start object or array.
+            Utf8JsonReader restore = reader;
+
+            if (!reader.Read())
+            {
+                return false;
+            }
+
+            // Perform the actual read-ahead.
+            JsonTokenType tokenType = reader.TokenType;
+            if (tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray)
+            {
+                // Attempt to skip to make sure we have all the data we need.
+                bool complete = reader.TrySkipPartial();
+
+                // We need to restore the state in all cases as we need to be positioned back before
+                // the current token to either attempt to skip again or to actually read the value.
+                reader = restore;
+
+                if (!complete)
                 {
-                    // Attempt to skip to make sure we have all the data we need.
-                    bool complete = reader.TrySkipPartial();
-
-                    // We need to restore the state in all cases as we need to be positioned back before
-                    // the current token to either attempt to skip again or to actually read the value.
-                    reader = restore;
-
-                    if (!complete)
-                    {
-                        // Couldn't read to the end of the object, exit out to get more data in the buffer.
-                        return false;
-                    }
-
-                    // Success, requeue the reader to the start token.
-                    reader.ReadWithVerify();
-                    Debug.Assert(tokenType == reader.TokenType);
+                    // Couldn't read to the end of the object, exit out to get more data in the buffer.
+                    return false;
                 }
 
-                return true;
+                // Success, requeue the reader to the start token.
+                reader.ReadWithVerify();
+                Debug.Assert(tokenType == reader.TokenType);
             }
+
+            return true;
         }
 
 #if !NET

src/libraries/System.Text.Json/src/System/Text/Json/Reader/JsonReaderOptions.cs

@@ -70,5 +70,14 @@ public int MaxDepth
         /// By default, it's set to false, and <exception cref="JsonException"/> is thrown if a trailing comma is encountered.
         /// </remarks>
         public bool AllowTrailingCommas { get; set; }
+
+        /// <summary>
+        /// Defines whether the <see cref="Utf8JsonReader"/> should tolerate
+        /// zero or more top-level JSON values that are whitespace separated.
+        /// </summary>
+        /// <remarks>
+        /// By default, it's set to false, and <exception cref="JsonException"/> is thrown if trailing content is encountered.
+        /// </remarks>
+        public bool AllowMultipleValues { get; set; }
     }
 }

src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.MultiSegment.cs

@@ -363,17 +363,13 @@ private bool ReadFirstTokenMultiSegment(byte first)
                     }
                     _tokenType = JsonTokenType.Number;
                     _consumed += numberOfBytes;
-                    return true;
                 }
                 else if (!ConsumeValueMultiSegment(first))
                 {
                     return false;
                 }
 
-                if (_tokenType == JsonTokenType.StartObject || _tokenType == JsonTokenType.StartArray)
-                {
-                    _isNotPrimitive = true;
-                }
+                _isNotPrimitive = _tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray;
                 // Intentionally fall out of the if-block to return true
             }
             return true;
@@ -1580,6 +1576,11 @@ private ConsumeTokenResult ConsumeNextTokenMultiSegment(byte marker)
 
             if (_bitStack.CurrentDepth == 0)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstTokenMultiSegment(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
             }
 
@@ -1711,6 +1712,11 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentTokenMultiSegment()
 
             if (_bitStack.CurrentDepth == 0 && _tokenType != JsonTokenType.None)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstTokenMultiSegment(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, first);
             }
 
@@ -2064,6 +2070,11 @@ private ConsumeTokenResult ConsumeNextTokenUntilAfterAllCommentsAreSkippedMultiS
             }
             else if (_bitStack.CurrentDepth == 0)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstTokenMultiSegment(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
             }
             else if (marker == JsonConstants.ListSeparator)

src/libraries/System.Text.Json/src/System/Text/Json/Reader/Utf8JsonReader.cs

@@ -4,7 +4,6 @@
 using System.Buffers;
 using System.Diagnostics;
 using System.Runtime.CompilerServices;
-using System.Runtime.InteropServices;
 
 namespace System.Text.Json
 {
@@ -55,6 +54,8 @@ public ref partial struct Utf8JsonReader
 
         internal readonly int ValueLength => HasValueSequence ? checked((int)ValueSequence.Length) : ValueSpan.Length;
 
+        internal readonly bool AllowMultipleValues => _readerOptions.AllowMultipleValues;
+
         /// <summary>
         /// Gets the value of the last processed token as a ReadOnlySpan&lt;byte&gt; slice
         /// of the input payload. If the JSON is provided within a ReadOnlySequence&lt;byte&gt;
@@ -280,7 +281,7 @@ public bool Read()
 
             if (!retVal)
             {
-                if (_isFinalBlock && TokenType == JsonTokenType.None)
+                if (_isFinalBlock && TokenType is JsonTokenType.None && !_readerOptions.AllowMultipleValues)
                 {
                     ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedJsonTokens);
                 }
@@ -929,7 +930,7 @@ private bool HasMoreData()
                         return false;
                     }
 
-                    if (_tokenType != JsonTokenType.EndArray && _tokenType != JsonTokenType.EndObject)
+                    if (_tokenType is not JsonTokenType.EndArray and not JsonTokenType.EndObject)
                     {
                         ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.InvalidEndOfJsonNonPrimitive);
                     }
@@ -991,17 +992,13 @@ private bool ReadFirstToken(byte first)
                     _tokenType = JsonTokenType.Number;
                     _consumed += numberOfBytes;
                     _bytePositionInLine += numberOfBytes;
-                    return true;
                 }
                 else if (!ConsumeValue(first))
                 {
                     return false;
                 }
 
-                if (_tokenType == JsonTokenType.StartObject || _tokenType == JsonTokenType.StartArray)
-                {
-                    _isNotPrimitive = true;
-                }
+                _isNotPrimitive = _tokenType is JsonTokenType.StartObject or JsonTokenType.StartArray;
                 // Intentionally fall out of the if-block to return true
             }
             return true;
@@ -1016,10 +1013,10 @@ private void SkipWhiteSpace()
                 byte val = localBuffer[_consumed];
 
                 // JSON RFC 8259 section 2 says only these 4 characters count, not all of the Unicode definitions of whitespace.
-                if (val != JsonConstants.Space &&
-                    val != JsonConstants.CarriageReturn &&
-                    val != JsonConstants.LineFeed &&
-                    val != JsonConstants.Tab)
+                if (val is not JsonConstants.Space and
+                           not JsonConstants.CarriageReturn and
+                           not JsonConstants.LineFeed and
+                           not JsonConstants.Tab)
                 {
                     break;
                 }
@@ -1747,6 +1744,11 @@ private ConsumeTokenResult ConsumeNextToken(byte marker)
 
             if (_bitStack.CurrentDepth == 0)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstToken(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
             }
 
@@ -1869,6 +1871,11 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentToken()
 
             if (_bitStack.CurrentDepth == 0 && _tokenType != JsonTokenType.None)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstToken(first) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, first);
             }
 
@@ -2033,7 +2040,7 @@ private ConsumeTokenResult ConsumeNextTokenFromLastNonCommentToken()
             }
             else
             {
-                Debug.Assert(_tokenType == JsonTokenType.EndArray || _tokenType == JsonTokenType.EndObject);
+                Debug.Assert(_tokenType is JsonTokenType.EndArray or JsonTokenType.EndObject);
                 if (_inObject)
                 {
                     Debug.Assert(first != JsonConstants.CloseBrace);
@@ -2207,6 +2214,11 @@ private ConsumeTokenResult ConsumeNextTokenUntilAfterAllCommentsAreSkipped(byte
             }
             else if (_bitStack.CurrentDepth == 0)
             {
+                if (_readerOptions.AllowMultipleValues)
+                {
+                    return ReadFirstToken(marker) ? ConsumeTokenResult.Success : ConsumeTokenResult.NotEnoughDataRollBackState;
+                }
+
                 ThrowHelper.ThrowJsonReaderException(ref this, ExceptionResource.ExpectedEndAfterSingleJson, marker);
             }
             else if (marker == JsonConstants.ListSeparator)