Merge pull request #49922 from vseanreesermsft/internal-merge-2.1-202…

…3-08-08-1011

Merging internal commits for release/2.1

eng/PatchConfig.props

@@ -134,4 +134,11 @@ Later on, this will be checked using this condition:
       Microsoft.AspNetCore.Identity;
     </PackagesInPatch>
   </PropertyGroup>
+  <PropertyGroup Condition=" '$(VersionPrefix)' == '2.1.40' ">
+    <PackagesInPatch>
+      Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv;
+      Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets;
+      Microsoft.AspNetCore.SignalR.Redis;
+    </PackagesInPatch>
+  </PropertyGroup>
 </Project>

src/Middleware/WebSockets/test/UnitTests/Microsoft.AspNetCore.WebSockets.Tests.csproj

@@ -6,7 +6,9 @@
 
   <ItemGroup>
     <PackageReference Include="Microsoft.AspNetCore.Server.IntegrationTesting" Version="$(MicrosoftAspNetCoreServerIntegrationTestingPackageVersion)" />
-    <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel" Version="$(MicrosoftAspNetCoreServerKestrelPackageVersion)" />
+    <Reference Include="Microsoft.AspNetCore.Server.Kestrel" />
+    <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions" />
+    <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets" />
     <Reference Include="Microsoft.AspNetCore.WebSockets" />
     <Reference Include="Microsoft.Extensions.Logging.Testing" />
   </ItemGroup>

src/Servers/Kestrel/Kestrel/src/Microsoft.AspNetCore.Server.Kestrel.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <Description>ASP.NET Core Kestrel cross-platform web server.</Description>
@@ -11,9 +11,12 @@
   <ItemGroup>
     <Reference Include="Microsoft.AspNetCore.Hosting" />
     <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Https" />
-    <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets" />
     <PackageReference Include="Microsoft.AspNetCore.Server.Kestrel.Core" Version="2.1.25" />
     <PackageReference Include="System.IO.Pipelines" Version="$(SystemIOPipelinesPackageVersion)" />
   </ItemGroup>
 
+  <ItemGroup>
+    <ProjectReference Include="..\..\Transport.Sockets\src\Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.csproj" />
+  </ItemGroup>
+
 </Project>

src/Servers/Kestrel/Kestrel/test/Microsoft.AspNetCore.Server.Kestrel.Tests.csproj

@@ -1,4 +1,4 @@
-<Project Sdk="Microsoft.NET.Sdk">
+<Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
     <TargetFrameworks>netcoreapp2.1;net461</TargetFrameworks>
@@ -10,8 +10,12 @@
   </ItemGroup>
 
   <ItemGroup>
-    <Reference Include="Microsoft.AspNetCore.Server.Kestrel" />
-    <Reference Include="Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv" />
+    <ProjectReference Include="..\..\Core\src\Microsoft.AspNetCore.Server.Kestrel.Core.csproj" />
+    <ProjectReference Include="..\..\Https\src\Microsoft.AspNetCore.Server.Kestrel.Https.csproj" />
+    <ProjectReference Include="..\..\Transport.Abstractions\src\Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.csproj" />
+    <ProjectReference Include="..\..\Transport.Libuv\src\Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.csproj" />
+    <ProjectReference Include="..\..\Transport.Sockets\src\Microsoft.AspNetCore.Server.Kestrel.Transport.Sockets.csproj" />
+    <ProjectReference Include="..\src\Microsoft.AspNetCore.Server.Kestrel.csproj" />
   </ItemGroup>
 
 </Project>

src/Servers/Kestrel/Transport.Libuv/src/Internal/ILibuvTrace.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -14,6 +14,8 @@ public interface ILibuvTrace : ILogger
 
         void ConnectionWriteFin(string connectionId);
 
+        void ConnectionWriteRst(string connectionId);
+
         void ConnectionWroteFin(string connectionId, int status);
 
         void ConnectionWrite(string connectionId, int count);

src/Servers/Kestrel/Transport.Libuv/src/Internal/LibuvConnection.cs

@@ -6,6 +6,7 @@
 using System.IO;
 using System.IO.Pipelines;
 using System.Net;
+using System.Net.Sockets;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Connections;
@@ -28,13 +29,21 @@ public partial class LibuvConnection : TransportConnection
         private readonly UvStreamHandle _socket;
         private readonly CancellationTokenSource _connectionClosedTokenSource = new CancellationTokenSource();
 
+        private readonly bool _finOnError;
+
         private volatile ConnectionAbortedException _abortReason;
 
         private MemoryHandle _bufferHandle;
 
         public LibuvConnection(UvStreamHandle socket, ILibuvTrace log, LibuvThread thread, IPEndPoint remoteEndPoint, IPEndPoint localEndPoint)
+            : this(socket, log, thread, remoteEndPoint, localEndPoint, finOnError: false)
+        {
+        }
+
+        internal LibuvConnection(UvStreamHandle socket, ILibuvTrace log, LibuvThread thread, IPEndPoint remoteEndPoint, IPEndPoint localEndPoint, bool finOnError)
         {
             _socket = socket;
+            _finOnError = finOnError; ;
 
             RemoteAddress = remoteEndPoint?.Address;
             RemotePort = remoteEndPoint?.Port ?? 0;
@@ -94,6 +103,13 @@ public async Task Start()
                 }
                 finally
                 {
+                    if (!_finOnError && _abortReason != null)
+                    {
+                        // When shutdown isn't clean (note that we're using _abortReason, rather than inputError, to exclude that case),
+                        // we set the DontLinger socket option to cause libuv to send a RST and release any buffered response data.
+                        SetDontLingerOption(_socket);
+                    }
+
                     // Now, complete the input so that no more reads can happen
                     Input.Complete(inputError ?? _abortReason ?? new ConnectionAbortedException());
                     Output.Complete(outputError);
@@ -102,8 +118,16 @@ public async Task Start()
                     // on the stream handle
                     Input.CancelPendingFlush();
 
-                    // Send a FIN
-                    Log.ConnectionWriteFin(ConnectionId);
+                    if (!_finOnError && _abortReason != null)
+                    {
+                        // Send a RST
+                        Log.ConnectionWriteRst(ConnectionId);
+                    }
+                    else
+                    {
+                        // Send a FIN
+                        Log.ConnectionWriteFin(ConnectionId);
+                    }
 
                     // We're done with the socket now
                     _socket.Dispose();
@@ -116,13 +140,39 @@ public async Task Start()
             }
         }
 
+        /// <remarks>
+        /// This should be called on <see cref="_socket"/> before it is disposed.
+        /// Both <see cref="Abort"/> and <see cref="Start"/> call dispose but, rather than predict
+        /// which will do so first (which varies), we make this method idempotent and call it in both.
+        /// </remarks>
+        private static void SetDontLingerOption(UvStreamHandle socket)
+        {
+            if (!socket.IsClosed && !socket.IsInvalid) {
+                var libuv = socket.Libuv;
+                if (libuv.IsWindows) // p/invoke of setsockopt is Windows-specific
+                {
+                    var pSocket = IntPtr.Zero;
+                    libuv.uv_fileno(socket, ref pSocket);
+                    libuv.setsockopt(pSocket, SocketOptionLevel.Socket, SocketOptionName.DontLinger, 0);
+                }
+            }
+        }
+
         public override void Abort(ConnectionAbortedException abortReason)
         {
             _abortReason = abortReason;
             Output.CancelPendingRead();
-
-            // This cancels any pending I/O.
-            Thread.Post(s => s.Dispose(), _socket);
+
+            Thread.Post(/*static*/ (self) =>
+            {
+                if (!self._finOnError)
+                {
+                    SetDontLingerOption(self._socket);
+                }
+
+                // This cancels any pending I/O.
+                self._socket.Dispose();
+            }, this);
         }
 
         // Called on Libuv thread

src/Servers/Kestrel/Transport.Libuv/src/Internal/LibuvTrace.cs

@@ -22,6 +22,9 @@ public class LibuvTrace : ILibuvTrace
         private static readonly Action<ILogger, string, Exception> _connectionWriteFin =
             LoggerMessage.Define<string>(LogLevel.Debug, new EventId(7, nameof(ConnectionWriteFin)), @"Connection id ""{ConnectionId}"" sending FIN.");
 
+        private static readonly Action<ILogger, string, Exception> _connectionWriteRst =
+            LoggerMessage.Define<string>(LogLevel.Debug, new EventId(8, nameof(ConnectionWriteRst)), @"Connection id ""{ConnectionId}"" sending RST.");
+
         private static readonly Action<ILogger, string, int, Exception> _connectionWroteFin =
             LoggerMessage.Define<string, int>(LogLevel.Debug, new EventId(8, nameof(ConnectionWroteFin)), @"Connection id ""{ConnectionId}"" sent FIN with status ""{Status}"".");
 
@@ -58,6 +61,10 @@ public void ConnectionWriteFin(string connectionId)
             _connectionWriteFin(_logger, connectionId, null);
         }
 
+        public void ConnectionWriteRst(string connectionId) {
+            _connectionWriteRst(_logger, connectionId, null);
+        }
+
         public void ConnectionWroteFin(string connectionId, int status)
         {
             _connectionWroteFin(_logger, connectionId, status, null);

src/Servers/Kestrel/Transport.Libuv/src/Internal/LibuvTransport.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -10,6 +10,7 @@
 using Microsoft.AspNetCore.Server.Kestrel.Transport.Abstractions.Internal;
 using Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Internal.Networking;
 using Microsoft.Extensions.Logging;
+using System.Diagnostics;
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv.Internal
 {
@@ -72,7 +73,22 @@ public async Task BindAsync()
         {
             // TODO: Move thread management to LibuvTransportFactory
             // TODO: Split endpoint management from thread management
-            for (var index = 0; index < TransportOptions.ThreadCount; index++)
+
+            // When `FinOnError` is false (the default), we need to be able to forcibly abort connections.
+            // On Windows, libuv 1.10.0 will call `shutdown`, preventing forcible abort, on any socket
+            // not flagged as `UV_HANDLE_SHARED_TCP_SOCKET`.  The only way we've found to cause socket
+            // to be flagged as `UV_HANDLE_SHARED_TCP_SOCKET` is to share it across a named pipe (which
+            // must, itself, be flagged `ipc`), which naturally happens when a `ListenerPrimary` dispatches
+            // a connection to a `ListenerSecondary`.  Therefore, in scenarios where this is required, we
+            // tell the `ListenerPrimary` to dispatch *all* connections to secondary and create an
+            // additional `ListenerSecondary` to replace the lost capacity.
+            var dispatchAllToSecondary = Libuv.IsWindows && !TransportContext.Options.FinOnError;
+
+            var threadCount = dispatchAllToSecondary
+                ? TransportOptions.ThreadCount + 1
+                : TransportOptions.ThreadCount;
+
+            for (var index = 0; index < threadCount; index++)
             {
                 Threads.Add(new LibuvThread(this));
             }
@@ -84,8 +100,10 @@ public async Task BindAsync()
 
             try
             {
-                if (TransportOptions.ThreadCount == 1)
+                if (threadCount == 1)
                 {
+                    Debug.Assert(!dispatchAllToSecondary, "Should have taken the primary/secondary code path");
+
                     var listener = new Listener(TransportContext);
                     _listeners.Add(listener);
                     await listener.StartAsync(_endPointInformation, Threads[0]).ConfigureAwait(false);
@@ -95,7 +113,7 @@ public async Task BindAsync()
                     var pipeName = (Libuv.IsWindows ? @"\\.\pipe\kestrel_" : "/tmp/kestrel_") + Guid.NewGuid().ToString("n");
                     var pipeMessage = Guid.NewGuid().ToByteArray();
 
-                    var listenerPrimary = new ListenerPrimary(TransportContext);
+                    var listenerPrimary = new ListenerPrimary(TransportContext, dispatchAllToSecondary);
                     _listeners.Add(listenerPrimary);
                     await listenerPrimary.StartAsync(pipeName, pipeMessage, _endPointInformation, Threads[0]).ConfigureAwait(false);
 

src/Servers/Kestrel/Transport.Libuv/src/Internal/ListenerContext.cs

@@ -63,7 +63,8 @@ protected void HandleConnectionAsync(UvStreamHandle socket)
                     }
                 }
 
-                var connection = new LibuvConnection(socket, TransportContext.Log, Thread, remoteEndPoint, localEndPoint);
+                var finOnError = TransportContext.Options.FinOnError;
+                var connection = new LibuvConnection(socket, TransportContext.Log, Thread, remoteEndPoint, localEndPoint, finOnError);
                 TransportContext.ConnectionDispatcher.OnConnection(connection);
                 _ = connection.Start();
             }

src/Servers/Kestrel/Transport.Libuv/src/Internal/ListenerPrimary.cs

@@ -3,6 +3,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.IO;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
@@ -22,6 +23,10 @@ public class ListenerPrimary : Listener
         private readonly List<UvPipeHandle> _dispatchPipes = new List<UvPipeHandle>();
         // The list of pipes we've created but may not be part of _dispatchPipes
         private readonly List<UvPipeHandle> _createdPipes = new List<UvPipeHandle>();
+
+        // If true, dispatch all connections to _dispatchPipes - don't process any in the primary
+        private readonly bool _dispatchAll;
+
         private int _dispatchIndex;
         private string _pipeName;
         private byte[] _pipeMessage;
@@ -32,8 +37,9 @@ public class ListenerPrimary : Listener
         // but it has no other functional significance
         private readonly ArraySegment<ArraySegment<byte>> _dummyMessage = new ArraySegment<ArraySegment<byte>>(new[] { new ArraySegment<byte>(new byte[] { 1, 2, 3, 4 }) });
 
-        public ListenerPrimary(LibuvTransportContext transportContext) : base(transportContext)
+        public ListenerPrimary(LibuvTransportContext transportContext, bool dispatchAll) : base(transportContext)
         {
+            _dispatchAll = dispatchAll;
         }
 
         /// <summary>
@@ -105,9 +111,22 @@ private void OnListenPipe(UvStreamHandle pipe, int status, UvException error)
 
         protected override void DispatchConnection(UvStreamHandle socket)
         {
-            var index = _dispatchIndex++ % (_dispatchPipes.Count + 1);
+            var modulus = _dispatchAll ? _dispatchPipes.Count : (_dispatchPipes.Count + 1);
+            if (modulus == 0) {
+                if (_createdPipes.Count == 0) {
+                    Log.LogError(0, $"Connection received before listeners were initialized - see https://aka.ms/dotnet/aspnet/finonerror for possible mitigations");
+                }
+                else {
+                    Log.LogError(0, "Unable to process connection since listeners failed to initialize - see https://aka.ms/dotnet/aspnet/finonerror for possible mitigations");
+                }
+
+                return;
+            }
+
+            var index = _dispatchIndex++ % modulus;
             if (index == _dispatchPipes.Count)
             {
+                Debug.Assert(!_dispatchAll, "Should have dispatched to a secondary listener");
                 base.DispatchConnection(socket);
             }
             else

src/Servers/Kestrel/Transport.Libuv/src/Internal/Networking/LibuvFunctions.cs

@@ -2,6 +2,7 @@
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
+using System.Net.Sockets;
 using System.Runtime.CompilerServices;
 using System.Runtime.InteropServices;
 
@@ -57,6 +58,7 @@ public LibuvFunctions()
             _uv_timer_start = NativeMethods.uv_timer_start;
             _uv_timer_stop = NativeMethods.uv_timer_stop;
             _uv_now = NativeMethods.uv_now;
+            _setsockopt = NativeMethods.setsockopt;
         }
 
         // Second ctor that doesn't set any fields only to be used by MockLibuv
@@ -421,6 +423,14 @@ public void tcp_getpeername(UvTcpHandle handle, out SockAddr addr, ref int namel
             ThrowIfErrored(_uv_tcp_getpeername(handle, out addr, ref namelen));
         }
 
+        protected delegate int setsockopt_func(IntPtr s, SocketOptionLevel level, SocketOptionName optname, ref int optval, int optlen);
+        protected setsockopt_func _setsockopt;
+        public void setsockopt(IntPtr s, SocketOptionLevel level, SocketOptionName optname, int optval)
+        {
+            var status = _setsockopt(s, level, optname, ref optval, optlen: sizeof(int));
+            ThrowIfErrored(status);
+        }
+
         public uv_buf_t buf_init(IntPtr memory, int len)
         {
             return new uv_buf_t(memory, len, IsWindows);
@@ -629,6 +639,9 @@ IntPtr lpCompletionRoutine
 
             [DllImport("WS2_32.dll", CallingConvention = CallingConvention.Winapi)]
             public static extern int WSAGetLastError();
+
+            [DllImport("WS2_32.dll", CallingConvention = CallingConvention.Winapi)]
+            public static extern int setsockopt(IntPtr s, SocketOptionLevel level, SocketOptionName optname, ref int optval, int optlen);
         }
     }
-}
+}

src/Servers/Kestrel/Transport.Libuv/src/LibuvTransportOptions.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -10,6 +10,17 @@ namespace Microsoft.AspNetCore.Server.Kestrel.Transport.Libuv
     /// </summary>
     public class LibuvTransportOptions
     {
+        private const string FinOnErrorSwitch = "Microsoft.AspNetCore.Server.Kestrel.FinOnError";
+        private static readonly bool _finOnError;
+
+        static LibuvTransportOptions()
+        {
+            AppContext.TryGetSwitch(FinOnErrorSwitch, out _finOnError);
+        }
+
+        // Opt-out flag for back compat
+        internal bool FinOnError { get; set; } = _finOnError;
+
         /// <summary>
         /// The number of libuv I/O threads used to process requests.
         /// </summary>