Reduce HTTP/2 allocations

- Remove per request allocations on the thread pool by implementing IThreadPoolWorkItem on Http2Stream
- Made generic version of Http2Stream to store the IHttpApplication instead of using a tuple
- Removed passing of IHttpApplication<TContext> everywhere

Author: davidfowl

src/Servers/Kestrel/Core/src/Internal/Http2/Http2Connection.cs

@@ -415,7 +415,7 @@ private Task ProcessFrameAsync<TContext>(IHttpApplication<TContext> application,
                 case Http2FrameType.WINDOW_UPDATE:
                     return ProcessWindowUpdateFrameAsync();
                 case Http2FrameType.CONTINUATION:
-                    return ProcessContinuationFrameAsync(application, payload);
+                    return ProcessContinuationFrameAsync(payload);
                 default:
                     return ProcessUnknownFrameAsync();
             }
@@ -558,7 +558,7 @@ private Task ProcessHeadersFrameAsync<TContext>(IHttpApplication<TContext> appli
                 }
 
                 // Start a new stream
-                _currentHeadersStream = new Http2Stream(new Http2StreamContext
+                _currentHeadersStream = new Http2Stream<TContext>(application, new Http2StreamContext
                 {
                     ConnectionId = ConnectionId,
                     StreamId = _incomingFrame.StreamId,
@@ -580,7 +580,7 @@ private Task ProcessHeadersFrameAsync<TContext>(IHttpApplication<TContext> appli
                 _headerFlags = _incomingFrame.HeadersFlags;
 
                 var headersPayload = payload.Slice(0, _incomingFrame.HeadersPayloadLength); // Minus padding
-                return DecodeHeadersAsync(application, _incomingFrame.HeadersEndHeaders, headersPayload);
+                return DecodeHeadersAsync(_incomingFrame.HeadersEndHeaders, headersPayload);
             }
         }
 
@@ -822,7 +822,7 @@ private Task ProcessWindowUpdateFrameAsync()
             return Task.CompletedTask;
         }
 
-        private Task ProcessContinuationFrameAsync<TContext>(IHttpApplication<TContext> application, ReadOnlySequence<byte> payload)
+        private Task ProcessContinuationFrameAsync(ReadOnlySequence<byte> payload)
         {
             if (_currentHeadersStream == null)
             {
@@ -847,7 +847,7 @@ private Task ProcessContinuationFrameAsync<TContext>(IHttpApplication<TContext>
                     TimeoutControl.CancelTimeout();
                 }
 
-                return DecodeHeadersAsync(application, _incomingFrame.ContinuationEndHeaders, payload);
+                return DecodeHeadersAsync(_incomingFrame.ContinuationEndHeaders, payload);
             }
         }
 
@@ -861,7 +861,7 @@ private Task ProcessUnknownFrameAsync()
             return Task.CompletedTask;
         }
 
-        private Task DecodeHeadersAsync<TContext>(IHttpApplication<TContext> application, bool endHeaders, ReadOnlySequence<byte> payload)
+        private Task DecodeHeadersAsync(bool endHeaders, ReadOnlySequence<byte> payload)
         {
             try
             {
@@ -870,7 +870,7 @@ private Task DecodeHeadersAsync<TContext>(IHttpApplication<TContext> application
 
                 if (endHeaders)
                 {
-                    StartStream(application);
+                    StartStream();
                     ResetRequestHeaderParsingState();
                 }
             }
@@ -896,7 +896,7 @@ private Task DecodeTrailersAsync(bool endHeaders, ReadOnlySequence<byte> payload
             return Task.CompletedTask;
         }
 
-        private void StartStream<TContext>(IHttpApplication<TContext> application)
+        private void StartStream()
         {
             if (!_isMethodConnect && (_parsedPseudoHeaderFields & _mandatoryRequestPseudoHeaderFields) != _mandatoryRequestPseudoHeaderFields)
             {
@@ -923,12 +923,7 @@ private void StartStream<TContext>(IHttpApplication<TContext> application)
             _activeStreamCount++;
             _streams[_incomingFrame.StreamId] = _currentHeadersStream;
             // Must not allow app code to block the connection handling loop.
-            ThreadPool.UnsafeQueueUserWorkItem(state =>
-            {
-                var (app, currentStream) = (Tuple<IHttpApplication<TContext>, Http2Stream>)state;
-                _ = currentStream.ProcessRequestsAsync(app);
-            },
-            new Tuple<IHttpApplication<TContext>, Http2Stream>(application, _currentHeadersStream));
+            ThreadPool.UnsafeQueueUserWorkItem(_currentHeadersStream, preferLocal: false);
         }
 
         private void ResetRequestHeaderParsingState()

src/Servers/Kestrel/Core/src/Internal/Http2/Http2Stream.cs

@@ -6,6 +6,7 @@
 using System.Diagnostics;
 using System.IO;
 using System.IO.Pipelines;
+using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Connections;
 using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http;
@@ -17,7 +18,7 @@
 
 namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
 {
-    public partial class Http2Stream : HttpProtocol
+    public abstract partial class Http2Stream : HttpProtocol, IThreadPoolWorkItem
     {
         private readonly Http2StreamContext _context;
         private readonly Http2OutputProducer _http2Output;
@@ -499,6 +500,11 @@ private Pipe CreateRequestBodyPipe(uint windowSize)
             }
         }
 
+        /// <summary>
+        /// Used to kick off the request processing loop by derived classes.
+        /// </summary>
+        public abstract void Execute();
+
         [Flags]
         private enum StreamCompletionFlags
         {

src/Servers/Kestrel/Core/src/Internal/Http2/Http2StreamOfT.cs

@@ -0,0 +1,23 @@
+using System;
+using System.Collections.Generic;
+using System.Text;
+using Microsoft.AspNetCore.Hosting.Server;
+
+namespace Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http2
+{
+    public class Http2Stream<TContext> : Http2Stream
+    {
+        private readonly IHttpApplication<TContext> _application;
+
+        public Http2Stream(IHttpApplication<TContext> application, Http2StreamContext context) : base(context)
+        {
+            _application = application;
+        }
+
+        public override void Execute()
+        {
+            // REVIEW: Should we store this in a field for easy debugging?
+            _ = ProcessRequestsAsync(_application);
+        }
+    }
+}

src/Servers/Kestrel/Core/test/HttpProtocolFeatureCollectionTests.cs

@@ -1,4 +1,4 @@
-// Copyright (c) .NET Foundation. All rights reserved.
+// Copyright (c) .NET Foundation. All rights reserved.
 // Licensed under the Apache License, Version 2.0. See License.txt in the project root for license information.
 
 using System;
@@ -40,7 +40,7 @@ public HttpProtocolFeatureCollectionTests()
             _http1Connection.Reset();
             _collection = _http1Connection;
 
-            var http2Stream = new Http2Stream(context);
+            var http2Stream = new TestHttp2Stream(context);
             http2Stream.Reset();
             _http2Collection = http2Stream;
         }
@@ -220,5 +220,16 @@ private int SetFeaturesToNonDefault()
         }
 
         private Http1Connection CreateHttp1Connection() => new TestHttp1Connection(_httpConnectionContext);
+
+        private class TestHttp2Stream : Http2Stream
+        {
+            public TestHttp2Stream(Http2StreamContext context) : base(context)
+            {
+            }
+
+            public override void Execute()
+            {
+            }
+        }
     }
 }