diff --git a/src/Servers/Kestrel/Core/src/Internal/Http3/Http3Connection.cs b/src/Servers/Kestrel/Core/src/Internal/Http3/Http3Connection.cs index fb22e0558c5d..c39c66312c2b 100644 --- a/src/Servers/Kestrel/Core/src/Internal/Http3/Http3Connection.cs +++ b/src/Servers/Kestrel/Core/src/Internal/Http3/Http3Connection.cs @@ -133,6 +133,11 @@ public void StopProcessingNextRequest(bool serverInitiated) if (Interlocked.CompareExchange(ref _gracefulCloseInitiator, initiator, GracefulCloseInitiator.None) == GracefulCloseInitiator.None) { + // https://quicwg.org/base-drafts/draft-ietf-quic-http.html#section-5.2-11 + // An endpoint that completes a graceful shutdown SHOULD use the H3_NO_ERROR error code + // when closing the connection. + _errorCodeFeature.Error = (long)Http3ErrorCode.NoError; + // Abort accept async loop to initiate graceful shutdown // TODO aborting connection isn't graceful due to runtime issue, will drop data on streams // Either we need to swap to using a cts here or fix runtime to gracefully close connection. @@ -249,10 +254,15 @@ public void OnTimeout(TimeoutReason reason) case TimeoutReason.TimeoutFeature: SendGoAway(GetHighestStreamId()).Preserve(); break; - case TimeoutReason.RequestHeaders: // Request header timeout is handled in starting stream queue - case TimeoutReason.KeepAlive: // Keep-alive is handled by msquic case TimeoutReason.ReadDataRate: + Abort(new ConnectionAbortedException(CoreStrings.BadRequest_RequestBodyTimeout), Http3ErrorCode.InternalError); + break; case TimeoutReason.WriteDataRate: + Log.ResponseMinimumDataRateNotSatisfied(_context.ConnectionId, traceIdentifier: null); + Abort(new ConnectionAbortedException(CoreStrings.ConnectionTimedBecauseResponseMininumDataRateNotSatisfied), Http3ErrorCode.InternalError); + break; + case TimeoutReason.RequestHeaders: // Request header timeout is handled in starting stream queue + case TimeoutReason.KeepAlive: // Keep-alive is handled by msquic case TimeoutReason.RequestBodyDrain: default: Debug.Assert(false, "Invalid TimeoutReason"); @@ -392,6 +402,9 @@ internal async Task InnerProcessStreamsAsync(IHttpApplication } [Fact] - public async Task GracefulServerShutdownSendsGoawayClosesConnection() + public async Task GracefulServerShutdownClosesConnection() { await InitializeConnectionAsync(_echoApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + // Trigger server shutdown. - MultiplexedConnectionContext.ConnectionClosingCts.Cancel(); + CloseConnectionGracefully(); + Assert.Null(await MultiplexedConnectionContext.AcceptAsync().DefaultTimeout()); + + await WaitForConnectionStopAsync(0, false, expectedErrorCode: Http3ErrorCode.NoError); } [Theory] diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TestBase.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TestBase.cs index e9986ea4e3a0..662ec4a1007f 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TestBase.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TestBase.cs @@ -6,12 +6,14 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.IO.Pipelines; using System.Linq; using System.Net.Http; using System.Net.Http.QPack; using System.Reflection; +using System.Text; using System.Threading; using System.Threading.Channels; using System.Threading.Tasks; @@ -38,6 +40,8 @@ public abstract class Http3TestBase : TestApplicationErrorLoggerLoggedTest, IDis { protected static readonly int MaxRequestHeaderFieldSize = 16 * 1024; protected static readonly string _4kHeaderValue = new string('a', 4096); + protected static readonly byte[] _helloWorldBytes = Encoding.ASCII.GetBytes("hello, world"); + protected static readonly byte[] _maxData = Encoding.ASCII.GetBytes(new string('a', 16 * 1024)); internal TestServiceContext _serviceContext; internal readonly TimeoutControl _timeoutControl; @@ -49,9 +53,9 @@ public abstract class Http3TestBase : TestApplicationErrorLoggerLoggedTest, IDis protected readonly TaskCompletionSource _closedStateReached = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); internal readonly ConcurrentDictionary _runningStreams = new ConcurrentDictionary(); - protected readonly RequestDelegate _noopApplication; protected readonly RequestDelegate _echoApplication; + protected readonly RequestDelegate _readRateApplication; protected readonly RequestDelegate _echoMethod; protected readonly RequestDelegate _echoPath; protected readonly RequestDelegate _echoHost; @@ -59,6 +63,27 @@ public abstract class Http3TestBase : TestApplicationErrorLoggerLoggedTest, IDis private Http3ControlStream _inboundControlStream; private long _currentStreamId; + protected static readonly IEnumerable> _browserRequestHeaders = new[] + { + new KeyValuePair(HeaderNames.Method, "GET"), + new KeyValuePair(HeaderNames.Path, "/"), + new KeyValuePair(HeaderNames.Scheme, "http"), + new KeyValuePair(HeaderNames.Authority, "localhost:80"), + new KeyValuePair("user-agent", "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:54.0) Gecko/20100101 Firefox/54.0"), + new KeyValuePair("accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"), + new KeyValuePair("accept-language", "en-US,en;q=0.5"), + new KeyValuePair("accept-encoding", "gzip, deflate, br"), + new KeyValuePair("upgrade-insecure-requests", "1"), + }; + + protected static IEnumerable> ReadRateRequestHeaders(int expectedBytes) => new[] + { + new KeyValuePair(HeaderNames.Method, "POST"), + new KeyValuePair(HeaderNames.Path, "/" + expectedBytes), + new KeyValuePair(HeaderNames.Scheme, "http"), + new KeyValuePair(HeaderNames.Authority, "localhost:80"), + }; + public Http3TestBase() { _timeoutControl = new TimeoutControl(_mockTimeoutHandler.Object); @@ -83,6 +108,26 @@ public Http3TestBase() } }; + _readRateApplication = async context => + { + var expectedBytes = int.Parse(context.Request.Path.Value.Substring(1), CultureInfo.InvariantCulture); + + var buffer = new byte[16 * 1024]; + var received = 0; + + while (received < expectedBytes) + { + received += await context.Request.Body.ReadAsync(buffer, 0, buffer.Length); + } + + var stalledReadTask = context.Request.Body.ReadAsync(buffer, 0, buffer.Length); + + // Write to the response so the test knows the app started the stalled read. + await context.Response.Body.WriteAsync(new byte[1], 0, 1); + + await stalledReadTask; + }; + _echoMethod = context => { context.Response.Headers["Method"] = context.Request.Method; @@ -151,7 +196,17 @@ internal async ValueTask GetInboundControlStream() return _inboundControlStream; } - internal async Task WaitForConnectionErrorAsync(bool ignoreNonGoAwayFrames, long expectedLastStreamId, Http3ErrorCode expectedErrorCode, params string[] expectedErrorMessage) + internal void CloseConnectionGracefully() + { + MultiplexedConnectionContext.ConnectionClosingCts.Cancel(); + } + + internal Task WaitForConnectionStopAsync(long expectedLastStreamId, bool ignoreNonGoAwayFrames, Http3ErrorCode? expectedErrorCode = null) + { + return WaitForConnectionErrorAsync(ignoreNonGoAwayFrames, expectedLastStreamId, expectedErrorCode: expectedErrorCode ?? 0, expectedErrorMessage: null); + } + + internal async Task WaitForConnectionErrorAsync(bool ignoreNonGoAwayFrames, long? expectedLastStreamId, Http3ErrorCode expectedErrorCode, params string[] expectedErrorMessage) where TException : Exception { var frame = await _inboundControlStream.ReceiveFrameAsync(); @@ -164,7 +219,10 @@ internal async Task WaitForConnectionErrorAsync(bool ignoreNonGoAway } } - VerifyGoAway(frame, expectedLastStreamId); + if (expectedLastStreamId != null) + { + VerifyGoAway(frame, expectedLastStreamId.GetValueOrDefault()); + } Assert.Equal((Http3ErrorCode)expectedErrorCode, (Http3ErrorCode)MultiplexedConnectionContext.Error); @@ -216,8 +274,6 @@ protected async Task InitializeConnectionAsync(RequestDelegate application) _connectionTask = Connection.ProcessStreamsAsync(new DummyApplication(application)); await GetInboundControlStream(); - - await Task.CompletedTask; } internal async ValueTask InitializeConnectionAndStreamsAsync(RequestDelegate application) @@ -394,7 +450,7 @@ public class Http3StreamBase : IProtocolErrorCodeFeature internal DuplexPipe.DuplexPipePair _pair; internal Http3TestBase _testBase; internal Http3Connection _connection; - private long _bytesReceived; + public long BytesReceived { get; private set; } public long Error { get; set; } public Task OnStreamCreatedTask => _onStreamCreatedTcs.Task; @@ -412,6 +468,12 @@ protected static async Task FlushAsync(PipeWriter writableBuffer) await writableBuffer.FlushAsync().AsTask().DefaultTimeout(); } + internal async Task ReceiveEndAsync() + { + var result = await _pair.Application.Input.ReadAsync().AsTask().DefaultTimeout(); + Assert.True(result.IsCompleted); + } + internal async Task ReceiveFrameAsync() { var frame = new Http3FrameWithPayload(); @@ -446,7 +508,7 @@ internal async Task ReceiveFrameAsync() } finally { - _bytesReceived += copyBuffer.Slice(copyBuffer.Start, consumed).Length; + BytesReceived += copyBuffer.Slice(copyBuffer.Start, consumed).Length; _pair.Application.Input.AdvanceTo(consumed, examined); } } @@ -655,6 +717,17 @@ void WriteSpan(PipeWriter pw) await FlushAsync(writableBuffer); } + internal async Task SendGoAwayAsync(long streamId, bool endStream = false) + { + var frame = new Http3RawFrame(); + frame.PrepareGoAway(); + + var data = new byte[VariableLengthIntegerHelper.GetByteCount(streamId)]; + VariableLengthIntegerHelper.WriteInteger(data, streamId); + + await SendFrameAsync(frame, data, endStream); + } + internal async Task SendSettingsAsync(List settings, bool endStream = false) { var frame = new Http3RawFrame(); @@ -738,6 +811,7 @@ public class TestMultiplexedConnectionContext : MultiplexedConnectionContext, IC }); private readonly Http3TestBase _testBase; + private long _error; public TestMultiplexedConnectionContext(Http3TestBase testBase) { @@ -759,7 +833,11 @@ public TestMultiplexedConnectionContext(Http3TestBase testBase) public CancellationTokenSource ConnectionClosingCts { get; set; } = new CancellationTokenSource(); - public long Error { get; set; } + public long Error + { + get => _error; + set => _error = value; + } public override void Abort() { diff --git a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TimeoutTests.cs b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TimeoutTests.cs index f52320ca7732..7eb5a8ac985a 100644 --- a/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TimeoutTests.cs +++ b/src/Servers/Kestrel/test/InMemory.FunctionalTests/Http3/Http3TimeoutTests.cs @@ -6,7 +6,11 @@ using System.Linq; using System.Net.Http; using System.Threading.Tasks; +using Microsoft.AspNetCore.Connections; +using Microsoft.AspNetCore.Http; +using Microsoft.AspNetCore.Server.Kestrel.Core.Features; using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Http3; +using Microsoft.AspNetCore.Server.Kestrel.Core.Internal.Infrastructure; using Microsoft.AspNetCore.Testing; using Microsoft.Net.Http.Headers; using Moq; @@ -181,5 +185,404 @@ public async Task ControlStream_RequestHeadersTimeoutMaxValue_ExpirationIsMaxVal Assert.Equal(TimeSpan.MaxValue.Ticks, serverInboundControlStream.HeaderTimeoutTicks); } + + [Fact] + public async Task DATA_Received_TooSlowlyOnSmallRead_AbortsConnectionAfterGracePeriod() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinRequestBodyDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + var requestStream = await InitializeConnectionAndStreamsAsync(_readRateApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + // _helloWorldBytes is 12 bytes, and 12 bytes / 240 bytes/sec = .05 secs which is far below the grace period. + await requestStream.SendHeadersAsync(ReadRateRequestHeaders(_helloWorldBytes.Length), endStream: false); + await requestStream.SendDataAsync(_helloWorldBytes, endStream: false); + + await requestStream.ExpectHeadersAsync(); + + await requestStream.ExpectDataAsync(); + + // Don't send any more data and advance just to and then past the grace period. + AdvanceClock(limits.MinRequestBodyDataRate.GracePeriod); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromTicks(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.ReadDataRate), Times.Once); + + await WaitForConnectionErrorAsync( + ignoreNonGoAwayFrames: false, + expectedLastStreamId: 8, + Http3ErrorCode.InternalError, + null); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + /* + * Additional work around closing connections is required before response drain can be supported. + [Fact] + public async Task ResponseDrain_SlowerThanMinimumDataRate_AbortsConnection() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + await InitializeConnectionAsync(_noopApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + CloseConnectionGracefully(); + + await inboundControlStream.ReceiveFrameAsync().DefaultTimeout(); + await inboundControlStream.ReceiveFrameAsync().DefaultTimeout(); + await inboundControlStream.ReceiveEndAsync().DefaultTimeout(); + + //await WaitForConnectionStopAsync(expectedLastStreamId: VariableLengthIntegerHelper.EightByteLimit, ignoreNonGoAwayFrames: false, expectedErrorCode: Http3ErrorCode.NoError); + + AdvanceClock(TimeSpan.FromSeconds(inboundControlStream.BytesReceived / limits.MinResponseDataRate.BytesPerSecond) + + limits.MinResponseDataRate.GracePeriod + Heartbeat.Interval - TimeSpan.FromSeconds(.5)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.WriteDataRate), Times.Once); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + */ + + private class EchoAppWithNotification + { + private readonly TaskCompletionSource _writeStartedTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + + public Task WriteStartedTask => _writeStartedTcs.Task; + + public async Task RunApp(HttpContext context) + { + await context.Response.Body.FlushAsync(); + + var buffer = new byte[16 * 1024]; + int received; + + while ((received = await context.Request.Body.ReadAsync(buffer, 0, buffer.Length)) > 0) + { + var writeTask = context.Response.Body.WriteAsync(buffer, 0, received); + _writeStartedTcs.TrySetResult(); + + await writeTask; + } + } + } + + [Fact] + [QuarantinedTest("https://github.com/dotnet/aspnetcore/issues/21520")] + public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnSmallWrite_AbortsConnectionAfterGracePeriod() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinResponseDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + // Disable response buffering so "socket" backpressure is observed immediately. + limits.MaxResponseBufferSize = 0; + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + var app = new EchoAppWithNotification(); + var requestStream = await InitializeConnectionAndStreamsAsync(app.RunApp); + + await requestStream.SendHeadersAsync(_browserRequestHeaders, endStream: false); + await requestStream.SendDataAsync(_helloWorldBytes, endStream: true); + + await requestStream.ExpectHeadersAsync(); + + await app.WriteStartedTask.DefaultTimeout(); + + // Complete timing of the request body so we don't induce any unexpected request body rate timeouts. + _timeoutControl.Tick(mockSystemClock.UtcNow); + + // Don't read data frame to induce "socket" backpressure. + AdvanceClock(TimeSpan.FromSeconds((requestStream.BytesReceived + _helloWorldBytes.Length) / limits.MinResponseDataRate.BytesPerSecond) + + limits.MinResponseDataRate.GracePeriod + Heartbeat.Interval - TimeSpan.FromSeconds(.5)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.WriteDataRate), Times.Once); + + // The "hello, world" bytes are buffered from before the timeout, but not an END_STREAM data frame. + var data = await requestStream.ExpectDataAsync(); + Assert.Equal(_helloWorldBytes.Length, data.Length); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + [Fact] + public async Task DATA_Sent_TooSlowlyDueToSocketBackPressureOnLargeWrite_AbortsConnectionAfterRateTimeout() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinResponseDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + // Disable response buffering so "socket" backpressure is observed immediately. + limits.MaxResponseBufferSize = 0; + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + var app = new EchoAppWithNotification(); + var requestStream = await InitializeConnectionAndStreamsAsync(app.RunApp); + + await requestStream.SendHeadersAsync(_browserRequestHeaders, endStream: false); + await requestStream.SendDataAsync(_maxData, endStream: true); + + await requestStream.ExpectHeadersAsync(); + + await app.WriteStartedTask.DefaultTimeout(); + + // Complete timing of the request body so we don't induce any unexpected request body rate timeouts. + _timeoutControl.Tick(mockSystemClock.UtcNow); + + var timeToWriteMaxData = TimeSpan.FromSeconds((requestStream.BytesReceived + _maxData.Length) / limits.MinResponseDataRate.BytesPerSecond) + + limits.MinResponseDataRate.GracePeriod + Heartbeat.Interval - TimeSpan.FromSeconds(.5); + + // Don't read data frame to induce "socket" backpressure. + AdvanceClock(timeToWriteMaxData); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.WriteDataRate), Times.Once); + + // The _maxData bytes are buffered from before the timeout, but not an END_STREAM data frame. + await requestStream.ExpectDataAsync(); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + [Fact] + public async Task DATA_Received_TooSlowlyOnLargeRead_AbortsConnectionAfterRateTimeout() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinRequestBodyDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + var requestStream = await InitializeConnectionAndStreamsAsync(_readRateApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + // _maxData is 16 KiB, and 16 KiB / 240 bytes/sec ~= 68 secs which is far above the grace period. + await requestStream.SendHeadersAsync(ReadRateRequestHeaders(_maxData.Length), endStream: false); + await requestStream.SendDataAsync(_maxData, endStream: false); + + await requestStream.ExpectHeadersAsync(); + + await requestStream.ExpectDataAsync(); + + // Due to the imprecision of floating point math and the fact that TimeoutControl derives rate from elapsed + // time for reads instead of vice versa like for writes, use a half-second instead of single-tick cushion. + var timeToReadMaxData = TimeSpan.FromSeconds(_maxData.Length / limits.MinRequestBodyDataRate.BytesPerSecond) - TimeSpan.FromSeconds(.5); + + // Don't send any more data and advance just to and then past the rate timeout. + AdvanceClock(timeToReadMaxData); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.ReadDataRate), Times.Once); + + await WaitForConnectionErrorAsync( + ignoreNonGoAwayFrames: false, + expectedLastStreamId: null, + Http3ErrorCode.InternalError, + null); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + [Fact] + public async Task DATA_Received_TooSlowlyOnMultipleStreams_AbortsConnectionAfterAdditiveRateTimeout() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinRequestBodyDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + await InitializeConnectionAsync(_readRateApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + var requestStream1 = await CreateRequestStream(); + + // _maxData is 16 KiB, and 16 KiB / 240 bytes/sec ~= 68 secs which is far above the grace period. + await requestStream1.SendHeadersAsync(ReadRateRequestHeaders(_maxData.Length), endStream: false); + await requestStream1.SendDataAsync(_maxData, endStream: false); + + await requestStream1.ExpectHeadersAsync(); + await requestStream1.ExpectDataAsync(); + + var requestStream2 = await CreateRequestStream(); + + await requestStream2.SendHeadersAsync(ReadRateRequestHeaders(_maxData.Length), endStream: false); + await requestStream2.SendDataAsync(_maxData, endStream: false); + + await requestStream2.ExpectHeadersAsync(); + await requestStream2.ExpectDataAsync(); + + var timeToReadMaxData = TimeSpan.FromSeconds(_maxData.Length / limits.MinRequestBodyDataRate.BytesPerSecond); + // Double the timeout for the second stream. + timeToReadMaxData += timeToReadMaxData; + + // Due to the imprecision of floating point math and the fact that TimeoutControl derives rate from elapsed + // time for reads instead of vice versa like for writes, use a half-second instead of single-tick cushion. + timeToReadMaxData -= TimeSpan.FromSeconds(.5); + + // Don't send any more data and advance just to and then past the rate timeout. + AdvanceClock(timeToReadMaxData); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.ReadDataRate), Times.Once); + + await WaitForConnectionErrorAsync( + ignoreNonGoAwayFrames: false, + expectedLastStreamId: null, + Http3ErrorCode.InternalError, + null); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + [Fact] + public async Task DATA_Received_TooSlowlyOnSecondStream_AbortsConnectionAfterNonAdditiveRateTimeout() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinRequestBodyDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + await InitializeConnectionAsync(_readRateApplication); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + var requestStream1 = await CreateRequestStream(); + + // _maxData is 16 KiB, and 16 KiB / 240 bytes/sec ~= 68 secs which is far above the grace period. + await requestStream1.SendHeadersAsync(ReadRateRequestHeaders(_maxData.Length), endStream: false); + await requestStream1.SendDataAsync(_maxData, endStream: true); + + await requestStream1.ExpectHeadersAsync(); + await requestStream1.ExpectDataAsync(); + + await requestStream1.ExpectReceiveEndOfStream(); + + var requestStream2 = await CreateRequestStream(); + + await requestStream2.SendHeadersAsync(ReadRateRequestHeaders(_maxData.Length), endStream: false); + await requestStream2.SendDataAsync(_maxData, endStream: false); + + await requestStream2.ExpectHeadersAsync(); + await requestStream2.ExpectDataAsync(); + + // Due to the imprecision of floating point math and the fact that TimeoutControl derives rate from elapsed + // time for reads instead of vice versa like for writes, use a half-second instead of single-tick cushion. + var timeToReadMaxData = TimeSpan.FromSeconds(_maxData.Length / limits.MinRequestBodyDataRate.BytesPerSecond) - TimeSpan.FromSeconds(.5); + + // Don't send any more data and advance just to and then past the rate timeout. + AdvanceClock(timeToReadMaxData); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromSeconds(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(TimeoutReason.ReadDataRate), Times.Once); + + await WaitForConnectionErrorAsync( + ignoreNonGoAwayFrames: false, + expectedLastStreamId: null, + Http3ErrorCode.InternalError, + null); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + + [Fact] + public async Task DATA_Received_SlowlyWhenRateLimitDisabledPerRequest_DoesNotAbortConnection() + { + var mockSystemClock = _serviceContext.MockSystemClock; + var limits = _serviceContext.ServerOptions.Limits; + + // Use non-default value to ensure the min request and response rates aren't mixed up. + limits.MinRequestBodyDataRate = new MinDataRate(480, TimeSpan.FromSeconds(2.5)); + + _timeoutControl.Initialize(mockSystemClock.UtcNow.Ticks); + + var requestStream = await InitializeConnectionAndStreamsAsync(context => + { + // Completely disable rate limiting for this stream. + context.Features.Get().MinDataRate = null; + return _readRateApplication(context); + }); + + var inboundControlStream = await GetInboundControlStream(); + await inboundControlStream.ExpectSettingsAsync(); + + // _helloWorldBytes is 12 bytes, and 12 bytes / 240 bytes/sec = .05 secs which is far below the grace period. + await requestStream.SendHeadersAsync(ReadRateRequestHeaders(_helloWorldBytes.Length), endStream: false); + await requestStream.SendDataAsync(_helloWorldBytes, endStream: false); + + await requestStream.ExpectHeadersAsync(); + + await requestStream.ExpectDataAsync(); + + // Don't send any more data and advance just to and then past the grace period. + AdvanceClock(limits.MinRequestBodyDataRate.GracePeriod); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + AdvanceClock(TimeSpan.FromTicks(1)); + + _mockTimeoutHandler.Verify(h => h.OnTimeout(It.IsAny()), Times.Never); + + await requestStream.SendDataAsync(_helloWorldBytes, endStream: true); + + await requestStream.ExpectReceiveEndOfStream(); + + _mockTimeoutHandler.VerifyNoOtherCalls(); + } + } }