diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp index a1322eaabb32..fcd4205138c9 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp @@ -22,7 +22,8 @@ const PCWSTR HandlerResolver::s_pwzAspnetcoreOutOfProcessRequestHandlerName = L" HandlerResolver::HandlerResolver(HMODULE hModule, const IHttpServer &pServer) : m_hModule(hModule), m_pServer(pServer), - m_loadedApplicationHostingModel(HOSTING_UNKNOWN) + m_loadedApplicationHostingModel(HOSTING_UNKNOWN), + m_shutdownDelay() { m_disallowRotationOnConfigChange = false; InitializeSRWLock(&m_requestHandlerLoadLock); @@ -171,6 +172,7 @@ HandlerResolver::GetApplicationFactory(const IHttpApplication& pApplication, con m_loadedApplicationHostingModel = options.QueryHostingModel(); m_loadedApplicationId = pApplication.GetApplicationId(); m_disallowRotationOnConfigChange = options.QueryDisallowRotationOnConfigChange(); + m_shutdownDelay = options.QueryShutdownDelay(); RETURN_IF_FAILED(LoadRequestHandlerAssembly(pApplication, shadowCopyPath, options, pApplicationFactory, errorContext)); @@ -197,6 +199,11 @@ bool HandlerResolver::GetDisallowRotationOnConfigChange() return m_disallowRotationOnConfigChange; } +std::chrono::milliseconds HandlerResolver::GetShutdownDelay() const +{ + return m_shutdownDelay; +} + HRESULT HandlerResolver::FindNativeAssemblyFromGlobalLocation( const ShimOptions& pConfiguration, diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h index a828773c20e1..54121f072cac 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h @@ -19,6 +19,7 @@ class HandlerResolver void ResetHostingModel(); APP_HOSTING_MODEL GetHostingModel(); bool GetDisallowRotationOnConfigChange(); + std::chrono::milliseconds GetShutdownDelay() const; private: HRESULT LoadRequestHandlerAssembly(const IHttpApplication &pApplication, const std::filesystem::path& shadowCopyPath, const ShimOptions& pConfiguration, std::unique_ptr& pApplicationFactory, ErrorContext& errorContext); @@ -40,6 +41,7 @@ class HandlerResolver APP_HOSTING_MODEL m_loadedApplicationHostingModel; HostFxr m_hHostFxrDll; bool m_disallowRotationOnConfigChange; + std::chrono::milliseconds m_shutdownDelay; static const PCWSTR s_pwzAspnetcoreInProcessRequestHandlerName; static const PCWSTR s_pwzAspnetcoreOutOfProcessRequestHandlerName; diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp index f85f5483a2c8..88865cd9b132 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp @@ -12,6 +12,8 @@ #define CS_ASPNETCORE_SHADOW_COPY_DIRECTORY L"shadowCopyDirectory" #define CS_ASPNETCORE_CLEAN_SHADOW_DIRECTORY_CONTENT L"cleanShadowCopyDirectory" #define CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG L"disallowRotationOnConfigChange" +#define CS_ASPNETCORE_SHUTDOWN_DELAY L"shutdownDelay" +#define CS_ASPNETCORE_SHUTDOWN_DELAY_ENV L"ANCM_shutdownDelay" ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) : m_hostingModel(HOSTING_UNKNOWN), @@ -53,7 +55,7 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) : auto disallowRotationOnConfigChange = find_element(handlerSettings, CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG).value_or(std::wstring()); m_fDisallowRotationOnConfigChange = equals_ignore_case(L"true", disallowRotationOnConfigChange); - + m_strProcessPath = section->GetRequiredString(CS_ASPNETCORE_PROCESS_EXE_PATH); m_strArguments = section->GetString(CS_ASPNETCORE_PROCESS_ARGUMENTS).value_or(CS_ASPNETCORE_PROCESS_ARGUMENTS_DEFAULT); m_fStdoutLogEnabled = section->GetRequiredBool(CS_ASPNETCORE_STDOUT_LOG_ENABLED); @@ -82,4 +84,38 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) : auto dotnetEnvironmentEnabled = equals_ignore_case(L"Development", dotnetEnvironment); m_fShowDetailedErrors = detailedErrorsEnabled || aspnetCoreEnvironmentEnabled || dotnetEnvironmentEnabled; + + // Specifies how long to delay (in milliseconds) after IIS tells us to stop before starting the application shutdown. + // See StartShutdown in globalmodule to see how it's used. + auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring()); + if (shutdownDelay.empty()) + { + // Fallback to environment variable if process specific config wasn't set + shutdownDelay = Environment::GetEnvironmentVariableValue(CS_ASPNETCORE_SHUTDOWN_DELAY_ENV) + .value_or(environmentVariables[CS_ASPNETCORE_SHUTDOWN_DELAY_ENV]); + if (shutdownDelay.empty()) + { + // Default if neither process specific config or environment variable aren't set + m_fShutdownDelay = std::chrono::seconds(1); + } + else + { + SetShutdownDelay(shutdownDelay); + } + } + else + { + SetShutdownDelay(shutdownDelay); + } +} + +void ShimOptions::SetShutdownDelay(const std::wstring& shutdownDelay) +{ + auto millsecondsValue = std::stoi(shutdownDelay); + if (millsecondsValue < 0) + { + throw ConfigurationLoadException(format( + L"'shutdownDelay' in web.config or '%s' environment variable is less than 0.", CS_ASPNETCORE_SHUTDOWN_DELAY_ENV)); + } + m_fShutdownDelay = std::chrono::milliseconds(millsecondsValue); } diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h index 5b3cf72d692b..4e13190be6dd 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h @@ -89,6 +89,12 @@ class ShimOptions: NonCopyable return m_fDisallowRotationOnConfigChange; } + std::chrono::milliseconds + QueryShutdownDelay() const noexcept + { + return m_fShutdownDelay; + } + ShimOptions(const ConfigurationSource &configurationSource); private: @@ -104,4 +110,7 @@ class ShimOptions: NonCopyable bool m_fCleanShadowCopyDirectory; bool m_fDisallowRotationOnConfigChange; std::wstring m_strShadowCopyingDirectory; + std::chrono::milliseconds m_fShutdownDelay; + + void SetShutdownDelay(const std::wstring& shutdownDelay); }; diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp index 1da43e14343a..48855946d29a 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp @@ -143,22 +143,26 @@ APPLICATION_MANAGER::RecycleApplicationFromManager( } } - // If we receive a request at this point. - // OutOfProcess: we will create a new application with new configuration - // InProcess: the request would have to be rejected, as we are about to call g_HttpServer->RecycleProcess - // on the worker process - if (!applicationsToRecycle.empty()) { for (auto& application : applicationsToRecycle) { try { - application->ShutDownApplication(/* fServerInitiated */ false); + if (UseLegacyShutdown()) + { + application->ShutDownApplication(/* fServerInitiated */ false); + } + else + { + // Recycle the process to trigger OnGlobalStopListening + // which will shutdown the server and stop listening for new requests for this app + m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand"); + } } catch (...) { - LOG_ERRORF(L"Failed to stop application '%ls'", application->QueryApplicationInfoKey().c_str()); + LOG_ERRORF(L"Failed to recycle application '%ls'", application->QueryApplicationInfoKey().c_str()); OBSERVE_CAUGHT_EXCEPTION() // Failed to recycle an application. Log an event @@ -176,28 +180,31 @@ APPLICATION_MANAGER::RecycleApplicationFromManager( } } - // Remove apps after calling shutdown on each of them - // This is exclusive to in-process, as the shutdown of an in-process app recycles - // the entire worker process. - if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS) + if (UseLegacyShutdown()) { - SRWExclusiveLock lock(m_srwLock); - const std::wstring configurationPath = pszApplicationId; - - auto itr = m_pApplicationInfoHash.begin(); - while (itr != m_pApplicationInfoHash.end()) + // Remove apps after calling shutdown on each of them + // This is exclusive to in-process, as the shutdown of an in-process app recycles + // the entire worker process. + if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS) { - if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath) - && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end()) - { - itr = m_pApplicationInfoHash.erase(itr); - } - else + SRWExclusiveLock lock(m_srwLock); + const std::wstring configurationPath = pszApplicationId; + + auto itr = m_pApplicationInfoHash.begin(); + while (itr != m_pApplicationInfoHash.end()) { - ++itr; + if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath) + && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end()) + { + itr = m_pApplicationInfoHash.erase(itr); + } + else + { + ++itr; + } } - } - } // Release Exclusive m_srwLock + } // Release Exclusive m_srwLock + } } CATCH_RETURN() @@ -211,14 +218,19 @@ APPLICATION_MANAGER::RecycleApplicationFromManager( VOID APPLICATION_MANAGER::ShutDown() { + // During shutdown we lock until we delete the application + SRWExclusiveLock lock(m_srwLock); + // We are guaranteed to only have one outstanding OnGlobalStopListening event at a time // However, it is possible to receive multiple OnGlobalStopListening events // Protect against this by checking if we already shut down. + if (g_fInShutdown) + { + return; + } + g_fInShutdown = TRUE; g_fInAppOfflineShutdown = true; - - // During shutdown we lock until we delete the application - SRWExclusiveLock lock(m_srwLock); for (auto & [str, applicationInfo] : m_pApplicationInfoHash) { applicationInfo->ShutDownApplication(/* fServerInitiated */ true); diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h index 2f9f8b84ce5c..efc466dc7ca9 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h @@ -47,6 +47,16 @@ class APPLICATION_MANAGER return !m_handlerResolver.GetDisallowRotationOnConfigChange(); } + std::chrono::milliseconds GetShutdownDelay() const + { + return m_handlerResolver.GetShutdownDelay(); + } + + bool UseLegacyShutdown() const + { + return m_handlerResolver.GetShutdownDelay() == std::chrono::milliseconds::zero(); + } + private: std::unordered_map> m_pApplicationInfoHash; diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp index 1fde8723bd77..4d55e36b80d9 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp @@ -125,13 +125,14 @@ HRESULT moduleFactory.release(), RQ_EXECUTE_REQUEST_HANDLER, 0)); -; + auto pGlobalModule = std::make_unique(std::move(applicationManager)); RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications( - pGlobalModule.release(), - GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop - GL_STOP_LISTENING)); // worker process stop or recycle + pGlobalModule.release(), + GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop + GL_STOP_LISTENING | // worker process will stop listening for http requests + GL_APPLICATION_STOP)); // app pool recycle or stop return S_OK; } diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp index 94668ed8a34e..9e69d586cb80 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp @@ -6,7 +6,7 @@ extern BOOL g_fInShutdown; ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptr pApplicationManager) noexcept - :m_pApplicationManager(std::move(pApplicationManager)) + : m_pApplicationManager(std::move(pApplicationManager)) { } @@ -16,26 +16,52 @@ ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptrShutDown(); - m_pApplicationManager = nullptr; + StartShutdown(); // Return processing to the pipeline. return GL_NOTIFICATION_CONTINUE; } +GLOBAL_NOTIFICATION_STATUS +ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop( + IN IHttpApplicationStopProvider* pProvider +) +{ + UNREFERENCED_PARAMETER(pProvider); + + // If we're already cleaned up just return. + // If user has opted out of the new shutdown behavior ignore this call as we never registered for it before + if (!m_pApplicationManager || m_pApplicationManager->UseLegacyShutdown()) + { + return GL_NOTIFICATION_CONTINUE; + } + + LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop"); + + if (!g_fInShutdown && !m_shutdown.joinable()) + { + // Apps with preload + always running that don't receive a request before recycle/shutdown will never call OnGlobalStopListening + // IISExpress can also close without calling OnGlobalStopListening which is where we usually would trigger shutdown + // so we should make sure to shutdown the server in those cases + StartShutdown(); + } + + return GL_NOTIFICATION_CONTINUE; +} + // // Is called when configuration changed // Recycled the corresponding core app if its configuration changed diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h index 80f047e08d74..3bcb30c0b2e8 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h @@ -4,6 +4,9 @@ #pragma once #include "applicationmanager.h" +#include + +extern BOOL g_fInShutdown; class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule { @@ -19,6 +22,12 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule VOID Terminate() override { LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::Terminate"); + + if (m_shutdown.joinable()) + { + m_shutdown.join(); + } + // Remove the class from memory. delete this; } @@ -33,6 +42,48 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule _In_ IGlobalConfigurationChangeProvider * pProvider ) override; + GLOBAL_NOTIFICATION_STATUS + OnGlobalApplicationStop( + IN IHttpApplicationStopProvider* pProvider + ) override; + private: std::shared_ptr m_pApplicationManager; + std::thread m_shutdown; + + void StartShutdown() + { + // Shutdown has already been started/finished + if (m_shutdown.joinable() || g_fInShutdown) + { + return; + } + + // If delay is zero we can go back to the old behavior of calling shutdown inline + // this is primarily so that we have a way for users to revert the new behavior if there are issues with it + if (m_pApplicationManager->UseLegacyShutdown()) + { + LOG_INFO(L"Shutdown starting."); + m_pApplicationManager->ShutDown(); + m_pApplicationManager = nullptr; + } + else + { + // Run shutdown on a background thread. It seems like IIS keeps giving us requests if OnGlobalStopListening is still running + // which will result in 503s from applicationmanager since we're shutting down and don't want to process new requests. + // But if we return ASAP from OnGlobalStopListening, by not shutting down inline and with a small delay to reduce races, + // IIS will actually stop giving us new requests and queue them instead for processing by the new app process. + m_shutdown = std::thread([this]() + { + auto delay = m_pApplicationManager->GetShutdownDelay(); + LOG_INFOF(L"Shutdown starting in %d ms.", delay.count()); + // Delay so that any incoming requests while we're returning from OnGlobalStopListening are allowed to be processed + std::this_thread::sleep_for(delay); + + LOG_INFO(L"Shutdown starting."); + m_pApplicationManager->ShutDown(); + m_pApplicationManager = nullptr; + }); + } + } }; diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp index 162c0fea907b..99dd210b3dd1 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp @@ -93,6 +93,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler( { if (g_fInShutdown) { + LOG_WARN(L"Received a request during shutdown. Will return a 503 response."); FINISHED(HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS)); } diff --git a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp index b876d6dc2656..5282792f1e6f 100644 --- a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp +++ b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp @@ -17,38 +17,40 @@ InProcessApplicationBase::StopInternal(bool fServerInitiated) { AppOfflineTrackingApplication::StopInternal(fServerInitiated); - // Stop was initiated by server no need to do anything, server would stop on it's own - if (fServerInitiated) + // Ignore fServerInitiated for IISExpress + // Recycle doesn't do anything in IISExpress, we need to explicitly shutdown + if (m_pHttpServer.IsCommandLineLaunch()) { + // Send WM_QUIT to the main window to initiate graceful shutdown + EnumWindows([](HWND hwnd, LPARAM) -> BOOL + { + DWORD processId; + + if (GetWindowThreadProcessId(hwnd, &processId) && + processId == GetCurrentProcessId() && + GetConsoleWindow() != hwnd) + { + PostMessage(hwnd, WM_QUIT, 0, 0); + return false; + } + + return true; + }, 0); + return; } - if (!m_pHttpServer.IsCommandLineLaunch()) + // Stop was initiated by server no need to do anything, server would stop on its own + if (fServerInitiated) { - // IIS scenario. - // We don't actually handle any shutdown logic here. - // Instead, we notify IIS that the process needs to be recycled, which will call - // ApplicationManager->Shutdown(). This will call shutdown on the application. - LOG_INFO(L"AspNetCore InProcess Recycle Process on Demand"); - m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand"); + return; } - else - { - // Send WM_QUIT to the main window to initiate graceful shutdown - EnumWindows([](HWND hwnd, LPARAM) -> BOOL - { - DWORD processId; - if (GetWindowThreadProcessId(hwnd, &processId) && - processId == GetCurrentProcessId() && - GetConsoleWindow() != hwnd) - { - PostMessage(hwnd, WM_QUIT, 0, 0); - return false; - } - - return true; - }, 0); - } + // IIS scenario. + // We don't actually handle any shutdown logic here. + // Instead, we notify IIS that the process needs to be recycled, which will call + // ApplicationManager->Shutdown(). This will call shutdown on the application. + LOG_INFO(L"AspNetCore InProcess Recycle Process on Demand"); + m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand"); } diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs index 857c87721bb1..9e08448d969b 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs @@ -79,7 +79,7 @@ private static string FormatEntries(IEnumerable entries) return string.Join(",", entries.Select(e => e.Message)); } - private static IEnumerable GetEntries(IISDeploymentResult deploymentResult) + internal static IEnumerable GetEntries(IISDeploymentResult deploymentResult) { var eventLog = new EventLog("Application"); @@ -162,9 +162,16 @@ public static string InProcessFailedToStart(IISDeploymentResult deploymentResult } } - public static string InProcessShutdown() + public static string ShutdownMessage(IISDeploymentResult deploymentResult) { - return "Application 'MACHINE/WEBROOT/APPHOST/.*?' has shutdown."; + if (deploymentResult.DeploymentParameters.HostingModel == HostingModel.InProcess) + { + return "Application 'MACHINE/WEBROOT/APPHOST/.*?' has shutdown."; + } + else + { + return "Application '/LM/W3SVC/1/ROOT' with physical root '.*?' shut down process with Id '.*?' listening on port '.*?'"; + } } public static string ShutdownFileChange(IISDeploymentResult deploymentResult) diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs index 0b8c5bbeff58..74f499ce488e 100644 --- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs +++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs @@ -8,6 +8,7 @@ using Microsoft.AspNetCore.Server.IntegrationTesting; using Microsoft.AspNetCore.Server.IntegrationTesting.IIS; using Microsoft.Extensions.Logging; +using Microsoft.Web.Administration; using Newtonsoft.Json; namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests; @@ -179,6 +180,15 @@ public static async Task AssertRecycledAsync(this IISDeploymentResult deployment } } + // Don't use with IISExpress, recycle isn't a valid operation + public static void Recycle(string appPoolName) + { + using var serverManager = new ServerManager(); + var appPool = serverManager.ApplicationPools.FirstOrDefault(ap => ap.Name == appPoolName); + Assert.NotNull(appPool); + appPool.Recycle(); + } + public static IEnumerable ToTheoryData(this Dictionary dictionary) { return dictionary.Keys.Select(k => new[] { k }); diff --git a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs index 872193edbb29..5d82142bcddd 100644 --- a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs +++ b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs @@ -257,7 +257,81 @@ await statusConnection.Receive("5", // Shutdown should be graceful here! EventLogHelpers.VerifyEventLogEvent(deploymentResult, - EventLogHelpers.InProcessShutdown(), Logger); + EventLogHelpers.ShutdownMessage(deploymentResult), Logger); + } + + [ConditionalFact] + [RequiresNewShim] + public async Task RequestsWhileRestartingAppFromConfigChangeAreProcessed() + { + var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite); + + if (deploymentParameters.ServerType == ServerType.IISExpress) + { + // IISExpress doesn't support recycle + return; + } + + var deploymentResult = await DeployAsync(deploymentParameters); + + var result = await deploymentResult.HttpClient.GetAsync("/HelloWorld"); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + result.Dispose(); + + // Just "touching" web.config should be enough to restart the process + deploymentResult.ModifyWebConfig(element => { }); + + // Default shutdown delay is 1 second, we want to send requests while the shutdown is happening + // So we send a bunch of requests and one of them hopefully will run during shutdown and be queued for processing by the new app + for (var i = 0; i < 2000; i++) + { + using var res = await deploymentResult.HttpClient.GetAsync("/HelloWorld"); + await Task.Delay(1); + Assert.Equal(HttpStatusCode.OK, res.StatusCode); + } + + await deploymentResult.AssertRecycledAsync(); + + // Shutdown should be graceful here! + EventLogHelpers.VerifyEventLogEvent(deploymentResult, + EventLogHelpers.ShutdownMessage(deploymentResult), Logger); + } + + [ConditionalFact] + [RequiresNewShim] + public async Task RequestsWhileRecyclingAppAreProcessed() + { + var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite); + + if (deploymentParameters.ServerType == ServerType.IISExpress) + { + // IISExpress doesn't support recycle + return; + } + + var deploymentResult = await DeployAsync(deploymentParameters); + + var result = await deploymentResult.HttpClient.GetAsync("/HelloWorld"); + Assert.Equal(HttpStatusCode.OK, result.StatusCode); + result.Dispose(); + + // Recycle app pool + Helpers.Recycle(deploymentResult.AppPoolName); + + // Default shutdown delay is 1 second, we want to send requests while the shutdown is happening + // So we send a bunch of requests and one of them hopefully will run during shutdown and be queued for processing by the new app + for (var i = 0; i < 2000; i++) + { + using var res = await deploymentResult.HttpClient.GetAsync("/HelloWorld"); + await Task.Delay(1); + Assert.Equal(HttpStatusCode.OK, res.StatusCode); + } + + await deploymentResult.AssertRecycledAsync(); + + // Shutdown should be graceful here! + EventLogHelpers.VerifyEventLogEvent(deploymentResult, + EventLogHelpers.ShutdownMessage(deploymentResult), Logger); } [ConditionalFact] diff --git a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs index 2f646a005206..ba407def5bcf 100644 --- a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs +++ b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs @@ -4,6 +4,7 @@ using System; using System.IO; using System.ServiceProcess; +using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using Microsoft.AspNetCore.Server.IIS.FunctionalTests.Utilities; @@ -37,9 +38,11 @@ public ApplicationInitializationTests(PublishedSitesFixture fixture) : base(fixt [ConditionalTheory] [RequiresIIS(IISCapability.ApplicationInitialization)] - [InlineData(HostingModel.InProcess)] - [InlineData(HostingModel.OutOfProcess)] - public async Task ApplicationPreloadStartsApp(HostingModel hostingModel) + [InlineData(HostingModel.InProcess, true)] + [InlineData(HostingModel.OutOfProcess, true)] + [InlineData(HostingModel.InProcess, false)] + [InlineData(HostingModel.OutOfProcess, false)] + public async Task ApplicationPreloadStartsApp(HostingModel hostingModel, bool delayShutdown) { // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900)) @@ -49,11 +52,26 @@ public async Task ApplicationPreloadStartsApp(HostingModel hostingModel) (args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\""); EnablePreload(baseDeploymentParameters); + baseDeploymentParameters.HandlerSettings["shutdownDelay"] = delayShutdown ? "1000" : "0"; var result = await DeployAsync(baseDeploymentParameters); await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), TimeoutExtensions.DefaultTimeoutValue); StopServer(); EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result), Logger); + + if (delayShutdown) + { + EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger); + } + else + { + Assert.True(result.HostProcess.HasExited); + + var entries = EventLogHelpers.GetEntries(result); + var expectedRegex = new Regex(EventLogHelpers.ShutdownMessage(result), RegexOptions.Singleline); + var matchedEntries = entries.Where(entry => expectedRegex.IsMatch(entry.Message)).ToArray(); + Assert.Empty(matchedEntries); + } } } @@ -84,6 +102,7 @@ public async Task ApplicationInitializationPageIsRequested(HostingModel hostingM await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), TimeoutExtensions.DefaultTimeoutValue); StopServer(); EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result), Logger); + EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger); } }