From 8b43cc43c58127ee8d74120b57e6217441bf6cbb Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Wed, 13 Dec 2023 13:01:05 -0800
Subject: [PATCH 01/11] Change how ANCM recycles app

---
 .../AspNetCore/HandlerResolver.cpp            |  9 ++-
 .../AspNetCore/HandlerResolver.h              |  2 +
 .../AspNetCore/ShimOptions.cpp                | 12 +++
 .../AspNetCore/ShimOptions.h                  |  7 ++
 .../AspNetCore/applicationmanager.cpp         | 47 ++++--------
 .../AspNetCore/applicationmanager.h           | 10 +++
 .../AspNetCoreModuleV2/AspNetCore/dllmain.cpp |  3 +-
 .../AspNetCore/globalmodule.cpp               | 33 +++++++-
 .../AspNetCore/globalmodule.h                 | 36 +++++++++
 .../AspNetCore/proxymodule.cpp                |  1 +
 .../Infrastructure/Helpers.cs                 | 10 +++
 .../test/Common.LongTests/ShutdownTests.cs    | 76 +++++++++++++++++++
 12 files changed, 206 insertions(+), 40 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp
index a1322eaabb32..fcd4205138c9 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.cpp
@@ -22,7 +22,8 @@ const PCWSTR HandlerResolver::s_pwzAspnetcoreOutOfProcessRequestHandlerName = L"
 HandlerResolver::HandlerResolver(HMODULE hModule, const IHttpServer &pServer)
     : m_hModule(hModule),
       m_pServer(pServer),
-      m_loadedApplicationHostingModel(HOSTING_UNKNOWN)
+      m_loadedApplicationHostingModel(HOSTING_UNKNOWN),
+      m_shutdownDelay()
 {
     m_disallowRotationOnConfigChange = false;
     InitializeSRWLock(&m_requestHandlerLoadLock);
@@ -171,6 +172,7 @@ HandlerResolver::GetApplicationFactory(const IHttpApplication& pApplication, con
     m_loadedApplicationHostingModel = options.QueryHostingModel();
     m_loadedApplicationId = pApplication.GetApplicationId();
     m_disallowRotationOnConfigChange = options.QueryDisallowRotationOnConfigChange();
+    m_shutdownDelay = options.QueryShutdownDelay();
 
     RETURN_IF_FAILED(LoadRequestHandlerAssembly(pApplication, shadowCopyPath, options, pApplicationFactory, errorContext));
 
@@ -197,6 +199,11 @@ bool HandlerResolver::GetDisallowRotationOnConfigChange()
     return m_disallowRotationOnConfigChange;
 }
 
+std::chrono::milliseconds HandlerResolver::GetShutdownDelay() const
+{
+    return m_shutdownDelay;
+}
+
 HRESULT
 HandlerResolver::FindNativeAssemblyFromGlobalLocation(
     const ShimOptions& pConfiguration,
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h
index a828773c20e1..54121f072cac 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/HandlerResolver.h
@@ -19,6 +19,7 @@ class HandlerResolver
     void ResetHostingModel();
     APP_HOSTING_MODEL GetHostingModel();
     bool GetDisallowRotationOnConfigChange();
+    std::chrono::milliseconds GetShutdownDelay() const;
 
 private:
     HRESULT LoadRequestHandlerAssembly(const IHttpApplication &pApplication, const std::filesystem::path& shadowCopyPath, const ShimOptions& pConfiguration, std::unique_ptr<ApplicationFactory>& pApplicationFactory, ErrorContext& errorContext);
@@ -40,6 +41,7 @@ class HandlerResolver
     APP_HOSTING_MODEL m_loadedApplicationHostingModel;
     HostFxr m_hHostFxrDll;
     bool m_disallowRotationOnConfigChange;
+    std::chrono::milliseconds m_shutdownDelay;
 
     static const PCWSTR          s_pwzAspnetcoreInProcessRequestHandlerName;
     static const PCWSTR          s_pwzAspnetcoreOutOfProcessRequestHandlerName;
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
index f85f5483a2c8..2a43ffd8cc2d 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
@@ -12,6 +12,7 @@
 #define CS_ASPNETCORE_SHADOW_COPY_DIRECTORY              L"shadowCopyDirectory"
 #define CS_ASPNETCORE_CLEAN_SHADOW_DIRECTORY_CONTENT     L"cleanShadowCopyDirectory"
 #define CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG             L"disallowRotationOnConfigChange"
+#define CS_ASPNETCORE_SHUTDOWN_DELAY                     L"shutdownDelay"
 
 ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
         m_hostingModel(HOSTING_UNKNOWN),
@@ -53,6 +54,17 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
 
     auto disallowRotationOnConfigChange = find_element(handlerSettings, CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG).value_or(std::wstring());
     m_fDisallowRotationOnConfigChange = equals_ignore_case(L"true", disallowRotationOnConfigChange);
+
+    auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
+    if (shutdownDelay.empty())
+    {
+        m_fShutdownDelay = std::chrono::seconds(1);
+    }
+    else
+    {
+        auto str = to_multi_byte_string(shutdownDelay, CP_UTF8);
+        m_fShutdownDelay = std::chrono::milliseconds(std::atoi(str.c_str()));
+    }
                 
     m_strProcessPath = section->GetRequiredString(CS_ASPNETCORE_PROCESS_EXE_PATH);
     m_strArguments = section->GetString(CS_ASPNETCORE_PROCESS_ARGUMENTS).value_or(CS_ASPNETCORE_PROCESS_ARGUMENTS_DEFAULT);
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
index 5b3cf72d692b..33a66b54bb81 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
@@ -89,6 +89,12 @@ class ShimOptions: NonCopyable
         return m_fDisallowRotationOnConfigChange;
     }
 
+    std::chrono::milliseconds
+        QueryShutdownDelay() const noexcept
+    {
+        return m_fShutdownDelay;
+    }
+
     ShimOptions(const ConfigurationSource &configurationSource);
 
 private:
@@ -104,4 +110,5 @@ class ShimOptions: NonCopyable
     bool                           m_fCleanShadowCopyDirectory;
     bool                           m_fDisallowRotationOnConfigChange;
     std::wstring                   m_strShadowCopyingDirectory;
+    std::chrono::milliseconds      m_fShutdownDelay;
 };
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
index 1da43e14343a..b76a7ccec802 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
@@ -143,22 +143,19 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             }
         }
 
-        // If we receive a request at this point.
-        // OutOfProcess: we will create a new application with new configuration
-        // InProcess: the request would have to be rejected, as we are about to call g_HttpServer->RecycleProcess
-        // on the worker process
-
         if (!applicationsToRecycle.empty())
         {
             for (auto& application : applicationsToRecycle)
             {
                 try
                 {
-                    application->ShutDownApplication(/* fServerInitiated */ false);
+                    // Recycle the process to trigger OnGlobalStopListening
+                    // which will shutdown the server and stop listening for new requests for this app
+                    m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
                 }
                 catch (...)
                 {
-                    LOG_ERRORF(L"Failed to stop application '%ls'", application->QueryApplicationInfoKey().c_str());
+                    LOG_ERRORF(L"Failed to recycle application '%ls'", application->QueryApplicationInfoKey().c_str());
                     OBSERVE_CAUGHT_EXCEPTION()
 
                     // Failed to recycle an application. Log an event
@@ -175,29 +172,6 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
                 }
             }
         }
-
-        // Remove apps after calling shutdown on each of them
-        // This is exclusive to in-process, as the shutdown of an in-process app recycles
-        // the entire worker process.
-        if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS)
-        {
-            SRWExclusiveLock lock(m_srwLock);
-            const std::wstring configurationPath = pszApplicationId;
-
-            auto itr = m_pApplicationInfoHash.begin();
-            while (itr != m_pApplicationInfoHash.end())
-            {
-                if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath)
-                    && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end())
-                {
-                    itr = m_pApplicationInfoHash.erase(itr);
-                }
-                else
-                {
-                    ++itr;
-                }
-            }
-        } // Release Exclusive m_srwLock
     }
     CATCH_RETURN()
 
@@ -211,17 +185,22 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
 VOID
 APPLICATION_MANAGER::ShutDown()
 {
+    // During shutdown we lock until we delete the application
+    SRWExclusiveLock lock(m_srwLock);
+
     // We are guaranteed to only have one outstanding OnGlobalStopListening event at a time
     // However, it is possible to receive multiple OnGlobalStopListening events
     // Protect against this by checking if we already shut down.
+    if (g_fInShutdown)
+    {
+        return;
+    }
+
     g_fInShutdown = TRUE;
     g_fInAppOfflineShutdown = true;
-
-    // During shutdown we lock until we delete the application
-    SRWExclusiveLock lock(m_srwLock);
     for (auto & [str, applicationInfo] : m_pApplicationInfoHash)
     {
-        applicationInfo->ShutDownApplication(/* fServerInitiated */ true);
+        applicationInfo->ShutDownApplication(/* fServerInitiated */ false);
         applicationInfo = nullptr;
     }
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
index 2f9f8b84ce5c..f9770011daf2 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
@@ -47,6 +47,16 @@ class APPLICATION_MANAGER
         return !m_handlerResolver.GetDisallowRotationOnConfigChange();
     }
 
+    std::chrono::milliseconds GetShutdownDelay() const
+    {
+        return m_handlerResolver.GetShutdownDelay();
+    }
+
+    bool IsCommandLineLaunch() const
+    {
+        return m_pHttpServer.IsCommandLineLaunch();
+    }
+
 private:
 
     std::unordered_map<std::wstring, std::shared_ptr<APPLICATION_INFO>>      m_pApplicationInfoHash;
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
index 1fde8723bd77..75f85aa28867 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
@@ -131,7 +131,8 @@ HRESULT
     RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
                                      pGlobalModule.release(),
                                      GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-                                     GL_STOP_LISTENING));   // worker process stop or recycle
+                                     GL_STOP_LISTENING | // worker process stop
+                                     GL_APPLICATION_STOP)); // app pool recycle
 
     return S_OK;
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index 94668ed8a34e..2ddcdeb106ad 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -16,26 +16,51 @@ ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptr<APPLICATION
 //
 GLOBAL_NOTIFICATION_STATUS
 ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening(
-    _In_ IGlobalStopListeningProvider * pProvider
+    _In_ IGlobalStopListeningProvider* pProvider
 )
 {
     UNREFERENCED_PARAMETER(pProvider);
 
     LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalStopListening");
 
-    if (g_fInShutdown)
+    if (g_fInShutdown || m_shutdown.joinable())
     {
         // Avoid receiving two shutdown notifications.
         return GL_NOTIFICATION_CONTINUE;
     }
 
-    m_pApplicationManager->ShutDown();
-    m_pApplicationManager = nullptr;
+    StartShutdown();
 
     // Return processing to the pipeline.
     return GL_NOTIFICATION_CONTINUE;
 }
 
+GLOBAL_NOTIFICATION_STATUS
+ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
+    IN IHttpApplicationStopProvider* pProvider
+)
+{
+    UNREFERENCED_PARAMETER(pProvider);
+    LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop");
+
+    if (m_shutdown.joinable())
+    {
+        m_shutdown.join();
+    }
+    else
+    {
+        if (!g_fInShutdown && m_pApplicationManager && m_pApplicationManager->IsCommandLineLaunch() &&
+            m_pApplicationManager->ShouldRecycleOnConfigChange())
+        {
+            // IISExpress can close without calling OnGlobalStopListening which is where we usually would trigger shutdown
+            // so we should make sure to shutdown the server
+            StartShutdown();
+        }
+    }
+
+    return GL_NOTIFICATION_CONTINUE;
+}
+
 //
 // Is called when configuration changed
 // Recycled the corresponding core app if its configuration changed
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index 80f047e08d74..3fd3db40f1cc 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -4,6 +4,7 @@
 #pragma once
 
 #include "applicationmanager.h"
+#include <thread>
 
 class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
 {
@@ -19,6 +20,12 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
     VOID Terminate() override
     {
         LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::Terminate");
+
+        if (m_shutdown.joinable())
+        {
+            m_shutdown.join();
+        }
+
         // Remove the class from memory.
         delete this;
     }
@@ -33,6 +40,35 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
         _In_ IGlobalConfigurationChangeProvider * pProvider
     ) override;
 
+    GLOBAL_NOTIFICATION_STATUS
+    OnGlobalApplicationStop(
+        IN IHttpApplicationStopProvider* pProvider
+    ) override;
+
 private:
     std::shared_ptr<APPLICATION_MANAGER> m_pApplicationManager;
+    std::thread m_shutdown;
+
+    void StartShutdown()
+    {
+        // Shutdown has already been started
+        if (m_shutdown.joinable())
+        {
+            return;
+        }
+
+        // Run shutdown on a background thread. It seems like IIS keeps giving us requests if OnGlobalStopListening is still running
+        // which will result in 503s since we're shutting down.
+        // But if we return ASAP from OnGlobalStopListening (by not shutting down inline),
+        // IIS will actually stop giving us new requests and queue them instead for processing by the new app process.
+        m_shutdown = std::thread([this]()
+            {
+                auto delay = m_pApplicationManager->GetShutdownDelay();
+                LOG_INFOF(L"Shutdown starting in %d ms.", delay.count());
+                // Delay so that any incoming requests while we're returning from OnGlobalStopListening are allowed to be processed
+                std::this_thread::sleep_for(delay);
+                m_pApplicationManager->ShutDown();
+                m_pApplicationManager = nullptr;
+            });
+    }
 };
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
index 162c0fea907b..5a752b66442e 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
@@ -93,6 +93,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
     {
         if (g_fInShutdown)
         {
+            LOG_INFO(L"Received request during shutdown. Will return 503.");
             FINISHED(HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS));
         }
 
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs
index 0b8c5bbeff58..74f499ce488e 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/Helpers.cs
@@ -8,6 +8,7 @@
 using Microsoft.AspNetCore.Server.IntegrationTesting;
 using Microsoft.AspNetCore.Server.IntegrationTesting.IIS;
 using Microsoft.Extensions.Logging;
+using Microsoft.Web.Administration;
 using Newtonsoft.Json;
 
 namespace Microsoft.AspNetCore.Server.IIS.FunctionalTests;
@@ -179,6 +180,15 @@ public static async Task AssertRecycledAsync(this IISDeploymentResult deployment
         }
     }
 
+    // Don't use with IISExpress, recycle isn't a valid operation
+    public static void Recycle(string appPoolName)
+    {
+        using var serverManager = new ServerManager();
+        var appPool = serverManager.ApplicationPools.FirstOrDefault(ap => ap.Name == appPoolName);
+        Assert.NotNull(appPool);
+        appPool.Recycle();
+    }
+
     public static IEnumerable<object[]> ToTheoryData<T>(this Dictionary<string, T> dictionary)
     {
         return dictionary.Keys.Select(k => new[] { k });
diff --git a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
index 872193edbb29..e89b1e766c3a 100644
--- a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
@@ -260,6 +260,82 @@ await statusConnection.Receive("5",
             EventLogHelpers.InProcessShutdown(), Logger);
     }
 
+    [ConditionalFact]
+    [RequiresNewShim]
+    [Repeat(10)] // temp for CI testing
+    public async Task RequestsWhileRestartingAppFromConfigChangeAreProcessed()
+    {
+        var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite);
+
+        if (deploymentParameters.ServerType == ServerType.IISExpress)
+        {
+            // IISExpress doesn't support recycle
+            return;
+        }
+
+        var deploymentResult = await DeployAsync(deploymentParameters);
+
+        var result = await deploymentResult.HttpClient.GetAsync("/HelloWorld");
+        Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        result.Dispose();
+
+        // Just "touching" web.config should be enough to restart the process
+        deploymentResult.ModifyWebConfig(element => { });
+
+        // Default shutdown delay is 1 second, we want to send requests while the shutdown is happening
+        // So we send a bunch of requests and one of them hopefully will run during shutdown and be queued for processing by the new app
+        for (var i = 0; i < 2000; i++)
+        {
+            using var res = await deploymentResult.HttpClient.GetAsync("/HelloWorld");
+            await Task.Delay(1);
+            Assert.Equal(HttpStatusCode.OK, res.StatusCode);
+        }
+
+        await deploymentResult.AssertRecycledAsync();
+
+        // Shutdown should be graceful here!
+        EventLogHelpers.VerifyEventLogEvent(deploymentResult,
+            EventLogHelpers.InProcessShutdown(), Logger);
+    }
+
+    [ConditionalFact]
+    [RequiresNewShim]
+    [Repeat(10)] // temp for CI testing
+    public async Task RequestsWhileRecyclingAppAreProcessed()
+    {
+        var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite);
+
+        if (deploymentParameters.ServerType == ServerType.IISExpress)
+        {
+            // IISExpress doesn't support recycle
+            return;
+        }
+
+        var deploymentResult = await DeployAsync(deploymentParameters);
+
+        var result = await deploymentResult.HttpClient.GetAsync("/HelloWorld");
+        Assert.Equal(HttpStatusCode.OK, result.StatusCode);
+        result.Dispose();
+
+        // Recycle app pool
+        Helpers.Recycle(deploymentResult.AppPoolName);
+
+        // Default shutdown delay is 1 second, we want to send requests while the shutdown is happening
+        // So we send a bunch of requests and one of them hopefully will run during shutdown and be queued for processing by the new app
+        for (var i = 0; i < 2000; i++)
+        {
+            using var res = await deploymentResult.HttpClient.GetAsync("/HelloWorld");
+            await Task.Delay(1);
+            Assert.Equal(HttpStatusCode.OK, res.StatusCode);
+        }
+
+        await deploymentResult.AssertRecycledAsync();
+
+        // Shutdown should be graceful here!
+        EventLogHelpers.VerifyEventLogEvent(deploymentResult,
+            EventLogHelpers.InProcessShutdown(), Logger);
+    }
+
     [ConditionalFact]
     public async Task AppOfflineDroppedWhileSiteRunning_SiteShutsDown_InProcess()
     {

From 2fcc48cd14056df9a9b6c169eb8fa91a41486a85 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Wed, 3 Jan 2024 10:22:32 -0800
Subject: [PATCH 02/11] app init and single join

---
 .../AspNetCore/globalmodule.cpp                 | 17 +++++------------
 .../Infrastructure/EventLogHelpers.cs           | 11 +++++++++--
 .../IIS/test/Common.LongTests/ShutdownTests.cs  |  6 +++---
 .../ApplicationInitializationTests.cs           |  2 ++
 4 files changed, 19 insertions(+), 17 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index 2ddcdeb106ad..5aa04bf2bc6a 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -43,19 +43,12 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
     UNREFERENCED_PARAMETER(pProvider);
     LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop");
 
-    if (m_shutdown.joinable())
+    if (!g_fInShutdown && !m_shutdown.joinable())
     {
-        m_shutdown.join();
-    }
-    else
-    {
-        if (!g_fInShutdown && m_pApplicationManager && m_pApplicationManager->IsCommandLineLaunch() &&
-            m_pApplicationManager->ShouldRecycleOnConfigChange())
-        {
-            // IISExpress can close without calling OnGlobalStopListening which is where we usually would trigger shutdown
-            // so we should make sure to shutdown the server
-            StartShutdown();
-        }
+        // Apps with preload + always running that don't receive a request before recycle/shutdown will never call OnGlobalStopListening
+        // IISExpress can also close without calling OnGlobalStopListening which is where we usually would trigger shutdown
+        // so we should make sure to shutdown the server in those cases
+        StartShutdown();
     }
 
     return GL_NOTIFICATION_CONTINUE;
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
index 857c87721bb1..ca61359c92ce 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
@@ -162,9 +162,16 @@ public static string InProcessFailedToStart(IISDeploymentResult deploymentResult
         }
     }
 
-    public static string InProcessShutdown()
+    public static string ShutdownMessage(IISDeploymentResult deploymentResult)
     {
-        return "Application 'MACHINE/WEBROOT/APPHOST/.*?' has shutdown.";
+        if (deploymentResult.DeploymentParameters.HostingModel == HostingModel.InProcess)
+        {
+            return "Application 'MACHINE/WEBROOT/APPHOST/.*?' has shutdown.";
+        }
+        else
+        {
+            return "Application '/LM/W3SVC/1/ROOT' with physical root '.*?' shut down process with Id '.*?' listening on port '.*?'";
+        }
     }
 
     public static string ShutdownFileChange(IISDeploymentResult deploymentResult)
diff --git a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
index e89b1e766c3a..77bfa31d379f 100644
--- a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
@@ -257,7 +257,7 @@ await statusConnection.Receive("5",
 
         // Shutdown should be graceful here!
         EventLogHelpers.VerifyEventLogEvent(deploymentResult,
-            EventLogHelpers.InProcessShutdown(), Logger);
+            EventLogHelpers.ShutdownMessage(deploymentResult), Logger);
     }
 
     [ConditionalFact]
@@ -295,7 +295,7 @@ public async Task RequestsWhileRestartingAppFromConfigChangeAreProcessed()
 
         // Shutdown should be graceful here!
         EventLogHelpers.VerifyEventLogEvent(deploymentResult,
-            EventLogHelpers.InProcessShutdown(), Logger);
+            EventLogHelpers.ShutdownMessage(deploymentResult), Logger);
     }
 
     [ConditionalFact]
@@ -333,7 +333,7 @@ public async Task RequestsWhileRecyclingAppAreProcessed()
 
         // Shutdown should be graceful here!
         EventLogHelpers.VerifyEventLogEvent(deploymentResult,
-            EventLogHelpers.InProcessShutdown(), Logger);
+            EventLogHelpers.ShutdownMessage(deploymentResult), Logger);
     }
 
     [ConditionalFact]
diff --git a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
index 2f646a005206..04833b0eb043 100644
--- a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
+++ b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
@@ -54,6 +54,7 @@ public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
             await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), TimeoutExtensions.DefaultTimeoutValue);
             StopServer();
             EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result), Logger);
+            EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger);
         }
     }
 
@@ -84,6 +85,7 @@ public async Task ApplicationInitializationPageIsRequested(HostingModel hostingM
             await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), TimeoutExtensions.DefaultTimeoutValue);
             StopServer();
             EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result), Logger);
+            EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger);
         }
     }
 

From 45b81c6f0db7d158683fa4c1c1aedd648cda95e4 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Wed, 17 Jan 2024 20:48:42 -0800
Subject: [PATCH 03/11] don't recycle again

---
 .../IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
index b76a7ccec802..85641fdde185 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
@@ -200,7 +200,7 @@ APPLICATION_MANAGER::ShutDown()
     g_fInAppOfflineShutdown = true;
     for (auto & [str, applicationInfo] : m_pApplicationInfoHash)
     {
-        applicationInfo->ShutDownApplication(/* fServerInitiated */ false);
+        applicationInfo->ShutDownApplication(/* fServerInitiated */ true);
         applicationInfo = nullptr;
     }
 }

From d8cfcf6f9dd27df40b5e4756f868b9bc3dfaeec5 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Thu, 18 Jan 2024 13:06:48 -0800
Subject: [PATCH 04/11] fix iisexpress

---
 .../InProcessApplicationBase.cpp              | 54 ++++++++++---------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
index b876d6dc2656..e8bee5a84f99 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
@@ -17,38 +17,40 @@ InProcessApplicationBase::StopInternal(bool fServerInitiated)
 {
     AppOfflineTrackingApplication::StopInternal(fServerInitiated);
 
-    // Stop was initiated by server no need to do anything, server would stop on it's own
-    if (fServerInitiated)
+    // Ignore fServerInitiated for IISExpress
+    // Recycle doesn't do anything in IISExpress, we need to explicitly shutdown
+    if (m_pHttpServer.IsCommandLineLaunch())
     {
+        // Send WM_QUIT to the main window to initiate graceful shutdown
+        EnumWindows([](HWND hwnd, LPARAM) -> BOOL
+            {
+                DWORD processId;
+
+                if (GetWindowThreadProcessId(hwnd, &processId) &&
+                    processId == GetCurrentProcessId() &&
+                    GetConsoleWindow() != hwnd)
+                {
+                    PostMessage(hwnd, WM_QUIT, 0, 0);
+                    return false;
+                }
+
+                return true;
+            }, 0);
+
         return;
     }
 
-    if (!m_pHttpServer.IsCommandLineLaunch())
+    // Stop was initiated by server no need to do anything, server would stop on it's own
+    if (fServerInitiated)
     {
-        // IIS scenario.
-        // We don't actually handle any shutdown logic here.
-        // Instead, we notify IIS that the process needs to be recycled, which will call
-        // ApplicationManager->Shutdown(). This will call shutdown on the application.
-        LOG_INFO(L"AspNetCore InProcess Recycle Process on Demand");
-        m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
+        return;
     }
-    else
-    {
-        // Send WM_QUIT to the main window to initiate graceful shutdown
-        EnumWindows([](HWND hwnd, LPARAM) -> BOOL
-        {
-            DWORD processId;
-
-            if (GetWindowThreadProcessId(hwnd, &processId) &&
-                processId == GetCurrentProcessId() &&
-                GetConsoleWindow() != hwnd)
-            {
-                PostMessage(hwnd, WM_QUIT, 0, 0);
-                return false;
-            }
 
-            return true;
-        }, 0);
-    }
+    // IIS scenario.
+    // We don't actually handle any shutdown logic here.
+    // Instead, we notify IIS that the process needs to be recycled, which will call
+    // ApplicationManager->Shutdown(). This will call shutdown on the application.
+    LOG_INFO(L"AspNetCore InProcess Recycle Process on Demand");
+    m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
 }
 

From fab63e490b829ce8152ee559d538853c8f45f539 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Thu, 29 Feb 2024 19:53:13 -0800
Subject: [PATCH 05/11] fallback

---
 .../AspNetCore/ShimOptions.cpp                |  2 +-
 .../AspNetCore/applicationmanager.h           |  5 ---
 .../AspNetCoreModuleV2/AspNetCore/dllmain.cpp | 23 +++++++++---
 .../AspNetCore/globalmodule.cpp               |  2 +-
 .../AspNetCore/globalmodule.h                 | 37 ++++++++++++-------
 5 files changed, 43 insertions(+), 26 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
index 2a43ffd8cc2d..d5580d7fad64 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
@@ -58,7 +58,7 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
     if (shutdownDelay.empty())
     {
-        m_fShutdownDelay = std::chrono::seconds(1);
+        m_fShutdownDelay = std::chrono::seconds(0);
     }
     else
     {
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
index f9770011daf2..bccc1d704c83 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
@@ -52,11 +52,6 @@ class APPLICATION_MANAGER
         return m_handlerResolver.GetShutdownDelay();
     }
 
-    bool IsCommandLineLaunch() const
-    {
-        return m_pHttpServer.IsCommandLineLaunch();
-    }
-
 private:
 
     std::unordered_map<std::wstring, std::shared_ptr<APPLICATION_INFO>>      m_pApplicationInfoHash;
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
index 75f85aa28867..87772f175395 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
@@ -125,14 +125,25 @@ HRESULT
                                   moduleFactory.release(),
                                   RQ_EXECUTE_REQUEST_HANDLER,
                                   0));
-;
+
+    auto delay = applicationManager->GetShutdownDelay();
     auto pGlobalModule = std::make_unique<ASPNET_CORE_GLOBAL_MODULE>(std::move(applicationManager));
 
-    RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
-                                     pGlobalModule.release(),
-                                     GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-                                     GL_STOP_LISTENING | // worker process stop
-                                     GL_APPLICATION_STOP)); // app pool recycle
+    if (delay == std::chrono::milliseconds::zero())
+    {
+        RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
+            pGlobalModule.release(),
+            GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
+            GL_STOP_LISTENING)); // worker process stop
+    }
+    else
+    {
+        RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
+            pGlobalModule.release(),
+            GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
+            GL_STOP_LISTENING | // worker process stop
+            GL_APPLICATION_STOP)); // app pool recycle
+    }
 
     return S_OK;
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index 5aa04bf2bc6a..581a4fbbfa6d 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -6,7 +6,7 @@
 extern BOOL         g_fInShutdown;
 
 ASPNET_CORE_GLOBAL_MODULE::ASPNET_CORE_GLOBAL_MODULE(std::shared_ptr<APPLICATION_MANAGER> pApplicationManager) noexcept
-    :m_pApplicationManager(std::move(pApplicationManager))
+    : m_pApplicationManager(std::move(pApplicationManager))
 {
 }
 
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index 3fd3db40f1cc..0ba0e8abc0d6 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -57,18 +57,29 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
             return;
         }
 
-        // Run shutdown on a background thread. It seems like IIS keeps giving us requests if OnGlobalStopListening is still running
-        // which will result in 503s since we're shutting down.
-        // But if we return ASAP from OnGlobalStopListening (by not shutting down inline),
-        // IIS will actually stop giving us new requests and queue them instead for processing by the new app process.
-        m_shutdown = std::thread([this]()
-            {
-                auto delay = m_pApplicationManager->GetShutdownDelay();
-                LOG_INFOF(L"Shutdown starting in %d ms.", delay.count());
-                // Delay so that any incoming requests while we're returning from OnGlobalStopListening are allowed to be processed
-                std::this_thread::sleep_for(delay);
-                m_pApplicationManager->ShutDown();
-                m_pApplicationManager = nullptr;
-            });
+        // If delay is zero we can go back to the old behavior of calling shutdown inline
+        // this is primarily so that we have a way for users to revert the new behavior if there are issues with it
+        if (m_pApplicationManager->GetShutdownDelay() == std::chrono::milliseconds::zero())
+        {
+            LOG_INFO(L"Shutdown starting.");
+            m_pApplicationManager->ShutDown();
+            m_pApplicationManager = nullptr;
+        }
+        else
+        {
+            // Run shutdown on a background thread. It seems like IIS keeps giving us requests if OnGlobalStopListening is still running
+            // which will result in 503s since we're shutting down.
+            // But if we return ASAP from OnGlobalStopListening (by not shutting down inline),
+            // IIS will actually stop giving us new requests and queue them instead for processing by the new app process.
+            m_shutdown = std::thread([this]()
+                {
+                    auto delay = m_pApplicationManager->GetShutdownDelay();
+                    LOG_INFOF(L"Shutdown starting in %d ms.", delay.count());
+                    // Delay so that any incoming requests while we're returning from OnGlobalStopListening are allowed to be processed
+                    std::this_thread::sleep_for(delay);
+                    m_pApplicationManager->ShutDown();
+                    m_pApplicationManager = nullptr;
+                });
+        }
     }
 };

From 8f4070820c7cece305c1752cba56bcf54830875a Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Tue, 2 Apr 2024 15:48:42 -0700
Subject: [PATCH 06/11] testing

---
 .../AspNetCore/applicationmanager.cpp          | 13 ++++++++++---
 .../AspNetCoreModuleV2/AspNetCore/dllmain.cpp  | 18 +++++++++---------
 .../AspNetCore/globalmodule.cpp                |  5 +++++
 3 files changed, 24 insertions(+), 12 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
index 85641fdde185..1e17305deb14 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
@@ -149,9 +149,16 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             {
                 try
                 {
-                    // Recycle the process to trigger OnGlobalStopListening
-                    // which will shutdown the server and stop listening for new requests for this app
-                    m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
+                    //if (GetShutdownDelay() == std::chrono::milliseconds::zero())
+                    //{
+                    //    application->ShutDownApplication(/* fServerInitiated */ false);
+                    //}
+                    //else
+                    {
+                        // Recycle the process to trigger OnGlobalStopListening
+                        // which will shutdown the server and stop listening for new requests for this app
+                        m_pHttpServer.RecycleProcess(L"AspNetCore InProcess Recycle Process on Demand");
+                    }
                 }
                 catch (...)
                 {
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
index 87772f175395..2776f3eb3d98 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
@@ -126,17 +126,17 @@ HRESULT
                                   RQ_EXECUTE_REQUEST_HANDLER,
                                   0));
 
-    auto delay = applicationManager->GetShutdownDelay();
+    //auto delay = applicationManager->GetShutdownDelay();
     auto pGlobalModule = std::make_unique<ASPNET_CORE_GLOBAL_MODULE>(std::move(applicationManager));
 
-    if (delay == std::chrono::milliseconds::zero())
-    {
-        RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
-            pGlobalModule.release(),
-            GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-            GL_STOP_LISTENING)); // worker process stop
-    }
-    else
+    //if (delay == std::chrono::milliseconds::zero())
+    //{
+    //    RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
+    //        pGlobalModule.release(),
+    //        GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
+    //        GL_STOP_LISTENING)); // worker process stop
+    //}
+    //else
     {
         RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
             pGlobalModule.release(),
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index 581a4fbbfa6d..d943ed2dfc17 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -40,6 +40,11 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
     IN IHttpApplicationStopProvider* pProvider
 )
 {
+    if (!m_pApplicationManager || m_pApplicationManager->GetShutdownDelay() == std::chrono::milliseconds::zero())
+    {
+        return GL_NOTIFICATION_CONTINUE;
+    }
+
     UNREFERENCED_PARAMETER(pProvider);
     LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop");
 

From 9e10362a02296821dadc6c6d83513fb06ffe43cb Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Wed, 3 Apr 2024 09:51:31 -0700
Subject: [PATCH 07/11] cleanup

---
 .../AspNetCore/ShimOptions.cpp                |  2 +-
 .../AspNetCore/applicationmanager.cpp         | 36 ++++++++++++++++---
 .../Infrastructure/EventLogHelpers.cs         |  2 +-
 .../ApplicationInitializationTests.cs         | 25 ++++++++++---
 4 files changed, 54 insertions(+), 11 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
index d5580d7fad64..2a43ffd8cc2d 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
@@ -58,7 +58,7 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
     if (shutdownDelay.empty())
     {
-        m_fShutdownDelay = std::chrono::seconds(0);
+        m_fShutdownDelay = std::chrono::seconds(1);
     }
     else
     {
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
index 1e17305deb14..f15d76f78e06 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
@@ -149,11 +149,11 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             {
                 try
                 {
-                    //if (GetShutdownDelay() == std::chrono::milliseconds::zero())
-                    //{
-                    //    application->ShutDownApplication(/* fServerInitiated */ false);
-                    //}
-                    //else
+                    if (GetShutdownDelay() == std::chrono::milliseconds::zero())
+                    {
+                        application->ShutDownApplication(/* fServerInitiated */ false);
+                    }
+                    else
                     {
                         // Recycle the process to trigger OnGlobalStopListening
                         // which will shutdown the server and stop listening for new requests for this app
@@ -179,6 +179,32 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
                 }
             }
         }
+
+        if (GetShutdownDelay() == std::chrono::milliseconds::zero())
+        {
+            // Remove apps after calling shutdown on each of them
+            // This is exclusive to in-process, as the shutdown of an in-process app recycles
+            // the entire worker process.
+            if (m_handlerResolver.GetHostingModel() == APP_HOSTING_MODEL::HOSTING_IN_PROCESS)
+            {
+                SRWExclusiveLock lock(m_srwLock);
+                const std::wstring configurationPath = pszApplicationId;
+
+                auto itr = m_pApplicationInfoHash.begin();
+                while (itr != m_pApplicationInfoHash.end())
+                {
+                    if (itr->second != nullptr && itr->second->ConfigurationPathApplies(configurationPath)
+                        && std::find(applicationsToRecycle.begin(), applicationsToRecycle.end(), itr->second) != applicationsToRecycle.end())
+                    {
+                        itr = m_pApplicationInfoHash.erase(itr);
+                    }
+                    else
+                    {
+                        ++itr;
+                    }
+                }
+            } // Release Exclusive m_srwLock
+        }
     }
     CATCH_RETURN()
 
diff --git a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
index ca61359c92ce..9e08448d969b 100644
--- a/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
+++ b/src/Servers/IIS/IIS/test/Common.FunctionalTests/Infrastructure/EventLogHelpers.cs
@@ -79,7 +79,7 @@ private static string FormatEntries(IEnumerable<EventLogEntry> entries)
         return string.Join(",", entries.Select(e => e.Message));
     }
 
-    private static IEnumerable<EventLogEntry> GetEntries(IISDeploymentResult deploymentResult)
+    internal static IEnumerable<EventLogEntry> GetEntries(IISDeploymentResult deploymentResult)
     {
         var eventLog = new EventLog("Application");
 
diff --git a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
index 04833b0eb043..ba407def5bcf 100644
--- a/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
+++ b/src/Servers/IIS/IIS/test/IIS.Shared.FunctionalTests/ApplicationInitializationTests.cs
@@ -4,6 +4,7 @@
 using System;
 using System.IO;
 using System.ServiceProcess;
+using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
 using Microsoft.AspNetCore.Server.IIS.FunctionalTests.Utilities;
@@ -37,9 +38,11 @@ public ApplicationInitializationTests(PublishedSitesFixture fixture) : base(fixt
 
     [ConditionalTheory]
     [RequiresIIS(IISCapability.ApplicationInitialization)]
-    [InlineData(HostingModel.InProcess)]
-    [InlineData(HostingModel.OutOfProcess)]
-    public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
+    [InlineData(HostingModel.InProcess, true)]
+    [InlineData(HostingModel.OutOfProcess, true)]
+    [InlineData(HostingModel.InProcess, false)]
+    [InlineData(HostingModel.OutOfProcess, false)]
+    public async Task ApplicationPreloadStartsApp(HostingModel hostingModel, bool delayShutdown)
     {
         // This test often hits a memory leak in warmup.dll module, it has been reported to IIS team
         using (AppVerifier.Disable(DeployerSelector.ServerType, 0x900))
@@ -49,12 +52,26 @@ public async Task ApplicationPreloadStartsApp(HostingModel hostingModel)
                 (args, contentRoot) => $"{args} CreateFile \"{Path.Combine(contentRoot, "Started.txt")}\"");
             EnablePreload(baseDeploymentParameters);
 
+            baseDeploymentParameters.HandlerSettings["shutdownDelay"] = delayShutdown ? "1000" : "0";
             var result = await DeployAsync(baseDeploymentParameters);
 
             await Helpers.Retry(async () => await File.ReadAllTextAsync(Path.Combine(result.ContentRoot, "Started.txt")), TimeoutExtensions.DefaultTimeoutValue);
             StopServer();
             EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.Started(result), Logger);
-            EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger);
+
+            if (delayShutdown)
+            {
+                EventLogHelpers.VerifyEventLogEvent(result, EventLogHelpers.ShutdownMessage(result), Logger);
+            }
+            else
+            {
+                Assert.True(result.HostProcess.HasExited);
+
+                var entries = EventLogHelpers.GetEntries(result);
+                var expectedRegex = new Regex(EventLogHelpers.ShutdownMessage(result), RegexOptions.Singleline);
+                var matchedEntries = entries.Where(entry => expectedRegex.IsMatch(entry.Message)).ToArray();
+                Assert.Empty(matchedEntries);
+            }
         }
     }
 

From 8c5328771b981a3ad0228cc71d84450751112425 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Fri, 12 Apr 2024 11:33:51 -0700
Subject: [PATCH 08/11] env var

---
 .../AspNetCore/ShimOptions.cpp                | 41 ++++++++++++++-----
 .../AspNetCore/ShimOptions.h                  |  2 +
 .../AspNetCoreModuleV2/AspNetCore/dllmain.cpp | 21 +++-------
 .../AspNetCore/globalmodule.cpp               |  5 ++-
 .../AspNetCore/globalmodule.h                 |  6 ++-
 .../AspNetCore/proxymodule.cpp                |  2 +-
 .../test/Common.LongTests/ShutdownTests.cs    |  2 -
 7 files changed, 46 insertions(+), 33 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
index 2a43ffd8cc2d..0be2bf0f0e54 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
@@ -13,6 +13,7 @@
 #define CS_ASPNETCORE_CLEAN_SHADOW_DIRECTORY_CONTENT     L"cleanShadowCopyDirectory"
 #define CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG             L"disallowRotationOnConfigChange"
 #define CS_ASPNETCORE_SHUTDOWN_DELAY                     L"shutdownDelay"
+#define CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV                 L"ANCM_shutdownDelay"
 
 ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
         m_hostingModel(HOSTING_UNKNOWN),
@@ -55,17 +56,6 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     auto disallowRotationOnConfigChange = find_element(handlerSettings, CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG).value_or(std::wstring());
     m_fDisallowRotationOnConfigChange = equals_ignore_case(L"true", disallowRotationOnConfigChange);
 
-    auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
-    if (shutdownDelay.empty())
-    {
-        m_fShutdownDelay = std::chrono::seconds(1);
-    }
-    else
-    {
-        auto str = to_multi_byte_string(shutdownDelay, CP_UTF8);
-        m_fShutdownDelay = std::chrono::milliseconds(std::atoi(str.c_str()));
-    }
-                
     m_strProcessPath = section->GetRequiredString(CS_ASPNETCORE_PROCESS_EXE_PATH);
     m_strArguments = section->GetString(CS_ASPNETCORE_PROCESS_ARGUMENTS).value_or(CS_ASPNETCORE_PROCESS_ARGUMENTS_DEFAULT);
     m_fStdoutLogEnabled = section->GetRequiredBool(CS_ASPNETCORE_STDOUT_LOG_ENABLED);
@@ -94,4 +84,33 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     auto dotnetEnvironmentEnabled = equals_ignore_case(L"Development", dotnetEnvironment);
 
     m_fShowDetailedErrors = detailedErrorsEnabled || aspnetCoreEnvironmentEnabled || dotnetEnvironmentEnabled;
+
+    // Specifies how long to delay (in milliseconds) after IIS tells us to stop before starting the application shutdown.
+    // See StartShutdown in globalmodule to see how it's used.
+    auto shutdownDelay = find_element(handlerSettings, CS_ASPNETCORE_SHUTDOWN_DELAY).value_or(std::wstring());
+    if (shutdownDelay.empty())
+    {
+        // Fallback to environment variable if process specific config wasn't set
+        shutdownDelay = Environment::GetEnvironmentVariableValue(CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV)
+            .value_or(environmentVariables[CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV]);
+        if (shutdownDelay.empty())
+        {
+            // Default if neither process specific config or environment variable aren't set
+            m_fShutdownDelay = std::chrono::seconds(1);
+        }
+        else
+        {
+            SetShutdownDelay(shutdownDelay);
+        }
+    }
+    else
+    {
+        SetShutdownDelay(shutdownDelay);
+    }
+}
+
+void ShimOptions::SetShutdownDelay(std::wstring& shutdownDelay)
+{
+    auto str = to_multi_byte_string(shutdownDelay, CP_UTF8);
+    m_fShutdownDelay = std::chrono::milliseconds(std::atoi(str.c_str()));
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
index 33a66b54bb81..df42e11b84b9 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
@@ -111,4 +111,6 @@ class ShimOptions: NonCopyable
     bool                           m_fDisallowRotationOnConfigChange;
     std::wstring                   m_strShadowCopyingDirectory;
     std::chrono::milliseconds      m_fShutdownDelay;
+
+    void SetShutdownDelay(std::wstring& shutdownDelay);
 };
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
index 2776f3eb3d98..4d55e36b80d9 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/dllmain.cpp
@@ -126,24 +126,13 @@ HRESULT
                                   RQ_EXECUTE_REQUEST_HANDLER,
                                   0));
 
-    //auto delay = applicationManager->GetShutdownDelay();
     auto pGlobalModule = std::make_unique<ASPNET_CORE_GLOBAL_MODULE>(std::move(applicationManager));
 
-    //if (delay == std::chrono::milliseconds::zero())
-    //{
-    //    RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
-    //        pGlobalModule.release(),
-    //        GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-    //        GL_STOP_LISTENING)); // worker process stop
-    //}
-    //else
-    {
-        RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
-            pGlobalModule.release(),
-            GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
-            GL_STOP_LISTENING | // worker process stop
-            GL_APPLICATION_STOP)); // app pool recycle
-    }
+    RETURN_IF_FAILED(pModuleInfo->SetGlobalNotifications(
+        pGlobalModule.release(),
+        GL_CONFIGURATION_CHANGE | // Configuration change triggers IIS application stop
+        GL_STOP_LISTENING | // worker process will stop listening for http requests
+        GL_APPLICATION_STOP)); // app pool recycle or stop
 
     return S_OK;
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index d943ed2dfc17..de8e56096cd7 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -40,12 +40,15 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
     IN IHttpApplicationStopProvider* pProvider
 )
 {
+    UNREFERENCED_PARAMETER(pProvider);
+
+    // If we're already cleaned up just return.
+    // If user has opted out of the new shutdown behavior ignore this call as we never registered for it before
     if (!m_pApplicationManager || m_pApplicationManager->GetShutdownDelay() == std::chrono::milliseconds::zero())
     {
         return GL_NOTIFICATION_CONTINUE;
     }
 
-    UNREFERENCED_PARAMETER(pProvider);
     LOG_INFO(L"ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop");
 
     if (!g_fInShutdown && !m_shutdown.joinable())
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index 0ba0e8abc0d6..294630fff364 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -68,8 +68,8 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
         else
         {
             // Run shutdown on a background thread. It seems like IIS keeps giving us requests if OnGlobalStopListening is still running
-            // which will result in 503s since we're shutting down.
-            // But if we return ASAP from OnGlobalStopListening (by not shutting down inline),
+            // which will result in 503s from applicationmanager since we're shutting down and don't want to process new requests.
+            // But if we return ASAP from OnGlobalStopListening, by not shutting down inline and with a small delay to reduce races,
             // IIS will actually stop giving us new requests and queue them instead for processing by the new app process.
             m_shutdown = std::thread([this]()
                 {
@@ -77,6 +77,8 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
                     LOG_INFOF(L"Shutdown starting in %d ms.", delay.count());
                     // Delay so that any incoming requests while we're returning from OnGlobalStopListening are allowed to be processed
                     std::this_thread::sleep_for(delay);
+
+                    LOG_INFO(L"Shutdown starting.");
                     m_pApplicationManager->ShutDown();
                     m_pApplicationManager = nullptr;
                 });
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
index 5a752b66442e..e4292be76b42 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
@@ -93,7 +93,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
     {
         if (g_fInShutdown)
         {
-            LOG_INFO(L"Received request during shutdown. Will return 503.");
+            LOG_INFO(L"Received a request during shutdown. Will return a 503 response.");
             FINISHED(HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS));
         }
 
diff --git a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
index 77bfa31d379f..5d82142bcddd 100644
--- a/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
+++ b/src/Servers/IIS/IIS/test/Common.LongTests/ShutdownTests.cs
@@ -262,7 +262,6 @@ await statusConnection.Receive("5",
 
     [ConditionalFact]
     [RequiresNewShim]
-    [Repeat(10)] // temp for CI testing
     public async Task RequestsWhileRestartingAppFromConfigChangeAreProcessed()
     {
         var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite);
@@ -300,7 +299,6 @@ public async Task RequestsWhileRestartingAppFromConfigChangeAreProcessed()
 
     [ConditionalFact]
     [RequiresNewShim]
-    [Repeat(10)] // temp for CI testing
     public async Task RequestsWhileRecyclingAppAreProcessed()
     {
         var deploymentParameters = Fixture.GetBaseDeploymentParameters(Fixture.InProcessTestSite);

From f25ca056f5a740406fbc35c642d365934a2c35c7 Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Mon, 15 Apr 2024 14:29:22 -0700
Subject: [PATCH 09/11] fb

---
 .../AspNetCore/ShimOptions.cpp                  | 17 +++++++++++------
 .../AspNetCoreModuleV2/AspNetCore/ShimOptions.h |  4 ++--
 .../AspNetCore/applicationmanager.cpp           |  4 ++--
 .../AspNetCore/applicationmanager.h             |  5 +++++
 .../AspNetCore/globalmodule.cpp                 |  2 +-
 .../AspNetCore/globalmodule.h                   |  2 +-
 .../AspNetCore/proxymodule.cpp                  |  2 +-
 .../InProcessApplicationBase.cpp                |  2 +-
 8 files changed, 24 insertions(+), 14 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
index 0be2bf0f0e54..88865cd9b132 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.cpp
@@ -13,7 +13,7 @@
 #define CS_ASPNETCORE_CLEAN_SHADOW_DIRECTORY_CONTENT     L"cleanShadowCopyDirectory"
 #define CS_ASPNETCORE_DISALLOW_ROTATE_CONFIG             L"disallowRotationOnConfigChange"
 #define CS_ASPNETCORE_SHUTDOWN_DELAY                     L"shutdownDelay"
-#define CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV                 L"ANCM_shutdownDelay"
+#define CS_ASPNETCORE_SHUTDOWN_DELAY_ENV                 L"ANCM_shutdownDelay"
 
 ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
         m_hostingModel(HOSTING_UNKNOWN),
@@ -91,8 +91,8 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     if (shutdownDelay.empty())
     {
         // Fallback to environment variable if process specific config wasn't set
-        shutdownDelay = Environment::GetEnvironmentVariableValue(CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV)
-            .value_or(environmentVariables[CS_ASPNEtCORE_SHUTDOWN_DELAY_ENV]);
+        shutdownDelay = Environment::GetEnvironmentVariableValue(CS_ASPNETCORE_SHUTDOWN_DELAY_ENV)
+            .value_or(environmentVariables[CS_ASPNETCORE_SHUTDOWN_DELAY_ENV]);
         if (shutdownDelay.empty())
         {
             // Default if neither process specific config or environment variable aren't set
@@ -109,8 +109,13 @@ ShimOptions::ShimOptions(const ConfigurationSource &configurationSource) :
     }
 }
 
-void ShimOptions::SetShutdownDelay(std::wstring& shutdownDelay)
+void ShimOptions::SetShutdownDelay(const std::wstring& shutdownDelay)
 {
-    auto str = to_multi_byte_string(shutdownDelay, CP_UTF8);
-    m_fShutdownDelay = std::chrono::milliseconds(std::atoi(str.c_str()));
+    auto millsecondsValue = std::stoi(shutdownDelay);
+    if (millsecondsValue < 0)
+    {
+        throw ConfigurationLoadException(format(
+            L"'shutdownDelay' in web.config or '%s' environment variable is less than 0.", CS_ASPNETCORE_SHUTDOWN_DELAY_ENV));
+    }
+    m_fShutdownDelay = std::chrono::milliseconds(millsecondsValue);
 }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
index df42e11b84b9..4e13190be6dd 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/ShimOptions.h
@@ -90,7 +90,7 @@ class ShimOptions: NonCopyable
     }
 
     std::chrono::milliseconds
-        QueryShutdownDelay() const noexcept
+    QueryShutdownDelay() const noexcept
     {
         return m_fShutdownDelay;
     }
@@ -112,5 +112,5 @@ class ShimOptions: NonCopyable
     std::wstring                   m_strShadowCopyingDirectory;
     std::chrono::milliseconds      m_fShutdownDelay;
 
-    void SetShutdownDelay(std::wstring& shutdownDelay);
+    void SetShutdownDelay(const std::wstring& shutdownDelay);
 };
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
index f15d76f78e06..48855946d29a 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.cpp
@@ -149,7 +149,7 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             {
                 try
                 {
-                    if (GetShutdownDelay() == std::chrono::milliseconds::zero())
+                    if (UseLegacyShutdown())
                     {
                         application->ShutDownApplication(/* fServerInitiated */ false);
                     }
@@ -180,7 +180,7 @@ APPLICATION_MANAGER::RecycleApplicationFromManager(
             }
         }
 
-        if (GetShutdownDelay() == std::chrono::milliseconds::zero())
+        if (UseLegacyShutdown())
         {
             // Remove apps after calling shutdown on each of them
             // This is exclusive to in-process, as the shutdown of an in-process app recycles
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
index bccc1d704c83..efc466dc7ca9 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/applicationmanager.h
@@ -52,6 +52,11 @@ class APPLICATION_MANAGER
         return m_handlerResolver.GetShutdownDelay();
     }
 
+    bool UseLegacyShutdown() const
+    {
+        return m_handlerResolver.GetShutdownDelay() == std::chrono::milliseconds::zero();
+    }
+
 private:
 
     std::unordered_map<std::wstring, std::shared_ptr<APPLICATION_INFO>>      m_pApplicationInfoHash;
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
index de8e56096cd7..9e69d586cb80 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.cpp
@@ -44,7 +44,7 @@ ASPNET_CORE_GLOBAL_MODULE::OnGlobalApplicationStop(
 
     // If we're already cleaned up just return.
     // If user has opted out of the new shutdown behavior ignore this call as we never registered for it before
-    if (!m_pApplicationManager || m_pApplicationManager->GetShutdownDelay() == std::chrono::milliseconds::zero())
+    if (!m_pApplicationManager || m_pApplicationManager->UseLegacyShutdown())
     {
         return GL_NOTIFICATION_CONTINUE;
     }
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index 294630fff364..e74a8ec2d66b 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -59,7 +59,7 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
 
         // If delay is zero we can go back to the old behavior of calling shutdown inline
         // this is primarily so that we have a way for users to revert the new behavior if there are issues with it
-        if (m_pApplicationManager->GetShutdownDelay() == std::chrono::milliseconds::zero())
+        if (m_pApplicationManager->UseLegacyShutdown())
         {
             LOG_INFO(L"Shutdown starting.");
             m_pApplicationManager->ShutDown();
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
index e4292be76b42..99dd210b3dd1 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/proxymodule.cpp
@@ -93,7 +93,7 @@ ASPNET_CORE_PROXY_MODULE::OnExecuteRequestHandler(
     {
         if (g_fInShutdown)
         {
-            LOG_INFO(L"Received a request during shutdown. Will return a 503 response.");
+            LOG_WARN(L"Received a request during shutdown. Will return a 503 response.");
             FINISHED(HRESULT_FROM_WIN32(ERROR_SERVER_SHUTDOWN_IN_PROGRESS));
         }
 
diff --git a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
index e8bee5a84f99..5282792f1e6f 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
+++ b/src/Servers/IIS/AspNetCoreModuleV2/InProcessRequestHandler/InProcessApplicationBase.cpp
@@ -40,7 +40,7 @@ InProcessApplicationBase::StopInternal(bool fServerInitiated)
         return;
     }
 
-    // Stop was initiated by server no need to do anything, server would stop on it's own
+    // Stop was initiated by server no need to do anything, server would stop on its own
     if (fServerInitiated)
     {
         return;

From 74160d33f1fe4e6b4796bf99d29ce142764c05ff Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Mon, 15 Apr 2024 19:08:29 -0700
Subject: [PATCH 10/11] fb

---
 src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index e74a8ec2d66b..65b9168d4b65 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -51,8 +51,8 @@ class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
 
     void StartShutdown()
     {
-        // Shutdown has already been started
-        if (m_shutdown.joinable())
+        // Shutdown has already been started/finished
+        if (m_shutdown.joinable() || g_fInShutdown)
         {
             return;
         }

From bd91533c075a8924dc617f61eac16d03bc13e37a Mon Sep 17 00:00:00 2001
From: Brennan <brecon@microsoft.com>
Date: Mon, 15 Apr 2024 21:14:04 -0700
Subject: [PATCH 11/11] compile

---
 src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
index 65b9168d4b65..3bcb30c0b2e8 100644
--- a/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
+++ b/src/Servers/IIS/AspNetCoreModuleV2/AspNetCore/globalmodule.h
@@ -6,6 +6,8 @@
 #include "applicationmanager.h"
 #include <thread>
 
+extern BOOL         g_fInShutdown;
+
 class ASPNET_CORE_GLOBAL_MODULE : NonCopyable, public CGlobalModule
 {