diff --git a/src/Microsoft.AspNetCore.Hosting/WebHostBuilder.cs b/src/Microsoft.AspNetCore.Hosting/WebHostBuilder.cs
index 5bd5ba6a..7774de93 100644
--- a/src/Microsoft.AspNetCore.Hosting/WebHostBuilder.cs
+++ b/src/Microsoft.AspNetCore.Hosting/WebHostBuilder.cs
@@ -285,12 +285,6 @@ private IServiceCollection BuildCommonServices(out AggregateException hostingSta
                 if (exceptions.Count > 0)
                 {
                     hostingStartupErrors = new AggregateException(exceptions);
-
-                    // Throw directly if we're not capturing startup errors
-                    if (!_options.CaptureStartupErrors)
-                    {
-                        throw hostingStartupErrors;
-                    }
                 }
             }
 
diff --git a/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs b/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
index 1e51be38..be0e86c5 100644
--- a/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
+++ b/test/Microsoft.AspNetCore.Hosting.Tests/WebHostBuilderTests.cs
@@ -975,7 +975,7 @@ public void Build_HostingStartupFromPrimaryAssemblyCanBeDisabled()
         }
 
         [Fact]
-        public void Build_ThrowsIfUnloadableAssemblyNameInHostingStartupAssemblies()
+        public void Build_DoesntThrowIfUnloadableAssemblyNameInHostingStartupAssemblies()
         {
             var builder = CreateWebHostBuilder()
                 .CaptureStartupErrors(false)
@@ -983,9 +983,10 @@ public void Build_ThrowsIfUnloadableAssemblyNameInHostingStartupAssemblies()
                 .Configure(app => { })
                 .UseServer(new TestServer());
 
-            var ex = Assert.Throws<AggregateException>(() => builder.Build());
-            Assert.IsType<InvalidOperationException>(ex.InnerExceptions[0]);
-            Assert.IsType<FileNotFoundException>(ex.InnerExceptions[0].InnerException);
+            using (builder.Build())
+            {
+                Assert.Equal("0", builder.GetSetting("testhostingstartup"));
+            }
         }
 
         [Fact]