Skip to content

Update links and disable warnings related to tests that don't work in Helix #10427

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 12 commits into from
May 21, 2019
Merged
2 changes: 1 addition & 1 deletion docs/Helix.md
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,6 @@ Most tests that don't just work on helix automatically are ones that depend on t
## How to skip tests on helix
There are two main ways to opt out of helix
- Skipping the entire test project via `<BuildHelixPayload>false</BuildHelixPayload>` in csproj (the default value for this is IsTestProject).
- Skipping an individual test via `[SkipOnHelix]` which might require including a compile reference to: `<Compile Include="$(SharedSourceRoot)test\SkipOnHelixAttribute.cs" />`
- Skipping an individual test via `[SkipOnHelix("url to github issue")]` which might require including a compile reference to: `<Compile Include="$(SharedSourceRoot)test\SkipOnHelixAttribute.cs" />`

Make sure to file an issue for any skipped tests and include that in a comment next to either of these
4 changes: 4 additions & 0 deletions src/Analyzers/Analyzers/test/AnalyzerTestBase.cs
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,11 @@ private static string GetProjectDirectory()
return AppContext.BaseDirectory;
}

// This test code needs to be updated to support distributed testing.
// See https://github.com/aspnet/AspNetCore/issues/10422
#pragma warning disable 0618
var solutionDirectory = TestPathUtilities.GetSolutionRootDirectory("Analyzers");
#pragma warning restore 0618
var projectDirectory = Path.Combine(solutionDirectory, "Analyzers", "test");
return projectDirectory;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<RootNamespace>Microsoft.AspNetCore.Analyzers</RootNamespace>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/10422 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ public RuntimeDependenciesResolverTest(ITestOutputHelper output)
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6549
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/10426")]
public void FindsReferenceAssemblyGraph_ForStandaloneApp()
{
// Arrange
Expand Down
2 changes: 1 addition & 1 deletion src/Components/Components/test/RendererTest.cs
Original file line number Diff line number Diff line change
Expand Up @@ -2577,7 +2577,7 @@ public void CanTriggerRenderingSynchronouslyFromInsideAfterRenderCallback()
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7487
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/7487")]
public async Task CanTriggerEventHandlerDisposedInEarlierPendingBatchAsync()
{
// This represents the scenario where the same event handler is being triggered
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -117,7 +117,7 @@ public void System_UsesProvidedDirectory_WithConfigurationCallback()
[ConditionalFact]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2177", FlakyOn.AzP.Windows)]
[X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public void System_UsesProvidedDirectoryAndCertificate()
{
var filePath = Path.Combine(GetTestFilesPath(), "TestCert.pfx");
Expand Down Expand Up @@ -167,7 +167,7 @@ public void System_UsesProvidedDirectoryAndCertificate()

[ConditionalFact]
[X509StoreIsAvailable(StoreName.My, StoreLocation.CurrentUser)]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public void System_UsesProvidedCertificateNotFromStore()
{
using (var store = new X509Store(StoreName.My, StoreLocation.CurrentUser))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<TargetFramework>netcoreapp3.0</TargetFramework>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/10424 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
4 changes: 2 additions & 2 deletions src/Hosting/Hosting/test/WebHostTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,7 @@ public async Task WebHostShutsDownWhenTokenTriggers()
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/7291")]
public async Task WebHostStopAsyncUsesDefaultTimeoutIfGivenTokenDoesNotFire()
{
var data = new Dictionary<string, string>
Expand Down Expand Up @@ -315,7 +315,7 @@ public async Task WebHostStopAsyncCanBeCancelledEarly()
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7291
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/7291")]
public void WebHostApplicationLifetimeEventsOrderedCorrectlyDuringShutdown()
{
using (var host = CreateBuilder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@
<TargetFramework>netcoreapp3.0</TargetFramework>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/8247 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
3 changes: 3 additions & 0 deletions src/Hosting/test/FunctionalTests/ShutdownTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,8 +49,11 @@ private async Task ExecuteShutdownTest(string testName, string shutdownMechanic)
{
var logger = loggerFactory.CreateLogger(testName);

// https://github.com/aspnet/AspNetCore/issues/8247
#pragma warning disable 0618
var applicationPath = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("Hosting"), "test", "testassets",
"Microsoft.AspNetCore.Hosting.TestSites");
#pragma warning restore 0618

var deploymentParameters = new DeploymentParameters(
applicationPath,
Expand Down
3 changes: 3 additions & 0 deletions src/Hosting/test/FunctionalTests/WebHostBuilderTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,10 @@ public async Task InjectedStartup_DefaultApplicationNameIsEntryAssembly(TestVari
{
var logger = loggerFactory.CreateLogger(nameof(InjectedStartup_DefaultApplicationNameIsEntryAssembly));

// https://github.com/aspnet/AspNetCore/issues/8247
#pragma warning disable 0618
var applicationPath = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("Hosting"), "test", "testassets", "IStartupInjectionAssemblyName");
#pragma warning restore 0618

var deploymentParameters = new DeploymentParameters(variant)
{
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,8 @@ public class ConfigureSigningCredentialsTests
UnsafeEphemeralKeySet : (RuntimeInformation.IsOSPlatform(OSPlatform.OSX) ? X509KeyStorageFlags.PersistKeySet :
X509KeyStorageFlags.DefaultKeySet);

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[ConditionalFact]
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
[FrameworkSkipCondition(RuntimeFrameworks.CLR)]
public void Configure_AddsDevelopmentKeyFromConfiguration()
{
Expand Down Expand Up @@ -64,7 +64,7 @@ public void Configure_AddsDevelopmentKeyFromConfiguration()
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public void Configure_LoadsPfxCertificateCredentialFromConfiguration()
{
// Arrange
Expand Down Expand Up @@ -94,7 +94,7 @@ public void Configure_LoadsPfxCertificateCredentialFromConfiguration()
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public void Configure_LoadsCertificateStoreCertificateCredentialFromConfiguration()
{
try
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ public class SigningKeysLoaderTests
[Fact]
public void LoadFromFile_ThrowsIfFileDoesNotExist()
{
// Arrange, Act & Assert
// Arrange, Act & Assert
var exception = Assert.Throws<InvalidOperationException>(() => SigningKeysLoader.LoadFromFile("./nonexisting.pfx", "", DefaultFlags));
Assert.Equal($"There was an error loading the certificate. The file './nonexisting.pfx' was not found.", exception.Message);
}
Expand Down Expand Up @@ -58,8 +58,8 @@ public static void LoadFromStoreCert_ThrowsIfThereIsNoCertificateAvailable()
Assert.Equal("Couldn't find a valid certificate with subject 'Invalid' on the 'CurrentUser\\My'", exception.Message);
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[ConditionalFact]
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public static void LoadFromStoreCert_SkipsCertificatesNotYetValid()
{
try
Expand All @@ -81,8 +81,8 @@ public static void LoadFromStoreCert_SkipsCertificatesNotYetValid()
}
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[ConditionalFact]
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public static void LoadFromStoreCert_PrefersCertificatesCloserToExpirationDate()
{
try
Expand All @@ -104,8 +104,8 @@ public static void LoadFromStoreCert_PrefersCertificatesCloserToExpirationDate()
}
}

[ConditionalFact]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/6720
[ConditionalFact]
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/6720")]
public static void LoadFromStoreCert_SkipsExpiredCertificates()
{
try
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<DefaultItemExcludes>$(DefaultItemExcludes);node_modules\**\*</DefaultItemExcludes>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/7990 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -68,7 +68,10 @@ public async Task RunClientTests(string startup)

private static async Task<SamplesDeploymentResult> CreateDeployments(ILoggerFactory loggerFactory, string startup)
{
// https://github.com/aspnet/AspNetCore/issues/7990
#pragma warning disable 0618
var solutionPath = TestPathUtilities.GetSolutionRootDirectory("Middleware");
#pragma warning restore 0618

var configuration =
#if RELEASE
Expand Down
3 changes: 3 additions & 0 deletions src/MusicStore/test/MusicStore.E2ETests/Common/Helpers.cs
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,10 @@ public class Helpers
{
public static string GetApplicationPath()
{
// https://github.com/aspnet/AspNetCore/issues/8343
#pragma warning disable 0618
var solutionDirectory = TestPathUtilities.GetSolutionRootDirectory("MusicStore");
#pragma warning restore 0618
return Path.GetFullPath(Path.Combine(solutionDirectory, "samples", "MusicStore"));
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<WarningsNotAsErrors>$(WarningsNotAsErrors);xUnit1004</WarningsNotAsErrors>
<!-- Temporarily suppress warnings about Microsoft.AspNetCore.Server.IntegrationTesting -->
<NoWarn>$(NoWarn);NU1605</NoWarn>
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/8343 -->
<BuildHelixPayload>false</BuildHelixPayload>
<IsTestProject>false</IsTestProject>
<IsReferenceAssemblyProject>false</IsReferenceAssemblyProject>
Expand Down
5 changes: 4 additions & 1 deletion src/Mvc/Mvc.Analyzers/test/Infrastructure/MvcTestSource.cs
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ public static TestSource Read(string testClassName, string testMethod)

private static string GetProjectDirectory()
{
// On helix we use the published test files
// On helix we use the published test files
if (SkipOnHelixAttribute.OnHelix())
{
return AppContext.BaseDirectory;
}

// https://github.com/aspnet/AspNetCore/issues/9431
#pragma warning disable 0618
var solutionDirectory = TestPathUtilities.GetSolutionRootDirectory("Mvc");
#pragma warning restore 0618
var projectDirectory = Path.Combine(solutionDirectory, "Mvc.Analyzers", "test");
return projectDirectory;
}
Expand Down
2 changes: 1 addition & 1 deletion src/Mvc/Mvc.Analyzers/test/Mvc.Analyzers.Test.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
<RootNamespace>Microsoft.AspNetCore.Mvc.Analyzers</RootNamespace>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/9431 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,16 @@ public static TestSource Read(string testClassName, string testMethod)

private static string GetProjectDirectory()
{
// On helix we use the published test files
// On helix we use the published test files
if (SkipOnHelixAttribute.OnHelix())
{
return AppContext.BaseDirectory;
}

// https://github.com/aspnet/AspNetCore/issues/9431
#pragma warning disable 0618
var solutionDirectory = TestPathUtilities.GetSolutionRootDirectory("Mvc");
#pragma warning restore 0618
var projectDirectory = Path.Combine(solutionDirectory, "Mvc.Api.Analyzers", "test");
return projectDirectory;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
<RootNamespace>Microsoft.AspNetCore.Mvc.Api.Analyzers</RootNamespace>

<!-- Tests do not work on Helix or when bin/ directory is not in project directory due to undeclared dependency on test content. -->
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/9431 -->
<BuildHelixPayload>false</BuildHelixPayload>
<BaseOutputPath />
<OutputPath />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,11 @@ public static void UpdateFile(Assembly assembly, string resourceName, string pre
{
// The build system compiles every file under the resources folder as a resource available at runtime
// with the same name as the file name. Need to update this file on disc.

// https://github.com/aspnet/AspNetCore/issues/10423
#pragma warning disable 0618
var solutionPath = TestPathUtilities.GetSolutionRootDirectory("Mvc");
#pragma warning restore 0618
var projectPath = Path.Combine(solutionPath, "test", assembly.GetName().Name);
var fullPath = Path.Combine(projectPath, resourceName);
WriteFile(fullPath, content);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<TestGroupName>Mvc.FunctionalTests</TestGroupName>
<!-- The test asset projects this depends on are not strong-named. -->
<SignAssembly>false</SignAssembly>
<!-- https://github.com/aspnet/AspNetCore/issues/6549 -->
<!-- https://github.com/aspnet/AspNetCore/issues/10423 -->
<BuildHelixPayload>false</BuildHelixPayload>
</PropertyGroup>

Expand Down
2 changes: 1 addition & 1 deletion src/Security/Authentication/test/SecureDataFormatTests.cs
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public void ProtectWithPurposeRoundTrips()
}

[ConditionalFact]
[SkipOnHelix]
[SkipOnHelix("https://github.com/aspnet/AspNetCore-Internal/issues/1974")]
public void UnprotectWithDifferentPurposeFails()
{
var provider = ServiceProvider.GetRequiredService<IDataProtectionProvider>();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,11 +20,15 @@ public class StartupTimeBenchmark
[IterationSetup]
public void Setup()
{
// Deployers do not work in distributed environments
// see https://github.com/aspnet/AspNetCore/issues/10268 and https://github.com/aspnet/Extensions/issues/1697
#pragma warning disable 0618
var deploymentParameters = new DeploymentParameters(Path.Combine(TestPathUtilities.GetSolutionRootDirectory("IISIntegration"), "test/testassets/InProcessWebSite"),
ServerType.IISExpress,
RuntimeFlavor.CoreClr,
RuntimeArchitecture.x64)
{
#pragma warning restore 0618
ServerConfigTemplateContent = File.ReadAllText("IISExpress.config"),
SiteName = "HttpTestSite",
TargetFramework = "netcoreapp2.1",
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,8 +113,7 @@ public async Task StartsWithDotnetOnThePath(string path)
[SkipIfNotAdmin]
[RequiresNewShim]
[RequiresIIS(IISCapability.PoolEnvironmentVariables)]
[SkipOnHelix]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2221", FlakyOn.Helix.All)]
[SkipOnHelix("https://github.com/aspnet/AspNetCore-Internal/issues/2221")]
public async Task StartsWithDotnetInstallLocation(RuntimeArchitecture runtimeArchitecture)
{
var deploymentParameters = Fixture.GetBaseDeploymentParameters();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,11 @@ public override Task<PublishedApplication> Publish(DeploymentParameters deployme

private string GetProjectReferencePublishLocation(DeploymentParameters deploymentParameters)
{
// Deployers do not work in distributed environments
// see https://github.com/aspnet/AspNetCore/issues/10268 and https://github.com/aspnet/Extensions/issues/1697
#pragma warning disable 0618
var testAssetsBasePath = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("IISIntegration"), "IIS", "test", "testassets", _applicationName);
#pragma warning restore 0618
var configuration = this.GetType().GetTypeInfo().Assembly.GetCustomAttribute<AssemblyConfigurationAttribute>().Configuration;
var path = Path.Combine(testAssetsBasePath, "bin", configuration, deploymentParameters.TargetFramework, "publish", GetProfileName(deploymentParameters));
return path;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,14 @@ public class MofFileTests
[ConditionalFact]
[OSSkipCondition(OperatingSystems.Linux | OperatingSystems.MacOSX)]
[RequiresIIS(IISCapability.TracingModule)]
[Flaky("https://github.com/aspnet/AspNetCore-Internal/issues/2222", FlakyOn.Helix.All)]
[SkipOnHelix]
[SkipOnHelix("https://github.com/aspnet/AspNetCore-Internal/issues/2222")]
public void CheckMofFile()
{
// This test code needs to be updated to support distributed testing.
// See https://github.com/aspnet/AspNetCore-Internal/issues/2222
#pragma warning disable 0618
var path = Path.Combine(TestPathUtilities.GetSolutionRootDirectory("IISIntegration"), "aspnetcoremodulev2", "aspnetcore", "ancm.mof");
#pragma warning restore 0618
var process = Process.Start("mofcomp.exe", path);
process.WaitForExit();
Assert.Equal(0, process.ExitCode);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ public class AddressRegistrationTests : TestApplicationErrorLoggerLoggedTest

[ConditionalFact]
[HostNameIsReachable]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7267
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/7267")]
public async Task RegisterAddresses_HostName_Success()
{
var hostName = Dns.GetHostName();
Expand Down Expand Up @@ -335,7 +335,7 @@ public async Task ListenAnyIP_IPv6_Success()

[ConditionalFact]
[HostNameIsReachable]
[SkipOnHelix] // https://github.com/aspnet/AspNetCore/issues/7267
[SkipOnHelix("https://github.com/aspnet/AspNetCore/issues/7267")]
public async Task ListenAnyIP_HostName_Success()
{
var hostName = Dns.GetHostName();
Expand Down
Loading