From 21f008e44dbf6326ffd73c4ee4bd95948b9a7ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Wed, 30 Aug 2023 12:35:45 +0200 Subject: [PATCH 01/36] wip --- eng/testing/tests.browser.targets | 2 +- .../CompletionCallbackExecutionSink.cs | 37 + .../ConsoleDiagnosticMessageSink.cs | 22 + .../Common/tests/WasmTestRunner/Extensions.cs | 10 + .../tests/WasmTestRunner/WasmTestRunner.cs | 100 +- .../WasmTestRunner/WasmTestRunner.csproj | 8 + .../tests/WasmTestRunner/XUnitFilter.cs | 313 +++++ .../tests/WasmTestRunner/XUnitFilterType.cs | 14 + .../WasmTestRunner/XUnitFiltersCollection.cs | 80 ++ .../tests/WasmTestRunner/XUnitTestRunner.cs | 1105 +++++++++++++++++ .../WasmTestRunner/XunitTestRunnerBase.cs | 87 ++ 11 files changed, 1761 insertions(+), 17 deletions(-) create mode 100644 src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/Extensions.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs create mode 100644 src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 8c4b3e46c9eac9..12bcc7b168facf 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -97,7 +97,7 @@ $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=1 - $(WasmXHarnessMonoArgs) --no-memory-snapshot + $(WasmXHarnessMonoArgs) --no-memory-snapshot -mt $(WasmXHarnessMonoArgs) --setenv=IsBrowserThreadingSupported=true diff --git a/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs b/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs new file mode 100644 index 00000000000000..c344797432cc84 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs @@ -0,0 +1,37 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Threading; +using Xunit; +using Xunit.Abstractions; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; + +internal class CompletionCallbackExecutionSink : global::Xunit.Sdk.LongLivedMarshalByRefObject, IExecutionSink +{ + private readonly Action _completionCallback; + private readonly IExecutionSink _innerSink; + + public ExecutionSummary ExecutionSummary => _innerSink.ExecutionSummary; + + public ManualResetEvent Finished => _innerSink.Finished; + + public CompletionCallbackExecutionSink(IExecutionSink innerSink, Action completionCallback) + { + _innerSink = innerSink; + _completionCallback = completionCallback; + } + + public void Dispose() => _innerSink.Dispose(); + + public bool OnMessageWithTypes(IMessageSinkMessage message, HashSet messageTypes) + { + var result = _innerSink.OnMessageWithTypes(message, messageTypes); + message.Dispatch(messageTypes, args => _completionCallback(ExecutionSummary)); + return result; + } +} diff --git a/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs b/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs new file mode 100644 index 00000000000000..c76ca047030712 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs @@ -0,0 +1,22 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Xunit.Abstractions; + +public class MyConsoleDiagnosticMessageSink : global::Xunit.Sdk.LongLivedMarshalByRefObject, IMessageSink +{ + public bool OnMessage(IMessageSinkMessage message) + { + if (message is IDiagnosticMessage diagnosticMessage) + { + Console.WriteLine(diagnosticMessage.Message); + } + + return true; + } +} diff --git a/src/libraries/Common/tests/WasmTestRunner/Extensions.cs b/src/libraries/Common/tests/WasmTestRunner/Extensions.cs new file mode 100644 index 00000000000000..a28df3be3d9c1e --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/Extensions.cs @@ -0,0 +1,10 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.XHarness.TestRunners.Common; + +internal static partial class Extensions +{ + public static string YesNo(this bool b) => b ? "yes" : "no"; +} diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 1307a18a9eb51d..fc10dbfafc6990 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -3,9 +3,13 @@ using System; using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Linq; +using System.Reflection; using System.Threading.Tasks; - +using Microsoft.DotNet.XHarness.TestRunners.Common; using Microsoft.DotNet.XHarness.TestRunners.Xunit; +using Xunit.Sdk; public class SimpleWasmTestRunner : WasmApplicationEntryPoint { @@ -13,7 +17,7 @@ public static async Task Main(string[] args) { if (args.Length == 0) { - Console.WriteLine ($"No args given"); + Console.WriteLine($"No args given"); return -1; } @@ -30,40 +34,104 @@ public static async Task Main(string[] args) switch (option) { case "-notrait": - excludedTraits.Add (args[i + 1]); + excludedTraits.Add(args[i + 1]); i++; break; case "-trait": - includedTraits.Add (args[i + 1]); + includedTraits.Add(args[i + 1]); i++; break; case "-namespace": - includedNamespaces.Add (args[i + 1]); + includedNamespaces.Add(args[i + 1]); i++; break; case "-class": - includedClasses.Add (args[i + 1]); + includedClasses.Add(args[i + 1]); i++; break; case "-method": - includedMethods.Add (args[i + 1]); + includedMethods.Add(args[i + 1]); i++; break; + case "-mt": + break; default: throw new ArgumentException($"Invalid argument '{option}'."); } } - var runner = new SimpleWasmTestRunner() + WasmApplicationEntryPointBase? runner = null; + if (args.Contains("-mt")) + { + Console.WriteLine("Using MultiThreadedTestRunner"); + runner = new MultiThreadedTestRunner() + { + TestAssembly = testAssembly, + ExcludedTraits = excludedTraits, + IncludedTraits = includedTraits, + IncludedNamespaces = includedNamespaces, + IncludedClasses = includedClasses, + IncludedMethods = includedMethods + }; + } + else + { + Console.WriteLine("Using SimpleWasmTestRunner"); + runner = new SimpleWasmTestRunner() + { + TestAssembly = testAssembly, + ExcludedTraits = excludedTraits, + IncludedTraits = includedTraits, + IncludedNamespaces = includedNamespaces, + IncludedClasses = includedClasses, + IncludedMethods = includedMethods + }; + } + + await runner.RunAsync(); + return runner.LastRunHadFailedTests ? 1 : 0; + } +} + +class MultiThreadedTestRunner : WasmApplicationEntryPointBase +{ + public virtual string TestAssembly { get; set; } = ""; + public virtual IEnumerable ExcludedTraits { get; set; } = Array.Empty(); + public virtual IEnumerable IncludedTraits { get; set; } = Array.Empty(); + public virtual IEnumerable IncludedClasses { get; set; } = Array.Empty(); + public virtual IEnumerable IncludedMethods { get; set; } = Array.Empty(); + public virtual IEnumerable IncludedNamespaces { get; set; } = Array.Empty(); + + protected override bool IsXunit => true; + + protected override TestRunner GetTestRunner(LogWriter logWriter) + { + var runner = new MyXUnitTestRunner(logWriter); + //var xUnitTestRunnerType = typeof(XunitTestRunnerBase).Assembly.GetType("Microsoft.DotNet.XHarness.TestRunners.Xunit.XUnitTestRunner"); + //var runner = (XunitTestRunnerBase)xUnitTestRunnerType!.GetConstructors().First().Invoke(new[] { logWriter }); + + //ConfigureRunnerFilters(runner, ApplicationOptions.Current); + var configureRunnerFiltersMethod = typeof(ApplicationEntryPoint).GetMethod("ConfigureRunnerFilters", BindingFlags.Static | BindingFlags.NonPublic); + configureRunnerFiltersMethod!.Invoke(null, new object[] { runner, ApplicationOptions.Current }); + + runner.SkipCategories(ExcludedTraits); + runner.SkipCategories(IncludedTraits, isExcluded: false); + foreach (var cls in IncludedClasses) { - TestAssembly = testAssembly, - ExcludedTraits = excludedTraits, - IncludedTraits = includedTraits, - IncludedNamespaces = includedNamespaces, - IncludedClasses = includedClasses, - IncludedMethods = includedMethods - }; + runner.SkipClass(cls, false); + } + foreach (var method in IncludedMethods) + { + runner.SkipMethod(method, false); + } + foreach (var ns in IncludedNamespaces) + { + runner.SkipNamespace(ns, isExcluded: false); + } - return await runner.Run(); + return runner; } + + protected override IEnumerable GetTestAssemblies() + => new[] { new TestAssemblyInfo(Assembly.LoadFrom(TestAssembly), TestAssembly) }; } diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj index 2f60f08a55c44a..8260105d391e09 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj @@ -5,7 +5,15 @@ $(NetCoreAppCurrent) + + + + + + + + diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs new file mode 100644 index 00000000000000..18c7894e528779 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs @@ -0,0 +1,313 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.IO; +using System.Linq; +using System.Text; +using Microsoft.DotNet.XHarness.TestRunners.Common; +using Xunit.Abstractions; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; + +internal class XUnitFilter +{ + public string? AssemblyName { get; private set; } + public string? SelectorName { get; private set; } + public string? SelectorValue { get; private set; } + + public bool Exclude { get; private set; } + public XUnitFilterType FilterType { get; private set; } + + public static XUnitFilter CreateSingleFilter(string singleTestName, bool exclude, string? assemblyName = null) + { + if (string.IsNullOrEmpty(singleTestName)) + { + throw new ArgumentException("must not be null or empty", nameof(singleTestName)); + } + + return new XUnitFilter + { + AssemblyName = assemblyName, + SelectorValue = singleTestName, + FilterType = XUnitFilterType.Single, + Exclude = exclude + }; + } + + public static XUnitFilter CreateAssemblyFilter(string assemblyName, bool exclude) + { + if (string.IsNullOrEmpty(assemblyName)) + { + throw new ArgumentException("must not be null or empty", nameof(assemblyName)); + } + + // ensure that the assembly name does have one of the valid extensions + var fileExtension = Path.GetExtension(assemblyName); + if (fileExtension != ".dll" && fileExtension != ".exe") + { + throw new ArgumentException($"Assembly name must have .dll or .exe as extensions. Found extension {fileExtension}"); + } + + return new XUnitFilter + { + AssemblyName = assemblyName, + FilterType = XUnitFilterType.Assembly, + Exclude = exclude + }; + } + + public static XUnitFilter CreateNamespaceFilter(string namespaceName, bool exclude, string? assemblyName = null) + { + if (string.IsNullOrEmpty(namespaceName)) + { + throw new ArgumentException("must not be null or empty", nameof(namespaceName)); + } + + return new XUnitFilter + { + AssemblyName = assemblyName, + SelectorValue = namespaceName, + FilterType = XUnitFilterType.Namespace, + Exclude = exclude + }; + } + + public static XUnitFilter CreateClassFilter(string className, bool exclude, string? assemblyName = null) + { + if (string.IsNullOrEmpty(className)) + { + throw new ArgumentException("must not be null or empty", nameof(className)); + } + + return new XUnitFilter + { + AssemblyName = assemblyName, + SelectorValue = className, + FilterType = XUnitFilterType.TypeName, + Exclude = exclude + }; + } + + public static XUnitFilter CreateTraitFilter(string traitName, string? traitValue, bool exclude) + { + if (string.IsNullOrEmpty(traitName)) + { + throw new ArgumentException("must not be null or empty", nameof(traitName)); + } + + return new XUnitFilter + { + AssemblyName = null, + SelectorName = traitName, + SelectorValue = traitValue ?? string.Empty, + FilterType = XUnitFilterType.Trait, + Exclude = exclude + }; + } + + private bool ApplyTraitFilter(ITestCase testCase, Func? reportFilteredTest = null) + { + Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; + + if (!testCase.HasTraits()) + { + return log(!Exclude); + } + + if (testCase.TryGetTrait(SelectorName!, out var values)) + { + if (values == null || values.Count == 0) + { + // We have no values and the filter doesn't specify one - that means we match on + // the trait name only. + if (string.IsNullOrEmpty(SelectorValue)) + { + return log(Exclude); + } + + return log(!Exclude); + } + + return values.Any(value => value.Equals(SelectorValue, StringComparison.InvariantCultureIgnoreCase)) ? + log(Exclude) : log(!Exclude); + } + + // no traits found, that means that we return the opposite of the setting of the filter + return log(!Exclude); + } + + private bool ApplyTypeNameFilter(ITestCase testCase, Func? reportFilteredTest = null) + { + Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; + var testClassName = testCase.GetTestClass(); + if (!string.IsNullOrEmpty(testClassName)) + { + if (string.Equals(testClassName, SelectorValue, StringComparison.InvariantCulture)) + { + return log(Exclude); + } + } + + return log(!Exclude); + } + + private bool ApplySingleFilter(ITestCase testCase, Func? reportFilteredTest = null) + { + Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; + if (string.Equals(testCase.DisplayName, SelectorValue, StringComparison.InvariantCulture)) + { + // if there is a match, return the exclude value + return log(Exclude); + } + // if there is not match, return the opposite + return log(!Exclude); + } + + private bool ApplyNamespaceFilter(ITestCase testCase, Func? reportFilteredTest = null) + { + Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; + var testClassNamespace = testCase.GetNamespace(); + if (string.IsNullOrEmpty(testClassNamespace)) + { + // if we exclude, since we have no namespace, we include the test + return log(!Exclude); + } + + if (string.Equals(testClassNamespace, SelectorValue, StringComparison.InvariantCultureIgnoreCase)) + { + return log(Exclude); + } + + // same logic as with no namespace + return log(!Exclude); + } + + public bool IsExcluded(TestAssemblyInfo assembly, Action? reportFilteredAssembly = null) + { + if (FilterType != XUnitFilterType.Assembly) + { + throw new InvalidOperationException("Filter is not targeting assemblies."); + } + + Func log = (result) => ReportFilteredAssembly(assembly, result, reportFilteredAssembly); + + if (string.Equals(AssemblyName, assembly.FullPath, StringComparison.Ordinal)) + { + return log(Exclude); + } + + string fileName = Path.GetFileName(assembly.FullPath); + if (string.Equals(fileName, AssemblyName, StringComparison.Ordinal)) + { + return log(Exclude); + } + + // No path of the name matched the filter, therefore return the opposite of the Exclude value + return log(!Exclude); + } + + public bool IsExcluded(ITestCase testCase, Action? log = null) + { + Func? reportFilteredTest = null; + if (log != null) + { + reportFilteredTest = (result) => ReportFilteredTest(testCase, result, log); + } + + return FilterType switch + { + XUnitFilterType.Trait => ApplyTraitFilter(testCase, reportFilteredTest), + XUnitFilterType.TypeName => ApplyTypeNameFilter(testCase, reportFilteredTest), + XUnitFilterType.Single => ApplySingleFilter(testCase, reportFilteredTest), + XUnitFilterType.Namespace => ApplyNamespaceFilter(testCase, reportFilteredTest), + _ => throw new InvalidOperationException($"Unsupported filter type {FilterType}") + }; + } + + private bool ReportFilteredTest(ITestCase testCase, bool excluded, Action? log = null) + { + const string includedText = "Included"; + const string excludedText = "Excluded"; + + if (log == null) + { + return excluded; + } + + var selector = FilterType == XUnitFilterType.Trait ? + $"'{SelectorName}':'{SelectorValue}'" : $"'{SelectorValue}'"; + + log($"[FILTER] {(excluded ? excludedText : includedText)} test (filtered by {FilterType}; {selector}): {testCase.DisplayName}"); + return excluded; + } + + private static bool ReportFilteredAssembly(TestAssemblyInfo assemblyInfo, bool excluded, Action? log = null) + { + if (log == null) + { + return excluded; + } + + const string includedPrefix = "Included"; + const string excludedPrefix = "Excluded"; + + log($"[FILTER] {(excluded ? excludedPrefix : includedPrefix)} assembly: {assemblyInfo.FullPath}"); + return excluded; + } + + private static void AppendDesc(StringBuilder sb, string name, string? value) + { + if (string.IsNullOrEmpty(value)) + { + return; + } + + sb.Append($"; {name}: {value}"); + } + + public override string ToString() + { + var sb = new StringBuilder("XUnitFilter ["); + + sb.Append($"Type: {FilterType}; "); + sb.Append(Exclude ? "exclude" : "include"); + + if (!string.IsNullOrEmpty(AssemblyName)) + { + sb.Append($"; AssemblyName: {AssemblyName}"); + } + + switch (FilterType) + { + case XUnitFilterType.Assembly: + break; + + case XUnitFilterType.Namespace: + AppendDesc(sb, "Namespace", SelectorValue); + break; + + case XUnitFilterType.Single: + AppendDesc(sb, "Method", SelectorValue); + break; + + case XUnitFilterType.Trait: + AppendDesc(sb, "Trait name", SelectorName); + AppendDesc(sb, "Trait value", SelectorValue); + break; + + case XUnitFilterType.TypeName: + AppendDesc(sb, "Class", SelectorValue); + break; + + default: + sb.Append("; Unknown filter type"); + break; + } + sb.Append(']'); + + return sb.ToString(); + } +} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs new file mode 100644 index 00000000000000..dee5e272585f29 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; + +internal enum XUnitFilterType +{ + Trait, + TypeName, + Assembly, + Single, + Namespace, +} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs new file mode 100644 index 00000000000000..97e2587131dd85 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs @@ -0,0 +1,80 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.DotNet.XHarness.TestRunners.Common; +using Xunit.Abstractions; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; + +/// +/// Class that contains a collection of filters and can be used to decide if a test should be executed or not. +/// +internal class XUnitFiltersCollection : List +{ + /// + /// Return all the filters that are applied to assemblies. + /// + public IEnumerable AssemblyFilters + => Enumerable.Where(this, f => f.FilterType == XUnitFilterType.Assembly); + + /// + /// Return all the filters that are applied to test cases. + /// + public IEnumerable TestCaseFilters + => Enumerable.Where(this, f => f.FilterType != XUnitFilterType.Assembly); + + // loop over all the filters, if we have conflicting filters, that is, one exclude and other one + // includes, we will always include since it is better to run a test thant to skip it and think + // you ran in. + private bool IsExcludedInternal(IEnumerable filters, Func isExcludedCb) + { + // No filters : include by default + // Any exclude filters : include by default + // Only include filters : exclude by default + var isExcluded = filters.Any() && filters.All(f => !f.Exclude); + foreach (var filter in filters) + { + var doesExclude = isExcludedCb(filter); + if (filter.Exclude) + { + isExcluded |= doesExclude; + } + else + { + // filter does not exclude, that means that if it include, we should include and break the + // loop, always include + if (!doesExclude) + { + return false; + } + } + } + + return isExcluded; + } + + public bool IsExcluded(TestAssemblyInfo assembly, Action? log = null) => + IsExcludedInternal(AssemblyFilters, f => f.IsExcluded(assembly, log)); + + public bool IsExcluded(ITestCase testCase, Action? log = null) + { + // Check each type of filter separately. For conflicts within a type of filter, we want the inclusion + // (the logic in IsExcludedInternal), but if all filters for a filter type exclude a test case, we want + // the exclusion. For example, if a test class is included, but it contains tests that have excluded + // traits, the behaviour should be to run all tests in that class without the excluded traits. + foreach (IGrouping filterGroup in TestCaseFilters.GroupBy(f => f.FilterType)) + { + if (IsExcludedInternal(filterGroup, f => f.IsExcluded(testCase, log))) + { + return true; + } + } + + return false; + } +} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs new file mode 100644 index 00000000000000..ad5586df29dde9 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -0,0 +1,1105 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +#nullable disable + +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using System.Xml; +using System.Xml.Linq; +using System.Xml.Xsl; +using Microsoft.DotNet.XHarness.Common; +using Microsoft.DotNet.XHarness.TestRunners.Common; +using Xunit; +using Xunit.Abstractions; + +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; + +internal class XsltIdGenerator +{ + // NUnit3 xml does not have schema, there is no much info about it, most examples just have incremental IDs. + private int _seed = 1000; + public int GenerateHash() => _seed++; +} + +internal class MyXUnitTestRunner : MyXunitTestRunnerBase +{ + private readonly TestMessageSink _messageSink; + + public int? MaxParallelThreads { get; set; } + + private XElement _assembliesElement; + + public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; + protected override string ResultsFileName { get; set; } = "TestResults.xUnit.xml"; + + public MyXUnitTestRunner(LogWriter logger) : base(logger) + { + _messageSink = new TestMessageSink(); + + _messageSink.Diagnostics.DiagnosticMessageEvent += HandleDiagnosticMessage; + _messageSink.Diagnostics.ErrorMessageEvent += HandleDiagnosticErrorMessage; + + _messageSink.Discovery.DiscoveryCompleteMessageEvent += HandleDiscoveryCompleteMessage; + _messageSink.Discovery.TestCaseDiscoveryMessageEvent += HandleDiscoveryTestCaseMessage; + + _messageSink.Runner.TestAssemblyDiscoveryFinishedEvent += HandleTestAssemblyDiscoveryFinished; + _messageSink.Runner.TestAssemblyDiscoveryStartingEvent += HandleTestAssemblyDiscoveryStarting; + _messageSink.Runner.TestAssemblyExecutionFinishedEvent += HandleTestAssemblyExecutionFinished; + _messageSink.Runner.TestAssemblyExecutionStartingEvent += HandleTestAssemblyExecutionStarting; + _messageSink.Runner.TestExecutionSummaryEvent += HandleTestExecutionSummary; + + _messageSink.Execution.AfterTestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("AfterTestFinishedEvent", args, HandleAfterTestFinished); + _messageSink.Execution.AfterTestStartingEvent += (MessageHandlerArgs args) => HandleEvent("AfterTestStartingEvent", args, HandleAfterTestStarting); + _messageSink.Execution.BeforeTestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("BeforeTestFinishedEvent", args, HandleBeforeTestFinished); + _messageSink.Execution.BeforeTestStartingEvent += (MessageHandlerArgs args) => HandleEvent("BeforeTestStartingEvent", args, HandleBeforeTestStarting); + _messageSink.Execution.TestAssemblyCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyCleanupFailureEvent", args, HandleTestAssemblyCleanupFailure); + _messageSink.Execution.TestAssemblyFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyFinishedEvent", args, HandleTestAssemblyFinished); + _messageSink.Execution.TestAssemblyStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyStartingEvent", args, HandleTestAssemblyStarting); + _messageSink.Execution.TestCaseCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCaseCleanupFailureEvent", args, HandleTestCaseCleanupFailure); + _messageSink.Execution.TestCaseFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestCaseFinishedEvent", args, HandleTestCaseFinished); + _messageSink.Execution.TestCaseStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestStartingEvent", args, HandleTestCaseStarting); + _messageSink.Execution.TestClassCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestClassCleanupFailureEvent", args, HandleTestClassCleanupFailure); + _messageSink.Execution.TestClassConstructionFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassConstructionFinishedEvent", args, HandleTestClassConstructionFinished); + _messageSink.Execution.TestClassConstructionStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassConstructionStartingEvent", args, HandleTestClassConstructionStarting); + _messageSink.Execution.TestClassDisposeFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassDisposeFinishedEvent", args, HandleTestClassDisposeFinished); + _messageSink.Execution.TestClassDisposeStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassDisposeStartingEvent", args, HandleTestClassDisposeStarting); + _messageSink.Execution.TestClassFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassFinishedEvent", args, HandleTestClassFinished); + _messageSink.Execution.TestClassStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassStartingEvent", args, HandleTestClassStarting); + _messageSink.Execution.TestCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCleanupFailureEvent", args, HandleTestCleanupFailure); + _messageSink.Execution.TestCollectionCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionCleanupFailureEvent", args, HandleTestCollectionCleanupFailure); + _messageSink.Execution.TestCollectionFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionFinishedEvent", args, HandleTestCollectionFinished); + _messageSink.Execution.TestCollectionStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionStartingEvent", args, HandleTestCollectionStarting); + _messageSink.Execution.TestFailedEvent += (MessageHandlerArgs args) => HandleEvent("TestFailedEvent", args, HandleTestFailed); + _messageSink.Execution.TestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestFinishedEvent", args, HandleTestFinished); + _messageSink.Execution.TestMethodCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodCleanupFailureEvent", args, HandleTestMethodCleanupFailure); + _messageSink.Execution.TestMethodFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodFinishedEvent", args, HandleTestMethodFinished); + _messageSink.Execution.TestMethodStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodStartingEvent", args, HandleTestMethodStarting); + _messageSink.Execution.TestOutputEvent += (MessageHandlerArgs args) => HandleEvent("TestOutputEvent", args, HandleTestOutput); + _messageSink.Execution.TestPassedEvent += (MessageHandlerArgs args) => HandleEvent("TestPassedEvent", args, HandleTestPassed); + _messageSink.Execution.TestSkippedEvent += (MessageHandlerArgs args) => HandleEvent("TestSkippedEvent", args, HandleTestSkipped); + _messageSink.Execution.TestStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestStartingEvent", args, HandleTestStarting); + } + + public void AddFilter(XUnitFilter filter) + { + if (filter != null) + { + _filters.Add(filter); + } + } + public void SetFilters(List newFilters) + { + if (newFilters == null) + { + _filters = null; + return; + } + + if (_filters == null) + { + _filters = new XUnitFiltersCollection(); + } + + _filters.AddRange(newFilters); + } + + private void HandleEvent(string name, MessageHandlerArgs args, Action> actualHandler) where T : class, IMessageSinkMessage + { + try + { + actualHandler(args); + } + catch (Exception ex) + { + OnError($"Handler for event {name} failed with exception"); + OnError(ex.ToString()); + } + } + + private void HandleTestStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test starting"); + LogTestDetails(args.Message.Test, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + } + + private void HandleTestSkipped(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + RaiseTestSkippedCase(args.Message, args.Message.TestCases, args.Message.TestCase); + } + + private void HandleTestPassed(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + PassedTests++; + OnInfo($"\t[PASS] {args.Message.TestCase.DisplayName}"); + LogTestDetails(args.Message.Test, log: OnDebug); + LogTestOutput(args.Message, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + // notify the completion of the test + OnTestCompleted(( + TestName: args.Message.Test.DisplayName, + TestResult: TestResult.Passed + )); + } + + private void HandleTestOutput(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo(args.Message.Output); + } + + private void HandleTestMethodStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test method starting"); + LogTestMethodDetails(args.Message.TestMethod.Method, log: OnDebug); + LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestMethodFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test method finished"); + LogTestMethodDetails(args.Message.TestMethod.Method, log: OnDebug); + LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnDebug); + LogSummary(args.Message, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestMethodCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError($"Test method cleanup failure{GetAssemblyInfo(args.Message.TestAssembly)}"); + LogTestMethodDetails(args.Message.TestMethod.Method, log: OnError); + LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnError); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleTestFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + ExecutedTests++; + OnDiagnostic("Test finished"); + LogTestDetails(args.Message.Test, log: OnDiagnostic); + LogTestOutput(args.Message, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + } + + private void HandleTestFailed(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + FailedTests++; + string assemblyInfo = GetAssemblyInfo(args.Message.TestAssembly); + var sb = new StringBuilder($"\t[FAIL] {args.Message.TestCase.DisplayName}"); + LogTestDetails(args.Message.Test, OnError, sb); + sb.AppendLine(); + if (!string.IsNullOrEmpty(assemblyInfo)) + { + sb.AppendLine($" Assembly: {assemblyInfo}"); + } + + LogSourceInformation(args.Message.TestCase.SourceInformation, OnError, sb); + LogFailureInformation(args.Message, OnError, sb); + sb.AppendLine(); + LogTestOutput(args.Message, OnError, sb); + sb.AppendLine(); + if (args.Message.TestCase.Traits != null && args.Message.TestCase.Traits.Count > 0) + { + foreach (var kvp in args.Message.TestCase.Traits) + { + string message = $" Test trait name: {kvp.Key}"; + OnError(message); + sb.AppendLine(message); + + foreach (string v in kvp.Value) + { + message = $" value: {v}"; + OnError(message); + sb.AppendLine(message); + } + } + sb.AppendLine(); + } + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + + FailureInfos.Add(new TestFailureInfo + { + TestName = args.Message.Test?.DisplayName, + Message = sb.ToString() + }); + OnInfo($"\t[FAIL] {args.Message.Test?.TestCase.DisplayName}"); + OnInfo(sb.ToString()); + OnTestCompleted(( + TestName: args.Message.Test?.DisplayName, + TestResult: TestResult.Failed + )); + } + + private void HandleTestCollectionStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"\n{args.Message.TestCollection.DisplayName}"); + OnDebug("Test collection starting"); + LogTestCollectionDetails(args.Message.TestCollection, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestCollectionFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test collection finished"); + LogSummary(args.Message, log: OnDebug); + LogTestCollectionDetails(args.Message.TestCollection, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestCollectionCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError("Error during test collection cleanup"); + LogTestCollectionDetails(args.Message.TestCollection, log: OnError); + ReportTestCases(" Associated", args.Message.TestCases, log: OnError); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleTestCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError($"Test cleanup failure{GetAssemblyInfo(args.Message.TestAssembly)}"); + LogTestDetails(args.Message.Test, log: OnError); + ReportTestCases(" Associated", args.Message.TestCases, log: OnError); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleTestClassStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test class starting"); + LogTestClassDetails(args.Message.TestClass, log: OnDiagnostic); + } + + private void HandleTestClassFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test class finished"); + OnInfo($"{args.Message.TestClass.Class.Name} {args.Message.ExecutionTime} ms"); + LogTestClassDetails(args.Message.TestClass, OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestClassDisposeStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test class dispose starting"); + LogTestDetails(args.Message.Test, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestClassDisposeFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test class dispose finished"); + LogTestDetails(args.Message.Test, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestClassConstructionStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test class construction starting"); + LogTestDetails(args.Message.Test, OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + } + + private void HandleTestClassConstructionFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test class construction finished"); + LogTestDetails(args.Message.Test, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + } + + private void HandleTestClassCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError($"Test class cleanup error{GetAssemblyInfo(args.Message.TestAssembly)}"); + LogTestClassDetails(args.Message.TestClass, log: OnError); + LogTestCollectionDetails(args.Message.TestCollection, log: OnError); + ReportTestCases(" Associated", args.Message.TestCases, log: OnError); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleTestCaseStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic("Test case starting"); + ReportTestCase(" Starting", args.Message.TestCase, log: OnDiagnostic); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); + } + + private void HandleTestCaseFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDebug("Test case finished executing"); + ReportTestCase(" Finished", args.Message.TestCase, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDebug); + LogSummary(args.Message, log: OnDebug); + } + + private void HandleTestCaseCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError("Test case cleanup failure"); + ReportTestCase(" Failed", args.Message.TestCase, log: OnError); + ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnError); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleTestAssemblyStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"[Test environment: {args.Message.TestEnvironment}]"); + OnInfo($"[Test framework: {args.Message.TestFrameworkDisplayName}]"); + LogAssemblyInformation(args.Message, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDebug); + } + + private void HandleTestAssemblyFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + TotalTests = args.Message.TestsRun; // HACK: We are not counting correctly all the tests + OnDebug("Execution process for assembly finished"); + LogAssemblyInformation(args.Message, log: OnDebug); + LogSummary(args.Message, log: OnDebug); + ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); + } + + private void HandleTestAssemblyCleanupFailure(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnError("Assembly cleanup failure"); + LogAssemblyInformation(args.Message, OnError); + ReportTestCases(" Associated", args.Message.TestCases, log: OnError); + LogFailureInformation(args.Message, log: OnError); + } + + private void HandleBeforeTestStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + // notify that a method is starting + OnTestStarted(args.Message.Test.DisplayName); + OnDiagnostic($"'Before' method for test '{args.Message.Test.DisplayName}' starting"); + } + + private void HandleBeforeTestFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic($"'Before' method for test '{args.Message.Test.DisplayName}' finished"); + } + + private void HandleAfterTestStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic($"'After' method for test '{args.Message.Test.DisplayName}' starting"); + } + + private void HandleAfterTestFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic($"'After' method for test '{args.Message.Test.DisplayName}' finished"); + } + + private void HandleTestExecutionSummary(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo("All tests finished"); + OnInfo($" Elapsed time: {args.Message.ElapsedClockTime}"); + + if (args.Message.Summaries == null || args.Message.Summaries.Count == 0) + { + return; + } + + foreach (KeyValuePair summary in args.Message.Summaries) + { + OnInfo(string.Empty); + OnInfo($" Assembly: {summary.Key}"); + LogSummary(summary.Value, log: OnDebug); + } + } + + private void HandleTestAssemblyExecutionStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"Execution starting for assembly {args.Message.Assembly.AssemblyFilename}"); + } + + private void HandleTestAssemblyExecutionFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"Execution finished for assembly {args.Message.Assembly.AssemblyFilename}"); + LogSummary(args.Message.ExecutionSummary, log: OnDebug); + } + + private void HandleTestAssemblyDiscoveryStarting(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"Discovery for assembly {args.Message.Assembly.AssemblyFilename} starting"); + OnInfo($" Will use AppDomain: {args.Message.AppDomain.YesNo()}"); + } + + private void HandleTestAssemblyDiscoveryFinished(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo($"Discovery for assembly {args.Message.Assembly.AssemblyFilename} finished"); + OnInfo($" Test cases discovered: {args.Message.TestCasesDiscovered}"); + OnInfo($" Test cases to run: {args.Message.TestCasesToRun}"); + } + + private void HandleDiagnosticMessage(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnDiagnostic(args.Message.Message); + } + + private void HandleDiagnosticErrorMessage(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + LogFailureInformation(args.Message); + } + + private void HandleDiscoveryCompleteMessage(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + OnInfo("Discovery complete"); + } + + private void HandleDiscoveryTestCaseMessage(MessageHandlerArgs args) + { + if (args == null || args.Message == null) + { + return; + } + + ITestCase singleTestCase = args.Message.TestCase; + ReportTestCases("Discovered", args.Message.TestCases, log: OnInfo, ignore: (ITestCase tc) => tc == singleTestCase); + ReportTestCase("Discovered", singleTestCase, log: OnInfo); + } + + private void RaiseTestSkippedCase(ITestResultMessage message, IEnumerable testCases, ITestCase testCase) + { + SkippedTests++; + OnInfo($"\t[IGNORED] {testCase.DisplayName}"); + LogTestDetails(message.Test, log: OnDebug); + LogTestOutput(message, log: OnDiagnostic); + ReportTestCases(" Associated", testCases, log: OnDiagnostic); + // notify that the test completed because it was skipped + OnTestCompleted(( + TestName: message.Test.DisplayName, + TestResult: TestResult.Skipped + )); + } + + private void ReportTestCases(string verb, IEnumerable testCases, ITestCase ignoreTestCase, Action log = null) => ReportTestCases(verb, testCases, log, (ITestCase tc) => ignoreTestCase == tc); + + private void ReportTestCases(string verb, IEnumerable testCases, Action log = null, Func ignore = null) + { + if (testCases == null) + { + return; + } + + foreach (ITestCase tc in testCases) + { + if (ignore != null && ignore(tc)) + { + continue; + } + + ReportTestCase(verb, tc, log); + } + } + + private void ReportTestCase(string verb, ITestCase testCase, Action log = null) + { + if (testCase == null) + { + return; + } + + EnsureLogger(log)($"{verb} test case: {testCase.DisplayName}"); + } + + private void LogAssemblyInformation(ITestAssemblyMessage message, Action log = null, StringBuilder sb = null) + { + if (message == null) + { + return; + } + + do_log($"[Assembly name: {message.TestAssembly.Assembly.Name}]", log, sb); + do_log($"[Assembly path: {message.TestAssembly.Assembly.AssemblyPath}]", OnDiagnostic, sb); + } + + private void LogFailureInformation(IFailureInformation info, Action log = null, StringBuilder sb = null) + { + if (info == null) + { + return; + } + + string message = ExceptionUtility.CombineMessages(info); + do_log($" Exception messages: {message}", log, sb); + + string traces = ExceptionUtility.CombineStackTraces(info); + do_log($" Exception stack traces: {traces}", log, sb); + } + + private Action EnsureLogger(Action log) => log ?? OnInfo; + +#pragma warning disable IDE0060 // Remove unused parameter + private static void LogTestMethodDetails(IMethodInfo method, Action log = null, StringBuilder sb = null) +#pragma warning restore IDE0060 // Remove unused parameter + { + // log = EnsureLogger(log); + // log ($" Test method name: {method.Type.Name}.{method.Name}"); + } + + private void LogTestOutput(ITestFinished test, Action log = null, StringBuilder sb = null) => LogTestOutput(test.ExecutionTime, test.Output, log, sb); + + private void LogTestOutput(ITestResultMessage test, Action log = null, StringBuilder sb = null) => LogTestOutput(test.ExecutionTime, test.Output, log, sb); + + private void LogTestOutput(decimal executionTime, string output, Action log = null, StringBuilder sb = null) + { + do_log($" Execution time: {executionTime}", log, sb); + if (!string.IsNullOrEmpty(output)) + { + do_log(" **** Output start ****", log, sb); + foreach (string line in output.Split('\n')) + { + do_log(line, log, sb); + } + + do_log(" **** Output end ****", log, sb); + } + } + + private void LogTestCollectionDetails(ITestCollection collection, Action log = null, StringBuilder sb = null) => do_log($" Test collection: {collection.DisplayName}", log, sb); + + private void LogTestClassDetails(ITestClass klass, Action log = null, StringBuilder sb = null) + { + do_log($" Class name: {klass.Class.Name}", log, sb); + do_log($" Class assembly: {klass.Class.Assembly.Name}", OnDebug, sb); + do_log($" Class assembly path: {klass.Class.Assembly.AssemblyPath}", OnDebug, sb); + } + + private void LogTestDetails(ITest test, Action log = null, StringBuilder sb = null) + { + do_log($" Test name: {test.DisplayName}", log, sb); + if (string.Compare(test.DisplayName, test.TestCase.DisplayName, StringComparison.Ordinal) != 0) + { + do_log($" Test case: {test.TestCase.DisplayName}", log, sb); + } + } + + private void LogSummary(IFinishedMessage summary, Action log = null, StringBuilder sb = null) + { + do_log($" Time: {summary.ExecutionTime}", log, sb); + do_log($" Total tests run: {summary.TestsRun}", log, sb); + do_log($" Skipped tests: {summary.TestsSkipped}", log, sb); + do_log($" Failed tests: {summary.TestsFailed}", log, sb); + } + + private void LogSummary(ExecutionSummary summary, Action log = null, StringBuilder sb = null) + { + do_log($" Time: {summary.Time}", log, sb); + do_log($" Total tests run: {summary.Total}", log, sb); + do_log($" Total errors: {summary.Errors}", log, sb); + do_log($" Skipped tests: {summary.Skipped}", log, sb); + do_log($" Failed tests: {summary.Failed}", log, sb); + } + + private void LogSourceInformation(ISourceInformation source, Action log = null, StringBuilder sb = null) + { + if (source == null || string.IsNullOrEmpty(source.FileName)) + { + return; + } + + string location = source.FileName; + if (source.LineNumber != null && source.LineNumber >= 0) + { + location += $":{source.LineNumber}"; + } + + do_log($" Source: {location}", log, sb); + sb?.AppendLine(); + } + + private static string GetAssemblyInfo(ITestAssembly assembly) + { + string name = assembly?.Assembly?.Name?.Trim(); + if (string.IsNullOrEmpty(name)) + { + return name; + } + + return $" [{name}]"; + } + + private void do_log(string message, Action log = null, StringBuilder sb = null) + { + log = EnsureLogger(log); + + if (sb != null) + { + sb.Append(message); + } + + log(message); + } + + public override async Task Run(IEnumerable testAssemblies) + { + if (testAssemblies == null) + { + throw new ArgumentNullException(nameof(testAssemblies)); + } + + if (_filters != null && _filters.Count > 0) + { + do_log("Configured filters:"); + foreach (XUnitFilter filter in _filters) + { + do_log($" {filter}"); + } + } + + _assembliesElement = new XElement("assemblies"); + Action log = LogExcludedTests ? (s) => do_log(s) : (Action)null; + foreach (TestAssemblyInfo assemblyInfo in testAssemblies) + { + if (assemblyInfo == null || assemblyInfo.Assembly == null) + { + continue; + } + + if (_filters.AssemblyFilters.Any() && _filters.IsExcluded(assemblyInfo, log)) + { + continue; + } + + if (string.IsNullOrEmpty(assemblyInfo.FullPath)) + { + OnWarning($"Assembly '{assemblyInfo.Assembly}' cannot be found on the filesystem. xUnit requires access to actual on-disk file."); + continue; + } + + OnInfo($"Assembly: {assemblyInfo.Assembly} ({assemblyInfo.FullPath})"); + XElement assemblyElement = null; + try + { + OnAssemblyStart(assemblyInfo.Assembly); + assemblyElement = await Run(assemblyInfo.Assembly, assemblyInfo.FullPath).ConfigureAwait(false); + } + catch (FileNotFoundException ex) + { + OnWarning($"Assembly '{assemblyInfo.Assembly}' using path '{assemblyInfo.FullPath}' cannot be found on the filesystem. xUnit requires access to actual on-disk file."); + OnWarning($"Exception is '{ex}'"); + } + finally + { + OnAssemblyFinish(assemblyInfo.Assembly); + if (assemblyElement != null) + { + _assembliesElement.Add(assemblyElement); + } + } + } + + LogFailureSummary(); + TotalTests += FilteredTests; // ensure that we do have in the total run the excluded ones. + } + + public override string WriteResultsToFile(XmlResultJargon jargon) + { + if (_assembliesElement == null) + { + return string.Empty; + } + // remove all the empty nodes + _assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove(); + string outputFilePath = GetResultsFilePath(); + var settings = new XmlWriterSettings { Indent = true }; + using (var xmlWriter = XmlWriter.Create(outputFilePath, settings)) + { + switch (jargon) + { + case XmlResultJargon.TouchUnit: + case XmlResultJargon.NUnitV2: + Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter); + break; + case XmlResultJargon.NUnitV3: + Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter); + break; + default: // xunit as default, includes when we got Missing + _assembliesElement.Save(xmlWriter); + break; + } + } + + return outputFilePath; + } + public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon) + { + if (_assembliesElement == null) + { + return; + } + // remove all the empty nodes + _assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove(); + var settings = new XmlWriterSettings { Indent = true }; + using (var xmlWriter = XmlWriter.Create(writer, settings)) + { + switch (jargon) + { + case XmlResultJargon.TouchUnit: + case XmlResultJargon.NUnitV2: + try + { + Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter); + } + catch (Exception e) + { + writer.WriteLine(e); + } + break; + case XmlResultJargon.NUnitV3: + try + { + Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter); + } + catch (Exception e) + { + writer.WriteLine(e); + } + break; + default: // xunit as default, includes when we got Missing + _assembliesElement.Save(xmlWriter); + break; + } + } + } + + private void Transform_Results(string xsltResourceName, XElement element, XmlWriter writer) + { + var xmlTransform = new System.Xml.Xsl.XslCompiledTransform(); + var name = GetType().Assembly.GetManifestResourceNames().Where(a => a.EndsWith(xsltResourceName, StringComparison.Ordinal)).FirstOrDefault(); + if (name == null) + { + return; + } + + using (var xsltStream = GetType().Assembly.GetManifestResourceStream(name)) + { + if (xsltStream == null) + { + throw new Exception($"Stream with name {name} cannot be found! We have {GetType().Assembly.GetManifestResourceNames()[0]}"); + } + // add the extension so that we can get the hash from the name of the test + // Create an XsltArgumentList. + var xslArg = new XsltArgumentList(); + + var generator = new XsltIdGenerator(); + xslArg.AddExtensionObject("urn:hash-generator", generator); + + using (var xsltReader = XmlReader.Create(xsltStream)) + using (var xmlReader = element.CreateReader()) + { + xmlTransform.Load(xsltReader); + xmlTransform.Transform(xmlReader, xslArg, writer); + } + } + } + + protected virtual Stream GetConfigurationFileStream(Assembly assembly) + { + if (assembly == null) + { + throw new ArgumentNullException(nameof(assembly)); + } + + string path = assembly.Location?.Trim(); + if (string.IsNullOrEmpty(path)) + { + return null; + } + + path = Path.Combine(path, ".xunit.runner.json"); + if (!File.Exists(path)) + { + return null; + } + + return File.OpenRead(path); + } + + protected virtual TestAssemblyConfiguration GetConfiguration(Assembly assembly) + { + if (assembly == null) + { + throw new ArgumentNullException(nameof(assembly)); + } + + Stream configStream = GetConfigurationFileStream(assembly); + if (configStream != null) + { + using (configStream) + { + return ConfigReader.Load(configStream); + } + } + + return null; + } + + protected virtual ITestFrameworkDiscoveryOptions GetFrameworkOptionsForDiscovery(TestAssemblyConfiguration configuration) + { + if (configuration == null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + return TestFrameworkOptions.ForDiscovery(configuration); + } + + protected virtual ITestFrameworkExecutionOptions GetFrameworkOptionsForExecution(TestAssemblyConfiguration configuration) + { + if (configuration == null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + return TestFrameworkOptions.ForExecution(configuration); + } + + private async Task Run(Assembly assembly, string assemblyPath) + { + //MF: XunitFrontController is checking file existence + //using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false)) + + var diagnosticSink = new MyConsoleDiagnosticMessageSink(); + using (var frontController = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyPath, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false)) + { + using (var discoverySink = new TestDiscoverySink()) + { + var configuration = GetConfiguration(assembly) ?? new TestAssemblyConfiguration() { PreEnumerateTheories = false }; + ITestFrameworkDiscoveryOptions discoveryOptions = GetFrameworkOptionsForDiscovery(configuration); + discoveryOptions.SetSynchronousMessageReporting(true); + Logger.OnDebug($"Starting test discovery in the '{assembly}' assembly"); + frontController.Find(false, discoverySink, discoveryOptions); + Logger.OnDebug($"Test discovery in assembly '{assembly}' completed"); + discoverySink.Finished.WaitOne(); + + if (discoverySink.TestCases == null || discoverySink.TestCases.Count == 0) + { + Logger.Info("No test cases discovered"); + return null; + } + + TotalTests += discoverySink.TestCases.Count; + List testCases; + if (_filters != null && _filters.TestCaseFilters.Any()) + { + Action log = LogExcludedTests ? (s) => do_log(s) : (Action)null; + testCases = discoverySink.TestCases.Where( + tc => !_filters.IsExcluded(tc, log)).ToList(); + FilteredTests += discoverySink.TestCases.Count - testCases.Count; + } + else + { + testCases = discoverySink.TestCases; + } + + var summaryTaskSource = new TaskCompletionSource(); + var resultsXmlAssembly = new XElement("assembly"); + var resultsSink = new DelegatingXmlCreationSink(new DelegatingExecutionSummarySink(_messageSink), resultsXmlAssembly); + var completionSink = new CompletionCallbackExecutionSink(resultsSink, summary => summaryTaskSource.SetResult(summary)); + + ITestFrameworkExecutionOptions executionOptions = GetFrameworkOptionsForExecution(configuration); + executionOptions.SetDisableParallelization(!RunInParallel); + executionOptions.SetSynchronousMessageReporting(true); + executionOptions.SetMaxParallelThreads(MaxParallelThreads); + + frontController.RunTests(testCases, completionSink, executionOptions); + await summaryTaskSource.Task.ConfigureAwait(false); + + return resultsXmlAssembly; + } + } + } +} diff --git a/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs b/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs new file mode 100644 index 00000000000000..ec490e552cd468 --- /dev/null +++ b/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs @@ -0,0 +1,87 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. +#nullable disable + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.DotNet.XHarness.TestRunners.Common; + +#nullable enable +namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; +public abstract class MyXunitTestRunnerBase : TestRunner +{ + private protected XUnitFiltersCollection _filters = new(); + + protected MyXunitTestRunnerBase(LogWriter logger) : base(logger) + { + } + + public override void SkipTests(IEnumerable tests) + { + if (tests.Any()) + { + // create a single filter per test + foreach (var t in tests) + { + if (t.StartsWith("KLASS:", StringComparison.Ordinal)) + { + var klass = t.Replace("KLASS:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("KLASS32:", StringComparison.Ordinal) && IntPtr.Size == 4) + { + var klass = t.Replace("KLASS32:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("KLASS64:", StringComparison.Ordinal) && IntPtr.Size == 8) + { + var klass = t.Replace("KLASS32:", ""); + _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); + } + else if (t.StartsWith("Platform32:", StringComparison.Ordinal) && IntPtr.Size == 4) + { + var filter = t.Replace("Platform32:", ""); + _filters.Add(XUnitFilter.CreateSingleFilter(filter, true)); + } + else + { + _filters.Add(XUnitFilter.CreateSingleFilter(t, true)); + } + } + } + } + + public override void SkipCategories(IEnumerable categories) => SkipCategories(categories, isExcluded: true); + + public virtual void SkipCategories(IEnumerable categories, bool isExcluded) + { + if (categories == null) + { + throw new ArgumentNullException(nameof(categories)); + } + + foreach (var c in categories) + { + var traitInfo = c.Split('='); + if (traitInfo.Length == 2) + { + _filters.Add(XUnitFilter.CreateTraitFilter(traitInfo[0], traitInfo[1], isExcluded)); + } + else + { + _filters.Add(XUnitFilter.CreateTraitFilter(c, null, isExcluded)); + } + } + } + + public override void SkipMethod(string method, bool isExcluded) + => _filters.Add(XUnitFilter.CreateSingleFilter(singleTestName: method, exclude: isExcluded)); + + public override void SkipClass(string className, bool isExcluded) + => _filters.Add(XUnitFilter.CreateClassFilter(className: className, exclude: isExcluded)); + + public virtual void SkipNamespace(string namespaceName, bool isExcluded) + => _filters.Add(XUnitFilter.CreateNamespaceFilter(namespaceName, exclude: isExcluded)); +} From dde8bd0762c7a659b867334c814c345256567b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Wed, 30 Aug 2023 13:54:15 +0200 Subject: [PATCH 02/36] Fix -mt argument position --- eng/testing/tests.browser.targets | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 12bcc7b168facf..143fbdae0d7c7a 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -97,7 +97,7 @@ $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=1 - $(WasmXHarnessMonoArgs) --no-memory-snapshot -mt + $(WasmXHarnessMonoArgs) --no-memory-snapshot $(WasmXHarnessMonoArgs) --setenv=IsBrowserThreadingSupported=true @@ -116,6 +116,7 @@ <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) + <_AppArgs Condition="'$(MonoWasmBuildVariant)' == 'multithread'">$(_AppArgs) -mt $HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) %24WasmTestAppArgs %HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs% -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_AppArgs) %WasmTestAppArgs% From 5e08c6b210db620ee9b18c66f9f97b2922c28fbf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 10 Nov 2023 14:48:14 +0100 Subject: [PATCH 03/36] Add test assembly to VFS --- eng/testing/tests.browser.targets | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index b69fea94c4f92c..2c649dc91280ab 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -166,6 +166,12 @@ PrepareForWasmBuildApp;$(WasmNestedPublishAppDependsOn) + + + + + + $(BundleDir) From 1fe73a4c28e1c63d9b3e6380bfeaa7ecb284bcc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 10 Nov 2023 14:50:32 +0100 Subject: [PATCH 04/36] Revert to use XunitFrontController --- .../Common/tests/WasmTestRunner/XUnitTestRunner.cs | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs index ad5586df29dde9..78b8aad1da6d33 100644 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -1050,10 +1050,9 @@ protected virtual ITestFrameworkExecutionOptions GetFrameworkOptionsForExecution private async Task Run(Assembly assembly, string assemblyPath) { //MF: XunitFrontController is checking file existence - //using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false)) - - var diagnosticSink = new MyConsoleDiagnosticMessageSink(); - using (var frontController = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyPath, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false)) + using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false)) + // var diagnosticSink = new MyConsoleDiagnosticMessageSink(); + // using (var frontController = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyPath, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false)) { using (var discoverySink = new TestDiscoverySink()) { From d07c3f876c97ae175f050b81bb26d0c23481ccf0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 10 Nov 2023 19:26:32 +0100 Subject: [PATCH 05/36] Fix duplicities in filesystem items --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 2c649dc91280ab..1f9f9ec5dd1b47 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -168,7 +168,7 @@ - + From 21b9f38d7d8e8148a7aaf66034af7a5eeb442171 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 13 Nov 2023 11:46:53 +0100 Subject: [PATCH 06/36] Allow to specify TargetDir for WasmFilesToIncludeFromPublishDir --- eng/testing/tests.browser.targets | 2 +- .../System.IO.FileSystem.DisabledFileLocking.Tests.csproj | 3 +++ .../tests/System.IO.FileSystem.Tests.csproj | 3 +++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 1f9f9ec5dd1b47..908bd63e58f652 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -222,7 +222,7 @@ diff --git a/src/libraries/System.IO.FileSystem/tests/DisabledFileLockingTests/System.IO.FileSystem.DisabledFileLocking.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/DisabledFileLockingTests/System.IO.FileSystem.DisabledFileLocking.Tests.csproj index 8f474ca378fc9d..62425e583e9145 100644 --- a/src/libraries/System.IO.FileSystem/tests/DisabledFileLockingTests/System.IO.FileSystem.DisabledFileLocking.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/DisabledFileLockingTests/System.IO.FileSystem.DisabledFileLocking.Tests.csproj @@ -7,6 +7,9 @@ --working-dir=/test-dir + + + diff --git a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj index 738960903f5045..c2d3f67d0d556c 100644 --- a/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj +++ b/src/libraries/System.IO.FileSystem/tests/System.IO.FileSystem.Tests.csproj @@ -6,6 +6,9 @@ --working-dir=/test-dir + + + From c1aecce928cf0a8aff6b58f5c7a9c3fd259ea0cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 13 Nov 2023 14:38:07 +0100 Subject: [PATCH 07/36] Rename -mt argument so it doesn't clash with -m filter. Sigh --- eng/testing/tests.browser.targets | 2 +- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 908bd63e58f652..3ffc880cb032e0 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -128,7 +128,7 @@ <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) - <_AppArgs Condition="'$(MonoWasmBuildVariant)' == 'multithread'">$(_AppArgs) -mt + <_AppArgs Condition="'$(MonoWasmBuildVariant)' == 'multithread'">$(_AppArgs) -threads $HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) %24WasmTestAppArgs %HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs% -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_AppArgs) %WasmTestAppArgs% diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index dc3e1640efec4f..d328ce972fd99f 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -53,7 +53,7 @@ public static async Task Main(string[] args) includedMethods.Add(args[i + 1]); i++; break; - case "-mt": + case "-threads": break; default: throw new ArgumentException($"Invalid argument '{option}'."); @@ -61,7 +61,7 @@ public static async Task Main(string[] args) } WasmApplicationEntryPointBase? runner = null; - if (args.Contains("-mt")) + if (args.Contains("-threads")) { Console.WriteLine("Using MultiThreadedTestRunner"); runner = new MultiThreadedTestRunner() From 0200dd86d7938ac7710bb079e01d34c92be814f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 13 Nov 2023 15:07:51 +0100 Subject: [PATCH 08/36] Log xunit start event is XHARNESS_LOG_TEST_START is set --- src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs index 78b8aad1da6d33..8dac73cf3418c7 100644 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -125,6 +125,11 @@ private void HandleEvent(string name, MessageHandlerArgs args, Action args) { + if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) + { + OnInfo($"\t[STRT] {args.Message.Test.DisplayName}"); + } + if (args == null || args.Message == null) { return; From e09edaf7176beb2e176609257ff99344ad02253b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 13 Nov 2023 15:09:47 +0100 Subject: [PATCH 09/36] Move start log after arg check --- .../Common/tests/WasmTestRunner/XUnitTestRunner.cs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs index 8dac73cf3418c7..4fa41f44967666 100644 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -125,14 +125,14 @@ private void HandleEvent(string name, MessageHandlerArgs args, Action args) { - if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) + if (args == null || args.Message == null) { - OnInfo($"\t[STRT] {args.Message.Test.DisplayName}"); + return; } - if (args == null || args.Message == null) + if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) { - return; + OnInfo($"\t[STRT] {args.Message.Test.DisplayName}"); } OnDebug("Test starting"); From f24ec62a18b2e878605817e49b228052782c09a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Wed, 15 Nov 2023 14:50:33 +0100 Subject: [PATCH 10/36] Disable PMDesignator_Get_ReturnsExpected_HybridGlobalization test on MT --- .../DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs index 5eeea693a400a5..530b6dfae52263 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs @@ -205,6 +205,7 @@ public static IEnumerable PMDesignator_Get_TestData_HybridGlobalizatio yield return new object[] { new CultureInfo("zh-TW").DateTimeFormat, "下午" }; } + [ActiveIssue("https://github.com/dotnet/runtime/issues/75123", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsHybridGlobalizationOnBrowser))] [MemberData(nameof(PMDesignator_Get_TestData_HybridGlobalization))] public void PMDesignator_Get_ReturnsExpected_HybridGlobalization(DateTimeFormatInfo format, string value) From 6c775c68bec2f6f36e2710f579cf6d3343c95b6d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 26 Jan 2024 12:25:17 +0100 Subject: [PATCH 11/36] Remove comments --- src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs | 3 --- 1 file changed, 3 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs index 4fa41f44967666..01fa14e7e5d8a1 100644 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -1054,10 +1054,7 @@ protected virtual ITestFrameworkExecutionOptions GetFrameworkOptionsForExecution private async Task Run(Assembly assembly, string assemblyPath) { - //MF: XunitFrontController is checking file existence using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false)) - // var diagnosticSink = new MyConsoleDiagnosticMessageSink(); - // using (var frontController = new Xunit2(AppDomainSupport.Denied, new NullSourceInformationProvider(), assemblyPath, configFileName: null, shadowCopy: false, shadowCopyFolder: null, diagnosticMessageSink: diagnosticSink, verifyTestAssemblyExists: false)) { using (var discoverySink = new TestDiscoverySink()) { From 76fdc01560dc139c33d4f86671f55bdac60f1a25 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 26 Jan 2024 14:43:25 +0100 Subject: [PATCH 12/36] Ignore warning --- src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs index 01fa14e7e5d8a1..262e9ef069833d 100644 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs @@ -1088,7 +1088,9 @@ private async Task Run(Assembly assembly, string assemblyPath) var summaryTaskSource = new TaskCompletionSource(); var resultsXmlAssembly = new XElement("assembly"); +#pragma warning disable CS0618 // Delegating*Sink types are marked obsolete, but we can't move to ExecutionSink yet: https://github.com/dotnet/arcade/issues/14375 var resultsSink = new DelegatingXmlCreationSink(new DelegatingExecutionSummarySink(_messageSink), resultsXmlAssembly); +#pragma warning restore var completionSink = new CompletionCallbackExecutionSink(resultsSink, summary => summaryTaskSource.SetResult(summary)); ITestFrameworkExecutionOptions executionOptions = GetFrameworkOptionsForExecution(configuration); From 87f4917e4e94e6c3f483a152074afba974d5719f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 09:18:57 +0100 Subject: [PATCH 13/36] Update WasmTestRunner based on changes in XHarness --- .../CompletionCallbackExecutionSink.cs | 37 - .../ConsoleDiagnosticMessageSink.cs | 22 - .../Common/tests/WasmTestRunner/Extensions.cs | 10 - .../tests/WasmTestRunner/WasmTestRunner.cs | 87 +- .../WasmTestRunner/WasmTestRunner.csproj | 8 - .../tests/WasmTestRunner/XUnitFilter.cs | 313 ----- .../tests/WasmTestRunner/XUnitFilterType.cs | 14 - .../WasmTestRunner/XUnitFiltersCollection.cs | 80 -- .../tests/WasmTestRunner/XUnitTestRunner.cs | 1108 ----------------- .../WasmTestRunner/XunitTestRunnerBase.cs | 87 -- src/mono/wasm/sln/WasmBuild.sln | 8 +- 11 files changed, 21 insertions(+), 1753 deletions(-) delete mode 100644 src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/Extensions.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs delete mode 100644 src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs diff --git a/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs b/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs deleted file mode 100644 index c344797432cc84..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/CompletionCallbackExecutionSink.cs +++ /dev/null @@ -1,37 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Threading; -using Xunit; -using Xunit.Abstractions; - -#nullable enable -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; - -internal class CompletionCallbackExecutionSink : global::Xunit.Sdk.LongLivedMarshalByRefObject, IExecutionSink -{ - private readonly Action _completionCallback; - private readonly IExecutionSink _innerSink; - - public ExecutionSummary ExecutionSummary => _innerSink.ExecutionSummary; - - public ManualResetEvent Finished => _innerSink.Finished; - - public CompletionCallbackExecutionSink(IExecutionSink innerSink, Action completionCallback) - { - _innerSink = innerSink; - _completionCallback = completionCallback; - } - - public void Dispose() => _innerSink.Dispose(); - - public bool OnMessageWithTypes(IMessageSinkMessage message, HashSet messageTypes) - { - var result = _innerSink.OnMessageWithTypes(message, messageTypes); - message.Dispatch(messageTypes, args => _completionCallback(ExecutionSummary)); - return result; - } -} diff --git a/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs b/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs deleted file mode 100644 index c76ca047030712..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/ConsoleDiagnosticMessageSink.cs +++ /dev/null @@ -1,22 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. - -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; -using Xunit.Abstractions; - -public class MyConsoleDiagnosticMessageSink : global::Xunit.Sdk.LongLivedMarshalByRefObject, IMessageSink -{ - public bool OnMessage(IMessageSinkMessage message) - { - if (message is IDiagnosticMessage diagnosticMessage) - { - Console.WriteLine(diagnosticMessage.Message); - } - - return true; - } -} diff --git a/src/libraries/Common/tests/WasmTestRunner/Extensions.cs b/src/libraries/Common/tests/WasmTestRunner/Extensions.cs deleted file mode 100644 index a28df3be3d9c1e..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/Extensions.cs +++ /dev/null @@ -1,10 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.DotNet.XHarness.TestRunners.Common; - -internal static partial class Extensions -{ - public static string YesNo(this bool b) => b ? "yes" : "no"; -} diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 628cd1c0d46352..b623cd5601d4bf 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -28,6 +28,7 @@ public static async Task Main(string[] args) var includedClasses = new List(); var includedMethods = new List(); var backgroundExec = false; + var isThreadless = true; for (int i = 1; i < args.Length; i++) { @@ -58,94 +59,34 @@ public static async Task Main(string[] args) backgroundExec = true; break; case "-threads": + isThreadless = false; break; default: throw new ArgumentException($"Invalid argument '{option}'."); } } - WasmApplicationEntryPointBase? runner = null; - if (args.Contains("-threads")) + var runner = new SimpleWasmTestRunner() { - Console.WriteLine("Using MultiThreadedTestRunner"); - runner = new MultiThreadedTestRunner() - { - TestAssembly = testAssembly, - ExcludedTraits = excludedTraits, - IncludedTraits = includedTraits, - IncludedNamespaces = includedNamespaces, - IncludedClasses = includedClasses, - IncludedMethods = includedMethods - }; - } - else - { - Console.WriteLine("Using SimpleWasmTestRunner"); - runner = new SimpleWasmTestRunner() - { - TestAssembly = testAssembly, - ExcludedTraits = excludedTraits, - IncludedTraits = includedTraits, - IncludedNamespaces = includedNamespaces, - IncludedClasses = includedClasses, - IncludedMethods = includedMethods - }; - } + TestAssembly = testAssembly, + ExcludedTraits = excludedTraits, + IncludedTraits = includedTraits, + IncludedNamespaces = includedNamespaces, + IncludedClasses = includedClasses, + IncludedMethods = includedMethods, + IsThreadless = isThreadless + }; if (OperatingSystem.IsBrowser()) { await Task.Yield(); } - if (backgroundExec) - { - await Task.Run(async () => await runner.RunAsync()); - return runner.LastRunHadFailedTests ? 1 : 0; - } - - await runner.RunAsync(); - return runner.LastRunHadFailedTests ? 1 : 0; - } -} - -class MultiThreadedTestRunner : WasmApplicationEntryPointBase -{ - public virtual string TestAssembly { get; set; } = ""; - public virtual IEnumerable ExcludedTraits { get; set; } = Array.Empty(); - public virtual IEnumerable IncludedTraits { get; set; } = Array.Empty(); - public virtual IEnumerable IncludedClasses { get; set; } = Array.Empty(); - public virtual IEnumerable IncludedMethods { get; set; } = Array.Empty(); - public virtual IEnumerable IncludedNamespaces { get; set; } = Array.Empty(); - - protected override bool IsXunit => true; - - protected override TestRunner GetTestRunner(LogWriter logWriter) - { - var runner = new MyXUnitTestRunner(logWriter); - //var xUnitTestRunnerType = typeof(XunitTestRunnerBase).Assembly.GetType("Microsoft.DotNet.XHarness.TestRunners.Xunit.XUnitTestRunner"); - //var runner = (XunitTestRunnerBase)xUnitTestRunnerType!.GetConstructors().First().Invoke(new[] { logWriter }); - - //ConfigureRunnerFilters(runner, ApplicationOptions.Current); - var configureRunnerFiltersMethod = typeof(ApplicationEntryPoint).GetMethod("ConfigureRunnerFilters", BindingFlags.Static | BindingFlags.NonPublic); - configureRunnerFiltersMethod!.Invoke(null, new object[] { runner, ApplicationOptions.Current }); - runner.SkipCategories(ExcludedTraits); - runner.SkipCategories(IncludedTraits, isExcluded: false); - foreach (var cls in IncludedClasses) - { - runner.SkipClass(cls, false); - } - foreach (var method in IncludedMethods) - { - runner.SkipMethod(method, false); - } - foreach (var ns in IncludedNamespaces) + if (backgroundExec) { - runner.SkipNamespace(ns, isExcluded: false); + return await Task.Run(() => runner.Run()); } - return runner; + return await runner.Run(); } - - protected override IEnumerable GetTestAssemblies() - => new[] { new TestAssemblyInfo(Assembly.LoadFrom(TestAssembly), TestAssembly) }; } diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj index 8260105d391e09..2f60f08a55c44a 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.csproj @@ -5,15 +5,7 @@ $(NetCoreAppCurrent) - - - - - - - - diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs deleted file mode 100644 index 18c7894e528779..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitFilter.cs +++ /dev/null @@ -1,313 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.IO; -using System.Linq; -using System.Text; -using Microsoft.DotNet.XHarness.TestRunners.Common; -using Xunit.Abstractions; - -#nullable enable -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; - -internal class XUnitFilter -{ - public string? AssemblyName { get; private set; } - public string? SelectorName { get; private set; } - public string? SelectorValue { get; private set; } - - public bool Exclude { get; private set; } - public XUnitFilterType FilterType { get; private set; } - - public static XUnitFilter CreateSingleFilter(string singleTestName, bool exclude, string? assemblyName = null) - { - if (string.IsNullOrEmpty(singleTestName)) - { - throw new ArgumentException("must not be null or empty", nameof(singleTestName)); - } - - return new XUnitFilter - { - AssemblyName = assemblyName, - SelectorValue = singleTestName, - FilterType = XUnitFilterType.Single, - Exclude = exclude - }; - } - - public static XUnitFilter CreateAssemblyFilter(string assemblyName, bool exclude) - { - if (string.IsNullOrEmpty(assemblyName)) - { - throw new ArgumentException("must not be null or empty", nameof(assemblyName)); - } - - // ensure that the assembly name does have one of the valid extensions - var fileExtension = Path.GetExtension(assemblyName); - if (fileExtension != ".dll" && fileExtension != ".exe") - { - throw new ArgumentException($"Assembly name must have .dll or .exe as extensions. Found extension {fileExtension}"); - } - - return new XUnitFilter - { - AssemblyName = assemblyName, - FilterType = XUnitFilterType.Assembly, - Exclude = exclude - }; - } - - public static XUnitFilter CreateNamespaceFilter(string namespaceName, bool exclude, string? assemblyName = null) - { - if (string.IsNullOrEmpty(namespaceName)) - { - throw new ArgumentException("must not be null or empty", nameof(namespaceName)); - } - - return new XUnitFilter - { - AssemblyName = assemblyName, - SelectorValue = namespaceName, - FilterType = XUnitFilterType.Namespace, - Exclude = exclude - }; - } - - public static XUnitFilter CreateClassFilter(string className, bool exclude, string? assemblyName = null) - { - if (string.IsNullOrEmpty(className)) - { - throw new ArgumentException("must not be null or empty", nameof(className)); - } - - return new XUnitFilter - { - AssemblyName = assemblyName, - SelectorValue = className, - FilterType = XUnitFilterType.TypeName, - Exclude = exclude - }; - } - - public static XUnitFilter CreateTraitFilter(string traitName, string? traitValue, bool exclude) - { - if (string.IsNullOrEmpty(traitName)) - { - throw new ArgumentException("must not be null or empty", nameof(traitName)); - } - - return new XUnitFilter - { - AssemblyName = null, - SelectorName = traitName, - SelectorValue = traitValue ?? string.Empty, - FilterType = XUnitFilterType.Trait, - Exclude = exclude - }; - } - - private bool ApplyTraitFilter(ITestCase testCase, Func? reportFilteredTest = null) - { - Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; - - if (!testCase.HasTraits()) - { - return log(!Exclude); - } - - if (testCase.TryGetTrait(SelectorName!, out var values)) - { - if (values == null || values.Count == 0) - { - // We have no values and the filter doesn't specify one - that means we match on - // the trait name only. - if (string.IsNullOrEmpty(SelectorValue)) - { - return log(Exclude); - } - - return log(!Exclude); - } - - return values.Any(value => value.Equals(SelectorValue, StringComparison.InvariantCultureIgnoreCase)) ? - log(Exclude) : log(!Exclude); - } - - // no traits found, that means that we return the opposite of the setting of the filter - return log(!Exclude); - } - - private bool ApplyTypeNameFilter(ITestCase testCase, Func? reportFilteredTest = null) - { - Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; - var testClassName = testCase.GetTestClass(); - if (!string.IsNullOrEmpty(testClassName)) - { - if (string.Equals(testClassName, SelectorValue, StringComparison.InvariantCulture)) - { - return log(Exclude); - } - } - - return log(!Exclude); - } - - private bool ApplySingleFilter(ITestCase testCase, Func? reportFilteredTest = null) - { - Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; - if (string.Equals(testCase.DisplayName, SelectorValue, StringComparison.InvariantCulture)) - { - // if there is a match, return the exclude value - return log(Exclude); - } - // if there is not match, return the opposite - return log(!Exclude); - } - - private bool ApplyNamespaceFilter(ITestCase testCase, Func? reportFilteredTest = null) - { - Func log = (result) => reportFilteredTest?.Invoke(result) ?? result; - var testClassNamespace = testCase.GetNamespace(); - if (string.IsNullOrEmpty(testClassNamespace)) - { - // if we exclude, since we have no namespace, we include the test - return log(!Exclude); - } - - if (string.Equals(testClassNamespace, SelectorValue, StringComparison.InvariantCultureIgnoreCase)) - { - return log(Exclude); - } - - // same logic as with no namespace - return log(!Exclude); - } - - public bool IsExcluded(TestAssemblyInfo assembly, Action? reportFilteredAssembly = null) - { - if (FilterType != XUnitFilterType.Assembly) - { - throw new InvalidOperationException("Filter is not targeting assemblies."); - } - - Func log = (result) => ReportFilteredAssembly(assembly, result, reportFilteredAssembly); - - if (string.Equals(AssemblyName, assembly.FullPath, StringComparison.Ordinal)) - { - return log(Exclude); - } - - string fileName = Path.GetFileName(assembly.FullPath); - if (string.Equals(fileName, AssemblyName, StringComparison.Ordinal)) - { - return log(Exclude); - } - - // No path of the name matched the filter, therefore return the opposite of the Exclude value - return log(!Exclude); - } - - public bool IsExcluded(ITestCase testCase, Action? log = null) - { - Func? reportFilteredTest = null; - if (log != null) - { - reportFilteredTest = (result) => ReportFilteredTest(testCase, result, log); - } - - return FilterType switch - { - XUnitFilterType.Trait => ApplyTraitFilter(testCase, reportFilteredTest), - XUnitFilterType.TypeName => ApplyTypeNameFilter(testCase, reportFilteredTest), - XUnitFilterType.Single => ApplySingleFilter(testCase, reportFilteredTest), - XUnitFilterType.Namespace => ApplyNamespaceFilter(testCase, reportFilteredTest), - _ => throw new InvalidOperationException($"Unsupported filter type {FilterType}") - }; - } - - private bool ReportFilteredTest(ITestCase testCase, bool excluded, Action? log = null) - { - const string includedText = "Included"; - const string excludedText = "Excluded"; - - if (log == null) - { - return excluded; - } - - var selector = FilterType == XUnitFilterType.Trait ? - $"'{SelectorName}':'{SelectorValue}'" : $"'{SelectorValue}'"; - - log($"[FILTER] {(excluded ? excludedText : includedText)} test (filtered by {FilterType}; {selector}): {testCase.DisplayName}"); - return excluded; - } - - private static bool ReportFilteredAssembly(TestAssemblyInfo assemblyInfo, bool excluded, Action? log = null) - { - if (log == null) - { - return excluded; - } - - const string includedPrefix = "Included"; - const string excludedPrefix = "Excluded"; - - log($"[FILTER] {(excluded ? excludedPrefix : includedPrefix)} assembly: {assemblyInfo.FullPath}"); - return excluded; - } - - private static void AppendDesc(StringBuilder sb, string name, string? value) - { - if (string.IsNullOrEmpty(value)) - { - return; - } - - sb.Append($"; {name}: {value}"); - } - - public override string ToString() - { - var sb = new StringBuilder("XUnitFilter ["); - - sb.Append($"Type: {FilterType}; "); - sb.Append(Exclude ? "exclude" : "include"); - - if (!string.IsNullOrEmpty(AssemblyName)) - { - sb.Append($"; AssemblyName: {AssemblyName}"); - } - - switch (FilterType) - { - case XUnitFilterType.Assembly: - break; - - case XUnitFilterType.Namespace: - AppendDesc(sb, "Namespace", SelectorValue); - break; - - case XUnitFilterType.Single: - AppendDesc(sb, "Method", SelectorValue); - break; - - case XUnitFilterType.Trait: - AppendDesc(sb, "Trait name", SelectorName); - AppendDesc(sb, "Trait value", SelectorValue); - break; - - case XUnitFilterType.TypeName: - AppendDesc(sb, "Class", SelectorValue); - break; - - default: - sb.Append("; Unknown filter type"); - break; - } - sb.Append(']'); - - return sb.ToString(); - } -} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs deleted file mode 100644 index dee5e272585f29..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitFilterType.cs +++ /dev/null @@ -1,14 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; - -internal enum XUnitFilterType -{ - Trait, - TypeName, - Assembly, - Single, - Namespace, -} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs deleted file mode 100644 index 97e2587131dd85..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitFiltersCollection.cs +++ /dev/null @@ -1,80 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.DotNet.XHarness.TestRunners.Common; -using Xunit.Abstractions; - -#nullable enable -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; - -/// -/// Class that contains a collection of filters and can be used to decide if a test should be executed or not. -/// -internal class XUnitFiltersCollection : List -{ - /// - /// Return all the filters that are applied to assemblies. - /// - public IEnumerable AssemblyFilters - => Enumerable.Where(this, f => f.FilterType == XUnitFilterType.Assembly); - - /// - /// Return all the filters that are applied to test cases. - /// - public IEnumerable TestCaseFilters - => Enumerable.Where(this, f => f.FilterType != XUnitFilterType.Assembly); - - // loop over all the filters, if we have conflicting filters, that is, one exclude and other one - // includes, we will always include since it is better to run a test thant to skip it and think - // you ran in. - private bool IsExcludedInternal(IEnumerable filters, Func isExcludedCb) - { - // No filters : include by default - // Any exclude filters : include by default - // Only include filters : exclude by default - var isExcluded = filters.Any() && filters.All(f => !f.Exclude); - foreach (var filter in filters) - { - var doesExclude = isExcludedCb(filter); - if (filter.Exclude) - { - isExcluded |= doesExclude; - } - else - { - // filter does not exclude, that means that if it include, we should include and break the - // loop, always include - if (!doesExclude) - { - return false; - } - } - } - - return isExcluded; - } - - public bool IsExcluded(TestAssemblyInfo assembly, Action? log = null) => - IsExcludedInternal(AssemblyFilters, f => f.IsExcluded(assembly, log)); - - public bool IsExcluded(ITestCase testCase, Action? log = null) - { - // Check each type of filter separately. For conflicts within a type of filter, we want the inclusion - // (the logic in IsExcludedInternal), but if all filters for a filter type exclude a test case, we want - // the exclusion. For example, if a test class is included, but it contains tests that have excluded - // traits, the behaviour should be to run all tests in that class without the excluded traits. - foreach (IGrouping filterGroup in TestCaseFilters.GroupBy(f => f.FilterType)) - { - if (IsExcludedInternal(filterGroup, f => f.IsExcluded(testCase, log))) - { - return true; - } - } - - return false; - } -} diff --git a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs deleted file mode 100644 index 262e9ef069833d..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/XUnitTestRunner.cs +++ /dev/null @@ -1,1108 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. -#nullable disable - -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using System.Text; -using System.Threading; -using System.Threading.Tasks; -using System.Xml; -using System.Xml.Linq; -using System.Xml.Xsl; -using Microsoft.DotNet.XHarness.Common; -using Microsoft.DotNet.XHarness.TestRunners.Common; -using Xunit; -using Xunit.Abstractions; - -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; - -internal class XsltIdGenerator -{ - // NUnit3 xml does not have schema, there is no much info about it, most examples just have incremental IDs. - private int _seed = 1000; - public int GenerateHash() => _seed++; -} - -internal class MyXUnitTestRunner : MyXunitTestRunnerBase -{ - private readonly TestMessageSink _messageSink; - - public int? MaxParallelThreads { get; set; } - - private XElement _assembliesElement; - - public AppDomainSupport AppDomainSupport { get; set; } = AppDomainSupport.Denied; - protected override string ResultsFileName { get; set; } = "TestResults.xUnit.xml"; - - public MyXUnitTestRunner(LogWriter logger) : base(logger) - { - _messageSink = new TestMessageSink(); - - _messageSink.Diagnostics.DiagnosticMessageEvent += HandleDiagnosticMessage; - _messageSink.Diagnostics.ErrorMessageEvent += HandleDiagnosticErrorMessage; - - _messageSink.Discovery.DiscoveryCompleteMessageEvent += HandleDiscoveryCompleteMessage; - _messageSink.Discovery.TestCaseDiscoveryMessageEvent += HandleDiscoveryTestCaseMessage; - - _messageSink.Runner.TestAssemblyDiscoveryFinishedEvent += HandleTestAssemblyDiscoveryFinished; - _messageSink.Runner.TestAssemblyDiscoveryStartingEvent += HandleTestAssemblyDiscoveryStarting; - _messageSink.Runner.TestAssemblyExecutionFinishedEvent += HandleTestAssemblyExecutionFinished; - _messageSink.Runner.TestAssemblyExecutionStartingEvent += HandleTestAssemblyExecutionStarting; - _messageSink.Runner.TestExecutionSummaryEvent += HandleTestExecutionSummary; - - _messageSink.Execution.AfterTestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("AfterTestFinishedEvent", args, HandleAfterTestFinished); - _messageSink.Execution.AfterTestStartingEvent += (MessageHandlerArgs args) => HandleEvent("AfterTestStartingEvent", args, HandleAfterTestStarting); - _messageSink.Execution.BeforeTestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("BeforeTestFinishedEvent", args, HandleBeforeTestFinished); - _messageSink.Execution.BeforeTestStartingEvent += (MessageHandlerArgs args) => HandleEvent("BeforeTestStartingEvent", args, HandleBeforeTestStarting); - _messageSink.Execution.TestAssemblyCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyCleanupFailureEvent", args, HandleTestAssemblyCleanupFailure); - _messageSink.Execution.TestAssemblyFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyFinishedEvent", args, HandleTestAssemblyFinished); - _messageSink.Execution.TestAssemblyStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestAssemblyStartingEvent", args, HandleTestAssemblyStarting); - _messageSink.Execution.TestCaseCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCaseCleanupFailureEvent", args, HandleTestCaseCleanupFailure); - _messageSink.Execution.TestCaseFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestCaseFinishedEvent", args, HandleTestCaseFinished); - _messageSink.Execution.TestCaseStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestStartingEvent", args, HandleTestCaseStarting); - _messageSink.Execution.TestClassCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestClassCleanupFailureEvent", args, HandleTestClassCleanupFailure); - _messageSink.Execution.TestClassConstructionFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassConstructionFinishedEvent", args, HandleTestClassConstructionFinished); - _messageSink.Execution.TestClassConstructionStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassConstructionStartingEvent", args, HandleTestClassConstructionStarting); - _messageSink.Execution.TestClassDisposeFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassDisposeFinishedEvent", args, HandleTestClassDisposeFinished); - _messageSink.Execution.TestClassDisposeStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassDisposeStartingEvent", args, HandleTestClassDisposeStarting); - _messageSink.Execution.TestClassFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestClassFinishedEvent", args, HandleTestClassFinished); - _messageSink.Execution.TestClassStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestClassStartingEvent", args, HandleTestClassStarting); - _messageSink.Execution.TestCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCleanupFailureEvent", args, HandleTestCleanupFailure); - _messageSink.Execution.TestCollectionCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionCleanupFailureEvent", args, HandleTestCollectionCleanupFailure); - _messageSink.Execution.TestCollectionFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionFinishedEvent", args, HandleTestCollectionFinished); - _messageSink.Execution.TestCollectionStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestCollectionStartingEvent", args, HandleTestCollectionStarting); - _messageSink.Execution.TestFailedEvent += (MessageHandlerArgs args) => HandleEvent("TestFailedEvent", args, HandleTestFailed); - _messageSink.Execution.TestFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestFinishedEvent", args, HandleTestFinished); - _messageSink.Execution.TestMethodCleanupFailureEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodCleanupFailureEvent", args, HandleTestMethodCleanupFailure); - _messageSink.Execution.TestMethodFinishedEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodFinishedEvent", args, HandleTestMethodFinished); - _messageSink.Execution.TestMethodStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestMethodStartingEvent", args, HandleTestMethodStarting); - _messageSink.Execution.TestOutputEvent += (MessageHandlerArgs args) => HandleEvent("TestOutputEvent", args, HandleTestOutput); - _messageSink.Execution.TestPassedEvent += (MessageHandlerArgs args) => HandleEvent("TestPassedEvent", args, HandleTestPassed); - _messageSink.Execution.TestSkippedEvent += (MessageHandlerArgs args) => HandleEvent("TestSkippedEvent", args, HandleTestSkipped); - _messageSink.Execution.TestStartingEvent += (MessageHandlerArgs args) => HandleEvent("TestStartingEvent", args, HandleTestStarting); - } - - public void AddFilter(XUnitFilter filter) - { - if (filter != null) - { - _filters.Add(filter); - } - } - public void SetFilters(List newFilters) - { - if (newFilters == null) - { - _filters = null; - return; - } - - if (_filters == null) - { - _filters = new XUnitFiltersCollection(); - } - - _filters.AddRange(newFilters); - } - - private void HandleEvent(string name, MessageHandlerArgs args, Action> actualHandler) where T : class, IMessageSinkMessage - { - try - { - actualHandler(args); - } - catch (Exception ex) - { - OnError($"Handler for event {name} failed with exception"); - OnError(ex.ToString()); - } - } - - private void HandleTestStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - if (Environment.GetEnvironmentVariable("XHARNESS_LOG_TEST_START") != null) - { - OnInfo($"\t[STRT] {args.Message.Test.DisplayName}"); - } - - OnDebug("Test starting"); - LogTestDetails(args.Message.Test, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - } - - private void HandleTestSkipped(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - RaiseTestSkippedCase(args.Message, args.Message.TestCases, args.Message.TestCase); - } - - private void HandleTestPassed(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - PassedTests++; - OnInfo($"\t[PASS] {args.Message.TestCase.DisplayName}"); - LogTestDetails(args.Message.Test, log: OnDebug); - LogTestOutput(args.Message, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - // notify the completion of the test - OnTestCompleted(( - TestName: args.Message.Test.DisplayName, - TestResult: TestResult.Passed - )); - } - - private void HandleTestOutput(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo(args.Message.Output); - } - - private void HandleTestMethodStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDebug("Test method starting"); - LogTestMethodDetails(args.Message.TestMethod.Method, log: OnDebug); - LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestMethodFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDebug("Test method finished"); - LogTestMethodDetails(args.Message.TestMethod.Method, log: OnDebug); - LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnDebug); - LogSummary(args.Message, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestMethodCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError($"Test method cleanup failure{GetAssemblyInfo(args.Message.TestAssembly)}"); - LogTestMethodDetails(args.Message.TestMethod.Method, log: OnError); - LogTestClassDetails(args.Message.TestMethod.TestClass, log: OnError); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleTestFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - ExecutedTests++; - OnDiagnostic("Test finished"); - LogTestDetails(args.Message.Test, log: OnDiagnostic); - LogTestOutput(args.Message, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - } - - private void HandleTestFailed(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - FailedTests++; - string assemblyInfo = GetAssemblyInfo(args.Message.TestAssembly); - var sb = new StringBuilder($"\t[FAIL] {args.Message.TestCase.DisplayName}"); - LogTestDetails(args.Message.Test, OnError, sb); - sb.AppendLine(); - if (!string.IsNullOrEmpty(assemblyInfo)) - { - sb.AppendLine($" Assembly: {assemblyInfo}"); - } - - LogSourceInformation(args.Message.TestCase.SourceInformation, OnError, sb); - LogFailureInformation(args.Message, OnError, sb); - sb.AppendLine(); - LogTestOutput(args.Message, OnError, sb); - sb.AppendLine(); - if (args.Message.TestCase.Traits != null && args.Message.TestCase.Traits.Count > 0) - { - foreach (var kvp in args.Message.TestCase.Traits) - { - string message = $" Test trait name: {kvp.Key}"; - OnError(message); - sb.AppendLine(message); - - foreach (string v in kvp.Value) - { - message = $" value: {v}"; - OnError(message); - sb.AppendLine(message); - } - } - sb.AppendLine(); - } - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - - FailureInfos.Add(new TestFailureInfo - { - TestName = args.Message.Test?.DisplayName, - Message = sb.ToString() - }); - OnInfo($"\t[FAIL] {args.Message.Test?.TestCase.DisplayName}"); - OnInfo(sb.ToString()); - OnTestCompleted(( - TestName: args.Message.Test?.DisplayName, - TestResult: TestResult.Failed - )); - } - - private void HandleTestCollectionStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"\n{args.Message.TestCollection.DisplayName}"); - OnDebug("Test collection starting"); - LogTestCollectionDetails(args.Message.TestCollection, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestCollectionFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDebug("Test collection finished"); - LogSummary(args.Message, log: OnDebug); - LogTestCollectionDetails(args.Message.TestCollection, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestCollectionCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError("Error during test collection cleanup"); - LogTestCollectionDetails(args.Message.TestCollection, log: OnError); - ReportTestCases(" Associated", args.Message.TestCases, log: OnError); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleTestCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError($"Test cleanup failure{GetAssemblyInfo(args.Message.TestAssembly)}"); - LogTestDetails(args.Message.Test, log: OnError); - ReportTestCases(" Associated", args.Message.TestCases, log: OnError); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleTestClassStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test class starting"); - LogTestClassDetails(args.Message.TestClass, log: OnDiagnostic); - } - - private void HandleTestClassFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDebug("Test class finished"); - OnInfo($"{args.Message.TestClass.Class.Name} {args.Message.ExecutionTime} ms"); - LogTestClassDetails(args.Message.TestClass, OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestClassDisposeStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test class dispose starting"); - LogTestDetails(args.Message.Test, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestClassDisposeFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test class dispose finished"); - LogTestDetails(args.Message.Test, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestClassConstructionStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test class construction starting"); - LogTestDetails(args.Message.Test, OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - } - - private void HandleTestClassConstructionFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test class construction finished"); - LogTestDetails(args.Message.Test, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - } - - private void HandleTestClassCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError($"Test class cleanup error{GetAssemblyInfo(args.Message.TestAssembly)}"); - LogTestClassDetails(args.Message.TestClass, log: OnError); - LogTestCollectionDetails(args.Message.TestCollection, log: OnError); - ReportTestCases(" Associated", args.Message.TestCases, log: OnError); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleTestCaseStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic("Test case starting"); - ReportTestCase(" Starting", args.Message.TestCase, log: OnDiagnostic); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDiagnostic); - } - - private void HandleTestCaseFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDebug("Test case finished executing"); - ReportTestCase(" Finished", args.Message.TestCase, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnDebug); - LogSummary(args.Message, log: OnDebug); - } - - private void HandleTestCaseCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError("Test case cleanup failure"); - ReportTestCase(" Failed", args.Message.TestCase, log: OnError); - ReportTestCases(" Associated", args.Message.TestCases, args.Message.TestCase, OnError); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleTestAssemblyStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"[Test environment: {args.Message.TestEnvironment}]"); - OnInfo($"[Test framework: {args.Message.TestFrameworkDisplayName}]"); - LogAssemblyInformation(args.Message, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDebug); - } - - private void HandleTestAssemblyFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - TotalTests = args.Message.TestsRun; // HACK: We are not counting correctly all the tests - OnDebug("Execution process for assembly finished"); - LogAssemblyInformation(args.Message, log: OnDebug); - LogSummary(args.Message, log: OnDebug); - ReportTestCases(" Associated", args.Message.TestCases, log: OnDiagnostic); - } - - private void HandleTestAssemblyCleanupFailure(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnError("Assembly cleanup failure"); - LogAssemblyInformation(args.Message, OnError); - ReportTestCases(" Associated", args.Message.TestCases, log: OnError); - LogFailureInformation(args.Message, log: OnError); - } - - private void HandleBeforeTestStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - // notify that a method is starting - OnTestStarted(args.Message.Test.DisplayName); - OnDiagnostic($"'Before' method for test '{args.Message.Test.DisplayName}' starting"); - } - - private void HandleBeforeTestFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic($"'Before' method for test '{args.Message.Test.DisplayName}' finished"); - } - - private void HandleAfterTestStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic($"'After' method for test '{args.Message.Test.DisplayName}' starting"); - } - - private void HandleAfterTestFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic($"'After' method for test '{args.Message.Test.DisplayName}' finished"); - } - - private void HandleTestExecutionSummary(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo("All tests finished"); - OnInfo($" Elapsed time: {args.Message.ElapsedClockTime}"); - - if (args.Message.Summaries == null || args.Message.Summaries.Count == 0) - { - return; - } - - foreach (KeyValuePair summary in args.Message.Summaries) - { - OnInfo(string.Empty); - OnInfo($" Assembly: {summary.Key}"); - LogSummary(summary.Value, log: OnDebug); - } - } - - private void HandleTestAssemblyExecutionStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"Execution starting for assembly {args.Message.Assembly.AssemblyFilename}"); - } - - private void HandleTestAssemblyExecutionFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"Execution finished for assembly {args.Message.Assembly.AssemblyFilename}"); - LogSummary(args.Message.ExecutionSummary, log: OnDebug); - } - - private void HandleTestAssemblyDiscoveryStarting(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"Discovery for assembly {args.Message.Assembly.AssemblyFilename} starting"); - OnInfo($" Will use AppDomain: {args.Message.AppDomain.YesNo()}"); - } - - private void HandleTestAssemblyDiscoveryFinished(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo($"Discovery for assembly {args.Message.Assembly.AssemblyFilename} finished"); - OnInfo($" Test cases discovered: {args.Message.TestCasesDiscovered}"); - OnInfo($" Test cases to run: {args.Message.TestCasesToRun}"); - } - - private void HandleDiagnosticMessage(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnDiagnostic(args.Message.Message); - } - - private void HandleDiagnosticErrorMessage(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - LogFailureInformation(args.Message); - } - - private void HandleDiscoveryCompleteMessage(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - OnInfo("Discovery complete"); - } - - private void HandleDiscoveryTestCaseMessage(MessageHandlerArgs args) - { - if (args == null || args.Message == null) - { - return; - } - - ITestCase singleTestCase = args.Message.TestCase; - ReportTestCases("Discovered", args.Message.TestCases, log: OnInfo, ignore: (ITestCase tc) => tc == singleTestCase); - ReportTestCase("Discovered", singleTestCase, log: OnInfo); - } - - private void RaiseTestSkippedCase(ITestResultMessage message, IEnumerable testCases, ITestCase testCase) - { - SkippedTests++; - OnInfo($"\t[IGNORED] {testCase.DisplayName}"); - LogTestDetails(message.Test, log: OnDebug); - LogTestOutput(message, log: OnDiagnostic); - ReportTestCases(" Associated", testCases, log: OnDiagnostic); - // notify that the test completed because it was skipped - OnTestCompleted(( - TestName: message.Test.DisplayName, - TestResult: TestResult.Skipped - )); - } - - private void ReportTestCases(string verb, IEnumerable testCases, ITestCase ignoreTestCase, Action log = null) => ReportTestCases(verb, testCases, log, (ITestCase tc) => ignoreTestCase == tc); - - private void ReportTestCases(string verb, IEnumerable testCases, Action log = null, Func ignore = null) - { - if (testCases == null) - { - return; - } - - foreach (ITestCase tc in testCases) - { - if (ignore != null && ignore(tc)) - { - continue; - } - - ReportTestCase(verb, tc, log); - } - } - - private void ReportTestCase(string verb, ITestCase testCase, Action log = null) - { - if (testCase == null) - { - return; - } - - EnsureLogger(log)($"{verb} test case: {testCase.DisplayName}"); - } - - private void LogAssemblyInformation(ITestAssemblyMessage message, Action log = null, StringBuilder sb = null) - { - if (message == null) - { - return; - } - - do_log($"[Assembly name: {message.TestAssembly.Assembly.Name}]", log, sb); - do_log($"[Assembly path: {message.TestAssembly.Assembly.AssemblyPath}]", OnDiagnostic, sb); - } - - private void LogFailureInformation(IFailureInformation info, Action log = null, StringBuilder sb = null) - { - if (info == null) - { - return; - } - - string message = ExceptionUtility.CombineMessages(info); - do_log($" Exception messages: {message}", log, sb); - - string traces = ExceptionUtility.CombineStackTraces(info); - do_log($" Exception stack traces: {traces}", log, sb); - } - - private Action EnsureLogger(Action log) => log ?? OnInfo; - -#pragma warning disable IDE0060 // Remove unused parameter - private static void LogTestMethodDetails(IMethodInfo method, Action log = null, StringBuilder sb = null) -#pragma warning restore IDE0060 // Remove unused parameter - { - // log = EnsureLogger(log); - // log ($" Test method name: {method.Type.Name}.{method.Name}"); - } - - private void LogTestOutput(ITestFinished test, Action log = null, StringBuilder sb = null) => LogTestOutput(test.ExecutionTime, test.Output, log, sb); - - private void LogTestOutput(ITestResultMessage test, Action log = null, StringBuilder sb = null) => LogTestOutput(test.ExecutionTime, test.Output, log, sb); - - private void LogTestOutput(decimal executionTime, string output, Action log = null, StringBuilder sb = null) - { - do_log($" Execution time: {executionTime}", log, sb); - if (!string.IsNullOrEmpty(output)) - { - do_log(" **** Output start ****", log, sb); - foreach (string line in output.Split('\n')) - { - do_log(line, log, sb); - } - - do_log(" **** Output end ****", log, sb); - } - } - - private void LogTestCollectionDetails(ITestCollection collection, Action log = null, StringBuilder sb = null) => do_log($" Test collection: {collection.DisplayName}", log, sb); - - private void LogTestClassDetails(ITestClass klass, Action log = null, StringBuilder sb = null) - { - do_log($" Class name: {klass.Class.Name}", log, sb); - do_log($" Class assembly: {klass.Class.Assembly.Name}", OnDebug, sb); - do_log($" Class assembly path: {klass.Class.Assembly.AssemblyPath}", OnDebug, sb); - } - - private void LogTestDetails(ITest test, Action log = null, StringBuilder sb = null) - { - do_log($" Test name: {test.DisplayName}", log, sb); - if (string.Compare(test.DisplayName, test.TestCase.DisplayName, StringComparison.Ordinal) != 0) - { - do_log($" Test case: {test.TestCase.DisplayName}", log, sb); - } - } - - private void LogSummary(IFinishedMessage summary, Action log = null, StringBuilder sb = null) - { - do_log($" Time: {summary.ExecutionTime}", log, sb); - do_log($" Total tests run: {summary.TestsRun}", log, sb); - do_log($" Skipped tests: {summary.TestsSkipped}", log, sb); - do_log($" Failed tests: {summary.TestsFailed}", log, sb); - } - - private void LogSummary(ExecutionSummary summary, Action log = null, StringBuilder sb = null) - { - do_log($" Time: {summary.Time}", log, sb); - do_log($" Total tests run: {summary.Total}", log, sb); - do_log($" Total errors: {summary.Errors}", log, sb); - do_log($" Skipped tests: {summary.Skipped}", log, sb); - do_log($" Failed tests: {summary.Failed}", log, sb); - } - - private void LogSourceInformation(ISourceInformation source, Action log = null, StringBuilder sb = null) - { - if (source == null || string.IsNullOrEmpty(source.FileName)) - { - return; - } - - string location = source.FileName; - if (source.LineNumber != null && source.LineNumber >= 0) - { - location += $":{source.LineNumber}"; - } - - do_log($" Source: {location}", log, sb); - sb?.AppendLine(); - } - - private static string GetAssemblyInfo(ITestAssembly assembly) - { - string name = assembly?.Assembly?.Name?.Trim(); - if (string.IsNullOrEmpty(name)) - { - return name; - } - - return $" [{name}]"; - } - - private void do_log(string message, Action log = null, StringBuilder sb = null) - { - log = EnsureLogger(log); - - if (sb != null) - { - sb.Append(message); - } - - log(message); - } - - public override async Task Run(IEnumerable testAssemblies) - { - if (testAssemblies == null) - { - throw new ArgumentNullException(nameof(testAssemblies)); - } - - if (_filters != null && _filters.Count > 0) - { - do_log("Configured filters:"); - foreach (XUnitFilter filter in _filters) - { - do_log($" {filter}"); - } - } - - _assembliesElement = new XElement("assemblies"); - Action log = LogExcludedTests ? (s) => do_log(s) : (Action)null; - foreach (TestAssemblyInfo assemblyInfo in testAssemblies) - { - if (assemblyInfo == null || assemblyInfo.Assembly == null) - { - continue; - } - - if (_filters.AssemblyFilters.Any() && _filters.IsExcluded(assemblyInfo, log)) - { - continue; - } - - if (string.IsNullOrEmpty(assemblyInfo.FullPath)) - { - OnWarning($"Assembly '{assemblyInfo.Assembly}' cannot be found on the filesystem. xUnit requires access to actual on-disk file."); - continue; - } - - OnInfo($"Assembly: {assemblyInfo.Assembly} ({assemblyInfo.FullPath})"); - XElement assemblyElement = null; - try - { - OnAssemblyStart(assemblyInfo.Assembly); - assemblyElement = await Run(assemblyInfo.Assembly, assemblyInfo.FullPath).ConfigureAwait(false); - } - catch (FileNotFoundException ex) - { - OnWarning($"Assembly '{assemblyInfo.Assembly}' using path '{assemblyInfo.FullPath}' cannot be found on the filesystem. xUnit requires access to actual on-disk file."); - OnWarning($"Exception is '{ex}'"); - } - finally - { - OnAssemblyFinish(assemblyInfo.Assembly); - if (assemblyElement != null) - { - _assembliesElement.Add(assemblyElement); - } - } - } - - LogFailureSummary(); - TotalTests += FilteredTests; // ensure that we do have in the total run the excluded ones. - } - - public override string WriteResultsToFile(XmlResultJargon jargon) - { - if (_assembliesElement == null) - { - return string.Empty; - } - // remove all the empty nodes - _assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove(); - string outputFilePath = GetResultsFilePath(); - var settings = new XmlWriterSettings { Indent = true }; - using (var xmlWriter = XmlWriter.Create(outputFilePath, settings)) - { - switch (jargon) - { - case XmlResultJargon.TouchUnit: - case XmlResultJargon.NUnitV2: - Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter); - break; - case XmlResultJargon.NUnitV3: - Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter); - break; - default: // xunit as default, includes when we got Missing - _assembliesElement.Save(xmlWriter); - break; - } - } - - return outputFilePath; - } - public override void WriteResultsToFile(TextWriter writer, XmlResultJargon jargon) - { - if (_assembliesElement == null) - { - return; - } - // remove all the empty nodes - _assembliesElement.Descendants().Where(e => e.Name == "collection" && !e.Descendants().Any()).Remove(); - var settings = new XmlWriterSettings { Indent = true }; - using (var xmlWriter = XmlWriter.Create(writer, settings)) - { - switch (jargon) - { - case XmlResultJargon.TouchUnit: - case XmlResultJargon.NUnitV2: - try - { - Transform_Results("NUnitXml.xslt", _assembliesElement, xmlWriter); - } - catch (Exception e) - { - writer.WriteLine(e); - } - break; - case XmlResultJargon.NUnitV3: - try - { - Transform_Results("NUnit3Xml.xslt", _assembliesElement, xmlWriter); - } - catch (Exception e) - { - writer.WriteLine(e); - } - break; - default: // xunit as default, includes when we got Missing - _assembliesElement.Save(xmlWriter); - break; - } - } - } - - private void Transform_Results(string xsltResourceName, XElement element, XmlWriter writer) - { - var xmlTransform = new System.Xml.Xsl.XslCompiledTransform(); - var name = GetType().Assembly.GetManifestResourceNames().Where(a => a.EndsWith(xsltResourceName, StringComparison.Ordinal)).FirstOrDefault(); - if (name == null) - { - return; - } - - using (var xsltStream = GetType().Assembly.GetManifestResourceStream(name)) - { - if (xsltStream == null) - { - throw new Exception($"Stream with name {name} cannot be found! We have {GetType().Assembly.GetManifestResourceNames()[0]}"); - } - // add the extension so that we can get the hash from the name of the test - // Create an XsltArgumentList. - var xslArg = new XsltArgumentList(); - - var generator = new XsltIdGenerator(); - xslArg.AddExtensionObject("urn:hash-generator", generator); - - using (var xsltReader = XmlReader.Create(xsltStream)) - using (var xmlReader = element.CreateReader()) - { - xmlTransform.Load(xsltReader); - xmlTransform.Transform(xmlReader, xslArg, writer); - } - } - } - - protected virtual Stream GetConfigurationFileStream(Assembly assembly) - { - if (assembly == null) - { - throw new ArgumentNullException(nameof(assembly)); - } - - string path = assembly.Location?.Trim(); - if (string.IsNullOrEmpty(path)) - { - return null; - } - - path = Path.Combine(path, ".xunit.runner.json"); - if (!File.Exists(path)) - { - return null; - } - - return File.OpenRead(path); - } - - protected virtual TestAssemblyConfiguration GetConfiguration(Assembly assembly) - { - if (assembly == null) - { - throw new ArgumentNullException(nameof(assembly)); - } - - Stream configStream = GetConfigurationFileStream(assembly); - if (configStream != null) - { - using (configStream) - { - return ConfigReader.Load(configStream); - } - } - - return null; - } - - protected virtual ITestFrameworkDiscoveryOptions GetFrameworkOptionsForDiscovery(TestAssemblyConfiguration configuration) - { - if (configuration == null) - { - throw new ArgumentNullException(nameof(configuration)); - } - - return TestFrameworkOptions.ForDiscovery(configuration); - } - - protected virtual ITestFrameworkExecutionOptions GetFrameworkOptionsForExecution(TestAssemblyConfiguration configuration) - { - if (configuration == null) - { - throw new ArgumentNullException(nameof(configuration)); - } - - return TestFrameworkOptions.ForExecution(configuration); - } - - private async Task Run(Assembly assembly, string assemblyPath) - { - using (var frontController = new XunitFrontController(AppDomainSupport, assemblyPath, null, false)) - { - using (var discoverySink = new TestDiscoverySink()) - { - var configuration = GetConfiguration(assembly) ?? new TestAssemblyConfiguration() { PreEnumerateTheories = false }; - ITestFrameworkDiscoveryOptions discoveryOptions = GetFrameworkOptionsForDiscovery(configuration); - discoveryOptions.SetSynchronousMessageReporting(true); - Logger.OnDebug($"Starting test discovery in the '{assembly}' assembly"); - frontController.Find(false, discoverySink, discoveryOptions); - Logger.OnDebug($"Test discovery in assembly '{assembly}' completed"); - discoverySink.Finished.WaitOne(); - - if (discoverySink.TestCases == null || discoverySink.TestCases.Count == 0) - { - Logger.Info("No test cases discovered"); - return null; - } - - TotalTests += discoverySink.TestCases.Count; - List testCases; - if (_filters != null && _filters.TestCaseFilters.Any()) - { - Action log = LogExcludedTests ? (s) => do_log(s) : (Action)null; - testCases = discoverySink.TestCases.Where( - tc => !_filters.IsExcluded(tc, log)).ToList(); - FilteredTests += discoverySink.TestCases.Count - testCases.Count; - } - else - { - testCases = discoverySink.TestCases; - } - - var summaryTaskSource = new TaskCompletionSource(); - var resultsXmlAssembly = new XElement("assembly"); -#pragma warning disable CS0618 // Delegating*Sink types are marked obsolete, but we can't move to ExecutionSink yet: https://github.com/dotnet/arcade/issues/14375 - var resultsSink = new DelegatingXmlCreationSink(new DelegatingExecutionSummarySink(_messageSink), resultsXmlAssembly); -#pragma warning restore - var completionSink = new CompletionCallbackExecutionSink(resultsSink, summary => summaryTaskSource.SetResult(summary)); - - ITestFrameworkExecutionOptions executionOptions = GetFrameworkOptionsForExecution(configuration); - executionOptions.SetDisableParallelization(!RunInParallel); - executionOptions.SetSynchronousMessageReporting(true); - executionOptions.SetMaxParallelThreads(MaxParallelThreads); - - frontController.RunTests(testCases, completionSink, executionOptions); - await summaryTaskSource.Task.ConfigureAwait(false); - - return resultsXmlAssembly; - } - } - } -} diff --git a/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs b/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs deleted file mode 100644 index ec490e552cd468..00000000000000 --- a/src/libraries/Common/tests/WasmTestRunner/XunitTestRunnerBase.cs +++ /dev/null @@ -1,87 +0,0 @@ -// Licensed to the .NET Foundation under one or more agreements. -// The .NET Foundation licenses this file to you under the MIT license. -// See the LICENSE file in the project root for more information. -#nullable disable - -using System; -using System.Collections.Generic; -using System.Linq; -using Microsoft.DotNet.XHarness.TestRunners.Common; - -#nullable enable -namespace Microsoft.DotNet.XHarness.TestRunners.Xunit; -public abstract class MyXunitTestRunnerBase : TestRunner -{ - private protected XUnitFiltersCollection _filters = new(); - - protected MyXunitTestRunnerBase(LogWriter logger) : base(logger) - { - } - - public override void SkipTests(IEnumerable tests) - { - if (tests.Any()) - { - // create a single filter per test - foreach (var t in tests) - { - if (t.StartsWith("KLASS:", StringComparison.Ordinal)) - { - var klass = t.Replace("KLASS:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("KLASS32:", StringComparison.Ordinal) && IntPtr.Size == 4) - { - var klass = t.Replace("KLASS32:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("KLASS64:", StringComparison.Ordinal) && IntPtr.Size == 8) - { - var klass = t.Replace("KLASS32:", ""); - _filters.Add(XUnitFilter.CreateClassFilter(klass, true)); - } - else if (t.StartsWith("Platform32:", StringComparison.Ordinal) && IntPtr.Size == 4) - { - var filter = t.Replace("Platform32:", ""); - _filters.Add(XUnitFilter.CreateSingleFilter(filter, true)); - } - else - { - _filters.Add(XUnitFilter.CreateSingleFilter(t, true)); - } - } - } - } - - public override void SkipCategories(IEnumerable categories) => SkipCategories(categories, isExcluded: true); - - public virtual void SkipCategories(IEnumerable categories, bool isExcluded) - { - if (categories == null) - { - throw new ArgumentNullException(nameof(categories)); - } - - foreach (var c in categories) - { - var traitInfo = c.Split('='); - if (traitInfo.Length == 2) - { - _filters.Add(XUnitFilter.CreateTraitFilter(traitInfo[0], traitInfo[1], isExcluded)); - } - else - { - _filters.Add(XUnitFilter.CreateTraitFilter(c, null, isExcluded)); - } - } - } - - public override void SkipMethod(string method, bool isExcluded) - => _filters.Add(XUnitFilter.CreateSingleFilter(singleTestName: method, exclude: isExcluded)); - - public override void SkipClass(string className, bool isExcluded) - => _filters.Add(XUnitFilter.CreateClassFilter(className: className, exclude: isExcluded)); - - public virtual void SkipNamespace(string namespaceName, bool isExcluded) - => _filters.Add(XUnitFilter.CreateNamespaceFilter(namespaceName, exclude: isExcluded)); -} diff --git a/src/mono/wasm/sln/WasmBuild.sln b/src/mono/wasm/sln/WasmBuild.sln index da3a82af643fb0..b84f2d7ba8c848 100755 --- a/src/mono/wasm/sln/WasmBuild.sln +++ b/src/mono/wasm/sln/WasmBuild.sln @@ -25,7 +25,9 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "ApplyUpdateReferencedAssemb EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.NET.Sdk.WebAssembly.Pack.Tasks", "..\..\..\tasks\Microsoft.NET.Sdk.WebAssembly.Pack.Tasks\Microsoft.NET.Sdk.WebAssembly.Pack.Tasks.csproj", "{5EEC2925-2021-4830-B7E9-72BB8B2C283D}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Wasi.Build.Tests", "..\..\wasi\Wasi.Build.Tests\Wasi.Build.Tests.csproj", "{3A3AEAE5-0110-45D3-89B0-B82AC430535C}" +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Wasi.Build.Tests", "..\..\wasi\Wasi.Build.Tests\Wasi.Build.Tests.csproj", "{3A3AEAE5-0110-45D3-89B0-B82AC430535C}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "WasmTestRunner", "..\..\..\libraries\Common\tests\WasmTestRunner\WasmTestRunner.csproj", "{2BBE4AA8-5424-44AB-933C-66554B688872}" EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution @@ -81,6 +83,10 @@ Global {3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Debug|Any CPU.Build.0 = Debug|Any CPU {3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Release|Any CPU.ActiveCfg = Release|Any CPU {3A3AEAE5-0110-45D3-89B0-B82AC430535C}.Release|Any CPU.Build.0 = Release|Any CPU + {2BBE4AA8-5424-44AB-933C-66554B688872}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {2BBE4AA8-5424-44AB-933C-66554B688872}.Debug|Any CPU.Build.0 = Debug|Any CPU + {2BBE4AA8-5424-44AB-933C-66554B688872}.Release|Any CPU.ActiveCfg = Release|Any CPU + {2BBE4AA8-5424-44AB-933C-66554B688872}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE From 3358756bc5409ed7250d31581d752cad0cafbc7b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 13:11:15 +0100 Subject: [PATCH 14/36] Remove unused using --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index b623cd5601d4bf..6033028a429926 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -7,7 +7,6 @@ using System.Linq; using System.Reflection; using System.Threading.Tasks; -using Microsoft.DotNet.XHarness.TestRunners.Common; using Microsoft.DotNet.XHarness.TestRunners.Xunit; using Xunit.Sdk; From 284a3fa84ea115f308d2de3042d7c8c3d36768ab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 13:11:52 +0100 Subject: [PATCH 15/36] Rename entrypoint class name --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 6033028a429926..e38b53c8911eaf 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -10,7 +10,7 @@ using Microsoft.DotNet.XHarness.TestRunners.Xunit; using Xunit.Sdk; -public class SimpleWasmTestRunner : WasmApplicationEntryPoint +public class WasmTestRunner : WasmApplicationEntryPoint { public static async Task Main(string[] args) { From d592e58b4d1602a70e755701a7845e9f48e57f53 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 13:14:00 +0100 Subject: [PATCH 16/36] Fix envvar value --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index d7651d6ff2d0b1..2176b06a2c9bd6 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -91,7 +91,7 @@ $(WasmTestAppArgs) -backgroundExec <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) - $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=1 + $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true $(WasmXHarnessMonoArgs) --no-memory-snapshot From 0e875a762536013423edb60af8b348b01282b7de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 13:14:27 +0100 Subject: [PATCH 17/36] REVERT: Test thread ID logging --- eng/testing/tests.browser.targets | 1 + 1 file changed, 1 insertion(+) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 2176b06a2c9bd6..2012abf36e0c08 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -91,6 +91,7 @@ $(WasmTestAppArgs) -backgroundExec <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) + $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_THREAD_ID=true $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true $(WasmXHarnessMonoArgs) --no-memory-snapshot From d169ae292e2d73da82f15bc859dae126486f86f3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 29 Jan 2024 13:48:13 +0100 Subject: [PATCH 18/36] Fix --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index e38b53c8911eaf..fb3da2f71bfc89 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -65,7 +65,7 @@ public static async Task Main(string[] args) } } - var runner = new SimpleWasmTestRunner() + var runner = new WasmTestRunner() { TestAssembly = testAssembly, ExcludedTraits = excludedTraits, From a6b175d151abe03abdea44b213534d227aa47261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 09:16:20 +0100 Subject: [PATCH 19/36] Revert "REVERT: Test thread ID logging" This reverts commit 0e875a762536013423edb60af8b348b01282b7de. --- eng/testing/tests.browser.targets | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 2012abf36e0c08..2176b06a2c9bd6 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -91,7 +91,6 @@ $(WasmTestAppArgs) -backgroundExec <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) - $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_THREAD_ID=true $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true $(WasmXHarnessMonoArgs) --no-memory-snapshot From 5cf933327d1c0f0a7aa4cd541825f51f8bf2c130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 09:20:28 +0100 Subject: [PATCH 20/36] Cleanup usings --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 0e8e58e761c8d7..95abe2c5e2a85d 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -3,12 +3,8 @@ using System; using System.Collections.Generic; -using System.Diagnostics.CodeAnalysis; -using System.Linq; -using System.Reflection; using System.Threading.Tasks; using Microsoft.DotNet.XHarness.TestRunners.Xunit; -using Xunit.Sdk; public class WasmTestRunner : WasmApplicationEntryPoint { From 456e6ab94c5c87c85fac5f6a3eda11d797ea3cb5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 13:29:49 +0100 Subject: [PATCH 21/36] Fix envvar value --- eng/testing/tests.wasi.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.wasi.targets b/eng/testing/tests.wasi.targets index 8516230d7e6da6..a132db0bf52af2 100644 --- a/eng/testing/tests.wasi.targets +++ b/eng/testing/tests.wasi.targets @@ -31,7 +31,7 @@ <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) -- $(WasmTestAppArgs) - $(WasmXHarnessMonoArgs) --env=XHARNESS_LOG_TEST_START=1 + $(WasmXHarnessMonoArgs) --env=XHARNESS_LOG_TEST_START=true From 803c3db7ead77ef7a5a9060539dfa434a328eb99 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 13:30:04 +0100 Subject: [PATCH 22/36] Try to enable PMDesignator_Get_ReturnsExpected_HybridGlobalization --- .../DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs b/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs index 530b6dfae52263..5eeea693a400a5 100644 --- a/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs +++ b/src/libraries/System.Runtime/tests/System.Globalization.Tests/DateTimeFormatInfo/DateTimeFormatInfoPMDesignator.cs @@ -205,7 +205,6 @@ public static IEnumerable PMDesignator_Get_TestData_HybridGlobalizatio yield return new object[] { new CultureInfo("zh-TW").DateTimeFormat, "下午" }; } - [ActiveIssue("https://github.com/dotnet/runtime/issues/75123", typeof(PlatformDetection), nameof(PlatformDetection.IsWasmThreadingSupported))] [ConditionalTheory(typeof(PlatformDetection), nameof(PlatformDetection.IsHybridGlobalizationOnBrowser))] [MemberData(nameof(PMDesignator_Get_TestData_HybridGlobalization))] public void PMDesignator_Get_ReturnsExpected_HybridGlobalization(DateTimeFormatInfo format, string value) From ada85ee021d7b8fd83c82c1e4da24d81b07ace56 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 13:30:48 +0100 Subject: [PATCH 23/36] Try non-background execution --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 2176b06a2c9bd6..5bc18b3f3568ab 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -87,7 +87,7 @@ <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(WasmMainAssemblyFileName)' != ''">--run $(WasmMainAssemblyFileName) <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll - <_XUnitBackgroundExec Condition="'$(_XUnitBackgroundExec)' == '' and '$(MonoWasmBuildVariant)' == 'multithread'">true + $(WasmTestAppArgs) -backgroundExec <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) From b73131fb71007621eacfd1f58c4e1bbcd8c23547 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 30 Jan 2024 17:44:02 +0100 Subject: [PATCH 24/36] Revert "Try non-background execution" This reverts commit ada85ee021d7b8fd83c82c1e4da24d81b07ace56. --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 5bc18b3f3568ab..2176b06a2c9bd6 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -87,7 +87,7 @@ <_AppArgs Condition="'$(IsFunctionalTest)' != 'true' and '$(WasmMainAssemblyFileName)' != ''">--run $(WasmMainAssemblyFileName) <_AppArgs Condition="'$(IsFunctionalTest)' == 'true'">--run $(AssemblyName).dll - + <_XUnitBackgroundExec Condition="'$(_XUnitBackgroundExec)' == '' and '$(MonoWasmBuildVariant)' == 'multithread'">true $(WasmTestAppArgs) -backgroundExec <_AppArgs Condition="'$(WasmTestAppArgs)' != ''">$(_AppArgs) $(WasmTestAppArgs) From d8636c518a9cfd5a055d3fe20dcd3c232adca813 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 2 Feb 2024 17:59:47 +0100 Subject: [PATCH 25/36] Pass verbosity to XHarness --- eng/testing/tests.browser.targets | 3 ++- eng/testing/tests.wasi.targets | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 2176b06a2c9bd6..0d63c1e9013aca 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -110,10 +110,11 @@ <_XHarnessArgs Condition="'$(_UseWasmSymbolicator)' == 'true'" >$(_XHarnessArgs) --symbolicator WasmSymbolicator.dll,Microsoft.WebAssembly.Internal.SymbolicatorWrapperForXHarness <_XHarnessArgs Condition="'$(_WasmBrowserPathForTests)' != ''" >$(_XHarnessArgs) "--browser-path=$(_WasmBrowserPathForTests)" <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" + <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) <_AppArgs Condition="'$(MonoWasmBuildVariant)' == 'multithread'">$(_AppArgs) -threads - + $HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) %24WasmTestAppArgs %HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs% -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_AppArgs) %WasmTestAppArgs% diff --git a/eng/testing/tests.wasi.targets b/eng/testing/tests.wasi.targets index a132db0bf52af2..d147fea218fe4a 100644 --- a/eng/testing/tests.wasi.targets +++ b/eng/testing/tests.wasi.targets @@ -51,7 +51,7 @@ <_InvariantGlobalization Condition="'$(InvariantGlobalization)' == 'true'">--env=DOTNET_SYSTEM_GLOBALIZATION_INVARIANT=true - + $HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_InvariantGlobalization) %24_InvariantGlobalization $(_AppArgs) %24WasmTestAppArgs %HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs% -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_InvariantGlobalization) %_InvariantGlobalization% $(_AppArgs) %WasmTestAppArgs% From 8eee15008323cf5928ad12a9eb1fe5a4b5856af2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Fri, 2 Feb 2024 18:07:37 +0100 Subject: [PATCH 26/36] Pass log level to xharness. Replace arg with property for STRT logging --- .../tests/WasmTestRunner/WasmTestRunner.cs | 28 ++++++++++--------- .../System.Collections.Immutable.Tests.csproj | 3 +- .../System.Net.Http.Functional.Tests.csproj | 3 +- .../System.Net.WebSockets.Client.Tests.csproj | 3 +- .../tests/System.Private.Xml.Tests.csproj | 3 +- ...me.InteropServices.JavaScript.Tests.csproj | 3 +- .../System.Text.Json.Tests.csproj | 3 +- 7 files changed, 27 insertions(+), 19 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 95abe2c5e2a85d..f0d1325d2e9916 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Threading.Tasks; +using Microsoft.DotNet.XHarness.TestRunners.Common; using Microsoft.DotNet.XHarness.TestRunners.Xunit; public class WasmTestRunner : WasmApplicationEntryPoint @@ -16,7 +17,10 @@ public static async Task Main(string[] args) return -1; } - var testAssembly = args[0]; + var runner = new WasmTestRunner(); + + runner.TestAssembly = args[0]; + var excludedTraits = new List(); var includedTraits = new List(); var includedNamespaces = new List(); @@ -24,7 +28,6 @@ public static async Task Main(string[] args) var includedMethods = new List(); var backgroundExec = false; var untilFailed = false; - var isThreadless = true; for (int i = 1; i < args.Length; i++) { @@ -58,23 +61,22 @@ public static async Task Main(string[] args) untilFailed = true; break; case "-threads": - isThreadless = false; + runner.IsThreadless = false; + break; + case "-verbosity": + runner.MinimumLogLevel = Enum.Parse(args[i + 1]); + i++; break; default: throw new ArgumentException($"Invalid argument '{option}'."); } } - var runner = new WasmTestRunner() - { - TestAssembly = testAssembly, - ExcludedTraits = excludedTraits, - IncludedTraits = includedTraits, - IncludedNamespaces = includedNamespaces, - IncludedClasses = includedClasses, - IncludedMethods = includedMethods, - IsThreadless = isThreadless - }; + runner.ExcludedTraits = excludedTraits; + runner.IncludedTraits = includedTraits; + runner.IncludedNamespaces = includedNamespaces; + runner.IncludedClasses = includedClasses; + runner.IncludedMethods = includedMethods; if (OperatingSystem.IsBrowser()) { diff --git a/src/libraries/System.Collections.Immutable/tests/System.Collections.Immutable.Tests.csproj b/src/libraries/System.Collections.Immutable/tests/System.Collections.Immutable.Tests.csproj index 77a909d923c9d7..8f16d85e3c7c8f 100644 --- a/src/libraries/System.Collections.Immutable/tests/System.Collections.Immutable.Tests.csproj +++ b/src/libraries/System.Collections.Immutable/tests/System.Collections.Immutable.Tests.csproj @@ -6,7 +6,8 @@ - --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true 01:15:00 diff --git a/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj b/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj index 2451c867a9489d..7bc530d01f2f77 100644 --- a/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj +++ b/src/libraries/System.Net.Http/tests/FunctionalTests/System.Net.Http.Functional.Tests.csproj @@ -24,7 +24,8 @@ $(TestArchiveRoot)browserornodejs/ $(TestArchiveTestsRoot)$(OSPlatformConfig)/ $(DefineConstants);TARGET_BROWSER - --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true 01:15:00 diff --git a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj index b797fcf1894599..82df867b45636a 100644 --- a/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj +++ b/src/libraries/System.Net.WebSockets.Client/tests/System.Net.WebSockets.Client.Tests.csproj @@ -13,7 +13,8 @@ $(TestArchiveRoot)browserornodejs/ $(TestArchiveTestsRoot)$(OSPlatformConfig)/ $(DefineConstants);TARGET_BROWSER - --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true 01:15:00 diff --git a/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj b/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj index e0dbfa21eec93c..9c3df02679ed2a 100644 --- a/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj +++ b/src/libraries/System.Private.Xml/tests/System.Private.Xml.Tests.csproj @@ -14,7 +14,8 @@ $(TestArchiveRoot)browserornodejs/ $(TestArchiveTestsRoot)$(OSPlatformConfig)/ $(DefineConstants);TARGET_BROWSER - --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true diff --git a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj index a4e88845ec0808..93bcdf307b96f3 100644 --- a/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj +++ b/src/libraries/System.Runtime.InteropServices.JavaScript/tests/System.Runtime.InteropServices.JavaScript.UnitTests/System.Runtime.InteropServices.JavaScript.Tests.csproj @@ -13,7 +13,8 @@ true - $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true diff --git a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj index b3a12d41a753bd..bd12a7c6b43da5 100644 --- a/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj +++ b/src/libraries/System.Text.Json/tests/System.Text.Json.Tests/System.Text.Json.Tests.csproj @@ -18,7 +18,8 @@ true - --setenv=XHARNESS_LOG_TEST_START=true --no-memory-snapshot + --no-memory-snapshot + true 01:15:00 From cd07f441ae83a35fd3f6ff08eee615733e983493 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 5 Feb 2024 14:18:03 +0100 Subject: [PATCH 27/36] Update condition to trigger threaded runner --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 22f38415818a26..bae89b2d544f62 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -113,7 +113,7 @@ <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) - <_AppArgs Condition="'$(MonoWasmBuildVariant)' == 'multithread'">$(_AppArgs) -threads + <_AppArgs Condition="'$(WasmEnableThreads)' == 'true'">$(_AppArgs) -threads $HARNESS_RUNNER $(_XHarnessArgs) %24XHARNESS_ARGS %24WasmXHarnessArgs -- $(WasmXHarnessMonoArgs) %24WasmXHarnessMonoArgs $(_AppArgs) %24WasmTestAppArgs %HARNESS_RUNNER% $(_XHarnessArgs) %XHARNESS_ARGS% %WasmXHarnessArgs% -- $(WasmXHarnessMonoArgs) %WasmXHarnessMonoArgs% $(_AppArgs) %WasmTestAppArgs% From a3cac819b5dc1b4376192cb17f74aff3fdd3cf7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Mon, 5 Feb 2024 14:45:40 +0100 Subject: [PATCH 28/36] How to iterate on xharness runner changes --- src/mono/browser/README.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/mono/browser/README.md b/src/mono/browser/README.md index 5ee64fcd4724d2..4fb494a86b199d 100644 --- a/src/mono/browser/README.md +++ b/src/mono/browser/README.md @@ -117,10 +117,24 @@ The wrapper script used to actually run these tests, accepts: ### Using a local build of xharness +XHarness consists of two pieces for WASM + +#### 1. CLI/host + * set `XHARNESS_CLI_PATH=/path/to/xharness/artifacts/bin/Microsoft.DotNet.XHarness.CLI/Debug/net7.0/Microsoft.DotNet.XHarness.CLI.dll` **Note:** Additional msbuild arguments can be passed with: `make .. MSBUILD_ARGS="/p:a=b"` +#### 2. Test runner running inside of the browser + +All library tests are hosted by `WasmTestRunner.csproj`. The project references XHarness nuget for running tests using Xunit. To make changes and iterate quickly + +- Add property `$(RestoreAdditionalProjectSources);LOCAL_CLONE_OF_XHARNESS\artifacts\packages\Debug\Shipping` in `WasmTestRunner.csproj`. +- Set environment variable in your terminal `$env:NUGET_PACKAGES="$pwd\.nuget"` (so that nuget packages are restored to local folder `.nuget`) +- Run "Pack" in the XHarness solution in Visual Studio on `Microsoft.DotNet.XHarness.TestRunners.Common` or `Microsoft.DotNet.XHarness.TestRunners.Xunit` based on your changes (it will generate a nuget package in `LOCAL_CLONE_OF_XHARNESS\artifacts\packages\Debug\Shipping`). +- Build WasmTestRunner `.\dotnet.cmd build -c Debug .\src\libraries\Common\tests\WasmTestRunner\WasmTestRunner.csproj`. +- If you need to iterate, delete Xunit or Common nuget cache `rm -r .\.nuget\microsoft.dotnet.xharness.testrunners.xunit\` or `rm -r .\.nuget\microsoft.dotnet.xharness.testrunners.common\`. + ### Symbolicating traces Exceptions thrown after the runtime starts get symbolicating from js itself. Exceptions before that, like asserts containing native traces get symbolicated by xharness using `src/mono/wasm/symbolicator`. From 44b56096a44f916f4c5820d192e03683daa1b66f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 6 Feb 2024 12:28:47 +0100 Subject: [PATCH 29/36] Testing RunInParallel --- eng/testing/tests.browser.targets | 1 + src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 3 +++ 2 files changed, 4 insertions(+) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index bae89b2d544f62..0f433cd0d435ae 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -112,6 +112,7 @@ <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) + $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_THREAD_ID=true <_AppArgs Condition="'$(WasmEnableThreads)' == 'true'">$(_AppArgs) -threads diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index f0d1325d2e9916..f884810d083dcc 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -9,6 +9,8 @@ public class WasmTestRunner : WasmApplicationEntryPoint { + protected override int? MaxParallelThreads => RunInParallel ? 2 : base.MaxParallelThreads; + public static async Task Main(string[] args) { if (args.Length == 0) @@ -62,6 +64,7 @@ public static async Task Main(string[] args) break; case "-threads": runner.IsThreadless = false; + runner.RunInParallel = true; break; case "-verbosity": runner.MinimumLogLevel = Enum.Parse(args[i + 1]); From abeaccb15253e5a367834a2e8f4465c0035e1073 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 13 Feb 2024 08:42:55 +0100 Subject: [PATCH 30/36] Fix build --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 2dfdf227bde227..f884810d083dcc 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -30,7 +30,6 @@ public static async Task Main(string[] args) var includedMethods = new List(); var backgroundExec = false; var untilFailed = false; - var minimumLogLevel = MinimumLogLevel.Info; for (int i = 1; i < args.Length; i++) { @@ -81,7 +80,6 @@ public static async Task Main(string[] args) runner.IncludedNamespaces = includedNamespaces; runner.IncludedClasses = includedClasses; runner.IncludedMethods = includedMethods; - MinimumLogLevel = minimumLogLevel if (OperatingSystem.IsBrowser()) { From 25f54df56062408efcca6e886e83906a421774d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Tue, 13 Feb 2024 12:05:09 +0100 Subject: [PATCH 31/36] System.Threading.Tasks.Tests logging start --- .../System.Threading.Tasks.Tests.csproj | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj index eb1b13766e64f9..19521db08c3331 100644 --- a/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj +++ b/src/libraries/System.Runtime/tests/System.Threading.Tasks.Tests/System.Threading.Tasks.Tests.csproj @@ -3,6 +3,7 @@ true true $(NetCoreAppCurrent) + true From dff6fa350fa39854f39633c72ceefbbec85d037b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Wed, 14 Feb 2024 09:25:22 +0100 Subject: [PATCH 32/36] Test 8 threads --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index f884810d083dcc..ed7ea245aa07f2 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -9,7 +9,7 @@ public class WasmTestRunner : WasmApplicationEntryPoint { - protected override int? MaxParallelThreads => RunInParallel ? 2 : base.MaxParallelThreads; + protected override int? MaxParallelThreads => RunInParallel ? 8 : base.MaxParallelThreads; public static async Task Main(string[] args) { From e8414a0433f6779772a79abcd4ae01d04e75b180 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Wed, 14 Feb 2024 12:17:39 +0100 Subject: [PATCH 33/36] Print number of threads for parallel run --- src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index ed7ea245aa07f2..60ea8038fcf440 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -65,6 +65,7 @@ public static async Task Main(string[] args) case "-threads": runner.IsThreadless = false; runner.RunInParallel = true; + Console.WriteLine($"Running in parallel with {runner.MaxParallelThreads} threads."); break; case "-verbosity": runner.MinimumLogLevel = Enum.Parse(args[i + 1]); From 4a1523813e608d1ebcfe5bd52a6263896d54d5c6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Thu, 15 Feb 2024 11:09:20 +0100 Subject: [PATCH 34/36] Remove RunInParallel --- .../Common/tests/WasmTestRunner/WasmTestRunner.cs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs index 60ea8038fcf440..2eb1c1e440c33b 100644 --- a/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs +++ b/src/libraries/Common/tests/WasmTestRunner/WasmTestRunner.cs @@ -9,7 +9,8 @@ public class WasmTestRunner : WasmApplicationEntryPoint { - protected override int? MaxParallelThreads => RunInParallel ? 8 : base.MaxParallelThreads; + // TODO: Set max threads for run in parallel + // protected override int? MaxParallelThreads => RunInParallel ? 8 : base.MaxParallelThreads; public static async Task Main(string[] args) { @@ -64,8 +65,9 @@ public static async Task Main(string[] args) break; case "-threads": runner.IsThreadless = false; - runner.RunInParallel = true; - Console.WriteLine($"Running in parallel with {runner.MaxParallelThreads} threads."); + // TODO: Enable run in parallel + // runner.RunInParallel = true; + // Console.WriteLine($"Running in parallel with {runner.MaxParallelThreads} threads."); break; case "-verbosity": runner.MinimumLogLevel = Enum.Parse(args[i + 1]); From f4635255c882b3b1994ea815fca1352bc38ed8cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Thu, 15 Feb 2024 11:10:55 +0100 Subject: [PATCH 35/36] Remove thjread ID logging for all tests --- eng/testing/tests.browser.targets | 1 - 1 file changed, 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index 0a3b687b8f2c87..b525e78c5d93a0 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -110,7 +110,6 @@ <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) - $(WasmXHarnessMonoArgs) --setenv=XHARNESS_LOG_THREAD_ID=true <_AppArgs Condition="'$(WasmEnableThreads)' == 'true'">$(_AppArgs) -threads From c8a34352ac58fb95ad148b9845a0a768d3f32f4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Fi=C5=A1era?= Date: Thu, 15 Feb 2024 11:15:32 +0100 Subject: [PATCH 36/36] Document verbosity values --- eng/testing/tests.browser.targets | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/testing/tests.browser.targets b/eng/testing/tests.browser.targets index b525e78c5d93a0..d27fa412d490b1 100644 --- a/eng/testing/tests.browser.targets +++ b/eng/testing/tests.browser.targets @@ -108,7 +108,7 @@ <_XHarnessArgs Condition="'$(_UseWasmSymbolicator)' == 'true'" >$(_XHarnessArgs) --symbolicator WasmSymbolicator.dll,Microsoft.WebAssembly.Internal.SymbolicatorWrapperForXHarness <_XHarnessArgs Condition="'$(_WasmBrowserPathForTests)' != ''" >$(_XHarnessArgs) "--browser-path=$(_WasmBrowserPathForTests)" <_XHarnessArgs Condition="'$(WasmXHarnessTestsTimeout)' != ''" >$(_XHarnessArgs) "--timeout=$(WasmXHarnessTestsTimeout)" - <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) + <_XHarnessArgs Condition="'$(WasmXHarnessVerbosity)' != ''" >$(_XHarnessArgs) --verbosity=$(WasmXHarnessVerbosity) <_XHarnessArgs Condition="'$(WasmXHarnessArgsCli)' != ''" >$(_XHarnessArgs) $(WasmXHarnessArgsCli) <_AppArgs Condition="'$(WasmEnableThreads)' == 'true'">$(_AppArgs) -threads