Added Linux & Mac changes for Arcade (#5479)

* Initial Windows, Linux, Macos builds test

* Add Linux/MacOS specific CI requirements

* Run Arcade CI tests on MacOS/Linux

* Fix final package building

* Add benchmark download to benchmars .csporj file

* Print detailed status of each unit test

* Install CentOS & Ubuntu build dependencies

* Use container names to differenciate between Ubuntu & CentOS

* Remove sudo usage in CentOS

* Fix Linux build dependencies

* Add -y param to apt install

* Remove installation of Linux dependencies

* Minor additions

* Rename Benchmarks to PerformanceTests for Arcade

* Changes

* Added benchmark doc changes

* Pre-merge changes

Author: mstfbl

.gitignore

@@ -6,6 +6,7 @@
 # Tool Runtime Dir
 /[Tt]ools/
 /.dotnet/
+/.packages/
 
 # User-specific files
 *.suo

.vsts-dotnet-ci.yml

@@ -11,42 +11,42 @@ resources:
     image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-20200515184230-2c829e8
 
 jobs:
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Centos_x64_NetCoreApp31
-#     buildScript: ./build.sh
-#     container: CentosContainer
-#     customMatrixes:
-#       Debug_Build:
-#         _configuration: Debug-netcoreapp3_1
-#         _config_short: DI
-#         _includeBenchmarkData: false
-#         _targetFramework: netcoreapp3.1
-#       Release_Build:
-#         _configuration: Release-netcoreapp3_1
-#         _config_short: RI
-#         _includeBenchmarkData: true
-#         _targetFramework: netcoreapp3.1
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Centos_x64_NetCoreApp31
+    buildScript: ./build.sh
+    container: CentosContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_1
+        _config_short: DI
+        _includeBenchmarkData: false
+        _targetFramework: netcoreapp3.1
+      Release_Build:
+        _configuration: Release-netcoreapp3_1
+        _config_short: RI
+        _includeBenchmarkData: true
+        _targetFramework: netcoreapp3.1
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: Ubuntu_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     container: UbuntuContainer
-#     innerLoop: true
-#     pool:
-#       name:  Hosted Ubuntu 1604
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Ubuntu_x64_NetCoreApp21
+    buildScript: ./build.sh
+    container: UbuntuContainer
+    innerLoop: true
+    pool:
+      name: Hosted Ubuntu 1604
 
-# - template: /build/ci/job-template.yml
-#   parameters:
-#     name: MacOS_x64_NetCoreApp21
-#     buildScript: ./build.sh
-#     innerLoop: true
-#     pool:
-#       name: Hosted macOS
+- template: /build/ci/job-template.yml
+  parameters:
+    name: MacOS_x64_NetCoreApp21
+    buildScript: ./build.sh
+    innerLoop: true
+    pool:
+      name: Hosted macOS
 
 - template: /build/ci/job-template.yml
   parameters:

Microsoft.ML.sln

@@ -48,7 +48,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "pkg", "pkg", "{D3D38B03-B55
 		pkg\Directory.Build.props = pkg\Directory.Build.props
 	EndProjectSection
 EndProject
-Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Benchmarks", "test\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
+Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.PerformanceTests", "test\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj", "{7A9DB75F-2CA5-4184-9EF5-1F17EB39483F}"
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Maml", "src\Microsoft.ML.Maml\Microsoft.ML.Maml.csproj", "{64F40A0D-D4C2-4AA7-8470-E9CC437827E4}"
 EndProject

build.sh

@@ -10,4 +10,4 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli
 done
 DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
 
-"$DIR/eng/common/build.sh" --restore --build --pack --warnAsError false "$@"
+"$DIR/eng/common/build.sh" --restore --build --warnAsError false "$@"

build/ci/job-template.yml

@@ -58,9 +58,10 @@ jobs:
           brew untap local/openssl |
           brew untap local/python2
         displayName: MacOS Homebrew bug Workaround
+    # Extra MacOS step required to install OS-specific dependencies
     - ${{ if eq(parameters.pool.name, 'Hosted macOS') }}:
       - script: brew update && brew unlink [email protected] && brew install mono-libgdiplus && brew install $(Build.SourcesDirectory)/build/libomp.rb && brew link libomp --force
-        displayName: Install build dependencies
+        displayName: Install MacOS build dependencies
     - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.pool.name, 'Hosted Ubuntu 1604')) }}:
       - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
         displayName: Set LD_LIBRARY_PATH for Ubuntu and CentOS to locate Native shared library in current running path
@@ -186,5 +187,5 @@ jobs:
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
     - ${{ if eq(parameters.nightlyBuild, 'false') }}:
-      - script: ${{ parameters.buildScript }} -pack  -ci
+      - script: ${{ parameters.buildScript }} -pack -ci
         displayName: Build Packages

src/Microsoft.ML.Core/Properties/AssemblyInfo.cs

@@ -13,7 +13,7 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.InferenceTesting" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]

src/Microsoft.ML.CpuMath/AssemblyInfo.cs

@@ -15,6 +15,6 @@
 [assembly: InternalsVisibleTo("Microsoft.ML.Sweeper" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.TimeSeries" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.Transforms" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks.Tests" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests.Tests" + PublicKey.Value)]
 
 [assembly: WantsToBeBestFriends]

src/Microsoft.ML.Data/Properties/AssemblyInfo.cs

@@ -12,10 +12,10 @@
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.OnnxTransformerTest" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Predictor.Tests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.TimeSeries.Tests" + PublicKey.Value)]
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.AutoML.Tests" + PublicKey.Value)]
 
-[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.EntryPoints" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.Maml" + PublicKey.Value)]
 [assembly: InternalsVisibleTo(assemblyName: "Microsoft.ML.ResultProcessor" + PublicKey.Value)]

src/Microsoft.ML.Maml/Properties/AssemblyInfo.cs

@@ -6,7 +6,7 @@
 using Microsoft.ML;
 
 [assembly: InternalsVisibleTo("Microsoft.ML.TestFramework" + PublicKey.Value)]
-[assembly: InternalsVisibleTo("Microsoft.ML.Benchmarks" + PublicKey.Value)]
+[assembly: InternalsVisibleTo("Microsoft.ML.PerformanceTests" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.ResultProcessor" + PublicKey.Value)]
 
 [assembly: InternalsVisibleTo("RunTests" + InternalPublicKey.Value)]

src/Redist/Microsoft.ML.DnnImageFeaturizer.ModelRedist/Microsoft.ML.DnnImageFeaturizer.ModelRedist.proj

@@ -123,4 +123,6 @@
 
   <Target Name="Test" />
 
+  <Target Name="PerformanceTest"/>
+
 </Project>

test/Microsoft.ML.Benchmarks.Tests/BenchmarksTest.cs

@@ -10,13 +10,13 @@
 using BenchmarkDotNet.Jobs;
 using BenchmarkDotNet.Loggers;
 using BenchmarkDotNet.Running;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.TestFramework;
 using Microsoft.ML.TestFramework.Attributes;
 using Xunit;
 using Xunit.Abstractions;
 
-namespace Microsoft.ML.Benchmarks.Tests
+namespace Microsoft.ML.PerformanceTests.Tests
 {
     public class TestConfig : RecommendedConfig
     {

test/Microsoft.ML.Benchmarks.Tests/Microsoft.ML.Benchmarks.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <ItemGroup>
-    <ProjectReference Include="..\Microsoft.ML.Benchmarks\Microsoft.ML.Benchmarks.csproj" />
+    <ProjectReference Include="..\Microsoft.ML.PerformanceTests\Microsoft.ML.PerformanceTests.csproj" />
 
     <NativeAssemblyReference Include="CpuMathNative" />
     <NativeAssemblyReference Include="MklImports" />

test/Microsoft.ML.Benchmarks/Program.cs

@@ -10,6 +10,8 @@
     -->
     <RuntimeIdentifier></RuntimeIdentifier>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
+    <IsUnitTestProject>false</IsUnitTestProject>
+    <IsPerformanceTestProject>true</IsPerformanceTestProject>
   </PropertyGroup>
 
   <ItemGroup>

test/Microsoft.ML.CpuMath.PerformanceTests/Microsoft.ML.CpuMath.PerformanceTests.csproj

@@ -8,7 +8,7 @@
 using Microsoft.ML.Runtime;
 using Microsoft.ML.TestFrameworkCommon;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class BenchmarkBase
     {

test/Microsoft.ML.Benchmarks/BenchmarkBase.cs renamed to test/Microsoft.ML.PerformanceTests/BenchmarkBase.cs

@@ -4,10 +4,10 @@
 
 using System;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class CacheDataViewBench : BenchmarkBase

test/Microsoft.ML.Benchmarks/CacheDataViewBench.cs renamed to test/Microsoft.ML.PerformanceTests/CacheDataViewBench.cs

@@ -11,7 +11,7 @@
 using Microsoft.ML.Transforms.Text;
 using Xunit;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class FeaturizeTextBench : BenchmarkBase

test/Microsoft.ML.Benchmarks/FeaturizeTextBench.cs renamed to test/Microsoft.ML.PerformanceTests/FeaturizeTextBench.cs

@@ -9,9 +9,9 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 using BenchmarkDotNet.Toolchains.DotNetCli;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public class RecommendedConfig : ManualConfig
     {

test/Microsoft.ML.Benchmarks/Harness/Configs.cs renamed to test/Microsoft.ML.PerformanceTests/Harness/Configs.cs

@@ -11,7 +11,7 @@
 using BenchmarkDotNet.Reports;
 using BenchmarkDotNet.Running;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     public abstract class WithExtraMetrics : BenchmarkBase
     {

test/Microsoft.ML.Benchmarks/Harness/Metrics.cs renamed to test/Microsoft.ML.PerformanceTests/Harness/Metrics.cs

@@ -10,7 +10,7 @@
 using BenchmarkDotNet.Toolchains;
 using BenchmarkDotNet.Toolchains.CsProj;
 
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// to avoid side effects of benchmarks affect each other BenchmarkDotNet runs every benchmark in a standalone, dedicated process

test/Microsoft.ML.Benchmarks/Harness/ProjectGenerator.cs renamed to test/Microsoft.ML.PerformanceTests/Harness/ProjectGenerator.cs

@@ -5,12 +5,12 @@
 using System;
 using System.Linq;
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Data;
 using Microsoft.ML.Runtime;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class HashBench : BenchmarkBase

test/Microsoft.ML.Benchmarks/HashBench.cs renamed to test/Microsoft.ML.PerformanceTests/HashBench.cs

@@ -3,7 +3,7 @@
 // See the LICENSE file in the project root for more information.
 
 using System;
-namespace Microsoft.ML.Benchmarks.Harness
+namespace Microsoft.ML.PerformanceTests.Harness
 {
     /// <summary>
     /// This attribute is used to identify the benchmarks 

test/Microsoft.ML.Benchmarks/Helpers/CIBenchmark.cs renamed to test/Microsoft.ML.PerformanceTests/Helpers/CIBenchmark.cs

@@ -5,7 +5,7 @@
 using System.IO;
 using System.Text;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     // Adding this class to not print anything to the console.
     // This is required for the current version of BenchmarkDotNet

test/Microsoft.ML.Benchmarks/Helpers/EmptyWriter.cs renamed to test/Microsoft.ML.PerformanceTests/Helpers/EmptyWriter.cs

@@ -7,7 +7,7 @@
 using Microsoft.ML.Trainers;
 using Microsoft.ML.Transforms;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class EnvironmentFactory
     {

test/Microsoft.ML.Benchmarks/Helpers/EnvironmentFactory.cs renamed to test/Microsoft.ML.PerformanceTests/Helpers/EnvironmentFactory.cs

@@ -2,7 +2,7 @@
 // The .NET Foundation licenses this file to you under the MIT license.
 // See the LICENSE file in the project root for more information.
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal class Errors
     {

test/Microsoft.ML.Benchmarks/Helpers/Errors.cs renamed to test/Microsoft.ML.PerformanceTests/Helpers/Errors.cs

@@ -4,7 +4,7 @@
 
 using System;
 using Microsoft.ML.Tools;
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     internal static class ExecuteMaml
     {

test/Microsoft.ML.Benchmarks/Helpers/ExecuteMaml.cs renamed to test/Microsoft.ML.PerformanceTests/Helpers/ExecuteMaml.cs

@@ -15,7 +15,7 @@
 using System.Net.Http;
 using Microsoft.ML.Vision;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [Config(typeof(TrainConfig))]
     public class ImageClassificationBench : BenchmarkBase

test/Microsoft.ML.Benchmarks/ImageClassificationBench.cs renamed to test/Microsoft.ML.PerformanceTests/ImageClassificationBench.cs

@@ -3,12 +3,12 @@
 // See the LICENSE file in the project root for more information.
 
 using BenchmarkDotNet.Attributes;
-using Microsoft.ML.Benchmarks.Harness;
+using Microsoft.ML.PerformanceTests.Harness;
 using Microsoft.ML.Calibrators;
 using Microsoft.ML.Data;
 using Microsoft.ML.Trainers;
 
-namespace Microsoft.ML.Benchmarks
+namespace Microsoft.ML.PerformanceTests
 {
     [CIBenchmark]
     public class KMeansAndLogisticRegressionBench : BenchmarkBase