Enabling building ML.NET for .NET Core 3.0 in the official builds. (dotnet#4096)

* Enabling building ML.NET for .NET Core 3.0 in the official builds.

* Update .sln for the new configuration

* Fix native build on windows.

* Fix netfx configuration by working around an sdk bug.

* Fix CpuMath tests on netfx.

Combine the two separate CpuMath UnitTest projects into a single project.

* Install the .NET Core 2.1 runtime so we can execute tests against it.

* Remove crossgen call since the crossgen executable is no longer shipped in the NuGet package.

We don't need many things from the Tools directory, so removing crossgen won't affect build performance very much.

Author: Eric Erhardt

.vsts-dotnet-ci.yml

@@ -18,11 +18,11 @@ jobs:
     container: CentosContainer
     customMatrixes:
       Debug_Build:
-        _configuration: Debug-Intrinsics
+        _configuration: Debug-netcoreapp3_0
         _config_short: DI
         _includeBenchmarkData: false
       Release_Build:
-        _configuration: Release-Intrinsics
+        _configuration: Release-netcoreapp3_0
         _config_short: RI
         _includeBenchmarkData: true
     pool:
@@ -49,11 +49,11 @@ jobs:
     buildScript: build.cmd
     customMatrixes:
       Debug_Build:
-        _configuration: Debug-Intrinsics
+        _configuration: Debug-netcoreapp3_0
         _config_short: DI
         _includeBenchmarkData: false
       Release_Build:
-        _configuration: Release-Intrinsics
+        _configuration: Release-netcoreapp3_0
         _config_short: RI
         _includeBenchmarkData: true
     pool:

Directory.Build.props

@@ -6,7 +6,7 @@
 
   <PropertyGroup>
     <Configuration Condition="'$(Configuration)'==''">Debug</Configuration>
-    <Configurations>Debug;Release;Debug-Intrinsics;Release-Intrinsics;Debug-netfx;Release-netfx</Configurations>
+    <Configurations>Debug;Release;Debug-netcoreapp3_0;Release-netcoreapp3_0;Debug-netfx;Release-netfx</Configurations>
     <Platform Condition="'$(Platform)'==''">AnyCPU</Platform>
     <TargetArchitecture Condition="'$(TargetArchitecture)' == ''">x64</TargetArchitecture>
     <NativeTargetArchitecture Condition="'$(NativeTargetArchitecture)' == ''">$(TargetArchitecture)</NativeTargetArchitecture>
@@ -106,11 +106,7 @@
     <PublicSign Condition="'$(OS)' != 'Windows_NT'">true</PublicSign>
   </PropertyGroup>
 
-  <PropertyGroup>
-    <UseIntrinsics Condition="'$(UseIntrinsics)' == ''">$(Configuration.EndsWith('-Intrinsics'))</UseIntrinsics>
-  </PropertyGroup>
-
-  <!-- Need to explicitly set these properties for the -Intrinsics or -Netfx configurations becuase they are typically based off 'Debug' or 'Release' configs -->
+  <!-- Need to explicitly set these properties for the -netcoreapp3_0 or -netfx configurations becuase they are typically based off 'Debug' or 'Release' configs -->
   <!-- Taken from https://github.com/dotnet/sdk/blob/073c98b92c81066c6c2e17c3674adbb6e833409a/src/Tasks/Microsoft.NET.Build.Tasks/targets/Microsoft.NET.Sdk.props#L41-L47 -->
   <PropertyGroup Condition="$(Configuration.StartsWith('Debug'))">
     <DebugSymbols>true</DebugSymbols>
@@ -121,7 +117,4 @@
     <Optimize>true</Optimize>
   </PropertyGroup>
 
-  <PropertyGroup>
-    <CustomAfterMicrosoftCommonTargets>$(RepoRoot)build\AfterCommonTargets.targets</CustomAfterMicrosoftCommonTargets>
-  </PropertyGroup>
 </Project>

DotnetCLIVersion.netcoreapp.latest.txt

@@ -1 +1 @@
-2.1.503
+3.0.100-preview7-012821

DotnetCLIVersion.txt

@@ -0,0 +1 @@
+2.1.12

DotnetExtraRuntimeVersion.txt

@@ -3,7 +3,7 @@
     "Configuration": {
       "description": "Sets the optimization level for the Build Configuration you want to build.",
       "valueType": "property",
-      "values": [ "Debug", "Release", "Debug-Intrinsics", "Release-Intrinsics", "Debug-netfx", "Release-netfx" ],
+      "values": [ "Debug", "Release", "Debug-netcoreapp3_0", "Release-netcoreapp3_0", "Debug-netfx", "Release-netfx" ],
       "defaultValue": "Debug"
     },
     "TargetArchitecture": {
@@ -106,16 +106,16 @@
             "Configuration": "Release"
           }
         },
-        "debug-intrinsics": {
-          "description": "Sets optimization level to debug for managed build configuration and builds against netcoreapp3.0. (/p:Configuration=Debug-Intrinsics)",
+        "debug-netcoreapp3_0": {
+          "description": "Sets optimization level to debug for managed build configuration and builds against netcoreapp3.0. (/p:Configuration=Debug-netcoreapp3_0)",
           "settings": {
-            "Configuration": "Debug-Intrinsics"
+            "Configuration": "Debug-netcoreapp3_0"
           }
         },
-        "release-intrinsics": {
-          "description": "Sets optimization level to release for managed build configuration and builds against netcoreapp3.0. (/p:Configuration=Release-Intrinsics)",
+        "release-netcoreapp3_0": {
+          "description": "Sets optimization level to release for managed build configuration and builds against netcoreapp3.0. (/p:Configuration=Release-netcoreapp3_0)",
           "settings": {
-            "Configuration": "Release-Intrinsics"
+            "Configuration": "Release-netcoreapp3_0"
           }
         },
         "debug-netfx": {

Microsoft.ML.sln

@@ -1,11 +1,11 @@
 In order to build ML.NET for .NET Core 3.0, you need to do a few manual steps.
 
 1. Delete the local `.\Tools\` folder from the root of the repo, to ensure you download the new version of the .NET Core SDK.
-2. Run `.\build.cmd -- /p:Configuration=Release-Intrinsics` or `.\build.cmd -Release-Intrinsics` from the root of the repo.
+2. Run `.\build.cmd -- /p:Configuration=Release-netcoreapp3_0` or `.\build.cmd -Release-netcoreapp3_0` from the root of the repo.
 3. If you want to build the NuGet packages, `.\build.cmd -buildPackages` after step 2.
 
 If you are using Visual Studio, you will need to do the following:
 
 1. Install the above .NET Core 3.0 SDK into %Program Files%. Or extract it to a directory and put that directory at the front of your %PATH%, so it is the first `dotnet.exe` on the PATH.
-2. In the Configuration Manager, switch the current configuration to `Debug-Intrinsics` or `Release-Intrinsics`.
+2. In the Configuration Manager, switch the current configuration to `Debug-netcoreapp3_0` or `Release-netcoreapp3_0`.
 3. Build and test as usual.

build/AfterCommonTargets.targets

@@ -46,25 +46,34 @@ if exist "%DotNetBuildToolsDir%" (
 echo Running %0 > "%INIT_TOOLS_LOG%"
 
 set /p DOTNET_VERSION=< "%~dp0DotnetCLIVersion.txt"
+set /p DOTNET_EXTRA_RUNTIME_VERSION=< "%~dp0DotnetExtraRuntimeVersion.txt"
 
 :Arg_Loop
 if [%1] == [] goto :ArchSet
 if /i [%1] == [x86]         ( set ARCH=x86)
-if /i [%1] == [-Debug-Intrinsics] ( set /p DOTNET_VERSION=< "%~dp0DotnetCLIVersion.netcoreapp.latest.txt")
-if /i [%1] == [-Release-Intrinsics] ( set /p DOTNET_VERSION=< "%~dp0DotnetCLIVersion.netcoreapp.latest.txt")
 shift
 goto :Arg_Loop
 
 :ArchSet
 if exist "%DOTNET_CMD%" goto :afterdotnetrestore
 
-echo Installing dotnet cli...
 if NOT exist "%DOTNET_PATH%" mkdir "%DOTNET_PATH%"
+
+:: install the extra runtime first, so the SDK install will overwrite the root dotnet executable
+echo Installing dotnet runtime %DOTNET_EXTRA_RUNTIME_VERSION%...
+set DOTNET_EXTRA_RUNTIME_ZIP_NAME=dotnet-runtime-%DOTNET_EXTRA_RUNTIME_VERSION%-win-%ARCH%.zip
+set DOTNET_EXTRA_RUNTIME_REMOTE_PATH=https://dotnetcli.azureedge.net/dotnet/Runtime/%DOTNET_EXTRA_RUNTIME_VERSION%/%DOTNET_EXTRA_RUNTIME_ZIP_NAME%
+set DOTNET_EXTRA_RUNTIME_LOCAL_PATH=%DOTNET_PATH%%DOTNET_EXTRA_RUNTIME_ZIP_NAME%
+echo Installing '%DOTNET_EXTRA_RUNTIME_REMOTE_PATH%' to '%DOTNET_EXTRA_RUNTIME_LOCAL_PATH%' >> "%INIT_TOOLS_LOG%"
+powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; $proxyCredentialsRequired = $false; do { try { $wc = New-Object Net.WebClient; if ($proxyCredentialsRequired) { [Net.WebRequest]::DefaultWebProxy.Credentials = [Net.CredentialCache]::DefaultNetworkCredentials; } $wc.DownloadFile('%DOTNET_EXTRA_RUNTIME_REMOTE_PATH%', '%DOTNET_EXTRA_RUNTIME_LOCAL_PATH%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $we = $_.Exception.InnerException -as [Net.WebException]; $proxyCredentialsRequired = ($we -ne $null -and ([Net.HttpWebResponse]$we.Response).StatusCode -eq [Net.HttpStatusCode]::ProxyAuthenticationRequired); Start-Sleep -Seconds (5 * $retryCount); $retryCount++; } } } while ($success -eq $false); Expand-Archive '%DOTNET_EXTRA_RUNTIME_LOCAL_PATH%' '%DOTNET_PATH%';" >> "%INIT_TOOLS_LOG%"
+
+echo Installing dotnet cli %DOTNET_VERSION%...
 set DOTNET_ZIP_NAME=dotnet-sdk-%DOTNET_VERSION%-win-%ARCH%.zip
 set DOTNET_REMOTE_PATH=https://dotnetcli.azureedge.net/dotnet/Sdk/%DOTNET_VERSION%/%DOTNET_ZIP_NAME%
 set DOTNET_LOCAL_PATH=%DOTNET_PATH%%DOTNET_ZIP_NAME%
 echo Installing '%DOTNET_REMOTE_PATH%' to '%DOTNET_LOCAL_PATH%' >> "%INIT_TOOLS_LOG%"
-powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; $proxyCredentialsRequired = $false; do { try { $wc = New-Object Net.WebClient; if ($proxyCredentialsRequired) { [Net.WebRequest]::DefaultWebProxy.Credentials = [Net.CredentialCache]::DefaultNetworkCredentials; } $wc.DownloadFile('%DOTNET_REMOTE_PATH%', '%DOTNET_LOCAL_PATH%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $we = $_.Exception.InnerException -as [Net.WebException]; $proxyCredentialsRequired = ($we -ne $null -and ([Net.HttpWebResponse]$we.Response).StatusCode -eq [Net.HttpStatusCode]::ProxyAuthenticationRequired); Start-Sleep -Seconds (5 * $retryCount); $retryCount++; } } } while ($success -eq $false); Add-Type -Assembly 'System.IO.Compression.FileSystem' -ErrorVariable AddTypeErrors; if ($AddTypeErrors.Count -eq 0) { [System.IO.Compression.ZipFile]::ExtractToDirectory('%DOTNET_LOCAL_PATH%', '%DOTNET_PATH%') } else { (New-Object -com shell.application).namespace('%DOTNET_PATH%').CopyHere((new-object -com shell.application).namespace('%DOTNET_LOCAL_PATH%').Items(),16) }" >> "%INIT_TOOLS_LOG%"
+powershell -NoProfile -ExecutionPolicy unrestricted -Command "$retryCount = 0; $success = $false; $proxyCredentialsRequired = $false; do { try { $wc = New-Object Net.WebClient; if ($proxyCredentialsRequired) { [Net.WebRequest]::DefaultWebProxy.Credentials = [Net.CredentialCache]::DefaultNetworkCredentials; } $wc.DownloadFile('%DOTNET_REMOTE_PATH%', '%DOTNET_LOCAL_PATH%'); $success = $true; } catch { if ($retryCount -ge 6) { throw; } else { $we = $_.Exception.InnerException -as [Net.WebException]; $proxyCredentialsRequired = ($we -ne $null -and ([Net.HttpWebResponse]$we.Response).StatusCode -eq [Net.HttpStatusCode]::ProxyAuthenticationRequired); Start-Sleep -Seconds (5 * $retryCount); $retryCount++; } } } while ($success -eq $false); Expand-Archive '%DOTNET_LOCAL_PATH%' '%DOTNET_PATH%' -Force; " >> "%INIT_TOOLS_LOG%"
+
 if NOT exist "%DOTNET_LOCAL_PATH%" (
   echo ERROR: Could not install dotnet cli correctly. 1>&2
   goto :error

build/Empty.targets

@@ -6,21 +6,15 @@ __PACKAGES_DIR="$__scriptpath/packages"
 __TOOLRUNTIME_DIR="$__scriptpath/Tools"
 __DOTNET_PATH="$__TOOLRUNTIME_DIR/dotnetcli"
 __DOTNET_CMD="$__DOTNET_PATH/dotnet"
-__TARGET_FRAMEWORK="netcoreapp2.1"
 if [ -z "${__BUILDTOOLS_SOURCE:-}" ]; then __BUILDTOOLS_SOURCE=https://dotnet.myget.org/F/dotnet-buildtools/api/v3/index.json; fi
 export __BUILDTOOLS_USE_CSPROJ=true
 __BUILD_TOOLS_PACKAGE_VERSION=$(cat "$__scriptpath/BuildToolsVersion.txt" | sed 's/\r$//') # remove CR if mounted repo on Windows drive
 
 DotNetCliFileName="DotnetCLIVersion.txt"
-
-for i do
-    if [[ "$i" == *"Intrinsics"* ]]; then
-        DotNetCliFileName="DotnetCLIVersion.netcoreapp.latest.txt"
-        __TARGET_FRAMEWORK="netcoreapp3.0"
-    fi
-done
+DotNetExtraRuntimeFileName="DotnetExtraRuntimeVersion.txt"
 
 __DOTNET_TOOLS_VERSION=$(cat "$__scriptpath/$DotNetCliFileName" | sed 's/\r$//') # remove CR if mounted repo on Windows drive
+__DOTNET_EXTRA_RUNTIME_VERSION=$(cat "$__scriptpath/$DotNetExtraRuntimeFileName" | sed 's/\r$//') # remove CR if mounted repo on Windows drive
 __BUILD_TOOLS_PATH="$__PACKAGES_DIR/microsoft.dotnet.buildtools/$__BUILD_TOOLS_PACKAGE_VERSION/lib"
 __INIT_TOOLS_RESTORE_PROJECT="$__scriptpath/init-tools.msbuild"
 __BUILD_TOOLS_SEMAPHORE="$__TOOLRUNTIME_DIR/$__BUILD_TOOLS_PACKAGE_VERSION/init-tools.complete"
@@ -129,10 +123,34 @@ if [ ! -e "$__DOTNET_PATH" ]; then
         esac
         __PKG_RID=$__PKG_RID-$__PKG_ARCH
         __DOTNET_PKG=dotnet-sdk-${__DOTNET_TOOLS_VERSION}-$__PKG_RID
+        __DOTNET_EXTRA_RUNTIME_PKG=dotnet-runtime-${__DOTNET_EXTRA_RUNTIME_VERSION}-$__PKG_RID
     fi
     mkdir -p "$__DOTNET_PATH"
 
-    echo "Installing dotnet cli..."
+	# install the extra runtime first, so the SDK install will overwrite the root dotnet executable
+    echo "Installing dotnet runtime ${__DOTNET_EXTRA_RUNTIME_VERSION}..."
+    __DOTNET_EXTRA_RUNTIME_LOCATION="https://dotnetcli.azureedge.net/dotnet/Runtime/${__DOTNET_EXTRA_RUNTIME_VERSION}/${__DOTNET_EXTRA_RUNTIME_PKG}.tar.gz"
+
+    install_dotnet_extra_runtime() {
+        if [[ -z "${DotNetExtraRuntimeTarPath-}" ]]; then
+            echo "Installing '${__DOTNET_EXTRA_RUNTIME_LOCATION}' to '$__DOTNET_PATH/dotnet.extra.runtime.tar'"
+            rm -rf -- "$__DOTNET_PATH/*"
+            # curl has HTTPS CA trust-issues less often than wget, so lets try that first.
+            if command -v curl > /dev/null; then
+                curl --retry 10 -sSL --create-dirs -o $__DOTNET_PATH/dotnet.extra.runtime.tar ${__DOTNET_EXTRA_RUNTIME_LOCATION}
+            else
+                wget -q -O $__DOTNET_PATH/dotnet.extra.runtime.tar ${__DOTNET_EXTRA_RUNTIME_LOCATION}
+            fi
+        else
+            echo "Copying '$DotNetExtraRuntimeTarPath' to '$__DOTNET_PATH/dotnet.extra.runtime.tar'"
+            cp $DotNetExtraRuntimeTarPath $__DOTNET_PATH/dotnet.extra.runtime.tar
+        fi
+        cd "$__DOTNET_PATH"
+        tar -xf "$__DOTNET_PATH/dotnet.extra.runtime.tar"
+    }
+    execute_with_retry install_dotnet_extra_runtime >> "$__init_tools_log" 2>&1
+
+    echo "Installing dotnet cli ${__DOTNET_TOOLS_VERSION}..."
     __DOTNET_LOCATION="https://dotnetcli.azureedge.net/dotnet/Sdk/${__DOTNET_TOOLS_VERSION}/${__DOTNET_PKG}.tar.gz"
 
     install_dotnet_cli() {
@@ -184,8 +202,6 @@ echo "Making all .sh files executable under Tools."
 ls "$__scriptpath/Tools/"*.sh | xargs chmod +x
 ls "$__scriptpath/Tools/scripts/docker/"*.sh | xargs chmod +x
 
-"$__scriptpath/Tools/crossgen.sh" "$__scriptpath/Tools" $__PKG_RID $__TARGET_FRAMEWORK
-
 mkdir -p "$(dirname "$__BUILD_TOOLS_SEMAPHORE")" && touch "$__BUILD_TOOLS_SEMAPHORE"
 
 echo "Done initializing tools."

config.json

@@ -1,8 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk" DefaultTargets="Pack">
 
   <PropertyGroup>
-    <TargetFramework Condition="'$(UseIntrinsics)' != 'true'">netstandard2.0</TargetFramework>
-    <TargetFrameworks Condition="'$(UseIntrinsics)' == 'true'">netstandard2.0;netcoreapp3.0</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0;netcoreapp3.0</TargetFrameworks>
     <PackageDescription>Microsoft.ML.CpuMath contains optimized math routines for ML.NET.</PackageDescription>
   </PropertyGroup>
 

docs/building/netcoreapp3.0-instructions.md

@@ -5,8 +5,7 @@
 using System.Runtime.CompilerServices;
 using Microsoft.ML.Internal.CpuMath.Core;
 
-[assembly: InternalsVisibleTo("Microsoft.ML.CpuMath.UnitTests.netstandard" + PublicKey.TestValue)]
-[assembly: InternalsVisibleTo("Microsoft.ML.CpuMath.UnitTests.netcoreapp" + PublicKey.TestValue)]
+[assembly: InternalsVisibleTo("Microsoft.ML.CpuMath.UnitTests" + PublicKey.TestValue)]
 [assembly: InternalsVisibleTo("Microsoft.ML.Data" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.FastTree" + PublicKey.Value)]
 [assembly: InternalsVisibleTo("Microsoft.ML.Mkl.Components" + PublicKey.Value)]

init-tools.cmd

@@ -1,8 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework Condition="'$(UseIntrinsics)' != 'true'">netstandard2.0</TargetFramework>
-    <TargetFrameworks Condition="'$(UseIntrinsics)' == 'true'">netstandard2.0;netcoreapp3.0</TargetFrameworks>
+    <TargetFrameworks>netstandard2.0;netcoreapp3.0</TargetFrameworks>
     <IncludeInPackage>Microsoft.ML.CpuMath</IncludeInPackage>
     <AllowUnsafeBlocks>true</AllowUnsafeBlocks>
     <DefineConstants>$(DefineConstants);CPUMATH_INFRASTRUCTURE</DefineConstants>

init-tools.sh

@@ -18,10 +18,10 @@ set MKL_LIB_PATH=""
 :Arg_Loop
 if [%1] == [] goto :ToolsVersion
 if /i [%1] == [Release]     ( set CMAKE_BUILD_TYPE=Release&&shift&goto Arg_Loop)
-if /i [%1] == [Release-Intrinsics]     ( set CMAKE_BUILD_TYPE=Release-Intrinsics&&shift&goto Arg_Loop)
+if /i [%1] == [Release-netcoreapp3_0]     ( set CMAKE_BUILD_TYPE=Release-netcoreapp3_0&&shift&goto Arg_Loop)
 if /i [%1] == [Release-netfx]     ( set CMAKE_BUILD_TYPE=Release-netfx&&shift&goto Arg_Loop)
 if /i [%1] == [Debug]       ( set CMAKE_BUILD_TYPE=Debug&&shift&goto Arg_Loop)
-if /i [%1] == [Debug-Intrinsics]       ( set CMAKE_BUILD_TYPE=Debug-Intrinsics&&shift&goto Arg_Loop)
+if /i [%1] == [Debug-netcoreapp3_0]       ( set CMAKE_BUILD_TYPE=Debug-netcoreapp3_0&&shift&goto Arg_Loop)
 if /i [%1] == [Debug-netfx]       ( set CMAKE_BUILD_TYPE=Debug-netfx&&shift&goto Arg_Loop)
 
 if /i [%1] == [x86]         ( set __BuildArch=x86&&set __VCBuildArch=x86&&shift&goto Arg_Loop)
@@ -104,9 +104,9 @@ if %__IntermediatesDir% == "" (
 set "__CMakeBinDir=%__CMakeBinDir:\=/%"
 set "__IntermediatesDir=%__IntermediatesDir:\=/%"
 
-:: Strip the "-Intrinsics" suffix from the build type
-if [%CMAKE_BUILD_TYPE:~-11%] == [-Intrinsics] (
-	set CMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE:~0,-11%
+:: Strip the "-netcoreapp3_0" suffix from the build type
+if [%CMAKE_BUILD_TYPE:~-14%] == [-netcoreapp3_0] (
+	set CMAKE_BUILD_TYPE=%CMAKE_BUILD_TYPE:~0,-14%
 )
 
 :: Strip the "-netfx" suffix from the build type