nightly build pipeline

Directory.Build.props

@@ -19,6 +19,7 @@
       https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json;
       https://dotnet.myget.org/F/dotnet-core/api/v3/index.json;
       https://dotnet.myget.org/F/roslyn-analyzers/api/v3/index.json;
+      https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json;
     </RestoreSources>
   </PropertyGroup>
 

Microsoft.ML.sln

@@ -285,6 +285,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.Vision", "src\
 EndProject
 Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.ML.TestFrameworkCommon", "test\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj", "{A22FAD27-77E8-4460-8B92-EC7090B7173A}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.NightlyBuild.Tests", "test\Microsoft.ML.NightlyBuild.Tests\Microsoft.ML.NightlyBuild.Tests.csproj", "{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Microsoft.ML.NugetPackageVersionUpdater", "test\Microsoft.ML.NugetPackageVersionUpdater\Microsoft.ML.NugetPackageVersionUpdater.csproj", "{C8DB58DC-6434-4431-A81F-263D86E2A5F3}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1716,6 +1720,42 @@ Global
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release|x64.ActiveCfg = Release|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug|x64.Build.0 = Debug|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netcoreapp3_0|Any CPU.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netcoreapp3_0|Any CPU.Build.0 = Debug-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netcoreapp3_0|x64.ActiveCfg = Debug-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netcoreapp3_0|x64.Build.0 = Debug-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netfx|Any CPU.ActiveCfg = Debug-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netfx|Any CPU.Build.0 = Debug-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netfx|x64.ActiveCfg = Debug-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Debug-netfx|x64.Build.0 = Debug-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release|Any CPU.Build.0 = Release|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release|x64.ActiveCfg = Release|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release|x64.Build.0 = Release|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netcoreapp3_0|Any CPU.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netcoreapp3_0|Any CPU.Build.0 = Release-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netcoreapp3_0|x64.ActiveCfg = Release-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netcoreapp3_0|x64.Build.0 = Release-netcoreapp3_0|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netfx|Any CPU.ActiveCfg = Release-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netfx|Any CPU.Build.0 = Release-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netfx|x64.ActiveCfg = Release-netfx|Any CPU
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3}.Release-netfx|x64.Build.0 = Release-netfx|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1806,6 +1846,8 @@ Global
 		{3817A875-278C-4140-BF66-3C4A8CA55F0D} = {D3D38B03-B557-484D-8348-8BADEE4DF592}
 		{419F93D5-4135-4DA0-A76E-EFC23E04093D} = {09EADF06-BE25-4228-AB53-95AE3E15B530}
 		{A22FAD27-77E8-4460-8B92-EC7090B7173A} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{A1CAC86F-F4BB-4B6D-9D18-E9AE15B3C66E} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
+		{C8DB58DC-6434-4431-A81F-263D86E2A5F3} = {AED9C836-31E3-4F3F-8ABC-929555D3F3C4}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {41165AF1-35BB-4832-A189-73060F82B01D}

build.proj

@@ -97,6 +97,11 @@
              Targets="RunTests" />
   </Target>
 
+  <Target Name="RunNightlyBuildTests">
+    <MSBuild Projects="test\run-night-build-tests.proj"
+             Targets="RunNightlyBuildTests" />
+  </Target>
+
   <!-- Override CleanAllProjects from dir.traversal.targets and just remove the full BinDir -->
   <Target Name="CleanAllProjects">
     <RemoveDir Directories="$(BinDir)" />

build/.night-build.yml

@@ -0,0 +1,117 @@
+################################################################################
+# ML.NET's nightly build
+################################################################################
+
+# no PR builds
+pr: none
+
+# no CI builds
+trigger: none
+
+# scheduled trigger, runs at UTC 8:00 every day which is midnight of GMT-8 
+schedules:
+- cron: "0 8 * * *"
+  displayName: Nightly Build at midnight
+  branches:
+    include:
+    - master
+    - releases/1.0
+    - features/automl
+    - features/integrationPackage
+  always: true
+
+resources:
+  containers:
+  - container: CentosContainer
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:centos-7-mlnet-8bba86b-20190314145033
+
+  - container: UbuntuContainer
+    image: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-16.04-mlnet-207e097-20190312152303
+
+jobs:
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Centos_x64_NetCoreApp30
+    buildScript: ./build.sh
+    container: CentosContainer
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_0
+        _config_short: DI
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netcoreapp3_0
+        _config_short: RI
+        _includeBenchmarkData: true
+    nightlyBuild: true
+    pool:
+      name: Hosted Ubuntu 1604
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Ubuntu_x64_NetCoreApp21
+    buildScript: ./build.sh
+    container: UbuntuContainer
+    nightlyBuild: true
+    pool:
+      name: Hosted Ubuntu 1604
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: MacOS_x64_NetCoreApp21
+    buildScript: ./build.sh
+    nightlyBuild: true
+    pool:
+      name: Hosted macOS High Sierra
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Windows_x64_NetCoreApp30
+    buildScript: build.cmd
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netcoreapp3_0
+        _config_short: DI
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netcoreapp3_0
+        _config_short: RI
+        _includeBenchmarkData: true
+    nightlyBuild: true
+    pool:
+      name: NetCorePublic-Pool
+      queue: buildpool.windows.10.amd64.vs2017.open
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Windows_x64_NetCoreApp21
+    buildScript: build.cmd
+    nightlyBuild: true
+    pool:
+      name: Hosted VS2017
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Windows_x64_NetFx461
+    buildScript: build.cmd
+    customMatrixes:
+      Debug_Build:
+        _configuration: Debug-netfx
+        _config_short: DFX
+        _includeBenchmarkData: false
+      Release_Build:
+        _configuration: Release-netfx
+        _config_short: RFX
+        _includeBenchmarkData: false
+    nightlyBuild: true
+    pool:
+      name: Hosted VS2017
+
+- template: /build/ci/job-template.yml
+  parameters:
+    name: Windows_x86_NetCoreApp21
+    architecture: x86
+    buildScript: build.cmd
+    nightlyBuild: true
+    pool:
+      name: Hosted VS2017

build/ci/job-template.yml

@@ -5,14 +5,24 @@ parameters:
   pool: {}
   customMatrixes: ''
   codeCoverage: false
+  nightlyBuild: false
   container: ''
 
 jobs:
   - job: ${{ parameters.name }}
-    ${{ if eq(parameters.codeCoverage, 'false') }}:
+    ${{ if eq(parameters.nightlyBuild, 'true') }}:
+      timeoutInMinutes: 20
+    ${{ if and(eq(parameters.nightlyBuild, 'false'), eq(parameters.codeCoverage, 'false')) }}:
       timeoutInMinutes: 75
     ${{ if eq(parameters.codeCoverage, 'true') }}:
       timeoutInMinutes: 60
+    variables:
+      dotnetPath: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet
+      nugetFeed: https://pkgs.dev.azure.com/dnceng/public/_packaging/MachineLearning/nuget/v3/index.json
+      nightlyBuildProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj
+      nightlyBuildRunPath: (Build.SourcesDirectory)/bin/AnyCPU.(_configuration)/Microsoft.ML.NightlyBuild.Tests/netcoreapp2.1
+      packageUpdaterProjPath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj
+      versionFilePath: $(Build.SourcesDirectory)/test/Microsoft.ML.NugetPackageVersionUpdater/latest_versions.txt
     strategy:
       matrix:
         ${{ if eq(parameters.customMatrixes, '') }}:
@@ -35,12 +45,27 @@ jobs:
     - ${{ if eq(parameters.pool.name, 'Hosted macOS High Sierra') }}:
       - script: brew update && brew install https://github.com/raw/Homebrew/homebrew-core/f5b1ac99a7fba27c19cee0bc4f036775c889b359/Formula/libomp.rb && brew install mono-libgdiplus gettext && brew link gettext --force && brew link libomp --force
         displayName: Install build dependencies
+    - ${{ if and( eq(parameters.nightlyBuild, 'true'), eq(parameters.name, 'Ubuntu_x64_NetCoreApp21')) }}:
+      - bash: echo "##vso[task.setvariable variable=LD_LIBRARY_PATH]$(nightlyBuildRunPath):$LD_LIBRARY_PATH"
+        displayName: Set LD_LIBRARY_PATH for Ubuntu to locate Native shared library in current running path
     - script: ${{ parameters.buildScript }} -$(_configuration) -buildArch=${{ parameters.architecture }}
       displayName: Build
-    - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
-      displayName: Download Benchmark Data
-    - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
-      displayName: Run Tests.
+    - ${{ if eq(parameters.nightlyBuild, 'true') }}:
+      - script: $(dotnetPath) restore $(nightlyBuildProjPath)
+        displayName: Restore nightly build project
+      - script: $(dotnetPath) list $(nightlyBuildProjPath) package --source $(nugetFeed) --outdated > $(versionFilePath)
+        displayName: List latest package versions
+      - script: $(dotnetPath) run --project $(packageUpdaterProjPath)
+        displayName: Update package versions for nightly build
+      - script: $(dotnetPath) msbuild -restore $(nightlyBuildProjPath) /p:ReferenceTypeForTestFramework="Nuget" /p:Configuration=$(_configuration) /p:TargetArchitecture=${{ parameters.architecture }}
+        displayName: Build Nightly-Build Project with latest package versions
+      - script: ${{ parameters.buildScript }} -$(_configuration) -runnightlybuildtests
+        displayName: Run Nightly Build Tests
+    - ${{ if eq(parameters.nightlyBuild, 'false') }}:
+      - script: ${{ parameters.buildScript }} -- /t:DownloadExternalTestFiles /p:IncludeBenchmarkData=$(_includeBenchmarkData)
+        displayName: Download Benchmark Data
+      - script: ${{ parameters.buildScript }} -$(_configuration) -runtests -coverage=${{ parameters.codeCoverage }}
+        displayName: Run Tests.
     - script: $(Build.SourcesDirectory)/Tools/dotnetcli/dotnet msbuild build/Codecoverage.proj /p:CodeCovToken=$(CODECOV_TOKEN)
       displayName: Upload coverage to codecov.io
       condition: and(succeeded(), eq(${{ parameters.codeCoverage }}, True))
@@ -77,7 +102,7 @@ jobs:
         pathToPublish: $(Build.ArtifactStagingDirectory)
         artifactName: ${{ parameters.name }} $(_config_short)
         artifactType: container
-    - ${{ if eq(parameters.buildScript, 'build.cmd') }}:
+    - ${{ if and(eq(parameters.nightlyBuild, 'false'), eq(parameters.buildScript, 'build.cmd')) }}:
       - powershell: |
                   $taskListOutput = & TaskList /FI "IMAGENAME eq dotnet.exe" | Out-String
                   if ($taskListOutput.Contains("dotnet.exe"))
@@ -107,5 +132,6 @@ jobs:
                   Remove-Item -Force -Recurse .\tmp\
                   Write-Output "Done cleaning up binary folder..."
         displayName: Clean up test data
-    - script: ${{ parameters.buildScript }} -buildPackages
-      displayName: Build Packages
+    - ${{ if eq(parameters.nightlyBuild, 'false') }}:
+      - script: ${{ parameters.buildScript }} -buildPackages
+        displayName: Build Packages

config.json

@@ -78,6 +78,12 @@
       "values": [],
       "defaultValue": ""
     },
+    "RunNightlyBuildTests": {
+      "description": "MsBuild target that run the nightly build tests. Call this after building.",
+        "valueType": "target",
+        "values": [],
+        "defaultValue": ""
+    },
     "Coverage": {
       "description": "Turn on code coverge.",
       "valueType": "property",
@@ -160,6 +166,12 @@
             "RunTests": "default"
           }
         },
+        "runnightlybuildtests": {
+          "description": "Runs the nightly build tests. Call this after building.",
+          "settings": {
+            "RunNightlyBuildTests": "default"
+          }
+        },
         "verbose": {
           "description": "Passes /flp:v=diag to the msbuild command or the value passed by the user.",
           "settings": {

test/Microsoft.ML.Functional.Tests/Microsoft.ML.Functional.Tests.csproj

@@ -5,6 +5,7 @@
     <SignAssembly>false</SignAssembly>
     <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
     <PublicSign>false</PublicSign>
+    <ReferenceTypeForTestFramework>Project</ReferenceTypeForTestFramework>
   </PropertyGroup>
 
   <ItemGroup>

test/Microsoft.ML.NightlyBuild.Tests/Microsoft.ML.NightlyBuild.Tests.csproj

@@ -0,0 +1,32 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
+    <SignAssembly>false</SignAssembly>
+    <!-- DON'T CHANGE THIS!!! We are turning off strong naming to ensure we never add `InternalsVisibleTo` for these tests -->
+    <PublicSign>false</PublicSign>
+    <IsPackable>false</IsPackable>
+  </PropertyGroup>
+
+  <Import Project="..\NightlyBuildDependency.props" />
+
+  <ItemGroup>
+    <Compile Include="..\Microsoft.ML.Functional.Tests\**\*.cs" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <NativeAssemblyReference Include="CpuMathNative" />
+    <NativeAssemblyReference Include="FastTreeNative" />
+    <NativeAssemblyReference Include="MatrixFactorizationNative" />
+    <NativeAssemblyReference Include="LdaNative" />
+    <NativeAssemblyReference Include="SymSgdNative" />
+    <NativeAssemblyReference Include="MklProxyNative" />
+    <NativeAssemblyReference Include="MklImports" />
+    <NativeAssemblyReference Condition="'$(OS)' == 'Windows_NT'" Include="libiomp5md" />
+  </ItemGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\Microsoft.ML.TestFrameworkCommon\Microsoft.ML.TestFrameworkCommon.csproj" />
+  </ItemGroup>
+
+</Project>

test/Microsoft.ML.NugetPackageVersionUpdater/Microsoft.ML.NugetPackageVersionUpdater.csproj

@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <StartupObject>Microsoft.ML.NugetPackageVersionUpdater.Program</StartupObject>
+    <RunWorkingDirectory>$(MSBuildProjectDirectory)</RunWorkingDirectory>
+  </PropertyGroup>
+
+</Project>