[Helix] Shared framework support + Templates tests

.azure/pipelines/ci.yml

@@ -154,8 +154,7 @@ stages:
         displayName: Build x64
 
       # Build the x86 shared framework
-      # TODO: make it possible to build for one Windows architecture at a time
-      # This is going to actually build x86 native assets. See https://github.com/dotnet/aspnetcore/issues/7196
+      # This is going to actually build x86 native assets.
       - script: ./build.cmd
                 -ci
                 -arch x86
@@ -631,7 +630,7 @@ stages:
         publishOnError: true
         includeForks: true
 
-# Helix x64
+  # Helix x64
   - template: jobs/default-build.yml
     parameters:
       condition: eq(variables['Build.Reason'], 'PullRequest')
@@ -640,9 +639,12 @@ stages:
       agentOs: Windows
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
+      - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
+        displayName: Build shared fx
       - script: .\restore.cmd -ci
         displayName: Restore
-      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
           HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -661,9 +663,13 @@ stages:
       agentOs: Windows
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
+      - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.daily.build.x64.binlog
+        displayName: Build shared fx
+      # Build the x86 shared framework
       - script: .\restore.cmd -ci
         displayName: Restore
-      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+      - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
         displayName: Run build.cmd helix target
         env:
           HelixApiAccessToken: $(HelixApiAccessToken) # Needed for internal queues
@@ -683,6 +689,7 @@ stages:
       agentOs: Linux
       timeoutInMinutes: 180
       steps:
+      # Build the shared framework
       - script: ./restore.sh -ci
         displayName: Restore
       - script: ./build.sh -ci --arch arm64 -test --no-build-nodejs -projects $(Build.SourcesDirectory)/eng/helix/helix.proj /p:IsHelixJob=true /p:IsHelixDaily=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl

.azure/pipelines/quarantined-tests.yml

@@ -27,9 +27,12 @@ jobs:
     agentOs: Windows
     timeoutInMinutes: 240
     steps:
+    # Build the shared framework
+    - script: ./build.cmd -ci -all -pack -arch x64 -buildNative /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log /bl:artifacts/log/helix.build.x64.binlog
+      displayName: Build shared fx    
     - script: .\restore.cmd -ci
       displayName: Restore
-    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
+    - script: .\build.cmd -ci -NoRestore -test -projects eng\helix\helix.proj /p:RunQuarantinedTests=true /p:IsRequiredCheck=true /p:IsHelixJob=true /p:BuildAllProjects=true /p:BuildNative=true /p:RunTemplateTests=true /p:ASPNETCORE_TEST_LOG_DIR=artifacts/log -bl
       displayName: Run build.cmd helix target
       env:
         SYSTEM_ACCESSTOKEN: $(System.AccessToken) # We need to set this env var to publish helix results to Azure Dev Ops

eng/helix/content/InstallAppRuntime.ps1

@@ -0,0 +1,54 @@
+ <# 
+ .SYNOPSIS 
+     Installs an AspNetCore shared framework on a machine
+ .DESCRIPTION 
+     This script installs an AspNetCore shared framework on a machine
+ .PARAMETER AppRuntimePath
+     The path to the app runtime package to install.
+ .PARAMETER InstallDir
+     The directory to install the shared framework to.     
+ .PARAMETER Framework
+     The framework directory to copy the shared framework from.
+ .PARAMETER RuntimeIdentifier
+     The runtime identifier for the shared framework.
+ #> 
+param(
+    [Parameter(Mandatory = $true)]
+    $AppRuntimePath,
+
+    [Parameter(Mandatory = $true)]
+    $InstallDir,
+
+    [Parameter(Mandatory = $true)]
+    $Framework,
+
+    [Parameter(Mandatory = $true)]
+    $RuntimeIdentifier)
+
+$ErrorActionPreference = 'Stop'
+$ProgressPreference = 'SilentlyContinue' # Workaround PowerShell/PowerShell#2138
+
+Set-StrictMode -Version 1
+
+Write-Host "Extracting to $InstallDir"
+
+$zipPackage = [io.path]::ChangeExtension($AppRuntimePath, ".zip")
+Write-Host "Renaming to $zipPackage"
+Rename-Item -Path $AppRuntimePath -NewName $zipPackage
+if (Get-Command -Name 'Microsoft.PowerShell.Archive\Expand-Archive' -ErrorAction Ignore) {
+    # Use built-in commands where possible as they are cross-plat compatible
+    Microsoft.PowerShell.Archive\Expand-Archive -Path $zipPackage -DestinationPath ".\tmpRuntime" -Force
+}
+else {
+    Remove-Item ".\tmpRuntime" -Recurse -ErrorAction Ignore
+    # Fallback to old approach for old installations of PowerShell
+    Add-Type -AssemblyName System.IO.Compression.FileSystem
+    [System.IO.Compression.ZipFile]::ExtractToDirectory($zipPackage, ".\tmpRuntime")
+}
+
+Get-ChildItem -Path ".\tmpRuntime" -Recurse
+
+Write-Host "Copying managed files to $InstallDir"
+Copy-Item -Path ".\tmpRuntime\runtimes\$RuntimeIdentifier\lib\$Framework\*" $InstallDir
+Write-Host "Copying native files to $InstallDir"
+Copy-Item -Path ".\tmpRuntime\runtimes\$RuntimeIdentifier\native\*" $InstallDir

eng/helix/content/InstallNode.ps1

@@ -29,9 +29,9 @@ if (Get-Command "node.exe" -ErrorAction SilentlyContinue)
     exit
 }
 
-if (Test-Path "$output_dir\node.exe")
+if (Test-Path "$InstallDir\node.exe")
 {
-    Write-Host "Node.exe found at $output_dir"
+    Write-Host "Node.exe found at $InstallDir"
     exit
 }
 

eng/helix/content/installappruntime.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+# Cause the script to fail if any subcommand fails
+set -e
+
+appRuntimePath=$1
+output_dir=$2
+framework=$3
+rid=$4
+tmpDir=./tmpRuntime
+
+echo "Installing shared framework from $appRuntimePath"
+cp $appRuntimePath sharedFx.zip
+
+mkdir -p $tmpDir
+unzip sharedFx.zip -d $tmpDir
+mkdir -p $output_dir
+echo "Copying to $output_dir"
+cp $tmpDir/runtimes/$rid/lib/$framework/* $output_dir
+cp $tmpDir/runtimes/$rid/native/* $output_dir

eng/helix/content/runtests.cmd

@@ -21,7 +21,22 @@ set PATH=%DOTNET_ROOT%;%PATH%;%HELIX_CORRELATION_PAYLOAD%\node\bin
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Version %$sdkVersion% -InstallDir %DOTNET_ROOT%"
 powershell.exe -NoProfile -ExecutionPolicy unrestricted -Command "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; &([scriptblock]::Create((Invoke-WebRequest -useb 'https://dot.net/v1/dotnet-install.ps1'))) -Architecture %$arch% -Runtime dotnet -Version %$runtimeVersion% -InstallDir %DOTNET_ROOT%"
 
+if EXIST ".\Microsoft.AspNetCore.App" (
+    echo "Found Microsoft.AspNetCore.App, copying to %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%"
+    xcopy /i /y ".\Microsoft.AspNetCore.App" %DOTNET_ROOT%\shared\Microsoft.AspNetCore.App\%runtimeVersion%\
+)
+
+echo "Current Directory: %HELIX_WORKITEM_ROOT%"
 set HELIX=%$helixQueue%
+set HELIX_DIR=%HELIX_WORKITEM_ROOT%
+set NUGET_FALLBACK_PACKAGES=%HELIX_DIR%
+set NUGET_RESTORE=%HELIX_DIR%\nugetRestore
+echo "Setting HELIX_DIR: %HELIX_DIR%"
+echo Creating nuget restore directory: %NUGET_RESTORE%
+mkdir %NUGET_RESTORE%
+mkdir logs
+
+dir
 
 %DOTNET_ROOT%\dotnet vstest %$target% -lt >discovered.txt
 find /c "Exception thrown" discovered.txt

eng/helix/content/runtests.sh

@@ -31,6 +31,14 @@ export DOTNET_SKIP_FIRST_TIME_EXPERIENCE=1
 
 # Used by SkipOnHelix attribute
 export helix="$helix_queue_name"
+export HELIX_DIR="$DIR"
+export NUGET_FALLBACK_PACKAGES="$DIR"
+export NUGET_RESTORE="$DIR/nugetRestore"
+echo "Creating nugetRestore directory: $NUGET_RESTORE"
+mkdir $NUGET_RESTORE
+mkdir logs
+
+ls -la
 
 RESET="\033[0m"
 RED="\033[0;31m"
@@ -82,6 +90,13 @@ if [ $? -ne 0 ]; then
     done
 fi
 
+# Copy over any local shared fx if found
+if [ -d "Microsoft.AspNetCore.App" ]
+then
+    echo "Found Microsoft.AspNetCore.App directory, copying to $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version."
+    cp -r Microsoft.AspNetCore.App $DOTNET_ROOT/shared/Microsoft.AspNetCore.App/$dotnet_runtime_version
+fi
+
 if [ -e /proc/self/coredump_filter ]; then
   # Include memory in private and shared file-backed mappings in the dump.
   # This ensures that we can see disassembly from our shared libraries when

eng/targets/Helix.props

@@ -21,6 +21,8 @@
     <HelixUseArchive>false</HelixUseArchive>
     <LoggingTestingDisableFileLogging Condition="'$(IsHelixJob)' == 'true'">false</LoggingTestingDisableFileLogging>
     <NodeVersion>10.15.3</NodeVersion>
+    <AppRuntimeVersion>5.0.0-ci</AppRuntimeVersion>
+    <TestDependsOnAspNetRuntime>false</TestDependsOnAspNetRuntime>
   </PropertyGroup>
 
   <ItemGroup Condition="'$(TargetFrameworkIdentifier)' == '.NETFramework'">

eng/targets/Helix.targets

@@ -16,6 +16,20 @@
     <HelixPreCommand Include="call RunPowershell.cmd InstallNode.ps1 $(NodeVersion) %25HELIX_CORRELATION_PAYLOAD%25\node\bin || exit /b 1" />
   </ItemGroup>
 
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(IsWindowsHelixQueue)' == 'true' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg" />
+    <HelixPreCommand Include="call RunPowershell.cmd InstallAppRuntime.ps1 Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg Microsoft.AspNetCore.App netcoreapp5.0 win-x64 || exit /b 1" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(IsWindowsHelixQueue)' == 'false' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg" />
+    <HelixPreCommand Include="./installappruntime.sh Microsoft.AspNetCore.App.Runtime.win-x64.$(AppRuntimeVersion).nupkg Microsoft.AspNetCore.App netcoreapp5.0 win-x64" />
+  </ItemGroup>
+
+  <ItemGroup Condition="'$(IsHelixJob)' == 'true' AND '$(TestDependsOnAspNetRuntime)' == 'true'">
+    <HelixContent Include="$(RepoRoot)artifacts\packages\Release\Shipping\*-ci.nupkg" />
+  </ItemGroup>  
+
   <!-- Item group has to be defined here becasue Helix.props is evaluated before xunit.runner.console.props  -->
   <ItemGroup Condition="$(BuildHelixPayload)">
     <Content Include="@(HelixContent)" />

src/ProjectTemplates/test/BlazorServerTemplateTest.cs

@@ -24,7 +24,8 @@ public BlazorServerTemplateTest(ProjectFactoryFixture projectFactory, BrowserFix
 
         public Project Project { get; private set; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("selenium")]
         public async Task BlazorServerTemplateWorks_NoAuth()
         {
             Project = await ProjectFactory.GetOrCreateProject("blazorservernoauth", Output);
@@ -79,9 +80,10 @@ public async Task BlazorServerTemplateWorks_NoAuth()
             }
         }
 
-        [Theory]
+        [ConditionalTheory]
         [InlineData(true)]
         [InlineData(false)]
+        [SkipOnHelix("ef restore no worky")]
         public async Task BlazorServerTemplateWorks_IndividualAuth(bool useLocalDB)
         {
             Project = await ProjectFactory.GetOrCreateProject("blazorserverindividual" + (useLocalDB ? "uld" : ""), Output);

src/ProjectTemplates/test/ByteOrderMarkTest.cs

@@ -5,6 +5,7 @@
 using System.IO;
 using System.Linq;
 using System.Text;
+using Microsoft.AspNetCore.Testing;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -19,7 +20,8 @@ public ByteOrderMarkTest(ITestOutputHelper output)
             _output = output;
         }
 
-        [Theory]
+        [ConditionalTheory]
+        [SkipOnHelix("missing files")]
         [InlineData("Web.ProjectTemplates")]
         [InlineData("Web.Spa.ProjectTemplates")]
         public void JSAndJSONInAllTemplates_ShouldNotContainBOM(string projectName)
@@ -60,7 +62,8 @@ public void JSAndJSONInAllTemplates_ShouldNotContainBOM(string projectName)
             Assert.False(filesWithBOMCharactersPresent);
         }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("missing files")]
         public void RazorFilesInWebProjects_ShouldContainBOM()
         {
             var projectName = "Web.ProjectTemplates";

src/ProjectTemplates/test/EmptyWebTemplateTest.cs

@@ -3,6 +3,7 @@
 
 using System.Threading.Tasks;
 using Templates.Test.Helpers;
+using Microsoft.AspNetCore.Testing;
 using Xunit;
 using Xunit.Abstractions;
 
@@ -22,7 +23,8 @@ public EmptyWebTemplateTest(ProjectFactoryFixture projectFactory, ITestOutputHel
 
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Cert failures", Queues = "OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task EmptyWebTemplateCSharp()
         {
             await EmtpyTemplateCore(languageOverride: null);

src/ProjectTemplates/test/GrpcTemplateTest.cs

@@ -4,6 +4,7 @@
 using System;
 using System.Runtime.InteropServices;
 using System.Threading.Tasks;
+using Microsoft.AspNetCore.Testing;
 using Templates.Test.Helpers;
 using Xunit;
 using Xunit.Abstractions;
@@ -23,7 +24,8 @@ public GrpcTemplateTest(ProjectFactoryFixture projectFactory, ITestOutputHelper
         public ProjectFactoryFixture ProjectFactory { get; }
         public ITestOutputHelper Output { get; }
 
-        [Fact]
+        [ConditionalFact]
+        [SkipOnHelix("Not supported queues", Queues = "Windows.7.Amd64;Windows.7.Amd64.Open;OSX.1014.Amd64;OSX.1014.Amd64.Open")]
         public async Task GrpcTemplate()
         {
             Project = await ProjectFactory.GetOrCreateProject("grpc", Output);

src/ProjectTemplates/test/Helpers/ProcessEx.cs

@@ -5,6 +5,7 @@
 using System.Collections.Concurrent;
 using System.Collections.Generic;
 using System.Diagnostics;
+using System.IO;
 using System.Linq;
 using System.Reflection;
 using System.Runtime.InteropServices;
@@ -99,6 +100,11 @@ public static ProcessEx Run(ITestOutputHelper output, string workingDirectory, s
             }
 
             startInfo.EnvironmentVariables["NUGET_PACKAGES"] = NUGET_PACKAGES;
+
+            if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("helix")))
+            {
+                startInfo.EnvironmentVariables["NUGET_FALLBACK_PACKAGES"] = Environment.GetEnvironmentVariable("NUGET_FALLBACK_PACKAGES");
+            }
 
             output.WriteLine($"==> {startInfo.FileName} {startInfo.Arguments} [{startInfo.WorkingDirectory}]");
             var proc = Process.Start(startInfo);
@@ -189,11 +195,12 @@ public void WaitForExit(bool assertSuccess, TimeSpan? timeSpan = null)
             }
         }
 
-        private static string GetNugetPackagesRestorePath() =>
-            typeof(ProcessEx).Assembly
+        private static string GetNugetPackagesRestorePath() => (string.IsNullOrEmpty(Environment.GetEnvironmentVariable("NUGET_RESTORE"))) 
+            ? typeof(ProcessEx).Assembly
                 .GetCustomAttributes<AssemblyMetadataAttribute>()
                 .First(attribute => attribute.Key == "TestPackageRestorePath")
-                .Value;
+                .Value
+            : Environment.GetEnvironmentVariable("NUGET_RESTORE");
 
         public void Dispose()
         {