Adds experimental package for Minimal APIs template

- Update SDK, Arcade and ./eng bits
- Add initial template project for 'dotnet new api'
- Contributes to dotnet/aspnetcore#34010

Author: DamianEdwards

eng/common/SetupNugetSources.ps1

@@ -158,4 +158,10 @@ if ($dotnet5Source -ne $null) {
     AddPackageSource -Sources $sources -SourceName "dotnet5-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet5-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
 }
 
+$dotnet6Source = $sources.SelectSingleNode("add[@key='dotnet6']")
+if ($dotnet6Source -ne $null) {
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2" -Creds $creds -Username $userName -Password $Password
+    AddPackageSource -Sources $sources -SourceName "dotnet6-internal-transport" -SourceEndPoint "https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2" -Creds $creds -Username $userName -Password $Password
+}
+
 $doc.Save($filename)

eng/common/SetupNugetSources.sh

@@ -129,6 +129,30 @@ if [ "$?" == "0" ]; then
     PackageSources+=('dotnet5-internal-transport')
 fi
 
+# Ensure dotnet6-internal and dotnet6-internal-transport are in the packageSources if the public dotnet6 feeds are present
+grep -i "<add key=\"dotnet6\"" $ConfigFile
+if [ "$?" == "0" ]; then
+    grep -i "<add key=\"dotnet6-internal\"" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal')
+
+    grep -i "<add key=\"dotnet6-internal-transport\">" $ConfigFile
+    if [ "$?" != "0" ]; then
+        echo "Adding dotnet6-internal-transport to the packageSources."
+        PackageSourcesNodeFooter="</packageSources>"
+        PackageSourceTemplate="${TB}<add key=\"dotnet6-internal-transport\" value=\"https://pkgs.dev.azure.com/dnceng/internal/_packaging/dotnet6-internal-transport/nuget/v2\" />"
+
+        sed -i.bak "s|$PackageSourcesNodeFooter|$PackageSourceTemplate${NL}$PackageSourcesNodeFooter|" $ConfigFile
+    fi
+    PackageSources+=('dotnet6-internal-transport')
+fi
+
 # I want things split line by line
 PrevIFS=$IFS
 IFS=$'\n'

eng/common/build.ps1

@@ -25,6 +25,7 @@ Param(
   [switch] $prepareMachine,
   [string] $runtimeSourceFeed = '',
   [string] $runtimeSourceFeedKey = '',
+  [switch] $excludePrereleaseVS,
   [switch] $help,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$properties
 )
@@ -65,6 +66,7 @@ function Print-Usage() {
   Write-Host "  -prepareMachine         Prepare machine for CI run, clean up processes after build"
   Write-Host "  -warnAsError <value>    Sets warnaserror msbuild parameter ('true' or 'false')"
   Write-Host "  -msbuildEngine <value>  Msbuild engine to use to run build ('dotnet', 'vs', or unspecified)."
+  Write-Host "  -excludePrereleaseVS    Set to exclude build engines in prerelease versions of Visual Studio"
   Write-Host ""
 
   Write-Host "Command line arguments not listed above are passed thru to msbuild."

eng/common/cross/build-android-rootfs.sh

@@ -106,7 +106,6 @@ __AndroidPackages+=" libandroid-glob"
 __AndroidPackages+=" liblzma"
 __AndroidPackages+=" krb5"
 __AndroidPackages+=" openssl"
-__AndroidPackages+=" openldap"
 
 for path in $(wget -qO- http://termux.net/dists/stable/main/binary-$__AndroidArch/Packages |\
     grep -A15 "Package: \(${__AndroidPackages// /\\|}\)" | grep -v "static\|tool" | grep Filename); do

eng/common/cross/build-rootfs.sh

@@ -55,13 +55,11 @@ __UbuntuPackages+=" libcurl4-openssl-dev"
 __UbuntuPackages+=" libkrb5-dev"
 __UbuntuPackages+=" libssl-dev"
 __UbuntuPackages+=" zlib1g-dev"
-__UbuntuPackages+=" libldap2-dev"
 
 __AlpinePackages+=" curl-dev"
 __AlpinePackages+=" krb5-dev"
 __AlpinePackages+=" openssl-dev"
 __AlpinePackages+=" zlib-dev"
-__AlpinePackages+=" openldap-dev"
 
 __FreeBSDBase="12.1-RELEASE"
 __FreeBSDPkg="1.12.0"
@@ -70,13 +68,15 @@ __FreeBSDPackages+=" icu"
 __FreeBSDPackages+=" libinotify"
 __FreeBSDPackages+=" lttng-ust"
 __FreeBSDPackages+=" krb5"
-__FreeBSDPackages+=" libslapi-2.4"
 
 __IllumosPackages="icu-64.2nb2"
 __IllumosPackages+=" mit-krb5-1.16.2nb4"
 __IllumosPackages+=" openssl-1.1.1e"
 __IllumosPackages+=" zlib-1.2.11"
-__IllumosPackages+=" openldap-client-2.4.49"
+
+# ML.NET dependencies
+__UbuntuPackages+=" libomp5"
+__UbuntuPackages+=" libomp-dev"
 
 __UseMirror=0
 

eng/common/dotnet-install.sh

@@ -70,7 +70,7 @@ case $cpuname in
     ;;
 esac
 
-dotnetRoot="$repo_root/.dotnet"
+dotnetRoot="${repo_root}.dotnet"
 if [[ $architecture != "" ]] && [[ $architecture != $buildarch ]]; then
   dotnetRoot="$dotnetRoot/$architecture"
 fi

eng/common/generate-locproject.ps1

@@ -14,7 +14,7 @@ $ErrorActionPreference = "Stop"
 
 Import-Module -Name (Join-Path $PSScriptRoot 'native\CommonLibrary.psm1')
 
-$exclusionsFilePath = "$SourcesDirectory\Localize\LocExclusions.json"
+$exclusionsFilePath = "$SourcesDirectory\eng\Localize\LocExclusions.json"
 $exclusions = @{ Exclusions = @() }
 if (Test-Path -Path $exclusionsFilePath)
 {
@@ -25,8 +25,15 @@ Push-Location "$SourcesDirectory" # push location for Resolve-Path -Relative to
 
 # Template files
 $jsonFiles = @()
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\en\..+\.json" } # .NET templating pattern
-$jsonFiles += Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
+$jsonTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "\.template\.config\\localize\\.+\.en\.json" } # .NET templating pattern
+$jsonTemplateFiles | ForEach-Object {
+    $null = $_.Name -Match "(.+)\.[\w-]+\.json" # matches '[filename].[langcode].json
+    
+    $destinationFile = "$($_.Directory.FullName)\$($Matches.1).json"
+    $jsonFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
+}
+
+$jsonWinformsTemplateFiles = Get-ChildItem -Recurse -Path "$SourcesDirectory" | Where-Object { $_.FullName -Match "en\\strings\.json" } # current winforms pattern
 
 $xlfFiles = @()
 
@@ -44,7 +51,7 @@ $langXlfFiles | ForEach-Object {
     $xlfFiles += Copy-Item "$($_.FullName)" -Destination $destinationFile -PassThru
 }
 
-$locFiles = $jsonFiles + $xlfFiles
+$locFiles = $jsonFiles + $jsonWinformsTemplateFiles + $xlfFiles
 
 $locJson = @{
     Projects = @(
@@ -92,14 +99,14 @@ Write-Host "LocProject.json generated:`n`n$json`n`n"
 Pop-Location
 
 if (!$UseCheckedInLocProjectJson) {
-    New-Item "$SourcesDirectory\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
-    Set-Content "$SourcesDirectory\Localize\LocProject.json" $json
+    New-Item "$SourcesDirectory\eng\Localize\LocProject.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\eng\Localize\LocProject.json" $json
 }
 else {
-    New-Item "$SourcesDirectory\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
-    Set-Content "$SourcesDirectory\Localize\LocProject-generated.json" $json
+    New-Item "$SourcesDirectory\eng\Localize\LocProject-generated.json" -Force # Need this to make sure the Localize directory is created
+    Set-Content "$SourcesDirectory\eng\Localize\LocProject-generated.json" $json
 
-    if ((Get-FileHash "$SourcesDirectory\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\Localize\LocProject.json").Hash) {
+    if ((Get-FileHash "$SourcesDirectory\eng\Localize\LocProject-generated.json").Hash -ne (Get-FileHash "$SourcesDirectory\eng\Localize\LocProject.json").Hash) {
         Write-PipelineTelemetryError -Category "OneLocBuild" -Message "Existing LocProject.json differs from generated LocProject.json. Download LocProject-generated.json and compare them."
 
         exit 1

eng/common/internal-feed-operations.ps1

@@ -45,11 +45,11 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  $nugetConfigPath = "$RepoRoot\NuGet.config"
+  $nugetConfigPath = Join-Path $RepoRoot "NuGet.config"
 
   if (-Not (Test-Path -Path $nugetConfigPath)) {
     Write-PipelineTelemetryError -Category 'Build' -Message 'NuGet.config file not found in repo root!'
-    ExitWithExitCode 1  
+    ExitWithExitCode 1
   }
 
   $endpoints = New-Object System.Collections.ArrayList
@@ -85,7 +85,7 @@ function SetupCredProvider {
 
 #Workaround for https://github.com/microsoft/msbuild/issues/4430
 function InstallDotNetSdkAndRestoreArcade {
-  $dotnetTempDir = "$RepoRoot\dotnet"
+  $dotnetTempDir = Join-Path $RepoRoot "dotnet"
   $dotnetSdkVersion="2.1.507" # After experimentation we know this version works when restoring the SDK (compared to 3.0.*)
   $dotnet = "$dotnetTempDir\dotnet.exe"
   $restoreProjPath = "$PSScriptRoot\restore.proj"

eng/common/internal-feed-operations.sh

@@ -39,7 +39,7 @@ function SetupCredProvider {
   # Then, we set the 'VSS_NUGET_EXTERNAL_FEED_ENDPOINTS' environment variable to restore from the stable 
   # feeds successfully
 
-  local nugetConfigPath="$repo_root/NuGet.config"
+  local nugetConfigPath="{$repo_root}NuGet.config"
 
   if [ ! "$nugetConfigPath" ]; then
     Write-PipelineTelemetryError -category 'Build' "NuGet.config file not found in repo's root!"

eng/common/msbuild.ps1

@@ -5,6 +5,7 @@ Param(
   [bool] $nodeReuse = $true,
   [switch] $ci,
   [switch] $prepareMachine,
+  [switch] $excludePrereleaseVS,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
 )
 

eng/common/post-build/sourcelink-validation.ps1

@@ -14,7 +14,9 @@ param(
 $global:RepoFiles = @{}
 
 # Maximum number of jobs to run in parallel
-$MaxParallelJobs = 6
+$MaxParallelJobs = 16
+
+$MaxRetries = 5
 
 # Wait time between check for system load
 $SecondsBetweenLoadChecks = 10
@@ -29,7 +31,10 @@ $ValidatePackage = {
   # Ensure input file exist
   if (!(Test-Path $PackagePath)) {
     Write-Host "Input file does not exist: $PackagePath"
-    return 1
+    return [pscustomobject]@{
+      result = 1
+      packagePath = $PackagePath
+    }
   }
 
   # Extensions for which we'll look for SourceLink information
@@ -59,7 +64,10 @@ $ValidatePackage = {
 
           # We ignore resource DLLs
           if ($FileName.EndsWith('.resources.dll')) {
-            return
+            return [pscustomobject]@{
+              result = 0
+              packagePath = $PackagePath
+            }
           }
 
           [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true)
@@ -91,36 +99,49 @@ $ValidatePackage = {
                     $Status = 200
                     $Cache = $using:RepoFiles
 
-                    if ( !($Cache.ContainsKey($FilePath)) ) {
-                      try {
-                        $Uri = $Link -as [System.URI]
-                      
-                        # Only GitHub links are valid
-                        if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
-                          $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                    $totalRetries = 0
+
+                    while ($totalRetries -lt $using:MaxRetries) {
+                      if ( !($Cache.ContainsKey($FilePath)) ) {
+                        try {
+                          $Uri = $Link -as [System.URI]
+                        
+                          # Only GitHub links are valid
+                          if ($Uri.AbsoluteURI -ne $null -and ($Uri.Host -match 'github' -or $Uri.Host -match 'githubusercontent')) {
+                            $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode
+                          }
+                          else {
+                            # If it's not a github link, we want to break out of the loop and not retry.
+                            $Status = 0
+                            $totalRetries = $using:MaxRetries
+                          }
                         }
-                        else {
+                        catch {
+                          Write-Host $_
                           $Status = 0
                         }
                       }
-                      catch {
-                        write-host $_
-                        $Status = 0
-                      }
-                    }
 
-                    if ($Status -ne 200) {
-                      if ($NumFailedLinks -eq 0) {
-                        if ($FailedFiles.Value -eq 0) {
-                          Write-Host
+                      if ($Status -ne 200) {
+                        $totalRetries++
+                        
+                        if ($totalRetries -ge $using:MaxRetries) {
+                          if ($NumFailedLinks -eq 0) {
+                            if ($FailedFiles.Value -eq 0) {
+                              Write-Host
+                            }
+  
+                            Write-Host "`tFile $RealPath has broken links:"
+                          }
+  
+                          Write-Host "`t`tFailed to retrieve $Link"
+  
+                          $NumFailedLinks++
                         }
-
-                        Write-Host "`tFile $RealPath has broken links:"
                       }
-
-                      Write-Host "`t`tFailed to retrieve $Link"
-
-                      $NumFailedLinks++
+                      else {
+                        break
+                      }
                     }
                   }
               }
@@ -136,7 +157,7 @@ $ValidatePackage = {
         }
   }
   catch {
-  
+    Write-Host $_
   }
   finally {
     $zip.Dispose() 
@@ -220,6 +241,7 @@ function ValidateSourceLinkLinks {
   # Process each NuGet package in parallel
   Get-ChildItem "$InputPath\*.symbols.nupkg" |
     ForEach-Object {
+      Write-Host "Starting $($_.FullName)"
       Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
       $NumJobs = @(Get-Job -State 'Running').Count
 
@@ -267,6 +289,10 @@ function InstallSourcelinkCli {
 try {
   InstallSourcelinkCli
 
+  foreach ($Job in @(Get-Job)) {
+    Remove-Job -Id $Job.Id
+  }
+
   ValidateSourceLinkLinks 
 }
 catch {