Add Compile to service_slim and pub-sub pattern

.config/dotnet-tools.json

@@ -0,0 +1,12 @@
+{
+  "version": 1,
+  "isRoot": true,
+  "tools": {
+    "fantomas-tool": {
+      "version": "4.6.0-alpha-007",
+      "commands": [
+        "fantomas"
+      ]
+    }
+  }
+}

.vscode/launch.json

@@ -0,0 +1,18 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": ".NET Core Launch (console)",
+            "type": "coreclr",
+            "request": "launch",
+            "program": "${workspaceFolder}/artifacts/bin/fcs-test/Debug/net6.0/fcs-test.dll",
+            "args": [],
+            "cwd": "${workspaceFolder}/fcs/fcs-test",
+            "console": "internalConsole",
+            "stopAtEntry": false
+        }
+    ]
+}

azure-pipelines.yml

@@ -353,6 +353,31 @@ stages:
           - script: .\tests\EndToEndBuildTests\EndToEndBuildTests.cmd -c Release
             displayName: End to end build tests
 
+        # Determinism
+        - job: Determinism_Debug
+          pool:
+            name: NetCore1ESPool-Public
+            demands: ImageOverride -equals Build.Windows.Amd64.VS2022.Pre.Open
+          timeoutInMinutes: 90
+          steps:
+          - checkout: none
+          - script: |
+              @echo on
+              git init
+              git remote add origin "$(Build.Repository.Uri)"
+              git fetch --progress --no-tags --depth=1 origin "$(Build.SourceVersion)"
+              git checkout "$(Build.SourceVersion)"
+            displayName: Shallow checkout
+          - script: .\eng\test-determinism.cmd -configuration Debug
+            displayName: Determinism tests with Debug configuration
+          - task: PublishPipelineArtifact@1
+            displayName: Publish Determinism Logs
+            inputs:
+              targetPath: '$(Build.SourcesDirectory)/artifacts/log/Debug'
+              artifactName: 'Determinism_Debug Attempt $(System.JobAttempt) Logs'
+            continueOnError: true
+            condition: not(succeeded())
+
         # Up-to-date - disabled due to it being flaky
         #- job: UpToDate_Windows
         #  pool:

docs/large-inputs-and-stack-overflows.md

@@ -21,6 +21,13 @@ The compiler performs constant folding for large constants so there are no costs
 
 Many sources of `StackOverflow` exceptions prior to F# 4.7 when processing these kinds of constructs were resolved by processing them on the heap via continuation passing techniques. This avoids filling data on the stack and appears to have negligible effects on overall throughout or memory usage of the compiler.
 
+There are two techniques to deal with this
+
+1. Linearizing processing of specific input shapes, keeping stacks small
+2. Using stack guards to simply temporarily move to a new thread when a certain threshold is reached.
+
+## Linearizing processing if certain inputs
+
 Aside from array expressions, most of the previously-listed inputs are called "linear" expressions. This means that there is a single linear hole in the shape of expressions. For example:
 
 * `expr :: HOLE` (list expressions or other right-linear constructions)
@@ -80,3 +87,31 @@ Some common aspects of this style of programming are:
 
 The previous example is considered incomplete, because arbitrary _combinations_ of `let` and sequential expressions aren't going to be dealt with in a tail-recursive way. The compiler generally tries to do these combinations as well.
 
+## Stack Guards
+
+The `StackGuard` type is used to count synchronous recursive processing and move to a new thread if a limit is reached. Compilation globals are re-installed. Sample:
+
+```fsharp
+let TcStackGuardDepth = StackGuard.GetDepthOption "Tc"
+
+...
+   stackGuard = StackGuard(TcMaxStackGuardDepth)
+
+let rec ....
+
+and TcExpr cenv ty (env: TcEnv) tpenv (expr: SynExpr) =
+
+    // Guard the stack for deeply nested expressions
+    cenv.stackGuard.Guard <| fun () ->
+
+    ...
+
+```
+
+Note stack guarding doesn't result in a tailcall so will appear in recursive stack frames, because a counter must be decremented after the call. This is used systematically for recursive processing of:
+
+* SyntaxTree SynExpr
+* TypedTree Expr
+
+We don't use it for other inputs.
+

eng/Version.Details.xml

@@ -8,9 +8,9 @@
     </Dependency>
   </ProductDependencies>
   <ToolsetDependencies>
-    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21527.1">
+    <Dependency Name="Microsoft.DotNet.Arcade.Sdk" Version="7.0.0-beta.21569.2">
       <Uri>https://github.com/dotnet/arcade</Uri>
-      <Sha>32ff2e3f45ae8fefed72a93ca17f4b01e106c7c9</Sha>
+      <Sha>97463777ee9a8445d4a4c5911ede0f0cd71fa8aa</Sha>
       <SourceBuild RepoName="arcade" ManagedOnly="true" />
     </Dependency>
   </ToolsetDependencies>

eng/common/darc-init.sh

@@ -53,7 +53,7 @@ fi
 function InstallDarcCli {
   local darc_cli_package_name="microsoft.dotnet.darc"
 
-  InitializeDotNetCli
+  InitializeDotNetCli true
   local dotnet_root=$_InitializeDotNetCli
 
   if [ -z "$toolpath" ]; then

eng/common/msbuild.ps1

@@ -6,6 +6,7 @@ Param(
   [switch] $ci,
   [switch] $prepareMachine,
   [switch] $excludePrereleaseVS,
+  [string] $msbuildEngine = $null,
   [Parameter(ValueFromRemainingArguments=$true)][String[]]$extraArgs
 )
 

eng/common/post-build/symbols-validation.ps1

@@ -134,17 +134,17 @@ $CountMissingSymbols = {
         # Save the output and get diagnostic output
         $output = & $dotnetSymbolExe --symbols --modules $WindowsPdbVerificationParam $TargetServerParam $FullPath -o $SymbolsPath --diagnostics | Out-String
 
-        if (Test-Path $PdbPath) {
-          return 'PDB'
+        if ((Test-Path $PdbPath) -and (Test-path $SymbolPath)) {
+          return 'Module and PDB for Module'
         }
-        elseif (Test-Path $NGenPdb) {
-          return 'NGen PDB'
+        elseif ((Test-Path $NGenPdb) -and (Test-Path $PdbPath) -and (Test-Path $SymbolPath)) {
+          return 'Dll, PDB and NGen PDB'
         }
-        elseif (Test-Path $SODbg) {
-          return 'DBG for SO'
+        elseif ((Test-Path $SODbg) -and (Test-Path $SymbolPath)) {
+          return 'So and DBG for SO'
         }  
-        elseif (Test-Path $DylibDwarf) {
-          return 'Dwarf for Dylib'
+        elseif ((Test-Path $DylibDwarf) -and (Test-Path $SymbolPath)) {
+          return 'Dylib and Dwarf for Dylib'
         }  
         elseif (Test-Path $SymbolPath) {
           return 'Module'

eng/common/sdl/execute-all-sdl-tools.ps1

@@ -124,7 +124,7 @@ try {
   Exec-BlockVerbosely {
     & $(Join-Path $PSScriptRoot 'run-sdl.ps1') `
       -GuardianCliLocation $guardianCliLocation `
-      -WorkingDirectory $workingDirectory `
+      -WorkingDirectory $SourceDirectory `
       -UpdateBaseline $UpdateBaseline `
       -GdnFolder $gdnFolder
   }

eng/common/templates/job/execute-sdl.yml

@@ -60,11 +60,7 @@ jobs:
     - name: GuardianPackagesConfigFile
       value: $(Build.SourcesDirectory)\eng\common\sdl\packages.config
   pool:
-    # To extract archives (.tar.gz, .zip), we need access to "tar", added in Windows 10/2019.
-    ${{ if eq(parameters.extractArchiveArtifacts, 'false') }}:
-      name: Hosted VS2017
-    ${{ if ne(parameters.extractArchiveArtifacts, 'false') }}:
-      vmImage: windows-2019
+    vmImage: windows-2019
   steps:
   - checkout: self
     clean: true

eng/common/templates/job/onelocbuild.yml

@@ -4,14 +4,15 @@ parameters:
 
   # Optional: A defined YAML pool - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#pool
   pool:
-    vmImage: vs2017-win2016
+    vmImage: 'windows-2019'
 
   CeapexPat: $(dn-bot-ceapex-package-r) # PAT for the loc AzDO instance https://dev.azure.com/ceapex
   GithubPat: $(BotAccount-dotnet-bot-repo-PAT)
 
   SourcesDirectory: $(Build.SourcesDirectory)
   CreatePr: true
   AutoCompletePr: false
+  ReusePr: true
   UseLfLineEndings: true
   UseCheckedInLocProjectJson: false
   LanguageSet: VS_Main_Languages
@@ -64,6 +65,8 @@ jobs:
         ${{ if eq(parameters.CreatePr, true) }}:
           isAutoCompletePrSelected: ${{ parameters.AutoCompletePr }}
           isUseLfLineEndingsSelected: ${{ parameters.UseLfLineEndings }}
+          ${{ if eq(parameters.RepoType, 'gitHub') }}:
+            isShouldReusePrSelected: ${{ parameters.ReusePr }}
         packageSourceAuth: patAuth
         patVariable: ${{ parameters.CeapexPat }}
         ${{ if eq(parameters.RepoType, 'gitHub') }}:

eng/common/templates/job/source-build.yml

@@ -31,11 +31,6 @@ parameters:
   #   container and pool.
   platform: {}
 
-  # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
-  # source-build builds run in Docker, including the default managed platform.
-  defaultContainerHostPool:
-    vmImage: ubuntu-20.04
-
 jobs:
 - job: ${{ parameters.jobNamePrefix }}_${{ parameters.platform.name }}
   displayName: Source-Build (${{ parameters.platform.name }})
@@ -47,7 +42,15 @@ jobs:
     container: ${{ parameters.platform.container }}
 
   ${{ if eq(parameters.platform.pool, '') }}:
-    pool: ${{ parameters.defaultContainerHostPool }}
+    # The default VM host AzDO pool. This should be capable of running Docker containers: almost all
+    # source-build builds run in Docker, including the default managed platform.
+    pool:
+      ${{ if eq(variables['System.TeamProject'], 'public') }}:
+        name: NetCore1ESPool-Public
+        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64.Open
+      ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+        name: NetCore1ESPool-Internal
+        demands: ImageOverride -equals Build.Ubuntu.1804.Amd64
   ${{ if ne(parameters.platform.pool, '') }}:
     pool: ${{ parameters.platform.pool }}
 

eng/common/templates/job/source-index-stage1.yml

@@ -5,8 +5,6 @@ parameters:
   sourceIndexBuildCommand: powershell -NoLogo -NoProfile -ExecutionPolicy Bypass -Command "eng/common/build.ps1 -restore -build -binarylog -ci"
   preSteps: []
   binlogPath: artifacts/log/Debug/Build.binlog
-  pool:
-    vmImage: vs2017-win2016
   condition: ''
   dependsOn: ''
 
@@ -24,7 +22,13 @@ jobs:
   - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
     - group: source-dot-net stage1 variables
 
-  pool: ${{ parameters.pool }}
+  pool:
+    ${{ if eq(variables['System.TeamProject'], 'public') }}:
+      name: NetCore1ESPool-Public
+      demands: ImageOverride -equals Build.Server.Amd64.VS2019.Open
+    ${{ if eq(variables['System.TeamProject'], 'internal') }}:
+      name: NetCore1ESPool-Internal
+      demands: ImageOverride -equals Build.Server.Amd64.VS2019
   steps:
   - ${{ each preStep in parameters.preSteps }}:
     - ${{ preStep }}

eng/common/templates/jobs/jobs.yml

@@ -83,7 +83,7 @@ jobs:
         - ${{ if eq(parameters.enableSourceBuild, true) }}:
           - Source_Build_Complete
         pool:
-          vmImage: vs2017-win2016
+          vmImage: 'windows-2019'
         runAsPublic: ${{ parameters.runAsPublic }}
         publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
         enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
@@ -96,4 +96,4 @@ jobs:
         dependsOn:
           - Asset_Registry_Publish
         pool:
-          vmImage: vs2017-win2016
+          vmImage: 'windows-2019'

eng/common/tools.ps1

@@ -301,31 +301,44 @@ function InstallDotNet([string] $dotnetRoot,
   if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
   if ($noPath) { $installParameters.NoPath = $True }
 
-  try {
-    & $installScript @installParameters
-  }
-  catch {
-    if ($runtimeSourceFeed -or $runtimeSourceFeedKey) {
-      Write-Host "Failed to install dotnet from public location. Trying from '$runtimeSourceFeed'"
-      if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+  $variations = @()
+  $variations += @($installParameters)
 
-      if ($runtimeSourceFeedKey) {
-        $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
-        $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
-        $installParameters.FeedCredential = $decodedString
-      }
+  $dotnetBuilds = $installParameters.Clone()
+  $dotnetbuilds.AzureFeed = "https://dotnetbuilds.azureedge.net/public"
+  $variations += @($dotnetBuilds)
 
-      try {
-        & $installScript @installParameters
-      }
-      catch {
-        Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from custom location '$runtimeSourceFeed'."
-        ExitWithExitCode 1
-      }
+  if ($runtimeSourceFeed) {
+    $runtimeSource = $installParameters.Clone()
+    $runtimeSource.AzureFeed = $runtimeSourceFeed
+    if ($runtimeSourceFeedKey) {
+      $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+      $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+      $runtimeSource.FeedCredential = $decodedString
+    }
+    $variations += @($runtimeSource)
+  }
+
+  $installSuccess = $false
+  foreach ($variation in $variations) {
+    if ($variation | Get-Member AzureFeed) {
+      $location = $variation.AzureFeed
     } else {
-      Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from public location."
-      ExitWithExitCode 1
+      $location = "public location";
+    }
+    Write-Host "Attempting to install dotnet from $location."
+    try {
+      & $installScript @variation
+      $installSuccess = $true
+      break
     }
+    catch {
+      Write-Host "Failed to install dotnet from $location."
+    }
+  }
+  if (-not $installSuccess) {
+    Write-PipelineTelemetryError -Category 'InitializeToolset' -Message "Failed to install dotnet from any of the specified locations."
+    ExitWithExitCode 1
   }
 }
 
@@ -887,7 +900,7 @@ function Try-LogClientIpAddress()
     Write-Host "Attempting to log this client's IP for Azure Package feed telemetry purposes"
     try
     {
-        $result = Invoke-WebRequest -Uri "http://co1.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
+        $result = Invoke-WebRequest -Uri "http://co1r5a.msedge.net/fdv2/diagnostics.aspx" -UseBasicParsing
         $lines = $result.Content.Split([Environment]::NewLine) 
         $socketIp = $lines | Select-String -Pattern "^Socket IP:.*"
         Write-Host $socketIp