diff --git a/.vsts.pipelines/builds/ci-internal.yml b/.vsts.pipelines/builds/ci-internal.yml
index ccf83c8f48..3c508d8314 100644
--- a/.vsts.pipelines/builds/ci-internal.yml
+++ b/.vsts.pipelines/builds/ci-internal.yml
@@ -3,3 +3,21 @@ jobs:
parameters:
windowsPoolName: NetCoreInternal-Pool
windowsQueueName: buildpool.windows.10.amd64.vs2017
+
+- template: ../jobs/collect-artifacts.yml
+ parameters:
+ dependsOn:
+ - centos71
+ - debian9
+ - fedora30
+ - ubuntu1804
+ pool:
+ vmImage: vs2017-win2016
+
+- template: /eng/common/templates/job/publish-build-assets.yml
+ parameters:
+ dependsOn:
+ - Collect_Artifacts
+ enablePublishBuildArtifacts: true
+ pool:
+ vmImage: vs2017-win2016
diff --git a/.vsts.pipelines/builds/matrix.yml b/.vsts.pipelines/builds/matrix.yml
index a896a13189..dadd43d9df 100644
--- a/.vsts.pipelines/builds/matrix.yml
+++ b/.vsts.pipelines/builds/matrix.yml
@@ -28,6 +28,7 @@ jobs:
matrix:
Production: {}
Online: { type: Online }
+ Offline: { type: Offline }
- template: ../jobs/ci-linux.yml
parameters:
@@ -43,6 +44,8 @@ jobs:
parameters:
job: ubuntu1804
imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-f90bc20-20180320154721
+ matrix:
+ Offline: { type: Offline }
- template: ../jobs/ci-local.yml
parameters:
diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml
index 73e1779a35..40db5b135c 100644
--- a/.vsts.pipelines/jobs/ci-linux.yml
+++ b/.vsts.pipelines/jobs/ci-linux.yml
@@ -74,6 +74,18 @@ jobs:
displayName: Build source-build
timeoutInMinutes: 150
+ - script: |
+ set -ex
+ df -h
+ $(docker.run) $(docker.drop.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \
+ --collect-artifacts \
+ /p:ArtifactOutput=/drop/artifacts/source-build/ \
+ /bl
+ du -h $(rootDirectory) | sort -h | tail -n 50
+ displayName: Copy source-build artifacts
+ condition: always()
+ continueOnError: true
+
# Generate prebuilt burndown data
- script: |
set -ex
@@ -137,7 +149,8 @@ jobs:
-path './bin/aspnet-debug/*' -o \
-iname '*.binlog' -o \
-iname '*.log' \) \
- -exec cp {} --parents /logs/source-build/logs \;"
+ -exec cp {} --parents /logs/source-build/logs \;
+ find . > /logs/source-build/logs/file-listing.txt"
du -h $(rootDirectory) | sort -h | tail -n 50
displayName: Copy source-build production build logs
condition: always()
@@ -197,6 +210,18 @@ jobs:
timeoutInMinutes: 120
condition: and(succeeded(), eq(variables['sb.tarball'], true))
+ - script: |
+ set -ex
+ df -h
+ $(docker.run) $(docker.drop.map) $(docker.tb.map) $(docker.tb.work) $(imageName) "$(tarballName)/build.sh" \
+ --collect-artifacts \
+ /p:ArtifactOutput=/drop/artifacts/tarball/ \
+ /bl
+ du -h $(rootDirectory) | sort -h | tail -n 50
+ displayName: Copy tarball artifacts
+ condition: eq(variables['sb.tarball'], true)
+ continueOnError: true
+
# Run smoke tests.
- script: |
set -ex
@@ -228,7 +253,8 @@ jobs:
-path './bin/aspnet-debug/*' -o \
-iname '*.binlog' -o \
-iname '*.log' \) \
- -exec cp {} --parents /logs/tarball/logs \;"
+ -exec cp {} --parents /logs/tarball/logs \;
+ find . > /logs/tarball/logs/file-listing.txt"
du -h $(rootDirectory) | sort -h | tail -n 50
displayName: Copy tarball logs
condition: eq(variables['sb.tarball'], true)
@@ -252,6 +278,7 @@ jobs:
# Copy artifacts to staging - Copy to VSTS owned folder is done outside of docker so copied files
# have correct ownership so VSTS can clean them up later.
- task: CopyFiles@2
+ displayName: Copy artifacts from Docker-owned staging directory to AzDO-agent-owned artifacts directory
condition: always()
continueOnError: true
inputs:
@@ -275,6 +302,14 @@ jobs:
PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/tarball
ArtifactName: Tarball $(artifactName)
ArtifactType: Container
+ - task: PublishBuildArtifacts@1
+ displayName: Publish build artifacts
+ condition: always()
+ continueOnError: true
+ inputs:
+ PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/artifacts
+ ArtifactName: Artifacts $(artifactName)
+ ArtifactType: Container
# Publish prebuilt report data to blob storage.
- template: ../steps/publish-prebuilt-data-sh.yml
diff --git a/.vsts.pipelines/jobs/collect-artifacts.yml b/.vsts.pipelines/jobs/collect-artifacts.yml
new file mode 100644
index 0000000000..06e9b57ee8
--- /dev/null
+++ b/.vsts.pipelines/jobs/collect-artifacts.yml
@@ -0,0 +1,145 @@
+jobs:
+- job: Collect_Artifacts
+
+ dependsOn: ${{ parameters.dependsOn }}
+
+ displayName: Collect Artifacts
+
+ pool: ${{ parameters.pool }}
+
+ steps:
+ - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download CentOS artifacts
+ inputs:
+ artifactName: Tarball centos71 Offline
+ downloadPath: '$(Build.StagingDirectory)/centos'
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)/centos'
+ script: |
+ rm */tarball*.tar.gz
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Debian artifacts
+ inputs:
+ artifactName: Tarball debian9 Offline
+ downloadPath: '$(Build.StagingDirectory)/debian'
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)/debian'
+ script: |
+ rm */tarball*.tar.gz
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Fedora artifacts
+ inputs:
+ artifactName: Tarball fedora30 Offline
+ downloadPath: '$(Build.StagingDirectory)/fedora'
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)/fedora'
+ script: |
+ rm */tarball*.tar.gz
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Ubuntu artifacts
+ inputs:
+ artifactName: Tarball ubuntu1804 Offline
+ downloadPath: '$(Build.StagingDirectory)/ubuntu'
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)/ubuntu'
+ script: |
+ rm */tarball*.tar.gz
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download portable artifacts
+ inputs:
+ artifactName: Tarball centos71 Offline Portable
+ downloadPath: '$(Build.StagingDirectory)/portable'
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)/portable'
+ script: |
+ rm */tarball*.tar.gz
+
+ - task: Bash@3
+ inputs:
+ targetType: 'inline'
+ workingDirectory: '$(Build.StagingDirectory)'
+ script: |
+ version=`echo centos/*/Private.SourceBuilt.Artifacts*.tar.gz | sed 's/\.tar\.gz$//' | sed 's/^centos\/.*\/Private\.SourceBuilt\.Artifacts\.//'`
+ echo "Artifacts version: $version"
+ mkdir collected
+ cd collected
+ # do CentOS last to prefer those versions of any duplicates
+ for os in portable debian fedora ubuntu centos; do
+ # DownloadArtifact creates another directory - use '*' to not deal with this
+ tar xzf ../$os/*/Private.SourceBuilt.Artifacts*.tar.gz
+ rm -rf ../$os
+ done
+ echo $(Build.SourceVersion) > .version
+ echo $(Build.BuildNumber) > .buildnumber
+ echo $(Build.BuildId) > .buildid
+ tar czf ../Private.SourceBuilt.Artifacts.$version.tar.gz * .version .buildnumber .buildid
+ rm -rf *
+ mv ../Private.SourceBuilt.Artifacts.*.tar.gz .
+ displayName: Collect RID-specific packages into one tarball
+ condition: succeeded()
+
+ - task: PowerShell@2
+ inputs:
+ targetType: 'filePath'
+ filePath: $(Build.SourcesDirectory)\publish.ps1
+ arguments: >-
+ /p:OfficialBuildId=$(Build.BuildNumber)
+ /p:AzureAccountName=$(publish.blobStorage.account)
+ /p:ContainerName=$(publish.blobStorage.container)
+ "/p:AzureAccessToken=$(publish.blobStorage.accessToken)"
+ "/p:ManifestName=$(BUILD.REPOSITORY.NAME)"
+ "/p:ManifestBuildId=$(BUILD.BUILDID)"
+ "/p:ManifestBranch=$(BUILD.SOURCEBRANCHNAME)"
+ /p:ManifestCommit=$(Build.SourceVersion)
+ /p:DotNetPublishToBlobFeed=true
+ workingDirectory: '$(Build.SourcesDirectory)'
+ displayName: Publish collected artifacts
+ condition: succeeded()
+
+ - task: PublishBuildArtifacts@1
+ displayName: Uploaded collected artifacts
+ condition: always()
+ continueOnError: true
+ inputs:
+ PathToPublish: $(Build.SourcesDirectory)/artifacts/tarball/
+ ArtifactName: Collected artifacts
+ ArtifactType: Container
+
+ - task: PublishBuildArtifacts@1
+ displayName: Uploaded asset manifests
+ condition: always()
+ continueOnError: true
+ inputs:
+ PathToPublish: $(Build.SourcesDirectory)/artifacts/asset-manifests/
+ ArtifactName: AssetManifests
+ ArtifactType: Container
+
+ - task: PublishBuildArtifacts@1
+ displayName: Upload publish logs
+ condition: always()
+ continueOnError: true
+ inputs:
+ PathToPublish: $(Build.SourcesDirectory)/artifacts/log/
+ ArtifactName: Publishing logs
+ ArtifactType: Container
+
diff --git a/build.proj b/build.proj
index 5a80d86843..81a6598f58 100644
--- a/build.proj
+++ b/build.proj
@@ -245,6 +245,10 @@
Overwrite="true" />
+
+
+
+
diff --git a/build.sh b/build.sh
index d582864d41..850873b46e 100755
--- a/build.sh
+++ b/build.sh
@@ -53,6 +53,9 @@ for arg do
(--generate-prebuilt-data) set -- "$@" "/t:GeneratePrebuiltBurndownData"
alternateTarget=true
;;
+ (--collect-artifacts) set -- "$@" "/t:CopyArtifactsToOutputFolder"
+ alternateTarget=true
+ ;;
(*) set -- "$@" "$arg" ;;
esac
done
diff --git a/eng/Tools.props b/eng/Tools.props
new file mode 100644
index 0000000000..1fbf39ed5e
--- /dev/null
+++ b/eng/Tools.props
@@ -0,0 +1,2 @@
+
+
diff --git a/eng/Versions.props b/eng/Versions.props
index 574d0c7c31..4938be4b8b 100644
--- a/eng/Versions.props
+++ b/eng/Versions.props
@@ -2,8 +2,11 @@
- 0.1.0
- alpha.1
+ 3
+ 1
+ $(BUILD_BUILDNUMBER)
+ $(VersionSuffix)+$(BUILD_SOURCEVERSION)
+ $(VersionSuffix)+$(GitCommitHash)
diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj
index 5d55e312b0..311e2bbe0f 100644
--- a/eng/common/PublishToSymbolServers.proj
+++ b/eng/common/PublishToSymbolServers.proj
@@ -37,6 +37,8 @@
3650
true
+ true
+ true
false
@@ -56,7 +58,7 @@
DryRun="false"
ConvertPortablePdbsToWindowsPdbs="false"
PdbConversionTreatAsWarning=""
- Condition="$(PublishToSymbolServer)"/>
+ Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/>
$(WorkItemDirectory)
- $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"
+ $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"
+ $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)"
+ $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand)
4:00
+
$(WorkItemDirectory)
- $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)"
+ $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)"
+ $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)"
+ $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults)
4:00
diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1
index 4a6706b638..ec41965fc8 100644
--- a/eng/common/performance/performance-setup.ps1
+++ b/eng/common/performance/performance-setup.ps1
@@ -1,6 +1,7 @@
Param(
[string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY,
[string] $CoreRootDirectory,
+ [string] $BaselineCoreRootDirectory,
[string] $Architecture="x64",
[string] $Framework="netcoreapp5.0",
[string] $CompilationMode="Tiered",
@@ -12,11 +13,13 @@ Param(
[string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj",
[string] $Kind="micro",
[switch] $Internal,
+ [switch] $Compare,
[string] $Configurations="CompilationMode=$CompilationMode"
)
-$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance")
+$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance")
$UseCoreRun = ($CoreRootDirectory -ne [string]::Empty)
+$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty)
$PayloadDirectory = (Join-Path $SourceDirectory "Payload")
$PerformanceDirectory = (Join-Path $PayloadDirectory "performance")
@@ -29,7 +32,13 @@ $HelixSourcePrefix = "pr"
$Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open"
if ($Framework.StartsWith("netcoreapp")) {
- $Queue = "Windows.10.Amd64.ClientRS4.Open"
+ $Queue = "Windows.10.Amd64.ClientRS5.Open"
+}
+
+if ($Compare) {
+ $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open"
+ $PerfLabArguments = ""
+ $ExtraBenchmarkDotNetArguments = ""
}
if ($Internal) {
@@ -56,6 +65,10 @@ if ($UseCoreRun) {
$NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root")
Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot
}
+if ($UseBaselineCoreRun) {
+ $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root")
+ Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot
+}
$DocsDir = (Join-Path $PerformanceDirectory "docs")
robocopy $DocsDir $WorkItemDirectory
@@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria
Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false
Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false
+Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false
# Helix Arguments
Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false
diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh
index 76126b1f86..2f2092166e 100755
--- a/eng/common/performance/performance-setup.sh
+++ b/eng/common/performance/performance-setup.sh
@@ -2,6 +2,7 @@
source_directory=$BUILD_SOURCESDIRECTORY
core_root_directory=
+baseline_core_root_directory=
architecture=x64
framework=netcoreapp5.0
compilation_mode=tiered
@@ -10,12 +11,14 @@ branch=$BUILD_SOURCEBRANCH
commit_sha=$BUILD_SOURCEVERSION
build_number=$BUILD_BUILDNUMBER
internal=false
+compare=false
kind="micro"
run_categories="coreclr corefx"
csproj="src\benchmarks\micro\MicroBenchmarks.csproj"
configurations=
run_from_perf_repo=false
use_core_run=true
+use_baseline_core_run=true
while (($# > 0)); do
lowerI="$(echo $1 | awk '{print tolower($0)}')"
@@ -28,6 +31,10 @@ while (($# > 0)); do
core_root_directory=$2
shift 2
;;
+ --baselinecorerootdirectory)
+ baseline_core_root_directory=$2
+ shift 2
+ ;;
--architecture)
architecture=$2
shift 2
@@ -72,6 +79,10 @@ while (($# > 0)); do
internal=true
shift 1
;;
+ --compare)
+ compare=true
+ shift 1
+ ;;
--configurations)
configurations=$2
shift 2
@@ -102,7 +113,7 @@ while (($# > 0)); do
esac
done
-if [[ "$repository" == "dotnet/performance" ]]; then
+if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then
run_from_perf_repo=true
fi
@@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then
use_core_run=false
fi
+if [ -z "$baseline_core_root_directory" ]; then
+ use_baseline_core_run=false
+fi
+
payload_directory=$source_directory/Payload
performance_directory=$payload_directory/performance
workitem_directory=$source_directory/workitem
@@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open
creator=$BUILD_DEFINITIONNAME
helix_source_prefix="pr"
+if [[ "$compare" == true ]]; then
+ extra_benchmark_dotnet_arguments=
+ perflab_arguments=
+
+ # No open queues for arm64
+ if [[ "$architecture" = "arm64" ]]; then
+ echo "Compare not available for arm64"
+ exit 1
+ fi
+
+ queue=Ubuntu.1804.Amd64.Tiger.Perf.Open
+fi
+
if [[ "$internal" == true ]]; then
perflab_arguments="--upload-to-perflab-container"
helix_source_prefix="official"
@@ -156,6 +184,11 @@ if [[ "$use_core_run" = true ]]; then
mv $core_root_directory $new_core_root
fi
+if [[ "$use_baseline_core_run" = true ]]; then
+ new_baseline_core_root=$payload_directory/Baseline_Core_Root
+ mv $baseline_core_root_directory $new_baseline_core_root
+fi
+
ci=true
_script_dir=$(pwd)/eng/common
@@ -163,6 +196,7 @@ _script_dir=$(pwd)/eng/common
# Make sure all of our variables are available for future steps
Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false
+Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false
Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false
Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false
Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false
@@ -178,4 +212,5 @@ Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" -
Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false
Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false
Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false
-Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
\ No newline at end of file
+Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false
+Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false
diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1
index 551ae113f8..aa9a025c0b 100644
--- a/eng/common/post-build/post-build-utils.ps1
+++ b/eng/common/post-build/post-build-utils.ps1
@@ -8,6 +8,7 @@ Set-StrictMode -Version 2.0
# scripts don't necessarily execute in the same agent that run the
# build.ps1/sh script this variable isn't automatically set.
$ci = $true
+$disableConfigureToolsetImport = "true"
. $PSScriptRoot\..\tools.ps1
function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") {
diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1
index 41e01ae6e6..bbfdacca13 100644
--- a/eng/common/post-build/sourcelink-validation.ps1
+++ b/eng/common/post-build/sourcelink-validation.ps1
@@ -1,8 +1,8 @@
param(
[Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored
[Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation
- [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
- [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages
+ [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade
+ [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages
[Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use
)
@@ -13,6 +13,12 @@ param(
# all files present in the repo at a specific commit point.
$global:RepoFiles = @{}
+# Maximum number of jobs to run in parallel
+$MaxParallelJobs = 6
+
+# Wait time between check for system load
+$SecondsBetweenLoadChecks = 10
+
$ValidatePackage = {
param(
[string] $PackagePath # Full path to a Symbols.NuGet package
@@ -22,8 +28,8 @@ $ValidatePackage = {
# Ensure input file exist
if (!(Test-Path $PackagePath)) {
- Write-PipelineTaskError "Input file does not exist: $PackagePath"
- ExitWithExitCode 1
+ Write-Host "Input file does not exist: $PackagePath"
+ return 1
}
# Extensions for which we'll look for SourceLink information
@@ -38,7 +44,7 @@ $ValidatePackage = {
Add-Type -AssemblyName System.IO.Compression.FileSystem
- [System.IO.Directory]::CreateDirectory($ExtractPath);
+ [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null
try {
$zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath)
@@ -138,16 +144,18 @@ $ValidatePackage = {
if ($FailedFiles -eq 0) {
Write-Host "Passed."
+ return 0
}
else {
- Write-PipelineTaskError "$PackagePath has broken SourceLink links."
+ Write-Host "$PackagePath has broken SourceLink links."
+ return 1
}
}
function ValidateSourceLinkLinks {
- if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
+ if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) {
if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) {
- Write-PipelineTaskError "GHRepoName should be in the format / or -"
+ Write-PipelineTaskError "GHRepoName should be in the format / or -. '$GHRepoName'"
ExitWithExitCode 1
}
else {
@@ -155,30 +163,33 @@ function ValidateSourceLinkLinks {
}
}
- if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
- Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string"
+ if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) {
+ Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'"
ExitWithExitCode 1
}
- $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
- $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
+ if ($GHRepoName -ne "" -and $GHCommit -ne "") {
+ $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1")
+ $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript")
- try {
- # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
- $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
+ try {
+ # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash
+ $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree
- foreach ($file in $Data) {
- $Extension = [System.IO.Path]::GetExtension($file.path)
+ foreach ($file in $Data) {
+ $Extension = [System.IO.Path]::GetExtension($file.path)
- if ($CodeExtensions.Contains($Extension)) {
- $RepoFiles[$file.path] = 1
+ if ($CodeExtensions.Contains($Extension)) {
+ $RepoFiles[$file.path] = 1
+ }
}
}
+ catch {
+ Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching."
+ }
}
- catch {
- Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL"
- Write-Host $_
- ExitWithExitCode 1
+ elseif ($GHRepoName -ne "" -or $GHCommit -ne "") {
+ Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed."
}
if (Test-Path $ExtractPath) {
@@ -186,14 +197,33 @@ function ValidateSourceLinkLinks {
}
# Process each NuGet package in parallel
- $Jobs = @()
Get-ChildItem "$InputPath\*.symbols.nupkg" |
ForEach-Object {
- $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName
+ Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null
+ $NumJobs = @(Get-Job -State 'Running').Count
+
+ while ($NumJobs -ge $MaxParallelJobs) {
+ Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again."
+ sleep $SecondsBetweenLoadChecks
+ $NumJobs = @(Get-Job -State 'Running').Count
+ }
+
+ foreach ($Job in @(Get-Job -State 'Completed')) {
+ Receive-Job -Id $Job.Id
+ Remove-Job -Id $Job.Id
+ }
}
- foreach ($Job in $Jobs) {
- Wait-Job -Id $Job.Id | Receive-Job
+ $ValidationFailures = 0
+ foreach ($Job in @(Get-Job)) {
+ $jobResult = Wait-Job -Id $Job.Id | Receive-Job
+ if ($jobResult -ne "0") {
+ $ValidationFailures++
+ }
+ }
+ if ($ValidationFailures -gt 0) {
+ Write-PipelineTaskError " $ValidationFailures package(s) failed validation."
+ ExitWithExitCode 1
}
}
diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1
index d5ec51b150..096ac321d1 100644
--- a/eng/common/post-build/symbols-validation.ps1
+++ b/eng/common/post-build/symbols-validation.ps1
@@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault {
# DWARF file for a .dylib
$DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf")
- $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools"
- $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe"
+ $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools"
+ $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe"
- & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
+ & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null
if (Test-Path $PdbPath) {
return "PDB"
@@ -159,25 +159,25 @@ function CheckSymbolsAvailable {
}
}
-function Installdotnetsymbol {
- $dotnetsymbolPackageName = "dotnet-symbol"
+function InstallDotnetSymbol {
+ $dotnetSymbolPackageName = "dotnet-symbol"
$dotnetRoot = InitializeDotNetCli -install:$true
$dotnet = "$dotnetRoot\dotnet.exe"
$toolList = & "$dotnet" tool list --global
- if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) {
- Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed."
+ if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) {
+ Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed."
}
else {
- Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..."
+ Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..."
Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed."
- & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global
+ & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global
}
}
try {
- Installdotnetsymbol
+ InstallDotnetSymbol
CheckSymbolsAvailable
}
diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1
index 6d9bdcf72b..eb21321ba2 100644
--- a/eng/common/sdl/execute-all-sdl-tools.ps1
+++ b/eng/common/sdl/execute-all-sdl-tools.ps1
@@ -1,30 +1,30 @@
Param(
- [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
- [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
- [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
- [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
- [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
- [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
- [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located
- [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
- [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
- [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
- [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
- [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
- [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
- [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
- [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
- [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
- [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
- [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
- [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
+ [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified)
+ [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified)
+ [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified
+ [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade)
+ [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master
+ [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located
+ [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located
+ [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault
+ [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code
+ [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts
+ [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs.
+ [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber)
+ [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed
+ [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+ [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs.
+ [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error
+ [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1")
+ [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1")
)
$ErrorActionPreference = "Stop"
@@ -45,6 +45,7 @@ if ($GuardianPackageName) {
$guardianCliLocation = $GuardianCliLocation
}
+$workingDirectory = (Split-Path $SourceDirectory -Parent)
$ValidPath = Test-Path $guardianCliLocation
if ($ValidPath -eq $False)
@@ -53,13 +54,13 @@ if ($ValidPath -eq $False)
exit 1
}
-& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
-$gdnFolder = Join-Path $ArtifactsDirectory ".gdn"
+& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel
+$gdnFolder = Join-Path $workingDirectory ".gdn"
if ($TsaOnboard) {
if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) {
- Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel
+ Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
@@ -71,10 +72,10 @@ if ($TsaOnboard) {
}
if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
if ($SourceToolsList -and $SourceToolsList.Count -gt 0) {
- & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
+ & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams
}
if ($UpdateBaseline) {
@@ -86,8 +87,8 @@ if ($TsaPublish) {
if (-not $TsaRepositoryName) {
$TsaRepositoryName = "$($Repository)-$($BranchName)"
}
- Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel"
- & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel
+ Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel"
+ & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1
index 1fdbb14329..d857ae2197 100644
--- a/eng/common/sdl/extract-artifact-packages.ps1
+++ b/eng/common/sdl/extract-artifact-packages.ps1
@@ -5,6 +5,14 @@ param(
$ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0
+
+# `tools.ps1` checks $ci to perform some actions. Since the post-build
+# scripts don't necessarily execute in the same agent that run the
+# build.ps1/sh script this variable isn't automatically set.
+$ci = $true
+$disableConfigureToolsetImport = "true"
+. $PSScriptRoot\..\tools.ps1
+
$ExtractPackage = {
param(
[string] $PackagePath # Full path to a NuGet package
diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1
index 26e01c0673..c737eb0e71 100644
--- a/eng/common/sdl/init-sdl.ps1
+++ b/eng/common/sdl/init-sdl.ps1
@@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop"
Set-StrictMode -Version 2.0
$LASTEXITCODE = 0
+# Don't display the console progress UI - it's a huge perf hit
+$ProgressPreference = 'SilentlyContinue'
+
# Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file
$encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken"))
$escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn")
diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config
index 3f97ac2f16..256ffbfb93 100644
--- a/eng/common/sdl/packages.config
+++ b/eng/common/sdl/packages.config
@@ -1,4 +1,4 @@
-
+
-
+
diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1
index b90c4399ba..9bc25314ae 100644
--- a/eng/common/sdl/run-sdl.ps1
+++ b/eng/common/sdl/run-sdl.ps1
@@ -32,16 +32,16 @@ foreach ($tool in $ToolsList) {
Write-Host $tool
# We have to manually configure tools that run on source to look at the source directory only
if ($tool -eq "credscan") {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
}
}
if ($tool -eq "policheck") {
- Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
- & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
+ Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})"
+ & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
@@ -56,4 +56,4 @@ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --bas
if ($LASTEXITCODE -ne 0) {
Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE."
exit $LASTEXITCODE
-}
\ No newline at end of file
+}
diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml
index 91621cf88f..2973bcaf3a 100644
--- a/eng/common/templates/job/execute-sdl.yml
+++ b/eng/common/templates/job/execute-sdl.yml
@@ -1,8 +1,16 @@
parameters:
overrideParameters: '' # Optional: to override values for parameters.
additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")'
- continueOnError: false # optional: determines whether to continue the build if the step errors;
+ # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named
+ # 'continueOnError', the parameter value is not correctly picked up.
+ # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter
+ sdlContinueOnError: false # optional: determines whether to continue the build if the step errors;
dependsOn: '' # Optional: dependencies of the job
+ artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts
+ # Usage:
+ # artifactNames:
+ # - 'BlobArtifacts'
+ # - 'Artifacts_Windows_NT_Release'
jobs:
- job: Run_SDL
@@ -15,23 +23,32 @@ jobs:
steps:
- checkout: self
clean: true
- - task: DownloadBuildArtifacts@0
- displayName: Download Build Artifacts
- inputs:
- buildType: current
- downloadType: specific files
- matchingPattern: "**"
- downloadPath: $(Build.SourcesDirectory)\artifacts
+ - ${{ if ne(parameters.artifactNames, '') }}:
+ - ${{ each artifactName in parameters.artifactNames }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Artifacts
+ inputs:
+ buildType: current
+ artifactName: ${{ artifactName }}
+ downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
+ - ${{ if eq(parameters.artifactNames, '') }}:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Build Artifacts
+ inputs:
+ buildType: current
+ downloadType: specific files
+ itemPattern: "**"
+ downloadPath: $(Build.ArtifactStagingDirectory)\artifacts
- powershell: eng/common/sdl/extract-artifact-packages.ps1
- -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
- -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts
+ -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
+ -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts
displayName: Extract Blob Artifacts
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- powershell: eng/common/sdl/extract-artifact-packages.ps1
- -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
- -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts
+ -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
+ -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts
displayName: Extract Package Artifacts
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- task: NuGetToolInstaller@1
displayName: 'Install NuGet.exe'
- task: NuGetCommand@2
@@ -45,12 +62,12 @@ jobs:
- ${{ if ne(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }}
displayName: Execute SDL
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
- ${{ if eq(parameters.overrideParameters, '') }}:
- powershell: eng/common/sdl/execute-all-sdl-tools.ps1
- -GuardianPackageName Microsoft.Guardian.Cli.0.7.1
+ -GuardianPackageName Microsoft.Guardian.Cli.0.7.2
-NugetPackageDirectory $(Build.SourcesDirectory)\.packages
-AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw)
${{ parameters.additionalParameters }}
displayName: Execute SDL
- continueOnError: ${{ parameters.continueOnError }}
+ continueOnError: ${{ parameters.sdlContinueOnError }}
diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml
index e54ce956f9..2692893de6 100644
--- a/eng/common/templates/job/generate-graph-files.yml
+++ b/eng/common/templates/job/generate-graph-files.yml
@@ -20,8 +20,6 @@ jobs:
variables:
# Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT
# DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat
- - group: Publish-Build-Assets
- - group: DotNet-AllOrgs-Darc-Pats
- name: _GraphArguments
value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT)
-azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat)
diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml
index 8db456bb7f..ecebd0f03e 100644
--- a/eng/common/templates/job/job.yml
+++ b/eng/common/templates/job/job.yml
@@ -1,67 +1,33 @@
+# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
+# and some (Microbuild) should only be applied to non-PR cases for internal builds.
+
parameters:
# Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
cancelTimeoutInMinutes: ''
-
condition: ''
-
- continueOnError: false
-
container: ''
-
+ continueOnError: false
dependsOn: ''
-
displayName: ''
-
- steps: []
-
pool: ''
-
+ steps: []
strategy: ''
-
timeoutInMinutes: ''
-
variables: []
-
workspace: ''
# Job base template specific parameters
- # Optional: Enable installing Microbuild plugin
- # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
- # _TeamName - the name of your team
- # _SignType - 'test' or 'real'
+ # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md
+ artifacts: ''
enableMicrobuild: false
-
- # Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false
-
- # Optional: Enable publishing to the build asset registry
enablePublishBuildAssets: false
-
- # Optional: Prevent gather/push manifest from executing when using publishing pipelines
- enablePublishUsingPipelines: false
-
- # Optional: Include PublishTestResults task
enablePublishTestResults: false
-
- # Optional: enable sending telemetry
- enableTelemetry: false
-
- # Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
- helixRepo: ''
-
- # Optional: define the helix type for telemetry (example: 'build/product/')
- helixType: ''
-
- # Required: name of the job
+ enablePublishUsingPipelines: false
name: ''
-
- # Optional: should run as a public build even in the internal project
- # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects.
+ preSteps: []
runAsPublic: false
-# Internal resources (telemetry, microbuild) can only be accessed from non-public projects,
-# and some (Microbuild) should only be applied to non-PR cases for internal builds.
-
jobs:
- job: ${{ parameters.name }}
@@ -93,7 +59,7 @@ jobs:
timeoutInMinutes: ${{ parameters.timeoutInMinutes }}
variables:
- - ${{ if eq(parameters.enableTelemetry, 'true') }}:
+ - ${{ if ne(parameters.enableTelemetry, 'false') }}:
- name: DOTNET_CLI_TELEMETRY_PROFILE
value: '$(Build.Repository.Uri)'
- ${{ each variable in parameters.variables }}:
@@ -125,21 +91,12 @@ jobs:
workspace: ${{ parameters.workspace }}
steps:
- - ${{ if eq(parameters.enableTelemetry, 'true') }}:
- # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
- - task: sendStartTelemetry@0
- displayName: 'Send Helix Start Telemetry'
- inputs:
- helixRepo: ${{ parameters.helixRepo }}
- ${{ if ne(parameters.helixType, '') }}:
- helixType: ${{ parameters.helixType }}
- buildConfig: $(_BuildConfig)
- runAsPublic: ${{ parameters.runAsPublic }}
- continueOnError: ${{ parameters.continueOnError }}
- condition: always()
+ - ${{ if ne(parameters.preSteps, '') }}:
+ - ${{ each preStep in parameters.preSteps }}:
+ - ${{ preStep }}
- - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if eq(parameters.enableMicrobuild, 'true') }}:
- task: MicroBuildSigningPlugin@2
displayName: Install MicroBuild plugin
inputs:
@@ -151,6 +108,16 @@ jobs:
continueOnError: ${{ parameters.continueOnError }}
condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT'))
+ - task: NuGetAuthenticate@0
+
+ - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}:
+ - task: DownloadPipelineArtifact@2
+ inputs:
+ buildType: current
+ artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }}
+ targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }}
+ itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }}
+
- ${{ each step in parameters.steps }}:
- ${{ step }}
@@ -163,20 +130,60 @@ jobs:
env:
TeamName: $(_TeamName)
- - ${{ if eq(parameters.enableTelemetry, 'true') }}:
- # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions
- - task: sendEndTelemetry@0
- displayName: 'Send Helix End Telemetry'
- continueOnError: ${{ parameters.continueOnError }}
- condition: always()
-
- - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
+ - ${{ if ne(parameters.artifacts.publish, '') }}:
+ - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}:
+ - task: CopyFiles@2
+ displayName: Gather binaries for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/bin'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin'
+ - task: CopyFiles@2
+ displayName: Gather packages for publish to artifacts
+ inputs:
+ SourceFolder: 'artifacts/packages'
+ Contents: '**'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages'
+ - task: PublishBuildArtifacts@1
+ displayName: Publish pipeline artifacts
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts'
+ PublishLocation: Container
+ ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }}
+ continueOnError: true
+ condition: always()
+ - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}:
+ - publish: artifacts/log
+ artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }}
+ displayName: Publish logs
+ continueOnError: true
+ condition: always()
+ - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: CopyFiles@2
+ displayName: Gather Asset Manifests
+ inputs:
+ SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest'
+ TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - task: PublishBuildArtifacts@1
+ displayName: Push Asset Manifests
+ inputs:
+ PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests'
+ PublishLocation: Container
+ ArtifactName: AssetManifests
+ continueOnError: ${{ parameters.continueOnError }}
+ condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true'))
+
+ - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}:
- task: PublishBuildArtifacts@1
displayName: Publish Logs
inputs:
PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)'
PublishLocation: Container
- ArtifactName: $(Agent.Os)_$(Agent.JobName)
+ ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }}
continueOnError: true
condition: always()
diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml
index ef809253d1..f877fd7a89 100644
--- a/eng/common/templates/job/performance.yml
+++ b/eng/common/templates/job/performance.yml
@@ -5,6 +5,7 @@ parameters:
displayName: '' # optional -- display name for the job. Will use jobName if not passed
pool: '' # required -- name of the Build pool
container: '' # required -- name of the container
+ osGroup: '' # required -- operating system for the job
extraSetupParameters: '' # optional -- extra arguments to pass to the setup script
frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against
continueOnError: 'false' # optional -- determines whether to continue the build if the step errors
@@ -44,12 +45,13 @@ jobs:
- HelixPreCommand: ''
- ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}:
+ - ${{ if eq( parameters.osGroup, 'Windows_NT') }}:
- HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"'
- IsInternal: -Internal
- - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}:
+ - ${{ if ne(parameters.osGroup, 'Windows_NT') }}:
- HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"'
- IsInternal: --internal
+
- group: DotNet-HelixApi-Access
- group: dotnet-benchview
diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml
index 9e77ef1b54..b722975f9c 100644
--- a/eng/common/templates/job/publish-build-assets.yml
+++ b/eng/common/templates/job/publish-build-assets.yml
@@ -47,6 +47,10 @@ jobs:
downloadPath: '$(Build.StagingDirectory)/Download'
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
+
+ - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - task: NuGetAuthenticate@0
+
- task: PowerShell@2
displayName: Publish Build Assets
inputs:
@@ -59,6 +63,7 @@ jobs:
/p:Configuration=$(_BuildConfig)
condition: ${{ parameters.condition }}
continueOnError: ${{ parameters.continueOnError }}
+
- task: powershell@2
displayName: Create ReleaseConfigs Artifact
inputs:
@@ -67,12 +72,14 @@ jobs:
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId)
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)"
Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild)
+
- task: PublishBuildArtifacts@1
displayName: Publish ReleaseConfigs Artifact
inputs:
PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt'
PublishLocation: Container
ArtifactName: ReleaseConfigs
+
- ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}:
- task: PublishBuildArtifacts@1
displayName: Publish Logs to VSTS
diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml
index 6a2f98c036..c08225a9a9 100644
--- a/eng/common/templates/jobs/jobs.yml
+++ b/eng/common/templates/jobs/jobs.yml
@@ -1,19 +1,10 @@
parameters:
- # Optional: 'true' if failures in job.yml job should not fail the job
+ # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md
continueOnError: false
- # Optional: Enable installing Microbuild plugin
- # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix
- # _TeamName - the name of your team
- # _SignType - 'test' or 'real'
- enableMicrobuild: false
-
# Optional: Include PublishBuildArtifacts task
enablePublishBuildArtifacts: false
- # Optional: Enable publishing to the build asset registry
- enablePublishBuildAssets: false
-
# Optional: Enable publishing using release pipelines
enablePublishUsingPipelines: false
@@ -23,19 +14,9 @@ parameters:
# Optional: Include toolset dependencies in the generated graph files
includeToolset: false
- # Optional: Include PublishTestResults task
- enablePublishTestResults: false
-
- # Optional: enable sending telemetry
- # if enabled then the 'helixRepo' parameter should also be specified
- enableTelemetry: false
-
# Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job
jobs: []
- # Optional: define the helix repo for telemetry (example: 'dotnet/arcade')
- helixRepo: ''
-
# Optional: Override automatically derived dependsOn value for "publish build assets" job
publishBuildAssetsDependsOn: ''
@@ -62,29 +43,30 @@ jobs:
name: ${{ job.job }}
-- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - template: ../job/publish-build-assets.yml
- parameters:
- continueOnError: ${{ parameters.continueOnError }}
- dependsOn:
- - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
- - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
- - ${{ job.job }}
- - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
- - ${{ each job in parameters.jobs }}:
- - ${{ job.job }}
- pool:
- vmImage: vs2017-win2016
- runAsPublic: ${{ parameters.runAsPublic }}
- publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
- enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
-
-- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
- - template: ../job/generate-graph-files.yml
- parameters:
- continueOnError: ${{ parameters.continueOnError }}
- includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
- dependsOn:
- - Asset_Registry_Publish
- pool:
- vmImage: vs2017-win2016
+- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}:
+ - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}:
+ - template: ../job/publish-build-assets.yml
+ parameters:
+ continueOnError: ${{ parameters.continueOnError }}
+ dependsOn:
+ - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.publishBuildAssetsDependsOn }}:
+ - ${{ job.job }}
+ - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}:
+ - ${{ each job in parameters.jobs }}:
+ - ${{ job.job }}
+ pool:
+ vmImage: vs2017-win2016
+ runAsPublic: ${{ parameters.runAsPublic }}
+ publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }}
+ enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }}
+
+ - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}:
+ - template: ../job/generate-graph-files.yml
+ parameters:
+ continueOnError: ${{ parameters.continueOnError }}
+ includeToolset: ${{ parameters.graphFileGeneration.includeToolset }}
+ dependsOn:
+ - Asset_Registry_Publish
+ pool:
+ vmImage: vs2017-win2016
diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml
new file mode 100644
index 0000000000..3a8755fbbb
--- /dev/null
+++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml
@@ -0,0 +1,159 @@
+parameters:
+ artifactsPublishingAdditionalParameters: ''
+ dependsOn:
+ - Validate
+ publishInstallersAndChecksums: false
+ symbolPublishingAdditionalParameters: ''
+ stageName: ''
+ channelName: ''
+ channelId: ''
+ transportFeed: ''
+ shippingFeed: ''
+ symbolsFeed: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+ dependsOn: ${{ parameters.dependsOn }}
+ variables:
+ - template: ../common-variables.yml
+ displayName: ${{ parameters.channelName }} Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job: publish_symbols
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ /p:PublishToMSDL=false
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'SymbolPublishing'
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }}))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+ continueOnError: true
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=true
+ /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey)
+ /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey)
+ /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+ /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:PublishToMSDL=false
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'AssetsPublishing'
+
+ - template: ../../steps/promote-build.yml
+ parameters:
+ ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml
new file mode 100644
index 0000000000..3f572f8b12
--- /dev/null
+++ b/eng/common/templates/post-build/channels/generic-public-channel.yml
@@ -0,0 +1,159 @@
+parameters:
+ artifactsPublishingAdditionalParameters: ''
+ dependsOn:
+ - Validate
+ publishInstallersAndChecksums: false
+ symbolPublishingAdditionalParameters: ''
+ stageName: ''
+ channelName: ''
+ channelId: ''
+ transportFeed: ''
+ shippingFeed: ''
+ symbolsFeed: ''
+
+stages:
+- stage: ${{ parameters.stageName }}
+ dependsOn: ${{ parameters.dependsOn }}
+ variables:
+ - template: ../common-variables.yml
+ displayName: ${{ parameters.channelName }} Publishing
+ jobs:
+ - template: ../setup-maestro-vars.yml
+
+ - job: publish_symbols
+ displayName: Symbol Publishing
+ dependsOn: setupMaestroVars
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} ))
+ variables:
+ - group: DotNet-Symbol-Server-Pats
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ artifactName: 'BlobArtifacts'
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download PDB Artifacts
+ inputs:
+ artifactName: 'PDBArtifacts'
+ continueOnError: true
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet
+ /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat)
+ /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat)
+ /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt'
+ /p:Configuration=Release
+ ${{ parameters.symbolPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'SymbolPublishing'
+
+ - job: publish_assets
+ displayName: Publish Assets
+ dependsOn: setupMaestroVars
+ variables:
+ - name: BARBuildId
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ]
+ - name: IsStableBuild
+ value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ]
+ condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }}))
+ pool:
+ vmImage: 'windows-2019'
+ steps:
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Package Artifacts
+ inputs:
+ buildType: current
+ artifactName: PackageArtifacts
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Blob Artifacts
+ inputs:
+ buildType: current
+ artifactName: BlobArtifacts
+ continueOnError: true
+
+ - task: DownloadBuildArtifacts@0
+ displayName: Download Asset Manifests
+ inputs:
+ buildType: current
+ artifactName: AssetManifests
+
+ - task: NuGetToolInstaller@1
+ displayName: 'Install NuGet.exe'
+
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
+ - task: PowerShell@2
+ displayName: Publish Assets
+ inputs:
+ filePath: eng\common\sdk-task.ps1
+ arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet
+ /p:ArtifactsCategory=$(_DotNetArtifactsCategory)
+ /p:IsStableBuild=$(IsStableBuild)
+ /p:IsInternalBuild=$(IsInternalBuild)
+ /p:RepositoryName=$(Build.Repository.Name)
+ /p:CommitSha=$(Build.SourceVersion)
+ /p:NugetPath=$(NuGetExeToolPath)
+ /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)'
+ /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)'
+ /p:BARBuildId=$(BARBuildId)
+ /p:MaestroApiEndpoint='$(MaestroApiEndPoint)'
+ /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)'
+ /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/'
+ /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/'
+ /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/'
+ /p:Configuration=Release
+ /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }}
+ /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl)
+ /p:InstallersAzureAccountKey=$(dotnetcli-storage-key)
+ /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl)
+ /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key)
+ /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}'
+ /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}'
+ /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}'
+ /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)'
+ ${{ parameters.artifactsPublishingAdditionalParameters }}
+
+ - template: ../../steps/publish-logs.yml
+ parameters:
+ StageLabel: '${{ parameters.stageName }}'
+ JobLabel: 'AssetsPublishing'
+
+ - template: ../../steps/promote-build.yml
+ parameters:
+ ChannelId: ${{ parameters.channelId }}
diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml
index b00d85d8ce..0a2c40c103 100644
--- a/eng/common/templates/post-build/common-variables.yml
+++ b/eng/common/templates/post-build/common-variables.yml
@@ -1,23 +1,34 @@
variables:
- - group: Publish-Build-Assets
+ - group: AzureDevOps-Artifact-Feeds-Pats
+ - group: DotNet-Blob-Feed
- group: DotNet-DotNetCli-Storage
+ - group: DotNet-MSRC-Storage
+ - group: Publish-Build-Assets
- # .NET Core 3 Dev
- - name: PublicDevRelease_30_Channel_Id
- value: 3
+ # .NET Core 3.1 Dev
+ - name: PublicDevRelease_31_Channel_Id
+ value: 128
# .NET Core 5 Dev
- name: NetCore_5_Dev_Channel_Id
value: 131
# .NET Tools - Validation
- - name: PublicValidationRelease_30_Channel_Id
+ - name: NetCore_Tools_Validation_Channel_Id
value: 9
# .NET Tools - Latest
- name: NetCore_Tools_Latest_Channel_Id
value: 2
+ # .NET 3 Tools - Validation
+ - name: NETCore_3_Tools_Validation_Channel_Id
+ value: 390
+
+ # .NET 3 Tools - Latest
+ - name: NetCore_3_Tools_Channel_Id
+ value: 344
+
# .NET Core 3.0 Internal Servicing
- name: InternalServicing_30_Channel_Id
value: 184
@@ -26,14 +37,22 @@ variables:
- name: PublicRelease_30_Channel_Id
value: 19
+ # .NET Core 3.1 Release
+ - name: PublicRelease_31_Channel_Id
+ value: 129
+
+ # General Testing
+ - name: GeneralTesting_Channel_Id
+ value: 529
+
+ # .NET Core 3.1 Blazor Features
+ - name: NetCore_31_Blazor_Features_Channel_Id
+ value: 531
+
# Whether the build is internal or not
- name: IsInternalBuild
value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }}
- # Storage account name for proxy-backed feeds
- - name: ProxyBackedFeedsAccountName
- value: dotnetfeed
-
# Default Maestro++ API Endpoint and API Version
- name: MaestroApiEndPoint
value: "https://maestro-prod.westus2.cloudapp.azure.com"
@@ -47,8 +66,23 @@ variables:
- name: SymbolToolVersion
value: 1.0.1
+ # Feed Configurations
+ # These should include the suffix "/index.json"
+
# Default locations for Installers and checksums
+ # Public Locations
- name: ChecksumsBlobFeedUrl
- value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
- - name: InstallersBlobFeedUrl
value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json
+ - name: InstallersBlobFeedUrl
+ value: https://dotnetcli.blob.core.windows.net/dotnet/index.json
+
+ # Private Locations
+ - name: InternalChecksumsBlobFeedUrl
+ value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalChecksumsBlobFeedKey
+ value: $(dotnetclichecksumsmsrc-storage-key)
+
+ - name: InternalInstallersBlobFeedUrl
+ value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json
+ - name: InternalInstallersBlobFeedKey
+ value: $(dotnetclimsrc-access-key)
diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml
index 29849c9dc6..5965cc2005 100644
--- a/eng/common/templates/post-build/post-build.yml
+++ b/eng/common/templates/post-build/post-build.yml
@@ -1,12 +1,14 @@
parameters:
- enableSourceLinkValidation: true
+ enableSourceLinkValidation: false
enableSigningValidation: true
- enableSymbolValidation: true
+ enableSymbolValidation: false
enableNugetValidation: true
publishInstallersAndChecksums: false
SDLValidationParameters:
enable: false
+ continueOnError: false
params: ''
+ artifactNames: ''
# These parameters let the user customize the call to sdk-task.ps1 for publishing
# symbols & general artifacts as well as for signing validation
@@ -15,11 +17,14 @@ parameters:
signingValidationAdditionalParameters: ''
# Which stages should finish execution before post-build stages start
- dependsOn: [build]
+ validateDependsOn:
+ - build
+ publishDependsOn:
+ - Validate
stages:
-- stage: validate
- dependsOn: ${{ parameters.dependsOn }}
+- stage: Validate
+ dependsOn: ${{ parameters.validateDependsOn }}
displayName: Validate
jobs:
- ${{ if eq(parameters.enableNugetValidation, 'true') }}:
@@ -44,6 +49,8 @@ stages:
- ${{ if eq(parameters.enableSigningValidation, 'true') }}:
- job:
displayName: Signing Validation
+ variables:
+ - template: common-variables.yml
pool:
vmImage: 'windows-2019'
steps:
@@ -53,6 +60,18 @@ stages:
buildType: current
artifactName: PackageArtifacts
+ # This is necessary whenever we want to publish/restore to an AzDO private feed
+ # Since sdk-task.ps1 tries to restore packages we need to do this authentication here
+ # otherwise it'll complain about accessing a private feed.
+ - task: NuGetAuthenticate@0
+ displayName: 'Authenticate to AzDO Feeds'
+
+ - task: PowerShell@2
+ displayName: Enable cross-org publishing
+ inputs:
+ filePath: eng\common\enable-cross-org-publishing.ps1
+ arguments: -token $(dn-bot-dnceng-artifact-feeds-rw)
+
- task: PowerShell@2
displayName: Validate
inputs:
@@ -60,9 +79,13 @@ stages:
arguments: -task SigningValidation -restore -msbuildEngine dotnet
/p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts'
/p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt'
- /p:Configuration=Release
${{ parameters.signingValidationAdditionalParameters }}
+ - template: ../steps/publish-logs.yml
+ parameters:
+ StageLabel: 'Validation'
+ JobLabel: 'Signing'
+
- ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}:
- job:
displayName: SourceLink Validation
@@ -86,44 +109,89 @@ stages:
-GHRepoName $(Build.Repository.Name)
-GHCommit $(Build.SourceVersion)
-SourcelinkCliVersion $(SourceLinkCLIVersion)
+ continueOnError: true
- ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}:
- template: /eng/common/templates/job/execute-sdl.yml
parameters:
additionalParameters: ${{ parameters.SDLValidationParameters.params }}
+ continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }}
+ artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }}
-- template: \eng\common\templates\post-build\channels\netcore-dev-5.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
- enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NetCore_Dev5_Publish'
+ channelName: '.NET Core 5 Dev'
+ channelId: 131
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json'
-- template: \eng\common\templates\post-build\channels\public-dev-release.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
- enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NetCore_Tools_Latest_Publish'
+ channelName: '.NET Tools - Latest'
+ channelId: 2
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-- template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
- enableSymbolValidation: ${{ parameters.enableSymbolValidation }}
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'PVR_Publish'
+ channelName: '.NET Tools - Validation'
+ channelId: 9
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json'
-- template: \eng\common\templates\post-build\channels\public-validation-release.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'General_Testing_Publish'
+ channelName: 'General Testing'
+ channelId: 529
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json'
-- template: \eng\common\templates\post-build\channels\public-release.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_Tooling_Dev_Publishing'
+ channelName: '.NET Core Tooling Dev'
+ channelId: 548
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
-- template: \eng\common\templates\post-build\channels\internal-servicing.yml
+- template: \eng\common\templates\post-build\channels\generic-public-channel.yml
parameters:
- symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }}
+ dependsOn: ${{ parameters.publishDependsOn }}
+ publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }}
+ symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }}
+ stageName: 'NETCore_Tooling_Release_Publishing'
+ channelName: '.NET Core Tooling Release'
+ channelId: 549
+ transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json'
+ symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json'
diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml
index 56242b068e..716b53f740 100644
--- a/eng/common/templates/post-build/setup-maestro-vars.yml
+++ b/eng/common/templates/post-build/setup-maestro-vars.yml
@@ -4,6 +4,8 @@ jobs:
pool:
vmImage: 'windows-2019'
steps:
+ - checkout: none
+
- task: DownloadBuildArtifacts@0
displayName: Download Release Configs
inputs:
@@ -14,5 +16,25 @@ jobs:
name: setReleaseVars
displayName: Set Release Configs Vars
inputs:
- filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1
- arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt'
+ targetType: inline
+ script: |
+ try {
+ $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt
+
+ $BarId = $Content | Select -Index 0
+
+ $Channels = ""
+ $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," }
+
+ $IsStableBuild = $Content | Select -Index 2
+
+ Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId"
+ Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels"
+ Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild"
+ }
+ catch {
+ Write-Host $_
+ Write-Host $_.Exception
+ Write-Host $_.ScriptStackTrace
+ exit 1
+ }
diff --git a/eng/common/templates/steps/promote-build.yml b/eng/common/templates/steps/promote-build.yml
new file mode 100644
index 0000000000..b90404435d
--- /dev/null
+++ b/eng/common/templates/steps/promote-build.yml
@@ -0,0 +1,13 @@
+parameters:
+ ChannelId: 0
+
+steps:
+- task: PowerShell@2
+ displayName: Add Build to Channel
+ inputs:
+ filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1
+ arguments: -BuildId $(BARBuildId)
+ -ChannelId ${{ parameters.ChannelId }}
+ -MaestroApiAccessToken $(MaestroApiAccessToken)
+ -MaestroApiEndPoint $(MaestroApiEndPoint)
+ -MaestroApiVersion $(MaestroApiVersion)
diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml
new file mode 100644
index 0000000000..8903ba57c0
--- /dev/null
+++ b/eng/common/templates/steps/publish-logs.yml
@@ -0,0 +1,23 @@
+parameters:
+ StageLabel: ''
+ JobLabel: ''
+
+steps:
+- task: Powershell@2
+ displayName: Prepare Binlogs to Upload
+ inputs:
+ targetType: inline
+ script: |
+ New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/
+ continueOnError: true
+ condition: always()
+
+- task: PublishBuildArtifacts@1
+ displayName: Publish Logs
+ inputs:
+ PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs'
+ PublishLocation: Container
+ ArtifactName: PostBuilLogs
+ continueOnError: true
+ condition: always()
diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml
index 05df886f55..30becf01ea 100644
--- a/eng/common/templates/steps/send-to-helix.yml
+++ b/eng/common/templates/steps/send-to-helix.yml
@@ -23,6 +23,7 @@ parameters:
EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control
WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget."
IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set
+ HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int)
Creator: '' # optional -- if the build is external, use this to specify who is sending the job
DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO
condition: succeeded() # optional -- condition for step to execute; defaults to succeeded()
@@ -55,6 +56,7 @@ steps:
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT'))
@@ -85,6 +87,7 @@ steps:
DotNetCliVersion: ${{ parameters.DotNetCliVersion }}
EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }}
WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }}
+ HelixBaseUri: ${{ parameters.HelixBaseUri }}
Creator: ${{ parameters.Creator }}
SYSTEM_ACCESSTOKEN: $(System.AccessToken)
condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT'))
diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1
index 9c12b1b4fd..42ca33ac33 100644
--- a/eng/common/tools.ps1
+++ b/eng/common/tools.ps1
@@ -49,6 +49,8 @@
# An array of names of processes to stop on script exit if prepareMachine is true.
$processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") }
+$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null }
+
set-strictmode -version 2.0
$ErrorActionPreference = "Stop"
[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
@@ -153,6 +155,7 @@ function InitializeDotNetCli([bool]$install) {
# Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build
Write-PipelinePrependPath -Path $dotnetRoot
+
Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0'
Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1'
@@ -163,6 +166,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) {
$installScript = Join-Path $dotnetRoot "dotnet-install.ps1"
if (!(Test-Path $installScript)) {
Create-Directory $dotnetRoot
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript
}
@@ -173,7 +177,14 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc
InstallDotNet $dotnetRoot $version $architecture
}
-function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) {
+function InstallDotNet([string] $dotnetRoot,
+ [string] $version,
+ [string] $architecture = "",
+ [string] $runtime = "",
+ [bool] $skipNonVersionedFiles = $false,
+ [string] $runtimeSourceFeed = "",
+ [string] $runtimeSourceFeedKey = "") {
+
$installScript = GetDotNetInstallScript $dotnetRoot
$installParameters = @{
Version = $version
@@ -184,10 +195,29 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit
if ($runtime) { $installParameters.Runtime = $runtime }
if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles }
- & $installScript @installParameters
- if ($lastExitCode -ne 0) {
- Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')."
- ExitWithExitCode $lastExitCode
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location."
+
+ # Only the runtime can be installed from a custom [private] location.
+ if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) {
+ if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed }
+
+ if ($runtimeSourceFeedKey) {
+ $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey)
+ $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes)
+ $installParameters.FeedCredential = $decodedString
+ }
+
+ try {
+ & $installScript @installParameters
+ }
+ catch {
+ Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'."
+ }
+ }
}
}
@@ -282,6 +312,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) {
Create-Directory $packageDir
Write-Host "Downloading $packageName $packageVersion"
+ $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit
Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath
Unzip $packagePath $packageDir
}
@@ -363,7 +394,6 @@ function InitializeBuildTool() {
Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'."
ExitWithExitCode 1
}
-
$buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" }
} elseif ($msbuildEngine -eq "vs") {
try {
@@ -416,7 +446,7 @@ function GetSdkTaskProject([string]$taskName) {
}
function InitializeNativeTools() {
- if (Get-Member -InputObject $GlobalJson -Name "native-tools") {
+ if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) {
$nativeArgs= @{}
if ($ci) {
$nativeArgs = @{
@@ -488,6 +518,18 @@ function Stop-Processes() {
function MSBuild() {
if ($pipelinesLog) {
$buildTool = InitializeBuildTool
+
+ # Work around issues with Azure Artifacts credential provider
+ # https://github.com/dotnet/arcade/issues/3932
+ if ($ci -and $buildTool.Tool -eq "dotnet") {
+ dotnet nuget locals http-cache -c
+
+ $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20
+ $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20'
+ Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20'
+ }
+
$toolsetBuildProject = InitializeToolset
$path = Split-Path -parent $toolsetBuildProject
$path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll")
@@ -585,3 +627,12 @@ Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir
Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir
Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir
Write-PipelineSetVariable -Name 'TMP' -Value $TempDir
+
+# Import custom tools configuration, if present in the repo.
+# Note: Import in global scope so that the script set top-level variables without qualification.
+if (!$disableConfigureToolsetImport) {
+ $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1"
+ if (Test-Path $configureToolsetScript) {
+ . $configureToolsetScript
+ }
+}
diff --git a/eng/common/tools.sh b/eng/common/tools.sh
index 3af9be6157..6a23ac0a34 100755
--- a/eng/common/tools.sh
+++ b/eng/common/tools.sh
@@ -4,6 +4,7 @@
# CI mode - set to true on CI server for PR validation build or official build.
ci=${ci:-false}
+disable_configure_toolset_import=${disable_configure_toolset_import:-}
# Set to true to use the pipelines logger which will enable Azure logging output.
# https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md
@@ -191,8 +192,28 @@ function InstallDotNet {
fi
bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || {
local exit_code=$?
- Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')."
- ExitWithExitCode $exit_code
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')."
+
+ if [[ -n "$runtimeArg" ]]; then
+ local runtimeSourceFeed=''
+ if [[ -n "${6:-}" ]]; then
+ runtimeSourceFeed="--azure-feed $6"
+ fi
+
+ local runtimeSourceFeedKey=''
+ if [[ -n "${7:-}" ]]; then
+ decodedFeedKey=`echo $7 | base64 --decode`
+ runtimeSourceFeedKey="--feed-credential $decodedFeedKey"
+ fi
+
+ if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then
+ bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ fi
+ fi
}
}
@@ -208,12 +229,19 @@ function GetDotNetInstallScript {
# Use curl if available, otherwise use wget
if command -v curl > /dev/null; then
- curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script"
- else
- wget -q -O "$install_script" "$install_script_url"
+ curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
+ else
+ wget -q -O "$install_script" "$install_script_url" || {
+ local exit_code=$?
+ Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')."
+ ExitWithExitCode $exit_code
+ }
fi
fi
-
# return value
_GetDotNetInstallScript="$install_script"
}
@@ -245,6 +273,9 @@ function GetNuGetPackageCachePath {
}
function InitializeNativeTools() {
+ if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then
+ return
+ fi
if grep -Fq "native-tools" $global_json_file
then
local nativeArgs=""
@@ -321,6 +352,18 @@ function MSBuild {
if [[ "$pipelines_log" == true ]]; then
InitializeBuildTool
InitializeToolset
+
+ # Work around issues with Azure Artifacts credential provider
+ # https://github.com/dotnet/arcade/issues/3932
+ if [[ "$ci" == true ]]; then
+ "$_InitializeBuildTool" nuget locals http-cache -c
+
+ export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20
+ export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20"
+ Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20"
+ fi
+
local toolset_dir="${_InitializeToolset%/*}"
local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll"
args=( "${args[@]}" "-logger:$logger_path" )
@@ -392,3 +435,18 @@ Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir"
Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir"
Write-PipelineSetVariable -name "Temp" -value "$temp_dir"
Write-PipelineSetVariable -name "TMP" -value "$temp_dir"
+
+# Import custom tools configuration, if present in the repo.
+if [[ -z "$disable_configure_toolset_import" ]]; then
+ configure_toolset_script="$eng_root/configure-toolset.sh"
+ if [[ -a "$configure_toolset_script" ]]; then
+ . "$configure_toolset_script"
+ fi
+fi
+
+# TODO: https://github.com/dotnet/arcade/issues/1468
+# Temporary workaround to avoid breaking change.
+# Remove once repos are updated.
+if [[ -n "${useInstalledDotNetCli:-}" ]]; then
+ use_installed_dotnet_cli="$useInstalledDotNetCli"
+fi
diff --git a/eng/emptyPackageProject/Class1.cs b/eng/emptyPackageProject/Class1.cs
new file mode 100644
index 0000000000..c2b154a9d4
--- /dev/null
+++ b/eng/emptyPackageProject/Class1.cs
@@ -0,0 +1,6 @@
+namespace MicrosoftDotnetSourceBuiltArtifacts
+{
+ public class Class1
+ {
+ }
+}
diff --git a/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj b/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj
new file mode 100644
index 0000000000..231c83fa69
--- /dev/null
+++ b/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj
@@ -0,0 +1,19 @@
+
+
+
+
+ https://github.com/dotnet/source-build
+ https://github.com/dotnet/source-build
+ git
+ 0000
+ $(ManifestCommit)
+ MIT
+ netcoreapp2.1
+ true
+ false
+ NU5105
+
+
+
+
diff --git a/eng/publishPackages/Directory.Build.props b/eng/publishPackages/Directory.Build.props
new file mode 100644
index 0000000000..8ebfe9720b
--- /dev/null
+++ b/eng/publishPackages/Directory.Build.props
@@ -0,0 +1,8 @@
+
+
+
+
+ $(CopyrightNetFoundation)
+ MIT
+
+
diff --git a/eng/publishPackages/Directory.Build.targets b/eng/publishPackages/Directory.Build.targets
new file mode 100644
index 0000000000..e2bdf743c6
--- /dev/null
+++ b/eng/publishPackages/Directory.Build.targets
@@ -0,0 +1,3 @@
+
+
+
diff --git a/eng/publishPackages/collectArtifacts.proj b/eng/publishPackages/collectArtifacts.proj
new file mode 100644
index 0000000000..fb8df8ee1c
--- /dev/null
+++ b/eng/publishPackages/collectArtifacts.proj
@@ -0,0 +1,21 @@
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/eng/publishPackages/publish.proj b/eng/publishPackages/publish.proj
new file mode 100644
index 0000000000..e755e0a31d
--- /dev/null
+++ b/eng/publishPackages/publish.proj
@@ -0,0 +1,76 @@
+
+
+ $(RepoRoot)artifacts/packages/
+ $(RepoRoot)artifacts/asset-manifests/
+ $(RepoRoot)artifacts/tarball/
+
+ $(RestoreSources);
+ https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json;
+
+
+ CreateEmptyPackage;
+
+
+ $(CreateTargets);
+
+
+ $(BuildDependsOn);
+ PreparePublish;
+ PublishPackagesToBlobFeed;
+ PublishFilesToBlobFeed;
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+ $(TarballDir)Private.SourceBuilt.Artifacts.*.tar.gz
+
+
+
+
+
+
+ https://$(AzureAccountName).blob.core.windows.net/$(ContainerName)/index.json
+ $(AzureAccessToken)
+
+
+
+
+
+ $(ManifestDir)source-build-artifacts-nupkg-manifest.xml
+
+
+
+
+
+
+
+
+ $(ManifestDir)source-build-artifacts-tar-manifest.xml
+
+
+
+
+
+
+
+
diff --git a/publish.ps1 b/publish.ps1
new file mode 100644
index 0000000000..123997c3a3
--- /dev/null
+++ b/publish.ps1
@@ -0,0 +1,4 @@
+. "$PSScriptRoot/eng/common/tools.ps1"
+InitializeToolset
+Write-Host "$PSScriptRoot/eng/common/build.ps1 -bl -restore -build -pack -publish /p:Projects=`"$PSScriptRoot/eng/publishPackages/publish.proj`" /p:PublishingToBlobStorage=true /p:DeterministicSourcePaths=false /p:EnableSourceLink=false $args"
+Invoke-Expression "$PSScriptRoot/eng/common/build.ps1 -bl -restore -build -pack -publish /p:Projects=`"$PSScriptRoot/eng/publishPackages/publish.proj`" /p:PublishingToBlobStorage=true /p:DeterministicSourcePaths=false /p:EnableSourceLink=false $args"
diff --git a/src/package-source-build/package.proj b/src/package-source-build/package.proj
index ecf7aa4f85..de2d69a785 100644
--- a/src/package-source-build/package.proj
+++ b/src/package-source-build/package.proj
@@ -14,7 +14,7 @@
- $(OutputPath)$(SourceBuiltArtifactsTarballName).$(VersionPrefix)-$(VersionSuffix).tar.gz
+ $(OutputPath)$(SourceBuiltArtifactsTarballName).$(SourceBuildOutputVersion).tar.gz
diff --git a/tools-local/prebuilt-baseline-online.xml b/tools-local/prebuilt-baseline-online.xml
index 8373234442..f5ff489a7a 100644
--- a/tools-local/prebuilt-baseline-online.xml
+++ b/tools-local/prebuilt-baseline-online.xml
@@ -77,6 +77,7 @@
+