diff --git a/.vsts.pipelines/builds/ci-internal.yml b/.vsts.pipelines/builds/ci-internal.yml index ccf83c8f48..3c508d8314 100644 --- a/.vsts.pipelines/builds/ci-internal.yml +++ b/.vsts.pipelines/builds/ci-internal.yml @@ -3,3 +3,21 @@ jobs: parameters: windowsPoolName: NetCoreInternal-Pool windowsQueueName: buildpool.windows.10.amd64.vs2017 + +- template: ../jobs/collect-artifacts.yml + parameters: + dependsOn: + - centos71 + - debian9 + - fedora30 + - ubuntu1804 + pool: + vmImage: vs2017-win2016 + +- template: /eng/common/templates/job/publish-build-assets.yml + parameters: + dependsOn: + - Collect_Artifacts + enablePublishBuildArtifacts: true + pool: + vmImage: vs2017-win2016 diff --git a/.vsts.pipelines/builds/matrix.yml b/.vsts.pipelines/builds/matrix.yml index a896a13189..dadd43d9df 100644 --- a/.vsts.pipelines/builds/matrix.yml +++ b/.vsts.pipelines/builds/matrix.yml @@ -28,6 +28,7 @@ jobs: matrix: Production: {} Online: { type: Online } + Offline: { type: Offline } - template: ../jobs/ci-linux.yml parameters: @@ -43,6 +44,8 @@ jobs: parameters: job: ubuntu1804 imageName: mcr.microsoft.com/dotnet-buildtools/prereqs:ubuntu-18.04-f90bc20-20180320154721 + matrix: + Offline: { type: Offline } - template: ../jobs/ci-local.yml parameters: diff --git a/.vsts.pipelines/jobs/ci-linux.yml b/.vsts.pipelines/jobs/ci-linux.yml index 73e1779a35..40db5b135c 100644 --- a/.vsts.pipelines/jobs/ci-linux.yml +++ b/.vsts.pipelines/jobs/ci-linux.yml @@ -74,6 +74,18 @@ jobs: displayName: Build source-build timeoutInMinutes: 150 + - script: | + set -ex + df -h + $(docker.run) $(docker.drop.map) $(docker.src.map) $(docker.src.work) $(imageName) ./build.sh \ + --collect-artifacts \ + /p:ArtifactOutput=/drop/artifacts/source-build/ \ + /bl + du -h $(rootDirectory) | sort -h | tail -n 50 + displayName: Copy source-build artifacts + condition: always() + continueOnError: true + # Generate prebuilt burndown data - script: | set -ex @@ -137,7 +149,8 @@ jobs: -path './bin/aspnet-debug/*' -o \ -iname '*.binlog' -o \ -iname '*.log' \) \ - -exec cp {} --parents /logs/source-build/logs \;" + -exec cp {} --parents /logs/source-build/logs \; + find . > /logs/source-build/logs/file-listing.txt" du -h $(rootDirectory) | sort -h | tail -n 50 displayName: Copy source-build production build logs condition: always() @@ -197,6 +210,18 @@ jobs: timeoutInMinutes: 120 condition: and(succeeded(), eq(variables['sb.tarball'], true)) + - script: | + set -ex + df -h + $(docker.run) $(docker.drop.map) $(docker.tb.map) $(docker.tb.work) $(imageName) "$(tarballName)/build.sh" \ + --collect-artifacts \ + /p:ArtifactOutput=/drop/artifacts/tarball/ \ + /bl + du -h $(rootDirectory) | sort -h | tail -n 50 + displayName: Copy tarball artifacts + condition: eq(variables['sb.tarball'], true) + continueOnError: true + # Run smoke tests. - script: | set -ex @@ -228,7 +253,8 @@ jobs: -path './bin/aspnet-debug/*' -o \ -iname '*.binlog' -o \ -iname '*.log' \) \ - -exec cp {} --parents /logs/tarball/logs \;" + -exec cp {} --parents /logs/tarball/logs \; + find . > /logs/tarball/logs/file-listing.txt" du -h $(rootDirectory) | sort -h | tail -n 50 displayName: Copy tarball logs condition: eq(variables['sb.tarball'], true) @@ -252,6 +278,7 @@ jobs: # Copy artifacts to staging - Copy to VSTS owned folder is done outside of docker so copied files # have correct ownership so VSTS can clean them up later. - task: CopyFiles@2 + displayName: Copy artifacts from Docker-owned staging directory to AzDO-agent-owned artifacts directory condition: always() continueOnError: true inputs: @@ -275,6 +302,14 @@ jobs: PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/tarball ArtifactName: Tarball $(artifactName) ArtifactType: Container + - task: PublishBuildArtifacts@1 + displayName: Publish build artifacts + condition: always() + continueOnError: true + inputs: + PathtoPublish: $(Build.ArtifactStagingDirectory)/drop/artifacts + ArtifactName: Artifacts $(artifactName) + ArtifactType: Container # Publish prebuilt report data to blob storage. - template: ../steps/publish-prebuilt-data-sh.yml diff --git a/.vsts.pipelines/jobs/collect-artifacts.yml b/.vsts.pipelines/jobs/collect-artifacts.yml new file mode 100644 index 0000000000..06e9b57ee8 --- /dev/null +++ b/.vsts.pipelines/jobs/collect-artifacts.yml @@ -0,0 +1,145 @@ +jobs: +- job: Collect_Artifacts + + dependsOn: ${{ parameters.dependsOn }} + + displayName: Collect Artifacts + + pool: ${{ parameters.pool }} + + steps: + - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: DownloadBuildArtifacts@0 + displayName: Download CentOS artifacts + inputs: + artifactName: Tarball centos71 Offline + downloadPath: '$(Build.StagingDirectory)/centos' + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)/centos' + script: | + rm */tarball*.tar.gz + + - task: DownloadBuildArtifacts@0 + displayName: Download Debian artifacts + inputs: + artifactName: Tarball debian9 Offline + downloadPath: '$(Build.StagingDirectory)/debian' + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)/debian' + script: | + rm */tarball*.tar.gz + + - task: DownloadBuildArtifacts@0 + displayName: Download Fedora artifacts + inputs: + artifactName: Tarball fedora30 Offline + downloadPath: '$(Build.StagingDirectory)/fedora' + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)/fedora' + script: | + rm */tarball*.tar.gz + + - task: DownloadBuildArtifacts@0 + displayName: Download Ubuntu artifacts + inputs: + artifactName: Tarball ubuntu1804 Offline + downloadPath: '$(Build.StagingDirectory)/ubuntu' + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)/ubuntu' + script: | + rm */tarball*.tar.gz + + - task: DownloadBuildArtifacts@0 + displayName: Download portable artifacts + inputs: + artifactName: Tarball centos71 Offline Portable + downloadPath: '$(Build.StagingDirectory)/portable' + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)/portable' + script: | + rm */tarball*.tar.gz + + - task: Bash@3 + inputs: + targetType: 'inline' + workingDirectory: '$(Build.StagingDirectory)' + script: | + version=`echo centos/*/Private.SourceBuilt.Artifacts*.tar.gz | sed 's/\.tar\.gz$//' | sed 's/^centos\/.*\/Private\.SourceBuilt\.Artifacts\.//'` + echo "Artifacts version: $version" + mkdir collected + cd collected + # do CentOS last to prefer those versions of any duplicates + for os in portable debian fedora ubuntu centos; do + # DownloadArtifact creates another directory - use '*' to not deal with this + tar xzf ../$os/*/Private.SourceBuilt.Artifacts*.tar.gz + rm -rf ../$os + done + echo $(Build.SourceVersion) > .version + echo $(Build.BuildNumber) > .buildnumber + echo $(Build.BuildId) > .buildid + tar czf ../Private.SourceBuilt.Artifacts.$version.tar.gz * .version .buildnumber .buildid + rm -rf * + mv ../Private.SourceBuilt.Artifacts.*.tar.gz . + displayName: Collect RID-specific packages into one tarball + condition: succeeded() + + - task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: $(Build.SourcesDirectory)\publish.ps1 + arguments: >- + /p:OfficialBuildId=$(Build.BuildNumber) + /p:AzureAccountName=$(publish.blobStorage.account) + /p:ContainerName=$(publish.blobStorage.container) + "/p:AzureAccessToken=$(publish.blobStorage.accessToken)" + "/p:ManifestName=$(BUILD.REPOSITORY.NAME)" + "/p:ManifestBuildId=$(BUILD.BUILDID)" + "/p:ManifestBranch=$(BUILD.SOURCEBRANCHNAME)" + /p:ManifestCommit=$(Build.SourceVersion) + /p:DotNetPublishToBlobFeed=true + workingDirectory: '$(Build.SourcesDirectory)' + displayName: Publish collected artifacts + condition: succeeded() + + - task: PublishBuildArtifacts@1 + displayName: Uploaded collected artifacts + condition: always() + continueOnError: true + inputs: + PathToPublish: $(Build.SourcesDirectory)/artifacts/tarball/ + ArtifactName: Collected artifacts + ArtifactType: Container + + - task: PublishBuildArtifacts@1 + displayName: Uploaded asset manifests + condition: always() + continueOnError: true + inputs: + PathToPublish: $(Build.SourcesDirectory)/artifacts/asset-manifests/ + ArtifactName: AssetManifests + ArtifactType: Container + + - task: PublishBuildArtifacts@1 + displayName: Upload publish logs + condition: always() + continueOnError: true + inputs: + PathToPublish: $(Build.SourcesDirectory)/artifacts/log/ + ArtifactName: Publishing logs + ArtifactType: Container + diff --git a/build.proj b/build.proj index 5a80d86843..81a6598f58 100644 --- a/build.proj +++ b/build.proj @@ -245,6 +245,10 @@ Overwrite="true" /> + + + + diff --git a/build.sh b/build.sh index d582864d41..850873b46e 100755 --- a/build.sh +++ b/build.sh @@ -53,6 +53,9 @@ for arg do (--generate-prebuilt-data) set -- "$@" "/t:GeneratePrebuiltBurndownData" alternateTarget=true ;; + (--collect-artifacts) set -- "$@" "/t:CopyArtifactsToOutputFolder" + alternateTarget=true + ;; (*) set -- "$@" "$arg" ;; esac done diff --git a/eng/Tools.props b/eng/Tools.props new file mode 100644 index 0000000000..1fbf39ed5e --- /dev/null +++ b/eng/Tools.props @@ -0,0 +1,2 @@ + + diff --git a/eng/Versions.props b/eng/Versions.props index 574d0c7c31..4938be4b8b 100644 --- a/eng/Versions.props +++ b/eng/Versions.props @@ -2,8 +2,11 @@ - 0.1.0 - alpha.1 + 3 + 1 + $(BUILD_BUILDNUMBER) + $(VersionSuffix)+$(BUILD_SOURCEVERSION) + $(VersionSuffix)+$(GitCommitHash) diff --git a/eng/common/PublishToSymbolServers.proj b/eng/common/PublishToSymbolServers.proj index 5d55e312b0..311e2bbe0f 100644 --- a/eng/common/PublishToSymbolServers.proj +++ b/eng/common/PublishToSymbolServers.proj @@ -37,6 +37,8 @@ 3650 true + true + true false @@ -56,7 +58,7 @@ DryRun="false" ConvertPortablePdbsToWindowsPdbs="false" PdbConversionTreatAsWarning="" - Condition="$(PublishToSymbolServer)"/> + Condition="$(PublishToSymbolServer) and $(PublishToMSDL)"/> $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --partition-count $(PartitionCount) --partition-index %(HelixWorkItem.Index)" + $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults);$(FinalCommand) 4:00 + $(WorkItemDirectory) - $(WorkItemCommand) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument) --artifacts $(ArtifactsDirectory)" + $(WorkItemCommand) --bdn-artifacts $(BaselineArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(BaselineCoreRunArgument)" + $(WorkItemCommand) --bdn-artifacts $(ArtifactsDirectory) --bdn-arguments="--anyCategories $(BDNCategories) $(ExtraBenchmarkDotNetArguments) $(CoreRunArgument)" + $(DotnetExe) run -f $(_Framework) -p $(ResultsComparer) --base $(BaselineArtifactsDirectory) --diff $(ArtifactsDirectory) --threshold 2$(Percent) --xml $(XMLResults) 4:00 diff --git a/eng/common/performance/performance-setup.ps1 b/eng/common/performance/performance-setup.ps1 index 4a6706b638..ec41965fc8 100644 --- a/eng/common/performance/performance-setup.ps1 +++ b/eng/common/performance/performance-setup.ps1 @@ -1,6 +1,7 @@ Param( [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, [string] $CoreRootDirectory, + [string] $BaselineCoreRootDirectory, [string] $Architecture="x64", [string] $Framework="netcoreapp5.0", [string] $CompilationMode="Tiered", @@ -12,11 +13,13 @@ Param( [string] $Csproj="src\benchmarks\micro\MicroBenchmarks.csproj", [string] $Kind="micro", [switch] $Internal, + [switch] $Compare, [string] $Configurations="CompilationMode=$CompilationMode" ) -$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") +$RunFromPerformanceRepo = ($Repository -eq "dotnet/performance") -or ($Repository -eq "dotnet-performance") $UseCoreRun = ($CoreRootDirectory -ne [string]::Empty) +$UseBaselineCoreRun = ($BaselineCoreRootDirectory -ne [string]::Empty) $PayloadDirectory = (Join-Path $SourceDirectory "Payload") $PerformanceDirectory = (Join-Path $PayloadDirectory "performance") @@ -29,7 +32,13 @@ $HelixSourcePrefix = "pr" $Queue = "Windows.10.Amd64.ClientRS4.DevEx.15.8.Open" if ($Framework.StartsWith("netcoreapp")) { - $Queue = "Windows.10.Amd64.ClientRS4.Open" + $Queue = "Windows.10.Amd64.ClientRS5.Open" +} + +if ($Compare) { + $Queue = "Windows.10.Amd64.19H1.Tiger.Perf.Open" + $PerfLabArguments = "" + $ExtraBenchmarkDotNetArguments = "" } if ($Internal) { @@ -56,6 +65,10 @@ if ($UseCoreRun) { $NewCoreRoot = (Join-Path $PayloadDirectory "Core_Root") Move-Item -Path $CoreRootDirectory -Destination $NewCoreRoot } +if ($UseBaselineCoreRun) { + $NewBaselineCoreRoot = (Join-Path $PayloadDirectory "Baseline_Core_Root") + Move-Item -Path $BaselineCoreRootDirectory -Destination $NewBaselineCoreRoot +} $DocsDir = (Join-Path $PerformanceDirectory "docs") robocopy $DocsDir $WorkItemDirectory @@ -80,7 +93,9 @@ Write-PipelineSetVariable -Name 'TargetCsproj' -Value "$Csproj" -IsMultiJobVaria Write-PipelineSetVariable -Name 'Kind' -Value "$Kind" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'Architecture' -Value "$Architecture" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'UseCoreRun' -Value "$UseCoreRun" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'UseBaselineCoreRun' -Value "$UseBaselineCoreRun" -IsMultiJobVariable $false Write-PipelineSetVariable -Name 'RunFromPerfRepo' -Value "$RunFromPerformanceRepo" -IsMultiJobVariable $false +Write-PipelineSetVariable -Name 'Compare' -Value "$Compare" -IsMultiJobVariable $false # Helix Arguments Write-PipelineSetVariable -Name 'Creator' -Value "$Creator" -IsMultiJobVariable $false diff --git a/eng/common/performance/performance-setup.sh b/eng/common/performance/performance-setup.sh index 76126b1f86..2f2092166e 100755 --- a/eng/common/performance/performance-setup.sh +++ b/eng/common/performance/performance-setup.sh @@ -2,6 +2,7 @@ source_directory=$BUILD_SOURCESDIRECTORY core_root_directory= +baseline_core_root_directory= architecture=x64 framework=netcoreapp5.0 compilation_mode=tiered @@ -10,12 +11,14 @@ branch=$BUILD_SOURCEBRANCH commit_sha=$BUILD_SOURCEVERSION build_number=$BUILD_BUILDNUMBER internal=false +compare=false kind="micro" run_categories="coreclr corefx" csproj="src\benchmarks\micro\MicroBenchmarks.csproj" configurations= run_from_perf_repo=false use_core_run=true +use_baseline_core_run=true while (($# > 0)); do lowerI="$(echo $1 | awk '{print tolower($0)}')" @@ -28,6 +31,10 @@ while (($# > 0)); do core_root_directory=$2 shift 2 ;; + --baselinecorerootdirectory) + baseline_core_root_directory=$2 + shift 2 + ;; --architecture) architecture=$2 shift 2 @@ -72,6 +79,10 @@ while (($# > 0)); do internal=true shift 1 ;; + --compare) + compare=true + shift 1 + ;; --configurations) configurations=$2 shift 2 @@ -102,7 +113,7 @@ while (($# > 0)); do esac done -if [[ "$repository" == "dotnet/performance" ]]; then +if [ "$repository" == "dotnet/performance" ] || [ "$repository" == "dotnet-performance" ]; then run_from_perf_repo=true fi @@ -114,6 +125,10 @@ if [ -z "$core_root_directory" ]; then use_core_run=false fi +if [ -z "$baseline_core_root_directory" ]; then + use_baseline_core_run=false +fi + payload_directory=$source_directory/Payload performance_directory=$payload_directory/performance workitem_directory=$source_directory/workitem @@ -123,6 +138,19 @@ queue=Ubuntu.1804.Amd64.Open creator=$BUILD_DEFINITIONNAME helix_source_prefix="pr" +if [[ "$compare" == true ]]; then + extra_benchmark_dotnet_arguments= + perflab_arguments= + + # No open queues for arm64 + if [[ "$architecture" = "arm64" ]]; then + echo "Compare not available for arm64" + exit 1 + fi + + queue=Ubuntu.1804.Amd64.Tiger.Perf.Open +fi + if [[ "$internal" == true ]]; then perflab_arguments="--upload-to-perflab-container" helix_source_prefix="official" @@ -156,6 +184,11 @@ if [[ "$use_core_run" = true ]]; then mv $core_root_directory $new_core_root fi +if [[ "$use_baseline_core_run" = true ]]; then + new_baseline_core_root=$payload_directory/Baseline_Core_Root + mv $baseline_core_root_directory $new_baseline_core_root +fi + ci=true _script_dir=$(pwd)/eng/common @@ -163,6 +196,7 @@ _script_dir=$(pwd)/eng/common # Make sure all of our variables are available for future steps Write-PipelineSetVariable -name "UseCoreRun" -value "$use_core_run" -is_multi_job_variable false +Write-PipelineSetVariable -name "UseBaselineCoreRun" -value "$use_baseline_core_run" -is_multi_job_variable false Write-PipelineSetVariable -name "Architecture" -value "$architecture" -is_multi_job_variable false Write-PipelineSetVariable -name "PayloadDirectory" -value "$payload_directory" -is_multi_job_variable false Write-PipelineSetVariable -name "PerformanceDirectory" -value "$performance_directory" -is_multi_job_variable false @@ -178,4 +212,5 @@ Write-PipelineSetVariable -name "RunFromPerfRepo" -value "$run_from_perf_repo" - Write-PipelineSetVariable -name "Creator" -value "$creator" -is_multi_job_variable false Write-PipelineSetVariable -name "HelixSourcePrefix" -value "$helix_source_prefix" -is_multi_job_variable false Write-PipelineSetVariable -name "Kind" -value "$kind" -is_multi_job_variable false -Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false \ No newline at end of file +Write-PipelineSetVariable -name "_BuildConfig" -value "$architecture.$kind.$framework" -is_multi_job_variable false +Write-PipelineSetVariable -name "Compare" -value "$compare" -is_multi_job_variable false diff --git a/eng/common/post-build/post-build-utils.ps1 b/eng/common/post-build/post-build-utils.ps1 index 551ae113f8..aa9a025c0b 100644 --- a/eng/common/post-build/post-build-utils.ps1 +++ b/eng/common/post-build/post-build-utils.ps1 @@ -8,6 +8,7 @@ Set-StrictMode -Version 2.0 # scripts don't necessarily execute in the same agent that run the # build.ps1/sh script this variable isn't automatically set. $ci = $true +$disableConfigureToolsetImport = "true" . $PSScriptRoot\..\tools.ps1 function Create-MaestroApiRequestHeaders([string]$ContentType = "application/json") { diff --git a/eng/common/post-build/sourcelink-validation.ps1 b/eng/common/post-build/sourcelink-validation.ps1 index 41e01ae6e6..bbfdacca13 100644 --- a/eng/common/post-build/sourcelink-validation.ps1 +++ b/eng/common/post-build/sourcelink-validation.ps1 @@ -1,8 +1,8 @@ param( [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation - [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade - [Parameter(Mandatory=$true)][string] $GHCommit, # GitHub commit SHA used to build the packages + [Parameter(Mandatory=$false)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$false)][string] $GHCommit, # GitHub commit SHA used to build the packages [Parameter(Mandatory=$true)][string] $SourcelinkCliVersion # Version of SourceLink CLI to use ) @@ -13,6 +13,12 @@ param( # all files present in the repo at a specific commit point. $global:RepoFiles = @{} +# Maximum number of jobs to run in parallel +$MaxParallelJobs = 6 + +# Wait time between check for system load +$SecondsBetweenLoadChecks = 10 + $ValidatePackage = { param( [string] $PackagePath # Full path to a Symbols.NuGet package @@ -22,8 +28,8 @@ $ValidatePackage = { # Ensure input file exist if (!(Test-Path $PackagePath)) { - Write-PipelineTaskError "Input file does not exist: $PackagePath" - ExitWithExitCode 1 + Write-Host "Input file does not exist: $PackagePath" + return 1 } # Extensions for which we'll look for SourceLink information @@ -38,7 +44,7 @@ $ValidatePackage = { Add-Type -AssemblyName System.IO.Compression.FileSystem - [System.IO.Directory]::CreateDirectory($ExtractPath); + [System.IO.Directory]::CreateDirectory($ExtractPath) | Out-Null try { $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) @@ -138,16 +144,18 @@ $ValidatePackage = { if ($FailedFiles -eq 0) { Write-Host "Passed." + return 0 } else { - Write-PipelineTaskError "$PackagePath has broken SourceLink links." + Write-Host "$PackagePath has broken SourceLink links." + return 1 } } function ValidateSourceLinkLinks { - if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + if ($GHRepoName -ne "" -and !($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { if (!($GHRepoName -Match "^[^\s-]+-[^\s]+$")) { - Write-PipelineTaskError "GHRepoName should be in the format / or -" + Write-PipelineTaskError "GHRepoName should be in the format / or -. '$GHRepoName'" ExitWithExitCode 1 } else { @@ -155,30 +163,33 @@ function ValidateSourceLinkLinks { } } - if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { - Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string" + if ($GHCommit -ne "" -and !($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-PipelineTaskError "GHCommit should be a 40 chars hexadecimal string. '$GHCommit'" ExitWithExitCode 1 } - $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") - $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + if ($GHRepoName -ne "" -and $GHCommit -ne "") { + $RepoTreeURL = -Join("http://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") - try { - # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash - $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL -UseBasicParsing | ConvertFrom-Json | Select-Object -ExpandProperty tree - foreach ($file in $Data) { - $Extension = [System.IO.Path]::GetExtension($file.path) + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) - if ($CodeExtensions.Contains($Extension)) { - $RepoFiles[$file.path] = 1 + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } } } + catch { + Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL . Execution will proceed without caching." + } } - catch { - Write-PipelineTaskError "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" - Write-Host $_ - ExitWithExitCode 1 + elseif ($GHRepoName -ne "" -or $GHCommit -ne "") { + Write-Host "For using the http caching mechanism both GHRepoName and GHCommit should be informed." } if (Test-Path $ExtractPath) { @@ -186,14 +197,33 @@ function ValidateSourceLinkLinks { } # Process each NuGet package in parallel - $Jobs = @() Get-ChildItem "$InputPath\*.symbols.nupkg" | ForEach-Object { - $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName | Out-Null + $NumJobs = @(Get-Job -State 'Running').Count + + while ($NumJobs -ge $MaxParallelJobs) { + Write-Host "There are $NumJobs validation jobs running right now. Waiting $SecondsBetweenLoadChecks seconds to check again." + sleep $SecondsBetweenLoadChecks + $NumJobs = @(Get-Job -State 'Running').Count + } + + foreach ($Job in @(Get-Job -State 'Completed')) { + Receive-Job -Id $Job.Id + Remove-Job -Id $Job.Id + } } - foreach ($Job in $Jobs) { - Wait-Job -Id $Job.Id | Receive-Job + $ValidationFailures = 0 + foreach ($Job in @(Get-Job)) { + $jobResult = Wait-Job -Id $Job.Id | Receive-Job + if ($jobResult -ne "0") { + $ValidationFailures++ + } + } + if ($ValidationFailures -gt 0) { + Write-PipelineTaskError " $ValidationFailures package(s) failed validation." + ExitWithExitCode 1 } } diff --git a/eng/common/post-build/symbols-validation.ps1 b/eng/common/post-build/symbols-validation.ps1 index d5ec51b150..096ac321d1 100644 --- a/eng/common/post-build/symbols-validation.ps1 +++ b/eng/common/post-build/symbols-validation.ps1 @@ -37,10 +37,10 @@ function FirstMatchingSymbolDescriptionOrDefault { # DWARF file for a .dylib $DylibDwarf = $SymbolPath.Replace($Extension, ".dylib.dwarf") - $dotnetsymbolExe = "$env:USERPROFILE\.dotnet\tools" - $dotnetsymbolExe = Resolve-Path "$dotnetsymbolExe\dotnet-symbol.exe" + $dotnetSymbolExe = "$env:USERPROFILE\.dotnet\tools" + $dotnetSymbolExe = Resolve-Path "$dotnetSymbolExe\dotnet-symbol.exe" - & $dotnetsymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null + & $dotnetSymbolExe --symbols --modules --windows-pdbs $TargetServerParam $FullPath -o $SymbolsPath | Out-Null if (Test-Path $PdbPath) { return "PDB" @@ -159,25 +159,25 @@ function CheckSymbolsAvailable { } } -function Installdotnetsymbol { - $dotnetsymbolPackageName = "dotnet-symbol" +function InstallDotnetSymbol { + $dotnetSymbolPackageName = "dotnet-symbol" $dotnetRoot = InitializeDotNetCli -install:$true $dotnet = "$dotnetRoot\dotnet.exe" $toolList = & "$dotnet" tool list --global - if (($toolList -like "*$dotnetsymbolPackageName*") -and ($toolList -like "*$dotnetsymbolVersion*")) { - Write-Host "dotnet-symbol version $dotnetsymbolVersion is already installed." + if (($toolList -like "*$dotnetSymbolPackageName*") -and ($toolList -like "*$dotnetSymbolVersion*")) { + Write-Host "dotnet-symbol version $dotnetSymbolVersion is already installed." } else { - Write-Host "Installing dotnet-symbol version $dotnetsymbolVersion..." + Write-Host "Installing dotnet-symbol version $dotnetSymbolVersion..." Write-Host "You may need to restart your command window if this is the first dotnet tool you have installed." - & "$dotnet" tool install $dotnetsymbolPackageName --version $dotnetsymbolVersion --verbosity "minimal" --global + & "$dotnet" tool install $dotnetSymbolPackageName --version $dotnetSymbolVersion --verbosity "minimal" --global } } try { - Installdotnetsymbol + InstallDotnetSymbol CheckSymbolsAvailable } diff --git a/eng/common/sdl/execute-all-sdl-tools.ps1 b/eng/common/sdl/execute-all-sdl-tools.ps1 index 6d9bdcf72b..eb21321ba2 100644 --- a/eng/common/sdl/execute-all-sdl-tools.ps1 +++ b/eng/common/sdl/execute-all-sdl-tools.ps1 @@ -1,30 +1,30 @@ Param( - [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) - [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) - [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified - [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) - [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master - [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located - [string] $ArtifactsDirectory = (Join-Path $env:BUILD_SOURCESDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located - [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault - [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code - [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts - [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. - [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. - [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) - [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed - [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. - [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. - [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. - [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. - [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. - [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error - [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") - [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") + [string] $GuardianPackageName, # Required: the name of guardian CLI package (not needed if GuardianCliLocation is specified) + [string] $NugetPackageDirectory, # Required: directory where NuGet packages are installed (not needed if GuardianCliLocation is specified) + [string] $GuardianCliLocation, # Optional: Direct location of Guardian CLI executable if GuardianPackageName & NugetPackageDirectory are not specified + [string] $Repository=$env:BUILD_REPOSITORY_NAME, # Required: the name of the repository (e.g. dotnet/arcade) + [string] $BranchName=$env:BUILD_SOURCEBRANCH, # Optional: name of branch or version of gdn settings; defaults to master + [string] $SourceDirectory=$env:BUILD_SOURCESDIRECTORY, # Required: the directory where source files are located + [string] $ArtifactsDirectory = (Join-Path $env:BUILD_ARTIFACTSTAGINGDIRECTORY ("artifacts")), # Required: the directory where build artifacts are located + [string] $AzureDevOpsAccessToken, # Required: access token for dnceng; should be provided via KeyVault + [string[]] $SourceToolsList, # Optional: list of SDL tools to run on source code + [string[]] $ArtifactToolsList, # Optional: list of SDL tools to run on built artifacts + [bool] $TsaPublish=$False, # Optional: true will publish results to TSA; only set to true after onboarding to TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaBranchName=$env:BUILD_SOURCEBRANCH, # Optional: required for TSA publish; defaults to $(Build.SourceBranchName); TSA is the automated framework used to upload test results as bugs. + [string] $TsaRepositoryName=$env:BUILD_REPOSITORY_NAME, # Optional: TSA repository name; will be generated automatically if not submitted; TSA is the automated framework used to upload test results as bugs. + [string] $BuildNumber=$env:BUILD_BUILDNUMBER, # Optional: required for TSA publish; defaults to $(Build.BuildNumber) + [bool] $UpdateBaseline=$False, # Optional: if true, will update the baseline in the repository; should only be run after fixing any issues which need to be fixed + [bool] $TsaOnboard=$False, # Optional: if true, will onboard the repository to TSA; should only be run once; TSA is the automated framework used to upload test results as bugs. + [string] $TsaInstanceUrl, # Optional: only needed if TsaOnboard or TsaPublish is true; the instance-url registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the codebase registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaProjectName, # Optional: only needed if TsaOnboard or TsaPublish is true; the name of the project registered with TSA; TSA is the automated framework used to upload test results as bugs. + [string] $TsaNotificationEmail, # Optional: only needed if TsaOnboard is true; the email(s) which will receive notifications of TSA bug filings (e.g. alias@microsoft.com); TSA is the automated framework used to upload test results as bugs. + [string] $TsaCodebaseAdmin, # Optional: only needed if TsaOnboard is true; the aliases which are admins of the TSA codebase (e.g. DOMAIN\alias); TSA is the automated framework used to upload test results as bugs. + [string] $TsaBugAreaPath, # Optional: only needed if TsaOnboard is true; the area path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $TsaIterationPath, # Optional: only needed if TsaOnboard is true; the iteration path where TSA will file bugs in AzDO; TSA is the automated framework used to upload test results as bugs. + [string] $GuardianLoggerLevel="Standard", # Optional: the logger level for the Guardian CLI; options are Trace, Verbose, Standard, Warning, and Error + [string[]] $CrScanAdditionalRunConfigParams, # Optional: Additional Params to custom build a CredScan run config in the format @("xyz:abc","sdf:1") + [string[]] $PoliCheckAdditionalRunConfigParams # Optional: Additional Params to custom build a Policheck run config in the format @("xyz:abc","sdf:1") ) $ErrorActionPreference = "Stop" @@ -45,6 +45,7 @@ if ($GuardianPackageName) { $guardianCliLocation = $GuardianCliLocation } +$workingDirectory = (Split-Path $SourceDirectory -Parent) $ValidPath = Test-Path $guardianCliLocation if ($ValidPath -eq $False) @@ -53,13 +54,13 @@ if ($ValidPath -eq $False) exit 1 } -& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $ArtifactsDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel -$gdnFolder = Join-Path $ArtifactsDirectory ".gdn" +& $(Join-Path $PSScriptRoot "init-sdl.ps1") -GuardianCliLocation $guardianCliLocation -Repository $RepoName -BranchName $BranchName -WorkingDirectory $workingDirectory -AzureDevOpsAccessToken $AzureDevOpsAccessToken -GuardianLoggerLevel $GuardianLoggerLevel +$gdnFolder = Join-Path $workingDirectory ".gdn" if ($TsaOnboard) { if ($TsaCodebaseName -and $TsaNotificationEmail -and $TsaCodebaseAdmin -and $TsaBugAreaPath) { - Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-onboard --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-onboard --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-onboard failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -71,10 +72,10 @@ if ($TsaOnboard) { } if ($ArtifactToolsList -and $ArtifactToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $ArtifactsDirectory -GdnFolder $gdnFolder -ToolsList $ArtifactToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($SourceToolsList -and $SourceToolsList.Count -gt 0) { - & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $ArtifactsDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams + & $(Join-Path $PSScriptRoot "run-sdl.ps1") -GuardianCliLocation $guardianCliLocation -WorkingDirectory $workingDirectory -TargetDirectory $SourceDirectory -GdnFolder $gdnFolder -ToolsList $SourceToolsList -AzureDevOpsAccessToken $AzureDevOpsAccessToken -UpdateBaseline $UpdateBaseline -GuardianLoggerLevel $GuardianLoggerLevel -CrScanAdditionalRunConfigParams $CrScanAdditionalRunConfigParams -PoliCheckAdditionalRunConfigParams $PoliCheckAdditionalRunConfigParams } if ($UpdateBaseline) { @@ -86,8 +87,8 @@ if ($TsaPublish) { if (-not $TsaRepositoryName) { $TsaRepositoryName = "$($Repository)-$($BranchName)" } - Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel" - & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $ArtifactsDirectory --logger-level $GuardianLoggerLevel + Write-Host "$guardianCliLocation tsa-publish --all-tools --repository-name `"$TsaRepositoryName`" --branch-name `"$TsaBranchName`" --build-number `"$BuildNumber`" --codebase-name `"$TsaCodebaseName`" --notification-alias `"$TsaNotificationEmail`" --codebase-admin `"$TsaCodebaseAdmin`" --instance-url `"$TsaInstanceUrl`" --project-name `"$TsaProjectName`" --area-path `"$TsaBugAreaPath`" --iteration-path `"$TsaIterationPath`" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel" + & $guardianCliLocation tsa-publish --all-tools --repository-name "$TsaRepositoryName" --branch-name "$TsaBranchName" --build-number "$BuildNumber" --onboard $True --codebase-name "$TsaCodebaseName" --notification-alias "$TsaNotificationEmail" --codebase-admin "$TsaCodebaseAdmin" --instance-url "$TsaInstanceUrl" --project-name "$TsaProjectName" --area-path "$TsaBugAreaPath" --iteration-path "$TsaIterationPath" --working-directory $workingDirectory --logger-level $GuardianLoggerLevel if ($LASTEXITCODE -ne 0) { Write-Host "Guardian tsa-publish failed with exit code $LASTEXITCODE." exit $LASTEXITCODE diff --git a/eng/common/sdl/extract-artifact-packages.ps1 b/eng/common/sdl/extract-artifact-packages.ps1 index 1fdbb14329..d857ae2197 100644 --- a/eng/common/sdl/extract-artifact-packages.ps1 +++ b/eng/common/sdl/extract-artifact-packages.ps1 @@ -5,6 +5,14 @@ param( $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 + +# `tools.ps1` checks $ci to perform some actions. Since the post-build +# scripts don't necessarily execute in the same agent that run the +# build.ps1/sh script this variable isn't automatically set. +$ci = $true +$disableConfigureToolsetImport = "true" +. $PSScriptRoot\..\tools.ps1 + $ExtractPackage = { param( [string] $PackagePath # Full path to a NuGet package diff --git a/eng/common/sdl/init-sdl.ps1 b/eng/common/sdl/init-sdl.ps1 index 26e01c0673..c737eb0e71 100644 --- a/eng/common/sdl/init-sdl.ps1 +++ b/eng/common/sdl/init-sdl.ps1 @@ -11,6 +11,9 @@ $ErrorActionPreference = "Stop" Set-StrictMode -Version 2.0 $LASTEXITCODE = 0 +# Don't display the console progress UI - it's a huge perf hit +$ProgressPreference = 'SilentlyContinue' + # Construct basic auth from AzDO access token; construct URI to the repository's gdn folder stored in that repository; construct location of zip file $encodedPat = [Convert]::ToBase64String([System.Text.Encoding]::ASCII.GetBytes(":$AzureDevOpsAccessToken")) $escapedRepository = [Uri]::EscapeDataString("/$Repository/$BranchName/.gdn") diff --git a/eng/common/sdl/packages.config b/eng/common/sdl/packages.config index 3f97ac2f16..256ffbfb93 100644 --- a/eng/common/sdl/packages.config +++ b/eng/common/sdl/packages.config @@ -1,4 +1,4 @@ - + - + diff --git a/eng/common/sdl/run-sdl.ps1 b/eng/common/sdl/run-sdl.ps1 index b90c4399ba..9bc25314ae 100644 --- a/eng/common/sdl/run-sdl.ps1 +++ b/eng/common/sdl/run-sdl.ps1 @@ -32,16 +32,16 @@ foreach ($tool in $ToolsList) { Write-Host $tool # We have to manually configure tools that run on source to look at the source directory only if ($tool -eq "credscan") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory : $TargetDirectory `" `" OutputType : pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory : $TargetDirectory " "OutputType : pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" TargetDirectory < $TargetDirectory `" `" OutputType < pre `" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " TargetDirectory < $TargetDirectory " "OutputType < pre" $(If ($CrScanAdditionalRunConfigParams) {$CrScanAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE } } if ($tool -eq "policheck") { - Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target : $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" - & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target : $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) + Write-Host "$GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args `" Target < $TargetDirectory `" $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams})" + & $GuardianCliLocation configure --working-directory $WorkingDirectory --tool $tool --output-path $gdnConfigFile --logger-level $GuardianLoggerLevel --noninteractive --force --args " Target < $TargetDirectory " $(If ($PoliCheckAdditionalRunConfigParams) {$PoliCheckAdditionalRunConfigParams}) if ($LASTEXITCODE -ne 0) { Write-Host "Guardian configure for $tool failed with exit code $LASTEXITCODE." exit $LASTEXITCODE @@ -56,4 +56,4 @@ Write-Host "$GuardianCliLocation run --working-directory $WorkingDirectory --bas if ($LASTEXITCODE -ne 0) { Write-Host "Guardian run for $ToolsList using $configParam failed with exit code $LASTEXITCODE." exit $LASTEXITCODE -} \ No newline at end of file +} diff --git a/eng/common/templates/job/execute-sdl.yml b/eng/common/templates/job/execute-sdl.yml index 91621cf88f..2973bcaf3a 100644 --- a/eng/common/templates/job/execute-sdl.yml +++ b/eng/common/templates/job/execute-sdl.yml @@ -1,8 +1,16 @@ parameters: overrideParameters: '' # Optional: to override values for parameters. additionalParameters: '' # Optional: parameters that need user specific values eg: '-SourceToolsList @("abc","def") -ArtifactToolsList @("ghi","jkl")' - continueOnError: false # optional: determines whether to continue the build if the step errors; + # There is some sort of bug (has been reported) in Azure DevOps where if this parameter is named + # 'continueOnError', the parameter value is not correctly picked up. + # This can also be remedied by the caller (post-build.yml) if it does not use a nested parameter + sdlContinueOnError: false # optional: determines whether to continue the build if the step errors; dependsOn: '' # Optional: dependencies of the job + artifactNames: '' # Optional: patterns supplied to DownloadBuildArtifacts + # Usage: + # artifactNames: + # - 'BlobArtifacts' + # - 'Artifacts_Windows_NT_Release' jobs: - job: Run_SDL @@ -15,23 +23,32 @@ jobs: steps: - checkout: self clean: true - - task: DownloadBuildArtifacts@0 - displayName: Download Build Artifacts - inputs: - buildType: current - downloadType: specific files - matchingPattern: "**" - downloadPath: $(Build.SourcesDirectory)\artifacts + - ${{ if ne(parameters.artifactNames, '') }}: + - ${{ each artifactName in parameters.artifactNames }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + artifactName: ${{ artifactName }} + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts + - ${{ if eq(parameters.artifactNames, '') }}: + - task: DownloadBuildArtifacts@0 + displayName: Download Build Artifacts + inputs: + buildType: current + downloadType: specific files + itemPattern: "**" + downloadPath: $(Build.ArtifactStagingDirectory)\artifacts - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts - -ExtractPath $(Build.SourcesDirectory)\artifacts\BlobArtifacts + -InputPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\BlobArtifacts displayName: Extract Blob Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - powershell: eng/common/sdl/extract-artifact-packages.ps1 - -InputPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts - -ExtractPath $(Build.SourcesDirectory)\artifacts\PackageArtifacts + -InputPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts + -ExtractPath $(Build.ArtifactStagingDirectory)\artifacts\PackageArtifacts displayName: Extract Package Artifacts - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - task: NuGetToolInstaller@1 displayName: 'Install NuGet.exe' - task: NuGetCommand@2 @@ -45,12 +62,12 @@ jobs: - ${{ if ne(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 ${{ parameters.overrideParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} - ${{ if eq(parameters.overrideParameters, '') }}: - powershell: eng/common/sdl/execute-all-sdl-tools.ps1 - -GuardianPackageName Microsoft.Guardian.Cli.0.7.1 + -GuardianPackageName Microsoft.Guardian.Cli.0.7.2 -NugetPackageDirectory $(Build.SourcesDirectory)\.packages -AzureDevOpsAccessToken $(dn-bot-dotnet-build-rw-code-rw) ${{ parameters.additionalParameters }} displayName: Execute SDL - continueOnError: ${{ parameters.continueOnError }} + continueOnError: ${{ parameters.sdlContinueOnError }} diff --git a/eng/common/templates/job/generate-graph-files.yml b/eng/common/templates/job/generate-graph-files.yml index e54ce956f9..2692893de6 100644 --- a/eng/common/templates/job/generate-graph-files.yml +++ b/eng/common/templates/job/generate-graph-files.yml @@ -20,8 +20,6 @@ jobs: variables: # Publish-Build-Assets provides: MaestroAccessToken, BotAccount-dotnet-maestro-bot-PAT # DotNet-AllOrgs-Darc-Pats provides: dn-bot-devdiv-dnceng-rw-code-pat - - group: Publish-Build-Assets - - group: DotNet-AllOrgs-Darc-Pats - name: _GraphArguments value: -gitHubPat $(BotAccount-dotnet-maestro-bot-PAT) -azdoPat $(dn-bot-devdiv-dnceng-rw-code-pat) diff --git a/eng/common/templates/job/job.yml b/eng/common/templates/job/job.yml index 8db456bb7f..ecebd0f03e 100644 --- a/eng/common/templates/job/job.yml +++ b/eng/common/templates/job/job.yml @@ -1,67 +1,33 @@ +# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, +# and some (Microbuild) should only be applied to non-PR cases for internal builds. + parameters: # Job schema parameters - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job cancelTimeoutInMinutes: '' - condition: '' - - continueOnError: false - container: '' - + continueOnError: false dependsOn: '' - displayName: '' - - steps: [] - pool: '' - + steps: [] strategy: '' - timeoutInMinutes: '' - variables: [] - workspace: '' # Job base template specific parameters - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' + # See schema documentation - https://github.com/dotnet/arcade/blob/master/Documentation/AzureDevOps/TemplateSchema.md + artifacts: '' enableMicrobuild: false - - # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false - - # Optional: Enable publishing to the build asset registry enablePublishBuildAssets: false - - # Optional: Prevent gather/push manifest from executing when using publishing pipelines - enablePublishUsingPipelines: false - - # Optional: Include PublishTestResults task enablePublishTestResults: false - - # Optional: enable sending telemetry - enableTelemetry: false - - # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') - helixRepo: '' - - # Optional: define the helix type for telemetry (example: 'build/product/') - helixType: '' - - # Required: name of the job + enablePublishUsingPipelines: false name: '' - - # Optional: should run as a public build even in the internal project - # if 'true', the build won't run any of the internal only steps, even if it is running in non-public projects. + preSteps: [] runAsPublic: false -# Internal resources (telemetry, microbuild) can only be accessed from non-public projects, -# and some (Microbuild) should only be applied to non-PR cases for internal builds. - jobs: - job: ${{ parameters.name }} @@ -93,7 +59,7 @@ jobs: timeoutInMinutes: ${{ parameters.timeoutInMinutes }} variables: - - ${{ if eq(parameters.enableTelemetry, 'true') }}: + - ${{ if ne(parameters.enableTelemetry, 'false') }}: - name: DOTNET_CLI_TELEMETRY_PROFILE value: '$(Build.Repository.Uri)' - ${{ each variable in parameters.variables }}: @@ -125,21 +91,12 @@ jobs: workspace: ${{ parameters.workspace }} steps: - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - - task: sendStartTelemetry@0 - displayName: 'Send Helix Start Telemetry' - inputs: - helixRepo: ${{ parameters.helixRepo }} - ${{ if ne(parameters.helixType, '') }}: - helixType: ${{ parameters.helixType }} - buildConfig: $(_BuildConfig) - runAsPublic: ${{ parameters.runAsPublic }} - continueOnError: ${{ parameters.continueOnError }} - condition: always() + - ${{ if ne(parameters.preSteps, '') }}: + - ${{ each preStep in parameters.preSteps }}: + - ${{ preStep }} - - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if eq(parameters.enableMicrobuild, 'true') }}: - task: MicroBuildSigningPlugin@2 displayName: Install MicroBuild plugin inputs: @@ -151,6 +108,16 @@ jobs: continueOnError: ${{ parameters.continueOnError }} condition: and(succeeded(), in(variables['_SignType'], 'real', 'test'), eq(variables['Agent.Os'], 'Windows_NT')) + - task: NuGetAuthenticate@0 + + - ${{ if or(eq(parameters.artifacts.download, 'true'), ne(parameters.artifacts.download, '')) }}: + - task: DownloadPipelineArtifact@2 + inputs: + buildType: current + artifactName: ${{ coalesce(parameters.artifacts.download.name, 'Artifacts_$(Agent.OS)_$(_BuildConfig)') }} + targetPath: ${{ coalesce(parameters.artifacts.download.path, 'artifacts') }} + itemPattern: ${{ coalesce(parameters.artifacts.download.pattern, '**') }} + - ${{ each step in parameters.steps }}: - ${{ step }} @@ -163,20 +130,60 @@ jobs: env: TeamName: $(_TeamName) - - ${{ if eq(parameters.enableTelemetry, 'true') }}: - # Telemetry tasks are built from https://github.com/dotnet/arcade-extensions - - task: sendEndTelemetry@0 - displayName: 'Send Helix End Telemetry' - continueOnError: ${{ parameters.continueOnError }} - condition: always() - - - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: + - ${{ if ne(parameters.artifacts.publish, '') }}: + - ${{ if or(eq(parameters.artifacts.publish.artifacts, 'true'), ne(parameters.artifacts.publish.artifacts, '')) }}: + - task: CopyFiles@2 + displayName: Gather binaries for publish to artifacts + inputs: + SourceFolder: 'artifacts/bin' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/bin' + - task: CopyFiles@2 + displayName: Gather packages for publish to artifacts + inputs: + SourceFolder: 'artifacts/packages' + Contents: '**' + TargetFolder: '$(Build.ArtifactStagingDirectory)/artifacts/packages' + - task: PublishBuildArtifacts@1 + displayName: Publish pipeline artifacts + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/artifacts' + PublishLocation: Container + ArtifactName: ${{ coalesce(parameters.artifacts.publish.artifacts.name , 'Artifacts_$(Agent.Os)_$(_BuildConfig)') }} + continueOnError: true + condition: always() + - ${{ if or(eq(parameters.artifacts.publish.logs, 'true'), ne(parameters.artifacts.publish.logs, '')) }}: + - publish: artifacts/log + artifact: ${{ coalesce(parameters.artifacts.publish.logs.name, 'Logs_Build_$(Agent.Os)_$(_BuildConfig)') }} + displayName: Publish logs + continueOnError: true + condition: always() + - ${{ if or(eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - ${{ if and(ne(parameters.enablePublishUsingPipelines, 'true'), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: CopyFiles@2 + displayName: Gather Asset Manifests + inputs: + SourceFolder: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)/AssetManifest' + TargetFolder: '$(Build.ArtifactStagingDirectory)/AssetManifests' + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - task: PublishBuildArtifacts@1 + displayName: Push Asset Manifests + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/AssetManifests' + PublishLocation: Container + ArtifactName: AssetManifests + continueOnError: ${{ parameters.continueOnError }} + condition: and(succeeded(), eq(variables['_DotNetPublishToBlobFeed'], 'true')) + + - ${{ if ne(parameters.enablePublishBuildArtifacts, 'false') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs inputs: PathtoPublish: '$(Build.SourcesDirectory)/artifacts/log/$(_BuildConfig)' PublishLocation: Container - ArtifactName: $(Agent.Os)_$(Agent.JobName) + ArtifactName: ${{ coalesce(parameters.enablePublishBuildArtifacts.artifactName, '$(Agent.Os)_$(Agent.JobName)' ) }} continueOnError: true condition: always() diff --git a/eng/common/templates/job/performance.yml b/eng/common/templates/job/performance.yml index ef809253d1..f877fd7a89 100644 --- a/eng/common/templates/job/performance.yml +++ b/eng/common/templates/job/performance.yml @@ -5,6 +5,7 @@ parameters: displayName: '' # optional -- display name for the job. Will use jobName if not passed pool: '' # required -- name of the Build pool container: '' # required -- name of the container + osGroup: '' # required -- operating system for the job extraSetupParameters: '' # optional -- extra arguments to pass to the setup script frameworks: ['netcoreapp3.0'] # optional -- list of frameworks to run against continueOnError: 'false' # optional -- determines whether to continue the build if the step errors @@ -44,12 +45,13 @@ jobs: - HelixPreCommand: '' - ${{ if and(ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - ${{ if eq(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if eq( parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'set "PERFLAB_UPLOAD_TOKEN=$(PerfCommandUploadToken)"' - IsInternal: -Internal - - ${{ if ne(variables['Agent.Os'], 'Windows_NT') }}: + - ${{ if ne(parameters.osGroup, 'Windows_NT') }}: - HelixPreCommand: 'export PERFLAB_UPLOAD_TOKEN="$(PerfCommandUploadTokenLinux)"' - IsInternal: --internal + - group: DotNet-HelixApi-Access - group: dotnet-benchview diff --git a/eng/common/templates/job/publish-build-assets.yml b/eng/common/templates/job/publish-build-assets.yml index 9e77ef1b54..b722975f9c 100644 --- a/eng/common/templates/job/publish-build-assets.yml +++ b/eng/common/templates/job/publish-build-assets.yml @@ -47,6 +47,10 @@ jobs: downloadPath: '$(Build.StagingDirectory)/Download' condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + + - ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - task: NuGetAuthenticate@0 + - task: PowerShell@2 displayName: Publish Build Assets inputs: @@ -59,6 +63,7 @@ jobs: /p:Configuration=$(_BuildConfig) condition: ${{ parameters.condition }} continueOnError: ${{ parameters.continueOnError }} + - task: powershell@2 displayName: Create ReleaseConfigs Artifact inputs: @@ -67,12 +72,14 @@ jobs: Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(BARBuildId) Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value "$(DefaultChannels)" Add-Content -Path "$(Build.StagingDirectory)/ReleaseConfigs.txt" -Value $(IsStableBuild) + - task: PublishBuildArtifacts@1 displayName: Publish ReleaseConfigs Artifact inputs: PathtoPublish: '$(Build.StagingDirectory)/ReleaseConfigs.txt' PublishLocation: Container ArtifactName: ReleaseConfigs + - ${{ if eq(parameters.enablePublishBuildArtifacts, 'true') }}: - task: PublishBuildArtifacts@1 displayName: Publish Logs to VSTS diff --git a/eng/common/templates/jobs/jobs.yml b/eng/common/templates/jobs/jobs.yml index 6a2f98c036..c08225a9a9 100644 --- a/eng/common/templates/jobs/jobs.yml +++ b/eng/common/templates/jobs/jobs.yml @@ -1,19 +1,10 @@ parameters: - # Optional: 'true' if failures in job.yml job should not fail the job + # See schema documentation in /Documentation/AzureDevOps/TemplateSchema.md continueOnError: false - # Optional: Enable installing Microbuild plugin - # if 'true', these "variables" must be specified in the variables object or as part of the queue matrix - # _TeamName - the name of your team - # _SignType - 'test' or 'real' - enableMicrobuild: false - # Optional: Include PublishBuildArtifacts task enablePublishBuildArtifacts: false - # Optional: Enable publishing to the build asset registry - enablePublishBuildAssets: false - # Optional: Enable publishing using release pipelines enablePublishUsingPipelines: false @@ -23,19 +14,9 @@ parameters: # Optional: Include toolset dependencies in the generated graph files includeToolset: false - # Optional: Include PublishTestResults task - enablePublishTestResults: false - - # Optional: enable sending telemetry - # if enabled then the 'helixRepo' parameter should also be specified - enableTelemetry: false - # Required: A collection of jobs to run - https://docs.microsoft.com/en-us/azure/devops/pipelines/yaml-schema?view=vsts&tabs=schema#job jobs: [] - # Optional: define the helix repo for telemetry (example: 'dotnet/arcade') - helixRepo: '' - # Optional: Override automatically derived dependsOn value for "publish build assets" job publishBuildAssetsDependsOn: '' @@ -62,29 +43,30 @@ jobs: name: ${{ job.job }} -- ${{ if and(eq(parameters.enablePublishBuildAssets, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: ../job/publish-build-assets.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - dependsOn: - - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.publishBuildAssetsDependsOn }}: - - ${{ job.job }} - - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: - - ${{ each job in parameters.jobs }}: - - ${{ job.job }} - pool: - vmImage: vs2017-win2016 - runAsPublic: ${{ parameters.runAsPublic }} - publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} - enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} - -- ${{ if and(eq(parameters.graphFileGeneration.enabled, true), eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: - - template: ../job/generate-graph-files.yml - parameters: - continueOnError: ${{ parameters.continueOnError }} - includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} - dependsOn: - - Asset_Registry_Publish - pool: - vmImage: vs2017-win2016 +- ${{ if and(eq(parameters.runAsPublic, 'false'), ne(variables['System.TeamProject'], 'public'), notin(variables['Build.Reason'], 'PullRequest')) }}: + - ${{ if or(eq(parameters.enablePublishBuildAssets, true), eq(parameters.artifacts.publish.manifests, 'true'), ne(parameters.artifacts.publish.manifests, '')) }}: + - template: ../job/publish-build-assets.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + dependsOn: + - ${{ if ne(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.publishBuildAssetsDependsOn }}: + - ${{ job.job }} + - ${{ if eq(parameters.publishBuildAssetsDependsOn, '') }}: + - ${{ each job in parameters.jobs }}: + - ${{ job.job }} + pool: + vmImage: vs2017-win2016 + runAsPublic: ${{ parameters.runAsPublic }} + publishUsingPipelines: ${{ parameters.enablePublishUsingPipelines }} + enablePublishBuildArtifacts: ${{ parameters.enablePublishBuildArtifacts }} + + - ${{ if eq(parameters.graphFileGeneration.enabled, true) }}: + - template: ../job/generate-graph-files.yml + parameters: + continueOnError: ${{ parameters.continueOnError }} + includeToolset: ${{ parameters.graphFileGeneration.includeToolset }} + dependsOn: + - Asset_Registry_Publish + pool: + vmImage: vs2017-win2016 diff --git a/eng/common/templates/post-build/channels/generic-internal-channel.yml b/eng/common/templates/post-build/channels/generic-internal-channel.yml new file mode 100644 index 0000000000..3a8755fbbb --- /dev/null +++ b/eng/common/templates/post-build/channels/generic-internal-channel.yml @@ -0,0 +1,159 @@ +parameters: + artifactsPublishingAdditionalParameters: '' + dependsOn: + - Validate + publishInstallersAndChecksums: false + symbolPublishingAdditionalParameters: '' + stageName: '' + channelName: '' + channelId: '' + transportFeed: '' + shippingFeed: '' + symbolsFeed: '' + +stages: +- stage: ${{ parameters.stageName }} + dependsOn: ${{ parameters.dependsOn }} + variables: + - template: ../common-variables.yml + displayName: ${{ parameters.channelName }} Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: publish_symbols + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + /p:PublishToMSDL=false + ${{ parameters.symbolPublishingAdditionalParameters }} + + - template: ../../steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'SymbolPublishing' + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + continueOnError: true + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=true + /p:ChecksumsTargetStaticFeed=$(InternalChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(InternalChecksumsBlobFeedKey) + /p:InstallersTargetStaticFeed=$(InternalInstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(InternalInstallersBlobFeedKey) + /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:PublishToMSDL=false + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'AssetsPublishing' + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/channels/generic-public-channel.yml b/eng/common/templates/post-build/channels/generic-public-channel.yml new file mode 100644 index 0000000000..3f572f8b12 --- /dev/null +++ b/eng/common/templates/post-build/channels/generic-public-channel.yml @@ -0,0 +1,159 @@ +parameters: + artifactsPublishingAdditionalParameters: '' + dependsOn: + - Validate + publishInstallersAndChecksums: false + symbolPublishingAdditionalParameters: '' + stageName: '' + channelName: '' + channelId: '' + transportFeed: '' + shippingFeed: '' + symbolsFeed: '' + +stages: +- stage: ${{ parameters.stageName }} + dependsOn: ${{ parameters.dependsOn }} + variables: + - template: ../common-variables.yml + displayName: ${{ parameters.channelName }} Publishing + jobs: + - template: ../setup-maestro-vars.yml + + - job: publish_symbols + displayName: Symbol Publishing + dependsOn: setupMaestroVars + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }} )) + variables: + - group: DotNet-Symbol-Server-Pats + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + artifactName: 'BlobArtifacts' + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download PDB Artifacts + inputs: + artifactName: 'PDBArtifacts' + continueOnError: true + + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishToSymbolServers -restore -msbuildEngine dotnet + /p:DotNetSymbolServerTokenMsdl=$(microsoft-symbol-server-pat) + /p:DotNetSymbolServerTokenSymWeb=$(symweb-symbol-server-pat) + /p:PDBArtifactsDirectory='$(Build.ArtifactStagingDirectory)/PDBArtifacts/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:SymbolPublishingExclusionsFile='$(Build.SourcesDirectory)/eng/SymbolPublishingExclusionsFile.txt' + /p:Configuration=Release + ${{ parameters.symbolPublishingAdditionalParameters }} + + - template: ../../steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'SymbolPublishing' + + - job: publish_assets + displayName: Publish Assets + dependsOn: setupMaestroVars + variables: + - name: BARBuildId + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.BARBuildId'] ] + - name: IsStableBuild + value: $[ dependencies.setupMaestroVars.outputs['setReleaseVars.IsStableBuild'] ] + condition: contains(dependencies.setupMaestroVars.outputs['setReleaseVars.InitialChannels'], format('[{0}]', ${{ parameters.channelId }})) + pool: + vmImage: 'windows-2019' + steps: + - task: DownloadBuildArtifacts@0 + displayName: Download Package Artifacts + inputs: + buildType: current + artifactName: PackageArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Blob Artifacts + inputs: + buildType: current + artifactName: BlobArtifacts + continueOnError: true + + - task: DownloadBuildArtifacts@0 + displayName: Download Asset Manifests + inputs: + buildType: current + artifactName: AssetManifests + + - task: NuGetToolInstaller@1 + displayName: 'Install NuGet.exe' + + # This is necessary whenever we want to publish/restore to an AzDO private feed + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + + - task: PowerShell@2 + displayName: Publish Assets + inputs: + filePath: eng\common\sdk-task.ps1 + arguments: -task PublishArtifactsInManifest -restore -msbuildEngine dotnet + /p:ArtifactsCategory=$(_DotNetArtifactsCategory) + /p:IsStableBuild=$(IsStableBuild) + /p:IsInternalBuild=$(IsInternalBuild) + /p:RepositoryName=$(Build.Repository.Name) + /p:CommitSha=$(Build.SourceVersion) + /p:NugetPath=$(NuGetExeToolPath) + /p:AzdoTargetFeedPAT='$(dn-bot-dnceng-universal-packages-rw)' + /p:AzureStorageTargetFeedPAT='$(dotnetfeed-storage-access-key-1)' + /p:BARBuildId=$(BARBuildId) + /p:MaestroApiEndpoint='$(MaestroApiEndPoint)' + /p:BuildAssetRegistryToken='$(MaestroApiAccessToken)' + /p:ManifestsBasePath='$(Build.ArtifactStagingDirectory)/AssetManifests/' + /p:BlobBasePath='$(Build.ArtifactStagingDirectory)/BlobArtifacts/' + /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts/' + /p:Configuration=Release + /p:PublishInstallersAndChecksums=${{ parameters.publishInstallersAndChecksums }} + /p:InstallersTargetStaticFeed=$(InstallersBlobFeedUrl) + /p:InstallersAzureAccountKey=$(dotnetcli-storage-key) + /p:ChecksumsTargetStaticFeed=$(ChecksumsBlobFeedUrl) + /p:ChecksumsAzureAccountKey=$(dotnetclichecksums-storage-key) + /p:AzureDevOpsStaticShippingFeed='${{ parameters.shippingFeed }}' + /p:AzureDevOpsStaticShippingFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticTransportFeed='${{ parameters.transportFeed }}' + /p:AzureDevOpsStaticTransportFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + /p:AzureDevOpsStaticSymbolsFeed='${{ parameters.symbolsFeed }}' + /p:AzureDevOpsStaticSymbolsFeedKey='$(dn-bot-dnceng-artifact-feeds-rw)' + ${{ parameters.artifactsPublishingAdditionalParameters }} + + - template: ../../steps/publish-logs.yml + parameters: + StageLabel: '${{ parameters.stageName }}' + JobLabel: 'AssetsPublishing' + + - template: ../../steps/promote-build.yml + parameters: + ChannelId: ${{ parameters.channelId }} diff --git a/eng/common/templates/post-build/common-variables.yml b/eng/common/templates/post-build/common-variables.yml index b00d85d8ce..0a2c40c103 100644 --- a/eng/common/templates/post-build/common-variables.yml +++ b/eng/common/templates/post-build/common-variables.yml @@ -1,23 +1,34 @@ variables: - - group: Publish-Build-Assets + - group: AzureDevOps-Artifact-Feeds-Pats + - group: DotNet-Blob-Feed - group: DotNet-DotNetCli-Storage + - group: DotNet-MSRC-Storage + - group: Publish-Build-Assets - # .NET Core 3 Dev - - name: PublicDevRelease_30_Channel_Id - value: 3 + # .NET Core 3.1 Dev + - name: PublicDevRelease_31_Channel_Id + value: 128 # .NET Core 5 Dev - name: NetCore_5_Dev_Channel_Id value: 131 # .NET Tools - Validation - - name: PublicValidationRelease_30_Channel_Id + - name: NetCore_Tools_Validation_Channel_Id value: 9 # .NET Tools - Latest - name: NetCore_Tools_Latest_Channel_Id value: 2 + # .NET 3 Tools - Validation + - name: NETCore_3_Tools_Validation_Channel_Id + value: 390 + + # .NET 3 Tools - Latest + - name: NetCore_3_Tools_Channel_Id + value: 344 + # .NET Core 3.0 Internal Servicing - name: InternalServicing_30_Channel_Id value: 184 @@ -26,14 +37,22 @@ variables: - name: PublicRelease_30_Channel_Id value: 19 + # .NET Core 3.1 Release + - name: PublicRelease_31_Channel_Id + value: 129 + + # General Testing + - name: GeneralTesting_Channel_Id + value: 529 + + # .NET Core 3.1 Blazor Features + - name: NetCore_31_Blazor_Features_Channel_Id + value: 531 + # Whether the build is internal or not - name: IsInternalBuild value: ${{ and(ne(variables['System.TeamProject'], 'public'), contains(variables['Build.SourceBranch'], 'internal')) }} - # Storage account name for proxy-backed feeds - - name: ProxyBackedFeedsAccountName - value: dotnetfeed - # Default Maestro++ API Endpoint and API Version - name: MaestroApiEndPoint value: "https://maestro-prod.westus2.cloudapp.azure.com" @@ -47,8 +66,23 @@ variables: - name: SymbolToolVersion value: 1.0.1 + # Feed Configurations + # These should include the suffix "/index.json" + # Default locations for Installers and checksums + # Public Locations - name: ChecksumsBlobFeedUrl - value: https://dotnetcli.blob.core.windows.net/dotnet/index.json - - name: InstallersBlobFeedUrl value: https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json + - name: InstallersBlobFeedUrl + value: https://dotnetcli.blob.core.windows.net/dotnet/index.json + + # Private Locations + - name: InternalChecksumsBlobFeedUrl + value: https://dotnetclichecksumsmsrc.blob.core.windows.net/dotnet/index.json + - name: InternalChecksumsBlobFeedKey + value: $(dotnetclichecksumsmsrc-storage-key) + + - name: InternalInstallersBlobFeedUrl + value: https://dotnetclimsrc.blob.core.windows.net/dotnet/index.json + - name: InternalInstallersBlobFeedKey + value: $(dotnetclimsrc-access-key) diff --git a/eng/common/templates/post-build/post-build.yml b/eng/common/templates/post-build/post-build.yml index 29849c9dc6..5965cc2005 100644 --- a/eng/common/templates/post-build/post-build.yml +++ b/eng/common/templates/post-build/post-build.yml @@ -1,12 +1,14 @@ parameters: - enableSourceLinkValidation: true + enableSourceLinkValidation: false enableSigningValidation: true - enableSymbolValidation: true + enableSymbolValidation: false enableNugetValidation: true publishInstallersAndChecksums: false SDLValidationParameters: enable: false + continueOnError: false params: '' + artifactNames: '' # These parameters let the user customize the call to sdk-task.ps1 for publishing # symbols & general artifacts as well as for signing validation @@ -15,11 +17,14 @@ parameters: signingValidationAdditionalParameters: '' # Which stages should finish execution before post-build stages start - dependsOn: [build] + validateDependsOn: + - build + publishDependsOn: + - Validate stages: -- stage: validate - dependsOn: ${{ parameters.dependsOn }} +- stage: Validate + dependsOn: ${{ parameters.validateDependsOn }} displayName: Validate jobs: - ${{ if eq(parameters.enableNugetValidation, 'true') }}: @@ -44,6 +49,8 @@ stages: - ${{ if eq(parameters.enableSigningValidation, 'true') }}: - job: displayName: Signing Validation + variables: + - template: common-variables.yml pool: vmImage: 'windows-2019' steps: @@ -53,6 +60,18 @@ stages: buildType: current artifactName: PackageArtifacts + # This is necessary whenever we want to publish/restore to an AzDO private feed + # Since sdk-task.ps1 tries to restore packages we need to do this authentication here + # otherwise it'll complain about accessing a private feed. + - task: NuGetAuthenticate@0 + displayName: 'Authenticate to AzDO Feeds' + + - task: PowerShell@2 + displayName: Enable cross-org publishing + inputs: + filePath: eng\common\enable-cross-org-publishing.ps1 + arguments: -token $(dn-bot-dnceng-artifact-feeds-rw) + - task: PowerShell@2 displayName: Validate inputs: @@ -60,9 +79,13 @@ stages: arguments: -task SigningValidation -restore -msbuildEngine dotnet /p:PackageBasePath='$(Build.ArtifactStagingDirectory)/PackageArtifacts' /p:SignCheckExclusionsFile='$(Build.SourcesDirectory)/eng/SignCheckExclusionsFile.txt' - /p:Configuration=Release ${{ parameters.signingValidationAdditionalParameters }} + - template: ../steps/publish-logs.yml + parameters: + StageLabel: 'Validation' + JobLabel: 'Signing' + - ${{ if eq(parameters.enableSourceLinkValidation, 'true') }}: - job: displayName: SourceLink Validation @@ -86,44 +109,89 @@ stages: -GHRepoName $(Build.Repository.Name) -GHCommit $(Build.SourceVersion) -SourcelinkCliVersion $(SourceLinkCLIVersion) + continueOnError: true - ${{ if eq(parameters.SDLValidationParameters.enable, 'true') }}: - template: /eng/common/templates/job/execute-sdl.yml parameters: additionalParameters: ${{ parameters.SDLValidationParameters.params }} + continueOnError: ${{ parameters.SDLValidationParameters.continueOnError }} + artifactNames: ${{ parameters.SDLValidationParameters.artifactNames }} -- template: \eng\common\templates\post-build\channels\netcore-dev-5.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Dev5_Publish' + channelName: '.NET Core 5 Dev' + channelId: 131 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-transport/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet5-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\public-dev-release.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NetCore_Tools_Latest_Publish' + channelName: '.NET Tools - Latest' + channelId: 2 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\netcore-tools-latest.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: - enableSymbolValidation: ${{ parameters.enableSymbolValidation }} - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'PVR_Publish' + channelName: '.NET Tools - Validation' + channelId: 9 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-eng-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\public-validation-release.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'General_Testing_Publish' + channelName: 'General Testing' + channelId: 529 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/general-testing-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\public-release.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Dev_Publishing' + channelName: '.NET Core Tooling Dev' + channelId: 548 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' -- template: \eng\common\templates\post-build\channels\internal-servicing.yml +- template: \eng\common\templates\post-build\channels\generic-public-channel.yml parameters: - symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} artifactsPublishingAdditionalParameters: ${{ parameters.artifactsPublishingAdditionalParameters }} + dependsOn: ${{ parameters.publishDependsOn }} + publishInstallersAndChecksums: ${{ parameters.publishInstallersAndChecksums }} + symbolPublishingAdditionalParameters: ${{ parameters.symbolPublishingAdditionalParameters }} + stageName: 'NETCore_Tooling_Release_Publishing' + channelName: '.NET Core Tooling Release' + channelId: 549 + transportFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + shippingFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools/nuget/v3/index.json' + symbolsFeed: 'https://pkgs.dev.azure.com/dnceng/public/_packaging/dotnet-tools-symbols/nuget/v3/index.json' diff --git a/eng/common/templates/post-build/setup-maestro-vars.yml b/eng/common/templates/post-build/setup-maestro-vars.yml index 56242b068e..716b53f740 100644 --- a/eng/common/templates/post-build/setup-maestro-vars.yml +++ b/eng/common/templates/post-build/setup-maestro-vars.yml @@ -4,6 +4,8 @@ jobs: pool: vmImage: 'windows-2019' steps: + - checkout: none + - task: DownloadBuildArtifacts@0 displayName: Download Release Configs inputs: @@ -14,5 +16,25 @@ jobs: name: setReleaseVars displayName: Set Release Configs Vars inputs: - filePath: $(Build.SourcesDirectory)/eng/common/post-build/setup-maestro-vars.ps1 - arguments: -ReleaseConfigsPath '$(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt' + targetType: inline + script: | + try { + $Content = Get-Content $(Build.StagingDirectory)/ReleaseConfigs/ReleaseConfigs.txt + + $BarId = $Content | Select -Index 0 + + $Channels = "" + $Content | Select -Index 1 | ForEach-Object { $Channels += "$_ ," } + + $IsStableBuild = $Content | Select -Index 2 + + Write-Host "##vso[task.setvariable variable=BARBuildId;isOutput=true]$BarId" + Write-Host "##vso[task.setvariable variable=InitialChannels;isOutput=true]$Channels" + Write-Host "##vso[task.setvariable variable=IsStableBuild;isOutput=true]$IsStableBuild" + } + catch { + Write-Host $_ + Write-Host $_.Exception + Write-Host $_.ScriptStackTrace + exit 1 + } diff --git a/eng/common/templates/steps/promote-build.yml b/eng/common/templates/steps/promote-build.yml new file mode 100644 index 0000000000..b90404435d --- /dev/null +++ b/eng/common/templates/steps/promote-build.yml @@ -0,0 +1,13 @@ +parameters: + ChannelId: 0 + +steps: +- task: PowerShell@2 + displayName: Add Build to Channel + inputs: + filePath: $(Build.SourcesDirectory)/eng/common/post-build/promote-build.ps1 + arguments: -BuildId $(BARBuildId) + -ChannelId ${{ parameters.ChannelId }} + -MaestroApiAccessToken $(MaestroApiAccessToken) + -MaestroApiEndPoint $(MaestroApiEndPoint) + -MaestroApiVersion $(MaestroApiVersion) diff --git a/eng/common/templates/steps/publish-logs.yml b/eng/common/templates/steps/publish-logs.yml new file mode 100644 index 0000000000..8903ba57c0 --- /dev/null +++ b/eng/common/templates/steps/publish-logs.yml @@ -0,0 +1,23 @@ +parameters: + StageLabel: '' + JobLabel: '' + +steps: +- task: Powershell@2 + displayName: Prepare Binlogs to Upload + inputs: + targetType: inline + script: | + New-Item -ItemType Directory $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + Move-Item -Path $(Build.SourcesDirectory)/artifacts/log/Debug/* $(Build.SourcesDirectory)/PostBuildLogs/${{parameters.StageLabel}}/${{parameters.JobLabel}}/ + continueOnError: true + condition: always() + +- task: PublishBuildArtifacts@1 + displayName: Publish Logs + inputs: + PathtoPublish: '$(Build.SourcesDirectory)/PostBuildLogs' + PublishLocation: Container + ArtifactName: PostBuilLogs + continueOnError: true + condition: always() diff --git a/eng/common/templates/steps/send-to-helix.yml b/eng/common/templates/steps/send-to-helix.yml index 05df886f55..30becf01ea 100644 --- a/eng/common/templates/steps/send-to-helix.yml +++ b/eng/common/templates/steps/send-to-helix.yml @@ -23,6 +23,7 @@ parameters: EnableXUnitReporter: false # optional -- true enables XUnit result reporting to Mission Control WaitForWorkItemCompletion: true # optional -- true will make the task wait until work items have been completed and fail the build if work items fail. False is "fire and forget." IsExternal: false # [DEPRECATED] -- doesn't do anything, jobs are external if HelixAccessToken is empty and Creator is set + HelixBaseUri: 'https://helix.dot.net/' # optional -- sets the Helix API base URI (allows targeting int) Creator: '' # optional -- if the build is external, use this to specify who is sending the job DisplayNamePrefix: 'Run Tests' # optional -- rename the beginning of the displayName of the steps in AzDO condition: succeeded() # optional -- condition for step to execute; defaults to succeeded() @@ -55,6 +56,7 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, eq(variables['Agent.Os'], 'Windows_NT')) @@ -85,6 +87,7 @@ steps: DotNetCliVersion: ${{ parameters.DotNetCliVersion }} EnableXUnitReporter: ${{ parameters.EnableXUnitReporter }} WaitForWorkItemCompletion: ${{ parameters.WaitForWorkItemCompletion }} + HelixBaseUri: ${{ parameters.HelixBaseUri }} Creator: ${{ parameters.Creator }} SYSTEM_ACCESSTOKEN: $(System.AccessToken) condition: and(${{ parameters.condition }}, ne(variables['Agent.Os'], 'Windows_NT')) diff --git a/eng/common/tools.ps1 b/eng/common/tools.ps1 index 9c12b1b4fd..42ca33ac33 100644 --- a/eng/common/tools.ps1 +++ b/eng/common/tools.ps1 @@ -49,6 +49,8 @@ # An array of names of processes to stop on script exit if prepareMachine is true. $processesToStopOnExit = if (Test-Path variable:processesToStopOnExit) { $processesToStopOnExit } else { @("msbuild", "dotnet", "vbcscompiler") } +$disableConfigureToolsetImport = if (Test-Path variable:disableConfigureToolsetImport) { $disableConfigureToolsetImport } else { $null } + set-strictmode -version 2.0 $ErrorActionPreference = "Stop" [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 @@ -153,6 +155,7 @@ function InitializeDotNetCli([bool]$install) { # Make Sure that our bootstrapped dotnet cli is available in future steps of the Azure Pipelines build Write-PipelinePrependPath -Path $dotnetRoot + Write-PipelineSetVariable -Name 'DOTNET_MULTILEVEL_LOOKUP' -Value '0' Write-PipelineSetVariable -Name 'DOTNET_SKIP_FIRST_TIME_EXPERIENCE' -Value '1' @@ -163,6 +166,7 @@ function GetDotNetInstallScript([string] $dotnetRoot) { $installScript = Join-Path $dotnetRoot "dotnet-install.ps1" if (!(Test-Path $installScript)) { Create-Directory $dotnetRoot + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dot.net/$dotnetInstallScriptVersion/dotnet-install.ps1" -OutFile $installScript } @@ -173,7 +177,14 @@ function InstallDotNetSdk([string] $dotnetRoot, [string] $version, [string] $arc InstallDotNet $dotnetRoot $version $architecture } -function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $architecture = "", [string] $runtime = "", [bool] $skipNonVersionedFiles = $false) { +function InstallDotNet([string] $dotnetRoot, + [string] $version, + [string] $architecture = "", + [string] $runtime = "", + [bool] $skipNonVersionedFiles = $false, + [string] $runtimeSourceFeed = "", + [string] $runtimeSourceFeedKey = "") { + $installScript = GetDotNetInstallScript $dotnetRoot $installParameters = @{ Version = $version @@ -184,10 +195,29 @@ function InstallDotNet([string] $dotnetRoot, [string] $version, [string] $archit if ($runtime) { $installParameters.Runtime = $runtime } if ($skipNonVersionedFiles) { $installParameters.SkipNonVersionedFiles = $skipNonVersionedFiles } - & $installScript @installParameters - if ($lastExitCode -ne 0) { - Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet cli (exit code '$lastExitCode')." - ExitWithExitCode $lastExitCode + try { + & $installScript @installParameters + } + catch { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from public location." + + # Only the runtime can be installed from a custom [private] location. + if ($runtime -and ($runtimeSourceFeed -or $runtimeSourceFeedKey)) { + if ($runtimeSourceFeed) { $installParameters.AzureFeed = $runtimeSourceFeed } + + if ($runtimeSourceFeedKey) { + $decodedBytes = [System.Convert]::FromBase64String($runtimeSourceFeedKey) + $decodedString = [System.Text.Encoding]::UTF8.GetString($decodedBytes) + $installParameters.FeedCredential = $decodedString + } + + try { + & $installScript @installParameters + } + catch { + Write-PipelineTelemetryError -Category "InitializeToolset" -Message "Failed to install dotnet runtime '$runtime' from custom location '$runtimeSourceFeed'." + } + } } } @@ -282,6 +312,7 @@ function InitializeXCopyMSBuild([string]$packageVersion, [bool]$install) { Create-Directory $packageDir Write-Host "Downloading $packageName $packageVersion" + $ProgressPreference = 'SilentlyContinue' # Don't display the console progress UI - it's a huge perf hit Invoke-WebRequest "https://dotnet.myget.org/F/roslyn-tools/api/v2/package/$packageName/$packageVersion/" -OutFile $packagePath Unzip $packagePath $packageDir } @@ -363,7 +394,6 @@ function InitializeBuildTool() { Write-PipelineTelemetryError -Category "InitializeToolset" -Message "/global.json must specify 'tools.dotnet'." ExitWithExitCode 1 } - $buildTool = @{ Path = Join-Path $dotnetRoot "dotnet.exe"; Command = "msbuild"; Tool = "dotnet"; Framework = "netcoreapp2.1" } } elseif ($msbuildEngine -eq "vs") { try { @@ -416,7 +446,7 @@ function GetSdkTaskProject([string]$taskName) { } function InitializeNativeTools() { - if (Get-Member -InputObject $GlobalJson -Name "native-tools") { + if (-Not (Test-Path variable:DisableNativeToolsetInstalls) -And (Get-Member -InputObject $GlobalJson -Name "native-tools")) { $nativeArgs= @{} if ($ci) { $nativeArgs = @{ @@ -488,6 +518,18 @@ function Stop-Processes() { function MSBuild() { if ($pipelinesLog) { $buildTool = InitializeBuildTool + + # Work around issues with Azure Artifacts credential provider + # https://github.com/dotnet/arcade/issues/3932 + if ($ci -and $buildTool.Tool -eq "dotnet") { + dotnet nuget locals http-cache -c + + $env:NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS = 20 + $env:NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS = 20 + Write-PipelineSetVariable -Name 'NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS' -Value '20' + Write-PipelineSetVariable -Name 'NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS' -Value '20' + } + $toolsetBuildProject = InitializeToolset $path = Split-Path -parent $toolsetBuildProject $path = Join-Path $path (Join-Path $buildTool.Framework "Microsoft.DotNet.Arcade.Sdk.dll") @@ -585,3 +627,12 @@ Write-PipelineSetVariable -Name 'Artifacts.Toolset' -Value $ToolsetDir Write-PipelineSetVariable -Name 'Artifacts.Log' -Value $LogDir Write-PipelineSetVariable -Name 'TEMP' -Value $TempDir Write-PipelineSetVariable -Name 'TMP' -Value $TempDir + +# Import custom tools configuration, if present in the repo. +# Note: Import in global scope so that the script set top-level variables without qualification. +if (!$disableConfigureToolsetImport) { + $configureToolsetScript = Join-Path $EngRoot "configure-toolset.ps1" + if (Test-Path $configureToolsetScript) { + . $configureToolsetScript + } +} diff --git a/eng/common/tools.sh b/eng/common/tools.sh index 3af9be6157..6a23ac0a34 100755 --- a/eng/common/tools.sh +++ b/eng/common/tools.sh @@ -4,6 +4,7 @@ # CI mode - set to true on CI server for PR validation build or official build. ci=${ci:-false} +disable_configure_toolset_import=${disable_configure_toolset_import:-} # Set to true to use the pipelines logger which will enable Azure logging output. # https://github.com/Microsoft/azure-pipelines-tasks/blob/master/docs/authoring/commands.md @@ -191,8 +192,28 @@ function InstallDotNet { fi bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg || { local exit_code=$? - Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK (exit code '$exit_code')." - ExitWithExitCode $exit_code + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from public location (exit code '$exit_code')." + + if [[ -n "$runtimeArg" ]]; then + local runtimeSourceFeed='' + if [[ -n "${6:-}" ]]; then + runtimeSourceFeed="--azure-feed $6" + fi + + local runtimeSourceFeedKey='' + if [[ -n "${7:-}" ]]; then + decodedFeedKey=`echo $7 | base64 --decode` + runtimeSourceFeedKey="--feed-credential $decodedFeedKey" + fi + + if [[ -n "$runtimeSourceFeed" || -n "$runtimeSourceFeedKey" ]]; then + bash "$install_script" --version $version --install-dir "$root" $archArg $runtimeArg $skipNonVersionedFilesArg $runtimeSourceFeed $runtimeSourceFeedKey || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to install dotnet SDK from custom location '$runtimeSourceFeed' (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + fi + fi } } @@ -208,12 +229,19 @@ function GetDotNetInstallScript { # Use curl if available, otherwise use wget if command -v curl > /dev/null; then - curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" - else - wget -q -O "$install_script" "$install_script_url" + curl "$install_script_url" -sSL --retry 10 --create-dirs -o "$install_script" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } + else + wget -q -O "$install_script" "$install_script_url" || { + local exit_code=$? + Write-PipelineTelemetryError -category 'InitializeToolset' "Failed to acquire dotnet install script (exit code '$exit_code')." + ExitWithExitCode $exit_code + } fi fi - # return value _GetDotNetInstallScript="$install_script" } @@ -245,6 +273,9 @@ function GetNuGetPackageCachePath { } function InitializeNativeTools() { + if [[ -n "${DisableNativeToolsetInstalls:-}" ]]; then + return + fi if grep -Fq "native-tools" $global_json_file then local nativeArgs="" @@ -321,6 +352,18 @@ function MSBuild { if [[ "$pipelines_log" == true ]]; then InitializeBuildTool InitializeToolset + + # Work around issues with Azure Artifacts credential provider + # https://github.com/dotnet/arcade/issues/3932 + if [[ "$ci" == true ]]; then + "$_InitializeBuildTool" nuget locals http-cache -c + + export NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS=20 + export NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS=20 + Write-PipelineSetVariable -name "NUGET_PLUGIN_HANDSHAKE_TIMEOUT_IN_SECONDS" -value "20" + Write-PipelineSetVariable -name "NUGET_PLUGIN_REQUEST_TIMEOUT_IN_SECONDS" -value "20" + fi + local toolset_dir="${_InitializeToolset%/*}" local logger_path="$toolset_dir/$_InitializeBuildToolFramework/Microsoft.DotNet.Arcade.Sdk.dll" args=( "${args[@]}" "-logger:$logger_path" ) @@ -392,3 +435,18 @@ Write-PipelineSetVariable -name "Artifacts.Toolset" -value "$toolset_dir" Write-PipelineSetVariable -name "Artifacts.Log" -value "$log_dir" Write-PipelineSetVariable -name "Temp" -value "$temp_dir" Write-PipelineSetVariable -name "TMP" -value "$temp_dir" + +# Import custom tools configuration, if present in the repo. +if [[ -z "$disable_configure_toolset_import" ]]; then + configure_toolset_script="$eng_root/configure-toolset.sh" + if [[ -a "$configure_toolset_script" ]]; then + . "$configure_toolset_script" + fi +fi + +# TODO: https://github.com/dotnet/arcade/issues/1468 +# Temporary workaround to avoid breaking change. +# Remove once repos are updated. +if [[ -n "${useInstalledDotNetCli:-}" ]]; then + use_installed_dotnet_cli="$useInstalledDotNetCli" +fi diff --git a/eng/emptyPackageProject/Class1.cs b/eng/emptyPackageProject/Class1.cs new file mode 100644 index 0000000000..c2b154a9d4 --- /dev/null +++ b/eng/emptyPackageProject/Class1.cs @@ -0,0 +1,6 @@ +namespace MicrosoftDotnetSourceBuiltArtifacts +{ + public class Class1 + { + } +} diff --git a/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj b/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj new file mode 100644 index 0000000000..231c83fa69 --- /dev/null +++ b/eng/emptyPackageProject/Private.SourceBuilt.Artifacts.csproj @@ -0,0 +1,19 @@ + + + + + https://github.com/dotnet/source-build + https://github.com/dotnet/source-build + git + 0000 + $(ManifestCommit) + MIT + netcoreapp2.1 + true + false + NU5105 + + + + diff --git a/eng/publishPackages/Directory.Build.props b/eng/publishPackages/Directory.Build.props new file mode 100644 index 0000000000..8ebfe9720b --- /dev/null +++ b/eng/publishPackages/Directory.Build.props @@ -0,0 +1,8 @@ + + + + + $(CopyrightNetFoundation) + MIT + + diff --git a/eng/publishPackages/Directory.Build.targets b/eng/publishPackages/Directory.Build.targets new file mode 100644 index 0000000000..e2bdf743c6 --- /dev/null +++ b/eng/publishPackages/Directory.Build.targets @@ -0,0 +1,3 @@ + + + diff --git a/eng/publishPackages/collectArtifacts.proj b/eng/publishPackages/collectArtifacts.proj new file mode 100644 index 0000000000..fb8df8ee1c --- /dev/null +++ b/eng/publishPackages/collectArtifacts.proj @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + diff --git a/eng/publishPackages/publish.proj b/eng/publishPackages/publish.proj new file mode 100644 index 0000000000..e755e0a31d --- /dev/null +++ b/eng/publishPackages/publish.proj @@ -0,0 +1,76 @@ + + + $(RepoRoot)artifacts/packages/ + $(RepoRoot)artifacts/asset-manifests/ + $(RepoRoot)artifacts/tarball/ + + $(RestoreSources); + https://dotnetfeed.blob.core.windows.net/dotnet-tools-internal/index.json; + + + CreateEmptyPackage; + + + $(CreateTargets); + + + $(BuildDependsOn); + PreparePublish; + PublishPackagesToBlobFeed; + PublishFilesToBlobFeed; + + + + + + + + + + + + + + + + + + + + + + $(TarballDir)Private.SourceBuilt.Artifacts.*.tar.gz + + + + + + + https://$(AzureAccountName).blob.core.windows.net/$(ContainerName)/index.json + $(AzureAccessToken) + + + + + + $(ManifestDir)source-build-artifacts-nupkg-manifest.xml + + + + + + + + + $(ManifestDir)source-build-artifacts-tar-manifest.xml + + + + + + + + diff --git a/publish.ps1 b/publish.ps1 new file mode 100644 index 0000000000..123997c3a3 --- /dev/null +++ b/publish.ps1 @@ -0,0 +1,4 @@ +. "$PSScriptRoot/eng/common/tools.ps1" +InitializeToolset +Write-Host "$PSScriptRoot/eng/common/build.ps1 -bl -restore -build -pack -publish /p:Projects=`"$PSScriptRoot/eng/publishPackages/publish.proj`" /p:PublishingToBlobStorage=true /p:DeterministicSourcePaths=false /p:EnableSourceLink=false $args" +Invoke-Expression "$PSScriptRoot/eng/common/build.ps1 -bl -restore -build -pack -publish /p:Projects=`"$PSScriptRoot/eng/publishPackages/publish.proj`" /p:PublishingToBlobStorage=true /p:DeterministicSourcePaths=false /p:EnableSourceLink=false $args" diff --git a/src/package-source-build/package.proj b/src/package-source-build/package.proj index ecf7aa4f85..de2d69a785 100644 --- a/src/package-source-build/package.proj +++ b/src/package-source-build/package.proj @@ -14,7 +14,7 @@ - $(OutputPath)$(SourceBuiltArtifactsTarballName).$(VersionPrefix)-$(VersionSuffix).tar.gz + $(OutputPath)$(SourceBuiltArtifactsTarballName).$(SourceBuildOutputVersion).tar.gz diff --git a/tools-local/prebuilt-baseline-online.xml b/tools-local/prebuilt-baseline-online.xml index 8373234442..f5ff489a7a 100644 --- a/tools-local/prebuilt-baseline-online.xml +++ b/tools-local/prebuilt-baseline-online.xml @@ -77,6 +77,7 @@ +