diff --git a/eng/common/templates/1es-official.yml b/eng/common/templates/1es-official.yml index 91bb72ef..357ab811 100644 --- a/eng/common/templates/1es-official.yml +++ b/eng/common/templates/1es-official.yml @@ -17,6 +17,9 @@ parameters: - name: stages type: stageList default: [] +- name: serviceConnections + type: object + default: [] - name: pool type: object default: @@ -59,4 +62,9 @@ extends: sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} tsa: enabled: true - stages: ${{ parameters.stages }} + stages: + - template: /eng/common/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/common/templates/1es-unofficial.yml b/eng/common/templates/1es-unofficial.yml index f0867abd..64dc5313 100644 --- a/eng/common/templates/1es-unofficial.yml +++ b/eng/common/templates/1es-unofficial.yml @@ -20,6 +20,9 @@ parameters: type: stageList default: [] # 1ES Pipeline Template parameters +- name: serviceConnections + type: object + default: [] - name: pool type: object default: @@ -67,4 +70,9 @@ extends: sourceAnalysisPool: ${{ parameters.sourceAnalysisPool }} tsa: enabled: true - stages: ${{ parameters.stages }} + stages: + - template: /eng/common/templates/stages/setup-service-connections.yml@self + parameters: + pool: ${{ parameters.pool }} + serviceConnections: ${{ parameters.serviceConnections }} + - ${{ parameters.stages }} diff --git a/eng/common/templates/jobs/build-images.yml b/eng/common/templates/jobs/build-images.yml index a1063904..6d2482e9 100644 --- a/eng/common/templates/jobs/build-images.yml +++ b/eng/common/templates/jobs/build-images.yml @@ -65,10 +65,14 @@ jobs: parameters: name: BuildImages displayName: Build Images - serviceConnection: $(build.serviceConnectionName) + serviceConnections: + - name: acr + id: $(build.serviceConnection.id) + tenantId: $(build.serviceConnection.tenantId) + clientId: $(build.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} - args: > + args: >- build --manifest $(manifest) $(imageBuilderPaths) diff --git a/eng/common/templates/jobs/copy-base-images-staging.yml b/eng/common/templates/jobs/copy-base-images-staging.yml index 169eeb14..71ddb4a5 100644 --- a/eng/common/templates/jobs/copy-base-images-staging.yml +++ b/eng/common/templates/jobs/copy-base-images-staging.yml @@ -24,7 +24,10 @@ jobs: additionalOptions: ${{ parameters.additionalOptions }} acr: server: $(acr-staging.server) - serviceConnection: $(internal-mirror.serviceConnectionName) + serviceConnection: + tenantId: $(internal-mirror.serviceConnection.tenantId) + clientId: $(internal-mirror.serviceConnection.clientId) + id: $(internal-mirror.serviceConnection.id) subscription: $(acr-staging.subscription) resourceGroup: $(acr-staging.resourceGroup) repoPrefix: $(mirrorRepoPrefix) diff --git a/eng/common/templates/jobs/generate-matrix.yml b/eng/common/templates/jobs/generate-matrix.yml index 81444303..8037aca5 100644 --- a/eng/common/templates/jobs/generate-matrix.yml +++ b/eng/common/templates/jobs/generate-matrix.yml @@ -61,6 +61,10 @@ jobs: parameters: name: matrix displayName: Generate ${{ parameters.matrixType }} Matrix - serviceConnection: $(build.serviceConnectionName) + serviceConnections: + - name: acr + tenantId: $(build.serviceConnection.tenantId) + clientId: $(build.serviceConnection.clientId) + id: $(build.serviceConnection.id) internalProjectName: internal args: $(generateBuildMatrixCommand) diff --git a/eng/common/templates/jobs/publish.yml b/eng/common/templates/jobs/publish.yml index bf08afad..9031256a 100644 --- a/eng/common/templates/jobs/publish.yml +++ b/eng/common/templates/jobs/publish.yml @@ -13,7 +13,7 @@ jobs: variables: - name: imageBuilder.commonCmdArgs - value: > + value: >- --manifest '$(manifest)' --registry-override '$(acr.server)' $(manifestVariables) @@ -75,9 +75,13 @@ jobs: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Copy Images - serviceConnection: $(publish.serviceConnectionName) + serviceConnections: + - name: acr + id: $(publish.serviceConnection.id) + tenantId: $(publish.serviceConnection.tenantId) + clientId: $(publish.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} - args: > + args: >- copyAcrImages '$(acr.subscription)' '$(acr.resourceGroup)' @@ -94,10 +98,14 @@ jobs: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Publish Manifest - serviceConnection: $(publish.serviceConnectionName) + serviceConnections: + - name: acr + id: $(publish.serviceConnection.id) + tenantId: $(publish.serviceConnection.tenantId) + clientId: $(publish.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} dockerClientOS: ${{ parameters.dockerClientOS }} - args: > + args: >- publishManifest '$(imageInfoContainerDir)/image-info.json' --repo-prefix '$(publishRepoPrefix)' @@ -152,10 +160,14 @@ jobs: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Ingest Kusto Image Info - serviceConnection: $(kusto.serviceConnectionName) + serviceConnections: + - name: kusto + id: $(kusto.serviceConnection.id) + tenantId: $(kusto.serviceConnection.tenantId) + clientId: $(kusto.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} condition: and(succeeded(), eq(variables['ingestKustoImageInfo'], 'true')) - args: > + args: >- ingestKustoImageInfo '$(imageInfoContainerDir)/image-info.json' '$(kusto.cluster)' @@ -170,10 +182,14 @@ jobs: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Generate EOL Annotation Data - serviceConnection: $(publish.serviceConnectionName) + serviceConnections: + - name: acr + id: $(publish.serviceConnection.id) + tenantId: $(publish.serviceConnection.tenantId) + clientId: $(publish.serviceConnection.clientId) internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: > + args: >- generateEolAnnotationData '$(artifactsPath)/eol-annotation-data/eol-annotation-data.json' '$(imageInfoContainerDir)/full-image-info-orig.json' diff --git a/eng/common/templates/stages/build-and-test.yml b/eng/common/templates/stages/build-and-test.yml index a4655f3c..1b5d4a8f 100644 --- a/eng/common/templates/stages/build-and-test.yml +++ b/eng/common/templates/stages/build-and-test.yml @@ -49,6 +49,7 @@ parameters: stages: - stage: Build condition: and(succeeded(), contains(variables['stages'], 'build')) + dependsOn: [] jobs: - template: /eng/common/templates/jobs/test-images-linux-client.yml@self parameters: diff --git a/eng/common/templates/stages/setup-service-connections.yml b/eng/common/templates/stages/setup-service-connections.yml new file mode 100644 index 00000000..7d125125 --- /dev/null +++ b/eng/common/templates/stages/setup-service-connections.yml @@ -0,0 +1,34 @@ +# This stage exists to tell Azure DevOps about all of the service connections +# that will be used in the pipeline. A service connection will not work unless +# it is declared in this stage's parameters, even if your pipeline has already +# been granted access to the service connection. This stage also does not need +# to complete before the service connection is used. +parameters: +- name: pool + type: object +# serviceConnections object shape: +# - name: string +- name: serviceConnections + type: object + default: [] + +stages: + +- stage: SetupServiceConnectionsStage + displayName: Setup service connections + jobs: + + - job: SetupServiceConnectionsJob + displayName: Setup service connections + pool: ${{ parameters.pool }} + steps: + + - ${{ each serviceConnection in parameters.serviceConnections }}: + - task: AzureCLI@2 + displayName: Setup ${{ serviceConnection.name }} + inputs: + azureSubscription: ${{ serviceConnection.name }} + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + az account show diff --git a/eng/common/templates/steps/annotate-eol-digests.yml b/eng/common/templates/steps/annotate-eol-digests.yml index 3f01cad0..35646b26 100644 --- a/eng/common/templates/steps/annotate-eol-digests.yml +++ b/eng/common/templates/steps/annotate-eol-digests.yml @@ -7,10 +7,14 @@ steps: parameters: name: AnnotateEOLImages displayName: Annotate EOL Images - serviceConnection: $(publish.serviceConnectionName) + serviceConnections: + - name: acr + id: $(publish.serviceConnection.id) + tenantId: $(publish.serviceConnection.tenantId) + clientId: $(publish.serviceConnection.clientId) internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: > + args: >- annotateEolDigests ${{ parameters.dataFile }} $(acr.server) @@ -28,9 +32,13 @@ steps: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Wait for Annotation Ingestion - serviceConnection: $(marStatus.serviceConnectionName) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) internalProjectName: internal condition: and(succeeded(), eq(variables['publishEolAnnotations'], 'true')) - args: > + args: >- waitForMarAnnotationIngestion $(artifactsPath)/annotation-digests/annotation-digests.txt diff --git a/eng/common/templates/steps/clean-acr-images.yml b/eng/common/templates/steps/clean-acr-images.yml index f02b0225..abfb9fb9 100644 --- a/eng/common/templates/steps/clean-acr-images.yml +++ b/eng/common/templates/steps/clean-acr-images.yml @@ -11,9 +11,13 @@ steps: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Clean ACR Images - ${{ parameters.repo }} - serviceConnection: $(clean.serviceConnectionName) + serviceConnections: + - name: acr + id: $(clean.serviceConnection.id) + tenantId: $(clean.serviceConnection.tenantId) + clientId: $(clean.serviceConnection.clientId) internalProjectName: ${{ parameters.internalProjectName }} - args: > + args: >- cleanAcrImages ${{ parameters.repo }} ${{ parameters.subscription }} diff --git a/eng/common/templates/steps/copy-base-images.yml b/eng/common/templates/steps/copy-base-images.yml index e24fc030..f78b1c5d 100644 --- a/eng/common/templates/steps/copy-base-images.yml +++ b/eng/common/templates/steps/copy-base-images.yml @@ -3,7 +3,10 @@ parameters: type: object default: server: "" - serviceConnection: "" + serviceConnection: + tenantId: "" + clientId: "" + id: "" subscription: "" resourceGroup: "" - name: repoPrefix @@ -25,13 +28,17 @@ steps: - template: /eng/common/templates/steps/run-imagebuilder.yml@self parameters: displayName: Copy Base Images - serviceConnection: ${{ parameters.acr.serviceConnection }} + serviceConnections: + - name: "acr" + tenantId: ${{ parameters.acr.serviceConnection.tenantId }} + clientId: ${{ parameters.acr.serviceConnection.clientId }} + id: ${{ parameters.acr.serviceConnection.id }} continueOnError: ${{ parameters.continueOnError }} internalProjectName: 'internal' # Use environment variable to reference $(dryRunArg). Since $(dryRunArg) might be undefined, # PowerShell will treat the Azure Pipelines variable macro syntax as a command and throw an # error - args: > + args: >- copyBaseImages '${{ parameters.acr.subscription }}' '${{ parameters.acr.resourceGroup }}' diff --git a/eng/common/templates/steps/init-docker-linux.yml b/eng/common/templates/steps/init-docker-linux.yml index aa74ac00..63b2e859 100644 --- a/eng/common/templates/steps/init-docker-linux.yml +++ b/eng/common/templates/steps/init-docker-linux.yml @@ -24,26 +24,27 @@ steps: # Setup Image Builder (Optional) ################################################################################ - ${{ if eq(parameters.setupImageBuilder, 'true') }}: + - script: $(engCommonPath)/pull-image.sh $(imageNames.imageBuilder) displayName: Pull Image Builder condition: and(succeeded(), ${{ parameters.condition }}) - - script: > + + - script: >- docker build -t $(imageNames.imageBuilder.withrepo) --build-arg IMAGE=$(imageNames.imageBuilder) -f $(engCommonPath)/Dockerfile.WithRepo . displayName: Build Image for Image Builder condition: and(succeeded(), ${{ parameters.condition }}) + - task: PowerShell@2 displayName: Define ImageBuilder Command Variables condition: and(succeeded(), ${{ parameters.condition }}) inputs: targetType: 'inline' script: | - $tokenHostPath = '$(Agent.TempDirectory)' - $tokenHostFilePath = "${tokenHostPath}/token" - $tokenContainerPath = "/tmp" - $tokenContainerFilePath = "${tokenContainerPath}/token" + $imageBuilderImageName = "$(imageNames.imageBuilder.withrepo)" + Write-Host "##vso[task.setvariable variable=imageBuilderImageName]$imageBuilderImageName" $dockerRunBaseCmd = @( "docker run --rm" @@ -58,10 +59,8 @@ steps: ) $authedDockerRunArgs = @( - '-e AZURE_TENANT_ID=$env:tenantId' - '-e AZURE_CLIENT_ID=$env:servicePrincipalId' - "-e AZURE_FEDERATED_TOKEN_FILE=$tokenContainerFilePath" - "-v ${tokenHostPath}:${tokenContainerPath}" + '-e SYSTEM_ACCESSTOKEN=$(System.AccessToken)' + '-e SYSTEM_OIDCREQUESTURI=$(System.OidcRequestUri)' ) $dockerRunCmd = $dockerRunBaseCmd + $dockerRunArgs @@ -72,7 +71,6 @@ steps: Write-Host "##vso[task.setvariable variable=runImageBuilderCmd]$runImageBuilderCmd" Write-Host "##vso[task.setvariable variable=runAuthedImageBuilderCmd]$runAuthedImageBuilderCmd" - Write-Host "##vso[task.setvariable variable=tokenHostFilePath]$tokenHostFilePath" ################################################################################ # Setup Test Runner (Optional) diff --git a/eng/common/templates/steps/run-imagebuilder.yml b/eng/common/templates/steps/run-imagebuilder.yml index ae20464c..0237ff47 100644 --- a/eng/common/templates/steps/run-imagebuilder.yml +++ b/eng/common/templates/steps/run-imagebuilder.yml @@ -5,9 +5,18 @@ parameters: - name: displayName type: string default: "Run ImageBuilder" -- name: serviceConnection - type: string - default: "" +- name: serviceConnections + type: object + default: + # name: the name of the service connection argument that will be passed to the ImageBuilder command. + # For example, if the argument is --acr-service-connection, the name would be "acr". + - name: "" + # The service connection's ID (GUID). + id: "" + # The client ID of the Managed Idendity backing the service connection (GUID). + clientId: "" + # The ID of the tenant that the Managed Identity is in (GUID). + tenantId: "" - name: internalProjectName type: string default: null @@ -25,21 +34,33 @@ parameters: default: "linux" steps: -- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest'), ne(parameters.serviceConnection, '')) }}: - - template: /eng/common/templates/steps/run-pwsh-with-auth.yml@self - parameters: - ${{ if ne(parameters.name, '') }}: - name: ${{ parameters.name }} - displayName: ${{ parameters.displayName }} - serviceConnection: ${{ parameters.serviceConnection }} - continueOnError: ${{ parameters.continueOnError }} - dockerClientOS: ${{ parameters.dockerClientOS }} - condition: ${{ parameters.condition }} - command: > - $env:idToken | Out-File -FilePath $(tokenHostFilePath); - $(runAuthedImageBuilderCmd) ${{ parameters.args }}; - Remove-Item -Path $(tokenHostFilePath) -Force +- ${{ if and(eq(variables['System.TeamProject'], parameters.internalProjectName), ne(variables['Build.Reason'], 'PullRequest')) }}: + + - task: PowerShell@2 + ${{ if ne(parameters.name, '') }}: + name: ${{ parameters.name }} + displayName: ${{ parameters.displayName }} + continueOnError: ${{ parameters.continueOnError }} + condition: ${{ parameters.condition }} + inputs: + targetType: 'inline' + script: | + $serviceConnections = '${{ convertToJson(parameters.serviceConnections) }}' + + Write-Host "Service connections:" + Write-Host "${serviceConnections}" + + $serviceConnectionsJson = $serviceConnections | ConvertFrom-Json + $serviceConnectionsArgs = @() + foreach ($connection in $serviceConnectionsJson) { + $serviceConnectionsArgs += "--$($connection.name)-service-connection" + $serviceConnectionsArgs += "$($connection.tenantId):$($connection.clientId):$($connection.id)" + } + + $(runAuthedImageBuilderCmd) ${{ parameters.args }} @serviceConnectionsArgs + - ${{ else }}: + - task: PowerShell@2 ${{ if ne(parameters.name, '') }}: name: ${{ parameters.name }} @@ -48,5 +69,5 @@ steps: condition: ${{ parameters.condition }} inputs: targetType: 'inline' - script: > + script: >- $(runImageBuilderCmd) ${{ parameters.args }} diff --git a/eng/common/templates/steps/test-images-linux-client.yml b/eng/common/templates/steps/test-images-linux-client.yml index f99fbb65..1af54d0c 100644 --- a/eng/common/templates/steps/test-images-linux-client.yml +++ b/eng/common/templates/steps/test-images-linux-client.yml @@ -48,7 +48,7 @@ steps: displayName: Docker login serviceConnection: $(test.serviceConnectionName) condition: and(succeeded(), ${{ parameters.condition }}) - command: > + command: >- $azLoginArgs = '--service-principal --tenant $env:AZURE_TENANT_ID -u $env:AZURE_CLIENT_ID --federated-token $env:AZURE_FEDERATED_TOKEN'; docker exec -e AZURE_TENANT_ID=$env:tenantId -e AZURE_CLIENT_ID=$env:servicePrincipalId -e AZURE_FEDERATED_TOKEN=$env:idToken $(testRunner.container) pwsh -File $(engCommonRelativePath)/Invoke-WithRetry.ps1 diff --git a/eng/common/templates/steps/test-images-windows-client.yml b/eng/common/templates/steps/test-images-windows-client.yml index d694d9b0..d53a42b6 100644 --- a/eng/common/templates/steps/test-images-windows-client.yml +++ b/eng/common/templates/steps/test-images-windows-client.yml @@ -17,7 +17,7 @@ steps: serviceConnection: $(test.serviceConnectionName) dockerClientOS: windows condition: and(succeeded(), ${{ parameters.condition }}) - command: > + command: >- az login --service-principal --tenant $env:tenantId -u $env:servicePrincipalId --federated-token $env:idToken; $accessToken = $(az acr login -n $(acr-staging.server) --expose-token --query accessToken --output tsv); docker login $(acr-staging.server) -u 00000000-0000-0000-0000-000000000000 -p $accessToken diff --git a/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml index a0be54d2..05112cc5 100644 --- a/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml +++ b/eng/common/templates/steps/wait-for-mcr-doc-ingestion.yml @@ -8,9 +8,13 @@ steps: parameters: displayName: Wait for MCR Doc Ingestion condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) - serviceConnection: $(marStatus.serviceConnectionName) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) internalProjectName: 'internal' - args: > + args: >- waitForMcrDocIngestion '${{ parameters.commitDigest }}' --timeout '$(mcrDocIngestionTimeout)' diff --git a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml index 83263c7f..a3c84ddf 100644 --- a/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml +++ b/eng/common/templates/steps/wait-for-mcr-image-ingestion.yml @@ -9,9 +9,13 @@ steps: parameters: displayName: Wait for Image Ingestion condition: and(${{ parameters.condition }}, eq(variables['waitForIngestionEnabled'], 'true')) - serviceConnection: $(marStatus.serviceConnectionName) + serviceConnections: + - name: mar + id: $(marStatus.serviceConnection.id) + tenantId: $(marStatus.serviceConnection.tenantId) + clientId: $(marStatus.serviceConnection.clientId) internalProjectName: 'internal' - args: > + args: >- waitForMcrImageIngestion '${{ parameters.imageInfoPath }}' --manifest '$(manifest)' diff --git a/eng/pipelines/dotnet-buildtools-image-builder-official.yml b/eng/pipelines/dotnet-buildtools-image-builder-official.yml index d15799bb..aa669fcb 100644 --- a/eng/pipelines/dotnet-buildtools-image-builder-official.yml +++ b/eng/pipelines/dotnet-buildtools-image-builder-official.yml @@ -28,6 +28,12 @@ variables: extends: template: /eng/common/templates/1es-official.yml@self parameters: + serviceConnections: + - name: $(internal-mirror.serviceConnectionName) + - name: $(build.serviceConnectionName) + - name: $(publish.serviceConnectionName) + - name: $(kusto.serviceConnectionName) + - name: $(marStatus.serviceConnectionName) stages: - template: /eng/common/templates/stages/dotnet/build-test-publish-repo.yml@self parameters: