From b586fae52446eadb0f909674ac404e9797680993 Mon Sep 17 00:00:00 2001 From: Hiroshi Yamauchi Date: Tue, 16 Apr 2024 11:00:29 -0700 Subject: [PATCH 1/5] Add cross-compiling support for the windows toolchain Co-authrored with @compnerd (cherry picked from commit f6ab729ffd8767373e291f035792db2600ebb1c8) --- utils/build.ps1 | 219 ++++++++++++++++++++++++++++++++++-------------- 1 file changed, 158 insertions(+), 61 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 3d76c1a4d9e22..9fc3c48e06789 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -80,6 +80,9 @@ For example: -BuildTo ToolsSupportCore When set, runs the script in a special mode which outputs a listing of command invocations in batch file format instead of executing them. +.PARAMETER HostArchName +The architecture where the toolchain will execute. + .EXAMPLE PS> .\Build.ps1 @@ -95,8 +98,9 @@ param( [string] $SwiftDebugFormat = "dwarf", [string[]] $WindowsSDKs = @("X64","X86","Arm64"), [string] $ProductVersion = "0.0.0", - [string] $PinnedBuild = "https://download.swift.org/swift-5.9-release/windows10/swift-5.9-RELEASE/swift-5.9-RELEASE-windows10.exe", - [string] $PinnedSHA256 = "EAB668ABFF903B4B8111FD27F49BAD470044B6403C6FA9CCD357AE831909856D", + [string] $PinnedBuild = "", + [string] $PinnedSHA256 = "", + [string] $PythonVersion = "3.9.10", [string] $WinSDKVersion = "", [switch] $SkipBuild = $false, [switch] $SkipRedistInstall = $false, @@ -104,6 +108,7 @@ param( [string[]] $Test = @(), [string] $Stage = "", [string] $BuildTo = "", + [string] $HostArchName = $(if ($env:PROCESSOR_ARCHITEW6432 -ne $null) { "$env:PROCESSOR_ARCHITEW6432" } else { "$env:PROCESSOR_ARCHITECTURE" }), [switch] $Clean, [switch] $DebugInfo, [switch] $EnableCaching, @@ -122,8 +127,23 @@ if ($null -ne $env:VSCMD_ARG_HOST_ARCH -or $null -ne $env:VSCMD_ARG_TGT_ARCH) { # Prevent elsewhere-installed swift modules from confusing our builds. $env:SDKROOT = "" -$NativeProcessorArchName = $env:PROCESSOR_ARCHITEW6432 -if ($null -eq $NativeProcessorArchName) { $NativeProcessorArchName = $env:PROCESSOR_ARCHITECTURE } + +$BuildArchName = $env:PROCESSOR_ARCHITEW6432 +if ($null -eq $BuildArchName) { $BuildArchName = $env:PROCESSOR_ARCHITECTURE } + +if ($PinnedBuild -eq "") { + switch ($BuildArchName) { + "AMD64" { + $PinnedBuild = "https://download.swift.org/swift-5.10-branch/windows10/swift-5.10-DEVELOPMENT-SNAPSHOT-2024-01-18-a/swift-5.10-DEVELOPMENT-SNAPSHOT-2024-01-18-a-windows10.exe" + $PinnedSHA256 = "006266d8c2a6a9c70e21b9d161ec35c07bcbb8a452b17e145899d814d07a29e7" + } + "ARM64" { + # TODO(hjyamauchi) once we have an arm64 release, fill in PinnedBuild and PinnedSHA256. + throw "Missing pinned toolchain for ARM64" + } + default { throw "Unsupported processor architecture" } + } +} # Store the revision zero variant of the Windows SDK version (no-op if unspecified) $WindowsSDKMajorMinorBuildMatch = [Regex]::Match($WinSDKVersion, "^\d+\.\d+\.\d+") @@ -132,7 +152,7 @@ $CustomWinSDKRoot = $null # Overwritten if we download a Windows SDK from nuget $vswhere = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Installer\vswhere.exe" $VSInstallRoot = & $vswhere -nologo -latest -products "*" -all -prerelease -property installationPath -$msbuild = "$VSInstallRoot\MSBuild\Current\Bin\$NativeProcessorArchName\MSBuild.exe" +$msbuild = "$VSInstallRoot\MSBuild\Current\Bin\$BuildArchName\MSBuild.exe" # Avoid $env:ProgramFiles in case this script is running as x86 $UnixToolsBinDir = "$env:SystemDrive\Program Files\Git\usr\bin" @@ -189,7 +209,7 @@ $ArchARM64 = @{ ShortName = "arm64"; LLVMName = "aarch64"; LLVMTarget = "aarch64-unknown-windows-msvc"; - CMakeName = "aarch64"; + CMakeName = "ARM64"; BinaryDir = "bin64a"; BuildID = 300; BinaryCache = "$BinaryCache\arm64"; @@ -199,12 +219,20 @@ $ArchARM64 = @{ ToolchainInstallRoot = "$BinaryCache\arm64\toolchains\$ProductVersion+Asserts"; } -$HostArch = switch ($NativeProcessorArchName) { +$HostArch = switch ($HostArchName) { + "AMD64" { $ArchX64 } + "ARM64" { $ArchARM64 } + default { throw "Unsupported processor architecture" } +} + +$BuildArch = switch ($BuildArchName) { "AMD64" { $ArchX64 } "ARM64" { $ArchARM64 } default { throw "Unsupported processor architecture" } } +$IsCrossCompiling = $HostArchName -ne $BuildArchName + $TimingData = New-Object System.Collections.Generic.List[System.Object] function Get-InstallDir($Arch) { @@ -263,9 +291,8 @@ function Get-TargetProjectBinaryCache($Arch, [TargetComponent]$Project) { } enum HostComponent { - BuildTools - Compilers - System + Compilers = 5 + System = 10 ToolsSupportCore LLBuild Yams @@ -293,6 +320,19 @@ function Get-HostProjectCMakeModules([HostComponent]$Project) { return "$BinaryCache\$($Project.value__)\cmake\modules" } +enum BuildComponent { + BuildTools + Compilers +} + +function Get-BuildProjectBinaryCache([BuildComponent]$Project) { + return "$BinaryCache\$($Project.value__)" +} + +function Get-BuildProjectCMakeModules([BuildComponent]$Project) { + return "$BinaryCache\$($Project.value__)\cmake\modules" +} + function Copy-File($Src, $Dst) { # Create the directory tree first so Copy-Item succeeds # If $Dst is the target directory, make sure it ends with "\" @@ -404,7 +444,7 @@ function Isolate-EnvVars([scriptblock]$Block) { } function Invoke-VsDevShell($Arch) { - $DevCmdArguments = "-no_logo -host_arch=$($HostArch.VSName) -arch=$($Arch.VSName)" + $DevCmdArguments = "-no_logo -host_arch=$($BuildArch.VSName) -arch=$($Arch.VSName)" if ($CustomWinSDKRoot) { $DevCmdArguments += " -winsdk=none" } elseif ($WinSDKVersion) { @@ -494,6 +534,28 @@ function Fetch-Dependencies { Invoke-Program -OutNull msiexec.exe /lvx! $BinaryCache\toolchains\$LogFile /qn /a $BinaryCache\toolchains\WixAttachedContainer\$_ ALLUSERS=0 TARGETDIR=$TARGETDIR } + function Download-Python($ArchName) { + $PythonAMD64URL = "https://www.nuget.org/api/v2/package/python/$PythonVersion" + $PythonAMD64Hash = "ac43b491e9488ac926ed31c5594f0c9409a21ecbaf99dc7a93f8c7b24cf85867" + + $PythonARM64URL = "https://www.nuget.org/api/v2/package/pythonarm64/$PythonVersion" + $PythonARM64Hash = "429ada77e7f30e4bd8ff22953a1f35f98b2728e84c9b1d006712561785641f69" + + DownloadAndVerify (Get-Variable -Name "Python${ArchName}URL").Value $BinaryCache\Python$ArchName-$PythonVersion.zip (Get-Variable -Name "Python${ArchName}Hash").Value + + if (-not $ToBatch) { + # TODO(compnerd) stamp/validate that we need to re-extract + New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\Python$ArchName-$PythonVersion | Out-Null + Write-Output "Extracting Python ($ArchName) ..." + Expand-Archive -Path $BinaryCache\Python$ArchName-$PythonVersion.zip -Destination $BinaryCache\Python$ArchName-$PythonVersion -Force + } + } + + Download-Python $HostArchName + if ($IsCrossCompiling) { + Download-Python $BuildArchName + } + if ($WinSDKVersion) { try { # Check whether VsDevShell can already resolve the requested Windows SDK Version @@ -610,7 +672,11 @@ function Build-CMakeProject { Invoke-VsDevShell $Arch } - $CompilersBinaryCache = Get-HostProjectBinaryCache Compilers + $CompilersBinaryCache = if ($IsCrossCompiling) { + Get-BuildProjectBinaryCache Compilers + } else { + Get-HostProjectBinaryCache Compilers + } $DriverBinaryCache = Get-HostProjectBinaryCache Driver if ($EnableCaching) { @@ -629,6 +695,11 @@ function Build-CMakeProject { # Add additional defines (unless already present) $Defines = $Defines.Clone() + if (($Platform -ne "Windows") -or ($Arch.CMakeName -ne $BuildArch.CMakeName)) { + TryAdd-KeyValue $Defines CMAKE_SYSTEM_NAME $Platform + TryAdd-KeyValue $Defines CMAKE_SYSTEM_PROCESSOR $Arch.CMakeName + } + TryAdd-KeyValue $Defines CMAKE_BUILD_TYPE Release TryAdd-KeyValue $Defines CMAKE_MT "mt" @@ -960,11 +1031,12 @@ function Build-CMark($Arch) { function Build-BuildTools($Arch) { Build-CMakeProject ` -Src $SourceCache\llvm-project\llvm ` - -Bin (Get-HostProjectBinaryCache BuildTools) ` + -Bin (Get-BuildProjectBinaryCache BuildTools) ` -Arch $Arch ` -UseMSVCCompilers C,CXX ` -BuildTargets llvm-tblgen,clang-tblgen,clang-pseudo-gen,clang-tidy-confusable-chars-gen,lldb-tblgen,llvm-config,swift-def-to-strings-converter,swift-serialize-diagnostics,swift-compatibility-symbols ` -Defines @{ + CMAKE_CROSSCOMPILING = "NO"; LLDB_ENABLE_PYTHON = "NO"; LLDB_INCLUDE_TESTS = "NO"; LLDB_ENABLE_SWIFT_SUPPORT = "NO"; @@ -1002,12 +1074,17 @@ function Build-Compilers() { [switch]$TestLLD = $false, [switch]$TestLLDB = $false, [switch]$TestLLVM = $false, - [switch]$TestSwift = $false + [switch]$TestSwift = $false, + [switch]$Build = $false ) Isolate-EnvVars { - $CompilersBinaryCache = Get-HostProjectBinaryCache Compilers - $BuildTools = Join-Path -Path (Get-HostProjectBinaryCache BuildTools) -ChildPath bin + $CompilersBinaryCache = if ($Build) { + Get-BuildProjectBinaryCache Compilers + } else { + Get-HostProjectBinaryCache Compilers + } + $BuildTools = Join-Path -Path (Get-BuildProjectBinaryCache BuildTools) -ChildPath bin if ($TestClang -or $TestLLD -or $TestLLDB -or $TestLLVM -or $TestSwift) { $env:Path = "$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$CompilersBinaryCache\tools\swift\libdispatch-windows-$($Arch.LLVMName)-prefix\bin;$CompilersBinaryCache\bin;$env:Path;$VSInstallRoot\DIA SDK\bin\$($HostArch.VSName);$UnixToolsBinDir" @@ -1038,17 +1115,16 @@ function Build-Compilers() { Build-CMakeProject ` -Src $SourceCache\llvm-project\llvm ` - -Bin (Get-HostProjectBinaryCache Compilers) ` + -Bin $CompilersBinaryCache ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` -UseMSVCCompilers C,CXX ` + -UsePinnedCompilers Swift ` -BuildTargets $Targets ` -CacheScript $SourceCache\swift\cmake\caches\Windows-$($Arch.LLVMName).cmake ` -Defines ($TestingDefines + @{ CLANG_TABLEGEN = (Join-Path -Path $BuildTools -ChildPath "clang-tblgen.exe"); CLANG_TIDY_CONFUSABLE_CHARS_GEN = (Join-Path -Path $BuildTools -ChildPath "clang-tidy-confusable-chars-gen.exe"); - CMAKE_Swift_COMPILER = (Join-Path -Path (Get-PinnedToolchainTool) -ChildPath "swiftc.exe"); - CMAKE_Swift_FLAGS = @("-sdk", (Get-PinnedToolchainSDK)); LLDB_PYTHON_EXE_RELATIVE_PATH = "python.exe"; LLDB_PYTHON_EXT_SUFFIX = ".pyd"; LLDB_PYTHON_RELATIVE_PATH = "lib/site-packages"; @@ -1058,6 +1134,10 @@ function Build-Compilers() { LLVM_NATIVE_TOOL_DIR = $BuildTools; LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe"); LLVM_USE_HOST_TOOLS = "NO"; + Python3_EXECUTABLE = "$python"; + Python3_INCLUDE_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\include"; + Python3_LIBRARY = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\libs\python39.lib"; + Python3_ROOT_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools"; SWIFT_BUILD_SWIFT_SYNTAX = "YES"; SWIFT_CLANG_LOCATION = (Get-PinnedToolchainTool); SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY = "YES"; @@ -1082,6 +1162,7 @@ function Build-LLVM([Platform]$Platform, $Arch) { -Bin (Get-TargetProjectBinaryCache $Arch LLVM) ` -Arch $Arch ` -Platform $Platform ` + -UseMSVCCompilers C,CXX ` -Defines @{ CMAKE_SYSTEM_NAME = if ($Platform -eq "Windows") { "Windows" } else { "Android" }; LLVM_HOST_TRIPLE = $Arch.LLVMTarget; @@ -1097,6 +1178,7 @@ function Build-ZLib([Platform]$Platform, $Arch) { -InstallTo $LibraryRoot\zlib-1.3\usr ` -Arch $Arch ` -Platform $Platform ` + -UseMSVCCompilers C ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1115,6 +1197,7 @@ function Build-XML2([Platform]$Platform, $Arch) { -InstallTo "$LibraryRoot\libxml2-2.11.5\usr" ` -Arch $Arch ` -Platform $Platform ` + -UseMSVCCompilers C,CXX ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1147,6 +1230,7 @@ function Build-CURL([Platform]$Platform, $Arch) { -InstallTo "$LibraryRoot\curl-8.4.0\usr" ` -Arch $Arch ` -Platform $Platform ` + -UseMSVCCompilers C ` -BuildTargets default ` -Defines ($PlatformDefines + @{ BUILD_SHARED_LIBS = "NO"; @@ -1243,14 +1327,13 @@ function Build-ICU([Platform]$Platform, $Arch) { } } - if ($Platform -ne "Windows" -or ($Arch -eq $ArchARM64 -and $HostArch -ne $ArchARM64)) { - # Use previously built x64 tools + if ($Platform -eq "Windows" -and (($Arch.CMakeName -eq $BuildArch.CMakeName) -or ($Arch.CMakeName -ne "ARM64"))) { + $BuildToolsDefines = @{BUILD_TOOLS = "YES"} + } else { $BuildToolsDefines = @{ BUILD_TOOLS = "NO"; - ICU_TOOLS_DIR = "$($ArchX64.BinaryCache)\windows\icu-69.1" + ICU_TOOLS_DIR = "$($BuildArch.BinaryCache)\windows\icu-69.1" } - } else { - $BuildToolsDefines = @{BUILD_TOOLS = "YES"} } Build-CMakeProject ` @@ -1259,6 +1342,7 @@ function Build-ICU([Platform]$Platform, $Arch) { -InstallTo "$LibraryRoot\icu-69.1\usr" ` -Arch $Arch ` -Platform $Platform ` + -UseMSVCCompilers C,CXX ` -BuildTargets default ` -Defines ($BuildToolsDefines + @{ BUILD_SHARED_LIBS = "NO"; @@ -1272,6 +1356,12 @@ function Build-Runtime([Platform]$Platform, $Arch) { Isolate-EnvVars { $env:Path = "$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$(Get-PinnedToolchainRuntime);${env:Path}" + $CompilersBinaryCache = if ($IsCrossCompiling) { + Get-BuildProjectBinaryCache Compilers + } else { + Get-HostProjectBinaryCache Compilers + } + Build-CMakeProject ` -Src $SourceCache\swift ` -Bin (Get-TargetProjectBinaryCache $Arch Runtime) ` @@ -1293,7 +1383,7 @@ function Build-Runtime([Platform]$Platform, $Arch) { SWIFT_ENABLE_EXPERIMENTAL_OBSERVATION = "YES"; SWIFT_ENABLE_EXPERIMENTAL_STRING_PROCESSING = "YES"; SWIFT_ENABLE_SYNCHRONIZATION = "YES"; - SWIFT_NATIVE_SWIFT_TOOLS_PATH = (Join-Path -Path (Get-HostProjectBinaryCache Compilers) -ChildPath "bin"); + SWIFT_NATIVE_SWIFT_TOOLS_PATH = (Join-Path -Path $CompilersBinaryCache -ChildPath "bin"); SWIFT_PATH_TO_LIBDISPATCH_SOURCE = "$SourceCache\swift-corelibs-libdispatch"; SWIFT_PATH_TO_STRING_PROCESSING_SOURCE = "$SourceCache\swift-experimental-string-processing"; CMAKE_SHARED_LINKER_FLAGS = @("/INCREMENTAL:NO", "/OPT:REF", "/OPT:ICF"); @@ -1316,8 +1406,6 @@ function Build-Dispatch([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers C,CXX,Swift ` -BuildTargets $Targets ` -Defines @{ - CMAKE_SYSTEM_NAME = "Windows"; - CMAKE_SYSTEM_PROCESSOR = $Arch.CMakeName; ENABLE_SWIFT = "YES"; } } @@ -1351,8 +1439,6 @@ function Build-Foundation([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers ASM,C,Swift ` -BuildTargets $Targets ` -Defines (@{ - CMAKE_SYSTEM_NAME = "Windows"; - CMAKE_SYSTEM_PROCESSOR = $Arch.CMakeName; # Turn off safeseh for lld as it has safeseh enabled by default # and fails with an ICU data object file icudt69l_dat.obj. This # matters to X86 only. @@ -1402,8 +1488,6 @@ function Build-XCTest([Platform]$Platform, $Arch, [switch]$Test = $false) { -UseBuiltCompilers Swift ` -BuildTargets $Targets ` -Defines (@{ - CMAKE_SYSTEM_NAME = "Windows"; - CMAKE_SYSTEM_PROCESSOR = $Arch.CMakeName; dispatch_DIR = "$DispatchBinaryCache\cmake\modules"; Foundation_DIR = "$FoundationBinaryCache\cmake\modules"; } + $TestingDefines) @@ -1518,6 +1602,7 @@ install(FILES sqlite3.h sqlite3ext.h DESTINATION include) -Bin "$($Arch.BinaryCache)\sqlite-3.43.2" ` -InstallTo $LibraryRoot\sqlite-3.43.2\usr ` -Arch $Arch ` + -UseMSVCCompilers C ` -BuildTargets default ` -Defines @{ BUILD_SHARED_LIBS = "NO"; @@ -1559,12 +1644,12 @@ function Build-LLBuild($Arch, [switch]$Test = $false) { # Build additional llvm executables needed by tests Isolate-EnvVars { Invoke-VsDevShell $HostArch - Invoke-Program ninja.exe -C (Get-HostProjectBinaryCache BuildTools) FileCheck not + Invoke-Program ninja.exe -C (Get-BuildProjectBinaryCache BuildTools) FileCheck not } $Targets = @("default", "test-llbuild") $TestingDefines = @{ - FILECHECK_EXECUTABLE = ([IO.Path]::Combine((Get-HostProjectBinaryCache BuildTools), "bin", "FileCheck.exe")); + FILECHECK_EXECUTABLE = ([IO.Path]::Combine((Get-BuildProjectBinaryCache BuildTools), "bin", "FileCheck.exe")); LIT_EXECUTABLE = "$SourceCache\llvm-project\llvm\utils\lit\lit.py"; } $env:Path = "$env:Path;$UnixToolsBinDir" @@ -1598,7 +1683,7 @@ function Build-Yams($Arch) { -Src $SourceCache\Yams ` -Bin (Get-HostProjectBinaryCache Yams) ` -Arch $Arch ` - -UseBuiltCompilers Swift ` + -UseBuiltCompilers C,Swift ` -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` -BuildTargets default ` -Defines @{ @@ -1662,7 +1747,7 @@ function Build-Collections($Arch) { -Bin (Get-HostProjectBinaryCache Collections) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` - -UseBuiltCompilers Swift ` + -UseBuiltCompilers C,Swift ` -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` -BuildTargets default ` -Defines @{ @@ -1675,7 +1760,7 @@ function Build-ASN1($Arch) { -Src $SourceCache\swift-asn1 ` -Bin (Get-HostProjectBinaryCache ASN1) ` -Arch $Arch ` - -UseBuiltCompilers Swift ` + -UseBuiltCompilers C,Swift ` -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` -BuildTargets default ` -Defines @{ @@ -1737,7 +1822,7 @@ function Build-Markdown($Arch) { -Bin (Get-HostProjectBinaryCache Markdown) ` -InstallTo "$($Arch.ToolchainInstallRoot)\usr" ` -Arch $Arch ` - -UseBuiltCompilers Swift ` + -UseBuiltCompilers C,Swift ` -SwiftSDK ([IO.Path]::Combine((Get-InstallDir $HostArch), "Platforms", "Windows.platform", "Developer", "SDKs", "Windows.sdk")) ` -BuildTargets default ` -Defines @{ @@ -1872,13 +1957,18 @@ function Test-PackageManager() { } function Build-Installer($Arch) { + # TODO(hjyamauchi) Re-enable the swift-inspect and swift-docc builds + # when cross-compiling https://github.com/apple/swift/issues/71655 + $INCLUDE_SWIFT_INSPECT = if ($IsCrossCompiling) { "false" } else { "true" } + $INCLUDE_SWIFT_DOCC = if ($IsCrossCompiling) { "false" } else { "true" } + $Properties = @{ BundleFlavor = "offline"; DEVTOOLS_ROOT = "$($Arch.ToolchainInstallRoot)\"; TOOLCHAIN_ROOT = "$($Arch.ToolchainInstallRoot)\"; - INCLUDE_SWIFT_INSPECT = "true"; + INCLUDE_SWIFT_INSPECT = $INCLUDE_SWIFT_INSPECT; SWIFT_INSPECT_BUILD = "$($Arch.BinaryCache)\swift-inspect\release"; - INCLUDE_SWIFT_DOCC = "true"; + INCLUDE_SWIFT_DOCC = $INCLUDE_SWIFT_DOCC; SWIFT_DOCC_BUILD = "$($Arch.BinaryCache)\swift-docc\release"; SWIFT_DOCC_RENDER_ARTIFACT_ROOT = "${SourceCache}\swift-docc-render-artifact"; } @@ -1929,13 +2019,18 @@ if (-not $SkipBuild) { } if (-not $SkipBuild) { + Invoke-BuildStep Build-CMark $BuildArch + Invoke-BuildStep Build-BuildTools $BuildArch + if ($IsCrossCompiling) { + Invoke-BuildStep Build-Compilers -Build $BuildArch + } + Invoke-BuildStep Build-CMark $HostArch - Invoke-BuildStep Build-BuildTools $HostArch Invoke-BuildStep Build-Compilers $HostArch } if ($Clean) { - 2..16 | % { Remove-Item -Force -Recurse "$BinaryCache\$_" -ErrorAction Ignore } + 10..27 | % { Remove-Item -Force -Recurse "$BinaryCache\$_" -ErrorAction Ignore } foreach ($Arch in $WindowsSDKArchs) { 0..3 | % { Remove-Item -Force -Recurse "$BinaryCache\$($Arch.BuildID + $_)" -ErrorAction Ignore } } @@ -1992,7 +2087,7 @@ if (-not $SkipBuild) { Install-HostToolchain -if (-not $SkipBuild) { +if (-not $SkipBuild -and -not $IsCrossCompiling) { Invoke-BuildStep Build-Inspect $HostArch Invoke-BuildStep Build-DocC $HostArch } @@ -2005,28 +2100,30 @@ if ($Stage) { Stage-BuildArtifacts $HostArch } -if ($Test -ne $null -and (Compare-Object $Test @("clang", "lld", "lldb", "llvm", "swift") -PassThru -IncludeEqual -ExcludeDifferent) -ne $null) { - $Tests = @{ - "-TestClang" = $Test -contains "clang"; - "-TestLLD" = $Test -contains "lld"; - "-TestLLDB" = $Test -contains "lldb"; - "-TestLLVM" = $Test -contains "llvm"; - "-TestSwift" = $Test -contains "swift"; +if (-not $IsCrossCompiling) { + if ($Test -ne $null -and (Compare-Object $Test @("clang", "lld", "lldb", "llvm", "swift") -PassThru -IncludeEqual -ExcludeDifferent) -ne $null) { + $Tests = @{ + "-TestClang" = $Test -contains "clang"; + "-TestLLD" = $Test -contains "lld"; + "-TestLLDB" = $Test -contains "lldb"; + "-TestLLVM" = $Test -contains "llvm"; + "-TestSwift" = $Test -contains "swift"; + } + Build-Compilers $HostArch @Tests } - Build-Compilers $HostArch @Tests -} -if ($Test -contains "dispatch") { - Build-Dispatch Windows $HostArch -Test -} -if ($Test -contains "foundation") { - Build-Foundation Windows $HostArch -Test -} -if ($Test -contains "xctest") { - Build-XCTest Windows $HostArch -Test + if ($Test -contains "dispatch") { + Build-Dispatch Windows $HostArch -Test + } + if ($Test -contains "foundation") { + Build-Foundation Windows $HostArch -Test + } + if ($Test -contains "xctest") { + Build-XCTest Windows $HostArch -Test + } + if ($Test -contains "llbuild") { Build-LLBuild $HostArch -Test } + if ($Test -contains "swiftpm") { Test-PackageManager $HostArch } } -if ($Test -contains "llbuild") { Build-LLBuild $HostArch -Test } -if ($Test -contains "swiftpm") { Test-PackageManager $HostArch } # Custom exception printing for more detailed exception information } catch { From 5496362e8810121d976abaa31ce2b7217bc5cbb9 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Tue, 16 Apr 2024 10:31:55 -0700 Subject: [PATCH 2/5] utils: refactor `Fetch-Dependencies` for future changes Future work to enable Android SDKs requires fetching additional dependencies (e.g. Android NDK). Refactor the fetch dependencies method to allow us to share the download logic rather than replicating it for each piece. This will also allow us to fetch the Python dependency for the ARM64 toolchain cross-compilation. (cherry picked from commit 726bf5cabf09d22bc9dfa537650672a733134112) --- utils/build.ps1 | 42 +++++++++++++++++++----------------------- 1 file changed, 19 insertions(+), 23 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 9fc3c48e06789..bb79e88b1bf96 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -486,41 +486,37 @@ function Fetch-Dependencies { $WebClient = New-Object Net.WebClient - $WiXVersion = "4.0.4" - $WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion" - $WiXHash = "A9CA12214E61BB49430A8C6E5E48AC5AE6F27DC82573B5306955C4D35F2D34E2" + function DownloadAndVerify($URL, $Destination, $Hash) { + if (Test-Path $Destination) { + return + } - if (-not (Test-Path $BinaryCache\WiX-$WiXVersion.zip)) { - Write-Output "WiX not found. Downloading from nuget.org ..." - New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache | Out-Null + Write-Output "$Destination not found. Downloading ..." if ($ToBatch) { - Write-Output "curl.exe -sL $WiXURL -o $BinaryCache\WiX-$WiXVersion.zip" + Write-Output "md `"$(Split-Path -Path $Destination -Parent)`"" + Write-Output "curl.exe -sL $URL -o $Destination" + Write-Output "(certutil -HashFile $Destination SHA256) == $Hash || (exit /b)" } else { - $WebClient.DownloadFile($WiXURL, "$BinaryCache\WiX-$WiXVersion.zip") - $SHA256 = Get-FileHash -Path "$BinaryCache\WiX-$WiXVersion.zip" -Algorithm SHA256 - if ($SHA256.Hash -ne $WiXHash) { - throw "WiX SHA256 mismatch ($($SHA256.Hash) vs $WiXHash)" + New-Item -ItemType Directory (Split-Path -Path $Destination -Parent) -ErrorAction Ignore | Out-Null + $WebClient.DownloadFile($URL, $Destination) + $SHA256 = Get-FileHash -Path $Destination -Algorithm SHA256 + if ($SHA256.Hash -ne $Hash) { + throw "SHA256 mismatch ($($SHA256.Hash) vs $Hash)" } } } + $WiXVersion = "4.0.4" + $WiXURL = "https://www.nuget.org/api/v2/package/wix/$WiXVersion" + $WiXHash = "A9CA12214E61BB49430A8C6E5E48AC5AE6F27DC82573B5306955C4D35F2D34E2" + DownloadAndVerify $WixURL "$BinaryCache\WiX-$WiXVersion.zip" $WiXHash + # TODO(compnerd) stamp/validate that we need to re-extract New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\WiX-$WiXVersion | Out-Null Write-Output "Extracting WiX ..." Expand-Archive -Path $BinaryCache\WiX-$WiXVersion.zip -Destination $BinaryCache\WiX-$WiXVersion -Force - if (-not (Test-Path $BinaryCache\$PinnedToolchain.exe)) { - Write-Output "Swift toolchain not found. Downloading from swift.org..." - if ($ToBatch) { - Write-Output "curl.exe -sL $PinnedBuild -o $BinaryCache\$PinnedToolchain.exe" - } else { - $WebClient.DownloadFile("$PinnedBuild", "$BinaryCache\$PinnedToolchain.exe") - $SHA256 = Get-FileHash -Path "$BinaryCache\$PinnedToolchain.exe" -Algorithm SHA256 - if ($SHA256.Hash -ne $PinnedSHA256) { - throw "$PinnedToolchain SHA256 mismatch ($($SHA256.Hash) vs $PinnedSHA256)" - } - } - } + DownloadAndVerify $PinnedBuild "$BinaryCache\$PinnedToolchain.exe" $PinnedSHA256 # TODO(compnerd) stamp/validate that we need to re-extract Write-Output "Extracting $PinnedToolchain ..." From 6586ed8fd376e11b6147d9753caa762668b8df6b Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Tue, 7 May 2024 14:45:44 -0700 Subject: [PATCH 3/5] utils: use cmark gfm for the build when running the compiler When using the just built compiler, we need to use the build CMark runtime as it is executing on the build. This used to work when host and build were guaranteed to be identical (i.e. no cross-compilation). This is now needed to support cross-compiling ARM64. (cherry picked from commit 421c0cf890dc59a9bf2764cbe08da096ac3936e7) --- utils/build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index bb79e88b1bf96..8b993239690bb 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -1350,7 +1350,7 @@ function Build-ICU([Platform]$Platform, $Arch) { function Build-Runtime([Platform]$Platform, $Arch) { Isolate-EnvVars { - $env:Path = "$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$(Get-PinnedToolchainRuntime);${env:Path}" + $env:Path = "$($BuildArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$(Get-PinnedToolchainRuntime);${env:Path}" $CompilersBinaryCache = if ($IsCrossCompiling) { Get-BuildProjectBinaryCache Compilers From 989cf45fe91c309b90005483f1b4d298ac852cf2 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Sat, 18 May 2024 09:30:55 -0700 Subject: [PATCH 4/5] utils: tweak the path before using a Swift toolchain We need to tweak the path before we can use the pinned or the just built toolchain. This becomes a problem when trying to cross-compile the toolchain. (cherry picked from commit 5440760612b7e90aa250a3322bf73524fd0ae175) --- utils/build.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 8b993239690bb..941870f768c19 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -866,7 +866,9 @@ function Build-CMakeProject { } if ($UseBuiltCompilers.Contains("Swift")) { - $env:Path = "$($HostArch.SDKInstallRoot)\usr\bin;$($HostArch.ToolchainInstallRoot)\usr\bin;${env:Path}" + $env:Path = "$($HostArch.SDKInstallRoot)\usr\bin;$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$($HostArch.ToolchainInstallRoot)\usr\bin;${env:Path}" + } else if ($UsePinnedCompilers.Contains("Swift")) { + $env:Path = "$(Get-PinnedToolchainRuntime);${env:Path}" } Invoke-Program cmake.exe @cmakeGenerateArgs @@ -1107,8 +1109,6 @@ function Build-Compilers() { } } - $env:Path = "$(Get-PinnedToolchainRuntime);${env:Path}" - Build-CMakeProject ` -Src $SourceCache\llvm-project\llvm ` -Bin $CompilersBinaryCache ` From 111f0cd6333f3dd66ab2a414e2aa97a312035c86 Mon Sep 17 00:00:00 2001 From: Saleem Abdulrasool Date: Sat, 18 May 2024 12:30:31 -0700 Subject: [PATCH 5/5] Update utils/build.ps1 Co-authored-by: Mishal Shah (cherry picked from commit eb6cd285daaaf0d5756dd89035c11c9fca0ce228) --- utils/build.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 941870f768c19..7d9af28220441 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -867,7 +867,7 @@ function Build-CMakeProject { if ($UseBuiltCompilers.Contains("Swift")) { $env:Path = "$($HostArch.SDKInstallRoot)\usr\bin;$($HostArch.BinaryCache)\cmark-gfm-0.29.0.gfm.13\src;$($HostArch.ToolchainInstallRoot)\usr\bin;${env:Path}" - } else if ($UsePinnedCompilers.Contains("Swift")) { + } elseif ($UsePinnedCompilers.Contains("Swift")) { $env:Path = "$(Get-PinnedToolchainRuntime);${env:Path}" } Invoke-Program cmake.exe @cmakeGenerateArgs