From 6a842fd38c4068fc005cf4d88bcb8d3caad65056 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 19 Sep 2024 18:04:56 +0800 Subject: [PATCH 1/3] [build.ps1] download Python 3.12+ from Python.org --- utils/build.ps1 | 73 +++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 15 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 47f28a021463f..cf587c782483c 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -122,7 +122,7 @@ param( [string] $PinnedBuild = "", [string] $PinnedSHA256 = "", [string] $PinnedVersion = "", - [string] $PythonVersion = "3.9.10", + [string] $PythonVersion = "3.12.6", [string] $AndroidNDKVersion = "r26b", [string] $WinSDKVersion = "", [switch] $Android = $false, @@ -222,6 +222,7 @@ $ArchX64 = @{ LLVMName = "x86_64"; LLVMTarget = "x86_64-unknown-windows-msvc"; CMakeName = "AMD64"; + PythonName = "amd64"; BinaryDir = "bin64"; BuildID = 100; BinaryCache = "$BinaryCache\x64"; @@ -238,6 +239,7 @@ $ArchX86 = @{ LLVMName = "i686"; LLVMTarget = "i686-unknown-windows-msvc"; CMakeName = "i686"; + PythonName = "win32"; BinaryDir = "bin32"; BuildID = 200; BinaryCache = "$BinaryCache\x86"; @@ -253,6 +255,7 @@ $ArchARM64 = @{ LLVMName = "aarch64"; LLVMTarget = "aarch64-unknown-windows-msvc"; CMakeName = "ARM64"; + PythonName = "arm64"; BinaryDir = "bin64a"; BuildID = 300; BinaryCache = "$BinaryCache\arm64"; @@ -375,6 +378,8 @@ function Get-HostSwiftSDK() { $NugetRoot = "$BinaryCache\nuget" $PinnedToolchain = [IO.Path]::GetFileNameWithoutExtension($PinnedBuild) +$PythonName = "Python{0}{1}" -f ([System.Version]$PythonVersion).Major, ([System.Version]$PythonVersion).Minor + $LibraryRoot = "$ImageRoot\Library" # For dev productivity, install the host toolchain directly using CMake. @@ -726,23 +731,59 @@ function Fetch-Dependencies { New-Item -ItemType Directory -ErrorAction Ignore $BinaryCache\toolchains | Out-Null Extract-Toolchain "$PinnedToolchain.exe" $BinaryCache $PinnedToolchain - function Download-Python($ArchName) { - $PythonAMD64URL = "https://www.nuget.org/api/v2/package/python/$PythonVersion" - $PythonAMD64Hash = "ac43b491e9488ac926ed31c5594f0c9409a21ecbaf99dc7a93f8c7b24cf85867" + function Download-PythonPackage($Version, $FileName) { + $PythonURL = "https://www.python.org/ftp/python/$Version/$FileName" + if (-not (Test-Path "$BinaryCache\$FileName.spdx.json")) { + curl.exe -sL "$PythonURL.spdx.json" -o "$BinaryCache\$FileName.spdx.json" + } + $PythonSBOM = Get-Content -Path "$BinaryCache\$FileName.spdx.json" | ConvertFrom-Json + $PythonSBOM = $PythonSBOM.packages | Where-Object { $_.SPDXID -eq "SPDXRef-PACKAGE-cpython" } + $PythonHash = ($PythonSBOM.checksums | Where-Object { $_.algorithm -eq "SHA256" }).checksumValue + + DownloadAndVerify $PythonURL $BinaryCache\$FileName $PythonHash + } + + function Extract-Python { + param + ( + [string]$InstallerExeName, + [string]$BinaryCache, + [string]$ArchName + ) - $PythonARM64URL = "https://www.nuget.org/api/v2/package/pythonarm64/$PythonVersion" - $PythonARM64Hash = "429ada77e7f30e4bd8ff22953a1f35f98b2728e84c9b1d006712561785641f69" + $source = Join-Path -Path $BinaryCache -ChildPath $InstallerExeName + $destination = Join-Path -Path $BinaryCache -ChildPath python\$PythonVersion\$ArchName - DownloadAndVerify (Get-Variable -Name "Python${ArchName}URL").Value $BinaryCache\Python$ArchName-$PythonVersion.zip (Get-Variable -Name "Python${ArchName}Hash").Value + # Check if the extracted directory already exists and is up to date. + if (Test-Path $destination) { + $installerWriteTime = (Get-Item $source).LastWriteTime + $extractedWriteTime = (Get-Item $destination).LastWriteTime + if ($installerWriteTime -le $extractedWriteTime) { + Write-Output "'$InstallerExeName' is already extracted and up to date." + return + } + } - if (-not $ToBatch) { - Extract-ZipFile Python$ArchName-$PythonVersion.zip $BinaryCache Python$ArchName-$PythonVersion + Write-Output "Extracting '$InstallerExeName' ..." + New-Item -ItemType Directory -ErrorAction Ignore "$BinaryCache\python\$PythonVersion\$ArchName\$PythonName" | Out-Null + Invoke-Program $BinaryCache\WiX-$WiXVersion\tools\net6.0\any\wix.exe -- burn extract $source -out $BinaryCache\python\$PythonVersion\$ArchName\ -outba $BinaryCache\python\$PythonVersion\$ArchName\ + Get-ChildItem "$BinaryCache\python\$PythonVersion\$ArchName\WixAttachedContainer" -Filter "*.msi" | ForEach-Object { + if (-not @("core.msi", "dev.msi", "exe.msi", "lib.msi").Contains($_.Name)) { + return + } + $LogFile = [System.IO.Path]::ChangeExtension($_.Name, "log") + Invoke-Program -OutNull msiexec.exe /lvx! $BinaryCache\python\$PythonVersion\$ArchName\$LogFile /qn /a $BinaryCache\python\$PythonVersion\$ArchName\WixAttachedContainer\$($_.Name) TargetDir=$BinaryCache\python\$PythonVersion\$ArchName\$PythonName } } - Download-Python $HostArchName + function Download-Python($Version, $Arch) { + Download-PythonPackage $Version "python-$Version-$($Arch.PythonName).exe" + Extract-Python "python-$Version-$($Arch.PythonName).exe" $BinaryCache $Arch.PythonName + } + + Download-Python $PythonVersion $HostArch if ($IsCrossCompiling) { - Download-Python $BuildArchName + Download-Python $PythonVersion $BuildArch } if ($Android) { @@ -1411,6 +1452,8 @@ function Build-Compilers() { } } + $PythonRoot = "$BinaryCache\python\$PythonVersion\$($Arch.PythonName)\$PythonName" + # The STL in VS 17.10 requires Clang 17 or higher, but Swift toolchains prior to version 6 include older versions # of Clang. If bootstrapping with an older toolchain, we need to relax to relax this requirement with # ALLOW_COMPILER_AND_STL_VERSION_MISMATCH. @@ -1442,10 +1485,10 @@ function Build-Compilers() { LLVM_NATIVE_TOOL_DIR = $BuildTools; LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe"); LLVM_USE_HOST_TOOLS = "NO"; - Python3_EXECUTABLE = "$python"; - Python3_INCLUDE_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\include"; - Python3_LIBRARY = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools\libs\python39.lib"; - Python3_ROOT_DIR = "$BinaryCache\Python$($Arch.CMakeName)-$PythonVersion\tools"; + Python3_EXECUTABLE = $python; + Python3_INCLUDE_DIR = "$PythonRoot\include"; + Python3_LIBRARY = "$PythonRoot\libs\$($PythonName.ToLower()).lib"; + Python3_ROOT_DIR = $PythonRoot; SWIFT_BUILD_SWIFT_SYNTAX = "YES"; SWIFT_CLANG_LOCATION = (Get-PinnedToolchainTool); SWIFT_ENABLE_EXPERIMENTAL_CONCURRENCY = "YES"; From c04a2498a310f50a0c673fa52b9a7d3627616d61 Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 19 Sep 2024 18:14:46 +0800 Subject: [PATCH 2/3] [build.ps1] ensure required modules in cached Python --- utils/build.ps1 | 37 ++++++++++++++++++++++++++++++++++++- 1 file changed, 36 insertions(+), 1 deletion(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index cf587c782483c..560b8ca385f9e 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -776,9 +776,44 @@ function Fetch-Dependencies { } } + function Ensure-PythonModules($Python) { + # First ensure pip is installed, else bootstrap it + try { + Invoke-Program -OutNull $Python -m pip *> $null + } catch { + Write-Output "Installing pip ..." + Invoke-Program -OutNull $Python '-I' -m ensurepip -U --default-pip + } + # 'packaging' is required for building LLVM 18+ + try { + Invoke-Program -OutNull $Python -c 'import packaging' *> $null + } catch { + $WheelURL = "https://files.pythonhosted.org/packages/08/aa/cc0199a5f0ad350994d660967a8efb233fe0416e4639146c089643407ce6/packaging-24.1-py3-none-any.whl" + $WheelHash = "5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124" + DownloadAndVerify $WheelURL "$BinaryCache\python\packaging-24.1-py3-none-any.whl" $WheelHash + Write-Output "Installing 'packaging-24.1-py3-none-any.whl' ..." + Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\packaging-24.1-py3-none-any.whl" *> $null + } + # 'setuptools' provides 'distutils' module for Python 3.12+, required for SWIG support + # https://github.com/swiftlang/llvm-project/issues/9289 + try { + Invoke-Program -OutNull $Python -c 'import distutils' *> $null + } catch { + $WheelURL = "https://files.pythonhosted.org/packages/ff/ae/f19306b5a221f6a436d8f2238d5b80925004093fa3edea59835b514d9057/setuptools-75.1.0-py3-none-any.whl" + $WheelHash = "35ab7fd3bcd95e6b7fd704e4a1539513edad446c097797f2985e0e4b960772f2" + DownloadAndVerify $WheelURL "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" $WheelHash + Write-Output "Installing 'setuptools-75.1.0-py3-none-any.whl' ..." + Invoke-Program -OutNull $Python '-I' -m pip install "$BinaryCache\python\setuptools-75.1.0-py3-none-any.whl" *> $null + } + } + function Download-Python($Version, $Arch) { Download-PythonPackage $Version "python-$Version-$($Arch.PythonName).exe" Extract-Python "python-$Version-$($Arch.PythonName).exe" $BinaryCache $Arch.PythonName + # Ensure Python modules that are required as host build tools + if ($Arch -eq $HostArch) { + Ensure-PythonModules $BinaryCache\python\$Version\$($Arch.PythonName)\$PythonName\python.exe + } } Download-Python $PythonVersion $HostArch @@ -1485,7 +1520,7 @@ function Build-Compilers() { LLVM_NATIVE_TOOL_DIR = $BuildTools; LLVM_TABLEGEN = (Join-Path $BuildTools -ChildPath "llvm-tblgen.exe"); LLVM_USE_HOST_TOOLS = "NO"; - Python3_EXECUTABLE = $python; + Python3_EXECUTABLE = "$PythonRoot\python.exe"; Python3_INCLUDE_DIR = "$PythonRoot\include"; Python3_LIBRARY = "$PythonRoot\libs\$($PythonName.ToLower()).lib"; Python3_ROOT_DIR = $PythonRoot; From d7057c9da2c093c5d52fbfb98d51a716789ce63e Mon Sep 17 00:00:00 2001 From: YR Chen Date: Thu, 19 Sep 2024 18:27:34 +0800 Subject: [PATCH 3/3] [build.ps1] don't use host Python any more --- utils/build.ps1 | 26 ++++++++++---------------- 1 file changed, 10 insertions(+), 16 deletions(-) diff --git a/utils/build.ps1 b/utils/build.ps1 index 560b8ca385f9e..9008eaf972add 100644 --- a/utils/build.ps1 +++ b/utils/build.ps1 @@ -188,13 +188,7 @@ $WiXVersion = "4.0.5" # Avoid $env:ProgramFiles in case this script is running as x86 $UnixToolsBinDir = "$env:SystemDrive\Program Files\Git\usr\bin" -$python = "${env:ProgramFiles(x86)}\Microsoft Visual Studio\Shared\Python39_64\python.exe" -if (-not (Test-Path $python)) { - $python = (where.exe python) | Select-Object -First 1 - if (-not (Test-Path $python)) { - throw "Python.exe not found" - } -} +$PythonName = "Python{0}{1}" -f ([System.Version]$PythonVersion).Major, ([System.Version]$PythonVersion).Minor if ($Android -and ($AndroidSDKs.Length -eq 0)) { # Enable all android SDKs by default. @@ -355,6 +349,10 @@ function Get-BisonExecutable { return Join-Path -Path $BinaryCache -ChildPath "win_flex_bison\win_bison.exe" } +function Get-PythonExecutable { + return [IO.Path]::Combine($BinaryCache, "python", $PythonVersion, $HostArch.PythonName, $PythonName, "python.exe") +} + function Get-InstallDir($Arch) { if ($Arch -eq $HostArch) { $ProgramFilesName = "Program Files" @@ -378,8 +376,6 @@ function Get-HostSwiftSDK() { $NugetRoot = "$BinaryCache\nuget" $PinnedToolchain = [IO.Path]::GetFileNameWithoutExtension($PinnedBuild) -$PythonName = "Python{0}{1}" -f ([System.Version]$PythonVersion).Major, ([System.Version]$PythonVersion).Minor - $LibraryRoot = "$ImageRoot\Library" # For dev productivity, install the host toolchain directly using CMake. @@ -810,16 +806,14 @@ function Fetch-Dependencies { function Download-Python($Version, $Arch) { Download-PythonPackage $Version "python-$Version-$($Arch.PythonName).exe" Extract-Python "python-$Version-$($Arch.PythonName).exe" $BinaryCache $Arch.PythonName - # Ensure Python modules that are required as host build tools - if ($Arch -eq $HostArch) { - Ensure-PythonModules $BinaryCache\python\$Version\$($Arch.PythonName)\$PythonName\python.exe - } } Download-Python $PythonVersion $HostArch if ($IsCrossCompiling) { Download-Python $PythonVersion $BuildArch } + # Ensure Python modules that are required as host build tools + Ensure-PythonModules "$(Get-PythonExecutable)" if ($Android) { # Only a specific NDK version is supported right now. @@ -1487,7 +1481,7 @@ function Build-Compilers() { } } - $PythonRoot = "$BinaryCache\python\$PythonVersion\$($Arch.PythonName)\$PythonName" + $PythonRoot = Get-PythonExecutable | Split-Path -Parent # The STL in VS 17.10 requires Clang 17 or higher, but Swift toolchains prior to version 6 include older versions # of Clang. If bootstrapping with an older toolchain, we need to relax to relax this requirement with @@ -1793,7 +1787,7 @@ function Build-Runtime([Platform]$Platform, $Arch) { }) } - Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" ` + Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'DEFAULT_USE_RUNTIME': 'MD' } }), encoding='utf-8'))" ` -OutFile "$($Arch.SDKInstallRoot)\SDKSettings.plist" } @@ -2020,7 +2014,7 @@ function Build-Testing([Platform]$Platform, $Arch, [switch]$Test = $false) { function Write-PlatformInfoPlist($Arch) { $PList = Join-Path -Path $Arch.PlatformInstallRoot -ChildPath "Info.plist" - Invoke-Program $python -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` + Invoke-Program "$(Get-PythonExecutable)" -c "import plistlib; print(str(plistlib.dumps({ 'DefaultProperties': { 'XCTEST_VERSION': 'development', 'SWIFT_TESTING_VERSION': 'development', 'SWIFTC_FLAGS': ['-use-ld=lld'] } }), encoding='utf-8'))" ` -OutFile "$PList" }