From 172e94b332b9089a0aa3c4b07ef62b1d09f95876 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Wed, 3 Apr 2024 16:00:49 +0300 Subject: [PATCH 1/3] Remove __DistroRid calculation --- Directory.Build.props | 9 +-- eng/common/cross/build-android-rootfs.sh | 2 +- eng/common/native/init-distro-rid.sh | 78 +------------------ .../templates-official/job/source-build.yml | 2 +- .../templates-official/steps/source-build.yml | 3 +- eng/common/templates/job/source-build.yml | 2 +- eng/common/templates/steps/source-build.yml | 3 +- eng/native/build-commons.sh | 5 +- eng/pipelines/common/global-build-job.yml | 6 +- eng/pipelines/common/platform-matrix.yml | 8 +- eng/pipelines/runtime.yml | 2 +- 11 files changed, 25 insertions(+), 95 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index b1ac2559f9ed1e..0fab591e81b869 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -246,17 +246,16 @@ <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) - <_parseDistroRid>$(__DistroRid) - <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) - <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) + <_parseDistroRid Condition="'$(PortableBuild)' == 'true'">$(_hostRid) + <_lastHyphenIndex Condition="'$(PortableBuild)' == 'true'">$(_parseDistroRid.LastIndexOf('-')) - <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) + <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_parseDistroRid.SubString(0, $(_lastHyphenIndex))) <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) $(_outputOS)-$(TargetArchitecture) diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index f163fb9dae9665..1b2b1d836f4626 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -118,7 +118,7 @@ done cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" -# Generate platform file for build.sh script to assign to __DistroRid +# Generate platform file for build.sh script to assign to __OutputRid echo "Generating platform file..." echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 228be0b1598629..a1ec204cce8594 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,78 +1,5 @@ #!/bin/sh -# getNonPortableDistroRid -# -# Input: -# targetOs: (str) -# targetArch: (str) -# rootfsDir: (str) -# -# Return: -# non-portable rid -getNonPortableDistroRid() -{ - targetOs="$1" - targetArch="$2" - rootfsDir="$3" - nonPortableRid="" - - if [ "$targetOs" = "linux" ]; then - # shellcheck disable=SC1091 - if [ -e "${rootfsDir}/etc/os-release" ]; then - . "${rootfsDir}/etc/os-release" - if [ "${ID}" = "rhel" ] || [ "${ID}" = "rocky" ] || [ "${ID}" = "alpine" ] || [ "${ID}" = "ol" ]; then - VERSION_ID="${VERSION_ID%.*}" # Remove the last version digit for these distros - fi - - if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then - nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" - else - # Rolling release distros either do not set VERSION_ID, set it as blank or - # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); - # so omit it here to be consistent with everything else. - nonPortableRid="${ID}-${targetArch}" - fi - elif [ -e "${rootfsDir}/android_platform" ]; then - # shellcheck disable=SC1091 - . "${rootfsDir}/android_platform" - nonPortableRid="$RID" - fi - fi - - if [ "$targetOs" = "freebsd" ]; then - # $rootfsDir can be empty. freebsd-version is a shell script and should always work. - __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) - nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" - elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then - __android_sdk_version=$(getprop ro.build.version.sdk) - nonPortableRid="android.$__android_sdk_version-${targetArch}" - elif [ "$targetOs" = "illumos" ]; then - __uname_version=$(uname -v) - case "$__uname_version" in - omnios-*) - __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="omnios.$__omnios_major_version-${targetArch}" - ;; - joyent_*) - __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="smartos.$__smartos_major_version-${targetArch}" - ;; - *) - nonPortableRid="illumos-${targetArch}" - ;; - esac - elif [ "$targetOs" = "solaris" ]; then - __uname_version=$(uname -v) - __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) - nonPortableRid="solaris.$__solaris_major_version-${targetArch}" - elif [ "$targetOs" = "haiku" ]; then - __uname_release="$(uname -r)" - nonPortableRid=haiku.r"$__uname_release"-"$targetArch" - fi - - echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' -} - # initDistroRidGlobal # # Input: @@ -86,7 +13,6 @@ getNonPortableDistroRid() # Notes: # It is important to note that the function does not return anything, but it # exports the following variables on success: -# __DistroRid : Non-portable rid of the target platform. # __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. initDistroRidGlobal() { @@ -105,8 +31,6 @@ initDistroRidGlobal() fi fi - __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") - if [ -z "${__PortableTargetOS:-}" ]; then __PortableTargetOS="$targetOs" @@ -122,5 +46,5 @@ initDistroRidGlobal() fi fi - export __DistroRid __PortableTargetOS + export __PortableTargetOS } diff --git a/eng/common/templates-official/job/source-build.yml b/eng/common/templates-official/job/source-build.yml index 50f04e642a3543..d60217130e4806 100644 --- a/eng/common/templates-official/job/source-build.yml +++ b/eng/common/templates-official/job/source-build.yml @@ -12,7 +12,7 @@ parameters: # The name of the job. This is included in the job ID. # targetRID: '' # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false + # portableBuild: true # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # linux-x64), and compiling against distro-provided packages rather than portable ones. # skipPublishValidation: false diff --git a/eng/common/templates-official/steps/source-build.yml b/eng/common/templates-official/steps/source-build.yml index 53ed57b6d48abc..453da0c0a1a951 100644 --- a/eng/common/templates-official/steps/source-build.yml +++ b/eng/common/templates-official/steps/source-build.yml @@ -92,7 +92,8 @@ steps: $targetRidArgs \ $runtimeOsArgs \ $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:PortableBuild=${{ parameters.platform.portableBuild }} \ + /p:OutputRid=${{ parameters.platform.outputRid }} \ /p:ArcadeBuildFromSource=true \ /p:DotNetBuildSourceOnly=true \ /p:DotNetBuildRepo=true \ diff --git a/eng/common/templates/job/source-build.yml b/eng/common/templates/job/source-build.yml index 8a3deef2b72746..5a9d1d22f04cc6 100644 --- a/eng/common/templates/job/source-build.yml +++ b/eng/common/templates/job/source-build.yml @@ -12,7 +12,7 @@ parameters: # The name of the job. This is included in the job ID. # targetRID: '' # The name of the target RID to use, instead of the one auto-detected by Arcade. - # nonPortable: false + # portableBuild: true # Enables non-portable mode. This means a more specific RID (e.g. fedora.32-x64 rather than # linux-x64), and compiling against distro-provided packages rather than portable ones. # skipPublishValidation: false diff --git a/eng/common/templates/steps/source-build.yml b/eng/common/templates/steps/source-build.yml index 32738aa938013e..041ce9ab95cc8f 100644 --- a/eng/common/templates/steps/source-build.yml +++ b/eng/common/templates/steps/source-build.yml @@ -92,7 +92,8 @@ steps: $targetRidArgs \ $runtimeOsArgs \ $baseOsArgs \ - /p:SourceBuildNonPortable=${{ parameters.platform.nonPortable }} \ + /p:PortableBuild=${{ parameters.platform.portableBuild }} \ + /p:OutputRid=${{ parameters.platform.outputRid }} \ /p:ArcadeBuildFromSource=true \ /p:DotNetBuildSourceOnly=true \ /p:DotNetBuildRepo=true \ diff --git a/eng/native/build-commons.sh b/eng/native/build-commons.sh index 0916abb396fa46..f0b20f23d7bb65 100755 --- a/eng/native/build-commons.sh +++ b/eng/native/build-commons.sh @@ -563,11 +563,12 @@ if [[ "$__CrossBuild" == 1 ]]; then fi fi -# init the target distro name (__DistroRid) and target portable os (__PortableTargetOS). +# init the __OutputRid and target portable os (__PortableTargetOS). initTargetDistroRid if [ -z "$__OutputRid" ]; then if [[ "$__PortableBuild" == 0 ]]; then - __OutputRid="$__DistroRid" + echo "ERROR: --outputrid is required for non-portable builds" + exit 1 else __OutputRid="$__PortableTargetOS-$__TargetArch" fi diff --git a/eng/pipelines/common/global-build-job.yml b/eng/pipelines/common/global-build-job.yml index aac4e64b80fb41..1077a56e3f445e 100644 --- a/eng/pipelines/common/global-build-job.yml +++ b/eng/pipelines/common/global-build-job.yml @@ -19,7 +19,8 @@ parameters: shouldContinueOnError: false isOfficialBuild: false isSourceBuild: false - isNonPortableSourceBuild: false + isPortableSourceBuild: true + outputRid: '' runtimeFlavor: 'coreclr' runtimeVariant: '' helixQueues: '' @@ -174,7 +175,8 @@ jobs: platform: baseOS: ${{ parameters.baseOS }} buildScript: $(Build.SourcesDirectory)$(dir)build$(scriptExt) - nonPortable: ${{ parameters.isNonPortableSourceBuild }} + portableBuild: ${{ parameters.isPortableSourceBuild }} + outputRid: ${{ parameters.outputRid }} targetRID: ${{ parameters.targetRid }} name: ${{ parameters.platform }} diff --git a/eng/pipelines/common/platform-matrix.yml b/eng/pipelines/common/platform-matrix.yml index 7245e521680464..164400a7ef8a32 100644 --- a/eng/pipelines/common/platform-matrix.yml +++ b/eng/pipelines/common/platform-matrix.yml @@ -313,7 +313,8 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} isSourceBuild: true - isNonPortableSourceBuild: true + isPortableSourceBuild: false + outputRid: 'centos.8-x64' ${{ insert }}: ${{ parameters.jobParameters }} # Non-existent RID Source Build @@ -336,7 +337,8 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} isSourceBuild: true - isNonPortableSourceBuild: true + isPortableSourceBuild: false + outputRid: 'banana.24-x64' ${{ insert }}: ${{ parameters.jobParameters }} # Portable Linux x64 Source Build @@ -358,7 +360,7 @@ jobs: buildConfig: ${{ parameters.buildConfig }} helixQueueGroup: ${{ parameters.helixQueueGroup }} isSourceBuild: true - isNonPortableSourceBuild: false + isPortableSourceBuild: true ${{ insert }}: ${{ parameters.jobParameters }} # GCC Linux x64 Build diff --git a/eng/pipelines/runtime.yml b/eng/pipelines/runtime.yml index b81da3971a1559..a1946e02abe9cc 100644 --- a/eng/pipelines/runtime.yml +++ b/eng/pipelines/runtime.yml @@ -477,7 +477,7 @@ extends: jobParameters: testScope: innerloop nameSuffix: CoreCLR_NonPortable - buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages -c $(_BuildConfig) /p:PortableBuild=false + buildArgs: -s clr.native+clr.tools+clr.corelib+clr.nativecorelib+clr.aot+clr.packages --outputrid tizen.9.0.0-armel -c $(_BuildConfig) /p:PortableBuild=false timeoutInMinutes: 120 condition: >- or( From 44dc0f6a43b05941254323934d3a7e0924db98ab Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:54:40 +0300 Subject: [PATCH 2/3] Address feedback --- Directory.Build.props | 9 +-- eng/common/cross/build-android-rootfs.sh | 2 +- eng/common/native/init-distro-rid.sh | 74 +++++++++++++++++++++++- eng/native/build-commons.sh | 5 +- 4 files changed, 81 insertions(+), 9 deletions(-) diff --git a/Directory.Build.props b/Directory.Build.props index 0fab591e81b869..b1ac2559f9ed1e 100644 --- a/Directory.Build.props +++ b/Directory.Build.props @@ -246,16 +246,17 @@ <_hostRid Condition="'$(MSBuildRuntimeType)' == 'core'">$([System.Runtime.InteropServices.RuntimeInformation]::RuntimeIdentifier) <_hostRid Condition="'$(MSBuildRuntimeType)' != 'core'">win-$([System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLowerInvariant) - <_parseDistroRid Condition="'$(PortableBuild)' == 'true'">$(_hostRid) - <_lastHyphenIndex Condition="'$(PortableBuild)' == 'true'">$(_parseDistroRid.LastIndexOf('-')) + <_parseDistroRid>$(__DistroRid) + <_parseDistroRid Condition="'$(_parseDistroRid)' == ''">$(_hostRid) + <_distroRidIndex>$(_parseDistroRid.LastIndexOf('-')) - <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_parseDistroRid.SubString(0, $(_lastHyphenIndex))) + <_outputOS>$(_parseDistroRid.SubString(0, $(_distroRidIndex))) <_outputOS Condition="'$(PortableBuild)' == 'true'">$(_portableOS) $(_outputOS)-$(TargetArchitecture) diff --git a/eng/common/cross/build-android-rootfs.sh b/eng/common/cross/build-android-rootfs.sh index 1b2b1d836f4626..f163fb9dae9665 100755 --- a/eng/common/cross/build-android-rootfs.sh +++ b/eng/common/cross/build-android-rootfs.sh @@ -118,7 +118,7 @@ done cp -R "$__TmpDir/data/data/com.termux/files/usr/"* "$__ToolchainDir/sysroot/usr/" -# Generate platform file for build.sh script to assign to __OutputRid +# Generate platform file for build.sh script to assign to __DistroRid echo "Generating platform file..." echo "RID=android.${__ApiLevel}-${__BuildArch}" > $__ToolchainDir/sysroot/android_platform diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index a1ec204cce8594..1db970f13237d8 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -1,5 +1,74 @@ #!/bin/sh +# getNonPortableDistroRid +# +# Input: +# targetOs: (str) +# targetArch: (str) +# rootfsDir: (str) +# +# Return: +# non-portable rid +getNonPortableDistroRid() +{ + targetOs="$1" + targetArch="$2" + rootfsDir="$3" + nonPortableRid="" + + if [ "$targetOs" = "linux" ]; then + # shellcheck disable=SC1091 + if [ -e "${rootfsDir}/etc/os-release" ]; then + . "${rootfsDir}/etc/os-release" + if echo "${VERSION_ID:-}" | grep -qE '^([[:digit:]]|\.)+$'; then + nonPortableRid="${ID}.${VERSION_ID}-${targetArch}" + else + # Rolling release distros either do not set VERSION_ID, set it as blank or + # set it to non-version looking string (such as TEMPLATE_VERSION_ID on ArchLinux); + # so omit it here to be consistent with everything else. + nonPortableRid="${ID}-${targetArch}" + fi + elif [ -e "${rootfsDir}/android_platform" ]; then + # shellcheck disable=SC1091 + . "${rootfsDir}/android_platform" + nonPortableRid="$RID" + fi + fi + + if [ "$targetOs" = "freebsd" ]; then + # $rootfsDir can be empty. freebsd-version is a shell script and should always work. + __freebsd_major_version=$("$rootfsDir"/bin/freebsd-version | cut -d'.' -f1) + nonPortableRid="freebsd.$__freebsd_major_version-${targetArch}" + elif command -v getprop >/dev/null && getprop ro.product.system.model | grep -qi android; then + __android_sdk_version=$(getprop ro.build.version.sdk) + nonPortableRid="android.$__android_sdk_version-${targetArch}" + elif [ "$targetOs" = "illumos" ]; then + __uname_version=$(uname -v) + case "$__uname_version" in + omnios-*) + __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="omnios.$__omnios_major_version-${targetArch}" + ;; + joyent_*) + __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) + nonPortableRid="smartos.$__smartos_major_version-${targetArch}" + ;; + *) + nonPortableRid="illumos-${targetArch}" + ;; + esac + elif [ "$targetOs" = "solaris" ]; then + __uname_version=$(uname -v) + __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1) + nonPortableRid="solaris.$__solaris_major_version-${targetArch}" + elif [ "$targetOs" = "haiku" ]; then + __uname_release="$(uname -r)" + nonPortableRid=haiku.r"$__uname_release"-"$targetArch" + fi + + echo "$nonPortableRid" | tr '[:upper:]' '[:lower:]' +} + # initDistroRidGlobal # # Input: @@ -13,6 +82,7 @@ # Notes: # It is important to note that the function does not return anything, but it # exports the following variables on success: +# __DistroRid : Non-portable rid of the target platform. # __PortableTargetOS : OS-part of the portable rid that corresponds to the target platform. initDistroRidGlobal() { @@ -31,6 +101,8 @@ initDistroRidGlobal() fi fi + __DistroRid=$(getNonPortableDistroRid "${targetOs}" "${targetArch}" "${rootfsDir}") + if [ -z "${__PortableTargetOS:-}" ]; then __PortableTargetOS="$targetOs" @@ -46,5 +118,5 @@ initDistroRidGlobal() fi fi - export __PortableTargetOS + export __DistroRid __PortableTargetOS } diff --git a/eng/native/build-commons.sh b/eng/native/build-commons.sh index f0b20f23d7bb65..0916abb396fa46 100755 --- a/eng/native/build-commons.sh +++ b/eng/native/build-commons.sh @@ -563,12 +563,11 @@ if [[ "$__CrossBuild" == 1 ]]; then fi fi -# init the __OutputRid and target portable os (__PortableTargetOS). +# init the target distro name (__DistroRid) and target portable os (__PortableTargetOS). initTargetDistroRid if [ -z "$__OutputRid" ]; then if [[ "$__PortableBuild" == 0 ]]; then - echo "ERROR: --outputrid is required for non-portable builds" - exit 1 + __OutputRid="$__DistroRid" else __OutputRid="$__PortableTargetOS-$__TargetArch" fi From 66fb899591aca4784d05989c3bce7980c96e0ef1 Mon Sep 17 00:00:00 2001 From: Adeel <3840695+am11@users.noreply.github.com> Date: Wed, 3 Apr 2024 17:57:40 +0300 Subject: [PATCH 3/3] Remove illumos distro versions calculation --- eng/common/native/init-distro-rid.sh | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/eng/common/native/init-distro-rid.sh b/eng/common/native/init-distro-rid.sh index 1db970f13237d8..83ea7aab0e081c 100644 --- a/eng/common/native/init-distro-rid.sh +++ b/eng/common/native/init-distro-rid.sh @@ -44,19 +44,7 @@ getNonPortableDistroRid() nonPortableRid="android.$__android_sdk_version-${targetArch}" elif [ "$targetOs" = "illumos" ]; then __uname_version=$(uname -v) - case "$__uname_version" in - omnios-*) - __omnios_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="omnios.$__omnios_major_version-${targetArch}" - ;; - joyent_*) - __smartos_major_version=$(echo "$__uname_version" | cut -c9-10) - nonPortableRid="smartos.$__smartos_major_version-${targetArch}" - ;; - *) - nonPortableRid="illumos-${targetArch}" - ;; - esac + nonPortableRid="illumos-${targetArch}" elif [ "$targetOs" = "solaris" ]; then __uname_version=$(uname -v) __solaris_major_version=$(echo "$__uname_version" | cut -d'.' -f1)