From 99c1c80f3adafc8078bda089af49339355c24997 Mon Sep 17 00:00:00 2001 From: Matous Kozak Date: Fri, 5 Sep 2025 17:41:50 +0100 Subject: [PATCH] Refactor iOS runtime measurements --- .../runtime-ios-scenarios-perf-jobs.yml | 18 ++++----- eng/pipelines/templates/runtime-perf-job.yml | 38 +++++++++---------- eng/pipelines/upload-build-artifacts-jobs.yml | 24 ++++++------ scripts/run_performance_job.py | 31 ++++++++++----- 4 files changed, 60 insertions(+), 51 deletions(-) diff --git a/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml index 3c14a3ef488..3daf312b4fa 100644 --- a/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml +++ b/eng/pipelines/runtime-ios-scenarios-perf-jobs.yml @@ -1,5 +1,4 @@ parameters: - hybridGlobalization: True runtimeRepoAlias: runtime performanceRepoAlias: self jobParameters: {} @@ -7,11 +6,10 @@ parameters: jobs: - template: /eng/pipelines/performance/templates/perf-ios-scenarios-build-jobs.yml@${{ parameters.runtimeRepoAlias }} parameters: - hybridGlobalization: ${{ parameters.hybridGlobalization }} mono: true nativeAot: true - # run mono iOS scenarios HybridGlobalization + # run iOS scenarios - Mono FullAOT - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} @@ -21,13 +19,13 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSMono + codeGenType: FullAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSLlvmBuild: False iOSStripSymbols: False - hybridGlobalization: ${{ parameters.hybridGlobalization }} runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} ${{ each parameter in parameters.jobParameters }}: @@ -42,13 +40,13 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSMono + codeGenType: FullAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSLlvmBuild: False iOSStripSymbols: True - hybridGlobalization: ${{ parameters.hybridGlobalization }} additionalJobIdentifier: iOSStripSymbols runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} @@ -64,13 +62,13 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSMono + codeGenType: FullAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSLlvmBuild: True iOSStripSymbols: False - hybridGlobalization: ${{ parameters.hybridGlobalization }} additionalJobIdentifier: iOSLlvmBuild runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} @@ -86,20 +84,20 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSMono + codeGenType: FullAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSLlvmBuild: True iOSStripSymbols: True - hybridGlobalization: ${{ parameters.hybridGlobalization }} additionalJobIdentifier: iOSLlvmBuild iOSStripSymbols runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} ${{ each parameter in parameters.jobParameters }}: ${{ parameter.key }}: ${{ parameter.value }} - # run NativeAOT iOS scenarios HybridGlobalization + # run iOS scenarios - CoreCLR NativeAOT - template: /eng/pipelines/common/platform-matrix.yml@${{ parameters.runtimeRepoAlias }} parameters: jobTemplate: /eng/pipelines/templates/runtime-perf-job.yml@${{ parameters.performanceRepoAlias }} @@ -109,12 +107,12 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSNativeAOT + codeGenType: NativeAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSStripSymbols: False - hybridGlobalization: ${{ parameters.hybridGlobalization }} runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} ${{ each parameter in parameters.jobParameters }}: @@ -129,12 +127,12 @@ jobs: - osx_x64 jobParameters: runtimeType: iOSNativeAOT + codeGenType: NativeAOT projectFile: $(Build.SourcesDirectory)/eng/testing/performance/ios_scenarios.proj runKind: ios_scenarios isScenario: true logicalMachine: 'perfiphone12mini' iOSStripSymbols: True - hybridGlobalization: ${{ parameters.hybridGlobalization }} additionalJobIdentifier: iOSStripSymbols runtimeRepoAlias: ${{ parameters.runtimeRepoAlias }} performanceRepoAlias: ${{ parameters.performanceRepoAlias }} diff --git a/eng/pipelines/templates/runtime-perf-job.yml b/eng/pipelines/templates/runtime-perf-job.yml index 541ea7ff1af..f9036c87354 100644 --- a/eng/pipelines/templates/runtime-perf-job.yml +++ b/eng/pipelines/templates/runtime-perf-job.yml @@ -179,29 +179,29 @@ jobs: # artifactName: 'AndroidBDNApk' # displayName: 'Mono Android BDN Apk' - ${{ elseif or(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.runtimeType, 'iOSNativeAOT')) }}: - # Download iOSMono and Native AOT tests + # Download iOS Mono and CoreCLR (NativeAOT) tests - template: /eng/pipelines/templates/download-artifact-step.yml parameters: unpackFolder: $(builtAppDir)/iosHelloWorld cleanUnpackFolder: false ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppNoLLVMSymbols' + artifactFileName: 'iOSSampleAppNoLLVMSymbols.zip' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppNoLLVMNoSymbols' + artifactFileName: 'iOSSampleAppNoLLVMNoSymbols.zip' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppLLVMSymbols' + artifactFileName: 'iOSSampleAppLLVMSymbols.zip' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppLLVMNoSymbols' + artifactFileName: 'iOSSampleAppLLVMNoSymbols.zip' ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppSymbols' + artifactFileName: 'iOSSampleAppSymbols.zip' ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' - artifactFileName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}.zip' + artifactName: 'iOSSampleAppNoSymbols' + artifactFileName: 'iOSSampleAppNoSymbols.zip' displayName: 'iOS Sample App' # same artifact as above but don't extract .zip - task: DownloadBuildArtifacts@0 @@ -211,17 +211,17 @@ jobs: downloadType: single downloadPath: '$(builtAppDir)/iosHelloWorldZip' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppNoLLVMSymbols' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'False'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppNoLLVMNoSymbols' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppLLVMSymbols' ${{ if and(eq(parameters.runtimeType, 'iOSMono'), eq(parameters.iOSLlvmBuild, 'True'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppLLVMNoSymbols' ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'False')) }}: - artifactName: 'iOSSampleAppSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppSymbols' ${{ if and(eq(parameters.runtimeType, 'iOSNativeAOT'), eq(parameters.iOSStripSymbols, 'True')) }}: - artifactName: 'iOSSampleAppNoSymbolsHybridGlobalization${{parameters.hybridGlobalization}}' + artifactName: 'iOSSampleAppNoSymbols' checkDownloadedFiles: true - task: DownloadBuildArtifacts@0 displayName: 'Download binlog files' diff --git a/eng/pipelines/upload-build-artifacts-jobs.yml b/eng/pipelines/upload-build-artifacts-jobs.yml index 2bb9c463e38..9ef05e1ff4d 100644 --- a/eng/pipelines/upload-build-artifacts-jobs.yml +++ b/eng/pipelines/upload-build-artifacts-jobs.yml @@ -119,14 +119,14 @@ jobs: buildType: 'mono_arm64_ios' dependencyJobName: build_ios_arm64_release_iOSMono artifacts: - - artifactName: 'iOSSampleAppLLVMNoSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppLLVMNoSymbolsHybridGlobalizationtrue.zip' ] - - artifactName: 'iOSSampleAppLLVMSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppLLVMSymbolsHybridGlobalizationtrue.zip' ] - - artifactName: 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppNoLLVMNoSymbolsHybridGlobalizationtrue.zip' ] - - artifactName: 'iOSSampleAppNoLLVMSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppNoLLVMSymbolsHybridGlobalizationtrue.zip' ] + - artifactName: 'iOSSampleAppLLVMNoSymbols' + files: [ 'iOSSampleAppLLVMNoSymbols.zip' ] + - artifactName: 'iOSSampleAppLLVMSymbols' + files: [ 'iOSSampleAppLLVMSymbols.zip' ] + - artifactName: 'iOSSampleAppNoLLVMNoSymbols' + files: [ 'iOSSampleAppNoLLVMNoSymbols.zip' ] + - artifactName: 'iOSSampleAppNoLLVMSymbols' + files: [ 'iOSSampleAppNoLLVMSymbols.zip' ] - ${{ if containsValue(parameters.buildType, 'monoBDN_arm64_android') }}: - template: /eng/pipelines/templates/upload-build-artifacts-job.yml@${{ parameters.performanceRepoAlias }} @@ -143,7 +143,7 @@ jobs: buildType: 'nativeAot_arm64_ios' dependencyJobName: build_ios_arm64_release_iOSNativeAOT artifacts: - - artifactName: 'iOSSampleAppSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppSymbolsHybridGlobalizationtrue.zip' ] - - artifactName: 'iOSSampleAppNoSymbolsHybridGlobalizationtrue' - files: [ 'iOSSampleAppNoSymbolsHybridGlobalizationtrue.zip' ] + - artifactName: 'iOSSampleAppSymbols' + files: [ 'iOSSampleAppSymbols.zip' ] + - artifactName: 'iOSSampleAppNoSymbols' + files: [ 'iOSSampleAppNoSymbols.zip' ] diff --git a/scripts/run_performance_job.py b/scripts/run_performance_job.py index 65d3aff549b..5885564f1b4 100644 --- a/scripts/run_performance_job.py +++ b/scripts/run_performance_job.py @@ -468,10 +468,23 @@ def get_run_configurations( # dotnet/runtime Android sample app scenarios if run_kind == "android_scenarios": + if not runtime_flavor in ("mono", "coreclr"): + raise Exception("Runtime flavor must be specified for runtime android scenarios") configurations["CodegenType"] = str(codegen_type) configurations["LinkingType"] = str(linking_type) configurations["RuntimeType"] = str(runtime_flavor) + # dotnet/runtime iOS sample app scenarios + if run_kind == "ios_scenarios": + if not runtime_flavor in ("mono", "coreclr"): + raise Exception("Runtime flavor must be specified for runtime ios scenarios") + configurations["CodegenType"] = str(codegen_type) + configurations["RuntimeType"] = str(runtime_flavor) + configurations["iOSStripSymbols"] = str(ios_strip_symbols) + + if runtime_flavor == "mono": + configurations["iOSLlvmBuild"] = str(ios_llvm_build) + # .NET Android and .NET MAUI Android sample app scenarios if run_kind == "maui_scenarios_android": if not runtime_flavor in ("mono", "coreclr"): @@ -486,15 +499,6 @@ def get_run_configurations( configurations["CodegenType"] = str(codegen_type) configurations["RuntimeType"] = str(runtime_flavor) - if runtime_type == "iOSMono": - configurations["iOSLlvmBuild"] = str(ios_llvm_build) - configurations["iOSStripSymbols"] = str(ios_strip_symbols) - configurations["RuntimeType"] = "Mono" - - if runtime_type == "iOSNativeAOT": - configurations["iOSStripSymbols"] = str(ios_strip_symbols) - configurations["RuntimeType"] = "NativeAOT" - return configurations def get_work_item_command(os_group: str, target_csproj: str, architecture: str, perf_lab_framework: str, internal: bool, wasm: bool, bdn_artifacts_dir: str): @@ -634,7 +638,14 @@ def run_performance_job(args: RunPerformanceJobArgs): args.runtime_flavor = "coreclr" else: raise Exception("Android scenarios only support Mono and CoreCLR runtimes") - + if args.run_kind == "ios_scenarios": + if args.runtime_type == "iOSMono": + args.runtime_flavor = "mono" + elif args.runtime_type == "iOSNativeAOT": + args.runtime_flavor = "coreclr" + else: + raise Exception("iOS scenarios only support Mono and CoreCLR runtimes") + branch = os.environ.get("BUILD_SOURCEBRANCH") cleaned_branch_name = "main" if branch is not None and branch.startswith("refs/heads/release"):