diff --git a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp index 5e719c6c8cbb7..2d84b4ae1ba5c 100644 --- a/llvm/lib/Transforms/Utils/SimplifyCFG.cpp +++ b/llvm/lib/Transforms/Utils/SimplifyCFG.cpp @@ -84,6 +84,7 @@ #include #include #include +#include #include #include #include @@ -6329,9 +6330,12 @@ static bool initializeUniqueCases(SwitchInst *SI, PHINode *&PHI, // Helper function that checks if it is possible to transform a switch with only // two cases (or two cases + default) that produces a result into a select. // TODO: Handle switches with more than 2 cases that map to the same result. +// The branch weights correspond to the provided Condition (i.e. if Condition is +// modified from the original SwitchInst, the caller must adjust the weights) static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, Constant *DefaultResult, Value *Condition, - IRBuilder<> &Builder, const DataLayout &DL) { + IRBuilder<> &Builder, const DataLayout &DL, + ArrayRef BranchWeights) { // If we are selecting between only two cases transform into a simple // select or a two-way select if default is possible. // Example: @@ -6340,6 +6344,10 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, // case 20: return 2; ----> %2 = icmp eq i32 %a, 20 // default: return 4; %3 = select i1 %2, i32 2, i32 %1 // } + + const bool HasBranchWeights = + !BranchWeights.empty() && !ProfcheckDisableMetadataFixes; + if (ResultVector.size() == 2 && ResultVector[0].second.size() == 1 && ResultVector[1].second.size() == 1) { ConstantInt *FirstCase = ResultVector[0].second[0]; @@ -6348,13 +6356,37 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, if (DefaultResult) { Value *ValueCompare = Builder.CreateICmpEQ(Condition, SecondCase, "switch.selectcmp"); - SelectValue = Builder.CreateSelect(ValueCompare, ResultVector[1].first, - DefaultResult, "switch.select"); + SelectInst *SelectValueInst = cast(Builder.CreateSelect( + ValueCompare, ResultVector[1].first, DefaultResult, "switch.select")); + SelectValue = SelectValueInst; + if (HasBranchWeights) { + // We start with 3 probabilities, where the numerator is the + // corresponding BranchWeights[i], and the denominator is the sum over + // BranchWeights. We want the probability and negative probability of + // Condition == SecondCase. + assert(BranchWeights.size() == 3); + setBranchWeights(SelectValueInst, BranchWeights[2], + BranchWeights[0] + BranchWeights[1], + /*IsExpected=*/false); + } } Value *ValueCompare = Builder.CreateICmpEQ(Condition, FirstCase, "switch.selectcmp"); - return Builder.CreateSelect(ValueCompare, ResultVector[0].first, - SelectValue, "switch.select"); + SelectInst *Ret = cast(Builder.CreateSelect( + ValueCompare, ResultVector[0].first, SelectValue, "switch.select")); + if (HasBranchWeights) { + // We may have had a DefaultResult. Base the position of the first and + // second's branch weights accordingly. Also the proability that Condition + // != FirstCase needs to take that into account. + assert(BranchWeights.size() >= 2); + size_t FirstCasePos = (Condition != nullptr); + size_t SecondCasePos = FirstCasePos + 1; + uint32_t DefaultCase = (Condition != nullptr) ? BranchWeights[0] : 0; + setBranchWeights(Ret, BranchWeights[FirstCasePos], + DefaultCase + BranchWeights[SecondCasePos], + /*IsExpected=*/false); + } + return Ret; } // Handle the degenerate case where two cases have the same result value. @@ -6390,8 +6422,16 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, Value *And = Builder.CreateAnd(Condition, AndMask); Value *Cmp = Builder.CreateICmpEQ( And, Constant::getIntegerValue(And->getType(), AndMask)); - return Builder.CreateSelect(Cmp, ResultVector[0].first, - DefaultResult); + SelectInst *Ret = cast( + Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult)); + if (HasBranchWeights) { + // We know there's a Default case. We base the resulting branch + // weights off its probability. + assert(BranchWeights.size() >= 2); + setBranchWeights(Ret, accumulate(drop_begin(BranchWeights), 0), + BranchWeights[0], /*IsExpected=*/false); + } + return Ret; } } @@ -6408,7 +6448,14 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, Value *And = Builder.CreateAnd(Condition, ~BitMask, "switch.and"); Value *Cmp = Builder.CreateICmpEQ( And, Constant::getNullValue(And->getType()), "switch.selectcmp"); - return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult); + SelectInst *Ret = cast( + Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult)); + if (HasBranchWeights) { + assert(BranchWeights.size() >= 2); + setBranchWeights(Ret, accumulate(drop_begin(BranchWeights), 0), + BranchWeights[0], /*IsExpected=*/false); + } + return Ret; } } @@ -6419,7 +6466,14 @@ static Value *foldSwitchToSelect(const SwitchCaseResultVectorTy &ResultVector, Value *Cmp2 = Builder.CreateICmpEQ(Condition, CaseValues[1], "switch.selectcmp.case2"); Value *Cmp = Builder.CreateOr(Cmp1, Cmp2, "switch.selectcmp"); - return Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult); + SelectInst *Ret = cast( + Builder.CreateSelect(Cmp, ResultVector[0].first, DefaultResult)); + if (HasBranchWeights) { + assert(BranchWeights.size() >= 2); + setBranchWeights(Ret, accumulate(drop_begin(BranchWeights), 0), + BranchWeights[0], /*IsExpected=*/false); + } + return Ret; } } @@ -6480,8 +6534,18 @@ static bool trySwitchToSelect(SwitchInst *SI, IRBuilder<> &Builder, assert(PHI != nullptr && "PHI for value select not found"); Builder.SetInsertPoint(SI); - Value *SelectValue = - foldSwitchToSelect(UniqueResults, DefaultResult, Cond, Builder, DL); + SmallVector BranchWeights; + if (!ProfcheckDisableMetadataFixes) { + [[maybe_unused]] auto HasWeights = + extractBranchWeights(getBranchWeightMDNode(*SI), BranchWeights); + assert(!HasWeights == (BranchWeights.empty())); + } + assert(BranchWeights.empty() || + (BranchWeights.size() >= + UniqueResults.size() + (DefaultResult != nullptr))); + + Value *SelectValue = foldSwitchToSelect(UniqueResults, DefaultResult, Cond, + Builder, DL, BranchWeights); if (!SelectValue) return false; diff --git a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll index fe2e897125eb8..39703e9b53b6b 100644 --- a/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll +++ b/llvm/test/Transforms/SimplifyCFG/switch-to-select-two-case.ll @@ -1,4 +1,4 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals ; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -S | FileCheck %s ; int foo1_with_default(int a) { @@ -11,20 +11,20 @@ ; return 4; ; } -define i32 @foo1_with_default(i32 %a) { +define i32 @foo1_with_default(i32 %a) !prof !0 { ; CHECK-LABEL: @foo1_with_default( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[A:%.*]], 20 -; CHECK-NEXT: [[SWITCH_SELECT:%.*]] = select i1 [[SWITCH_SELECTCMP]], i32 2, i32 4 +; CHECK-NEXT: [[SWITCH_SELECT:%.*]] = select i1 [[SWITCH_SELECTCMP]], i32 2, i32 4, !prof [[PROF1:![0-9]+]] ; CHECK-NEXT: [[SWITCH_SELECTCMP1:%.*]] = icmp eq i32 [[A]], 10 -; CHECK-NEXT: [[SWITCH_SELECT2:%.*]] = select i1 [[SWITCH_SELECTCMP1]], i32 10, i32 [[SWITCH_SELECT]] +; CHECK-NEXT: [[SWITCH_SELECT2:%.*]] = select i1 [[SWITCH_SELECTCMP1]], i32 10, i32 [[SWITCH_SELECT]], !prof [[PROF2:![0-9]+]] ; CHECK-NEXT: ret i32 [[SWITCH_SELECT2]] ; entry: switch i32 %a, label %sw.epilog [ i32 10, label %sw.bb i32 20, label %sw.bb1 - ] + ], !prof !1 sw.bb: br label %return @@ -41,20 +41,20 @@ return: } ; Same as above, but both cases have the same value. -define i32 @same_value(i32 %a) { +define i32 @same_value(i32 %a) !prof !0 { ; CHECK-LABEL: @same_value( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SWITCH_SELECTCMP_CASE1:%.*]] = icmp eq i32 [[A:%.*]], 10 ; CHECK-NEXT: [[SWITCH_SELECTCMP_CASE2:%.*]] = icmp eq i32 [[A]], 20 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = or i1 [[SWITCH_SELECTCMP_CASE1]], [[SWITCH_SELECTCMP_CASE2]] -; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i32 10, i32 4 +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i32 10, i32 4, !prof [[PROF3:![0-9]+]] ; CHECK-NEXT: ret i32 [[TMP0]] ; entry: switch i32 %a, label %sw.epilog [ i32 10, label %sw.bb i32 20, label %sw.bb - ] + ], !prof !1 sw.bb: br label %return @@ -67,17 +67,17 @@ return: ret i32 %retval.0 } -define i1 @switch_to_select_same2_case_results_different_default(i8 %0) { +define i1 @switch_to_select_same2_case_results_different_default(i8 %0) !prof !0 { ; CHECK-LABEL: @switch_to_select_same2_case_results_different_default( ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i8 [[TMP0:%.*]], -5 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %0, label %2 [ i8 4, label %3 i8 0, label %3 - ] + ], !prof !1 2: br label %3 @@ -87,18 +87,18 @@ define i1 @switch_to_select_same2_case_results_different_default(i8 %0) { ret i1 %4 } -define i1 @switch_to_select_same2_case_results_different_default_and_positive_offset_for_case(i8 %0) { +define i1 @switch_to_select_same2_case_results_different_default_and_positive_offset_for_case(i8 %0) !prof !0 { ; CHECK-LABEL: @switch_to_select_same2_case_results_different_default_and_positive_offset_for_case( ; CHECK-NEXT: [[TMP2:%.*]] = sub i8 [[TMP0:%.*]], 43 ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i8 [[TMP2]], -3 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP3:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP3]] ; switch i8 %0, label %2 [ i8 43, label %3 i8 45, label %3 - ] + ], !prof !1 2: br label %3 @@ -108,20 +108,20 @@ define i1 @switch_to_select_same2_case_results_different_default_and_positive_of ret i1 %4 } -define i8 @switch_to_select_same2_case_results_different_default_and_negative_offset_for_case(i32 %i) { +define i8 @switch_to_select_same2_case_results_different_default_and_negative_offset_for_case(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_same2_case_results_different_default_and_negative_offset_for_case( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[I:%.*]], -5 ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i32 [[TMP0]], -3 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i8 3, i8 42 +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i8 3, i8 42, !prof [[PROF3]] ; CHECK-NEXT: ret i8 [[TMP1]] ; entry: switch i32 %i, label %default [ i32 -3, label %end i32 -5, label %end - ] + ], !prof !1 default: br label %end @@ -131,12 +131,12 @@ end: ret i8 %t0 } -define i1 @switch_to_select_same4_case_results_different_default(i32 %i) { +define i1 @switch_to_select_same4_case_results_different_default(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_same4_case_results_different_default( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i32 [[I:%.*]], -7 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF4:![0-9]+]] ; CHECK-NEXT: ret i1 [[TMP0]] ; entry: @@ -145,7 +145,7 @@ entry: i32 2, label %lor.end i32 4, label %lor.end i32 6, label %lor.end - ] + ], !prof !2 lor.rhs: br label %lor.end @@ -155,12 +155,12 @@ lor.end: ret i1 %0 } -define i1 @switch_to_select_same4_case_results_different_default_alt_bitmask(i32 %i) { +define i1 @switch_to_select_same4_case_results_different_default_alt_bitmask(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_same4_case_results_different_default_alt_bitmask( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i32 [[I:%.*]], -11 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP0]] ; entry: @@ -169,7 +169,7 @@ entry: i32 2, label %lor.end i32 8, label %lor.end i32 10, label %lor.end - ] + ], !prof !2 lor.rhs: br label %lor.end @@ -179,13 +179,13 @@ lor.end: ret i1 %0 } -define i1 @switch_to_select_same4_case_results_different_default_positive_offset(i32 %i) { +define i1 @switch_to_select_same4_case_results_different_default_positive_offset(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_same4_case_results_different_default_positive_offset( ; CHECK-NEXT: entry: ; CHECK-NEXT: [[TMP0:%.*]] = sub i32 [[I:%.*]], 2 ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i32 [[TMP0]], -11 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i32 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP1]] ; entry: @@ -194,7 +194,7 @@ entry: i32 4, label %lor.end i32 10, label %lor.end i32 12, label %lor.end - ] + ], !prof !2 lor.rhs: br label %lor.end @@ -204,7 +204,7 @@ lor.end: ret i1 %0 } -define i1 @switch_to_select_invalid_mask(i32 %i) { +define i1 @switch_to_select_invalid_mask(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_invalid_mask( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[I:%.*]], label [[LOR_RHS:%.*]] [ @@ -212,7 +212,7 @@ define i1 @switch_to_select_invalid_mask(i32 %i) { ; CHECK-NEXT: i32 4, label [[LOR_END]] ; CHECK-NEXT: i32 10, label [[LOR_END]] ; CHECK-NEXT: i32 12, label [[LOR_END]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF5:![0-9]+]] ; CHECK: lor.rhs: ; CHECK-NEXT: br label [[LOR_END]] ; CHECK: lor.end: @@ -225,7 +225,7 @@ entry: i32 4, label %lor.end i32 10, label %lor.end i32 12, label %lor.end - ] + ], !prof !2 lor.rhs: br label %lor.end @@ -235,14 +235,14 @@ lor.end: ret i1 %0 } -define i1 @switch_to_select_nonpow2_cases(i32 %i) { +define i1 @switch_to_select_nonpow2_cases(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_nonpow2_cases( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[I:%.*]], label [[LOR_RHS:%.*]] [ ; CHECK-NEXT: i32 0, label [[LOR_END:%.*]] ; CHECK-NEXT: i32 2, label [[LOR_END]] ; CHECK-NEXT: i32 4, label [[LOR_END]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF6:![0-9]+]] ; CHECK: lor.rhs: ; CHECK-NEXT: br label [[LOR_END]] ; CHECK: lor.end: @@ -254,7 +254,7 @@ entry: i32 0, label %lor.end i32 2, label %lor.end i32 4, label %lor.end - ] + ], !prof !3 lor.rhs: br label %lor.end @@ -265,7 +265,7 @@ lor.end: } ; TODO: we can produce the optimal code when there is no default also -define i8 @switch_to_select_two_case_results_no_default(i32 %i) { +define i8 @switch_to_select_two_case_results_no_default(i32 %i) !prof !0 { ; CHECK-LABEL: @switch_to_select_two_case_results_no_default( ; CHECK-NEXT: entry: ; CHECK-NEXT: switch i32 [[I:%.*]], label [[DEFAULT:%.*]] [ @@ -273,7 +273,7 @@ define i8 @switch_to_select_two_case_results_no_default(i32 %i) { ; CHECK-NEXT: i32 2, label [[END]] ; CHECK-NEXT: i32 4, label [[CASE3:%.*]] ; CHECK-NEXT: i32 6, label [[CASE3]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF5]] ; CHECK: case3: ; CHECK-NEXT: br label [[END]] ; CHECK: default: @@ -288,7 +288,7 @@ entry: i32 2, label %case2 i32 4, label %case3 i32 6, label %case4 - ] + ], !prof !2 case1: br label %end @@ -310,12 +310,12 @@ end: ret i8 %t0 } -define i1 @no_range(i8 %f) { +define i1 @no_range(i8 %f) !prof !0 { ; CHECK-LABEL: @no_range( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 60 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 60 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF7:![0-9]+]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -335,7 +335,7 @@ define i1 @no_range(i8 %f) { i8 253, label %bb2 i8 254, label %bb2 i8 255, label %bb2 - ] + ], !prof !4 bb1: br label %bb3 bb2: @@ -345,7 +345,7 @@ bb3: ret i1 %phi } -define i1 @negative_no_range(i8 %f) { +define i1 @negative_no_range(i8 %f) !prof !0 { ; CHECK-LABEL: @negative_no_range( ; CHECK-NEXT: switch i8 [[F:%.*]], label [[BB3:%.*]] [ ; CHECK-NEXT: i8 52, label [[BB2:%.*]] @@ -364,12 +364,12 @@ define i1 @negative_no_range(i8 %f) { ; CHECK-NEXT: i8 -3, label [[BB2]] ; CHECK-NEXT: i8 -2, label [[BB2]] ; CHECK-NEXT: i8 -1, label [[BB2]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF8:![0-9]+]] ; CHECK: bb2: ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: -; CHECK-NEXT: [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] -; CHECK-NEXT: ret i1 [[_0_SROA_0_0]] +; CHECK-NEXT: [[PHI:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i1 [[PHI]] ; switch i8 %f, label %bb1 [ i8 52, label %bb2 @@ -388,7 +388,7 @@ define i1 @negative_no_range(i8 %f) { i8 253, label %bb2 i8 254, label %bb2 i8 255, label %bb2 - ] + ], !prof !4 bb1: br label %bb3 bb2: @@ -400,18 +400,19 @@ bb3: ; Using ranges. -define i1 @range0to4odd(i8 range(i8 0, 4) %f) { +define i1 @range0to4odd(i8 range(i8 0, 4) %f) !prof !0 { ; CHECK-LABEL: @range0to4odd( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ i8 1, label %bb2 i8 3, label %bb2 - ] + ], !prof !1 + bb1: br label %bb3 bb2: @@ -421,18 +422,18 @@ bb3: ret i1 %phi } -define i1 @range1to4odd(i8 range(i8 1, 4) %f) { +define i1 @range1to4odd(i8 range(i8 1, 4) %f) !prof !0 { ; CHECK-LABEL: @range1to4odd( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ i8 1, label %bb2 i8 3, label %bb2 - ] + ], !prof !1 bb1: br label %bb3 bb2: @@ -442,12 +443,12 @@ bb3: ret i1 %phi } -define i1 @range0to8odd(i8 range(i8 0, 8) %f) { +define i1 @range0to8odd(i8 range(i8 0, 8) %f) !prof !0 { ; CHECK-LABEL: @range0to8odd( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 1 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 1 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -455,7 +456,7 @@ define i1 @range0to8odd(i8 range(i8 0, 8) %f) { i8 3, label %bb2 i8 5, label %bb2 i8 7, label %bb2 - ] + ], !prof !2 bb1: br label %bb3 bb2: @@ -465,12 +466,12 @@ bb3: ret i1 %phi } -define i1 @range0to8most_significant_bit(i8 range(i8 0, 8) %f) { +define i1 @range0to8most_significant_bit(i8 range(i8 0, 8) %f) !prof !0 { ; CHECK-LABEL: @range0to8most_significant_bit( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 4 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 4 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -478,7 +479,7 @@ define i1 @range0to8most_significant_bit(i8 range(i8 0, 8) %f) { i8 5, label %bb2 i8 6, label %bb2 i8 7, label %bb2 - ] + ], !prof !2 bb1: br label %bb3 bb2: @@ -488,12 +489,12 @@ bb3: ret i1 %phi } -define i1 @range0to15_middle_two_bits(i8 range(i8 0, 16) %f) { +define i1 @range0to15_middle_two_bits(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @range0to15_middle_two_bits( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 6 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -501,7 +502,8 @@ define i1 @range0to15_middle_two_bits(i8 range(i8 0, 16) %f) { i8 7, label %bb2 i8 14, label %bb2 i8 15, label %bb2 - ] + ], !prof !2 + bb1: br label %bb3 bb2: @@ -511,24 +513,25 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15( ; CHECK-NEXT: switch i8 [[F:%.*]], label [[BB3:%.*]] [ ; CHECK-NEXT: i8 6, label [[BB2:%.*]] ; CHECK-NEXT: i8 7, label [[BB2]] ; CHECK-NEXT: i8 14, label [[BB2]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF6]] ; CHECK: bb2: ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: -; CHECK-NEXT: [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] -; CHECK-NEXT: ret i1 [[_0_SROA_0_0]] +; CHECK-NEXT: [[PHI:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i1 [[PHI]] ; switch i8 %f, label %bb1 [ i8 6, label %bb2 i8 7, label %bb2 i8 14, label %bb2 - ] + ], !prof !3 + bb1: br label %bb3 bb2: @@ -538,19 +541,19 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15_pow_2(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15_pow_2(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15_pow_2( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = sub i8 [[F:%.*]], 6 ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -2 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP1]] ; switch i8 %f, label %bb1 [ i8 6, label %bb2 i8 7, label %bb2 - ] + ], !prof !1 bb1: br label %bb3 bb2: @@ -560,19 +563,19 @@ bb3: ret i1 %phi } -define i1 @negative_range0to5even(i8 range(i8 0, 5) %f) { +define i1 @negative_range0to5even(i8 range(i8 0, 5) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to5even( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = sub i8 [[F:%.*]], 2 ; CHECK-NEXT: [[SWITCH_AND:%.*]] = and i8 [[TMP0]], -3 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = icmp eq i8 [[SWITCH_AND]], 0 -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP1]] ; switch i8 %f, label %bb1 [ i8 2, label %bb2 i8 4, label %bb2 - ] + ], !prof !1 bb1: br label %bb3 bb2: @@ -582,16 +585,17 @@ bb3: ret i1 %phi } -define i1 @range0to15_corner_case(i8 range(i8 0, 16) %f) { +define i1 @range0to15_corner_case(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @range0to15_corner_case( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[COND:%.*]] = icmp eq i8 [[F:%.*]], 15 -; CHECK-NEXT: [[DOT:%.*]] = select i1 [[COND]], i1 true, i1 false +; CHECK-NEXT: [[DOT:%.*]] = select i1 [[COND]], i1 true, i1 false, !prof [[PROF9:![0-9]+]] ; CHECK-NEXT: ret i1 [[DOT]] ; switch i8 %f, label %bb1 [ i8 15, label %bb2 - ] + ], !prof !5 + bb1: br label %bb3 bb2: @@ -601,19 +605,19 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15_corner_case(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15_corner_case(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15_corner_case( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[SWITCH_SELECTCMP_CASE1:%.*]] = icmp eq i8 [[F:%.*]], 15 ; CHECK-NEXT: [[SWITCH_SELECTCMP_CASE2:%.*]] = icmp eq i8 [[F]], 8 ; CHECK-NEXT: [[SWITCH_SELECTCMP:%.*]] = or i1 [[SWITCH_SELECTCMP_CASE1]], [[SWITCH_SELECTCMP_CASE2]] -; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false +; CHECK-NEXT: [[TMP0:%.*]] = select i1 [[SWITCH_SELECTCMP]], i1 true, i1 false, !prof [[PROF3]] ; CHECK-NEXT: ret i1 [[TMP0]] ; switch i8 %f, label %bb1 [ i8 15, label %bb2 - i8 8, label %bb2 - ] + i8 8, label %bb2 + ], !prof !1 bb1: br label %bb3 bb2: @@ -626,12 +630,12 @@ bb3: ; Out of range scenarios. Check if the cases, that have a value out of range ; are eliminated and the optimization is performed. -define i1 @range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) { +define i1 @range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @range0to15_out_of_range_non_prime( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 6 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -640,7 +644,7 @@ define i1 @range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) { i8 14, label %bb2 i8 15, label %bb2 i8 22, label %bb2 - ] + ], !prof !6 bb1: br label %bb3 bb2: @@ -650,12 +654,12 @@ bb3: ret i1 %phi } -define i1 @range0to15_out_of_range_non_prime_more(i8 range(i8 0, 16) %f) { +define i1 @range0to15_out_of_range_non_prime_more(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @range0to15_out_of_range_non_prime_more( ; CHECK-NEXT: bb3: ; CHECK-NEXT: [[TMP0:%.*]] = and i8 [[F:%.*]], 6 ; CHECK-NEXT: [[TMP1:%.*]] = icmp eq i8 [[TMP0]], 6 -; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false +; CHECK-NEXT: [[TMP2:%.*]] = select i1 [[TMP1]], i1 true, i1 false, !prof [[PROF4]] ; CHECK-NEXT: ret i1 [[TMP2]] ; switch i8 %f, label %bb1 [ @@ -665,7 +669,7 @@ define i1 @range0to15_out_of_range_non_prime_more(i8 range(i8 0, 16) %f) { i8 15, label %bb2 i8 22, label %bb2 i8 23, label %bb2 - ] + ], !prof !7 bb1: br label %bb3 bb2: @@ -675,25 +679,25 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15_out_of_range_non_prime(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15_out_of_range_non_prime( ; CHECK-NEXT: switch i8 [[F:%.*]], label [[BB3:%.*]] [ ; CHECK-NEXT: i8 6, label [[BB2:%.*]] ; CHECK-NEXT: i8 14, label [[BB2]] ; CHECK-NEXT: i8 15, label [[BB2]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF6]] ; CHECK: bb2: ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: -; CHECK-NEXT: [[TMP2:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] -; CHECK-NEXT: ret i1 [[TMP2]] +; CHECK-NEXT: [[PHI:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i1 [[PHI]] ; switch i8 %f, label %bb1 [ i8 6, label %bb2 i8 14, label %bb2 i8 15, label %bb2 i8 23, label %bb2 - ] + ], !prof !2 bb1: br label %bb3 bb2: @@ -703,25 +707,25 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15_out_of_range(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15_out_of_range(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15_out_of_range( ; CHECK-NEXT: switch i8 [[F:%.*]], label [[BB3:%.*]] [ ; CHECK-NEXT: i8 6, label [[BB2:%.*]] ; CHECK-NEXT: i8 7, label [[BB2]] ; CHECK-NEXT: i8 14, label [[BB2]] -; CHECK-NEXT: ] +; CHECK-NEXT: ], !prof [[PROF6]] ; CHECK: bb2: ; CHECK-NEXT: br label [[BB3]] ; CHECK: bb3: -; CHECK-NEXT: [[_0_SROA_0_0:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] -; CHECK-NEXT: ret i1 [[_0_SROA_0_0]] +; CHECK-NEXT: [[PHI:%.*]] = phi i1 [ true, [[BB2]] ], [ false, [[TMP0:%.*]] ] +; CHECK-NEXT: ret i1 [[PHI]] ; switch i8 %f, label %bb1 [ i8 6, label %bb2 i8 7, label %bb2 i8 14, label %bb2 - i8 150, label %bb2 - ] + i8 -106, label %bb2 + ], !prof !2 bb1: br label %bb3 bb2: @@ -731,7 +735,7 @@ bb3: ret i1 %phi } -define i1 @negative_range0to15_all_out_of_range(i8 range(i8 0, 16) %f) { +define i1 @negative_range0to15_all_out_of_range(i8 range(i8 0, 16) %f) !prof !0 { ; CHECK-LABEL: @negative_range0to15_all_out_of_range( ; CHECK-NEXT: bb1: ; CHECK-NEXT: ret i1 false @@ -741,7 +745,7 @@ define i1 @negative_range0to15_all_out_of_range(i8 range(i8 0, 16) %f) { i8 23, label %bb2 i8 30, label %bb2 i8 31, label %bb2 - ] + ], !prof !2 bb1: br label %bb3 bb2: @@ -750,3 +754,24 @@ bb3: %phi = phi i1 [ false, %bb1 ], [ true, %bb2 ] ret i1 %phi } + +!0 = !{!"function_entry_count", i64 1000} +!1 = !{!"branch_weights", i32 3, i32 5, i32 7} +!2 = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13} +!3 = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11} +!4 = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13, i32 17, i32 19, i32 23, i32 29, i32 31, i32 37, i32 41, i32 43, i32 47, i32 53, i32 59, i32 61} +!5 = !{!"branch_weights", i32 3, i32 5} +!6 = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13, i32 17} +!7 = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13, i32 17, i32 19} +;. +; CHECK: [[META0:![0-9]+]] = !{!"function_entry_count", i64 1000} +; CHECK: [[PROF1]] = !{!"branch_weights", i32 7, i32 8} +; CHECK: [[PROF2]] = !{!"branch_weights", i32 5, i32 10} +; CHECK: [[PROF3]] = !{!"branch_weights", i32 12, i32 3} +; CHECK: [[PROF4]] = !{!"branch_weights", i32 36, i32 3} +; CHECK: [[PROF5]] = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13} +; CHECK: [[PROF6]] = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11} +; CHECK: [[PROF7]] = !{!"branch_weights", i32 496, i32 3} +; CHECK: [[PROF8]] = !{!"branch_weights", i32 3, i32 5, i32 7, i32 11, i32 13, i32 17, i32 19, i32 23, i32 29, i32 31, i32 37, i32 41, i32 43, i32 47, i32 53, i32 59, i32 61} +; CHECK: [[PROF9]] = !{!"branch_weights", i32 5, i32 3} +;.