Skip to content

Commit 2e0f986

Browse files
SC llvm teamSC llvm team
SC llvm team
authored and
SC llvm team
committed
Merged main:d486b768238c into amd-gfx:577c5c77a94e
Local branch amd-gfx 577c5c7 Merged main:1b9805c14dba into amd-gfx:0cab57531d07 Remote branch main d486b76 [AArch64] Unroll some loops with early-continues on Apple Silicon. (llvm#118499)
2 parents 577c5c7 + d486b76 commit 2e0f986

File tree

5 files changed

+423
-44
lines changed

5 files changed

+423
-44
lines changed

‎libcxx/docs/Status/Cxx17Papers.csv

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -90,7 +90,7 @@
9090
"`P0521R0 <https://wg21.link/P0521R0>`__","Proposed Resolution for CA 14 (shared_ptr use_count/unique)","2016-11 (Issaquah)","|Complete|","18",""
9191
"","","","","",""
9292
"`P0156R2 <https://wg21.link/P0156R2>`__","Variadic Lock guard(rev 5)","2017-02 (Kona)","|Complete|","5",""
93-
"`P0270R3 <https://wg21.link/P0270R3>`__","Removing C dependencies from signal handler wording","2017-02 (Kona)","","",""
93+
"`P0270R3 <https://wg21.link/P0270R3>`__","Removing C dependencies from signal handler wording","2017-02 (Kona)","|Nothing To Do|","",""
9494
"`P0298R3 <https://wg21.link/P0298R3>`__","A byte type definition","2017-02 (Kona)","|Complete|","5",""
9595
"`P0317R1 <https://wg21.link/P0317R1>`__","Directory Entry Caching for Filesystem","2017-02 (Kona)","|Complete|","7",""
9696
"`P0430R2 <https://wg21.link/P0430R2>`__","File system library on non-POSIX-like operating systems","2017-02 (Kona)","|Complete|","7",""

‎libcxx/test/std/utilities/meta/derived_from_integral_constant.compile.pass.cpp

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
//===----------------------------------------------------------------------===//
2+
//
3+
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4+
// See https://llvm.org/LICENSE.txt for license information.
5+
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6+
//
7+
//===----------------------------------------------------------------------===//
8+
9+
// UNSUPPORTED: c++03
10+
11+
// Check that type traits derive from integral_constant
12+
13+
// ADDITIONAL_COMPILE_FLAGS: -D_LIBCPP_DISABLE_DEPRECATION_WARNINGS
14+
15+
#include <cstddef>
16+
#include <type_traits>
17+
18+
#include "test_macros.h"
19+
20+
static_assert(std::is_base_of<std::false_type, std::is_void<int>>::value, "");
21+
static_assert(std::is_base_of<std::true_type, std::is_integral<int>>::value, "");
22+
static_assert(std::is_base_of<std::false_type, std::is_floating_point<int>>::value, "");
23+
static_assert(std::is_base_of<std::false_type, std::is_array<int>>::value, "");
24+
static_assert(std::is_base_of<std::false_type, std::is_enum<int>>::value, "");
25+
static_assert(std::is_base_of<std::false_type, std::is_union<int>>::value, "");
26+
static_assert(std::is_base_of<std::false_type, std::is_class<int>>::value, "");
27+
static_assert(std::is_base_of<std::false_type, std::is_function<int>>::value, "");
28+
static_assert(std::is_base_of<std::false_type, std::is_pointer<int>>::value, "");
29+
static_assert(std::is_base_of<std::false_type, std::is_lvalue_reference<int>>::value, "");
30+
static_assert(std::is_base_of<std::false_type, std::is_rvalue_reference<int>>::value, "");
31+
static_assert(std::is_base_of<std::false_type, std::is_member_object_pointer<int>>::value, "");
32+
static_assert(std::is_base_of<std::false_type, std::is_member_function_pointer<int>>::value, "");
33+
static_assert(std::is_base_of<std::true_type, std::is_fundamental<int>>::value, "");
34+
static_assert(std::is_base_of<std::true_type, std::is_arithmetic<int>>::value, "");
35+
static_assert(std::is_base_of<std::true_type, std::is_scalar<int>>::value, "");
36+
static_assert(std::is_base_of<std::false_type, std::is_object<int&>>::value, "");
37+
static_assert(std::is_base_of<std::false_type, std::is_compound<int>>::value, "");
38+
static_assert(std::is_base_of<std::false_type, std::is_reference<int>>::value, "");
39+
static_assert(std::is_base_of<std::false_type, std::is_member_pointer<int>>::value, "");
40+
static_assert(std::is_base_of<std::false_type, std::is_const<int>>::value, "");
41+
static_assert(std::is_base_of<std::false_type, std::is_volatile<int>>::value, "");
42+
static_assert(std::is_base_of<std::true_type, std::is_trivial<int>>::value, "");
43+
static_assert(std::is_base_of<std::true_type, std::is_trivially_copyable<int>>::value, "");
44+
static_assert(std::is_base_of<std::true_type, std::is_standard_layout<int>>::value, "");
45+
static_assert(std::is_base_of<std::true_type, std::is_pod<int>>::value, "");
46+
static_assert(std::is_base_of<std::false_type, std::is_empty<int>>::value, "");
47+
static_assert(std::is_base_of<std::false_type, std::is_polymorphic<int>>::value, "");
48+
static_assert(std::is_base_of<std::false_type, std::is_abstract<int>>::value, "");
49+
static_assert(std::is_base_of<std::true_type, std::is_signed<int>>::value, "");
50+
static_assert(std::is_base_of<std::false_type, std::is_unsigned<int>>::value, "");
51+
static_assert(std::is_base_of<std::true_type, std::is_constructible<int>>::value, "");
52+
static_assert(std::is_base_of<std::true_type, std::is_trivially_constructible<int>>::value, "");
53+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_constructible<int>>::value, "");
54+
static_assert(std::is_base_of<std::true_type, std::is_default_constructible<int>>::value, "");
55+
static_assert(std::is_base_of<std::true_type, std::is_trivially_default_constructible<int>>::value, "");
56+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_default_constructible<int>>::value, "");
57+
static_assert(std::is_base_of<std::true_type, std::is_copy_constructible<int>>::value, "");
58+
static_assert(std::is_base_of<std::true_type, std::is_trivially_copy_constructible<int>>::value, "");
59+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_copy_constructible<int>>::value, "");
60+
static_assert(std::is_base_of<std::true_type, std::is_move_constructible<int>>::value, "");
61+
static_assert(std::is_base_of<std::true_type, std::is_trivially_move_constructible<int>>::value, "");
62+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_move_constructible<int>>::value, "");
63+
static_assert(std::is_base_of<std::false_type, std::is_assignable<int, int>>::value, "");
64+
static_assert(std::is_base_of<std::false_type, std::is_trivially_assignable<int, int>>::value, "");
65+
static_assert(std::is_base_of<std::false_type, std::is_nothrow_assignable<int, int>>::value, "");
66+
static_assert(std::is_base_of<std::true_type, std::is_copy_assignable<int>>::value, "");
67+
static_assert(std::is_base_of<std::true_type, std::is_trivially_copy_assignable<int>>::value, "");
68+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_copy_assignable<int>>::value, "");
69+
static_assert(std::is_base_of<std::true_type, std::is_move_assignable<int>>::value, "");
70+
static_assert(std::is_base_of<std::true_type, std::is_trivially_move_assignable<int>>::value, "");
71+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_move_assignable<int>>::value, "");
72+
static_assert(std::is_base_of<std::true_type, std::is_destructible<int>>::value, "");
73+
static_assert(std::is_base_of<std::true_type, std::is_trivially_destructible<int>>::value, "");
74+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_destructible<int>>::value, "");
75+
static_assert(std::is_base_of<std::false_type, std::has_virtual_destructor<int>>::value, "");
76+
static_assert(std::is_base_of<std::integral_constant<std::size_t, 1>, std::alignment_of<char>>::value, "");
77+
static_assert(std::is_base_of<std::integral_constant<std::size_t, 0>, std::rank<char>>::value, "");
78+
static_assert(std::is_base_of<std::integral_constant<std::size_t, 0>, std::extent<char>>::value, "");
79+
static_assert(std::is_base_of<std::true_type, std::is_same<int, int>>::value, "");
80+
static_assert(std::is_base_of<std::false_type, std::is_base_of<int, int>>::value, "");
81+
static_assert(std::is_base_of<std::true_type, std::is_convertible<int, int>>::value, "");
82+
#if TEST_STD_VER <= 17
83+
static_assert(std::is_base_of<std::true_type, std::is_literal_type<int>>::value, "");
84+
#endif
85+
#if TEST_STD_VER >= 14
86+
static_assert(std::is_base_of<std::false_type, std::is_null_pointer<int>>::value, "");
87+
static_assert(std::is_base_of<std::false_type, std::is_final<int>>::value, "");
88+
#endif
89+
#if TEST_STD_VER >= 17
90+
static_assert(std::is_base_of<std::true_type, std::has_unique_object_representations<int>>::value, "");
91+
static_assert(std::is_base_of<std::false_type, std::is_aggregate<int>>::value, "");
92+
static_assert(std::is_base_of<std::false_type, std::is_swappable_with<int, int>>::value, "");
93+
static_assert(std::is_base_of<std::true_type, std::is_swappable<int>>::value, "");
94+
static_assert(std::is_base_of<std::false_type, std::is_nothrow_swappable_with<int, int>>::value, "");
95+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_swappable<int>>::value, "");
96+
static_assert(std::is_base_of<std::false_type, std::is_invocable<int>>::value, "");
97+
static_assert(std::is_base_of<std::false_type, std::is_invocable_r<int, int>>::value, "");
98+
static_assert(std::is_base_of<std::false_type, std::is_nothrow_invocable<int, int>>::value, "");
99+
static_assert(std::is_base_of<std::false_type, std::is_nothrow_invocable_r<int, int>>::value, "");
100+
#endif
101+
#if TEST_STD_VER >= 20
102+
static_assert(std::is_base_of<std::false_type, std::is_bounded_array<int>>::value, "");
103+
static_assert(std::is_base_of<std::false_type, std::is_unbounded_array<int>>::value, "");
104+
static_assert(std::is_base_of<std::true_type, std::is_nothrow_convertible<int, int>>::value, "");
105+
#endif
106+
#if TEST_STD_VER >= 23
107+
# if defined(__cpp_lib_is_implicit_lifetime) && __cpp_lib_is_implicit_lifetime >= 202302L
108+
static_assert(std::is_base_of<std::true_type, std::is_implicit_lifetime<int>>::value, "");
109+
# endif
110+
static_assert(std::is_base_of<std::false_type, std::is_scoped_enum<int>>::value, "");
111+
#endif
112+
#if TEST_STD_VER >= 26
113+
# if defined(__cpp_lib_is_virtual_base_of) && __cpp_lib_is_virtual_base_of >= 202406L
114+
static_assert(std::is_base_of<std::false_type, std::is_virtual_base_of<int, int>>::value, "");
115+
# endif
116+
#endif

‎llvm/include/llvm/Config/llvm-config.h.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,7 @@
1616

1717
/* Indicate that this is LLVM compiled from the amd-gfx branch. */
1818
#define LLVM_HAVE_BRANCH_AMD_GFX
19-
#define LLVM_MAIN_REVISION 522244
19+
#define LLVM_MAIN_REVISION 522247
2020

2121
/* Define if LLVM_ENABLE_DUMP is enabled */
2222
#cmakedefine LLVM_ENABLE_DUMP

‎llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp

Lines changed: 72 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -4085,51 +4085,86 @@ getAppleRuntimeUnrollPreferences(Loop *L, ScalarEvolution &SE,
40854085

40864086
// Try to unroll small, single block loops, if they have load/store
40874087
// dependencies, to expose more parallel memory access streams.
4088-
if (L->getHeader() != L->getLoopLatch() || Size > 8)
4089-
return;
4088+
BasicBlock *Header = L->getHeader();
4089+
if (Header == L->getLoopLatch()) {
4090+
if (Size > 8)
4091+
return;
40904092

4091-
SmallPtrSet<Value *, 8> LoadedValues;
4092-
SmallVector<StoreInst *> Stores;
4093-
for (auto *BB : L->blocks()) {
4094-
for (auto &I : *BB) {
4095-
Value *Ptr = getLoadStorePointerOperand(&I);
4096-
if (!Ptr)
4097-
continue;
4098-
const SCEV *PtrSCEV = SE.getSCEV(Ptr);
4099-
if (SE.isLoopInvariant(PtrSCEV, L))
4100-
continue;
4101-
if (isa<LoadInst>(&I))
4102-
LoadedValues.insert(&I);
4103-
else
4104-
Stores.push_back(cast<StoreInst>(&I));
4093+
SmallPtrSet<Value *, 8> LoadedValues;
4094+
SmallVector<StoreInst *> Stores;
4095+
for (auto *BB : L->blocks()) {
4096+
for (auto &I : *BB) {
4097+
Value *Ptr = getLoadStorePointerOperand(&I);
4098+
if (!Ptr)
4099+
continue;
4100+
const SCEV *PtrSCEV = SE.getSCEV(Ptr);
4101+
if (SE.isLoopInvariant(PtrSCEV, L))
4102+
continue;
4103+
if (isa<LoadInst>(&I))
4104+
LoadedValues.insert(&I);
4105+
else
4106+
Stores.push_back(cast<StoreInst>(&I));
4107+
}
41054108
}
4106-
}
41074109

4108-
// Try to find an unroll count that maximizes the use of the instruction
4109-
// window, i.e. trying to fetch as many instructions per cycle as possible.
4110-
unsigned MaxInstsPerLine = 16;
4111-
unsigned UC = 1;
4112-
unsigned BestUC = 1;
4113-
unsigned SizeWithBestUC = BestUC * Size;
4114-
while (UC <= 8) {
4115-
unsigned SizeWithUC = UC * Size;
4116-
if (SizeWithUC > 48)
4117-
break;
4118-
if ((SizeWithUC % MaxInstsPerLine) == 0 ||
4119-
(SizeWithBestUC % MaxInstsPerLine) < (SizeWithUC % MaxInstsPerLine)) {
4120-
BestUC = UC;
4121-
SizeWithBestUC = BestUC * Size;
4110+
// Try to find an unroll count that maximizes the use of the instruction
4111+
// window, i.e. trying to fetch as many instructions per cycle as possible.
4112+
unsigned MaxInstsPerLine = 16;
4113+
unsigned UC = 1;
4114+
unsigned BestUC = 1;
4115+
unsigned SizeWithBestUC = BestUC * Size;
4116+
while (UC <= 8) {
4117+
unsigned SizeWithUC = UC * Size;
4118+
if (SizeWithUC > 48)
4119+
break;
4120+
if ((SizeWithUC % MaxInstsPerLine) == 0 ||
4121+
(SizeWithBestUC % MaxInstsPerLine) < (SizeWithUC % MaxInstsPerLine)) {
4122+
BestUC = UC;
4123+
SizeWithBestUC = BestUC * Size;
4124+
}
4125+
UC++;
41224126
}
4123-
UC++;
4127+
4128+
if (BestUC == 1 || none_of(Stores, [&LoadedValues](StoreInst *SI) {
4129+
return LoadedValues.contains(SI->getOperand(0));
4130+
}))
4131+
return;
4132+
4133+
UP.Runtime = true;
4134+
UP.DefaultUnrollRuntimeCount = BestUC;
4135+
return;
41244136
}
41254137

4126-
if (BestUC == 1 || none_of(Stores, [&LoadedValues](StoreInst *SI) {
4127-
return LoadedValues.contains(SI->getOperand(0));
4128-
}))
4138+
// Try to runtime-unroll loops with early-continues depending on loop-varying
4139+
// loads; this helps with branch-prediction for the early-continues.
4140+
auto *Term = dyn_cast<BranchInst>(Header->getTerminator());
4141+
auto *Latch = L->getLoopLatch();
4142+
SmallVector<BasicBlock *> Preds(predecessors(Latch));
4143+
if (!Term || !Term->isConditional() || Preds.size() == 1 ||
4144+
none_of(Preds, [Header](BasicBlock *Pred) { return Header == Pred; }) ||
4145+
none_of(Preds, [L](BasicBlock *Pred) { return L->contains(Pred); }))
41294146
return;
41304147

4131-
UP.Runtime = true;
4132-
UP.DefaultUnrollRuntimeCount = BestUC;
4148+
std::function<bool(Instruction *, unsigned)> DependsOnLoopLoad =
4149+
[&](Instruction *I, unsigned Depth) -> bool {
4150+
if (isa<PHINode>(I) || L->isLoopInvariant(I) || Depth > 8)
4151+
return false;
4152+
4153+
if (isa<LoadInst>(I))
4154+
return true;
4155+
4156+
return any_of(I->operands(), [&](Value *V) {
4157+
auto *I = dyn_cast<Instruction>(V);
4158+
return I && DependsOnLoopLoad(I, Depth + 1);
4159+
});
4160+
};
4161+
CmpPredicate Pred;
4162+
Instruction *I;
4163+
if (match(Term, m_Br(m_ICmp(Pred, m_Instruction(I), m_Value()), m_Value(),
4164+
m_Value())) &&
4165+
DependsOnLoopLoad(I, 0)) {
4166+
UP.Runtime = true;
4167+
}
41334168
}
41344169

41354170
void AArch64TTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,

0 commit comments

Comments
 (0)