[BOLT][AArch64] Basic support for Linux kernel #118022
Closed
Conversation
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
FLZ101
requested review from
aaupov,
maksfb,
rafaelauler,
ayermolo and
dcci
as code owners
|
@llvm/pr-subscribers-bolt Author: Franklin (FLZ101) ChangesThis PR mainly makes parts specific to X86 generic s.t. they also work on AArch64. Full diff: https://github.com/llvm/llvm-project/pull/118022.diff 4 Files Affected:
diff --git a/bolt/include/bolt/Core/BinaryContext.h b/bolt/include/bolt/Core/BinaryContext.h
index c9b0e103ed5145..b24df78d994231 100644
--- a/bolt/include/bolt/Core/BinaryContext.h
+++ b/bolt/include/bolt/Core/BinaryContext.h
@@ -612,6 +612,9 @@ class BinaryContext {
/// Addresses reserved for kernel on x86_64 start at this location.
static constexpr uint64_t KernelStartX86_64 = 0xFFFF'FFFF'8000'0000;
+ /// https://www.kernel.org/doc/html/v6.12/arch/arm64/memory.html
+ static constexpr uint64_t KernelStartAArch64 = 0xFFF0'0000'0000'0000;
+
/// Map address to a constant island owner (constant data in code section)
std::map<uint64_t, BinaryFunction *> AddressToConstantIslandMap;
diff --git a/bolt/lib/Core/JumpTable.cpp b/bolt/lib/Core/JumpTable.cpp
index 65e1032c579b5a..d3ca951d7e453d 100644
--- a/bolt/lib/Core/JumpTable.cpp
+++ b/bolt/lib/Core/JumpTable.cpp
@@ -85,7 +85,7 @@ void bolt::JumpTable::updateOriginal() {
uint64_t EntryOffset = BaseOffset;
for (MCSymbol *Entry : Entries) {
const uint64_t RelType =
- Type == JTT_NORMAL ? ELF::R_X86_64_64 : ELF::R_X86_64_PC32;
+ Type == JTT_NORMAL ? Relocation::getAbs64() : Relocation::getPC32();
const uint64_t RelAddend =
Type == JTT_NORMAL ? 0 : EntryOffset - BaseOffset;
// Replace existing relocation with the new one to allow any modifications
diff --git a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
index 03b414b71caca7..5023cd98ebf03d 100644
--- a/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
+++ b/bolt/lib/Rewrite/LinuxKernelRewriter.cpp
@@ -196,7 +196,6 @@ class LinuxKernelRewriter final : public MetadataRewriter {
/// Section containing the Linux exception table.
ErrorOr<BinarySection &> ExceptionsSection = std::errc::bad_address;
- static constexpr size_t EXCEPTION_TABLE_ENTRY_SIZE = 12;
/// Functions with exception handling code.
DenseSet<BinaryFunction *> FunctionsWithExceptions;
@@ -225,6 +224,17 @@ class LinuxKernelRewriter final : public MetadataRewriter {
ErrorOr<BinarySection &> PCIFixupSection = std::errc::bad_address;
static constexpr size_t PCI_FIXUP_ENTRY_SIZE = 16;
+ size_t getExceptionTableEntrySize() {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ return 12;
+ case llvm::Triple::aarch64:
+ return 8;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+ }
+
/// Process linux kernel special sections and their relocations.
void processLKSections();
@@ -474,8 +484,8 @@ void LinuxKernelRewriter::processInstructionFixups() {
continue;
Fixup.Section.addRelocation(Fixup.Offset, &Fixup.Label,
- Fixup.IsPCRelative ? ELF::R_X86_64_PC32
- : ELF::R_X86_64_64,
+ Fixup.IsPCRelative ? Relocation::getPC32()
+ : Relocation::getAbs64(),
/*Addend*/ 0);
}
}
@@ -998,7 +1008,7 @@ Error LinuxKernelRewriter::rewriteStaticCalls() {
StaticCallSection->getAddress() +
(Entry.ID - 1) * STATIC_CALL_ENTRY_SIZE;
StaticCallSection->addRelocation(EntryOffset, Entry.Label,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ Relocation::getPC32(), /*Addend*/ 0);
}
return Error::success();
@@ -1023,7 +1033,8 @@ Error LinuxKernelRewriter::readExceptionTable() {
if (!ExceptionsSection)
return Error::success();
- if (ExceptionsSection->getSize() % EXCEPTION_TABLE_ENTRY_SIZE)
+ size_t EntrySize = getExceptionTableEntrySize();
+ if (ExceptionsSection->getSize() % EntrySize)
return createStringError(errc::executable_format_error,
"exception table size error");
@@ -1038,7 +1049,19 @@ Error LinuxKernelRewriter::readExceptionTable() {
SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
const uint64_t FixupAddress =
SectionAddress + Cursor.tell() + (int32_t)DE.getU32(Cursor);
- const uint64_t Data = DE.getU32(Cursor);
+
+ auto ReadData = [this, &DE, &Cursor]() {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ return DE.getU32(Cursor);
+ case llvm::Triple::aarch64:
+ return 0U;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+ };
+
+ const uint64_t Data = ReadData();
// Consume the status of the cursor.
if (!Cursor)
@@ -1100,8 +1123,7 @@ Error LinuxKernelRewriter::readExceptionTable() {
}
}
- BC.outs() << "BOLT-INFO: parsed "
- << ExceptionsSection->getSize() / EXCEPTION_TABLE_ENTRY_SIZE
+ BC.outs() << "BOLT-INFO: parsed " << ExceptionsSection->getSize() / EntrySize
<< " exception table entries\n";
return Error::success();
@@ -1305,7 +1327,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
MCSymbol *Label =
BC.MIB->getOrCreateInstLabel(Inst, "__BUG_", BC.Ctx.get());
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, Label, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, Label,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1315,7 +1338,8 @@ Error LinuxKernelRewriter::rewriteBugTable() {
for (const uint32_t ID : FunctionBugList[&BF]) {
if (!EmittedIDs.count(ID)) {
const uint64_t EntryOffset = (ID - 1) * BUG_TABLE_ENTRY_SIZE;
- BugTableSection->addRelocation(EntryOffset, nullptr, ELF::R_X86_64_PC32,
+ BugTableSection->addRelocation(EntryOffset, nullptr,
+ Relocation::getPC32(),
/*Addend*/ 0);
}
}
@@ -1589,6 +1613,41 @@ Error LinuxKernelRewriter::readPCIFixupTable() {
return Error::success();
}
+static bool checkStaticKeysJumpInstSize(const BinaryContext &BC, size_t Size) {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ return Size == 2 || Size == 5;
+ case llvm::Triple::aarch64:
+ return Size == 4;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+}
+
+static bool checkStaticKeysJumpInstSize(const BinaryContext &BC, size_t Size,
+ uint64_t &NumShort, uint64_t &NumLong) {
+ switch (BC.TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Size == 2) {
+ ++NumShort;
+ return true;
+ }
+ if (Size == 5) {
+ ++NumLong;
+ return true;
+ }
+ return false;
+ case llvm::Triple::aarch64:
+ if (Size == 4) {
+ ++NumLong;
+ return true;
+ }
+ return false;
+ default:
+ llvm_unreachable("Unsupported architecture");
+ }
+}
+
/// Runtime code modification used by static keys is the most ubiquitous
/// self-modifying feature of the Linux kernel. The idea is to eliminate the
/// condition check and associated conditional jump on a hot path if that
@@ -1719,7 +1778,7 @@ Error LinuxKernelRewriter::readStaticKeysJumpTable() {
JumpAddress);
const uint64_t Size = BC.computeInstructionSize(*Inst);
- if (Size != 2 && Size != 5) {
+ if (!checkStaticKeysJumpInstSize(BC, Size)) {
return createStringError(
errc::executable_format_error,
"unexpected static keys jump size at address 0x%" PRIx64,
@@ -1805,11 +1864,7 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
const bool IsBranch = Info.Likely ^ Info.InitValue;
uint32_t Size = *BC.MIB->getSize(Inst);
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
+ if (!checkStaticKeysJumpInstSize(BC, Size, NumShort, NumLong))
llvm_unreachable("Wrong size for static keys jump instruction.");
MCInst NewInst;
@@ -1839,10 +1894,10 @@ Error LinuxKernelRewriter::rewriteStaticKeysJumpTable() {
StaticKeysJumpSection->getAddress() +
(EntryID - 1) * 16;
StaticKeysJumpSection->addRelocation(EntryOffset, Label,
- ELF::R_X86_64_PC32,
+ Relocation::getPC32(),
/*Addend*/ 0);
- StaticKeysJumpSection->addRelocation(EntryOffset + 4, Target,
- ELF::R_X86_64_PC32, /*Addend*/ 0);
+ StaticKeysJumpSection->addRelocation(
+ EntryOffset + 4, Target, Relocation::getPC32(), /*Addend*/ 0);
}
}
}
@@ -1915,11 +1970,7 @@ Error LinuxKernelRewriter::updateStaticKeysJumpTablePostEmit() {
}
assert(BC.MIB->isBranch(Inst) && "Branch instruction expected.");
- if (Size == 2)
- ++NumShort;
- else if (Size == 5)
- ++NumLong;
- else
+ if (!checkStaticKeysJumpInstSize(BC, Size, NumShort, NumLong))
llvm_unreachable("Unexpected size for static keys jump instruction.");
// Check if we need to convert jump instruction into a nop.
diff --git a/bolt/lib/Rewrite/RewriteInstance.cpp b/bolt/lib/Rewrite/RewriteInstance.cpp
index 7059a3dd231099..99b38f104ccb88 100644
--- a/bolt/lib/Rewrite/RewriteInstance.cpp
+++ b/bolt/lib/Rewrite/RewriteInstance.cpp
@@ -529,9 +529,18 @@ Error RewriteInstance::discoverStorage() {
BC->SegmentMapInfo[Phdr.p_vaddr] = SegmentInfo{
Phdr.p_vaddr, Phdr.p_memsz, Phdr.p_offset,
Phdr.p_filesz, Phdr.p_align, ((Phdr.p_flags & ELF::PF_X) != 0)};
- if (BC->TheTriple->getArch() == llvm::Triple::x86_64 &&
- Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
- BC->IsLinuxKernel = true;
+
+ switch (BC->TheTriple->getArch()) {
+ case llvm::Triple::x86_64:
+ if (Phdr.p_vaddr >= BinaryContext::KernelStartX86_64)
+ BC->IsLinuxKernel = true;
+ break;
+ case llvm::Triple::aarch64:
+ if (Phdr.p_vaddr >= BinaryContext::KernelStartAArch64)
+ BC->IsLinuxKernel = true;
+ break;
+ }
+
break;
case ELF::PT_INTERP:
BC->HasInterpHeader = true;
|
|
Tests will be pushed soon. |
Make parts specific to X86 also work on AArch64
FLZ101
force-pushed
the
feature-bolt-aarch64-linux-kernel
branch
from
4b0de86 to
07b5962
Compare
maksfb
reviewed
Dec 3, 2024
| @@ -1023,7 +1033,8 @@ Error LinuxKernelRewriter::readExceptionTable() { | |||
| if (!ExceptionsSection) | |||
| return Error::success(); | |||
|
|
|||
| if (ExceptionsSection->getSize() % EXCEPTION_TABLE_ENTRY_SIZE) | |||
| size_t EntrySize = getExceptionTableEntrySize(); | |||
There was a problem hiding this comment.
Suggested change
| size_t EntrySize = getExceptionTableEntrySize(); | |
| const size_t EntrySize = getExceptionTableEntrySize(); |
Sign up for free
to join this conversation on GitHub.
Already have an account?
Sign in to comment
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
Make parts specific to X86 also work on AArch64.