From 559fe8e78990997821defc9b97190a97bb44c841 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stefan=20Gr=C3=A4nitz?= Date: Sat, 13 Jan 2024 23:58:36 +0100 Subject: [PATCH] [JITLink][AArch32] Multi-stub support for armv7/thumbv7 --- .../llvm/ExecutionEngine/JITLink/aarch32.h | 59 ++------- llvm/lib/ExecutionEngine/JITLink/aarch32.cpp | 125 ++++++++++++++++-- .../JITLink/AArch32/ELF_stubs_arm.s | 53 ++++++++ .../JITLink/AArch32/ELF_stubs_multi.s | 50 +++++++ llvm/tools/llvm-jitlink/llvm-jitlink.cpp | 46 ++++++- 5 files changed, 274 insertions(+), 59 deletions(-) create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_arm.s create mode 100644 llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_multi.s diff --git a/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h b/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h index 081f77a85e182..ed53fa409ade8 100644 --- a/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h +++ b/llvm/include/llvm/ExecutionEngine/JITLink/aarch32.h @@ -341,64 +341,31 @@ class GOTBuilder : public TableManager { Section *GOTSection = nullptr; }; -/// Stubs builder for v7 emits non-position-independent Thumb stubs. -/// -/// Right now we only have one default stub kind, but we want to extend this -/// and allow creation of specific kinds in the future (e.g. branch range -/// extension or interworking). -/// -/// Let's keep it simple for the moment and not wire this through a GOT. -/// -class StubsManager_v7 : public TableManager { +/// Stubs builder for v7 emits non-position-independent Arm and Thumb stubs. +class StubsManager_v7 { public: StubsManager_v7() = default; /// Name of the object file section that will contain all our stubs. static StringRef getSectionName() { - return "__llvm_jitlink_aarch32_STUBS_Thumbv7"; + return "__llvm_jitlink_aarch32_STUBS_v7"; } /// Implements link-graph traversal via visitExistingEdges(). - bool visitEdge(LinkGraph &G, Block *B, Edge &E) { - if (E.getTarget().isDefined()) - return false; - - switch (E.getKind()) { - case Thumb_Call: - case Thumb_Jump24: { - DEBUG_WITH_TYPE("jitlink", { - dbgs() << " Fixing " << G.getEdgeKindName(E.getKind()) << " edge at " - << B->getFixupAddress(E) << " (" << B->getAddress() << " + " - << formatv("{0:x}", E.getOffset()) << ")\n"; - }); - E.setTarget(this->getEntryForTarget(G, E.getTarget())); - return true; - } - } - return false; - } - - /// Create a branch range extension stub with Thumb encoding for v7 CPUs. - Symbol &createEntry(LinkGraph &G, Symbol &Target); + bool visitEdge(LinkGraph &G, Block *B, Edge &E); private: - /// Create a new node in the link-graph for the given stub template. - template - Block &addStub(LinkGraph &G, const uint8_t (&Code)[Size], - uint64_t Alignment) { - ArrayRef Template(reinterpret_cast(Code), Size); - return G.createContentBlock(getStubsSection(G), Template, - orc::ExecutorAddr(), Alignment, 0); - } - - /// Get or create the object file section that will contain all our stubs. - Section &getStubsSection(LinkGraph &G) { - if (!StubsSection) - StubsSection = &G.createSection(getSectionName(), - orc::MemProt::Read | orc::MemProt::Exec); - return *StubsSection; + // Two slots per external: Arm and Thumb + using StubMapEntry = std::tuple; + + Symbol *&getStubSymbolSlot(StringRef Name, bool Thumb) { + StubMapEntry &Stubs = StubMap.try_emplace(Name).first->second; + if (Thumb) + return std::get<1>(Stubs); + return std::get<0>(Stubs); } + DenseMap StubMap; Section *StubsSection = nullptr; }; diff --git a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp index 1797a0068cd7c..9508cde07b42a 100644 --- a/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/aarch32.cpp @@ -15,6 +15,7 @@ #include "llvm/ADT/StringExtras.h" #include "llvm/BinaryFormat/ELF.h" #include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/ExecutionEngine/Orc/Shared/MemoryFlags.h" #include "llvm/Object/ELFObjectFile.h" #include "llvm/Support/Endian.h" #include "llvm/Support/ManagedStatic.h" @@ -724,27 +725,127 @@ bool GOTBuilder::visitEdge(LinkGraph &G, Block *B, Edge &E) { return true; } +const uint8_t Armv7ABS[] = { + 0x00, 0xc0, 0x00, 0xe3, // movw r12, #0x0000 ; lower 16-bit + 0x00, 0xc0, 0x40, 0xe3, // movt r12, #0x0000 ; upper 16-bit + 0x1c, 0xff, 0x2f, 0xe1 // bx r12 +}; + const uint8_t Thumbv7ABS[] = { 0x40, 0xf2, 0x00, 0x0c, // movw r12, #0x0000 ; lower 16-bit 0xc0, 0xf2, 0x00, 0x0c, // movt r12, #0x0000 ; upper 16-bit 0x60, 0x47 // bx r12 }; -Symbol &StubsManager_v7::createEntry(LinkGraph &G, Symbol &Target) { +/// Create a new node in the link-graph for the given stub template. +template +static Block &allocStub(LinkGraph &G, Section &S, const uint8_t (&Code)[Size]) { constexpr uint64_t Alignment = 4; - Block &B = addStub(G, Thumbv7ABS, Alignment); - LLVM_DEBUG({ - const char *StubPtr = B.getContent().data(); - HalfWords Reg12 = encodeRegMovtT1MovwT3(12); - assert(checkRegister(StubPtr, Reg12) && - checkRegister(StubPtr + 4, Reg12) && - "Linker generated stubs may only corrupt register r12 (IP)"); - }); + ArrayRef Template(reinterpret_cast(Code), Size); + return G.createContentBlock(S, Template, orc::ExecutorAddr(), Alignment, 0); +} + +static Block &createStubThumbv7(LinkGraph &G, Section &S, Symbol &Target) { + Block &B = allocStub(G, S, Thumbv7ABS); B.addEdge(Thumb_MovwAbsNC, 0, Target, 0); B.addEdge(Thumb_MovtAbs, 4, Target, 0); - Symbol &Stub = G.addAnonymousSymbol(B, 0, B.getSize(), true, false); - Stub.setTargetFlags(ThumbSymbol); - return Stub; + + [[maybe_unused]] const char *StubPtr = B.getContent().data(); + [[maybe_unused]] HalfWords Reg12 = encodeRegMovtT1MovwT3(12); + assert(checkRegister(StubPtr, Reg12) && + checkRegister(StubPtr + 4, Reg12) && + "Linker generated stubs may only corrupt register r12 (IP)"); + return B; +} + +static Block &createStubArmv7(LinkGraph &G, Section &S, Symbol &Target) { + Block &B = allocStub(G, S, Armv7ABS); + B.addEdge(Arm_MovwAbsNC, 0, Target, 0); + B.addEdge(Arm_MovtAbs, 4, Target, 0); + + [[maybe_unused]] const char *StubPtr = B.getContent().data(); + [[maybe_unused]] uint32_t Reg12 = encodeRegMovtA1MovwA2(12); + assert(checkRegister(StubPtr, Reg12) && + checkRegister(StubPtr + 4, Reg12) && + "Linker generated stubs may only corrupt register r12 (IP)"); + return B; +} + +static bool needsStub(const Edge &E) { + Symbol &Target = E.getTarget(); + + // Create stubs for external branch targets. + if (!Target.isDefined()) { + switch (E.getKind()) { + case Arm_Call: + case Arm_Jump24: + case Thumb_Call: + case Thumb_Jump24: + return true; + default: + return false; + } + } + + // For local targets, create interworking stubs if we switch Arm/Thumb with an + // instruction that cannot switch the instruction set state natively. + bool TargetIsThumb = Target.getTargetFlags() & ThumbSymbol; + switch (E.getKind()) { + case Arm_Jump24: + return TargetIsThumb; // Branch to Thumb needs interworking stub + case Thumb_Jump24: + return !TargetIsThumb; // Branch to Arm needs interworking stub + default: + break; + } + + return false; +} + +bool StubsManager_v7::visitEdge(LinkGraph &G, Block *B, Edge &E) { + if (!needsStub(E)) + return false; + + // Stub Arm/Thumb follows instruction set state at relocation site. + // TODO: We may reduce them at relaxation time and reuse freed slots. + bool MakeThumb = (E.getKind() > LastArmRelocation); + LLVM_DEBUG(dbgs() << " Preparing " << (MakeThumb ? "Thumb" : "Arm") + << " stub for " << G.getEdgeKindName(E.getKind()) + << " edge at " << B->getFixupAddress(E) << " (" + << B->getAddress() << " + " + << formatv("{0:x}", E.getOffset()) << ")\n"); + + Symbol &Target = E.getTarget(); + assert(Target.hasName() && "Edge cannot point to anonymous target"); + Symbol *&StubSymbol = getStubSymbolSlot(Target.getName(), MakeThumb); + + if (!StubSymbol) { + if (!StubsSection) + StubsSection = &G.createSection(getSectionName(), + orc::MemProt::Read | orc::MemProt::Exec); + Block &B = MakeThumb ? createStubThumbv7(G, *StubsSection, Target) + : createStubArmv7(G, *StubsSection, Target); + StubSymbol = &G.addAnonymousSymbol(B, 0, B.getSize(), true, false); + if (MakeThumb) + StubSymbol->setTargetFlags(ThumbSymbol); + + LLVM_DEBUG({ + dbgs() << " Created " << (MakeThumb ? "Thumb" : "Arm") << " entry for " + << Target.getName() << " in " << StubsSection->getName() << ": " + << *StubSymbol << "\n"; + }); + } + + assert(MakeThumb == (StubSymbol->getTargetFlags() & ThumbSymbol) && + "Instruction set states of stub and relocation site should be equal"); + LLVM_DEBUG({ + dbgs() << " Using " << (MakeThumb ? "Thumb" : "Arm") << " entry " + << *StubSymbol << " in " + << StubSymbol->getBlock().getSection().getName() << "\n"; + }); + + E.setTarget(*StubSymbol); + return true; } const char *getEdgeKindName(Edge::Kind K) { diff --git a/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_arm.s b/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_arm.s new file mode 100644 index 0000000000000..fb2e0eb2c0bf2 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_arm.s @@ -0,0 +1,53 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=armv7-linux-gnueabi -arm-add-build-attributes \ +# RUN: -filetype=obj -o %t/out.o %s +# RUN: llvm-jitlink -noexec -slab-address 0x76ff0000 \ +# RUN: -slab-allocate 10Kb -slab-page-size 4096 \ +# RUN: -abs ext=0x76bbe880 \ +# RUN: -check %s %t/out.o + + .text + .syntax unified + +# Check that calls/jumps to external functions trigger the generation of +# branch-range extension stubs. These stubs don't follow the default PLT model +# where the branch-target address is loaded from a GOT entry. Instead, they +# hard-code it in the immediate field. + +# The external function ext will return to the caller directly. +# jitlink-check: decode_operand(test_arm_jump, 0) = stub_addr(out.o, ext) - (test_arm_jump + 8) + .globl test_arm_jump + .type test_arm_jump,%function + .p2align 2 +test_arm_jump: + b ext + .size test_arm_jump, .-test_arm_jump + +# The branch-with-link sets the LR register so that the external function ext +# returns to us. We have to save the register (push) and return to main manually +# (pop). This adds the +4 offset for the bl instruction we decode: +# jitlink-check: decode_operand(test_arm_call + 4, 0) = stub_addr(out.o, ext) - (test_arm_call + 8) - 4 + .globl test_arm_call + .type test_arm_call,%function + .p2align 2 +test_arm_call: + push {lr} + bl ext + pop {pc} + .size test_arm_call, .-test_arm_call + +# This test is executable with both, Arm and Thumb `ext` functions. It only has +# to return with `bx lr`. For example: +# > echo "void ext() {}" | clang -target armv7-linux-gnueabihf -o ext-arm.o -c -xc - +# > llvm-jitlink ext-arm.o out.o +# + .globl main + .type main,%function + .p2align 2 +main: + push {lr} + bl test_arm_call + bl test_arm_jump + movw r0, #0 + pop {pc} + .size main, .-main diff --git a/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_multi.s b/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_multi.s new file mode 100644 index 0000000000000..d575f114dcba1 --- /dev/null +++ b/llvm/test/ExecutionEngine/JITLink/AArch32/ELF_stubs_multi.s @@ -0,0 +1,50 @@ +# RUN: rm -rf %t && mkdir -p %t +# RUN: llvm-mc -triple=armv7-linux-gnueabi -arm-add-build-attributes \ +# RUN: -filetype=obj -o %t/out.o %s +# RUN: llvm-jitlink -noexec -slab-address 0x76ff0000 \ +# RUN: -slab-allocate=10Kb -slab-page-size=4096 \ +# RUN: -abs ext=0x76bbe880 -check %s %t/out.o + + .text + .syntax unified + +# Check that a single external symbol can have multiple stubs. We access them +# with the extra stub-index argument to stub_addr(). Stubs are sorted by +# ascending size (because the default memory manager lays out blocks by size). + +# Thumb relocation site emits thumb stub +# jitlink-check: decode_operand(test_stub_thumb, 0) = stub_addr(out.o, ext, thumb) - (test_stub_thumb + 4) + .globl test_stub_thumb + .type test_stub_thumb,%function + .p2align 1 + .code 16 + .thumb_func +test_stub_thumb: + b ext + .size test_stub_thumb, .-test_stub_thumb + +# Arm relocation site emits arm stub +# jitlink-check: decode_operand(test_stub_arm, 0) = stub_addr(out.o, ext, arm) - (test_stub_arm + 8) + .globl test_stub_arm + .type test_stub_arm,%function + .p2align 2 + .code 32 +test_stub_arm: + b ext + .size test_stub_arm, .-test_stub_arm + +# This test is executable with both, Arm and Thumb `ext` functions. It only has +# to return (directly to main) with `bx lr`. For example: +# > echo "void ext() {}" | clang -target armv7-linux-gnueabihf -o ext-arm.o -c -xc - +# > llvm-jitlink ext-arm.o out.o +# + .globl main + .type main,%function + .p2align 2 +main: + push {lr} + bl test_stub_arm + bl test_stub_thumb + movw r0, #0 + pop {pc} + .size main, .-main diff --git a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp index 7e213777a6727..b2a133860197d 100644 --- a/llvm/tools/llvm-jitlink/llvm-jitlink.cpp +++ b/llvm/tools/llvm-jitlink/llvm-jitlink.cpp @@ -1265,8 +1265,52 @@ Session::findSectionInfo(StringRef FileName, StringRef SectionName) { return SecInfoItr->second; } +class MemoryMatcher { +public: + MemoryMatcher(ArrayRef Content) + : Pos(Content.data()), End(Pos + Content.size()) {} + + template bool matchMask(MaskType Mask) { + if (Mask == (Mask & *reinterpret_cast(Pos))) { + Pos += sizeof(MaskType); + return true; + } + return false; + } + + template bool matchEqual(ValueType Value) { + if (Value == *reinterpret_cast(Pos)) { + Pos += sizeof(ValueType); + return true; + } + return false; + } + + bool done() const { return Pos == End; } + +private: + const char *Pos; + const char *End; +}; + static StringRef detectStubKind(const Session::MemoryRegionInfo &Stub) { - // Implement acutal stub kind detection + constexpr uint32_t Armv7MovWTle = 0xe300c000; + constexpr uint32_t Armv7BxR12le = 0xe12fff1c; + constexpr uint32_t Thumbv7MovWTle = 0x0c00f240; + constexpr uint16_t Thumbv7BxR12le = 0x4760; + + MemoryMatcher M(Stub.getContent()); + if (M.matchMask(Thumbv7MovWTle)) { + if (M.matchMask(Thumbv7MovWTle)) + if (M.matchEqual(Thumbv7BxR12le)) + if (M.done()) + return "thumbv7_abs_le"; + } else if (M.matchMask(Armv7MovWTle)) { + if (M.matchMask(Armv7MovWTle)) + if (M.matchEqual(Armv7BxR12le)) + if (M.done()) + return "armv7_abs_le"; + } return ""; }