diff --git a/llvm/lib/Target/X86/CMakeLists.txt b/llvm/lib/Target/X86/CMakeLists.txt index 0b7a98ad6341d..ef54ca3e435c5 100644 --- a/llvm/lib/Target/X86/CMakeLists.txt +++ b/llvm/lib/Target/X86/CMakeLists.txt @@ -99,8 +99,10 @@ add_llvm_target(X86CodeGen ${sources} Core GlobalISel Instrumentation + IRPrinter MC ProfileData + ScalarOpts SelectionDAG Support Target diff --git a/llvm/lib/Target/X86/X86PassRegistry.def b/llvm/lib/Target/X86/X86PassRegistry.def new file mode 100644 index 0000000000000..b591fc8ccd0d4 --- /dev/null +++ b/llvm/lib/Target/X86/X86PassRegistry.def @@ -0,0 +1,76 @@ +//===- X86PassRegistry.def - Registry of passes for X86 ---------*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// X86 pass registry +// +//===----------------------------------------------------------------------===// + +// NOTE: NO INCLUDE GUARD DESIRED! + +#ifndef FUNCTION_PASS +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef FUNCTION_PASS + +#ifndef DUMMY_FUNCTION_PASS +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_FUNCTION_PASS("x86-win-eh-state", X86WinEHStatePass, ()) +#undef DUMMY_FUNCTION_PASS + +#ifndef MACHINE_FUNCTION_PASS +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif + +#undef MACHINE_FUNCTION_PASS + +// PASS_NAME is for mocking machine passes, remove it after all machine passes +// are added new pass manager interface. +#ifndef DUMMY_MACHINE_FUNCTION_PASS +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#endif +DUMMY_MACHINE_FUNCTION_PASS("x86-isel-dag", X86ISelDagPass, (getTM(), getOptLevel())) +DUMMY_MACHINE_FUNCTION_PASS("x86-global-basereg", X86GlobalBaseRegPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-cmov-converter", X86CmovConverterDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-setcc", X86FixupSetCCPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-opt-leas", X86OptimizeLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-callframe-opt", X86CallFrameOptimizationPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-store-foword-block", X86AvoidStoreForwardingBlocksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-speculative-load-hardening", X86SpeculativeLoadHardeningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-flags-copy-lowering", X86FlagsCopyLoweringDummyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-domain-reassign", X86DomainReassignmentPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fp-stackifier", X86FloatingPointStackifierPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-expand-pseudo", X86ExpandPseudoPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-exec-domain-fix", X86ExecutionDomainFixPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-indirectbr-tracking", X86IndirectBranchTrackingPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-issue-vzero-upper", X86IssueVZeroUpperPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-bwinsts", X86FixupBWInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-pad-short-funcs", X86PadShortFunctionsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-leas", X86FixupLEAsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-evex-to-vex-insts", X86EvexToVexInstsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-discriminate-memops", X86DiscriminateMemOpsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-prefetch", X86InsertPrefetchPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-insert-x87-wait", X86InsertX87waitPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-avoid-trailing-call", X86AvoidTrailingCallPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-asm-printer", X86AsmPrinterPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("cleanup-local-dyn-tls", CleanupLocalDynamicTLSPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-inst-tuning", X86FixupInstTuningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-fixup-vector-constants", X86FixupVectorConstantsPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("ehcontguard-catchret", EHContGuardCatchretPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-lvi-ret", X86LoadValueInjectionRetHardeningPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-lower-tile-copy", X86LowerTileCopyPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-seses", X86SpeculativeExecutionSideEffectSuppressionPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-retpoline-thunks", X86IndirectThunksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-return-thunks", X86ReturnThunksPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("tileconfig", X86TileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("tile-pre-config", X86PreTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("fastpretileconfig", X86FastPreTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("fasttileconfig", X86FastTileConfigPass, ()) +DUMMY_MACHINE_FUNCTION_PASS("x86-dyn-alloca-expander", X86DynAllocaExpanderPass, ()) +#undef DUMMY_MACHINE_FUNCTION_PASS diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp index 5668b514d6dec..60d36a43a6f23 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.cpp +++ b/llvm/lib/Target/X86/X86TargetMachine.cpp @@ -23,6 +23,7 @@ #include "llvm/ADT/SmallString.h" #include "llvm/ADT/StringRef.h" #include "llvm/Analysis/TargetTransformInfo.h" +#include "llvm/CodeGen/CodeGenPassBuilder.h" #include "llvm/CodeGen/ExecutionDomainFix.h" #include "llvm/CodeGen/GlobalISel/CSEInfo.h" #include "llvm/CodeGen/GlobalISel/CallLowering.h" @@ -39,6 +40,7 @@ #include "llvm/IR/DataLayout.h" #include "llvm/IR/Function.h" #include "llvm/MC/MCAsmInfo.h" +#include "llvm/MC/MCStreamer.h" #include "llvm/MC/TargetRegistry.h" #include "llvm/Pass.h" #include "llvm/Support/CodeGen.h" @@ -659,3 +661,346 @@ bool X86PassConfig::addRegAssignAndRewriteOptimized() { } return TargetPassConfig::addRegAssignAndRewriteOptimized(); } + +namespace { + +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public PassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(Function &, FunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + }; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + struct PASS_NAME : public PassInfoMixin { \ + template PASS_NAME(Ts &&...) {} \ + PreservedAnalyses run(MachineFunction &, \ + MachineFunctionAnalysisManager &) { \ + return PreservedAnalyses::all(); \ + } \ + static AnalysisKey Key; \ + }; \ + AnalysisKey PASS_NAME::Key; +#include "X86PassRegistry.def" + +/// X86 Code Generator Pass Configuration Options. +struct X86CodeGenPassBuilder + : public CodeGenPassBuilder { + X86CodeGenPassBuilder(X86TargetMachine &TM, + CGPassBuilderOption Opt = CGPassBuilderOption(), + PassInstrumentationCallbacks *PIC = nullptr) + : CodeGenPassBuilder(TM, Opt, PIC) { + // Target-specific `CGPassBuilderOption` could be overridden here. + } + + std::pair getTargetPassNameFromLegacyName(StringRef) const; + + bool parseTargetMIRPass(MachineFunctionPassManager &MFPM, + StringRef Text) const; + void addIRPasses(AddIRPass &) const; + void addPreISel(AddIRPass &) const; + Error addInstSelector(AddMachinePass &) const; + Error addIRTranslator(AddMachinePass &) const; + Error addLegalizeMachineIR(AddMachinePass &) const; + Error addRegBankSelect(AddMachinePass &) const; + Error addGlobalInstructionSelect(AddMachinePass &) const; + void addILPOpts(AddMachinePass &) const; + void addMachineSSAOptimization(AddMachinePass &) const; + void addPreRegAlloc(AddMachinePass &) const; + Error addPostFastRegAllocRewrite(AddMachinePass &) const; + void addPostRegAlloc(AddMachinePass &) const; + void addPreEmitPass(AddMachinePass &) const; + void addPreEmitPass2(AddMachinePass &) const; + void addPreSched2(AddMachinePass &) const; + Error addRegAssignAndRewriteOptimized(AddMachinePass &) const; + void addAsmPrinter(AddMachinePass &, CreateMCStreamer) const; +}; + +} // namespace + +void X86CodeGenPassBuilder::addIRPasses(AddIRPass &addPass) const { + addPass(AtomicExpandPass()); + + CodeGenPassBuilder::addIRPasses(addPass); + + if (TM.getOptLevel() != CodeGenOptLevel::None) + addPass(InterleavedAccessPass()); + + // Add passes that handle indirect branch removal and insertion of a retpoline + // thunk. These will be a no-op unless a function subtarget has the retpoline + // feature enabled. + addPass(IndirectBrExpandPass()); + + // Add Control Flow Guard checks. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows()) { + if (TT.getArch() == Triple::x86_64) + addPass(CFGuardDispatchPass()); + else + addPass(CFGuardCheckPass()); + } +} + +Error X86CodeGenPassBuilder::addInstSelector(AddMachinePass &addPass) const { + // Install an instruction selector. + addPass(X86ISelDagPass(getTM(), getOptLevel())); + + // For ELF, cleanup any local-dynamic TLS accesses. + if (TM.getTargetTriple().isOSBinFormatELF() && + TM.getOptLevel() != CodeGenOptLevel::None) + addPass(CleanupLocalDynamicTLSPass()); + + addPass(X86GlobalBaseRegPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addIRTranslator(AddMachinePass &addPass) const { + addPass(IRTranslatorPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addLegalizeMachineIR( + AddMachinePass &addPass) const { + addPass(LegalizerPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addRegBankSelect(AddMachinePass &addPass) const { + addPass(RegBankSelectPass()); + return Error::success(); +} + +Error X86CodeGenPassBuilder::addGlobalInstructionSelect( + AddMachinePass &addPass) const { + addPass(InstructionSelectPass()); + return Error::success(); +} + +void X86CodeGenPassBuilder::addILPOpts(AddMachinePass &addPass) const { + addPass(EarlyIfConverterPass()); + if (EnableMachineCombinerPass) + addPass(MachineCombinerPass()); + addPass(X86CmovConverterDummyPass()); +} + +void X86CodeGenPassBuilder::addPreISel(AddIRPass &addPass) const { + // Only add this pass for 32-bit x86 Windows. + const Triple &TT = TM.getTargetTriple(); + if (TT.isOSWindows() && TT.getArch() == Triple::x86) + addPass(X86WinEHStatePass()); +} + +void X86CodeGenPassBuilder::addPreRegAlloc(AddMachinePass &addPass) const { + if (TM.getOptLevel() != CodeGenOptLevel::None) { + addPass(LiveRangeShrinkPass()); + addPass(X86FixupSetCCPass()); + addPass(X86OptimizeLEAsPass()); + addPass(X86CallFrameOptimizationPass()); + addPass(X86AvoidStoreForwardingBlocksPass()); + } + + addPass(X86SpeculativeLoadHardeningPass()); + addPass(X86FlagsCopyLoweringDummyPass()); // TODO: port to NPM and rename + addPass(X86DynAllocaExpanderPass()); + + if (getOptLevel() != CodeGenOptLevel::None) + addPass(X86PreTileConfigPass()); + else + addPass(X86FastPreTileConfigPass()); +} + +Error X86CodeGenPassBuilder::addPostFastRegAllocRewrite( + AddMachinePass &addPass) const { + addPass(X86FastTileConfigPass()); + return Error::success(); +} + +void X86CodeGenPassBuilder::addMachineSSAOptimization( + AddMachinePass &addPass) const { + addPass(X86DomainReassignmentPass()); + CodeGenPassBuilder::addMachineSSAOptimization(addPass); +} + +void X86CodeGenPassBuilder::addPostRegAlloc(AddMachinePass &addPass) const { + addPass(X86LowerTileCopyPass()); + addPass(X86FloatingPointStackifierPass()); + // When -O0 is enabled, the Load Value Injection Hardening pass will fall back + // to using the Speculative Execution Side Effect Suppression pass for + // mitigation. This is to prevent slow downs due to + // analyses needed by the LVIHardening pass when compiling at -O0. + if (getOptLevel() != CodeGenOptLevel::None) + addPass(X86LoadValueInjectionRetHardeningPass()); +} + +void X86CodeGenPassBuilder::addPreSched2(AddMachinePass &addPass) const { + addPass(X86ExpandPseudoPass()); + addPass(KCFIPass()); +} + +void X86CodeGenPassBuilder::addPreEmitPass(AddMachinePass &addPass) const { + if (getOptLevel() != CodeGenOptLevel::None) { + addPass(X86ExecutionDomainFixPass()); + addPass(BreakFalseDepsPass()); + } + + addPass(X86IndirectBranchTrackingPass()); + + addPass(X86IssueVZeroUpperPass()); + + if (getOptLevel() != CodeGenOptLevel::None) { + addPass(X86FixupBWInstsPass()); + addPass(X86PadShortFunctionsPass()); + addPass(X86FixupLEAsPass()); + addPass(X86FixupInstTuningPass()); + addPass(X86FixupVectorConstantsPass()); + } + addPass(X86EvexToVexInstsPass()); + addPass(X86DiscriminateMemOpsPass()); + addPass(X86InsertPrefetchPass()); + addPass(X86InsertX87waitPass()); +} + +void X86CodeGenPassBuilder::addPreEmitPass2(AddMachinePass &addPass) const { + const Triple &TT = TM.getTargetTriple(); + const MCAsmInfo *MAI = TM.getMCAsmInfo(); + + // The X86 Speculative Execution Pass must run after all control + // flow graph modifying passes. As a result it was listed to run right before + // the X86 Retpoline Thunks pass. The reason it must run after control flow + // graph modifications is that the model of LFENCE in LLVM has to be updated + // (FIXME: https://bugs.llvm.org/show_bug.cgi?id=45167). Currently the + // placement of this pass was hand checked to ensure that the subsequent + // passes don't move the code around the LFENCEs in a way that will hurt the + // correctness of this pass. This placement has been shown to work based on + // hand inspection of the codegen output. + addPass(X86SpeculativeExecutionSideEffectSuppressionPass()); + addPass(X86IndirectThunksPass()); + addPass(X86ReturnThunksPass()); + + // Insert extra int3 instructions after trailing call instructions to avoid + // issues in the unwinder. + if (TT.isOSWindows() && TT.getArch() == Triple::x86_64) + addPass(X86AvoidTrailingCallPass()); + + // Verify basic block incoming and outgoing cfa offset and register values and + // correct CFA calculation rule where needed by inserting appropriate CFI + // instructions. + if (!TT.isOSDarwin() && + (!TT.isOSWindows() || + MAI->getExceptionHandlingType() == ExceptionHandling::DwarfCFI)) + addPass(CFIInstrInserterPass()); + // Identify valid longjmp targets for Windows Control Flow Guard. + if (TT.isOSWindows()) { + // Identify valid longjmp targets for Windows Control Flow Guard. + addPass(CFGuardLongjmpPass()); + // Identify valid eh continuation targets for Windows EHCont Guard. + addPass(EHContGuardCatchretPass()); + } + addPass(X86LoadValueInjectionRetHardeningPass()); + + // Insert pseudo probe annotation for callsite profiling + addPass(PseudoProbeInserterPass()); + + // KCFI indirect call checks are lowered to a bundle, and on Darwin platforms, + // also CALL_RVMARKER. + addPass(UnpackMachineBundlesPass([&TT](const MachineFunction &MF) { + // Only run bundle expansion if the module uses kcfi, or there are relevant + // ObjC runtime functions present in the module. + const Function &F = MF.getFunction(); + const Module *M = F.getParent(); + return M->getModuleFlag("kcfi") || + (TT.isOSDarwin() && + (M->getFunction("objc_retainAutoreleasedReturnValue") || + M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))); + })); +} + +Error X86CodeGenPassBuilder::addRegAssignAndRewriteOptimized( + AddMachinePass &addPass) const { + if (Opt.RegAlloc != RegAllocType::Default && EnableTileRAPass) { + addPass(RAGreedyPass(onlyAllocateTileRegisters)); + addPass(X86TileConfigPass()); + } + return CodeGenPassBuilder::addRegAssignAndRewriteFast(addPass); +} + +void X86CodeGenPassBuilder::addAsmPrinter(AddMachinePass &addPass, + CreateMCStreamer callback) const { + addPass(X86AsmPrinterPass(callback)); +} + +std::pair +X86CodeGenPassBuilder::getTargetPassNameFromLegacyName(StringRef Name) const { + std::pair Ret; + +#define FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define DUMMY_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, false}; +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) \ + Ret = {#PASS_NAME, true}; +#include "X86PassRegistry.def" + + return Ret; +} + +Error X86TargetMachine::buildCodeGenPipeline( + ModulePassManager &MPM, MachineFunctionPassManager &MFPM, + raw_pwrite_stream &Out, raw_pwrite_stream *DwoOut, CodeGenFileType FileType, + CGPassBuilderOption Opts, MachineFunctionAnalysisManager &MFAM, + PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.buildPipeline(MPM, MFPM, Out, DwoOut, FileType); +} + +std::pair +X86TargetMachine::getPassNameFromLegacyName(StringRef Name) { + X86CodeGenPassBuilder X86CGPB{*this}; + return X86CGPB.getPassNameFromLegacyName(Name); +} + +bool X86CodeGenPassBuilder::parseTargetMIRPass(MachineFunctionPassManager &MFPM, + StringRef Name) const { +#define ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + if (Name == NAME) { \ + { \ + addMachinePass(MFPM, PASS_NAME CONSTRUCTOR); \ + } \ + return false; \ + } + +#define MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define DUMMY_MACHINE_MODULE_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#define DUMMY_MACHINE_FUNCTION_PASS(NAME, PASS_NAME, CONSTRUCTOR) \ + ADD_PASS(NAME, PASS_NAME, CONSTRUCTOR) +#include "X86PassRegistry.def" +#undef ADD_PASS + return true; +} + +Error X86TargetMachine::parseMIRPipeline(MachineFunctionPassManager &MFPM, + StringRef PipelineText, + CGPassBuilderOption Opts, + MachineFunctionAnalysisManager &MFAM, + PassInstrumentationCallbacks *PIC) { + X86CodeGenPassBuilder X86CGPB{*this, Opts, PIC}; + X86CGPB.registerAnalyses(MFAM); + return X86CGPB.parseMIRPipeline(MFPM, PipelineText); +} diff --git a/llvm/lib/Target/X86/X86TargetMachine.h b/llvm/lib/Target/X86/X86TargetMachine.h index 4836be4db0e8e..bb06cedd10b47 100644 --- a/llvm/lib/Target/X86/X86TargetMachine.h +++ b/llvm/lib/Target/X86/X86TargetMachine.h @@ -50,6 +50,19 @@ class X86TargetMachine final : public LLVMTargetMachine { // Set up the pass pipeline. TargetPassConfig *createPassConfig(PassManagerBase &PM) override; + Error buildCodeGenPipeline(ModulePassManager &MPM, + MachineFunctionPassManager &MFPM, + raw_pwrite_stream &, raw_pwrite_stream *, + CodeGenFileType, CGPassBuilderOption, + MachineFunctionAnalysisManager &, + PassInstrumentationCallbacks *) override; + + std::pair getPassNameFromLegacyName(StringRef) override; + + Error parseMIRPipeline(MachineFunctionPassManager &MFPM, StringRef, + CGPassBuilderOption, MachineFunctionAnalysisManager &, + PassInstrumentationCallbacks *) override; + TargetLoweringObjectFile *getObjFileLowering() const override { return TLOF.get(); }