[AMDGPU] Implement wwm-register allocation

llvm/include/llvm/CodeGen/LiveStacks.h

@@ -34,6 +34,7 @@ class TargetRegisterInfo;
 
 class LiveStacks : public MachineFunctionPass {
   const TargetRegisterInfo *TRI = nullptr;
+  const MachineRegisterInfo *MRI = nullptr;
 
   /// Special pool allocator for VNInfo's (LiveInterval val#).
   ///

llvm/include/llvm/CodeGen/MachineInstr.h

@@ -1586,7 +1586,7 @@ class MachineInstr
   const TargetRegisterClass *getRegClassConstraintEffectForVReg(
       Register Reg, const TargetRegisterClass *CurRC,
       const TargetInstrInfo *TII, const TargetRegisterInfo *TRI,
-      bool ExploreBundle = false) const;
+      const MachineRegisterInfo &MRI, bool ExploreBundle = false) const;
 
   /// Applies the constraints (def/use) implied by the \p OpIdx operand
   /// to the given \p CurRC.
@@ -1600,7 +1600,8 @@ class MachineInstr
   const TargetRegisterClass *
   getRegClassConstraintEffect(unsigned OpIdx, const TargetRegisterClass *CurRC,
                               const TargetInstrInfo *TII,
-                              const TargetRegisterInfo *TRI) const;
+                              const TargetRegisterInfo *TRI,
+                              const MachineRegisterInfo &MRI) const;
 
   /// Add a tie between the register operands at DefIdx and UseIdx.
   /// The tie will cause the register allocator to ensure that the two
@@ -2005,7 +2006,8 @@ class MachineInstr
   /// If the related operand does not constrained Reg, this returns CurRC.
   const TargetRegisterClass *getRegClassConstraintEffectForVRegImpl(
       unsigned OpIdx, Register Reg, const TargetRegisterClass *CurRC,
-      const TargetInstrInfo *TII, const TargetRegisterInfo *TRI) const;
+      const TargetInstrInfo *TII, const TargetRegisterInfo *TRI,
+      const MachineRegisterInfo &MRI) const;
 
   /// Stores extra instruction information inline or allocates as ExtraInfo
   /// based on the number of pointers.

llvm/include/llvm/CodeGen/MachineRegisterInfo.h

@@ -94,6 +94,16 @@ class MachineRegisterInfo {
   /// all registers that were disabled are removed from the list.
   SmallVector<MCPhysReg, 16> UpdatedCSRs;
 
+  /// The Synthetic field of regclasses. Targets can alter this vector
+  /// to enable classes dynamically during codegen.
+  BitVector RegClassSyntheticInfo;
+
+  /// To have more restrictions on the allocation order for each register class.
+  /// For some targets, two distinct register classes can have the same set of
+  /// registers, but their allocatable set can vary. This vector helps targets
+  /// to dynamically determine the actual allocatable set for RCs.
+  std::vector<BitVector> RegClassAllocationMasks;
+
   /// RegAllocHints - This vector records register allocation hints for
   /// virtual registers. For each virtual register, it keeps a pair of hint
   /// type and hints vector making up the allocation hints. Only the first
@@ -257,6 +267,30 @@ class MachineRegisterInfo {
   /// Notice that it will override ant previously disabled/saved CSRs.
   void setCalleeSavedRegs(ArrayRef<MCPhysReg> CSRs);
 
+  /// Initialize the RegClassSyntheticInfo. It sets the bit position as
+  /// exactly as the tablegened Synthetic field. Targets can later flip this
+  /// field to enable/disable the regclass whenever required.
+  void initializeRegClassSyntheticInfo();
+
+  /// Change the synthetic info for the regclass \p RC from \p Value.
+  void changeSyntheticInfoForRC(const TargetRegisterClass *RC, bool Value);
+
+  /// This function checks if \p RC is enabled or not so that it can be included
+  /// in various regclass related queries.
+  bool isEnabled(const TargetRegisterClass *RC) const;
+
+  /// Update dynamically determined allocation mask for register classes.
+  void updateAllocationMaskForRCs(std::vector<BitVector> &&Mask);
+
+  // Return the allocation mask for regclass \p RC.
+  const BitVector &getAllocationMaskForRC(const TargetRegisterClass &RC) const;
+
+  /// True when the target has dynamically determined allocation mask for its
+  /// register classes.
+  bool hasAllocationMaskForRCs() const {
+    return RegClassAllocationMasks.size();
+  }
+
   // Strictly for use by MachineInstr.cpp.
   void addRegOperandToUseList(MachineOperand *MO);
 

llvm/include/llvm/CodeGen/RegisterBankInfo.h

@@ -445,7 +445,8 @@ class RegisterBankInfo {
   /// Get the MinimalPhysRegClass for Reg.
   /// \pre Reg is a physical register.
   const TargetRegisterClass *
-  getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI) const;
+  getMinimalPhysRegClass(Register Reg, const TargetRegisterInfo &TRI,
+                         const MachineRegisterInfo &MRI) const;
 
   /// Try to get the mapping of \p MI.
   /// See getInstrMapping for more details on what a mapping represents.

llvm/include/llvm/CodeGen/TargetLowering.h

@@ -4384,6 +4384,7 @@ class TargetLowering : public TargetLoweringBase {
   /// Returns 'true' is the edge is necessary, 'false' otherwise
   virtual bool checkForPhysRegDependency(SDNode *Def, SDNode *User, unsigned Op,
                                          const TargetRegisterInfo *TRI,
+                                         const MachineRegisterInfo &MRI,
                                          const TargetInstrInfo *TII,
                                          unsigned &PhysReg, int &Cost) const {
     return false;

llvm/include/llvm/CodeGen/TargetRegisterInfo.h

@@ -121,6 +121,9 @@ class TargetRegisterClass {
   /// Return true if this register class has a defined BaseClassOrder.
   bool isBaseClass() const { return MC->isBaseClass(); }
 
+  /// Return true if this register class is marked synthetic.
+  bool isSynthetic() const { return MC->isSynthetic(); }
+
   /// Return true if the specified TargetRegisterClass
   /// is a proper sub-class of this TargetRegisterClass.
   bool hasSubClass(const TargetRegisterClass *RC) const {
@@ -338,20 +341,23 @@ class TargetRegisterInfo : public MCRegisterInfo {
   /// Returns the Register Class of a physical register of the given type,
   /// picking the most sub register class of the right type that contains this
   /// physreg.
-  const TargetRegisterClass *getMinimalPhysRegClass(MCRegister Reg,
-                                                    MVT VT = MVT::Other) const;
+  const TargetRegisterClass *
+  getMinimalPhysRegClass(MCRegister Reg, const MachineRegisterInfo &MRI,
+                         MVT VT = MVT::Other) const;
 
   /// Returns the Register Class of a physical register of the given type,
   /// picking the most sub register class of the right type that contains this
   /// physreg. If there is no register class compatible with the given type,
   /// returns nullptr.
-  const TargetRegisterClass *getMinimalPhysRegClassLLT(MCRegister Reg,
-                                                       LLT Ty = LLT()) const;
+  const TargetRegisterClass *
+  getMinimalPhysRegClassLLT(MCRegister Reg, const MachineRegisterInfo &MRI,
+                            LLT Ty = LLT()) const;
 
   /// Return the maximal subclass of the given register class that is
   /// allocatable or NULL.
   const TargetRegisterClass *
-    getAllocatableClass(const TargetRegisterClass *RC) const;
+  getAllocatableClass(const TargetRegisterClass *RC,
+                      const MachineRegisterInfo &MRI) const;
 
   /// Returns a bitset indexed by register number indicating if a register is
   /// allocatable or not. If a register class is specified, returns the subset
@@ -630,7 +636,8 @@ class TargetRegisterInfo : public MCRegisterInfo {
   /// TableGen will synthesize missing A sub-classes.
   virtual const TargetRegisterClass *
   getMatchingSuperRegClass(const TargetRegisterClass *A,
-                           const TargetRegisterClass *B, unsigned Idx) const;
+                           const TargetRegisterClass *B, unsigned Idx,
+                           const MachineRegisterInfo &MRI) const;
 
   // For a copy-like instruction that defines a register of class DefRC with
   // subreg index DefSubReg, reading from another source with class SrcRC and
@@ -639,7 +646,8 @@ class TargetRegisterInfo : public MCRegisterInfo {
   virtual bool shouldRewriteCopySrc(const TargetRegisterClass *DefRC,
                                     unsigned DefSubReg,
                                     const TargetRegisterClass *SrcRC,
-                                    unsigned SrcSubReg) const;
+                                    unsigned SrcSubReg,
+                                    const MachineRegisterInfo &MRI) const;
 
   /// Returns the largest legal sub-class of RC that
   /// supports the sub-register index Idx.
@@ -769,10 +777,11 @@ class TargetRegisterInfo : public MCRegisterInfo {
   /// corresponding argument register class.
   ///
   /// The function returns NULL if no register class can be found.
-  const TargetRegisterClass*
+  const TargetRegisterClass *
   getCommonSuperRegClass(const TargetRegisterClass *RCA, unsigned SubA,
                          const TargetRegisterClass *RCB, unsigned SubB,
-                         unsigned &PreA, unsigned &PreB) const;
+                         unsigned &PreA, unsigned &PreB,
+                         const MachineRegisterInfo &MRI) const;
 
   //===--------------------------------------------------------------------===//
   // Register Class Information
@@ -809,8 +818,8 @@ class TargetRegisterInfo : public MCRegisterInfo {
   /// Find the largest common subclass of A and B.
   /// Return NULL if there is no common subclass.
   const TargetRegisterClass *
-  getCommonSubClass(const TargetRegisterClass *A,
-                    const TargetRegisterClass *B) const;
+  getCommonSubClass(const TargetRegisterClass *A, const TargetRegisterClass *B,
+                    const MachineRegisterInfo &MRI) const;
 
   /// Returns a TargetRegisterClass used for pointer values.
   /// If a target supports multiple different pointer register classes,

llvm/include/llvm/MC/MCRegisterInfo.h

@@ -47,6 +47,7 @@ class MCRegisterClass {
   const int8_t CopyCost;
   const bool Allocatable;
   const bool BaseClass;
+  const bool Synthetic;
 
   /// getID() - Return the register class ID number.
   ///
@@ -101,6 +102,9 @@ class MCRegisterClass {
 
   /// Return true if this register class has a defined BaseClassOrder.
   bool isBaseClass() const { return BaseClass; }
+  /// isSynthetic - Return true if this is a synthetic class. This field helps
+  /// targets to dynamically enable the regclass during codegen.
+  bool isSynthetic() const { return Synthetic; }
 };
 
 /// MCRegisterDesc - This record contains information about a particular

llvm/include/llvm/Target/Target.td

@@ -338,6 +338,12 @@ class RegisterClass<string namespace, list<ValueType> regTypes, int alignment,
   // Target-specific flags. This becomes the TSFlags field in TargetRegisterClass.
   bits<8> TSFlags = 0;
 
+  // If set to true, the register class won't take part in various regclass queries
+  // by default. This allows targets to dynamically enable classes for a period
+  // during codegen so that they can be turned allocatable, copyable, spillable,
+  // and/or make them available for various regclass queries.
+  bit Synthetic = false;
+
   // If set then consider this register class to be the base class for registers in
   // its MemberList.  The base class for registers present in multiple base register
   // classes will be resolved in the order defined by this value, with lower values

llvm/lib/CodeGen/AggressiveAntiDepBreaker.cpp

@@ -604,7 +604,7 @@ bool AggressiveAntiDepBreaker::FindSuitableFreeRegisters(
   // check every use of the register and find the largest register class
   // that can be used in all of them.
   const TargetRegisterClass *SuperRC =
-    TRI->getMinimalPhysRegClass(SuperReg, MVT::Other);
+      TRI->getMinimalPhysRegClass(SuperReg, MRI, MVT::Other);
 
   ArrayRef<MCPhysReg> Order = RegClassInfo.getOrder(SuperRC);
   if (Order.empty()) {

llvm/lib/CodeGen/AsmPrinter/DwarfCompileUnit.cpp

@@ -813,7 +813,7 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(
   auto AddEntry = [&](const DbgValueLocEntry &Entry,
                       DIExpressionCursor &Cursor) {
     if (Entry.isLocation()) {
-      if (!DwarfExpr.addMachineRegExpression(TRI, Cursor,
+      if (!DwarfExpr.addMachineRegExpression(TRI, Asm->MF->getRegInfo(), Cursor,
                                              Entry.getLoc().getReg()))
         return false;
     } else if (Entry.isInt()) {
@@ -910,7 +910,8 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(const Loc::MMI &MMI,
       addOpAddress(*Loc, FrameSymbol);
     else
       DwarfExpr.addMachineRegExpression(
-          *Asm->MF->getSubtarget().getRegisterInfo(), Cursor, FrameReg);
+          *Asm->MF->getSubtarget().getRegisterInfo(), Asm->MF->getRegInfo(),
+          Cursor, FrameReg);
     DwarfExpr.addExpression(std::move(Cursor));
   }
   if (Asm->TM.getTargetTriple().isNVPTX() && DD->tuneForGDB()) {
@@ -940,7 +941,8 @@ void DwarfCompileUnit::applyConcreteDbgVariableAttributes(
     DIExpressionCursor Cursor(Expr.getElements());
     DwarfExpr.beginEntryValueExpression(Cursor);
     DwarfExpr.addMachineRegExpression(
-        *Asm->MF->getSubtarget().getRegisterInfo(), Cursor, Register);
+        *Asm->MF->getSubtarget().getRegisterInfo(), Asm->MF->getRegInfo(),
+        Cursor, Register);
     DwarfExpr.addExpression(std::move(Cursor));
   }
   addBlock(VariableDie, dwarf::DW_AT_location, DwarfExpr.finalize());
@@ -1557,7 +1559,8 @@ void DwarfCompileUnit::addAddress(DIE &Die, dwarf::Attribute Attribute,
 
   DIExpressionCursor Cursor({});
   const TargetRegisterInfo &TRI = *Asm->MF->getSubtarget().getRegisterInfo();
-  if (!DwarfExpr.addMachineRegExpression(TRI, Cursor, Location.getReg()))
+  if (!DwarfExpr.addMachineRegExpression(TRI, Asm->MF->getRegInfo(), Cursor,
+                                         Location.getReg()))
     return;
   DwarfExpr.addExpression(std::move(Cursor));
 
@@ -1587,7 +1590,8 @@ void DwarfCompileUnit::addComplexAddress(const DIExpression *DIExpr, DIE &Die,
     DwarfExpr.beginEntryValueExpression(Cursor);
 
   const TargetRegisterInfo &TRI = *Asm->MF->getSubtarget().getRegisterInfo();
-  if (!DwarfExpr.addMachineRegExpression(TRI, Cursor, Location.getReg()))
+  if (!DwarfExpr.addMachineRegExpression(TRI, Asm->MF->getRegInfo(), Cursor,
+                                         Location.getReg()))
     return;
   DwarfExpr.addExpression(std::move(Cursor));
 

llvm/lib/CodeGen/AsmPrinter/DwarfDebug.cpp

@@ -2652,7 +2652,8 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
     DwarfExpr.beginEntryValueExpression(ExprCursor);
 
     const TargetRegisterInfo &TRI = *AP.MF->getSubtarget().getRegisterInfo();
-    if (!DwarfExpr.addMachineRegExpression(TRI, ExprCursor, Location.getReg()))
+    if (!DwarfExpr.addMachineRegExpression(TRI, AP.MF->getRegInfo(), ExprCursor,
+                                           Location.getReg()))
       return;
     return DwarfExpr.addExpression(std::move(ExprCursor));
   }
@@ -2673,7 +2674,8 @@ void DwarfDebug::emitDebugLocValue(const AsmPrinter &AP, const DIBasicType *BT,
         DwarfExpr.setMemoryLocationKind();
 
       const TargetRegisterInfo &TRI = *AP.MF->getSubtarget().getRegisterInfo();
-      if (!DwarfExpr.addMachineRegExpression(TRI, Cursor, Location.getReg()))
+      if (!DwarfExpr.addMachineRegExpression(TRI, AP.MF->getRegInfo(), Cursor,
+                                             Location.getReg()))
         return false;
     } else if (Entry.isTargetIndexLocation()) {
       TargetIndexLocation Loc = Entry.getTargetIndexLocation();

llvm/lib/CodeGen/AsmPrinter/DwarfExpression.cpp

@@ -97,6 +97,7 @@ void DwarfExpression::addAnd(unsigned Mask) {
 }
 
 bool DwarfExpression::addMachineReg(const TargetRegisterInfo &TRI,
+                                    const MachineRegisterInfo &MRI,
                                     llvm::Register MachineReg,
                                     unsigned MaxSize) {
   if (!MachineReg.isPhysical()) {
@@ -134,7 +135,7 @@ bool DwarfExpression::addMachineReg(const TargetRegisterInfo &TRI,
   // For example, Q0 on ARM is a composition of D0+D1.
   unsigned CurPos = 0;
   // The size of the register in bits.
-  const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(MachineReg);
+  const TargetRegisterClass *RC = TRI.getMinimalPhysRegClass(MachineReg, MRI);
   unsigned RegSize = TRI.getRegSizeInBits(*RC);
   // Keep track of the bits in the register we already emitted, so we
   // can avoid emitting redundant aliasing subregs. Because this is
@@ -248,11 +249,13 @@ void DwarfExpression::addConstantFP(const APFloat &APF, const AsmPrinter &AP) {
 }
 
 bool DwarfExpression::addMachineRegExpression(const TargetRegisterInfo &TRI,
+                                              const MachineRegisterInfo &MRI,
                                               DIExpressionCursor &ExprCursor,
                                               llvm::Register MachineReg,
                                               unsigned FragmentOffsetInBits) {
   auto Fragment = ExprCursor.getFragmentInfo();
-  if (!addMachineReg(TRI, MachineReg, Fragment ? Fragment->SizeInBits : ~1U)) {
+  if (!addMachineReg(TRI, MRI, MachineReg,
+                     Fragment ? Fragment->SizeInBits : ~1U)) {
     LocationKind = Unknown;
     return false;
   }

llvm/lib/CodeGen/AsmPrinter/DwarfExpression.h

@@ -245,7 +245,8 @@ class DwarfExpression {
   /// multiple subregisters that alias the register.
   ///
   /// \return false if no DWARF register exists for MachineReg.
-  bool addMachineReg(const TargetRegisterInfo &TRI, llvm::Register MachineReg,
+  bool addMachineReg(const TargetRegisterInfo &TRI,
+                     const MachineRegisterInfo &MRI, llvm::Register MachineReg,
                      unsigned MaxSize = ~1U);
 
   /// Emit a DW_OP_piece or DW_OP_bit_piece operation for a variable fragment.
@@ -325,6 +326,7 @@ class DwarfExpression {
   /// \return                         false if no DWARF register exists
   ///                                 for MachineReg.
   bool addMachineRegExpression(const TargetRegisterInfo &TRI,
+                               const MachineRegisterInfo &MRI,
                                DIExpressionCursor &Expr,
                                llvm::Register MachineReg,
                                unsigned FragmentOffsetInBits = 0);

llvm/lib/CodeGen/DetectDeadLanes.cpp

@@ -97,12 +97,12 @@ static bool isCrossCopy(const MachineRegisterInfo &MRI,
   unsigned PreA, PreB; // Unused.
   if (SrcSubIdx && DstSubIdx)
     return !TRI.getCommonSuperRegClass(SrcRC, SrcSubIdx, DstRC, DstSubIdx, PreA,
-                                       PreB);
+                                       PreB, MRI);
   if (SrcSubIdx)
-    return !TRI.getMatchingSuperRegClass(SrcRC, DstRC, SrcSubIdx);
+    return !TRI.getMatchingSuperRegClass(SrcRC, DstRC, SrcSubIdx, MRI);
   if (DstSubIdx)
-    return !TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSubIdx);
-  return !TRI.getCommonSubClass(SrcRC, DstRC);
+    return !TRI.getMatchingSuperRegClass(DstRC, SrcRC, DstSubIdx, MRI);
+  return !TRI.getCommonSubClass(SrcRC, DstRC, MRI);
 }
 
 void DeadLaneDetector::addUsedLanesOnOperand(const MachineOperand &MO,