[RISCV] Support XSfmm LLVM IR and CodeGen

llvm/include/llvm/IR/IntrinsicsRISCVXsf.td

@@ -180,4 +180,99 @@ let TargetPrefix = "riscv" in {
   // XSfvfnrclipxfqf
   defm int_riscv_sf_vfnrclip_x_f_qf : RISCVSFCustomVFNRCLIP;
   defm int_riscv_sf_vfnrclip_xu_f_qf : RISCVSFCustomVFNRCLIP;
+
+  // XSfmm
+  // Output: (output_len)
+  // Input: (input_len, vsew, twiden)
+  class RISCVSFVSet
+      : DefaultAttrsIntrinsic<[llvm_anyint_ty],
+                              [LLVMMatchType<0>, LLVMMatchType<0>, LLVMMatchType<0>],
+                              [ImmArg<ArgIndex<1>>, ImmArg<ArgIndex<2>>, IntrNoMem]>;
+
+  // Input: (tss, base, tn)
+  // IntrReadMem, IntrHasSideEffects does not work for pattern matching.
+  class RISCVSFTileLoad
+      : DefaultAttrsIntrinsic<[],
+                              [llvm_anyint_ty, llvm_ptr_ty, LLVMMatchType<0>],
+                              [NoCapture<ArgIndex<1>>]>,
+        RISCVVIntrinsic;
+
+  // Input: (tss, base, tn)
+  class RISCVSFTileStore
+      : DefaultAttrsIntrinsic<[],
+                              [llvm_anyint_ty, llvm_ptr_ty, LLVMMatchType<0>],
+                              [NoCapture<ArgIndex<1>>, IntrWriteMem,
+                               IntrHasSideEffects]>,
+        RISCVVIntrinsic;
+
+  // Output: ()
+  // Input: (mtd, mat1, mat2, tm, tn, tk, twiden)
+  class RISCVSFCustomMatMul<bit is_float = false>
+      : DefaultAttrsIntrinsic<[], [llvm_anyint_ty, llvm_anyvector_ty,
+                                   !if(is_float, LLVMMatchType<1>,
+                                                 llvm_anyvector_ty),
+                                   LLVMMatchType<0>, LLVMMatchType<0>,
+                                   LLVMMatchType<0>, LLVMMatchType<0>],
+                              [IntrNoMem, IntrHasSideEffects,
+                               ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<6>>]>,
+        RISCVVIntrinsic;
+
+  def int_riscv_sf_vsettnt  : RISCVSFVSet;
+  def int_riscv_sf_vsettm   : RISCVSFVSet;
+  def int_riscv_sf_vsettk   : RISCVSFVSet;
+
+  def int_riscv_sf_vlte8    : RISCVSFTileLoad;
+  def int_riscv_sf_vlte16   : RISCVSFTileLoad;
+  def int_riscv_sf_vlte32   : RISCVSFTileLoad;
+  def int_riscv_sf_vlte64   : RISCVSFTileLoad;
+  def int_riscv_sf_vste8    : RISCVSFTileStore;
+  def int_riscv_sf_vste16   : RISCVSFTileStore;
+  def int_riscv_sf_vste32   : RISCVSFTileStore;
+  def int_riscv_sf_vste64   : RISCVSFTileStore;
+
+  // Output: (vd)
+  // Input: (tss, tn)
+  def int_riscv_sf_vtmv_v_t
+      : DefaultAttrsIntrinsic<[llvm_anyvector_ty],
+                              [llvm_anyint_ty, LLVMMatchType<1>],
+                              [IntrNoMem, IntrHasSideEffects]>,
+      RISCVVIntrinsic {
+    let VLOperand = 2;
+  }
+  // Output: ()
+  // Input: (tss, vs2, tn)
+  def int_riscv_sf_vtmv_t_v
+      : DefaultAttrsIntrinsic<[], [LLVMMatchType<1>, llvm_anyvector_ty,
+                                   llvm_anyint_ty], [IntrNoMem, IntrHasSideEffects]>,
+      RISCVVIntrinsic {
+    let VLOperand = 2;
+  }
+
+  foreach a = ["u", "s"] in {
+    foreach b = ["u", "s"] in {
+      def int_riscv_sf_mm_ # a # _ # b   : RISCVSFCustomMatMul;
+    }
+  }
+
+  def int_riscv_sf_mm_f_f : RISCVSFCustomMatMul<true>;
+  foreach e1 = [5, 4] in
+    foreach e2 = [5, 4] in
+      def int_riscv_sf_mm_e # e1 # m # !sub(7, e1) # _e # e2 # m # !sub(7, e2)
+          : RISCVSFCustomMatMul<true>;
+
+  // Output: ()
+  // Input: (mtd)
+  def int_riscv_sf_vtzero_t
+      : DefaultAttrsIntrinsic<[],
+                              [llvm_anyint_ty, LLVMMatchType<0>,LLVMMatchType<0>,
+                               LLVMMatchType<0>, LLVMMatchType<0>],
+                              [ImmArg<ArgIndex<0>>, ImmArg<ArgIndex<3>>,
+                               ImmArg<ArgIndex<4>>, IntrNoMem, IntrHasSideEffects]>,
+        RISCVVIntrinsic;
+
+  // Output: ()
+  // Input: ()
+  def int_riscv_sf_vtdiscard
+      : DefaultAttrsIntrinsic<[], [], [IntrNoMem, IntrHasSideEffects]>,
+        RISCVVIntrinsic;
 } // TargetPrefix = "riscv"

llvm/lib/Target/RISCV/AsmParser/RISCVAsmParser.cpp

@@ -1616,6 +1616,10 @@ bool RISCVAsmParser::matchAndEmitInstruction(SMLoc IDLoc, unsigned &Opcode,
                                       "operand must be a valid system register "
                                       "name or an integer in the range");
   }
+  case Match_InvalidXSfmmVType: {
+    SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
+    return generateXSfmmVTypeError(ErrorLoc);
+  }
   case Match_InvalidVTypeI: {
     SMLoc ErrorLoc = ((RISCVOperand &)*Operands[ErrorInfo]).getStartLoc();
     return generateVTypeError(ErrorLoc);

llvm/lib/Target/RISCV/MCTargetDesc/RISCVBaseInfo.h

@@ -138,6 +138,25 @@ enum {
   // 3 -> SEW * 4
   DestEEWShift = ElementsDependOnMaskShift + 1,
   DestEEWMask = 3ULL << DestEEWShift,
+
+  // 0 -> Don't care about altfmt bit in VTYPE.
+  // 1 -> Is not altfmt.
+  // 2 -> Is altfmt(BF16).
+  AltFmtTypeShift = DestEEWShift + 2,
+  AltFmtTypeMask = 3ULL << AltFmtTypeShift,
+
+  IsWidenShift = AltFmtTypeShift + 2,
+  IsWidenMask = 1ULL << IsWidenShift,
+
+  // XSfmmbase
+  HasTWidenOpShift = IsWidenShift + 1,
+  HasTWidenOpMask = 1ULL << HasTWidenOpShift,
+
+  HasTMOpShift = HasTWidenOpShift + 1,
+  HasTMOpMask = 1ULL << HasTMOpShift,
+
+  HasTKOpShift = HasTMOpShift + 1,
+  HasTKOpMask = 1ULL << HasTKOpShift,
 };
 
 // Helper functions to read TSFlags.
@@ -179,6 +198,11 @@ static inline bool hasRoundModeOp(uint64_t TSFlags) {
   return TSFlags & HasRoundModeOpMask;
 }
 
+enum class AltFmtType { DontCare, NotAltFmt, AltFmt };
+static inline AltFmtType getAltFmtType(uint64_t TSFlags) {
+  return static_cast<AltFmtType>((TSFlags & AltFmtTypeMask) >> AltFmtTypeShift);
+}
+
 /// \returns true if this instruction uses vxrm
 static inline bool usesVXRM(uint64_t TSFlags) { return TSFlags & UsesVXRMMask; }
 
@@ -194,11 +218,47 @@ static inline bool elementsDependOnMask(uint64_t TSFlags) {
   return TSFlags & ElementsDependOnMaskMask;
 }
 
+// XSfmmbase
+static inline bool hasTWidenOp(uint64_t TSFlags) {
+  return TSFlags & HasTWidenOpMask;
+}
+
+static inline bool hasTMOp(uint64_t TSFlags) { return TSFlags & HasTMOpMask; }
+
+static inline bool hasTKOp(uint64_t TSFlags) { return TSFlags & HasTKOpMask; }
+
+static inline unsigned getTNOpNum(const MCInstrDesc &Desc) {
+  const uint64_t TSFlags = Desc.TSFlags;
+  assert(hasTWidenOp(TSFlags) && hasVLOp(TSFlags));
+  unsigned Offset = 3;
+  if (hasTKOp(TSFlags))
+    Offset = 4;
+  return Desc.getNumOperands() - Offset;
+}
+
+static inline unsigned getTMOpNum(const MCInstrDesc &Desc) {
+  const uint64_t TSFlags = Desc.TSFlags;
+  assert(hasTWidenOp(TSFlags) && hasTMOp(TSFlags));
+  if (hasTKOp(TSFlags))
+    return Desc.getNumOperands() - 5;
+  // vtzero.t
+  return Desc.getNumOperands() - 4;
+}
+
+static inline unsigned getTKOpNum(const MCInstrDesc &Desc) {
+  const uint64_t TSFlags = Desc.TSFlags;
+  assert(hasTWidenOp(TSFlags) && hasTKOp(TSFlags));
+  return Desc.getNumOperands() - 3;
+}
+
 static inline unsigned getVLOpNum(const MCInstrDesc &Desc) {
   const uint64_t TSFlags = Desc.TSFlags;
   // This method is only called if we expect to have a VL operand, and all
   // instructions with VL also have SEW.
   assert(hasSEWOp(TSFlags) && hasVLOp(TSFlags));
+  // In Xsfmmbase, TN is alias for VL, so here we use the same TSFlags bit.
+  if (hasTWidenOp(TSFlags))
+    return getTNOpNum(Desc);
   unsigned Offset = 2;
   if (hasVecPolicyOp(TSFlags))
     Offset = 3;
@@ -216,7 +276,7 @@ static inline unsigned getSEWOpNum(const MCInstrDesc &Desc) {
   const uint64_t TSFlags = Desc.TSFlags;
   assert(hasSEWOp(TSFlags));
   unsigned Offset = 1;
-  if (hasVecPolicyOp(TSFlags))
+  if (hasVecPolicyOp(TSFlags) || hasTWidenOp(TSFlags))
     Offset = 2;
   return Desc.getNumOperands() - Offset;
 }
@@ -233,6 +293,9 @@ static inline int getFRMOpNum(const MCInstrDesc &Desc) {
   if (!hasRoundModeOp(TSFlags) || usesVXRM(TSFlags))
     return -1;
 
+  if (hasTWidenOp(TSFlags) && hasTMOp(TSFlags))
+    return getTMOpNum(Desc) - 1;
+
   // The operand order
   // --------------------------------------
   // | n-1 (if any)   | n-2  | n-3 | n-4 |
@@ -375,6 +438,7 @@ enum OperandType : unsigned {
   // instructions to represent a value that be passed as AVL to either vsetvli
   // or vsetivli.
   OPERAND_AVL,
+  OPERAND_XSFMM_VTYPE,
 };
 } // namespace RISCVOp
 

llvm/lib/Target/RISCV/RISCVAsmPrinter.cpp

@@ -1100,6 +1100,12 @@ static bool lowerRISCVVMachineInstrToMCInst(const MachineInstr *MI,
     --NumOps;
   if (RISCVII::hasRoundModeOp(TSFlags))
     --NumOps;
+  if (RISCVII::hasTWidenOp(TSFlags))
+    --NumOps;
+  if (RISCVII::hasTMOp(TSFlags))
+    --NumOps;
+  if (RISCVII::hasTKOp(TSFlags))
+    --NumOps;
 
   bool hasVLOutput = RISCV::isFaultFirstLoad(*MI);
   for (unsigned OpNo = 0; OpNo != NumOps; ++OpNo) {