diff --git a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp index 35c3bc9708d91..026e0a365b8dc 100644 --- a/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp +++ b/llvm/lib/Target/RISCV/RISCVVectorPeephole.cpp @@ -66,6 +66,7 @@ class RISCVVectorPeephole : public MachineFunctionPass { bool convertToWholeRegister(MachineInstr &MI) const; bool convertToUnmasked(MachineInstr &MI) const; bool convertVMergeToVMv(MachineInstr &MI) const; + bool foldUndefPassthruVMV_V_V(MachineInstr &MI); bool foldVMV_V_V(MachineInstr &MI); bool isAllOnesMask(const MachineInstr *MaskDef) const; @@ -472,6 +473,38 @@ bool RISCVVectorPeephole::ensureDominates(const MachineOperand &MO, return true; } +/// If a PseudoVMV_V_V's passthru is undef then we can replace it with its input +bool RISCVVectorPeephole::foldUndefPassthruVMV_V_V(MachineInstr &MI) { + if (RISCV::getRVVMCOpcode(MI.getOpcode()) != RISCV::VMV_V_V) + return false; + if (MI.getOperand(1).getReg() != RISCV::NoRegister) + return false; + + // If the input was a pseudo with a policy operand, we can give it a tail + // agnostic policy if MI's undef tail subsumes the input's. + MachineInstr *Src = MRI->getVRegDef(MI.getOperand(2).getReg()); + if (Src && !Src->hasUnmodeledSideEffects() && + MRI->hasOneUse(MI.getOperand(2).getReg()) && + RISCVII::hasVLOp(Src->getDesc().TSFlags) && + RISCVII::hasVecPolicyOp(Src->getDesc().TSFlags) && + getSEWLMULRatio(MI) == getSEWLMULRatio(*Src)) { + const MachineOperand &MIVL = MI.getOperand(3); + const MachineOperand &SrcVL = + Src->getOperand(RISCVII::getVLOpNum(Src->getDesc())); + + MachineOperand &SrcPolicy = + Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc())); + + if (isVLKnownLE(MIVL, SrcVL)) + SrcPolicy.setImm(SrcPolicy.getImm() | RISCVII::TAIL_AGNOSTIC); + } + + MRI->replaceRegWith(MI.getOperand(0).getReg(), MI.getOperand(2).getReg()); + MI.eraseFromParent(); + V0Defs.erase(&MI); + return true; +} + /// If a PseudoVMV_V_V is the only user of its input, fold its passthru and VL /// into it. /// @@ -531,9 +564,8 @@ bool RISCVVectorPeephole::foldVMV_V_V(MachineInstr &MI) { // If MI was tail agnostic and the VL didn't increase, preserve it. int64_t Policy = RISCVII::TAIL_UNDISTURBED_MASK_UNDISTURBED; - bool TailAgnostic = (MI.getOperand(5).getImm() & RISCVII::TAIL_AGNOSTIC) || - Passthru.getReg() == RISCV::NoRegister; - if (TailAgnostic && isVLKnownLE(MI.getOperand(3), SrcVL)) + if ((MI.getOperand(5).getImm() & RISCVII::TAIL_AGNOSTIC) && + isVLKnownLE(MI.getOperand(3), SrcVL)) Policy |= RISCVII::TAIL_AGNOSTIC; Src->getOperand(RISCVII::getVecPolicyOpNum(Src->getDesc())).setImm(Policy); @@ -584,6 +616,10 @@ bool RISCVVectorPeephole::runOnMachineFunction(MachineFunction &MF) { Changed |= convertToUnmasked(MI); Changed |= convertToWholeRegister(MI); Changed |= convertVMergeToVMv(MI); + if (foldUndefPassthruVMV_V_V(MI)) { + Changed |= true; + continue; // MI is erased + } Changed |= foldVMV_V_V(MI); } } diff --git a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir index 1419eede6ca9d..19a918148e6eb 100644 --- a/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir +++ b/llvm/test/CodeGen/RISCV/rvv/rvv-peephole-vmerge-to-vmv.mir @@ -15,7 +15,6 @@ body: | ; CHECK-NEXT: %avl:gprnox0 = COPY $x1 ; CHECK-NEXT: %mask:vmv0 = PseudoVMSET_M_B8 %avl, 5 /* e32 */ ; CHECK-NEXT: $v0 = COPY %mask - ; CHECK-NEXT: %x:vr = PseudoVMV_V_V_M1 $noreg, %true, %avl, 5 /* e32 */, 0 /* tu, mu */ %false:vr = COPY $v8 %true:vr = COPY $v9 %avl:gprnox0 = COPY $x1 diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll index c1602c912da63..7f248a39b54fa 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll +++ b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.ll @@ -199,10 +199,8 @@ define @unfoldable_mismatched_sew( %passthr define @undef_passthru( %passthru, %x, %y, iXLen %avl) { ; CHECK-LABEL: undef_passthru: ; CHECK: # %bb.0: -; CHECK-NEXT: vsetvli zero, a0, e64, m1, tu, ma +; CHECK-NEXT: vsetvli zero, a0, e64, m1, ta, ma ; CHECK-NEXT: vadd.vv v8, v9, v10 -; CHECK-NEXT: vsetvli zero, zero, e64, m1, ta, ma -; CHECK-NEXT: vmv.v.v v8, v8 ; CHECK-NEXT: ret %a = call @llvm.riscv.vadd.nxv1i64.nxv1i64( %passthru, %x, %y, iXLen %avl) %b = call @llvm.riscv.vmv.v.v.nxv1i64( undef, %a, iXLen %avl) diff --git a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir index 7e2ac0e26f251..6858231bf0e6c 100644 --- a/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir +++ b/llvm/test/CodeGen/RISCV/rvv/vmv.v.v-peephole.mir @@ -69,8 +69,7 @@ body: | ; CHECK: liveins: $v8 ; CHECK-NEXT: {{ $}} ; CHECK-NEXT: %passthru:vr = COPY $v8 - ; CHECK-NEXT: %x:vr = PseudoVADD_VV_M1 %passthru, $noreg, $noreg, 4, 5 /* e32 */, 0 /* tu, mu */ - ; CHECK-NEXT: %y:vr = PseudoVMV_V_V_M1 $noreg, %x, 4, 5 /* e32 */, 1 /* ta, mu */ + ; CHECK-NEXT: %x:vr = PseudoVADD_VV_M1 %passthru, $noreg, $noreg, 4, 5 /* e32 */, 1 /* ta, mu */ %passthru:vr = COPY $v8 %x:vr = PseudoVADD_VV_M1 %passthru, $noreg, $noreg, 4, 5 /* e32 */, 0 /* tu, mu */ %y:vr = PseudoVMV_V_V_M1 $noreg, %x, 4, 5 /* e32 */, 1 /* ta, mu */