@@ -947,11 +947,11 @@ SDValue TargetLowering::SimplifyMultipleUseDemandedVectorElts(
947947
948948// Attempt to form ext(avgfloor(A, B)) from shr(add(ext(A), ext(B)), 1).
949949// or to form ext(avgceil(A, B)) from shr(add(ext(A), ext(B), 1), 1).
950- static SDValue combineShiftToAVG (SDValue Op, SelectionDAG &DAG,
950+ static SDValue combineShiftToAVG (SDValue Op,
951+ TargetLowering::TargetLoweringOpt &TLO,
951952 const TargetLowering &TLI,
952953 const APInt &DemandedBits,
953- const APInt &DemandedElts,
954- unsigned Depth) {
954+ const APInt &DemandedElts, unsigned Depth) {
955955 assert ((Op.getOpcode () == ISD::SRL || Op.getOpcode () == ISD::SRA) &&
956956 " SRL or SRA node is required here!"
957957 // Is the right shift using an immediate value of 1?
@@ -1002,6 +1002,7 @@ static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG,
10021002 // If the shift is unsigned (srl):
10031003 // - Needs >= 1 zero bit for both operands.
10041004 // - Needs 1 demanded bit zero and >= 2 sign bits.
1005+ SelectionDAG &DAG = TLO.DAG ;
10051006 unsigned ShiftOpc = Op.getOpcode ();
10061007 bool IsSigned = false ;
10071008 unsigned KnownBits;
@@ -1057,10 +1058,10 @@ static SDValue combineShiftToAVG(SDValue Op, SelectionDAG &DAG,
10571058 EVT NVT = EVT::getIntegerVT (*DAG.getContext (), llvm::bit_ceil (MinWidth));
10581059 if (VT.isVector ())
10591060 NVT = EVT::getVectorVT (*DAG.getContext (), NVT, VT.getVectorElementCount ());
1060- if (!TLI.isOperationLegalOrCustom (AVGOpc, NVT)) {
1061+ if (TLO. LegalOperations () && !TLI.isOperationLegal (AVGOpc, NVT)) {
10611062 // If we could not transform, and (both) adds are nuw/nsw, we can use the
10621063 // larger type size to do the transform.
1063- if (!TLI.isOperationLegalOrCustom (AVGOpc, VT))
1064+ if (TLO. LegalOperations () && !TLI.isOperationLegal (AVGOpc, VT))
10641065 return SDValue ();
10651066 if (DAG.willNotOverflowAdd (IsSigned, Add.getOperand (0 ),
10661067 Add.getOperand (1 )) &&
@@ -1907,11 +1908,6 @@ bool TargetLowering::SimplifyDemandedBits(
19071908 SDValue Op1 = Op.getOperand (1 );
19081909 EVT ShiftVT = Op1.getValueType ();
19091910
1910- // Try to match AVG patterns.
1911- if (SDValue AVG = combineShiftToAVG (Op, TLO.DAG , *this , DemandedBits,
1912- DemandedElts, Depth + 1 ))
1913- return TLO.CombineTo (Op, AVG);
1914-
19151911 if (const APInt *SA =
19161912 TLO.DAG .getValidShiftAmountConstant (Op, DemandedElts)) {
19171913 unsigned ShAmt = SA->getZExtValue ();
@@ -1992,6 +1988,12 @@ bool TargetLowering::SimplifyDemandedBits(
19921988 // shift amounts.
19931989 Known = TLO.DAG .computeKnownBits (Op, DemandedElts, Depth);
19941990 }
1991+
1992+ // Try to match AVG patterns (after shift simplification).
1993+ if (SDValue AVG = combineShiftToAVG (Op, TLO, *this , DemandedBits,
1994+ DemandedElts, Depth + 1 ))
1995+ return TLO.CombineTo (Op, AVG);
1996+
19951997 break ;
19961998 }
19971999 case ISD::SRA: {
@@ -2013,11 +2015,6 @@ bool TargetLowering::SimplifyDemandedBits(
20132015 if (DemandedBits.isOne ())
20142016 return TLO.CombineTo (Op, TLO.DAG .getNode (ISD::SRL, dl, VT, Op0, Op1));
20152017
2016- // Try to match AVG patterns.
2017- if (SDValue AVG = combineShiftToAVG (Op, TLO.DAG , *this , DemandedBits,
2018- DemandedElts, Depth + 1 ))
2019- return TLO.CombineTo (Op, AVG);
2020-
20212018 if (const APInt *SA =
20222019 TLO.DAG .getValidShiftAmountConstant (Op, DemandedElts)) {
20232020 unsigned ShAmt = SA->getZExtValue ();
@@ -2103,6 +2100,12 @@ bool TargetLowering::SimplifyDemandedBits(
21032100 }
21042101 }
21052102 }
2103+
2104+ // Try to match AVG patterns (after shift simplification).
2105+ if (SDValue AVG = combineShiftToAVG (Op, TLO, *this , DemandedBits,
2106+ DemandedElts, Depth + 1 ))
2107+ return TLO.CombineTo (Op, AVG);
2108+
21062109 break ;
21072110 }
21082111 case ISD::FSHL:
@@ -9200,6 +9203,49 @@ SDValue TargetLowering::expandABD(SDNode *N, SelectionDAG &DAG) const {
92009203 DAG.getNode (ISD::SUB, dl, VT, RHS, LHS));
92019204}
92029205
9206+ SDValue TargetLowering::expandAVG (SDNode *N, SelectionDAG &DAG) const {
9207+ SDLoc dl (N);
9208+ EVT VT = N->getValueType (0 );
9209+ SDValue LHS = N->getOperand (0 );
9210+ SDValue RHS = N->getOperand (1 );
9211+
9212+ unsigned Opc = N->getOpcode ();
9213+ bool IsFloor = Opc == ISD::AVGFLOORS || Opc == ISD::AVGFLOORU;
9214+ bool IsSigned = Opc == ISD::AVGCEILS || Opc == ISD::AVGFLOORS;
9215+ unsigned ShiftOpc = IsSigned ? ISD::SRA : ISD::SRL;
9216+ assert ((Opc == ISD::AVGFLOORS || Opc == ISD::AVGCEILS ||
9217+ Opc == ISD::AVGFLOORU || Opc == ISD::AVGCEILU) &&
9218+ " Unknown AVG node"
9219+
9220+ // If the operands are already extended, we can add+shift.
9221+ bool IsExt =
9222+ (IsSigned && DAG.ComputeNumSignBits (LHS) >= 2 &&
9223+ DAG.ComputeNumSignBits (RHS) >= 2 ) ||
9224+ (!IsSigned && DAG.computeKnownBits (LHS).countMinLeadingZeros () >= 1 &&
9225+ DAG.computeKnownBits (RHS).countMinLeadingZeros () >= 1 );
9226+ if (IsExt) {
9227+ SDValue Sum = DAG.getNode (ISD::ADD, dl, VT, LHS, RHS);
9228+ if (!IsFloor)
9229+ Sum = DAG.getNode (ISD::ADD, dl, VT, Sum, DAG.getConstant (1 , dl, VT));
9230+ return DAG.getNode (ShiftOpc, dl, VT, Sum,
9231+ DAG.getShiftAmountConstant (1 , VT, dl));
9232+ }
9233+
9234+ // avgceils(lhs, rhs) -> sub(or(lhs,rhs),ashr(xor(lhs,rhs),1))
9235+ // avgceilu(lhs, rhs) -> sub(or(lhs,rhs),lshr(xor(lhs,rhs),1))
9236+ // avgfloors(lhs, rhs) -> add(and(lhs,rhs),ashr(xor(lhs,rhs),1))
9237+ // avgflooru(lhs, rhs) -> add(and(lhs,rhs),lshr(xor(lhs,rhs),1))
9238+ unsigned SumOpc = IsFloor ? ISD::ADD : ISD::SUB;
9239+ unsigned SignOpc = IsFloor ? ISD::AND : ISD::OR;
9240+ LHS = DAG.getFreeze (LHS);
9241+ RHS = DAG.getFreeze (RHS);
9242+ SDValue Sign = DAG.getNode (SignOpc, dl, VT, LHS, RHS);
9243+ SDValue Xor = DAG.getNode (ISD::XOR, dl, VT, LHS, RHS);
9244+ SDValue Shift =
9245+ DAG.getNode (ShiftOpc, dl, VT, Xor, DAG.getShiftAmountConstant (1 , VT, dl));
9246+ return DAG.getNode (SumOpc, dl, VT, Sign, Shift);
9247+ }
9248+
92039249SDValue TargetLowering::expandBSWAP (SDNode *N, SelectionDAG &DAG) const {
92049250 SDLoc dl (N);
92059251 EVT VT = N->getValueType (0 );
0 commit comments