diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 6e8138725375a..65528de8e7ac1 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2690,6 +2690,54 @@ struct MemorySanitizerVisitor : public InstVisitor { SC.Done(&I); } + // Perform a bitwise OR on the horizontal pairs (or other specified grouping) + // of elements. + // + // For example, suppose we have: + // VectorA: + // VectorB: + // ReductionFactor: 3. + // The output would be: + // + // + // This is convenient for instrumenting horizontal add/sub. + // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic(). + Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, + Value *VectorA, Value *VectorB) { + assert(isa(VectorA->getType())); + unsigned TotalNumElems = + cast(VectorA->getType())->getNumElements(); + + if (VectorB) { + assert(VectorA->getType() == VectorB->getType()); + TotalNumElems = TotalNumElems * 2; + } + + assert(TotalNumElems % ReductionFactor == 0); + + Value *Or = nullptr; + + IRBuilder<> IRB(&I); + for (unsigned i = 0; i < ReductionFactor; i++) { + SmallVector Mask; + for (unsigned X = 0; X < TotalNumElems; X += ReductionFactor) + Mask.push_back(X + i); + + Value *Masked; + if (VectorB) + Masked = IRB.CreateShuffleVector(VectorA, VectorB, Mask); + else + Masked = IRB.CreateShuffleVector(VectorA, Mask); + + if (Or) + Or = IRB.CreateOr(Or, Masked); + else + Or = Masked; + } + + return Or; + } + /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent /// fields. /// @@ -2711,31 +2759,16 @@ struct MemorySanitizerVisitor : public InstVisitor { 2 * ReturnType->getNumElements()); IRBuilder<> IRB(&I); - unsigned Width = ParamType->getNumElements() * I.arg_size(); // Horizontal OR of shadow - SmallVector EvenMask; - SmallVector OddMask; - for (unsigned X = 0; X < Width; X += 2) { - EvenMask.push_back(X); - OddMask.push_back(X + 1); - } - Value *FirstArgShadow = getShadow(&I, 0); - Value *EvenShadow; - Value *OddShadow; - if (I.arg_size() == 2) { - Value *SecondArgShadow = getShadow(&I, 1); - EvenShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask); - OddShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask); - } else { - EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask); - OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask); - } + Value *SecondArgShadow = nullptr; + if (I.arg_size() == 2) + SecondArgShadow = getShadow(&I, 1); + + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow, + SecondArgShadow); - Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); setShadow(&I, OrShadow); @@ -2768,23 +2801,14 @@ struct MemorySanitizerVisitor : public InstVisitor { IRBuilder<> IRB(&I); - unsigned TotalNumElems = ParamType->getNumElements() * I.arg_size(); FixedVectorType *ReinterpretShadowTy = nullptr; assert(isAligned(Align(ReinterpretElemWidth), ParamType->getPrimitiveSizeInBits())); ReinterpretShadowTy = FixedVectorType::get( IRB.getIntNTy(ReinterpretElemWidth), ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth); - TotalNumElems = ReinterpretShadowTy->getNumElements() * I.arg_size(); // Horizontal OR of shadow - SmallVector EvenMask; - SmallVector OddMask; - for (unsigned X = 0; X < TotalNumElems - 1; X += 2) { - EvenMask.push_back(X); - OddMask.push_back(X + 1); - } - Value *FirstArgShadow = getShadow(&I, 0); FirstArgShadow = IRB.CreateBitCast(FirstArgShadow, ReinterpretShadowTy); @@ -2796,22 +2820,15 @@ struct MemorySanitizerVisitor : public InstVisitor { Align(2), cast(FirstArgShadow->getType())->getNumElements())); - Value *EvenShadow; - Value *OddShadow; + Value *SecondArgShadow = nullptr; if (I.arg_size() == 2) { - Value *SecondArgShadow = getShadow(&I, 1); + SecondArgShadow = getShadow(&I, 1); SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, ReinterpretShadowTy); - - EvenShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask); - OddShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask); - } else { - EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask); - OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask); } - Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow); + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow, + SecondArgShadow); + OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); setShadow(&I, OrShadow); @@ -3219,7 +3236,7 @@ struct MemorySanitizerVisitor : public InstVisitor { /// Caller guarantees that this intrinsic does not access memory. /// /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by - /// by this handler. + /// by this handler. See horizontalReduce(). [[maybe_unused]] bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I, unsigned int trailingFlags) {