From f8f42afe366f1b4113e6149b1651d91b535dc3d7 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 07:01:10 +0000 Subject: [PATCH 1/7] [NFCI][msan] Refactor into 'horizontalReduce' The functionality is used by two helper functions, and will be used even more in the future (e.g., https://github.com/llvm/llvm-project/pull/152941). --- .../Instrumentation/MemorySanitizer.cpp | 91 ++++++++++--------- 1 file changed, 48 insertions(+), 43 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 6e8138725375a..5e618fec6dd0c 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2690,6 +2690,41 @@ struct MemorySanitizerVisitor : public InstVisitor { SC.Done(&I); } + // Perform a bitwise OR on the horizontal pairs (or other specified grouping) + // of elements. This is convenient for instrumenting horizontal add/sub. + Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, + Value *VectorA, Value* VectorB) { + IRBuilder<> IRB(&I); + assert(isa(VectorA->getType())); + unsigned TotalNumElems = cast(VectorA->getType())->getNumElements(); + + if (VectorB) { + assert(VectorA->getType() == VectorB->getType()); + TotalNumElems = TotalNumElems * 2; + } + + Value *Or = nullptr; + + for (unsigned i = 0; i < ReductionFactor; i++) { + SmallVector Mask; + for (unsigned X = 0; X < TotalNumElems; X += ReductionFactor) + Mask.push_back(X + i); + + Value *Masked; + if (VectorB) + Masked = IRB.CreateShuffleVector(VectorA, VectorB, Mask); + else + Masked = IRB.CreateShuffleVector(VectorA, Mask); + + if (Or) + Or = IRB.CreateOr(Or, Masked); + else + Or = Masked; + } + + return Or; + } + /// Propagate shadow for 1- or 2-vector intrinsics that combine adjacent /// fields. /// @@ -2711,31 +2746,16 @@ struct MemorySanitizerVisitor : public InstVisitor { 2 * ReturnType->getNumElements()); IRBuilder<> IRB(&I); - unsigned Width = ParamType->getNumElements() * I.arg_size(); // Horizontal OR of shadow - SmallVector EvenMask; - SmallVector OddMask; - for (unsigned X = 0; X < Width; X += 2) { - EvenMask.push_back(X); - OddMask.push_back(X + 1); - } - Value *FirstArgShadow = getShadow(&I, 0); - Value *EvenShadow; - Value *OddShadow; - if (I.arg_size() == 2) { - Value *SecondArgShadow = getShadow(&I, 1); - EvenShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask); - OddShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask); - } else { - EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask); - OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask); - } + Value *SecondArgShadow = nullptr; + if (I.arg_size() == 2) + SecondArgShadow = getShadow(&I, 1); + + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, + FirstArgShadow, SecondArgShadow); - Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); setShadow(&I, OrShadow); @@ -2768,23 +2788,14 @@ struct MemorySanitizerVisitor : public InstVisitor { IRBuilder<> IRB(&I); - unsigned TotalNumElems = ParamType->getNumElements() * I.arg_size(); FixedVectorType *ReinterpretShadowTy = nullptr; assert(isAligned(Align(ReinterpretElemWidth), ParamType->getPrimitiveSizeInBits())); ReinterpretShadowTy = FixedVectorType::get( IRB.getIntNTy(ReinterpretElemWidth), ParamType->getPrimitiveSizeInBits() / ReinterpretElemWidth); - TotalNumElems = ReinterpretShadowTy->getNumElements() * I.arg_size(); // Horizontal OR of shadow - SmallVector EvenMask; - SmallVector OddMask; - for (unsigned X = 0; X < TotalNumElems - 1; X += 2) { - EvenMask.push_back(X); - OddMask.push_back(X + 1); - } - Value *FirstArgShadow = getShadow(&I, 0); FirstArgShadow = IRB.CreateBitCast(FirstArgShadow, ReinterpretShadowTy); @@ -2796,22 +2807,16 @@ struct MemorySanitizerVisitor : public InstVisitor { Align(2), cast(FirstArgShadow->getType())->getNumElements())); - Value *EvenShadow; - Value *OddShadow; + Value *SecondArgShadow = nullptr; if (I.arg_size() == 2) { - Value *SecondArgShadow = getShadow(&I, 1); - SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, ReinterpretShadowTy); - - EvenShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, EvenMask); - OddShadow = - IRB.CreateShuffleVector(FirstArgShadow, SecondArgShadow, OddMask); - } else { - EvenShadow = IRB.CreateShuffleVector(FirstArgShadow, EvenMask); - OddShadow = IRB.CreateShuffleVector(FirstArgShadow, OddMask); + SecondArgShadow = getShadow(&I, 1); + SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, + ReinterpretShadowTy); } - Value *OrShadow = IRB.CreateOr(EvenShadow, OddShadow); + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, + FirstArgShadow, SecondArgShadow); + OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); setShadow(&I, OrShadow); From e15df775e080fa4ef500a3ac078ba716b2342967 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 07:11:48 +0000 Subject: [PATCH 2/7] Assert that total number of elements is divisible by reduction factor --- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 5e618fec6dd0c..94aaddc014ba0 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2694,7 +2694,6 @@ struct MemorySanitizerVisitor : public InstVisitor { // of elements. This is convenient for instrumenting horizontal add/sub. Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, Value *VectorA, Value* VectorB) { - IRBuilder<> IRB(&I); assert(isa(VectorA->getType())); unsigned TotalNumElems = cast(VectorA->getType())->getNumElements(); @@ -2703,8 +2702,11 @@ struct MemorySanitizerVisitor : public InstVisitor { TotalNumElems = TotalNumElems * 2; } + assert(TotalNumElems % ReductionFactor == 0); + Value *Or = nullptr; + IRBuilder<> IRB(&I); for (unsigned i = 0; i < ReductionFactor; i++) { SmallVector Mask; for (unsigned X = 0; X < TotalNumElems; X += ReductionFactor) From bfca6c329d583bb3cd4d06fbd24f900725f3ba8a Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 07:13:06 +0000 Subject: [PATCH 3/7] Format --- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 94aaddc014ba0..78c428d9a7873 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2695,7 +2695,8 @@ struct MemorySanitizerVisitor : public InstVisitor { Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, Value *VectorA, Value* VectorB) { assert(isa(VectorA->getType())); - unsigned TotalNumElems = cast(VectorA->getType())->getNumElements(); + unsigned TotalNumElems + = cast(VectorA->getType())->getNumElements(); if (VectorB) { assert(VectorA->getType() == VectorB->getType()); From 76a4ff432c2e4d082228c7b6e6251234d2144f50 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 20:50:04 +0000 Subject: [PATCH 4/7] Reformat --- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 78c428d9a7873..6d5e372ba07aa 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2693,10 +2693,10 @@ struct MemorySanitizerVisitor : public InstVisitor { // Perform a bitwise OR on the horizontal pairs (or other specified grouping) // of elements. This is convenient for instrumenting horizontal add/sub. Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, - Value *VectorA, Value* VectorB) { + Value *VectorA, Value *VectorB) { assert(isa(VectorA->getType())); - unsigned TotalNumElems - = cast(VectorA->getType())->getNumElements(); + unsigned TotalNumElems = + cast(VectorA->getType())->getNumElements(); if (VectorB) { assert(VectorA->getType() == VectorB->getType()); From 5707f0de1f09a87015cf1e5a34f35ba7abc26450 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 20:56:02 +0000 Subject: [PATCH 5/7] Example --- .../Transforms/Instrumentation/MemorySanitizer.cpp | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 6d5e372ba07aa..21710684aa392 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2691,7 +2691,17 @@ struct MemorySanitizerVisitor : public InstVisitor { } // Perform a bitwise OR on the horizontal pairs (or other specified grouping) - // of elements. This is convenient for instrumenting horizontal add/sub. + // of elements. + // + // For example, suppose we have: + // VectorA: + // VectorB: + // ReductionFactor: 3. + // The output would be: + // + // + // This is convenient for instrumenting horizontal add/sub. + // For bitwise OR on "vertical" pairs, see maybeHandleSimpleNomemIntrinsic(). Value *horizontalReduce(IntrinsicInst &I, unsigned ReductionFactor, Value *VectorA, Value *VectorB) { assert(isa(VectorA->getType())); @@ -3227,7 +3237,7 @@ struct MemorySanitizerVisitor : public InstVisitor { /// Caller guarantees that this intrinsic does not access memory. /// /// TODO: "horizontal"/"pairwise" intrinsics are often incorrectly matched by - /// by this handler. + /// by this handler. See horizontalReduce(). [[maybe_unused]] bool maybeHandleSimpleNomemIntrinsic(IntrinsicInst &I, unsigned int trailingFlags) { From ac5c10272336fe160a795e3272619a4ea7312b4c Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 21:01:18 +0000 Subject: [PATCH 6/7] Re-reformat --- .../lib/Transforms/Instrumentation/MemorySanitizer.cpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 21710684aa392..0d81a97287643 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2690,7 +2690,7 @@ struct MemorySanitizerVisitor : public InstVisitor { SC.Done(&I); } - // Perform a bitwise OR on the horizontal pairs (or other specified grouping) + // Perform a bitwise OR on the horizontal pairs (or other specified grouping) // of elements. // // For example, suppose we have: @@ -2766,8 +2766,8 @@ struct MemorySanitizerVisitor : public InstVisitor { if (I.arg_size() == 2) SecondArgShadow = getShadow(&I, 1); - Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, - FirstArgShadow, SecondArgShadow); + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, FirstArgShadow, + SecondArgShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); @@ -2827,8 +2827,8 @@ struct MemorySanitizerVisitor : public InstVisitor { ReinterpretShadowTy); } - Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, - FirstArgShadow, SecondArgShadow); + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, FirstArgShadow, + SecondArgShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); From 7df10d8e450e87a2c70c5351195863a3d040b649 Mon Sep 17 00:00:00 2001 From: Thurston Dang Date: Mon, 11 Aug 2025 21:35:59 +0000 Subject: [PATCH 7/7] Re-re-reformat --- llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp index 0d81a97287643..65528de8e7ac1 100644 --- a/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp +++ b/llvm/lib/Transforms/Instrumentation/MemorySanitizer.cpp @@ -2766,7 +2766,7 @@ struct MemorySanitizerVisitor : public InstVisitor { if (I.arg_size() == 2) SecondArgShadow = getShadow(&I, 1); - Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, FirstArgShadow, + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow, SecondArgShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I)); @@ -2823,11 +2823,10 @@ struct MemorySanitizerVisitor : public InstVisitor { Value *SecondArgShadow = nullptr; if (I.arg_size() == 2) { SecondArgShadow = getShadow(&I, 1); - SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, - ReinterpretShadowTy); + SecondArgShadow = IRB.CreateBitCast(SecondArgShadow, ReinterpretShadowTy); } - Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/ 2, FirstArgShadow, + Value *OrShadow = horizontalReduce(I, /*ReductionFactor=*/2, FirstArgShadow, SecondArgShadow); OrShadow = CreateShadowCast(IRB, OrShadow, getShadowTy(&I));