Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions lib/SPIRV/SPIRVInternal.h
Original file line number Diff line number Diff line change
Expand Up @@ -271,6 +271,7 @@ typedef SPIRVMap<SPIRVExtInstSetKind, std::string, SPIRVExtSetShortName>

#define SPIRV_MD_PARAMETER_DECORATIONS "spirv.ParameterDecorations"
#define SPIRV_MD_DECORATIONS "spirv.Decorations"
#define SPIRV_MD_INTEL_CACHE_DECORATIONS "spirv.DecorationCacheControlINTEL"

#define OCL_TYPE_NAME_SAMPLER_T "sampler_t"
#define SPIR_TYPE_NAME_EVENT_T "opencl.event_t"
Expand Down
62 changes: 62 additions & 0 deletions lib/SPIRV/SPIRVRegularizeLLVM.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -495,6 +495,65 @@ void regularizeWithOverflowInstrinsics(StringRef MangledName, CallInst *Call,
}
ToErase.push_back(Call);
}

// CacheControls(Load/Store)INTEL decorations can be represented as metadata
// placed on memory accessing instruction with the following form:
// !spirv.DecorationCacheControlINTEL !X
// !X = !{i32 %decoration_kind%, i32 %level%, i32 %control%,
// i32 %operand of the instruction to decorate%}
// This function creates a dummy GEP accessing pointer operand of the
// instruction and creates !spirv.Decorations metadata attached to it.
void prepareCacheControlsTranslation(Metadata *MD, Instruction *Inst) {
if (!Inst->mayReadOrWriteMemory())
return;
auto *ArgDecoMD = dyn_cast<MDNode>(MD);
assert(ArgDecoMD && "Decoration list must be a metadata node");
for (unsigned I = 0, E = ArgDecoMD->getNumOperands(); I != E; ++I) {
auto *DecoMD = dyn_cast<MDNode>(ArgDecoMD->getOperand(I));
if (!DecoMD) {
assert(!"Decoration does not name metadata");
return;
}

constexpr size_t CacheControlsNumOps = 4;
if (DecoMD->getNumOperands() != CacheControlsNumOps) {
assert(!"Cache controls metadata on instruction must have 4 operands");
return;
}

auto *const KindMD = cast<ConstantAsMetadata>(DecoMD->getOperand(0));
auto *const LevelMD = cast<ConstantAsMetadata>(DecoMD->getOperand(1));
auto *const ControlMD = cast<ConstantAsMetadata>(DecoMD->getOperand(2));

const size_t TargetArgNo =
mdconst::dyn_extract<ConstantInt>(DecoMD->getOperand(3))
->getZExtValue();
Value *PtrInstOp = Inst->getOperand(TargetArgNo);
if (!PtrInstOp->getType()->isPointerTy()) {
assert(!"Cache controls must decorate a pointer");
return;
}

// Create dummy GEP for SSA copy of the pointer operand. Lets do our best
// to guess pointee type here, but if we won't - just pointer is also fine,
// if necessary TypeScavenger will adjust types and create bitcasts.
IRBuilder Builder(Inst);
Type *GEPTy = PtrInstOp->getType();
if (auto *LI = dyn_cast<LoadInst>(Inst))
GEPTy = LI->getType();
else if (auto *SI = dyn_cast<StoreInst>(Inst))
GEPTy = SI->getValueOperand()->getType();
auto *GEP =
cast<Instruction>(Builder.CreateConstGEP1_32(GEPTy, PtrInstOp, 0));
Inst->setOperand(TargetArgNo, GEP);

SmallVector<Metadata *, 4> MDs;
std::vector<Metadata *> OPs = {KindMD, LevelMD, ControlMD};
MDs.push_back(MDNode::get(Inst->getContext(), OPs));
MDNode *MDList = MDNode::get(Inst->getContext(), MDs);
GEP->setMetadata(SPIRV_MD_DECORATIONS, MDList);
}
}
} // namespace

/// Remove entities not representable by SPIR-V
Expand All @@ -511,9 +570,12 @@ bool SPIRVRegularizeLLVMBase::regularize() {
continue;
}

// TODO: query intrinsic calls from their declarations
std::vector<Instruction *> ToErase;
for (BasicBlock &BB : *F) {
for (Instruction &II : BB) {
if (auto *MD = II.getMetadata(SPIRV_MD_INTEL_CACHE_DECORATIONS))
prepareCacheControlsTranslation(MD, &II);
if (auto *Call = dyn_cast<CallInst>(&II)) {
Call->setTailCall(false);
Function *CF = Call->getCalledFunction();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
; RUN: llvm-as %s -o %t.bc
; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM

; CHECK-SPIRV-DAG: TypeInt [[#Int32:]] 32 0
; CHECK-SPIRV-DAG: Constant [[#Int32]] [[#Zero:]] 0
; CHECK-SPIRV-DAG: Decorate [[#Load1GEPPtr:]] CacheControlLoadINTEL 0 1
; CHECK-SPIRV-DAG: Decorate [[#Load2GEPPtr:]] CacheControlLoadINTEL 1 1
; CHECK-SPIRV-DAG: Decorate [[#Store1GEPPtr:]] CacheControlStoreINTEL 0 1
; CHECK-SPIRV-DAG: Decorate [[#Store2GEPPtr:]] CacheControlStoreINTEL 1 1

; CHECK-SPIRV: FunctionParameter [[#]] [[#Buffer:]]
; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load1GEPPtr:]] [[#Buffer]] [[#Zero]]
; CHECK-SPIRV: Load [[#]] [[#]] [[#Load1GEPPtr]]
; CHECK-SPIRV: PtrAccessChain [[#]] [[#Load2GEPPtr:]] [[#]] [[#Zero]]
; CHECK-SPIRV: Load [[#]] [[#]] [[#Load2GEPPtr]]
; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store1GEPPtr:]] [[#]] [[#Zero]]
; CHECK-SPIRV: Store [[#Store1GEPPtr]]
; CHECK-SPIRV: PtrAccessChain [[#]] [[#Store2GEPPtr:]] [[#]] [[#Zero]]
; CHECK-SPIRV: Store [[#Store2GEPPtr]]

; CHECK-LLVM: %[[#GEPLoad1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad1]], align 4
; CHECK-LLVM: %[[#GEPLoad2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
; CHECK-LLVM: load i32, ptr addrspace(1) %[[#GEPLoad2]], align 4
; CHECK-LLVM: %[[#GEPStore1:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache3:]]
; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore1]], align 4
; CHECK-LLVM: %[[#GEPStore2:]] = getelementptr i32, ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache4:]]
; CHECK-LLVM: store i32 %[[#]], ptr addrspace(1) %[[#GEPStore2]], align 4
; CHECK-LLVM: ![[#Cache1]] = !{![[#DecLoad1:]]}
; CHECK-LLVM: ![[#DecLoad1]] = !{i32 6442, i32 0, i32 1}
; CHECK-LLVM: ![[#Cache2]] = !{![[#DecLoad2:]]}
; CHECK-LLVM: ![[#DecLoad2]] = !{i32 6442, i32 1, i32 1}
; CHECK-LLVM: ![[#Cache3:]] = !{![[#DecStore1:]]}
; CHECK-LLVM: ![[#DecStore1]] = !{i32 6443, i32 0, i32 1}
; CHECK-LLVM: ![[#Cache4:]] = !{![[#DecStore2:]]}
; CHECK-LLVM: ![[#DecStore2]] = !{i32 6443, i32 1, i32 1}

target triple = "spir64-unknown-unknown"

define spir_kernel void @test(ptr addrspace(1) %buffer) {
entry:
%0 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !3
%1 = load i32, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !5
store i32 %0, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !7
store i32 %1, ptr addrspace(1) %buffer, align 4, !spirv.DecorationCacheControlINTEL !9
ret void
}

!spirv.MemoryModel = !{!0}
!spirv.Source = !{!1}
!opencl.spir.version = !{!2}
!opencl.ocl.version = !{!2}

!0 = !{i32 2, i32 2}
!1 = !{i32 3, i32 102000}
!2 = !{i32 1, i32 2}
!3 = !{!4}
!4 = !{i32 6442, i32 0, i32 1, i32 0}
!5 = !{!6}
!6 = !{i32 6442, i32 1, i32 1, i32 0}
!7 = !{!8}
!8 = !{i32 6443, i32 0, i32 1, i32 1}
!9 = !{!10}
!10 = !{i32 6443, i32 1, i32 1, i32 1}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
; RUN: llvm-as %s -o %t.bc
; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls -spirv-text %t.bc -o - | FileCheck %s --check-prefix=CHECK-SPIRV
; RUN: llvm-spirv --spirv-ext=+SPV_INTEL_cache_controls %t.bc -o %t.spv
; RUN: llvm-spirv -r %t.spv --spirv-target-env=SPV-IR -o - | llvm-dis -o - | FileCheck %s --check-prefix=CHECK-LLVM

; CHECK-SPIRV: Decorate [[#GEP1:]] CacheControlLoadINTEL 0 1
; CHECK-SPIRV: Decorate [[#GEP2:]] CacheControlStoreINTEL 0 1
; CHECK-SPIRV: TypeInt [[#Int32Ty:]] 32 0
; CHECK-SPIRV: Constant [[#Int32Ty]] [[#Zero:]] 0
; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP1]] [[#]] [[#Zero]]
; CHECK-SPIRV: PtrAccessChain [[#]] [[#GEP2]] [[#]] [[#Zero]]
; CHECK-SPIRV: FunctionCall [[#]] [[#]] [[#]] [[#GEP1]] [[#GEP2]]

; CHECK-LLVM: %[[#GEP1:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache1:]]
; CHECK-LLVM: %[[#GEP2:]] = getelementptr ptr addrspace(1), ptr addrspace(1) %{{.*}}, i32 0, !spirv.Decorations ![[#Cache2:]]
; CHECK-LLVM: call spir_func void @foo(ptr addrspace(1) %[[#GEP1]], ptr addrspace(1) %[[#GEP2]])
; CHECK-LLVM: ![[#Cache1]] = !{![[#LoadCache:]]}
; CHECK-LLVM: ![[#LoadCache]] = !{i32 6442, i32 0, i32 1}
; CHECK-LLVM: ![[#Cache2]] = !{![[#StoreCache:]]}
; CHECK-LLVM: ![[#StoreCache]] = !{i32 6443, i32 0, i32 1}

target triple = "spir64-unknown-unknown"

define spir_kernel void @test(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2) {
entry:
call void @foo(ptr addrspace(1) %buffer1, ptr addrspace(1) %buffer2), !spirv.DecorationCacheControlINTEL !3
ret void
}

declare void @foo(ptr addrspace(1), ptr addrspace(1))

!spirv.MemoryModel = !{!0}
!spirv.Source = !{!1}
!opencl.spir.version = !{!2}
!opencl.ocl.version = !{!2}

!0 = !{i32 2, i32 2}
!1 = !{i32 3, i32 102000}
!2 = !{i32 1, i32 2}
!3 = !{!4, !5}
!4 = !{i32 6442, i32 0, i32 1, i32 0}
!5 = !{i32 6443, i32 0, i32 1, i32 1}