[FlattenCFG] Fixup Phi nodes during CFG flattening

llvm/lib/Transforms/Utils/FlattenCFG.cpp

@@ -134,10 +134,6 @@ class FlattenCFGOpt {
 ///  its predecessor.  In Case 2, BB (BB3) only has conditional branches
 ///  as its predecessors.
 bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
-  PHINode *PHI = dyn_cast<PHINode>(BB->begin());
-  if (PHI)
-    return false; // For simplicity, avoid cases containing PHI nodes.
-
   BasicBlock *LastCondBlock = nullptr;
   BasicBlock *FirstCondBlock = nullptr;
   BasicBlock *UnCondBlock = nullptr;
@@ -208,8 +204,12 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
 
     if (Idx == -1)
       Idx = CIdx;
-    else if (CIdx != Idx)
-      return false;
+    else if (CIdx != Idx) {
+      // Inverse Branch Condition
+      IRBuilder<>::InsertPointGuard Guard(Builder);
+      Builder.SetInsertPoint(PBI);
+      InvertBranch(PBI, Builder);
+    }
 
     // PS is the successor which is not BB. Check successors to identify
     // the last conditional branch.
@@ -269,11 +269,6 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
   if (!PBI1 || !PBI1->isUnconditional())
     return false;
 
-  // PS2 should not contain PHI node.
-  PHI = dyn_cast<PHINode>(PS2->begin());
-  if (PHI)
-    return false;
-
   // Do the transformation.
   BasicBlock *CB;
   BranchInst *PBI = cast<BranchInst>(FirstCondBlock->getTerminator());
@@ -291,17 +286,45 @@ bool FlattenCFGOpt::FlattenParallelAndOr(BasicBlock *BB, IRBuilder<> &Builder) {
     // Merge conditions.
     Builder.SetInsertPoint(PBI);
     Value *NC;
-    if (Idx == 0)
-      // Case 2, use parallel or.
-      NC = Builder.CreateOr(PC, CC);
-    else
+    if (UnCondBlock)
       // Case 1, use parallel and.
       NC = Builder.CreateAnd(PC, CC);
+    else
+      // Case 2, use parallel or.
+      NC = Builder.CreateOr(PC, CC);
+
+    // Fixup PHI node if needed
+    for (BasicBlock *CBS : successors(PBI)) {
+      for (PHINode &Phi : CBS->phis()) {
+        Value *origPhi0 = nullptr;
+        Value *newPhi = nullptr;
+        if (llvm::is_contained(Phi.blocks(), FirstCondBlock)) {
+          origPhi0 = Phi.removeIncomingValue(FirstCondBlock, false);
+          newPhi = origPhi0;
+        }
+        if (llvm::is_contained(Phi.blocks(), CB)) {
+          Value *origPhi1 = Phi.removeIncomingValue(CB, false);
+          newPhi = origPhi1;
+
+          if (origPhi0) {
+            // Swap branch given the conditions
+            if (PBI->getSuccessor(0) == CBS) {
+              newPhi = Builder.CreateSelect(PC, origPhi0, origPhi1);
+            } else {
+              newPhi = Builder.CreateSelect(PC, origPhi1, origPhi0);
+            }
+          }
+        }
+        if (newPhi)
+          Phi.addIncoming(newPhi, FirstCondBlock);
+      }
+    }
 
     PBI->replaceUsesOfWith(CC, NC);
     PC = NC;
     if (CB == LastCondBlock)
       Iteration = false;
+
     // Remove internal conditional branches.
     CB->dropAllReferences();
     // make CB unreachable and let downstream to delete the block.

llvm/test/CodeGen/AMDGPU/GlobalISel/vni8-across-blocks.ll

@@ -480,29 +480,58 @@ define amdgpu_kernel void @v8i8_phi_chain(ptr addrspace(1) %src1, ptr addrspace(
 ; GFX906-LABEL: v8i8_phi_chain:
 ; GFX906:       ; %bb.0: ; %entry
 ; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
-; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 3, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
-; GFX906-NEXT:    s_xor_b64 s[0:1], vcc, -1
+; GFX906-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
+; GFX906-NEXT:    v_cmp_le_u32_e32 vcc, 15, v0
+; GFX906-NEXT:    v_cmp_gt_u32_e64 s[0:1], 7, v0
+; GFX906-NEXT:    s_or_b64 s[2:3], vcc, s[0:1]
 ; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[8:9]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
+; GFX906-NEXT:    global_load_dwordx2 v[3:4], v5, s[8:9]
+; GFX906-NEXT:    global_load_dwordx2 v[1:2], v5, s[10:11]
+; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], s[2:3]
 ; GFX906-NEXT:    s_cbranch_execz .LBB8_2
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v3, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_andn2_b64 s[0:1], s[0:1], exec
-; GFX906-NEXT:    s_and_b64 s[4:5], exec, vcc
-; GFX906-NEXT:    s_or_b64 s[0:1], s[0:1], s[4:5]
-; GFX906-NEXT:  .LBB8_2: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], s[0:1]
-; GFX906-NEXT:    s_cbranch_execz .LBB8_4
-; GFX906-NEXT:  ; %bb.3: ; %bb.2
-; GFX906-NEXT:    v_mov_b32_e32 v0, 0
+; GFX906-NEXT:  ; %bb.1: ; %bb.2
+; GFX906-NEXT:    s_waitcnt vmcnt(1)
+; GFX906-NEXT:    v_lshrrev_b32_e32 v0, 8, v3
+; GFX906-NEXT:    v_lshrrev_b32_e32 v7, 8, v4
+; GFX906-NEXT:    v_lshrrev_b32_e32 v9, 24, v4
 ; GFX906-NEXT:    s_waitcnt vmcnt(0)
+; GFX906-NEXT:    v_lshrrev_b32_e32 v10, 8, v1
+; GFX906-NEXT:    v_lshrrev_b32_e32 v13, 8, v2
+; GFX906-NEXT:    v_lshrrev_b32_e32 v15, 24, v2
+; GFX906-NEXT:    v_lshrrev_b32_e32 v5, 16, v3
+; GFX906-NEXT:    v_lshrrev_b32_e32 v6, 24, v3
+; GFX906-NEXT:    v_lshrrev_b32_e32 v8, 16, v4
+; GFX906-NEXT:    v_lshrrev_b32_e32 v11, 16, v1
+; GFX906-NEXT:    v_lshrrev_b32_e32 v12, 24, v1
+; GFX906-NEXT:    v_lshrrev_b32_e32 v14, 16, v2
+; GFX906-NEXT:    v_cndmask_b32_e32 v0, v10, v0, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v2, v2, v4, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v4, v13, v7, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v7, v15, v9, vcc
+; GFX906-NEXT:    v_mov_b32_e32 v9, 8
+; GFX906-NEXT:    v_cndmask_b32_e32 v1, v1, v3, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v3, v11, v5, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v5, v12, v6, vcc
+; GFX906-NEXT:    v_cndmask_b32_e32 v6, v14, v8, vcc
+; GFX906-NEXT:    v_mov_b32_e32 v8, 0xff
+; GFX906-NEXT:    v_lshlrev_b32_sdwa v0, v9, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT:    v_and_or_b32 v0, v1, v8, v0
+; GFX906-NEXT:    v_and_b32_e32 v1, 0xff, v3
+; GFX906-NEXT:    v_and_b32_e32 v3, 0xff, v5
+; GFX906-NEXT:    v_lshlrev_b32_e32 v1, 16, v1
+; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX906-NEXT:    v_or3_b32 v1, v0, v1, v3
+; GFX906-NEXT:    v_lshlrev_b32_sdwa v0, v9, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT:    v_and_or_b32 v0, v2, v8, v0
+; GFX906-NEXT:    v_and_b32_e32 v2, 0xff, v6
+; GFX906-NEXT:    v_and_b32_e32 v3, 0xff, v7
+; GFX906-NEXT:    v_lshlrev_b32_e32 v2, 16, v2
+; GFX906-NEXT:    v_lshlrev_b32_e32 v3, 24, v3
+; GFX906-NEXT:    v_or3_b32 v2, v0, v2, v3
+; GFX906-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[12:13]
-; GFX906-NEXT:  .LBB8_4: ; %bb.3
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
+; GFX906-NEXT:  .LBB8_2: ; %bb.3
+; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX906-NEXT:    v_mov_b32_e32 v0, 0
 ; GFX906-NEXT:    s_waitcnt vmcnt(0)
 ; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[14:15]
@@ -535,29 +564,50 @@ define amdgpu_kernel void @v8i8_multi_block(ptr addrspace(1) %src1, ptr addrspac
 ; GFX906:       ; %bb.0: ; %entry
 ; GFX906-NEXT:    s_load_dwordx8 s[8:15], s[4:5], 0x24
 ; GFX906-NEXT:    v_lshlrev_b32_e32 v5, 3, v0
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 15, v0
+; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
+; GFX906-NEXT:    v_cmp_gt_u32_e64 s[0:1], 15, v0
+; GFX906-NEXT:    s_and_b64 s[2:3], s[0:1], vcc
 ; GFX906-NEXT:    s_waitcnt lgkmcnt(0)
-; GFX906-NEXT:    global_load_dwordx2 v[3:4], v5, s[8:9]
+; GFX906-NEXT:    global_load_dwordx2 v[1:2], v5, s[8:9]
+; GFX906-NEXT:    global_load_dwordx2 v[3:4], v5, s[10:11]
+; GFX906-NEXT:    s_mov_b64 vcc, s[0:1]
+; GFX906-NEXT:    v_mov_b32_e32 v6, 8
+; GFX906-NEXT:    v_mov_b32_e32 v5, 0xff
+; GFX906-NEXT:    s_waitcnt vmcnt(1)
+; GFX906-NEXT:    v_lshrrev_b32_e32 v7, 8, v1
 ; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    v_mov_b32_e32 v1, v3
-; GFX906-NEXT:    v_mov_b32_e32 v2, v4
-; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB9_4
-; GFX906-NEXT:  ; %bb.1: ; %bb.1
-; GFX906-NEXT:    global_load_dwordx2 v[1:2], v5, s[10:11]
-; GFX906-NEXT:    v_cmp_gt_u32_e32 vcc, 7, v0
-; GFX906-NEXT:    s_and_saveexec_b64 s[2:3], vcc
-; GFX906-NEXT:    s_cbranch_execz .LBB9_3
-; GFX906-NEXT:  ; %bb.2: ; %bb.2
+; GFX906-NEXT:    v_lshrrev_b32_e32 v9, 8, v3
+; GFX906-NEXT:    v_lshrrev_b32_e32 v8, 8, v2
+; GFX906-NEXT:    v_lshrrev_b32_e32 v10, 8, v4
+; GFX906-NEXT:    v_cndmask_b32_e64 v7, v7, v9, s[0:1]
+; GFX906-NEXT:    v_cndmask_b32_sdwa v9, v1, v3, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX906-NEXT:    v_cndmask_b32_e64 v0, v1, v3, s[0:1]
+; GFX906-NEXT:    v_cndmask_b32_e64 v8, v8, v10, s[0:1]
+; GFX906-NEXT:    v_lshlrev_b32_sdwa v7, v6, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT:    v_and_b32_e32 v9, 0xff, v9
+; GFX906-NEXT:    v_cndmask_b32_e64 v11, v2, v4, s[0:1]
+; GFX906-NEXT:    v_cndmask_b32_sdwa v10, v1, v3, vcc dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX906-NEXT:    v_lshlrev_b32_sdwa v6, v6, v8 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0
+; GFX906-NEXT:    v_and_or_b32 v0, v0, v5, v7
+; GFX906-NEXT:    v_lshlrev_b32_e32 v7, 16, v9
+; GFX906-NEXT:    v_cndmask_b32_sdwa v8, v2, v4, vcc dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
+; GFX906-NEXT:    v_and_or_b32 v6, v11, v5, v6
+; GFX906-NEXT:    v_or3_b32 v5, v0, v7, v10
+; GFX906-NEXT:    v_and_b32_e32 v0, 0xff, v8
+; GFX906-NEXT:    v_lshlrev_b32_e32 v0, 16, v0
+; GFX906-NEXT:    v_cndmask_b32_sdwa v7, v2, v4, vcc dst_sel:BYTE_3 dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:BYTE_3
+; GFX906-NEXT:    v_or3_b32 v6, v6, v0, v7
+; GFX906-NEXT:    s_and_saveexec_b64 s[0:1], s[2:3]
+; GFX906-NEXT:    s_cbranch_execz .LBB9_2
+; GFX906-NEXT:  ; %bb.1: ; %bb.2
+; GFX906-NEXT:    v_mov_b32_e32 v6, v4
 ; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    global_store_dwordx2 v0, v[3:4], s[12:13]
-; GFX906-NEXT:  .LBB9_3: ; %Flow
-; GFX906-NEXT:    s_or_b64 exec, exec, s[2:3]
-; GFX906-NEXT:  .LBB9_4: ; %bb.3
+; GFX906-NEXT:    v_mov_b32_e32 v5, v3
+; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[12:13]
+; GFX906-NEXT:  .LBB9_2: ; %bb.3
 ; GFX906-NEXT:    s_or_b64 exec, exec, s[0:1]
 ; GFX906-NEXT:    v_mov_b32_e32 v0, 0
-; GFX906-NEXT:    s_waitcnt vmcnt(0)
-; GFX906-NEXT:    global_store_dwordx2 v0, v[1:2], s[14:15]
+; GFX906-NEXT:    global_store_dwordx2 v0, v[5:6], s[14:15]
 ; GFX906-NEXT:    s_endpgm
 entry:
   %idx = call i32 @llvm.amdgcn.workitem.id.x()

llvm/test/CodeGen/AMDGPU/blender-no-live-segment-at-def-implicit-def.ll

@@ -12,73 +12,59 @@ define amdgpu_kernel void @blender_no_live_segment_at_def_error(<4 x float> %ext
 ; CHECK-NEXT:    s_load_dwordx8 s[48:55], s[8:9], 0x0
 ; CHECK-NEXT:    s_add_u32 s0, s0, s17
 ; CHECK-NEXT:    s_addc_u32 s1, s1, 0
-; CHECK-NEXT:    s_mov_b32 s12, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
 ; CHECK-NEXT:    s_cmp_lg_u32 s52, 0
-; CHECK-NEXT:    s_cbranch_scc1 .LBB0_9
-; CHECK-NEXT:  ; %bb.1: ; %if.end13.i.i
-; CHECK-NEXT:    s_cmp_eq_u32 s54, 0
 ; CHECK-NEXT:    s_cbranch_scc1 .LBB0_4
-; CHECK-NEXT:  ; %bb.2: ; %if.else251.i.i
-; CHECK-NEXT:    s_cmp_lg_u32 s55, 0
-; CHECK-NEXT:    s_mov_b32 s17, 0
-; CHECK-NEXT:    s_cselect_b32 s12, -1, 0
-; CHECK-NEXT:    s_and_b32 vcc_lo, exec_lo, s12
-; CHECK-NEXT:    s_cbranch_vccz .LBB0_5
-; CHECK-NEXT:  ; %bb.3:
-; CHECK-NEXT:    s_mov_b32 s18, 0
-; CHECK-NEXT:    s_branch .LBB0_6
-; CHECK-NEXT:  .LBB0_4:
-; CHECK-NEXT:    s_mov_b32 s14, s12
-; CHECK-NEXT:    s_mov_b32 s15, s12
-; CHECK-NEXT:    s_mov_b32 s13, s12
-; CHECK-NEXT:    s_mov_b64 s[50:51], s[14:15]
-; CHECK-NEXT:    s_mov_b64 s[48:49], s[12:13]
-; CHECK-NEXT:    s_branch .LBB0_8
-; CHECK-NEXT:  .LBB0_5: ; %if.then263.i.i
+; CHECK-NEXT:  ; %bb.1: ; %if.end13.i.i
 ; CHECK-NEXT:    v_cmp_lt_f32_e64 s12, s53, 0
-; CHECK-NEXT:    s_mov_b32 s18, 1.0
-; CHECK-NEXT:    s_mov_b32 s17, 0x7fc00000
-; CHECK-NEXT:  .LBB0_6: ; %Flow
-; CHECK-NEXT:    s_mov_b32 s48, 1.0
-; CHECK-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s12
+; CHECK-NEXT:    s_cmp_lg_u32 s55, 0
+; CHECK-NEXT:    s_cselect_b32 s17, -1, 0
+; CHECK-NEXT:    s_or_b32 s12, s17, s12
+; CHECK-NEXT:    s_cmp_lg_u32 s54, 0
+; CHECK-NEXT:    s_cselect_b32 s13, -1, 0
+; CHECK-NEXT:    s_and_b32 s18, s13, exec_lo
+; CHECK-NEXT:    s_cselect_b32 s48, 1.0, 0
+; CHECK-NEXT:    s_and_b32 s12, s13, s12
 ; CHECK-NEXT:    s_mov_b32 s49, s48
 ; CHECK-NEXT:    s_mov_b32 s50, s48
+; CHECK-NEXT:    s_andn2_b32 vcc_lo, exec_lo, s12
 ; CHECK-NEXT:    s_mov_b32 s51, s48
-; CHECK-NEXT:    s_cbranch_vccnz .LBB0_8
-; CHECK-NEXT:  ; %bb.7: ; %if.end273.i.i
+; CHECK-NEXT:    s_cbranch_vccnz .LBB0_3
+; CHECK-NEXT:  ; %bb.2: ; %if.end273.i.i
 ; CHECK-NEXT:    s_add_u32 s12, s8, 40
 ; CHECK-NEXT:    s_addc_u32 s13, s9, 0
-; CHECK-NEXT:    s_getpc_b64 s[20:21]
-; CHECK-NEXT:    s_add_u32 s20, s20, _Z3dotDv3_fS_@gotpcrel32@lo+4
-; CHECK-NEXT:    s_addc_u32 s21, s21, _Z3dotDv3_fS_@gotpcrel32@hi+12
+; CHECK-NEXT:    s_getpc_b64 s[18:19]
+; CHECK-NEXT:    s_add_u32 s18, s18, _Z3dotDv3_fS_@gotpcrel32@lo+4
+; CHECK-NEXT:    s_addc_u32 s19, s19, _Z3dotDv3_fS_@gotpcrel32@hi+12
+; CHECK-NEXT:    v_cndmask_b32_e64 v3, 1.0, 0, s17
+; CHECK-NEXT:    s_load_dwordx2 s[18:19], s[18:19], 0x0
+; CHECK-NEXT:    v_cndmask_b32_e64 v4, 0x7fc00000, 0, s17
 ; CHECK-NEXT:    v_lshlrev_b32_e32 v2, 20, v2
-; CHECK-NEXT:    s_load_dwordx2 s[20:21], s[20:21], 0x0
-; CHECK-NEXT:    v_lshlrev_b32_e32 v3, 10, v1
-; CHECK-NEXT:    v_add_f32_e64 v1, s17, s18
+; CHECK-NEXT:    v_lshlrev_b32_e32 v5, 10, v1
 ; CHECK-NEXT:    s_mov_b64 s[34:35], s[8:9]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[12:13]
+; CHECK-NEXT:    v_add_f32_e32 v1, v4, v3
 ; CHECK-NEXT:    s_mov_b32 s12, s14
-; CHECK-NEXT:    v_or3_b32 v31, v0, v3, v2
-; CHECK-NEXT:    v_mov_b32_e32 v0, v1
-; CHECK-NEXT:    v_mov_b32_e32 v1, 0
+; CHECK-NEXT:    v_or3_b32 v31, v0, v5, v2
 ; CHECK-NEXT:    v_mov_b32_e32 v2, 0
 ; CHECK-NEXT:    s_mov_b32 s13, s15
+; CHECK-NEXT:    v_mov_b32_e32 v0, v1
+; CHECK-NEXT:    v_mov_b32_e32 v1, 0
 ; CHECK-NEXT:    s_mov_b32 s14, s16
 ; CHECK-NEXT:    s_mov_b32 s48, 0
 ; CHECK-NEXT:    s_waitcnt lgkmcnt(0)
-; CHECK-NEXT:    s_swappc_b64 s[30:31], s[20:21]
+; CHECK-NEXT:    s_swappc_b64 s[30:31], s[18:19]
 ; CHECK-NEXT:    s_mov_b64 s[8:9], s[34:35]
 ; CHECK-NEXT:    s_mov_b32 s49, s48
 ; CHECK-NEXT:    s_mov_b32 s50, s48
 ; CHECK-NEXT:    s_mov_b32 s51, s48
-; CHECK-NEXT:  .LBB0_8: ; %if.end294.i.i
+; CHECK-NEXT:  .LBB0_3: ; %if.end294.i.i
 ; CHECK-NEXT:    v_mov_b32_e32 v0, 0
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:12
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:8
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0 offset:4
 ; CHECK-NEXT:    buffer_store_dword v0, off, s[0:3], 0
-; CHECK-NEXT:  .LBB0_9: ; %kernel_direct_lighting.exit
+; CHECK-NEXT:  .LBB0_4: ; %kernel_direct_lighting.exit
 ; CHECK-NEXT:    s_load_dwordx2 s[4:5], s[8:9], 0x20
 ; CHECK-NEXT:    v_mov_b32_e32 v0, s48
 ; CHECK-NEXT:    v_mov_b32_e32 v4, 0