Skip to content

Commit d9913e0

Browse files
Gbell26Gbell26
committed
bpf, x86: Support load-acquire and store-release instructions
JIRA: https://issues.redhat.com/browse/RHEL-78202 commit 5341c9a Author: Peilin Ye <[email protected]> Date: Tue Mar 4 01:06:40 2025 +0000 bpf, x86: Support load-acquire and store-release instructions Recently we introduced BPF load-acquire (BPF_LOAD_ACQ) and store-release (BPF_STORE_REL) instructions. For x86-64, simply implement them as regular BPF_LDX/BPF_STX loads and stores. The verifier always rejects misaligned load-acquires/store-releases (even if BPF_F_ANY_ALIGNMENT is set), so emitted MOV* instructions are guaranteed to be atomic. Arena accesses are supported. 8- and 16-bit load-acquires are zero-extending (i.e., MOVZBQ, MOVZWQ). Rename emit_atomic{,_index}() to emit_atomic_rmw{,_index}() to make it clear that they only handle read-modify-write atomics, and extend their @atomic_op parameter from u8 to u32, since we are starting to use more than the lowest 8 bits of the 'imm' field. Signed-off-by: Peilin Ye <[email protected]> Link: https://lore.kernel.org/r/d22bb3c69f126af1d962b7314f3489eff606a3b7.1741049567.git.yepeilin@google.com Signed-off-by: Alexei Starovoitov <[email protected]> Signed-off-by: Gregory Bell <[email protected]>
1 parent 4bd4119 commit d9913e0

File tree

1 file changed

+82
-17
lines changed

1 file changed

+82
-17
lines changed

‎arch/x86/net/bpf_jit_comp.c‎

Lines changed: 82 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1247,8 +1247,8 @@ static void emit_st_r12(u8 **pprog, u32 size, u32 dst_reg, int off, int imm)
12471247
emit_st_index(pprog, size, dst_reg, X86_REG_R12, off, imm);
12481248
}
12491249

1250-
static int emit_atomic(u8 **pprog, u8 atomic_op,
1251-
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
1250+
static int emit_atomic_rmw(u8 **pprog, u32 atomic_op,
1251+
u32 dst_reg, u32 src_reg, s16 off, u8 bpf_size)
12521252
{
12531253
u8 *prog = *pprog;
12541254

@@ -1288,8 +1288,9 @@ static int emit_atomic(u8 **pprog, u8 atomic_op,
12881288
return 0;
12891289
}
12901290

1291-
static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
1292-
u32 dst_reg, u32 src_reg, u32 index_reg, int off)
1291+
static int emit_atomic_rmw_index(u8 **pprog, u32 atomic_op, u32 size,
1292+
u32 dst_reg, u32 src_reg, u32 index_reg,
1293+
int off)
12931294
{
12941295
u8 *prog = *pprog;
12951296

@@ -1302,7 +1303,7 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
13021303
EMIT1(add_3mod(0x48, dst_reg, src_reg, index_reg));
13031304
break;
13041305
default:
1305-
pr_err("bpf_jit: 1 and 2 byte atomics are not supported\n");
1306+
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
13061307
return -EFAULT;
13071308
}
13081309

@@ -1336,6 +1337,49 @@ static int emit_atomic_index(u8 **pprog, u8 atomic_op, u32 size,
13361337
return 0;
13371338
}
13381339

1340+
static int emit_atomic_ld_st(u8 **pprog, u32 atomic_op, u32 dst_reg,
1341+
u32 src_reg, s16 off, u8 bpf_size)
1342+
{
1343+
switch (atomic_op) {
1344+
case BPF_LOAD_ACQ:
1345+
/* dst_reg = smp_load_acquire(src_reg + off16) */
1346+
emit_ldx(pprog, bpf_size, dst_reg, src_reg, off);
1347+
break;
1348+
case BPF_STORE_REL:
1349+
/* smp_store_release(dst_reg + off16, src_reg) */
1350+
emit_stx(pprog, bpf_size, dst_reg, src_reg, off);
1351+
break;
1352+
default:
1353+
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
1354+
atomic_op);
1355+
return -EFAULT;
1356+
}
1357+
1358+
return 0;
1359+
}
1360+
1361+
static int emit_atomic_ld_st_index(u8 **pprog, u32 atomic_op, u32 size,
1362+
u32 dst_reg, u32 src_reg, u32 index_reg,
1363+
int off)
1364+
{
1365+
switch (atomic_op) {
1366+
case BPF_LOAD_ACQ:
1367+
/* dst_reg = smp_load_acquire(src_reg + idx_reg + off16) */
1368+
emit_ldx_index(pprog, size, dst_reg, src_reg, index_reg, off);
1369+
break;
1370+
case BPF_STORE_REL:
1371+
/* smp_store_release(dst_reg + idx_reg + off16, src_reg) */
1372+
emit_stx_index(pprog, size, dst_reg, src_reg, index_reg, off);
1373+
break;
1374+
default:
1375+
pr_err("bpf_jit: unknown atomic load/store opcode %02x\n",
1376+
atomic_op);
1377+
return -EFAULT;
1378+
}
1379+
1380+
return 0;
1381+
}
1382+
13391383
#define DONT_CLEAR 1
13401384

13411385
bool ex_handler_bpf(const struct exception_table_entry *x, struct pt_regs *regs)
@@ -2160,6 +2204,13 @@ st: if (is_imm8(insn->off))
21602204
}
21612205
break;
21622206

2207+
case BPF_STX | BPF_ATOMIC | BPF_B:
2208+
case BPF_STX | BPF_ATOMIC | BPF_H:
2209+
if (!bpf_atomic_is_load_store(insn)) {
2210+
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
2211+
return -EFAULT;
2212+
}
2213+
fallthrough;
21632214
case BPF_STX | BPF_ATOMIC | BPF_W:
21642215
case BPF_STX | BPF_ATOMIC | BPF_DW:
21652216
if (insn->imm == (BPF_AND | BPF_FETCH) ||
@@ -2195,10 +2246,10 @@ st: if (is_imm8(insn->off))
21952246
EMIT2(simple_alu_opcodes[BPF_OP(insn->imm)],
21962247
add_2reg(0xC0, AUX_REG, real_src_reg));
21972248
/* Attempt to swap in new value */
2198-
err = emit_atomic(&prog, BPF_CMPXCHG,
2199-
real_dst_reg, AUX_REG,
2200-
insn->off,
2201-
BPF_SIZE(insn->code));
2249+
err = emit_atomic_rmw(&prog, BPF_CMPXCHG,
2250+
real_dst_reg, AUX_REG,
2251+
insn->off,
2252+
BPF_SIZE(insn->code));
22022253
if (WARN_ON(err))
22032254
return err;
22042255
/*
@@ -2213,17 +2264,35 @@ st: if (is_imm8(insn->off))
22132264
break;
22142265
}
22152266

2216-
err = emit_atomic(&prog, insn->imm, dst_reg, src_reg,
2217-
insn->off, BPF_SIZE(insn->code));
2267+
if (bpf_atomic_is_load_store(insn))
2268+
err = emit_atomic_ld_st(&prog, insn->imm, dst_reg, src_reg,
2269+
insn->off, BPF_SIZE(insn->code));
2270+
else
2271+
err = emit_atomic_rmw(&prog, insn->imm, dst_reg, src_reg,
2272+
insn->off, BPF_SIZE(insn->code));
22182273
if (err)
22192274
return err;
22202275
break;
22212276

2277+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_B:
2278+
case BPF_STX | BPF_PROBE_ATOMIC | BPF_H:
2279+
if (!bpf_atomic_is_load_store(insn)) {
2280+
pr_err("bpf_jit: 1- and 2-byte RMW atomics are not supported\n");
2281+
return -EFAULT;
2282+
}
2283+
fallthrough;
22222284
case BPF_STX | BPF_PROBE_ATOMIC | BPF_W:
22232285
case BPF_STX | BPF_PROBE_ATOMIC | BPF_DW:
22242286
start_of_ldx = prog;
2225-
err = emit_atomic_index(&prog, insn->imm, BPF_SIZE(insn->code),
2226-
dst_reg, src_reg, X86_REG_R12, insn->off);
2287+
2288+
if (bpf_atomic_is_load_store(insn))
2289+
err = emit_atomic_ld_st_index(&prog, insn->imm,
2290+
BPF_SIZE(insn->code), dst_reg,
2291+
src_reg, X86_REG_R12, insn->off);
2292+
else
2293+
err = emit_atomic_rmw_index(&prog, insn->imm, BPF_SIZE(insn->code),
2294+
dst_reg, src_reg, X86_REG_R12,
2295+
insn->off);
22272296
if (err)
22282297
return err;
22292298
goto populate_extable;
@@ -3825,12 +3894,8 @@ bool bpf_jit_supports_insn(struct bpf_insn *insn, bool in_arena)
38253894
if (!in_arena)
38263895
return true;
38273896
switch (insn->code) {
3828-
case BPF_STX | BPF_ATOMIC | BPF_B:
3829-
case BPF_STX | BPF_ATOMIC | BPF_H:
38303897
case BPF_STX | BPF_ATOMIC | BPF_W:
38313898
case BPF_STX | BPF_ATOMIC | BPF_DW:
3832-
if (bpf_atomic_is_load_store(insn))
3833-
return false;
38343899
if (insn->imm == (BPF_AND | BPF_FETCH) ||
38353900
insn->imm == (BPF_OR | BPF_FETCH) ||
38363901
insn->imm == (BPF_XOR | BPF_FETCH))

0 commit comments

Comments
 (0)