nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

bpf, x86_64: Use bpf_jit_binary_pack_alloc

Browse source 
Use bpf_jit_binary_pack_alloc in x86_64 jit. The jit engine first writes
the program to the rw buffer. When the jit is done, the program is copied
to the final location with bpf_jit_binary_pack_finalize.

Note that we need to do bpf_tail_call_direct_fixup after finalize.
Therefore, the text_live = false logic in __bpf_arch_text_poke is no
longer needed.

Signed-off-by: Song Liu <songliubraving@fb.com>
Signed-off-by: Alexei Starovoitov <ast@kernel.org>
Link: https://lore.kernel.org/bpf/20220204185742.271030-10-song@kernel.org
This commit is contained in:
Song Liu 2022-02-04 10:57:42 -08:00 committed by Alexei Starovoitov
parent 33c9805860
commit 1022a5498f
1 changed files with 31 additions and 27 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

58
arch/x86/net/bpf_jit_comp.c
View file

@ -330,8 +330,7 @@ static int emit_jump(u8 **pprog, void *func, void *ip)
} }
static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t, static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
void *old_addr, void *new_addr, void *old_addr, void *new_addr)
const bool text_live)
{ {
const u8 *nop_insn = x86_nops[5]; const u8 *nop_insn = x86_nops[5];
u8 old_insn[X86_PATCH_SIZE]; u8 old_insn[X86_PATCH_SIZE];
@ -365,10 +364,7 @@ static int __bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
goto out; goto out;
ret = 1; ret = 1;
if (memcmp(ip, new_insn, X86_PATCH_SIZE)) { if (memcmp(ip, new_insn, X86_PATCH_SIZE)) {
if (text_live) text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL);
text_poke_bp(ip, new_insn, X86_PATCH_SIZE, NULL);
else
memcpy(ip, new_insn, X86_PATCH_SIZE);
ret = 0; ret = 0;
} }
out: out:
@ -384,7 +380,7 @@ int bpf_arch_text_poke(void *ip, enum bpf_text_poke_type t,
/* BPF poking in modules is not supported */ /* BPF poking in modules is not supported */
return -EINVAL; return -EINVAL;
return __bpf_arch_text_poke(ip, t, old_addr, new_addr, true); return __bpf_arch_text_poke(ip, t, old_addr, new_addr);
} }
#define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8) #define EMIT_LFENCE() EMIT3(0x0F, 0xAE, 0xE8)
@ -558,24 +554,15 @@ static void bpf_tail_call_direct_fixup(struct bpf_prog *prog)
mutex_lock(&array->aux->poke_mutex); mutex_lock(&array->aux->poke_mutex);
target = array->ptrs[poke->tail_call.key]; target = array->ptrs[poke->tail_call.key];
if (target) { if (target) {
/* Plain memcpy is used when image is not live yet
* and still not locked as read-only. Once poke
* location is active (poke->tailcall_target_stable),
* any parallel bpf_arch_text_poke() might occur
* still on the read-write image until we finally
* locked it as read-only. Both modifications on
* the given image are under text_mutex to avoid
* interference.
*/
ret = __bpf_arch_text_poke(poke->tailcall_target, ret = __bpf_arch_text_poke(poke->tailcall_target,
BPF_MOD_JUMP, NULL, BPF_MOD_JUMP, NULL,
(u8 *)target->bpf_func + (u8 *)target->bpf_func +
poke->adj_off, false); poke->adj_off);
BUG_ON(ret < 0); BUG_ON(ret < 0);
ret = __bpf_arch_text_poke(poke->tailcall_bypass, ret = __bpf_arch_text_poke(poke->tailcall_bypass,
BPF_MOD_JUMP, BPF_MOD_JUMP,
(u8 *)poke->tailcall_target + (u8 *)poke->tailcall_target +
X86_PATCH_SIZE, NULL, false); X86_PATCH_SIZE, NULL);
BUG_ON(ret < 0); BUG_ON(ret < 0);
} }
WRITE_ONCE(poke->tailcall_target_stable, true); WRITE_ONCE(poke->tailcall_target_stable, true);
@ -866,7 +853,7 @@ static void emit_nops(u8 **pprog, int len)
#define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp))) #define INSN_SZ_DIFF (((addrs[i] - addrs[i - 1]) - (prog - temp)))
static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image, u8 *rw_image,
int oldproglen, struct jit_context *ctx, bool jmp_padding) int oldproglen, struct jit_context *ctx, bool jmp_padding)
{ {
bool tail_call_reachable = bpf_prog->aux->tail_call_reachable; bool tail_call_reachable = bpf_prog->aux->tail_call_reachable;
@ -893,8 +880,8 @@ static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
push_callee_regs(&prog, callee_regs_used); push_callee_regs(&prog, callee_regs_used);
ilen = prog - temp; ilen = prog - temp;
if (image) if (rw_image)
memcpy(image + proglen, temp, ilen); memcpy(rw_image + proglen, temp, ilen);
proglen += ilen; proglen += ilen;
addrs[0] = proglen; addrs[0] = proglen;
prog = temp; prog = temp;
@ -1323,6 +1310,9 @@ st: if (is_imm8(insn->off))
pr_err("extable->insn doesn't fit into 32-bit\n"); pr_err("extable->insn doesn't fit into 32-bit\n");
return -EFAULT; return -EFAULT;
} }
/* switch ex to rw buffer for writes */
ex = (void *)rw_image + ((void *)ex - (void *)image);
ex->insn = delta; ex->insn = delta;
ex->data = EX_TYPE_BPF; ex->data = EX_TYPE_BPF;
@ -1705,7 +1695,7 @@ st: if (is_imm8(insn->off))
pr_err("bpf_jit: fatal error\n"); pr_err("bpf_jit: fatal error\n");
return -EFAULT; return -EFAULT;
} }
memcpy(image + proglen, temp, ilen); memcpy(rw_image + proglen, temp, ilen);
} }
proglen += ilen; proglen += ilen;
addrs[i] = proglen; addrs[i] = proglen;
@ -2246,6 +2236,7 @@ int arch_prepare_bpf_dispatcher(void *image, s64 *funcs, int num_funcs)
} }
struct x64_jit_data { struct x64_jit_data {
struct bpf_binary_header *rw_header;
struct bpf_binary_header *header; struct bpf_binary_header *header;
int *addrs; int *addrs;
u8 *image; u8 *image;
@ -2258,6 +2249,7 @@ struct x64_jit_data {
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
{ {
struct bpf_binary_header *rw_header = NULL;
struct bpf_binary_header *header = NULL; struct bpf_binary_header *header = NULL;
struct bpf_prog *tmp, *orig_prog = prog; struct bpf_prog *tmp, *orig_prog = prog;
struct x64_jit_data *jit_data; struct x64_jit_data *jit_data;
@ -2266,6 +2258,7 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
bool tmp_blinded = false; bool tmp_blinded = false;
bool extra_pass = false; bool extra_pass = false;
bool padding = false; bool padding = false;
u8 *rw_image = NULL;
u8 *image = NULL; u8 *image = NULL;
int *addrs; int *addrs;
int pass; int pass;
@ -2301,6 +2294,8 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
oldproglen = jit_data->proglen; oldproglen = jit_data->proglen;
image = jit_data->image; image = jit_data->image;
header = jit_data->header; header = jit_data->header;
rw_header = jit_data->rw_header;
rw_image = (void *)rw_header + ((void *)image - (void *)header);
extra_pass = true; extra_pass = true;
padding = true; padding = true;
goto skip_init_addrs; goto skip_init_addrs;
@ -2331,12 +2326,12 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
for (pass = 0; pass < MAX_PASSES || image; pass++) { for (pass = 0; pass < MAX_PASSES || image; pass++) {
if (!padding && pass >= PADDING_PASSES) if (!padding && pass >= PADDING_PASSES)
padding = true; padding = true;
proglen = do_jit(prog, addrs, image, oldproglen, &ctx, padding); proglen = do_jit(prog, addrs, image, rw_image, oldproglen, &ctx, padding);
if (proglen <= 0) { if (proglen <= 0) {
out_image: out_image:
image = NULL; image = NULL;
if (header) if (header)
bpf_jit_binary_free(header); bpf_jit_binary_pack_free(header, rw_header);
prog = orig_prog; prog = orig_prog;
goto out_addrs; goto out_addrs;
} }
@ -2360,8 +2355,9 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
sizeof(struct exception_table_entry); sizeof(struct exception_table_entry);
/* allocate module memory for x86 insns and extable */ /* allocate module memory for x86 insns and extable */
header = bpf_jit_binary_alloc(roundup(proglen, align) + extable_size, header = bpf_jit_binary_pack_alloc(roundup(proglen, align) + extable_size,
&image, align, jit_fill_hole); &image, align, &rw_header, &rw_image,
jit_fill_hole);
if (!header) { if (!header) {
prog = orig_prog; prog = orig_prog;
goto out_addrs; goto out_addrs;
@ -2377,14 +2373,22 @@ struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog)
if (image) { if (image) {
if (!prog->is_func || extra_pass) { if (!prog->is_func || extra_pass) {
/*
* bpf_jit_binary_pack_finalize fails in two scenarios:
* 1) header is not pointing to proper module memory;
* 2) the arch doesn't support bpf_arch_text_copy().
*
* Both cases are serious bugs that we should not continue.
*/
BUG_ON(bpf_jit_binary_pack_finalize(prog, header, rw_header));
bpf_tail_call_direct_fixup(prog); bpf_tail_call_direct_fixup(prog);
bpf_jit_binary_lock_ro(header);
} else { } else {
jit_data->addrs = addrs; jit_data->addrs = addrs;
jit_data->ctx = ctx; jit_data->ctx = ctx;
jit_data->proglen = proglen; jit_data->proglen = proglen;
jit_data->image = image; jit_data->image = image;
jit_data->header = header; jit_data->header = header;
jit_data->rw_header = rw_header;
} }
prog->bpf_func = (void *)image; prog->bpf_func = (void *)image;
prog->jited = 1; prog->jited = 1;