nagai/kernel
3
0
Fork 0
forked from mirrors/linux
Code Pull requests Activity

MIPS: eBPF: Initial eBPF support for MIPS32 architecture.

Browse source 
Currently MIPS32 supports a JIT for classic BPF only, not extended BPF.
This patch adds JIT support for extended BPF on MIPS32, so code is
actually JIT'ed instead of being only interpreted. Instructions with
64-bit operands are not supported at this point.
We can delete classic BPF because the kernel will translate classic BPF
programs into extended BPF and JIT them, eliminating the need for
classic BPF.

Signed-off-by: Hassan Naveed <hnaveed@wavecomp.com>
Signed-off-by: Paul Burton <paul.burton@mips.com>
Cc: kafai@fb.com
Cc: songliubraving@fb.com
Cc: yhs@fb.com
Cc: netdev@vger.kernel.org
Cc: bpf@vger.kernel.org
Cc: linux-mips@vger.kernel.org
Cc: Ralf Baechle <ralf@linux-mips.org>
Cc: James Hogan <jhogan@kernel.org>
Cc: Alexei Starovoitov <ast@kernel.org>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Cc: open list:MIPS <linux-mips@linux-mips.org>
Cc: open list <linux-kernel@vger.kernel.org>
This commit is contained in:
Hassan Naveed 2019-03-12 22:48:12 +00:00 committed by Paul Burton
parent 6c2c8a1888
commit 716850ab10
No known key found for this signature in database
GPG key ID: 3EA79FACB57500DD
5 changed files with 70 additions and 1602 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

3
arch/mips/Kconfig
View file

@ -44,8 +44,7 @@ config MIPS
select HAVE_ARCH_SECCOMP_FILTER
select HAVE_ARCH_TRACEHOOK
select HAVE_ARCH_TRANSPARENT_HUGEPAGE if CPU_SUPPORTS_HUGEPAGES && 64BIT
select HAVE_CBPF_JIT if (!64BIT && !CPU_MICROMIPS)
select HAVE_EBPF_JIT if (64BIT && !CPU_MICROMIPS)
select HAVE_EBPF_JIT if (!CPU_MICROMIPS)
select HAVE_CONTEXT_TRACKING
select HAVE_COPY_THREAD_TLS
select HAVE_C_RECORDMCOUNT

1
arch/mips/net/Makefile
View file

@ -1,4 +1,3 @@
# MIPS networking code
obj-$(CONFIG_MIPS_CBPF_JIT) += bpf_jit.o bpf_jit_asm.o
obj-$(CONFIG_MIPS_EBPF_JIT) += ebpf_jit.o

1270
arch/mips/net/bpf_jit.c
View file

File diff suppressed because it is too large Load diff

285
arch/mips/net/bpf_jit_asm.S
View file

@ -1,285 +0,0 @@
/*
* bpf_jib_asm.S: Packet/header access helper functions for MIPS/MIPS64 BPF
* compiler.
*
* Copyright (C) 2015 Imagination Technologies Ltd.
* Author: Markos Chandras <markos.chandras@imgtec.com>
*
* This program is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License as published by the
* Free Software Foundation; version 2 of the License.
*/
#include <asm/asm.h>
#include <asm/isa-rev.h>
#include <asm/regdef.h>
#include "bpf_jit.h"
/* ABI
*
* r_skb_hl skb header length
* r_skb_data skb data
* r_off(a1) offset register
* r_A BPF register A
* r_X PF register X
* r_skb(a0) *skb
* r_M *scratch memory
* r_skb_le skb length
* r_s0 Scratch register 0
* r_s1 Scratch register 1
*
* On entry:
* a0: *skb
* a1: offset (imm or imm + X)
*
* All non-BPF-ABI registers are free for use. On return, we only
* care about r_ret. The BPF-ABI registers are assumed to remain
* unmodified during the entire filter operation.
*/
#define skb a0
#define offset a1
#define SKF_LL_OFF (-0x200000) /* Can't include linux/filter.h in assembly */
/* We know better :) so prevent assembler reordering etc */
.set noreorder
#define is_offset_negative(TYPE) \
/* If offset is negative we have more work to do */ \
slti t0, offset, 0; \
bgtz t0, bpf_slow_path_##TYPE##_neg; \
/* Be careful what follows in DS. */
#define is_offset_in_header(SIZE, TYPE) \
/* Reading from header? */ \
addiu $r_s0, $r_skb_hl, -SIZE; \
slt t0, $r_s0, offset; \
bgtz t0, bpf_slow_path_##TYPE; \
LEAF(sk_load_word)
is_offset_negative(word)
FEXPORT(sk_load_word_positive)
is_offset_in_header(4, word)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
.set reorder
lw $r_A, 0(t1)
.set noreorder
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh t0, $r_A
rotr $r_A, t0, 16
# else
sll t0, $r_A, 24
srl t1, $r_A, 24
srl t2, $r_A, 8
or t0, t0, t1
andi t2, t2, 0xff00
andi t1, $r_A, 0xff00
or t0, t0, t2
sll t1, t1, 8
or $r_A, t0, t1
# endif
#endif
jr $r_ra
move $r_ret, zero
END(sk_load_word)
LEAF(sk_load_half)
is_offset_negative(half)
FEXPORT(sk_load_half_positive)
is_offset_in_header(2, half)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
lhu $r_A, 0(t1)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh $r_A, $r_A
# else
sll t0, $r_A, 8
srl t1, $r_A, 8
andi t0, t0, 0xff00
or $r_A, t0, t1
# endif
#endif
jr $r_ra
move $r_ret, zero
END(sk_load_half)
LEAF(sk_load_byte)
is_offset_negative(byte)
FEXPORT(sk_load_byte_positive)
is_offset_in_header(1, byte)
/* Offset within header boundaries */
PTR_ADDU t1, $r_skb_data, offset
lbu $r_A, 0(t1)
jr $r_ra
move $r_ret, zero
END(sk_load_byte)
/*
* call skb_copy_bits:
* (prototype in linux/skbuff.h)
*
* int skb_copy_bits(sk_buff *skb, int offset, void *to, int len)
*
* o32 mandates we leave 4 spaces for argument registers in case
* the callee needs to use them. Even though we don't care about
* the argument registers ourselves, we need to allocate that space
* to remain ABI compliant since the callee may want to use that space.
* We also allocate 2 more spaces for $r_ra and our return register (*to).
*
* n64 is a bit different. The *caller* will allocate the space to preserve
* the arguments. So in 64-bit kernels, we allocate the 4-arg space for no
* good reason but it does not matter that much really.
*
* (void *to) is returned in r_s0
*
*/
#ifdef CONFIG_CPU_LITTLE_ENDIAN
#define DS_OFFSET(SIZE) (4 * SZREG)
#else
#define DS_OFFSET(SIZE) ((4 * SZREG) + (4 - SIZE))
#endif
#define bpf_slow_path_common(SIZE) \
/* Quick check. Are we within reasonable boundaries? */ \
LONG_ADDIU $r_s1, $r_skb_len, -SIZE; \
sltu $r_s0, offset, $r_s1; \
beqz $r_s0, fault; \
/* Load 4th argument in DS */ \
LONG_ADDIU a3, zero, SIZE; \
PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
PTR_LA t0, skb_copy_bits; \
PTR_S $r_ra, (5 * SZREG)($r_sp); \
/* Assign low slot to a2 */ \
PTR_ADDIU a2, $r_sp, DS_OFFSET(SIZE); \
jalr t0; \
/* Reset our destination slot (DS but it's ok) */ \
INT_S zero, (4 * SZREG)($r_sp); \
/* \
* skb_copy_bits returns 0 on success and -EFAULT \
* on error. Our data live in a2. Do not bother with \
* our data if an error has been returned. \
*/ \
/* Restore our frame */ \
PTR_L $r_ra, (5 * SZREG)($r_sp); \
INT_L $r_s0, (4 * SZREG)($r_sp); \
bltz v0, fault; \
PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
move $r_ret, zero; \
NESTED(bpf_slow_path_word, (6 * SZREG), $r_sp)
bpf_slow_path_common(4)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
wsbh t0, $r_s0
jr $r_ra
rotr $r_A, t0, 16
# else
sll t0, $r_s0, 24
srl t1, $r_s0, 24
srl t2, $r_s0, 8
or t0, t0, t1
andi t2, t2, 0xff00
andi t1, $r_s0, 0xff00
or t0, t0, t2
sll t1, t1, 8
jr $r_ra
or $r_A, t0, t1
# endif
#else
jr $r_ra
move $r_A, $r_s0
#endif
END(bpf_slow_path_word)
NESTED(bpf_slow_path_half, (6 * SZREG), $r_sp)
bpf_slow_path_common(2)
#ifdef CONFIG_CPU_LITTLE_ENDIAN
# if MIPS_ISA_REV >= 2
jr $r_ra
wsbh $r_A, $r_s0
# else
sll t0, $r_s0, 8
andi t1, $r_s0, 0xff00
andi t0, t0, 0xff00
srl t1, t1, 8
jr $r_ra
or $r_A, t0, t1
# endif
#else
jr $r_ra
move $r_A, $r_s0
#endif
END(bpf_slow_path_half)
NESTED(bpf_slow_path_byte, (6 * SZREG), $r_sp)
bpf_slow_path_common(1)
jr $r_ra
move $r_A, $r_s0
END(bpf_slow_path_byte)
/*
* Negative entry points
*/
.macro bpf_is_end_of_data
li t0, SKF_LL_OFF
/* Reading link layer data? */
slt t1, offset, t0
bgtz t1, fault
/* Be careful what follows in DS. */
.endm
/*
* call skb_copy_bits:
* (prototype in linux/filter.h)
*
* void *bpf_internal_load_pointer_neg_helper(const struct sk_buff *skb,
* int k, unsigned int size)
*
* see above (bpf_slow_path_common) for ABI restrictions
*/
#define bpf_negative_common(SIZE) \
PTR_ADDIU $r_sp, $r_sp, -(6 * SZREG); \
PTR_LA t0, bpf_internal_load_pointer_neg_helper; \
PTR_S $r_ra, (5 * SZREG)($r_sp); \
jalr t0; \
li a2, SIZE; \
PTR_L $r_ra, (5 * SZREG)($r_sp); \
/* Check return pointer */ \
beqz v0, fault; \
PTR_ADDIU $r_sp, $r_sp, 6 * SZREG; \
/* Preserve our pointer */ \
move $r_s0, v0; \
/* Set return value */ \
move $r_ret, zero; \
bpf_slow_path_word_neg:
bpf_is_end_of_data
NESTED(sk_load_word_negative, (6 * SZREG), $r_sp)
bpf_negative_common(4)
jr $r_ra
lw $r_A, 0($r_s0)
END(sk_load_word_negative)
bpf_slow_path_half_neg:
bpf_is_end_of_data
NESTED(sk_load_half_negative, (6 * SZREG), $r_sp)
bpf_negative_common(2)
jr $r_ra
lhu $r_A, 0($r_s0)
END(sk_load_half_negative)
bpf_slow_path_byte_neg:
bpf_is_end_of_data
NESTED(sk_load_byte_negative, (6 * SZREG), $r_sp)
bpf_negative_common(1)
jr $r_ra
lbu $r_A, 0($r_s0)
END(sk_load_byte_negative)
fault:
jr $r_ra
addiu $r_ret, zero, 1

113
arch/mips/net/ebpf_jit.c
View file

@ -126,15 +126,21 @@ static enum reg_val_type get_reg_val_type(const struct jit_ctx *ctx,
}
/* Simply emit the instruction if the JIT memory space has been allocated */
#define emit_instr(ctx, func, ...) \
do { \
if ((ctx)->target != NULL) { \
u32 *p = &(ctx)->target[ctx->idx]; \
uasm_i_##func(&p, ##__VA_ARGS__); \
} \
(ctx)->idx++; \
#define emit_instr_long(ctx, func64, func32, ...) \
do { \
if ((ctx)->target != NULL) { \
u32 *p = &(ctx)->target[ctx->idx]; \
if (IS_ENABLED(CONFIG_64BIT)) \
uasm_i_##func64(&p, ##__VA_ARGS__); \
else \
uasm_i_##func32(&p, ##__VA_ARGS__); \
} \
(ctx)->idx++; \
} while (0)
#define emit_instr(ctx, func, ...) \
emit_instr_long(ctx, func, func, ##__VA_ARGS__)
static unsigned int j_target(struct jit_ctx *ctx, int target_idx)
{
unsigned long target_va, base_va;
@ -274,17 +280,17 @@ static int gen_int_prologue(struct jit_ctx *ctx)
* If RA we are doing a function call and may need
* extra 8-byte tmp area.
*/
stack_adjust += 16;
stack_adjust += 2 * sizeof(long);
if (ctx->flags & EBPF_SAVE_S0)
stack_adjust += 8;
stack_adjust += sizeof(long);
if (ctx->flags & EBPF_SAVE_S1)
stack_adjust += 8;
stack_adjust += sizeof(long);
if (ctx->flags & EBPF_SAVE_S2)
stack_adjust += 8;
stack_adjust += sizeof(long);
if (ctx->flags & EBPF_SAVE_S3)
stack_adjust += 8;
stack_adjust += sizeof(long);
if (ctx->flags & EBPF_SAVE_S4)
stack_adjust += 8;
stack_adjust += sizeof(long);
BUILD_BUG_ON(MAX_BPF_STACK & 7);
locals_size = (ctx->flags & EBPF_SEEN_FP) ? MAX_BPF_STACK : 0;
@ -298,41 +304,49 @@ static int gen_int_prologue(struct jit_ctx *ctx)
* On tail call we skip this instruction, and the TCC is
* passed in $v1 from the caller.
*/
emit_instr(ctx, daddiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
emit_instr(ctx, addiu, MIPS_R_V1, MIPS_R_ZERO, MAX_TAIL_CALL_CNT);
if (stack_adjust)
emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, -stack_adjust);
emit_instr_long(ctx, daddiu, addiu,
MIPS_R_SP, MIPS_R_SP, -stack_adjust);
else
return 0;
store_offset = stack_adjust - 8;
store_offset = stack_adjust - sizeof(long);
if (ctx->flags & EBPF_SAVE_RA) {
emit_instr(ctx, sd, MIPS_R_RA, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_RA, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S0) {
emit_instr(ctx, sd, MIPS_R_S0, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_S0, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S1) {
emit_instr(ctx, sd, MIPS_R_S1, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_S1, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S2) {
emit_instr(ctx, sd, MIPS_R_S2, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_S2, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S3) {
emit_instr(ctx, sd, MIPS_R_S3, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_S3, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S4) {
emit_instr(ctx, sd, MIPS_R_S4, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, sd, sw,
MIPS_R_S4, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if ((ctx->flags & EBPF_SEEN_TC) && !(ctx->flags & EBPF_TCC_IN_V1))
emit_instr(ctx, daddu, MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
emit_instr_long(ctx, daddu, addu,
MIPS_R_S4, MIPS_R_V1, MIPS_R_ZERO);
return 0;
}
@ -341,7 +355,7 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
{
const struct bpf_prog *prog = ctx->skf;
int stack_adjust = ctx->stack_size;
int store_offset = stack_adjust - 8;
int store_offset = stack_adjust - sizeof(long);
enum reg_val_type td;
int r0 = MIPS_R_V0;
@ -353,33 +367,40 @@ static int build_int_epilogue(struct jit_ctx *ctx, int dest_reg)
}
if (ctx->flags & EBPF_SAVE_RA) {
emit_instr(ctx, ld, MIPS_R_RA, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_RA, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S0) {
emit_instr(ctx, ld, MIPS_R_S0, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_S0, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S1) {
emit_instr(ctx, ld, MIPS_R_S1, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_S1, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S2) {
emit_instr(ctx, ld, MIPS_R_S2, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_S2, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S3) {
emit_instr(ctx, ld, MIPS_R_S3, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_S3, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
if (ctx->flags & EBPF_SAVE_S4) {
emit_instr(ctx, ld, MIPS_R_S4, store_offset, MIPS_R_SP);
store_offset -= 8;
emit_instr_long(ctx, ld, lw,
MIPS_R_S4, store_offset, MIPS_R_SP);
store_offset -= sizeof(long);
}
emit_instr(ctx, jr, dest_reg);
if (stack_adjust)
emit_instr(ctx, daddiu, MIPS_R_SP, MIPS_R_SP, stack_adjust);
emit_instr_long(ctx, daddiu, addiu,
MIPS_R_SP, MIPS_R_SP, stack_adjust);
else
emit_instr(ctx, nop);
@ -646,6 +667,10 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
s64 t64s;
int bpf_op = BPF_OP(insn->code);
if (IS_ENABLED(CONFIG_32BIT) && ((BPF_CLASS(insn->code) == BPF_ALU64)
|| (bpf_op == BPF_DW)))
return -EINVAL;
switch (insn->code) {
case BPF_ALU64 | BPF_ADD | BPF_K: /* ALU64_IMM */
case BPF_ALU64 | BPF_SUB | BPF_K: /* ALU64_IMM */
@ -1283,7 +1308,7 @@ static int build_one_insn(const struct bpf_insn *insn, struct jit_ctx *ctx,
case BPF_JMP | BPF_CALL:
ctx->flags |= EBPF_SAVE_RA;
t64s = (s64)insn->imm + (s64)__bpf_call_base;
t64s = (s64)insn->imm + (long)__bpf_call_base;
emit_const_to_reg(ctx, MIPS_R_T9, (u64)t64s);
emit_instr(ctx, jalr, MIPS_R_RA, MIPS_R_T9);
/* delay slot */