mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

bpf: Simplify bpf_prog_pack_[size|mask]

Browse source 
Simplify the logic that selects bpf_prog_pack_size, and always use
(PMD_SIZE * num_possible_nodes()). This is a good tradeoff, as most of
the performance benefit observed is from less direct map fragmentation [0].

Also, module_alloc(4MB) may not allocate 4MB aligned memory. Therefore,
we cannot use (ptr & bpf_prog_pack_mask) to find the correct address of
bpf_prog_pack. Fix this by checking the header address falls in the range
of pack->ptr and (pack->ptr + bpf_prog_pack_size).

  [0] https://lore.kernel.org/bpf/20220707223546.4124919-1-song@kernel.org/

Signed-off-by: Song Liu <song@kernel.org>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Reviewed-by: Stanislav Fomichev <sdf@google.com>
Link: https://lore.kernel.org/bpf/20220713204950.3015201-1-song@kernel.org
This commit is contained in:
Song Liu 2022-07-13 13:49:50 -07:00 committed by Daniel Borkmann
parent 00963a2e75
commit ea2babac63
1 changed files with 17 additions and 54 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

71
kernel/bpf/core.c
View file

@ -825,15 +825,6 @@ struct bpf_prog_pack {
#define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE) #define BPF_PROG_SIZE_TO_NBITS(size) (round_up(size, BPF_PROG_CHUNK_SIZE) / BPF_PROG_CHUNK_SIZE)
static size_t bpf_prog_pack_size = -1;
static size_t bpf_prog_pack_mask = -1;
static int bpf_prog_chunk_count(void)
{
WARN_ON_ONCE(bpf_prog_pack_size == -1);
return bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE;
}
static DEFINE_MUTEX(pack_mutex); static DEFINE_MUTEX(pack_mutex);
static LIST_HEAD(pack_list); static LIST_HEAD(pack_list);
@ -841,55 +832,33 @@ static LIST_HEAD(pack_list);
* CONFIG_MMU=n. Use PAGE_SIZE in these cases. * CONFIG_MMU=n. Use PAGE_SIZE in these cases.
*/ */
#ifdef PMD_SIZE #ifdef PMD_SIZE
#define BPF_HPAGE_SIZE PMD_SIZE #define BPF_PROG_PACK_SIZE (PMD_SIZE * num_possible_nodes())
#define BPF_HPAGE_MASK PMD_MASK
#else #else
#define BPF_HPAGE_SIZE PAGE_SIZE #define BPF_PROG_PACK_SIZE PAGE_SIZE
#define BPF_HPAGE_MASK PAGE_MASK
#endif #endif
static size_t select_bpf_prog_pack_size(void) #define BPF_PROG_CHUNK_COUNT (BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE)
{
size_t size;
void *ptr;
size = BPF_HPAGE_SIZE * num_online_nodes();
ptr = module_alloc(size);
/* Test whether we can get huge pages. If not just use PAGE_SIZE
* packs.
*/
if (!ptr || !is_vm_area_hugepages(ptr)) {
size = PAGE_SIZE;
bpf_prog_pack_mask = PAGE_MASK;
} else {
bpf_prog_pack_mask = BPF_HPAGE_MASK;
}
vfree(ptr);
return size;
}
static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns) static struct bpf_prog_pack *alloc_new_pack(bpf_jit_fill_hole_t bpf_fill_ill_insns)
{ {
struct bpf_prog_pack *pack; struct bpf_prog_pack *pack;
pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(bpf_prog_chunk_count())), pack = kzalloc(struct_size(pack, bitmap, BITS_TO_LONGS(BPF_PROG_CHUNK_COUNT)),
GFP_KERNEL); GFP_KERNEL);
if (!pack) if (!pack)
return NULL; return NULL;
pack->ptr = module_alloc(bpf_prog_pack_size); pack->ptr = module_alloc(BPF_PROG_PACK_SIZE);
if (!pack->ptr) { if (!pack->ptr) {
kfree(pack); kfree(pack);
return NULL; return NULL;
} }
bpf_fill_ill_insns(pack->ptr, bpf_prog_pack_size); bpf_fill_ill_insns(pack->ptr, BPF_PROG_PACK_SIZE);
bitmap_zero(pack->bitmap, bpf_prog_pack_size / BPF_PROG_CHUNK_SIZE); bitmap_zero(pack->bitmap, BPF_PROG_PACK_SIZE / BPF_PROG_CHUNK_SIZE);
list_add_tail(&pack->list, &pack_list); list_add_tail(&pack->list, &pack_list);
set_vm_flush_reset_perms(pack->ptr); set_vm_flush_reset_perms(pack->ptr);
set_memory_ro((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); set_memory_ro((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
set_memory_x((unsigned long)pack->ptr, bpf_prog_pack_size / PAGE_SIZE); set_memory_x((unsigned long)pack->ptr, BPF_PROG_PACK_SIZE / PAGE_SIZE);
return pack; return pack;
} }
@ -901,10 +870,7 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn
void *ptr = NULL; void *ptr = NULL;
mutex_lock(&pack_mutex); mutex_lock(&pack_mutex);
if (bpf_prog_pack_size == -1) if (size > BPF_PROG_PACK_SIZE) {
bpf_prog_pack_size = select_bpf_prog_pack_size();
if (size > bpf_prog_pack_size) {
size = round_up(size, PAGE_SIZE); size = round_up(size, PAGE_SIZE);
ptr = module_alloc(size); ptr = module_alloc(size);
if (ptr) { if (ptr) {
@ -916,9 +882,9 @@ static void *bpf_prog_pack_alloc(u32 size, bpf_jit_fill_hole_t bpf_fill_ill_insn
goto out; goto out;
} }
list_for_each_entry(pack, &pack_list, list) { list_for_each_entry(pack, &pack_list, list) {
pos = bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, pos = bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
nbits, 0); nbits, 0);
if (pos < bpf_prog_chunk_count()) if (pos < BPF_PROG_CHUNK_COUNT)
goto found_free_area; goto found_free_area;
} }
@ -942,18 +908,15 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)
struct bpf_prog_pack *pack = NULL, *tmp; struct bpf_prog_pack *pack = NULL, *tmp;
unsigned int nbits; unsigned int nbits;
unsigned long pos; unsigned long pos;
void *pack_ptr;
mutex_lock(&pack_mutex); mutex_lock(&pack_mutex);
if (hdr->size > bpf_prog_pack_size) { if (hdr->size > BPF_PROG_PACK_SIZE) {
module_memfree(hdr); module_memfree(hdr);
goto out; goto out;
} }
pack_ptr = (void *)((unsigned long)hdr & bpf_prog_pack_mask);
list_for_each_entry(tmp, &pack_list, list) { list_for_each_entry(tmp, &pack_list, list) {
if (tmp->ptr == pack_ptr) { if ((void *)hdr >= tmp->ptr && (tmp->ptr + BPF_PROG_PACK_SIZE) > (void *)hdr) {
pack = tmp; pack = tmp;
break; break;
} }
@ -963,14 +926,14 @@ static void bpf_prog_pack_free(struct bpf_binary_header *hdr)
goto out; goto out;
nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size); nbits = BPF_PROG_SIZE_TO_NBITS(hdr->size);
pos = ((unsigned long)hdr - (unsigned long)pack_ptr) >> BPF_PROG_CHUNK_SHIFT; pos = ((unsigned long)hdr - (unsigned long)pack->ptr) >> BPF_PROG_CHUNK_SHIFT;
WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size), WARN_ONCE(bpf_arch_text_invalidate(hdr, hdr->size),
"bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n"); "bpf_prog_pack bug: missing bpf_arch_text_invalidate?\n");
bitmap_clear(pack->bitmap, pos, nbits); bitmap_clear(pack->bitmap, pos, nbits);
if (bitmap_find_next_zero_area(pack->bitmap, bpf_prog_chunk_count(), 0, if (bitmap_find_next_zero_area(pack->bitmap, BPF_PROG_CHUNK_COUNT, 0,
bpf_prog_chunk_count(), 0) == 0) { BPF_PROG_CHUNK_COUNT, 0) == 0) {
list_del(&pack->list); list_del(&pack->list);
module_memfree(pack->ptr); module_memfree(pack->ptr);
kfree(pack); kfree(pack);