mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-04 10:40:15 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

net: move skbuff_cache(s) to net_hotdata

Browse source 
skbuff_cache, skbuff_fclone_cache and skb_small_head_cache
are used in rx/tx fast paths.

Move them to net_hotdata for better cache locality.

Signed-off-by: Eric Dumazet <edumazet@google.com>
Acked-by: Soheil Hassas Yeganeh <soheil@google.com>
Reviewed-by: David Ahern <dsahern@kernel.org>
Link: https://lore.kernel.org/r/20240306160031.874438-11-edumazet@google.com
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
This commit is contained in:
Eric Dumazet 2024-03-06 16:00:23 +00:00 committed by Jakub Kicinski
parent 71c0de9bac
commit aa70d2d16f
6 changed files with 32 additions and 29 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

1
include/linux/skbuff.h
View file

@ -1271,7 +1271,6 @@ static inline void consume_skb(struct sk_buff *skb)
void __consume_stateless_skb(struct sk_buff *skb); void __consume_stateless_skb(struct sk_buff *skb);
void __kfree_skb(struct sk_buff *skb); void __kfree_skb(struct sk_buff *skb);
extern struct kmem_cache *skbuff_cache;
void kfree_skb_partial(struct sk_buff *skb, bool head_stolen); void kfree_skb_partial(struct sk_buff *skb, bool head_stolen);
bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from, bool skb_try_coalesce(struct sk_buff *to, struct sk_buff *from,

3
include/net/hotdata.h
View file

@ -16,6 +16,9 @@ struct net_hotdata {
#endif #endif
struct list_head offload_base; struct list_head offload_base;
struct list_head ptype_all; struct list_head ptype_all;
struct kmem_cache *skbuff_cache;
struct kmem_cache *skbuff_fclone_cache;
struct kmem_cache *skb_small_head_cache;
int gro_normal_batch; int gro_normal_batch;
int netdev_budget; int netdev_budget;
int netdev_budget_usecs; int netdev_budget_usecs;

4
kernel/bpf/cpumap.c
View file

@ -24,6 +24,7 @@
#include <linux/filter.h> #include <linux/filter.h>
#include <linux/ptr_ring.h> #include <linux/ptr_ring.h>
#include <net/xdp.h> #include <net/xdp.h>
#include <net/hotdata.h>
#include <linux/sched.h> #include <linux/sched.h>
#include <linux/workqueue.h> #include <linux/workqueue.h>
@ -326,7 +327,8 @@ static int cpu_map_kthread_run(void *data)
/* Support running another XDP prog on this CPU */ /* Support running another XDP prog on this CPU */
nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list); nframes = cpu_map_bpf_prog_run(rcpu, frames, xdp_n, &stats, &list);
if (nframes) { if (nframes) {
m = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, skbs); m = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
gfp, nframes, skbs);
if (unlikely(m == 0)) { if (unlikely(m == 0)) {
for (i = 0; i < nframes; i++) for (i = 0; i < nframes; i++)
skbs[i] = NULL; /* effect: xdp_return_frame */ skbs[i] = NULL; /* effect: xdp_return_frame */

4
net/bpf/test_run.c
View file

@ -12,6 +12,7 @@
#include <linux/rcupdate_trace.h> #include <linux/rcupdate_trace.h>
#include <linux/sched/signal.h> #include <linux/sched/signal.h>
#include <net/bpf_sk_storage.h> #include <net/bpf_sk_storage.h>
#include <net/hotdata.h>
#include <net/sock.h> #include <net/sock.h>
#include <net/tcp.h> #include <net/tcp.h>
#include <net/net_namespace.h> #include <net/net_namespace.h>
@ -254,7 +255,8 @@ static int xdp_recv_frames(struct xdp_frame **frames, int nframes,
int i, n; int i, n;
LIST_HEAD(list); LIST_HEAD(list);
n = kmem_cache_alloc_bulk(skbuff_cache, gfp, nframes, (void **)skbs); n = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, nframes,
(void **)skbs);
if (unlikely(n == 0)) { if (unlikely(n == 0)) {
for (i = 0; i < nframes; i++) for (i = 0; i < nframes; i++)
xdp_return_frame(frames[i]); xdp_return_frame(frames[i]);

44
net/core/skbuff.c
View file

@ -69,6 +69,7 @@
#include <net/sock.h> #include <net/sock.h>
#include <net/checksum.h> #include <net/checksum.h>
#include <net/gso.h> #include <net/gso.h>
#include <net/hotdata.h>
#include <net/ip6_checksum.h> #include <net/ip6_checksum.h>
#include <net/xfrm.h> #include <net/xfrm.h>
#include <net/mpls.h> #include <net/mpls.h>
@ -88,15 +89,10 @@
#include "dev.h" #include "dev.h"
#include "sock_destructor.h" #include "sock_destructor.h"
struct kmem_cache *skbuff_cache __ro_after_init;
static struct kmem_cache *skbuff_fclone_cache __ro_after_init;
#ifdef CONFIG_SKB_EXTENSIONS #ifdef CONFIG_SKB_EXTENSIONS
static struct kmem_cache *skbuff_ext_cache __ro_after_init; static struct kmem_cache *skbuff_ext_cache __ro_after_init;
#endif #endif
static struct kmem_cache *skb_small_head_cache __ro_after_init;
#define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER) #define SKB_SMALL_HEAD_SIZE SKB_HEAD_ALIGN(MAX_TCP_HEADER)
/* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two. /* We want SKB_SMALL_HEAD_CACHE_SIZE to not be a power of two.
@ -349,7 +345,7 @@ static struct sk_buff *napi_skb_cache_get(void)
struct sk_buff *skb; struct sk_buff *skb;
if (unlikely(!nc->skb_count)) { if (unlikely(!nc->skb_count)) {
nc->skb_count = kmem_cache_alloc_bulk(skbuff_cache, nc->skb_count = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache,
GFP_ATOMIC, GFP_ATOMIC,
NAPI_SKB_CACHE_BULK, NAPI_SKB_CACHE_BULK,
nc->skb_cache); nc->skb_cache);
@ -358,7 +354,7 @@ static struct sk_buff *napi_skb_cache_get(void)
} }
skb = nc->skb_cache[--nc->skb_count]; skb = nc->skb_cache[--nc->skb_count];
kasan_mempool_unpoison_object(skb, kmem_cache_size(skbuff_cache)); kasan_mempool_unpoison_object(skb, kmem_cache_size(net_hotdata.skbuff_cache));
return skb; return skb;
} }
@ -416,7 +412,7 @@ struct sk_buff *slab_build_skb(void *data)
struct sk_buff *skb; struct sk_buff *skb;
unsigned int size; unsigned int size;
skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
if (unlikely(!skb)) if (unlikely(!skb))
return NULL; return NULL;
@ -467,7 +463,7 @@ struct sk_buff *__build_skb(void *data, unsigned int frag_size)
{ {
struct sk_buff *skb; struct sk_buff *skb;
skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
if (unlikely(!skb)) if (unlikely(!skb))
return NULL; return NULL;
@ -578,7 +574,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
obj_size = SKB_HEAD_ALIGN(*size); obj_size = SKB_HEAD_ALIGN(*size);
if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE && if (obj_size <= SKB_SMALL_HEAD_CACHE_SIZE &&
!(flags & KMALLOC_NOT_NORMAL_BITS)) { !(flags & KMALLOC_NOT_NORMAL_BITS)) {
obj = kmem_cache_alloc_node(skb_small_head_cache, obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache,
flags | __GFP_NOMEMALLOC | __GFP_NOWARN, flags | __GFP_NOMEMALLOC | __GFP_NOWARN,
node); node);
*size = SKB_SMALL_HEAD_CACHE_SIZE; *size = SKB_SMALL_HEAD_CACHE_SIZE;
@ -586,7 +582,7 @@ static void *kmalloc_reserve(unsigned int *size, gfp_t flags, int node,
goto out; goto out;
/* Try again but now we are using pfmemalloc reserves */ /* Try again but now we are using pfmemalloc reserves */
ret_pfmemalloc = true; ret_pfmemalloc = true;
obj = kmem_cache_alloc_node(skb_small_head_cache, flags, node); obj = kmem_cache_alloc_node(net_hotdata.skb_small_head_cache, flags, node);
goto out; goto out;
} }
@ -649,7 +645,7 @@ struct sk_buff *__alloc_skb(unsigned int size, gfp_t gfp_mask,
u8 *data; u8 *data;
cache = (flags & SKB_ALLOC_FCLONE) cache = (flags & SKB_ALLOC_FCLONE)
? skbuff_fclone_cache : skbuff_cache; ? net_hotdata.skbuff_fclone_cache : net_hotdata.skbuff_cache;
if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX)) if (sk_memalloc_socks() && (flags & SKB_ALLOC_RX))
gfp_mask |= __GFP_MEMALLOC; gfp_mask |= __GFP_MEMALLOC;
@ -1095,7 +1091,7 @@ static int skb_pp_frag_ref(struct sk_buff *skb)
static void skb_kfree_head(void *head, unsigned int end_offset) static void skb_kfree_head(void *head, unsigned int end_offset)
{ {
if (end_offset == SKB_SMALL_HEAD_HEADROOM) if (end_offset == SKB_SMALL_HEAD_HEADROOM)
kmem_cache_free(skb_small_head_cache, head); kmem_cache_free(net_hotdata.skb_small_head_cache, head);
else else
kfree(head); kfree(head);
} }
@ -1162,7 +1158,7 @@ static void kfree_skbmem(struct sk_buff *skb)
switch (skb->fclone) { switch (skb->fclone) {
case SKB_FCLONE_UNAVAILABLE: case SKB_FCLONE_UNAVAILABLE:
kmem_cache_free(skbuff_cache, skb); kmem_cache_free(net_hotdata.skbuff_cache, skb);
return; return;
case SKB_FCLONE_ORIG: case SKB_FCLONE_ORIG:
@ -1183,7 +1179,7 @@ static void kfree_skbmem(struct sk_buff *skb)
if (!refcount_dec_and_test(&fclones->fclone_ref)) if (!refcount_dec_and_test(&fclones->fclone_ref))
return; return;
fastpath: fastpath:
kmem_cache_free(skbuff_fclone_cache, fclones); kmem_cache_free(net_hotdata.skbuff_fclone_cache, fclones);
} }
void skb_release_head_state(struct sk_buff *skb) void skb_release_head_state(struct sk_buff *skb)
@ -1280,7 +1276,7 @@ static void kfree_skb_add_bulk(struct sk_buff *skb,
sa->skb_array[sa->skb_count++] = skb; sa->skb_array[sa->skb_count++] = skb;
if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) { if (unlikely(sa->skb_count == KFREE_SKB_BULK_SIZE)) {
kmem_cache_free_bulk(skbuff_cache, KFREE_SKB_BULK_SIZE, kmem_cache_free_bulk(net_hotdata.skbuff_cache, KFREE_SKB_BULK_SIZE,
sa->skb_array); sa->skb_array);
sa->skb_count = 0; sa->skb_count = 0;
} }
@ -1305,7 +1301,7 @@ kfree_skb_list_reason(struct sk_buff *segs, enum skb_drop_reason reason)
} }
if (sa.skb_count) if (sa.skb_count)
kmem_cache_free_bulk(skbuff_cache, sa.skb_count, sa.skb_array); kmem_cache_free_bulk(net_hotdata.skbuff_cache, sa.skb_count, sa.skb_array);
} }
EXPORT_SYMBOL(kfree_skb_list_reason); EXPORT_SYMBOL(kfree_skb_list_reason);
@ -1467,9 +1463,9 @@ static void napi_skb_cache_put(struct sk_buff *skb)
if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) { if (unlikely(nc->skb_count == NAPI_SKB_CACHE_SIZE)) {
for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++) for (i = NAPI_SKB_CACHE_HALF; i < NAPI_SKB_CACHE_SIZE; i++)
kasan_mempool_unpoison_object(nc->skb_cache[i], kasan_mempool_unpoison_object(nc->skb_cache[i],
kmem_cache_size(skbuff_cache)); kmem_cache_size(net_hotdata.skbuff_cache));
kmem_cache_free_bulk(skbuff_cache, NAPI_SKB_CACHE_HALF, kmem_cache_free_bulk(net_hotdata.skbuff_cache, NAPI_SKB_CACHE_HALF,
nc->skb_cache + NAPI_SKB_CACHE_HALF); nc->skb_cache + NAPI_SKB_CACHE_HALF);
nc->skb_count = NAPI_SKB_CACHE_HALF; nc->skb_count = NAPI_SKB_CACHE_HALF;
} }
@ -2066,7 +2062,7 @@ struct sk_buff *skb_clone(struct sk_buff *skb, gfp_t gfp_mask)
if (skb_pfmemalloc(skb)) if (skb_pfmemalloc(skb))
gfp_mask |= __GFP_MEMALLOC; gfp_mask |= __GFP_MEMALLOC;
n = kmem_cache_alloc(skbuff_cache, gfp_mask); n = kmem_cache_alloc(net_hotdata.skbuff_cache, gfp_mask);
if (!n) if (!n)
return NULL; return NULL;
@ -5005,7 +5001,7 @@ static void skb_extensions_init(void) {}
void __init skb_init(void) void __init skb_init(void)
{ {
skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache", net_hotdata.skbuff_cache = kmem_cache_create_usercopy("skbuff_head_cache",
sizeof(struct sk_buff), sizeof(struct sk_buff),
0, 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC| SLAB_HWCACHE_ALIGN|SLAB_PANIC|
@ -5013,7 +5009,7 @@ void __init skb_init(void)
offsetof(struct sk_buff, cb), offsetof(struct sk_buff, cb),
sizeof_field(struct sk_buff, cb), sizeof_field(struct sk_buff, cb),
NULL); NULL);
skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache", net_hotdata.skbuff_fclone_cache = kmem_cache_create("skbuff_fclone_cache",
sizeof(struct sk_buff_fclones), sizeof(struct sk_buff_fclones),
0, 0,
SLAB_HWCACHE_ALIGN|SLAB_PANIC, SLAB_HWCACHE_ALIGN|SLAB_PANIC,
@ -5022,7 +5018,7 @@ void __init skb_init(void)
* struct skb_shared_info is located at the end of skb->head, * struct skb_shared_info is located at the end of skb->head,
* and should not be copied to/from user. * and should not be copied to/from user.
*/ */
skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head", net_hotdata.skb_small_head_cache = kmem_cache_create_usercopy("skbuff_small_head",
SKB_SMALL_HEAD_CACHE_SIZE, SKB_SMALL_HEAD_CACHE_SIZE,
0, 0,
SLAB_HWCACHE_ALIGN | SLAB_PANIC, SLAB_HWCACHE_ALIGN | SLAB_PANIC,
@ -5895,7 +5891,7 @@ void kfree_skb_partial(struct sk_buff *skb, bool head_stolen)
{ {
if (head_stolen) { if (head_stolen) {
skb_release_head_state(skb); skb_release_head_state(skb);
kmem_cache_free(skbuff_cache, skb); kmem_cache_free(net_hotdata.skbuff_cache, skb);
} else { } else {
__kfree_skb(skb); __kfree_skb(skb);
} }

5
net/core/xdp.c
View file

@ -16,6 +16,7 @@
#include <linux/bug.h> #include <linux/bug.h>
#include <net/page_pool/helpers.h> #include <net/page_pool/helpers.h>
#include <net/hotdata.h>
#include <net/xdp.h> #include <net/xdp.h>
#include <net/xdp_priv.h> /* struct xdp_mem_allocator */ #include <net/xdp_priv.h> /* struct xdp_mem_allocator */
#include <trace/events/xdp.h> #include <trace/events/xdp.h>
@ -589,7 +590,7 @@ EXPORT_SYMBOL_GPL(xdp_warn);
int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp) int xdp_alloc_skb_bulk(void **skbs, int n_skb, gfp_t gfp)
{ {
n_skb = kmem_cache_alloc_bulk(skbuff_cache, gfp, n_skb, skbs); n_skb = kmem_cache_alloc_bulk(net_hotdata.skbuff_cache, gfp, n_skb, skbs);
if (unlikely(!n_skb)) if (unlikely(!n_skb))
return -ENOMEM; return -ENOMEM;
@ -658,7 +659,7 @@ struct sk_buff *xdp_build_skb_from_frame(struct xdp_frame *xdpf,
{ {
struct sk_buff *skb; struct sk_buff *skb;
skb = kmem_cache_alloc(skbuff_cache, GFP_ATOMIC); skb = kmem_cache_alloc(net_hotdata.skbuff_cache, GFP_ATOMIC);
if (unlikely(!skb)) if (unlikely(!skb))
return NULL; return NULL;