mirrors/linux
1
0
Fork 1
mirror of https://github.com/torvalds/linux.git synced 2025-11-03 18:20:25 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'net-gro-cleanups-and-fast-path-refinement'

Browse source 
Eric Dumazet says:

====================
net: gro: cleanups and fast path refinement

Current GRO stack has a 'fast path' for a subset of drivers,
users of napi_frags_skb().

With TCP zerocopy/direct uses, header split at receive is becoming
more important, and GRO fast path is disabled.

This series makes GRO (a bit) more efficient for almost all use cases.
====================

Link: https://lore.kernel.org/r/20240301193740.3436871-1-edumazet@google.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
This commit is contained in:
Paolo Abeni 2024-03-05 13:30:15 +01:00
commit d35c9659e5
6 changed files with 46 additions and 38 deletions
repo.diff.show_diff_stats Download patch file Download diff file Expand all files Collapse all files

2
drivers/net/geneve.c
View file

@ -508,7 +508,7 @@ static struct sk_buff *geneve_gro_receive(struct sock *sk,
gh_len = geneve_hlen(gh); gh_len = geneve_hlen(gh);
hlen = off_gnv + gh_len; hlen = off_gnv + gh_len;
if (skb_gro_header_hard(skb, hlen)) { if (!skb_gro_may_pull(skb, hlen)) {
gh = skb_gro_header_slow(skb, hlen, off_gnv); gh = skb_gro_header_slow(skb, hlen, off_gnv);
if (unlikely(!gh)) if (unlikely(!gh))
goto out; goto out;

34
include/net/gro.h
View file

@ -139,21 +139,16 @@ static inline void skb_gro_pull(struct sk_buff *skb, unsigned int len)
NAPI_GRO_CB(skb)->data_offset += len; NAPI_GRO_CB(skb)->data_offset += len;
} }
static inline void *skb_gro_header_fast(struct sk_buff *skb, static inline void *skb_gro_header_fast(const struct sk_buff *skb,
unsigned int offset) unsigned int offset)
{ {
return NAPI_GRO_CB(skb)->frag0 + offset; return NAPI_GRO_CB(skb)->frag0 + offset;
} }
static inline int skb_gro_header_hard(struct sk_buff *skb, unsigned int hlen) static inline bool skb_gro_may_pull(const struct sk_buff *skb,
unsigned int hlen)
{ {
return NAPI_GRO_CB(skb)->frag0_len < hlen; return likely(hlen <= NAPI_GRO_CB(skb)->frag0_len);
}
static inline void skb_gro_frag0_invalidate(struct sk_buff *skb)
{
NAPI_GRO_CB(skb)->frag0 = NULL;
NAPI_GRO_CB(skb)->frag0_len = 0;
} }
static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen, static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
@ -162,28 +157,30 @@ static inline void *skb_gro_header_slow(struct sk_buff *skb, unsigned int hlen,
if (!pskb_may_pull(skb, hlen)) if (!pskb_may_pull(skb, hlen))
return NULL; return NULL;
skb_gro_frag0_invalidate(skb);
return skb->data + offset; return skb->data + offset;
} }
static inline void *skb_gro_header(struct sk_buff *skb, static inline void *skb_gro_header(struct sk_buff *skb, unsigned int hlen,
unsigned int hlen, unsigned int offset) unsigned int offset)
{ {
void *ptr; void *ptr;
ptr = skb_gro_header_fast(skb, offset); ptr = skb_gro_header_fast(skb, offset);
if (skb_gro_header_hard(skb, hlen)) if (!skb_gro_may_pull(skb, hlen))
ptr = skb_gro_header_slow(skb, hlen, offset); ptr = skb_gro_header_slow(skb, hlen, offset);
return ptr; return ptr;
} }
static inline void *skb_gro_network_header(struct sk_buff *skb) static inline void *skb_gro_network_header(const struct sk_buff *skb)
{ {
return (NAPI_GRO_CB(skb)->frag0 ?: skb->data) + if (skb_gro_may_pull(skb, skb_gro_offset(skb)))
skb_network_offset(skb); return skb_gro_header_fast(skb, skb_network_offset(skb));
return skb_network_header(skb);
} }
static inline __wsum inet_gro_compute_pseudo(struct sk_buff *skb, int proto) static inline __wsum inet_gro_compute_pseudo(const struct sk_buff *skb,
int proto)
{ {
const struct iphdr *iph = skb_gro_network_header(skb); const struct iphdr *iph = skb_gro_network_header(skb);
@ -421,7 +418,8 @@ static inline struct udphdr *udp_gro_udphdr(struct sk_buff *skb)
return uh; return uh;
} }
static inline __wsum ip6_gro_compute_pseudo(struct sk_buff *skb, int proto) static inline __wsum ip6_gro_compute_pseudo(const struct sk_buff *skb,
int proto)
{ {
const struct ipv6hdr *iph = skb_gro_network_header(skb); const struct ipv6hdr *iph = skb_gro_network_header(skb);

25
net/core/gro.c
View file

@ -369,15 +369,21 @@ static void gro_list_prepare(const struct list_head *head,
static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff) static inline void skb_gro_reset_offset(struct sk_buff *skb, u32 nhoff)
{ {
const struct skb_shared_info *pinfo = skb_shinfo(skb); const struct skb_shared_info *pinfo;
const skb_frag_t *frag0 = &pinfo->frags[0]; const skb_frag_t *frag0;
unsigned int headlen;
NAPI_GRO_CB(skb)->data_offset = 0; NAPI_GRO_CB(skb)->data_offset = 0;
NAPI_GRO_CB(skb)->frag0 = NULL; headlen = skb_headlen(skb);
NAPI_GRO_CB(skb)->frag0_len = 0; NAPI_GRO_CB(skb)->frag0 = skb->data;
NAPI_GRO_CB(skb)->frag0_len = headlen;
if (headlen)
return;
if (!skb_headlen(skb) && pinfo->nr_frags && pinfo = skb_shinfo(skb);
!PageHighMem(skb_frag_page(frag0)) && frag0 = &pinfo->frags[0];
if (pinfo->nr_frags && !PageHighMem(skb_frag_page(frag0)) &&
(!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) { (!NET_IP_ALIGN || !((skb_frag_off(frag0) + nhoff) & 3))) {
NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0); NAPI_GRO_CB(skb)->frag0 = skb_frag_address(frag0);
NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int, NAPI_GRO_CB(skb)->frag0_len = min_t(unsigned int,
@ -700,7 +706,7 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
skb_reset_mac_header(skb); skb_reset_mac_header(skb);
skb_gro_reset_offset(skb, hlen); skb_gro_reset_offset(skb, hlen);
if (unlikely(skb_gro_header_hard(skb, hlen))) { if (unlikely(!skb_gro_may_pull(skb, hlen))) {
eth = skb_gro_header_slow(skb, hlen, 0); eth = skb_gro_header_slow(skb, hlen, 0);
if (unlikely(!eth)) { if (unlikely(!eth)) {
net_warn_ratelimited("%s: dropping impossible skb from %s\n", net_warn_ratelimited("%s: dropping impossible skb from %s\n",
@ -710,7 +716,10 @@ static struct sk_buff *napi_frags_skb(struct napi_struct *napi)
} }
} else { } else {
eth = (const struct ethhdr *)skb->data; eth = (const struct ethhdr *)skb->data;
gro_pull_from_frag0(skb, hlen);
if (NAPI_GRO_CB(skb)->frag0 != skb->data)
gro_pull_from_frag0(skb, hlen);
NAPI_GRO_CB(skb)->frag0 += hlen; NAPI_GRO_CB(skb)->frag0 += hlen;
NAPI_GRO_CB(skb)->frag0_len -= hlen; NAPI_GRO_CB(skb)->frag0_len -= hlen;
} }

2
net/ipv4/fou_core.c
View file

@ -351,7 +351,7 @@ static struct sk_buff *gue_gro_receive(struct sock *sk,
optlen = guehdr->hlen << 2; optlen = guehdr->hlen << 2;
len += optlen; len += optlen;
if (skb_gro_header_hard(skb, len)) { if (!skb_gro_may_pull(skb, len)) {
guehdr = skb_gro_header_slow(skb, len, off); guehdr = skb_gro_header_slow(skb, len, off);
if (unlikely(!guehdr)) if (unlikely(!guehdr))
goto out; goto out;

2
net/ipv4/gre_offload.c
View file

@ -174,7 +174,7 @@ static struct sk_buff *gre_gro_receive(struct list_head *head,
grehlen += GRE_HEADER_SECTION; grehlen += GRE_HEADER_SECTION;
hlen = off + grehlen; hlen = off + grehlen;
if (skb_gro_header_hard(skb, hlen)) { if (!skb_gro_may_pull(skb, hlen)) {
greh = skb_gro_header_slow(skb, hlen, off); greh = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!greh)) if (unlikely(!greh))
goto out; goto out;

19
net/ipv4/tcp_offload.c
View file

@ -204,7 +204,7 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
goto out; goto out;
hlen = off + thlen; hlen = off + thlen;
if (skb_gro_header_hard(skb, hlen)) { if (!skb_gro_may_pull(skb, hlen)) {
th = skb_gro_header_slow(skb, hlen, off); th = skb_gro_header_slow(skb, hlen, off);
if (unlikely(!th)) if (unlikely(!th))
goto out; goto out;
@ -299,18 +299,20 @@ struct sk_buff *tcp_gro_receive(struct list_head *head, struct sk_buff *skb)
void tcp_gro_complete(struct sk_buff *skb) void tcp_gro_complete(struct sk_buff *skb)
{ {
struct tcphdr *th = tcp_hdr(skb); struct tcphdr *th = tcp_hdr(skb);
struct skb_shared_info *shinfo;
if (skb->encapsulation)
skb->inner_transport_header = skb->transport_header;
skb->csum_start = (unsigned char *)th - skb->head; skb->csum_start = (unsigned char *)th - skb->head;
skb->csum_offset = offsetof(struct tcphdr, check); skb->csum_offset = offsetof(struct tcphdr, check);
skb->ip_summed = CHECKSUM_PARTIAL; skb->ip_summed = CHECKSUM_PARTIAL;
skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count; shinfo = skb_shinfo(skb);
shinfo->gso_segs = NAPI_GRO_CB(skb)->count;
if (th->cwr) if (th->cwr)
skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_ECN; shinfo->gso_type |= SKB_GSO_TCP_ECN;
if (skb->encapsulation)
skb->inner_transport_header = skb->transport_header;
} }
EXPORT_SYMBOL(tcp_gro_complete); EXPORT_SYMBOL(tcp_gro_complete);
@ -335,10 +337,9 @@ INDIRECT_CALLABLE_SCOPE int tcp4_gro_complete(struct sk_buff *skb, int thoff)
th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr, th->check = ~tcp_v4_check(skb->len - thoff, iph->saddr,
iph->daddr, 0); iph->daddr, 0);
skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4;
if (NAPI_GRO_CB(skb)->is_atomic) skb_shinfo(skb)->gso_type |= SKB_GSO_TCPV4 |
skb_shinfo(skb)->gso_type |= SKB_GSO_TCP_FIXEDID; (NAPI_GRO_CB(skb)->is_atomic * SKB_GSO_TCP_FIXEDID);
tcp_gro_complete(skb); tcp_gro_complete(skb);
return 0; return 0;